def test_write_against_file_contents(self): filename = u'tests/testdata/simple_1.concrete' with open(filename, 'rb') as f: f_buf = f.read() comm = read_communication_from_buffer(f_buf) buf = write_communication_to_buffer(comm) self.assertEquals(f_buf, buf)
def test_read_against_file_contents_no_add_references(self): filename = u'tests/testdata/simple_1.concrete' with open(filename, 'rb') as f: buf = f.read() comm = read_communication_from_buffer(buf, add_references=False) self.assertFalse(hasattr(comm, 'sentenceForUUID')) self.assertEquals('one', comm.id)
def test_read_write_fixed_point(self): comm = create_comm('comm-1') buf_1 = write_communication_to_buffer(comm) buf_2 = write_communication_to_buffer( read_communication_from_buffer(buf_1) ) self.assertEquals(buf_1, buf_2)
def test_write_against_file_contents(self): filename = u'tests/testdata/simple_1.concrete' key = 'comm' with open(filename, 'rb') as f: f_buf = f.read() comm = read_communication_from_buffer(f_buf) with RedisServer(loglevel='warning') as server: redis_db = Redis(port=server.port) write_communication_to_redis_key(redis_db, key, comm) self.assertEquals(f_buf, redis_db.get(key))
def read_communication_from_redis_key(redis_db, key, add_references=True): ''' Return a serialized communication from a string key. If block is True, poll server until key appears at specified interval or until specified timeout (indefinitely if timeout is zero). Return None if block is False and key does not exist or if block is True and key does not exist after specified timeout. ''' buf = redis_db.get(key) if buf is None: return None else: return read_communication_from_buffer(buf, add_references=add_references)
def get_restful_comm(restful_host, restful_port, restful_pattern, communication_loc): url = urlparse('%s:%s' % (restful_host, restful_port)) if url.netloc is None or len(url.netloc) == 0: h = 'http://%s' % (restful_host) else: h = '%s://%s' % (url.scheme, url.netloc) loc_pattern = restful_pattern % (communication_loc) logging.info('using location pattern %s' % loc_pattern) full = '%s:%s/%s' % (h, restful_port, loc_pattern) logging.info("querying %s" % full) resp = urlopen(full) if resp is None: error("Got back a None from querying %s" % (full)) if resp.code != 200: error("received code %d and message %s from %s" % ( resp.code, resp.msg, full)) comm_buf = resp.read() resp.close() comm = read_communication_from_buffer(comm_buf) return comm
def get_restful_comm(restful_host, restful_port, restful_pattern, communication_loc): url = urlparse('%s:%s' % (restful_host, restful_port)) if url.netloc is None or len(url.netloc) == 0: h = 'http://%s' % (restful_host) else: h = '%s://%s' % (url.scheme, url.netloc) loc_pattern = restful_pattern % (communication_loc) logging.info('using location pattern %s' % loc_pattern) full = '%s:%s/%s' % (h, restful_port, loc_pattern) logging.info("querying %s" % full) resp = urlopen(full) if resp is None: error("Got back a None from querying %s" % (full)) if resp.code != 200: error("received code %d and message %s from %s" % (resp.code, resp.msg, full)) comm_buf = resp.read() resp.close() comm = read_communication_from_buffer(comm_buf) return comm
def _load_from_buffer(self, buf): return read_communication_from_buffer( buf, add_references=self.add_references)
def get_redis_comm(redis_host, redis_port, redis_comm, comm_lookup_by, redis_comm_map, redis_comm_index, redis_direction, communication_loc): def comm_lookup(comm): if comm_lookup_by == 'id': return comm.id elif comm_lookup_by == 'uuid': return comm.uuid.uuidString else: error('unsupported comm_lookup_by %s' % comm_lookup_by) input_db = None if redis_comm_index is not None: if redis_direction == RIGHT_TO_LEFT: redis_comm_index = -(redis_comm_index + 1) else: redis_comm_index = redis_comm_index else: redis_comm_index = None if redis_comm is not None and redis_comm_index is not None: error("Cannot include both --redis-comm and --redis-comm-index") input_db = Redis(redis_host, redis_port) reader = RedisCommunicationReader(input_db, communication_loc, add_references=True, right_to_left=(redis_direction == RIGHT_TO_LEFT)) if redis_comm: # look up comm in collection by comm field (comm_lookup_by) key_type = input_db.type(communication_loc) if ((key_type == 'list' and redis_comm_map is None) or (key_type == 'set')): # do linear scan for co in reader: if comm_lookup(co) == redis_comm: comm = co break elif key_type == 'list' and redis_comm_map is not None: # look up list index using field->index map def no_comm(comm_idx): error(('Unable to find a communication with identifier "%s"' ' with value "%s" using pivoting map "%s", which' ' returned (list) index %s, under the %s %s') % (comm_lookup_by, redis_comm, redis_comm_map, str(comm_idx), key_type, communication_loc)) comm_idx = input_db.hget(redis_comm_map, redis_comm) if comm_idx is None: no_comm(comm_idx) comm_idx = int(comm_idx) if redis_direction == RIGHT_TO_LEFT: comm_idx = - (comm_idx + 1) comm_buf = input_db.lindex(communication_loc, comm_idx) comm = read_communication_from_buffer(comm_buf) if comm_lookup(comm) != redis_comm: error(('Cannot find the appropriate document with %s' ' indexing') % redis_direction) elif key_type == 'hash': # do O(1) hash lookup comm_buf = input_db.hget(communication_loc, redis_comm) comm = read_communication_from_buffer(comm_buf) else: error('Unknown key type %s' % (key_type)) if comm is None: error(('Unable to find communication with id %s at %s:%s under' ' key %s') % (redis_comm, redis_host, redis_port, communication_loc)) elif redis_comm_index is not None: # lookup comm by index in list comm_buf = input_db.lindex(communication_loc, redis_comm_index) if comm_buf is None: error(('Unable to find communication with id %s at %s:%s under' ' key %s') % (redis_comm, redis_host, redis_port, communication_loc)) else: comm = read_communication_from_buffer(comm_buf) logging.info('%dth Communication has id %s' % (redis_comm_index + 1, comm.id)) else: # take first comm in collection # (or return single comm stored in simple key) for co in reader: comm = co break if comm is None: error('Unable to find any communications at %s:%s under key %s' % (redis_host, redis_port, communication_loc)) return comm
def get_redis_comm(redis_host, redis_port, redis_comm, comm_lookup_by, redis_comm_map, redis_comm_index, redis_direction, communication_loc): def comm_lookup(comm): if comm_lookup_by == 'id': return comm.id elif comm_lookup_by == 'uuid': return comm.uuid.uuidString else: error('unsupported comm_lookup_by %s' % comm_lookup_by) input_db = None if redis_comm_index is not None: if redis_direction == RIGHT_TO_LEFT: redis_comm_index = -(redis_comm_index + 1) else: redis_comm_index = redis_comm_index else: redis_comm_index = None if redis_comm is not None and redis_comm_index is not None: error("Cannot include both --redis-comm and --redis-comm-index") input_db = Redis(redis_host, redis_port) reader = RedisCommunicationReader( input_db, communication_loc, add_references=True, right_to_left=(redis_direction == RIGHT_TO_LEFT)) if redis_comm: # look up comm in collection by comm field (comm_lookup_by) key_type = input_db.type(communication_loc) if ((key_type == 'list' and redis_comm_map is None) or (key_type == 'set')): # do linear scan for co in reader: if comm_lookup(co) == redis_comm: comm = co break elif key_type == 'list' and redis_comm_map is not None: # look up list index using field->index map def no_comm(comm_idx): error(('Unable to find a communication with identifier "%s"' ' with value "%s" using pivoting map "%s", which' ' returned (list) index %s, under the %s %s') % (comm_lookup_by, redis_comm, redis_comm_map, str(comm_idx), key_type, communication_loc)) comm_idx = input_db.hget(redis_comm_map, redis_comm) if comm_idx is None: no_comm(comm_idx) comm_idx = int(comm_idx) if redis_direction == RIGHT_TO_LEFT: comm_idx = -(comm_idx + 1) comm_buf = input_db.lindex(communication_loc, comm_idx) comm = read_communication_from_buffer(comm_buf) if comm_lookup(comm) != redis_comm: error(('Cannot find the appropriate document with %s' ' indexing') % redis_direction) elif key_type == 'hash': # do O(1) hash lookup comm_buf = input_db.hget(communication_loc, redis_comm) comm = read_communication_from_buffer(comm_buf) else: error('Unknown key type %s' % (key_type)) if comm is None: error(('Unable to find communication with id %s at %s:%s under' ' key %s') % (redis_comm, redis_host, redis_port, communication_loc)) elif redis_comm_index is not None: # lookup comm by index in list comm_buf = input_db.lindex(communication_loc, redis_comm_index) if comm_buf is None: error(('Unable to find communication with id %s at %s:%s under' ' key %s') % (redis_comm, redis_host, redis_port, communication_loc)) else: comm = read_communication_from_buffer(comm_buf) logging.info('%dth Communication has id %s' % (redis_comm_index + 1, comm.id)) else: # take first comm in collection # (or return single comm stored in simple key) for co in reader: comm = co break if comm is None: error('Unable to find any communications at %s:%s under key %s' % (redis_host, redis_port, communication_loc)) return comm