def test_write_against_file_contents(self):
     filename = u'tests/testdata/simple_1.concrete'
     with open(filename, 'rb') as f:
         f_buf = f.read()
         comm = read_communication_from_buffer(f_buf)
     buf = write_communication_to_buffer(comm)
     self.assertEquals(f_buf, buf)
 def test_read_against_file_contents_no_add_references(self):
     filename = u'tests/testdata/simple_1.concrete'
     with open(filename, 'rb') as f:
         buf = f.read()
         comm = read_communication_from_buffer(buf, add_references=False)
         self.assertFalse(hasattr(comm, 'sentenceForUUID'))
         self.assertEquals('one', comm.id)
 def test_read_write_fixed_point(self):
     comm = create_comm('comm-1')
     buf_1 = write_communication_to_buffer(comm)
     buf_2 = write_communication_to_buffer(
         read_communication_from_buffer(buf_1)
     )
     self.assertEquals(buf_1, buf_2)
示例#4
0
 def test_write_against_file_contents(self):
     filename = u'tests/testdata/simple_1.concrete'
     key = 'comm'
     with open(filename, 'rb') as f:
         f_buf = f.read()
         comm = read_communication_from_buffer(f_buf)
         with RedisServer(loglevel='warning') as server:
             redis_db = Redis(port=server.port)
             write_communication_to_redis_key(redis_db, key, comm)
             self.assertEquals(f_buf, redis_db.get(key))
示例#5
0
def read_communication_from_redis_key(redis_db, key, add_references=True):
    '''
    Return a serialized communication from a string key.  If block
    is True, poll server until key appears at specified interval
    or until specified timeout (indefinitely if timeout is zero).
    Return None if block is False and key does not exist or if
    block is True and key does not exist after specified timeout.
    '''
    buf = redis_db.get(key)
    if buf is None:
        return None
    else:
        return read_communication_from_buffer(buf,
                                              add_references=add_references)
示例#6
0
文件: qlook.py 项目: hltcoe/quicklime
def get_restful_comm(restful_host, restful_port, restful_pattern, communication_loc):
    url = urlparse('%s:%s' % (restful_host, restful_port))
    if url.netloc is None or len(url.netloc) == 0:
        h = 'http://%s' % (restful_host)
    else:
        h = '%s://%s' % (url.scheme, url.netloc)
    loc_pattern = restful_pattern % (communication_loc)
    logging.info('using location pattern %s' % loc_pattern)
    full = '%s:%s/%s' % (h, restful_port, loc_pattern)
    logging.info("querying %s" % full)
    resp = urlopen(full)
    if resp is None:
        error("Got back a None from querying %s" % (full))
    if resp.code != 200:
        error("received code %d and message %s from %s" % (
            resp.code, resp.msg, full))
    comm_buf = resp.read()
    resp.close()
    comm = read_communication_from_buffer(comm_buf)
    return comm
示例#7
0
def get_restful_comm(restful_host, restful_port, restful_pattern,
                     communication_loc):
    url = urlparse('%s:%s' % (restful_host, restful_port))
    if url.netloc is None or len(url.netloc) == 0:
        h = 'http://%s' % (restful_host)
    else:
        h = '%s://%s' % (url.scheme, url.netloc)
    loc_pattern = restful_pattern % (communication_loc)
    logging.info('using location pattern %s' % loc_pattern)
    full = '%s:%s/%s' % (h, restful_port, loc_pattern)
    logging.info("querying %s" % full)
    resp = urlopen(full)
    if resp is None:
        error("Got back a None from querying %s" % (full))
    if resp.code != 200:
        error("received code %d and message %s from %s" %
              (resp.code, resp.msg, full))
    comm_buf = resp.read()
    resp.close()
    comm = read_communication_from_buffer(comm_buf)
    return comm
示例#8
0
 def _load_from_buffer(self, buf):
     return read_communication_from_buffer(
         buf, add_references=self.add_references)
示例#9
0
文件: qlook.py 项目: hltcoe/quicklime
def get_redis_comm(redis_host, redis_port, redis_comm, comm_lookup_by,
                   redis_comm_map, redis_comm_index, redis_direction,
                   communication_loc):
    def comm_lookup(comm):
        if comm_lookup_by == 'id':
            return comm.id
        elif comm_lookup_by == 'uuid':
            return comm.uuid.uuidString
        else:
            error('unsupported comm_lookup_by %s' % comm_lookup_by)

    input_db = None

    if redis_comm_index is not None:
        if redis_direction == RIGHT_TO_LEFT:
            redis_comm_index = -(redis_comm_index + 1)
        else:
            redis_comm_index = redis_comm_index
    else:
        redis_comm_index = None

    if redis_comm is not None and redis_comm_index is not None:
        error("Cannot include both --redis-comm and --redis-comm-index")

    input_db = Redis(redis_host, redis_port)
    reader = RedisCommunicationReader(input_db, communication_loc,
                                      add_references=True,
                                      right_to_left=(redis_direction ==
                                                     RIGHT_TO_LEFT))

    if redis_comm:
        # look up comm in collection by comm field (comm_lookup_by)
        key_type = input_db.type(communication_loc)

        if ((key_type == 'list' and redis_comm_map is None) or
                (key_type == 'set')):
            # do linear scan
            for co in reader:
                if comm_lookup(co) == redis_comm:
                    comm = co
                    break

        elif key_type == 'list' and redis_comm_map is not None:
            # look up list index using field->index map
            def no_comm(comm_idx):
                error(('Unable to find a communication with identifier "%s"'
                       ' with value "%s" using pivoting map "%s", which'
                       ' returned (list) index %s, under the %s %s') %
                      (comm_lookup_by,
                       redis_comm,
                       redis_comm_map,
                       str(comm_idx),
                       key_type,
                       communication_loc))

            comm_idx = input_db.hget(redis_comm_map, redis_comm)
            if comm_idx is None:
                no_comm(comm_idx)
            comm_idx = int(comm_idx)
            if redis_direction == RIGHT_TO_LEFT:
                comm_idx = - (comm_idx + 1)

            comm_buf = input_db.lindex(communication_loc, comm_idx)
            comm = read_communication_from_buffer(comm_buf)

            if comm_lookup(comm) != redis_comm:
                error(('Cannot find the appropriate document with %s'
                       ' indexing') % redis_direction)

        elif key_type == 'hash':
            # do O(1) hash lookup
            comm_buf = input_db.hget(communication_loc, redis_comm)
            comm = read_communication_from_buffer(comm_buf)

        else:
            error('Unknown key type %s' % (key_type))

        if comm is None:
            error(('Unable to find communication with id %s at %s:%s under'
                   ' key %s') %
                  (redis_comm, redis_host, redis_port,
                   communication_loc))

    elif redis_comm_index is not None:
        # lookup comm by index in list
        comm_buf = input_db.lindex(communication_loc, redis_comm_index)
        if comm_buf is None:
            error(('Unable to find communication with id %s at %s:%s under'
                   ' key %s') %
                  (redis_comm, redis_host, redis_port,
                   communication_loc))
        else:
            comm = read_communication_from_buffer(comm_buf)
            logging.info('%dth Communication has id %s' %
                         (redis_comm_index + 1, comm.id))

    else:
        # take first comm in collection
        # (or return single comm stored in simple key)
        for co in reader:
            comm = co
            break

        if comm is None:
            error('Unable to find any communications at %s:%s under key %s' %
                  (redis_host, redis_port, communication_loc))
    return comm
示例#10
0
def get_redis_comm(redis_host, redis_port, redis_comm, comm_lookup_by,
                   redis_comm_map, redis_comm_index, redis_direction,
                   communication_loc):
    def comm_lookup(comm):
        if comm_lookup_by == 'id':
            return comm.id
        elif comm_lookup_by == 'uuid':
            return comm.uuid.uuidString
        else:
            error('unsupported comm_lookup_by %s' % comm_lookup_by)

    input_db = None

    if redis_comm_index is not None:
        if redis_direction == RIGHT_TO_LEFT:
            redis_comm_index = -(redis_comm_index + 1)
        else:
            redis_comm_index = redis_comm_index
    else:
        redis_comm_index = None

    if redis_comm is not None and redis_comm_index is not None:
        error("Cannot include both --redis-comm and --redis-comm-index")

    input_db = Redis(redis_host, redis_port)
    reader = RedisCommunicationReader(
        input_db,
        communication_loc,
        add_references=True,
        right_to_left=(redis_direction == RIGHT_TO_LEFT))

    if redis_comm:
        # look up comm in collection by comm field (comm_lookup_by)
        key_type = input_db.type(communication_loc)

        if ((key_type == 'list' and redis_comm_map is None)
                or (key_type == 'set')):
            # do linear scan
            for co in reader:
                if comm_lookup(co) == redis_comm:
                    comm = co
                    break

        elif key_type == 'list' and redis_comm_map is not None:
            # look up list index using field->index map
            def no_comm(comm_idx):
                error(('Unable to find a communication with identifier "%s"'
                       ' with value "%s" using pivoting map "%s", which'
                       ' returned (list) index %s, under the %s %s') %
                      (comm_lookup_by, redis_comm, redis_comm_map,
                       str(comm_idx), key_type, communication_loc))

            comm_idx = input_db.hget(redis_comm_map, redis_comm)
            if comm_idx is None:
                no_comm(comm_idx)
            comm_idx = int(comm_idx)
            if redis_direction == RIGHT_TO_LEFT:
                comm_idx = -(comm_idx + 1)

            comm_buf = input_db.lindex(communication_loc, comm_idx)
            comm = read_communication_from_buffer(comm_buf)

            if comm_lookup(comm) != redis_comm:
                error(('Cannot find the appropriate document with %s'
                       ' indexing') % redis_direction)

        elif key_type == 'hash':
            # do O(1) hash lookup
            comm_buf = input_db.hget(communication_loc, redis_comm)
            comm = read_communication_from_buffer(comm_buf)

        else:
            error('Unknown key type %s' % (key_type))

        if comm is None:
            error(('Unable to find communication with id %s at %s:%s under'
                   ' key %s') %
                  (redis_comm, redis_host, redis_port, communication_loc))

    elif redis_comm_index is not None:
        # lookup comm by index in list
        comm_buf = input_db.lindex(communication_loc, redis_comm_index)
        if comm_buf is None:
            error(('Unable to find communication with id %s at %s:%s under'
                   ' key %s') %
                  (redis_comm, redis_host, redis_port, communication_loc))
        else:
            comm = read_communication_from_buffer(comm_buf)
            logging.info('%dth Communication has id %s' %
                         (redis_comm_index + 1, comm.id))

    else:
        # take first comm in collection
        # (or return single comm stored in simple key)
        for co in reader:
            comm = co
            break

        if comm is None:
            error('Unable to find any communications at %s:%s under key %s' %
                  (redis_host, redis_port, communication_loc))
    return comm