Exemplo n.º 1
0
 def setUp(self):
     self.comm1 = create_comm('comm-1')
     self.comm2 = create_comm('comm-2')
     self.comm3 = create_comm('comm-3')
     self.buf1 = write_communication_to_buffer(self.comm1)
     self.buf2 = write_communication_to_buffer(self.comm2)
     self.buf3 = write_communication_to_buffer(self.comm3)
Exemplo n.º 2
0
 def test_hash(self):
     key = 'dataset'
     with RedisServer(loglevel='warning') as server:
         redis_db = Redis(port=server.port)
         redis_db.hset(key, self.comm1.uuid.uuidString,
                       write_communication_to_buffer(self.comm1))
         redis_db.hset(key, self.comm2.uuid.uuidString,
                       write_communication_to_buffer(self.comm2))
         redis_db.hset(key, self.comm3.uuid.uuidString,
                       write_communication_to_buffer(self.comm3))
         reader = RedisCommunicationReader(redis_db, key, key_type='hash')
         comms = [c for c in reader]
         ids = [c.id for c in comms]
         # assert no duplicates
         self.assertEquals(3, len(ids))
         self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
         self.assertEquals(3, len(reader))
         self.assertEquals('comm-2', reader[self.comm2.uuid.uuidString].id)
         self.assertTrue(hasattr(comms[0], 'sentenceForUUID'))
         self.assertTrue(hasattr(comms[1], 'sentenceForUUID'))
         self.assertTrue(hasattr(comms[2], 'sentenceForUUID'))
         # assert data still there
         ids = [c.id for c in reader]
         self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
         self.assertEquals(3, redis_db.hlen(key))
Exemplo n.º 3
0
 def test_set(self):
     key = 'dataset'
     with RedisServer(loglevel='warning') as server:
         redis_db = Redis(port=server.port)
         redis_db.sadd(key, write_communication_to_buffer(self.comm1))
         redis_db.sadd(key, write_communication_to_buffer(self.comm2))
         redis_db.sadd(key, write_communication_to_buffer(self.comm3))
         reader = RedisCommunicationReader(redis_db, key, key_type='set')
         comms = [c for c in reader]
         ids = [c.id for c in comms]
         # assert no duplicates
         self.assertEquals(3, len(ids))
         self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
         self.assertEquals(3, len(reader))
         self.assertTrue(hasattr(comms[0], 'sentenceForUUID'))
         self.assertTrue(hasattr(comms[1], 'sentenceForUUID'))
         self.assertTrue(hasattr(comms[2], 'sentenceForUUID'))
         batch_ids = [c.id for c in reader.batch(2)]
         # do this weird thing because set(['foo']) != set([u'foo'])
         self.assertTrue(
             ('comm-1' in batch_ids and 'comm-2' in batch_ids) or
             ('comm-1' in batch_ids and 'comm-3' in batch_ids) or
             ('comm-2' in batch_ids and 'comm-3' in batch_ids)
         )
         # assert data still there
         ids = [c.id for c in reader]
         self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
         self.assertEquals(3, redis_db.scard(key))
Exemplo n.º 4
0
 def test_read_write_fixed_point(self):
     comm = create_comm('comm-1')
     buf_1 = write_communication_to_buffer(comm)
     buf_2 = write_communication_to_buffer(
         read_communication_from_buffer(buf_1)
     )
     self.assertEquals(buf_1, buf_2)
Exemplo n.º 5
0
 def test_list_implicit(self):
     key = 'dataset'
     with RedisServer(loglevel='warning') as server:
         redis_db = Redis(port=server.port)
         redis_db.lpush(key, write_communication_to_buffer(self.comm1))
         redis_db.lpush(key, write_communication_to_buffer(self.comm2))
         redis_db.lpush(key, write_communication_to_buffer(self.comm3))
         reader = RedisCommunicationReader(redis_db, key)
         ids = [c.id for c in reader]
         self.assertEquals(['comm-1', 'comm-2', 'comm-3'], ids)
Exemplo n.º 6
0
 def test_set_implicit(self):
     key = 'dataset'
     with RedisServer(loglevel='warning') as server:
         redis_db = Redis(port=server.port)
         redis_db.sadd(key, write_communication_to_buffer(self.comm1))
         redis_db.sadd(key, write_communication_to_buffer(self.comm2))
         redis_db.sadd(key, write_communication_to_buffer(self.comm3))
         reader = RedisCommunicationReader(redis_db, key)
         ids = [c.id for c in reader]
         # assert no duplicates
         self.assertEquals(3, len(ids))
         self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
Exemplo n.º 7
0
 def test_list_no_add_references(self):
     key = 'dataset'
     with RedisServer(loglevel='warning') as server:
         redis_db = Redis(port=server.port)
         redis_db.lpush(key, write_communication_to_buffer(self.comm1))
         redis_db.lpush(key, write_communication_to_buffer(self.comm2))
         redis_db.lpush(key, write_communication_to_buffer(self.comm3))
         reader = RedisCommunicationReader(redis_db, key, key_type='list',
                                           add_references=False)
         comms = [c for c in reader]
         ids = [c.id for c in comms]
         self.assertEquals(['comm-1', 'comm-2', 'comm-3'], ids)
         self.assertFalse(hasattr(comms[0], 'sentenceForUUID'))
         self.assertFalse(hasattr(comms[1], 'sentenceForUUID'))
         self.assertFalse(hasattr(comms[2], 'sentenceForUUID'))
Exemplo n.º 8
0
def json_str_to_concrete_bytes(tweet_str):
    tweet_str = tweet_str.decode('utf-8')
    b = json_tweet_string_to_Communication(tweet_str, True, True)
    if b is None:
        return None
    else:
        return write_communication_to_buffer(b)
Exemplo n.º 9
0
def json_str_to_concrete_bytes(tweet_str):
    tweet_str = tweet_str.decode('utf-8')
    b = json_tweet_string_to_Communication(tweet_str, True, True)
    if b is None:
        return None
    else:
        return write_communication_to_buffer(b)
Exemplo n.º 10
0
 def test_write_against_file_contents(self):
     filename = u'tests/testdata/simple_1.concrete'
     with open(filename, 'rb') as f:
         f_buf = f.read()
         comm = read_communication_from_buffer(f_buf)
     buf = write_communication_to_buffer(comm)
     self.assertEquals(f_buf, buf)
Exemplo n.º 11
0
 def test_hash_empty(self):
     key = 'dataset'
     with RedisServer(loglevel='warning') as server:
         redis_db = Redis(port=server.port)
         reader = RedisCommunicationReader(redis_db, key, key_type='hash')
         self.assertEquals(0, len(reader))
         redis_db.hset(key, self.comm1.uuid.uuidString,
                       write_communication_to_buffer(self.comm1))
         redis_db.hset(key, self.comm2.uuid.uuidString,
                       write_communication_to_buffer(self.comm2))
         redis_db.hset(key, self.comm3.uuid.uuidString,
                       write_communication_to_buffer(self.comm3))
         ids = [c.id for c in reader]
         # assert no duplicates
         self.assertEquals(3, len(ids))
         self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
         self.assertEquals(3, len(reader))
Exemplo n.º 12
0
 def test_list(self):
     key = 'dataset'
     with RedisServer(loglevel='warning') as server:
         redis_db = Redis(port=server.port)
         redis_db.lpush(key, write_communication_to_buffer(self.comm1))
         redis_db.lpush(key, write_communication_to_buffer(self.comm2))
         redis_db.lpush(key, write_communication_to_buffer(self.comm3))
         reader = RedisCommunicationReader(redis_db, key, key_type='list')
         comms = [c for c in reader]
         ids = [c.id for c in comms]
         self.assertEquals(['comm-1', 'comm-2', 'comm-3'], ids)
         self.assertEquals(3, len(reader))
         self.assertEquals('comm-2', reader[1].id)
         self.assertTrue(hasattr(comms[0], 'sentenceForUUID'))
         self.assertTrue(hasattr(comms[1], 'sentenceForUUID'))
         self.assertTrue(hasattr(comms[2], 'sentenceForUUID'))
         # assert data still there
         ids = [c.id for c in reader]
         self.assertEquals(['comm-1', 'comm-2', 'comm-3'], ids)
         self.assertEquals(3, redis_db.llen(key))
Exemplo n.º 13
0
 def test_list_block_pop_timeout(self):
     key = 'dataset'
     with RedisServer(loglevel='warning') as server:
         redis_db = Redis(port=server.port)
         redis_db.lpush(key, write_communication_to_buffer(self.comm1))
         redis_db.lpush(key, write_communication_to_buffer(self.comm2))
         redis_db.lpush(key, write_communication_to_buffer(self.comm3))
         reader = RedisCommunicationReader(redis_db, key, key_type='list',
                                           pop=True, block=True,
                                           block_timeout=1)
         it = iter(reader)
         ids = []
         ids.append(it.next().id)
         ids.append(it.next().id)
         self.assertEquals(1, redis_db.llen(key))
         ids.append(it.next().id)
         self.assertEquals(['comm-1', 'comm-2', 'comm-3'], ids)
         with self.assertRaises(StopIteration):
             print 'Waiting for timeout (1 sec)...'
             it.next()
Exemplo n.º 14
0
 def test_list_pop_left_to_right(self):
     key = 'dataset'
     with RedisServer(loglevel='warning') as server:
         redis_db = Redis(port=server.port)
         redis_db.lpush(key, write_communication_to_buffer(self.comm1))
         redis_db.lpush(key, write_communication_to_buffer(self.comm2))
         redis_db.lpush(key, write_communication_to_buffer(self.comm3))
         reader = RedisCommunicationReader(redis_db, key, key_type='list',
                                           pop=True, right_to_left=False)
         it = iter(reader)
         ids = []
         ids.append(it.next().id)
         ids.append(it.next().id)
         self.assertEquals(1, redis_db.llen(key))
         ids.append(it.next().id)
         self.assertEquals(['comm-3', 'comm-2', 'comm-1'], ids)
         # assert data is gone
         self.assertEquals([], [c.id for c in reader])
         self.assertFalse(redis_db.exists(key))
         with self.assertRaises(StopIteration):
             it.next()
Exemplo n.º 15
0
 def test_list_block_pop(self):
     key = 'dataset'
     with RedisServer(loglevel='warning') as server:
         redis_db = Redis(port=server.port)
         redis_db.lpush(key, write_communication_to_buffer(self.comm1))
         redis_db.lpush(key, write_communication_to_buffer(self.comm2))
         redis_db.lpush(key, write_communication_to_buffer(self.comm3))
         reader = RedisCommunicationReader(redis_db, key, key_type='list',
                                           pop=True, block=True)
         it = iter(reader)
         ids = []
         ids.append(it.next().id)
         ids.append(it.next().id)
         self.assertEquals(1, redis_db.llen(key))
         ids.append(it.next().id)
         self.assertEquals(['comm-1', 'comm-2', 'comm-3'], ids)
         proc = Process(target=_add_comm_to_list,
                        args=(3, server.port, 'comm-4', key))
         proc.start()
         print 'Waiting for new comm to be added (3 sec)...'
         self.assertEquals('comm-4', iter(reader).next().id)
         proc.join()
Exemplo n.º 16
0
 def test_set_pop(self):
     key = 'dataset'
     with RedisServer(loglevel='warning') as server:
         redis_db = Redis(port=server.port)
         redis_db.sadd(key, write_communication_to_buffer(self.comm1))
         redis_db.sadd(key, write_communication_to_buffer(self.comm2))
         redis_db.sadd(key, write_communication_to_buffer(self.comm3))
         reader = RedisCommunicationReader(redis_db, key, key_type='set',
                                           pop=True)
         it = iter(reader)
         ids = []
         ids.append(it.next().id)
         ids.append(it.next().id)
         self.assertEquals(1, redis_db.scard(key))
         ids.append(it.next().id)
         # assert no duplicates
         self.assertEquals(3, len(ids))
         self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
         # assert data is gone
         self.assertEquals([], [c.id for c in reader])
         self.assertFalse(redis_db.exists(key))
         with self.assertRaises(StopIteration):
             it.next()
Exemplo n.º 17
0
 def _write_to_buffer(self, comm):
     return write_communication_to_buffer(comm)
Exemplo n.º 18
0
def write_communication_to_redis_key(redis_db, key, comm):
    '''
    Serialize communication and store result in redis key.
    '''
    redis_db.set(key, write_communication_to_buffer(comm))
Exemplo n.º 19
0
def _add_comm_to_list(sleep, port, comm_id, key):
    time.sleep(sleep)
    redis_db = Redis(port=port)
    comm = create_comm(comm_id)
    buf = write_communication_to_buffer(comm)
    redis_db.lpush(key, buf)