def setUp(self):
self.comm1 = create_comm('comm-1')
self.comm2 = create_comm('comm-2')
self.comm3 = create_comm('comm-3')
self.buf1 = write_communication_to_buffer(self.comm1)
self.buf2 = write_communication_to_buffer(self.comm2)
self.buf3 = write_communication_to_buffer(self.comm3)
def test_hash(self):
key = 'dataset'
with RedisServer(loglevel='warning') as server:
redis_db = Redis(port=server.port)
redis_db.hset(key, self.comm1.uuid.uuidString,
write_communication_to_buffer(self.comm1))
redis_db.hset(key, self.comm2.uuid.uuidString,
write_communication_to_buffer(self.comm2))
redis_db.hset(key, self.comm3.uuid.uuidString,
write_communication_to_buffer(self.comm3))
reader = RedisCommunicationReader(redis_db, key, key_type='hash')
comms = [c for c in reader]
ids = [c.id for c in comms]
# assert no duplicates
self.assertEquals(3, len(ids))
self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
self.assertEquals(3, len(reader))
self.assertEquals('comm-2', reader[self.comm2.uuid.uuidString].id)
self.assertTrue(hasattr(comms[0], 'sentenceForUUID'))
self.assertTrue(hasattr(comms[1], 'sentenceForUUID'))
self.assertTrue(hasattr(comms[2], 'sentenceForUUID'))
# assert data still there
ids = [c.id for c in reader]
self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
self.assertEquals(3, redis_db.hlen(key))
def test_set(self):
key = 'dataset'
with RedisServer(loglevel='warning') as server:
redis_db = Redis(port=server.port)
redis_db.sadd(key, write_communication_to_buffer(self.comm1))
redis_db.sadd(key, write_communication_to_buffer(self.comm2))
redis_db.sadd(key, write_communication_to_buffer(self.comm3))
reader = RedisCommunicationReader(redis_db, key, key_type='set')
comms = [c for c in reader]
ids = [c.id for c in comms]
# assert no duplicates
self.assertEquals(3, len(ids))
self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
self.assertEquals(3, len(reader))
self.assertTrue(hasattr(comms[0], 'sentenceForUUID'))
self.assertTrue(hasattr(comms[1], 'sentenceForUUID'))
self.assertTrue(hasattr(comms[2], 'sentenceForUUID'))
batch_ids = [c.id for c in reader.batch(2)]
# do this weird thing because set(['foo']) != set([u'foo'])
self.assertTrue(
('comm-1' in batch_ids and 'comm-2' in batch_ids) or
('comm-1' in batch_ids and 'comm-3' in batch_ids) or
('comm-2' in batch_ids and 'comm-3' in batch_ids)
)
# assert data still there
ids = [c.id for c in reader]
self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
self.assertEquals(3, redis_db.scard(key))
def test_read_write_fixed_point(self):
comm = create_comm('comm-1')
buf_1 = write_communication_to_buffer(comm)
buf_2 = write_communication_to_buffer(
read_communication_from_buffer(buf_1)
)
self.assertEquals(buf_1, buf_2)
def test_list_implicit(self):
key = 'dataset'
with RedisServer(loglevel='warning') as server:
redis_db = Redis(port=server.port)
redis_db.lpush(key, write_communication_to_buffer(self.comm1))
redis_db.lpush(key, write_communication_to_buffer(self.comm2))
redis_db.lpush(key, write_communication_to_buffer(self.comm3))
reader = RedisCommunicationReader(redis_db, key)
ids = [c.id for c in reader]
self.assertEquals(['comm-1', 'comm-2', 'comm-3'], ids)
def test_set_implicit(self):
key = 'dataset'
with RedisServer(loglevel='warning') as server:
redis_db = Redis(port=server.port)
redis_db.sadd(key, write_communication_to_buffer(self.comm1))
redis_db.sadd(key, write_communication_to_buffer(self.comm2))
redis_db.sadd(key, write_communication_to_buffer(self.comm3))
reader = RedisCommunicationReader(redis_db, key)
ids = [c.id for c in reader]
# assert no duplicates
self.assertEquals(3, len(ids))
self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
def test_list_no_add_references(self):
key = 'dataset'
with RedisServer(loglevel='warning') as server:
redis_db = Redis(port=server.port)
redis_db.lpush(key, write_communication_to_buffer(self.comm1))
redis_db.lpush(key, write_communication_to_buffer(self.comm2))
redis_db.lpush(key, write_communication_to_buffer(self.comm3))
reader = RedisCommunicationReader(redis_db, key, key_type='list',
add_references=False)
comms = [c for c in reader]
ids = [c.id for c in comms]
self.assertEquals(['comm-1', 'comm-2', 'comm-3'], ids)
self.assertFalse(hasattr(comms[0], 'sentenceForUUID'))
self.assertFalse(hasattr(comms[1], 'sentenceForUUID'))
self.assertFalse(hasattr(comms[2], 'sentenceForUUID'))
def json_str_to_concrete_bytes(tweet_str):
tweet_str = tweet_str.decode('utf-8')
b = json_tweet_string_to_Communication(tweet_str, True, True)
if b is None:
return None
else:
return write_communication_to_buffer(b)
def json_str_to_concrete_bytes(tweet_str):
tweet_str = tweet_str.decode('utf-8')
b = json_tweet_string_to_Communication(tweet_str, True, True)
if b is None:
return None
else:
return write_communication_to_buffer(b)
def test_write_against_file_contents(self):
filename = u'tests/testdata/simple_1.concrete'
with open(filename, 'rb') as f:
f_buf = f.read()
comm = read_communication_from_buffer(f_buf)
buf = write_communication_to_buffer(comm)
self.assertEquals(f_buf, buf)
def test_hash_empty(self):
key = 'dataset'
with RedisServer(loglevel='warning') as server:
redis_db = Redis(port=server.port)
reader = RedisCommunicationReader(redis_db, key, key_type='hash')
self.assertEquals(0, len(reader))
redis_db.hset(key, self.comm1.uuid.uuidString,
write_communication_to_buffer(self.comm1))
redis_db.hset(key, self.comm2.uuid.uuidString,
write_communication_to_buffer(self.comm2))
redis_db.hset(key, self.comm3.uuid.uuidString,
write_communication_to_buffer(self.comm3))
ids = [c.id for c in reader]
# assert no duplicates
self.assertEquals(3, len(ids))
self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
self.assertEquals(3, len(reader))
def test_list(self):
key = 'dataset'
with RedisServer(loglevel='warning') as server:
redis_db = Redis(port=server.port)
redis_db.lpush(key, write_communication_to_buffer(self.comm1))
redis_db.lpush(key, write_communication_to_buffer(self.comm2))
redis_db.lpush(key, write_communication_to_buffer(self.comm3))
reader = RedisCommunicationReader(redis_db, key, key_type='list')
comms = [c for c in reader]
ids = [c.id for c in comms]
self.assertEquals(['comm-1', 'comm-2', 'comm-3'], ids)
self.assertEquals(3, len(reader))
self.assertEquals('comm-2', reader[1].id)
self.assertTrue(hasattr(comms[0], 'sentenceForUUID'))
self.assertTrue(hasattr(comms[1], 'sentenceForUUID'))
self.assertTrue(hasattr(comms[2], 'sentenceForUUID'))
# assert data still there
ids = [c.id for c in reader]
self.assertEquals(['comm-1', 'comm-2', 'comm-3'], ids)
self.assertEquals(3, redis_db.llen(key))
def test_list_block_pop_timeout(self):
key = 'dataset'
with RedisServer(loglevel='warning') as server:
redis_db = Redis(port=server.port)
redis_db.lpush(key, write_communication_to_buffer(self.comm1))
redis_db.lpush(key, write_communication_to_buffer(self.comm2))
redis_db.lpush(key, write_communication_to_buffer(self.comm3))
reader = RedisCommunicationReader(redis_db, key, key_type='list',
pop=True, block=True,
block_timeout=1)
it = iter(reader)
ids = []
ids.append(it.next().id)
ids.append(it.next().id)
self.assertEquals(1, redis_db.llen(key))
ids.append(it.next().id)
self.assertEquals(['comm-1', 'comm-2', 'comm-3'], ids)
with self.assertRaises(StopIteration):
print 'Waiting for timeout (1 sec)...'
it.next()
def test_list_pop_left_to_right(self):
key = 'dataset'
with RedisServer(loglevel='warning') as server:
redis_db = Redis(port=server.port)
redis_db.lpush(key, write_communication_to_buffer(self.comm1))
redis_db.lpush(key, write_communication_to_buffer(self.comm2))
redis_db.lpush(key, write_communication_to_buffer(self.comm3))
reader = RedisCommunicationReader(redis_db, key, key_type='list',
pop=True, right_to_left=False)
it = iter(reader)
ids = []
ids.append(it.next().id)
ids.append(it.next().id)
self.assertEquals(1, redis_db.llen(key))
ids.append(it.next().id)
self.assertEquals(['comm-3', 'comm-2', 'comm-1'], ids)
# assert data is gone
self.assertEquals([], [c.id for c in reader])
self.assertFalse(redis_db.exists(key))
with self.assertRaises(StopIteration):
it.next()
def test_list_block_pop(self):
key = 'dataset'
with RedisServer(loglevel='warning') as server:
redis_db = Redis(port=server.port)
redis_db.lpush(key, write_communication_to_buffer(self.comm1))
redis_db.lpush(key, write_communication_to_buffer(self.comm2))
redis_db.lpush(key, write_communication_to_buffer(self.comm3))
reader = RedisCommunicationReader(redis_db, key, key_type='list',
pop=True, block=True)
it = iter(reader)
ids = []
ids.append(it.next().id)
ids.append(it.next().id)
self.assertEquals(1, redis_db.llen(key))
ids.append(it.next().id)
self.assertEquals(['comm-1', 'comm-2', 'comm-3'], ids)
proc = Process(target=_add_comm_to_list,
args=(3, server.port, 'comm-4', key))
proc.start()
print 'Waiting for new comm to be added (3 sec)...'
self.assertEquals('comm-4', iter(reader).next().id)
proc.join()
def test_set_pop(self):
key = 'dataset'
with RedisServer(loglevel='warning') as server:
redis_db = Redis(port=server.port)
redis_db.sadd(key, write_communication_to_buffer(self.comm1))
redis_db.sadd(key, write_communication_to_buffer(self.comm2))
redis_db.sadd(key, write_communication_to_buffer(self.comm3))
reader = RedisCommunicationReader(redis_db, key, key_type='set',
pop=True)
it = iter(reader)
ids = []
ids.append(it.next().id)
ids.append(it.next().id)
self.assertEquals(1, redis_db.scard(key))
ids.append(it.next().id)
# assert no duplicates
self.assertEquals(3, len(ids))
self.assertEquals(set(['comm-1', 'comm-2', 'comm-3']), set(ids))
# assert data is gone
self.assertEquals([], [c.id for c in reader])
self.assertFalse(redis_db.exists(key))
with self.assertRaises(StopIteration):
it.next()
def _write_to_buffer(self, comm):
return write_communication_to_buffer(comm)
def write_communication_to_redis_key(redis_db, key, comm):
'''
Serialize communication and store result in redis key.
'''
redis_db.set(key, write_communication_to_buffer(comm))
def _add_comm_to_list(sleep, port, comm_id, key):
time.sleep(sleep)
redis_db = Redis(port=port)
comm = create_comm(comm_id)
buf = write_communication_to_buffer(comm)
redis_db.lpush(key, buf)
