def test_index(self):
''' Inserts events in to a stream and ensures that the index column
family contains all the of expected row keys. '''
stream_name = 'TestCassandraBackend_test_index'
stream = self.namespace.get_stream(stream_name, self.width, self.shards)
for i in xrange(100):
self.put(stream_name, [{TIMESTAMP_FIELD:
epoch_time_to_kronos_time(i % 20)}])
events = self.get(stream_name, 0, epoch_time_to_kronos_time(20))
self.assertEqual(len(events), 100)
time_shards = defaultdict(set)
for shard_desc in stream.get_overlapping_shards(
0, epoch_time_to_kronos_time(20)):
self.assertEqual(shard_desc['width'], self.width)
time_shards[shard_desc['start_time']] |= {shard_desc['shard']}
for shards in time_shards.itervalues():
self.assertEqual(len(shards), self.shards)
self.assertEqual(shards, set(xrange(self.shards)))
self.assertEqual(set(time_shards),
set(round_down(epoch_time_to_kronos_time(t), self.width)
for t in xrange(20)))
def insert(self, events):
if not events:
return
batch_stmt = BatchStatement(batch_type=BatchType.UNLOGGED,
consistency_level=ConsistencyLevel.QUORUM)
shard_idx = {}
for _id, event in events:
shard_time = round_down(event[TIMESTAMP_FIELD], self.width)
shard = shard_idx.get(shard_time,
random.randint(0, self.shards - 1))
# Insert to index.
try:
self.index_cache.get((shard_time, shard))
except KeyError:
batch_stmt.add(BoundStatement(self.namespace.INDEX_INSERT_STMT,
routing_key=self.stream,
consistency_level=ConsistencyLevel.QUORUM)
.bind((self.stream, shard_time, self.width, shard)))
self.index_cache.set((shard_time, shard), None)
# Insert to stream.
shard_key = StreamShard.get_key(self.stream, shard_time, shard)
batch_stmt.add(BoundStatement(self.namespace.INSERT_STMT,
routing_key=shard_key,
consistency_level=ConsistencyLevel.QUORUM)
.bind((shard_key,
_id,
marshal.dumps(event))))
shard_idx[shard_time] = (shard + 1) % self.shards # Round robin.
self.session.execute(batch_stmt)
def test_index(self):
''' Inserts events in to a stream and ensures that the index column
family contains all the of expected row keys. '''
stream_name = 'TestCassandraBackend_test_index'
stream = self.namespace.get_stream(stream_name, self.width, self.shards)
for i in xrange(100):
self.put(stream_name, [{TIMESTAMP_FIELD:
epoch_time_to_kronos_time(i % 20)}])
events = self.get(stream_name, 0, epoch_time_to_kronos_time(20))
self.assertEqual(len(events), 100)
time_shards = defaultdict(set)
for shard_desc in stream.get_overlapping_shards(
0, epoch_time_to_kronos_time(20)):
self.assertEqual(shard_desc['width'], self.width)
time_shards[shard_desc['start_time']] |= {shard_desc['shard']}
for shards in time_shards.itervalues():
self.assertEqual(len(shards), self.shards)
self.assertEqual(shards, set(xrange(self.shards)))
self.assertEqual(set(time_shards),
set(round_down(epoch_time_to_kronos_time(t), self.width)
for t in xrange(20)))