def test_rollover(self): settings.storage.elasticsearch.rollover_size = 10 settings.storage.elasticsearch.rollover_check_period_seconds = 2 reload_router() indices = set() for i in xrange(50): indices.add(router.get_backend('elasticsearch') .index_manager.get_index('kronos')) self.put('test_rollover', [{TIMESTAMP_FIELD: 0} for _ in xrange(5)]) gevent.sleep(0.05) # Has index rolled over? self.assertTrue(len(indices) > 1) # No events were lost? events = self.get('test_rollover', 0, 1) self.assertEqual(len(events), 50 * 5) self.assertEqual(events, sorted(events, key=lambda e: TimeUUID(e[ID_FIELD]))) es = router.get_backend('elasticsearch').es index = router.get_backend('elasticsearch').index_manager.get_index( 'kronos') indices.discard(index) for index in indices: self.assertTrue(es.count(index=index)['count'] >= 10)
def test_rollover(self): settings.storage.elasticsearch.rollover_size = 10 settings.storage.elasticsearch.rollover_check_period_seconds = 2 reload_router() indices = set() for i in xrange(50): indices.add(router.get_backend('elasticsearch') .index_manager.get_index('kronos')) self.put('test_rollover', [{TIMESTAMP_FIELD: 0} for _ in xrange(5)]) gevent.sleep(0.05) # Has index rolled over? self.assertTrue(len(indices) > 1) # No events were lost? events = self.get('test_rollover', 0, 1) self.assertEqual(len(events), 50 * 5) self.assertEqual(events, sorted(events, key=lambda e: TimeUUID(e[ID_FIELD]))) es = router.get_backend('elasticsearch').es index = router.get_backend('elasticsearch').index_manager.get_index( 'kronos') indices.discard(index) for index in indices: self.assertTrue(es.count(index=index)['count'] >= 10)
def test_concurrent_rollover(self): settings.storage.elasticsearch.rollover_size = 10 reload_router() index_manager1 = router.get_backend('elasticsearch').index_manager index_manager1.kill_rollover_worker() # Create another IndexManager. This should pull index names from the # previously created directory by index_manager1. index_manager2 = IndexManager(router.get_backend('elasticsearch')) index_manager2.kill_rollover_worker() index1 = index_manager1.get_index('kronos') self.assertEqual(index_manager1.get_index('kronos'), index_manager2.get_index('kronos')) self.put('test_concurrent_rollover', [{TIMESTAMP_FIELD: 0} for _ in xrange(10)]) # Update index_manager2. It should rollover. index_manager2.update() self.assertEqual(index1, index_manager1.get_index('kronos')) self.assertNotEqual(index_manager1.get_index('kronos'), index_manager2.get_index('kronos')) # This should read updated index set by index_manager2. index_manager1.update() self.assertEqual(index_manager1.get_index('kronos'), index_manager2.get_index('kronos'))
def test_concurrent_rollover(self): settings.storage.elasticsearch.rollover_size = 10 reload_router() index_manager1 = router.get_backend('elasticsearch').index_manager index_manager1.kill_rollover_worker() # Create another IndexManager. This should pull index names from the # previously created directory by index_manager1. index_manager2 = IndexManager(router.get_backend('elasticsearch')) index_manager2.kill_rollover_worker() index1 = index_manager1.get_index('kronos') self.assertEqual(index_manager1.get_index('kronos'), index_manager2.get_index('kronos')) self.put('test_concurrent_rollover', [{TIMESTAMP_FIELD: 0} for _ in xrange(10)]) # Update index_manager2. It should rollover. index_manager2.update() self.assertEqual(index1, index_manager1.get_index('kronos')) self.assertNotEqual(index_manager1.get_index('kronos'), index_manager2.get_index('kronos')) # This should read updated index set by index_manager2. index_manager1.update() self.assertEqual(index_manager1.get_index('kronos'), index_manager2.get_index('kronos'))
def clear_backend(backend_name): from kronos.storage.router import router router.get_backend(backend_name)._clear()
def clear_backend(backend_name): from kronos.storage.router import router router.get_backend(backend_name)._clear()
def setUp(self): super(TestCassandraBackend, self).setUp() self.namespace = router.get_backend('cassandra').namespaces['kronos'] self.shards = settings.storage['cassandra']['shards_per_bucket'] self.width_seconds = settings.storage['cassandra']['timewidth_seconds'] self.width = epoch_time_to_kronos_time(self.width_seconds)
def test_aliasing(self): settings.storage.elasticsearch.rollover_size = 10 reload_router(kill_update_thread=True) index1 = router.get_backend('elasticsearch').index_manager.get_index( 'kronos') time1 = datetime_to_kronos_time(datetime(2014, 1, 1, 0)) time2 = datetime_to_kronos_time(datetime(2014, 1, 2, 0)) time3 = datetime_to_kronos_time(datetime(2014, 1, 3, 0)) self.put('test_aliasing', [{TIMESTAMP_FIELD: time1, 'i': i, 'j': 0} for i in xrange(5)]) self.put('test_aliasing', [{TIMESTAMP_FIELD: time2, 'i': i, 'j': 0} for i in xrange(5)]) router.get_backend('elasticsearch').index_manager.update() index2 = router.get_backend('elasticsearch').index_manager.get_index( 'kronos') self.put('test_aliasing', [{TIMESTAMP_FIELD: time2, 'i': i, 'j': 1} for i in xrange(5)]) self.put('test_aliasing', [{TIMESTAMP_FIELD: time3, 'i': i, 'j': 1} for i in xrange(5)]) router.get_backend('elasticsearch').index_manager.update() index3 = router.get_backend('elasticsearch').index_manager.get_index( 'kronos') # Has index rolled over each time? self.assertTrue(index1 != index2) self.assertTrue(index2 != index3) self.assertTrue(index1 != index3) events = self.get('test_aliasing', time1, time3) self.assertEqual(len(events), 20) self.assertEqual(events, sorted(events, key=lambda e: TimeUUID(e[ID_FIELD]))) events = self.get('test_aliasing', time1, time2) self.assertEqual(len(events), 15) self.assertEqual(events, sorted(events, key=lambda e: TimeUUID(e[ID_FIELD]))) events = self.get('test_aliasing', time2, time3) self.assertEqual(len(events), 15) self.assertEqual(events, sorted(events, key=lambda e: TimeUUID(e[ID_FIELD]))) es = router.get_backend('elasticsearch').es aliases = es.indices.get_aliases(index=[index1, index2, index3]) self.assertEqual(len(aliases), 2) self.assertTrue(index1 in aliases) self.assertTrue(index2 in aliases) self.assertEqual(set(aliases[index1]['aliases']) & set(aliases[index2]['aliases']), set(['kronos_test:kronos:2014.01.02'])) self.assertEqual(set(aliases[index1]['aliases']), set(['kronos_test:kronos:2014.01.01', 'kronos_test:kronos:2014.01.02'])) self.assertEqual(set(aliases[index2]['aliases']), set(['kronos_test:kronos:2014.01.02', 'kronos_test:kronos:2014.01.03'])) self.assertEqual(es.count(index=index1)['count'], 10) self.assertEqual(es.count(index=index2)['count'], 10) self.assertEqual(es.count(index=index3, ignore_unavailable=True).get('count', 0), 0)
def reload_router(kill_update_thread=False): router.reload() backend = router.get_backend('elasticsearch') if kill_update_thread: backend.index_manager.kill_rollover_worker()
def setUp(self): super(TestCassandraBackend, self).setUp() self.namespace = router.get_backend('cassandra').namespaces['kronos'] self.shards = settings.storage['cassandra']['shards_per_bucket'] self.width_seconds = settings.storage['cassandra']['timewidth_seconds'] self.width = epoch_time_to_kronos_time(self.width_seconds)
def setUp(self): super(TestCassandraBackend, self).setUp() self.namespace = router.get_backend("cassandra").namespaces["kronos"] self.shards = settings.storage["cassandra"]["shards_per_bucket"] self.width_seconds = settings.storage["cassandra"]["timewidth_seconds"] self.width = epoch_time_to_kronos_time(self.width_seconds)
def test_aliasing(self): settings.storage.elasticsearch.rollover_size = 10 reload_router(kill_update_thread=True) index1 = router.get_backend('elasticsearch').index_manager.get_index( 'kronos') time1 = datetime_to_kronos_time(datetime(2014, 1, 1, 0)) time2 = datetime_to_kronos_time(datetime(2014, 1, 2, 0)) time3 = datetime_to_kronos_time(datetime(2014, 1, 3, 0)) self.put('test_aliasing', [{TIMESTAMP_FIELD: time1, 'i': i, 'j': 0} for i in xrange(5)]) self.put('test_aliasing', [{TIMESTAMP_FIELD: time2, 'i': i, 'j': 0} for i in xrange(5)]) router.get_backend('elasticsearch').index_manager.update() index2 = router.get_backend('elasticsearch').index_manager.get_index( 'kronos') self.put('test_aliasing', [{TIMESTAMP_FIELD: time2, 'i': i, 'j': 1} for i in xrange(5)]) self.put('test_aliasing', [{TIMESTAMP_FIELD: time3, 'i': i, 'j': 1} for i in xrange(5)]) router.get_backend('elasticsearch').index_manager.update() index3 = router.get_backend('elasticsearch').index_manager.get_index( 'kronos') # Has index rolled over each time? self.assertTrue(index1 != index2) self.assertTrue(index2 != index3) self.assertTrue(index1 != index3) events = self.get('test_aliasing', time1, time3) self.assertEqual(len(events), 20) self.assertEqual(events, sorted(events, key=lambda e: TimeUUID(e[ID_FIELD]))) events = self.get('test_aliasing', time1, time2) self.assertEqual(len(events), 15) self.assertEqual(events, sorted(events, key=lambda e: TimeUUID(e[ID_FIELD]))) events = self.get('test_aliasing', time2, time3) self.assertEqual(len(events), 15) self.assertEqual(events, sorted(events, key=lambda e: TimeUUID(e[ID_FIELD]))) es = router.get_backend('elasticsearch').es aliases = es.indices.get_aliases(index=[index1, index2, index3]) self.assertEqual(len(aliases), 2) self.assertTrue(index1 in aliases) self.assertTrue(index2 in aliases) self.assertEqual(set(aliases[index1]['aliases']) & set(aliases[index2]['aliases']), set(['kronos_test:kronos:2014.01.02'])) self.assertEqual(set(aliases[index1]['aliases']), set(['kronos_test:kronos:2014.01.01', 'kronos_test:kronos:2014.01.02'])) self.assertEqual(set(aliases[index2]['aliases']), set(['kronos_test:kronos:2014.01.02', 'kronos_test:kronos:2014.01.03'])) self.assertEqual(es.count(index=index1)['count'], 10) self.assertEqual(es.count(index=index2)['count'], 10) self.assertEqual(es.count(index=index3, ignore_unavailable=True).get('count', 0), 0)
def reload_router(kill_update_thread=False): router.reload() backend = router.get_backend('elasticsearch') if kill_update_thread: backend.index_manager.kill_rollover_worker()