def test_rollover(self):
    settings.storage.elasticsearch.rollover_size = 10
    settings.storage.elasticsearch.rollover_check_period_seconds = 2
    reload_router()
    
    indices = set()
    for i in xrange(50):
      indices.add(router.get_backend('elasticsearch')
                  .index_manager.get_index('kronos'))
      self.put('test_rollover', [{TIMESTAMP_FIELD: 0} for _ in xrange(5)])
      gevent.sleep(0.05)

    # Has index rolled over?
    self.assertTrue(len(indices) > 1)

    # No events were lost?
    events = self.get('test_rollover', 0, 1)
    self.assertEqual(len(events), 50 * 5)
    self.assertEqual(events, sorted(events,
                                    key=lambda e: TimeUUID(e[ID_FIELD])))

    es = router.get_backend('elasticsearch').es
    index = router.get_backend('elasticsearch').index_manager.get_index(
      'kronos')
    indices.discard(index)
    for index in indices:
      self.assertTrue(es.count(index=index)['count'] >= 10)
Beispiel #2
0
  def test_rollover(self):
    settings.storage.elasticsearch.rollover_size = 10
    settings.storage.elasticsearch.rollover_check_period_seconds = 2
    reload_router()

    indices = set()
    for i in xrange(50):
      indices.add(router.get_backend('elasticsearch')
                  .index_manager.get_index('kronos'))
      self.put('test_rollover', [{TIMESTAMP_FIELD: 0} for _ in xrange(5)])
      gevent.sleep(0.05)

    # Has index rolled over?
    self.assertTrue(len(indices) > 1)

    # No events were lost?
    events = self.get('test_rollover', 0, 1)
    self.assertEqual(len(events), 50 * 5)
    self.assertEqual(events, sorted(events,
                                    key=lambda e: TimeUUID(e[ID_FIELD])))

    es = router.get_backend('elasticsearch').es
    index = router.get_backend('elasticsearch').index_manager.get_index(
      'kronos')
    indices.discard(index)
    for index in indices:
      self.assertTrue(es.count(index=index)['count'] >= 10)
  def test_concurrent_rollover(self):
    settings.storage.elasticsearch.rollover_size = 10
    reload_router()

    index_manager1 = router.get_backend('elasticsearch').index_manager
    index_manager1.kill_rollover_worker()

    # Create another IndexManager. This should pull index names from the
    # previously created directory by index_manager1.
    index_manager2 = IndexManager(router.get_backend('elasticsearch'))
    index_manager2.kill_rollover_worker()

    index1 = index_manager1.get_index('kronos')
    self.assertEqual(index_manager1.get_index('kronos'),
                     index_manager2.get_index('kronos'))

    self.put('test_concurrent_rollover',
             [{TIMESTAMP_FIELD: 0} for _ in xrange(10)])

    # Update index_manager2. It should rollover.
    index_manager2.update()
    self.assertEqual(index1, index_manager1.get_index('kronos'))
    self.assertNotEqual(index_manager1.get_index('kronos'),
                        index_manager2.get_index('kronos'))

    # This should read updated index set by index_manager2.
    index_manager1.update()
    self.assertEqual(index_manager1.get_index('kronos'),
                     index_manager2.get_index('kronos'))
Beispiel #4
0
  def test_concurrent_rollover(self):
    settings.storage.elasticsearch.rollover_size = 10
    reload_router()

    index_manager1 = router.get_backend('elasticsearch').index_manager
    index_manager1.kill_rollover_worker()

    # Create another IndexManager. This should pull index names from the
    # previously created directory by index_manager1.
    index_manager2 = IndexManager(router.get_backend('elasticsearch'))
    index_manager2.kill_rollover_worker()

    index1 = index_manager1.get_index('kronos')
    self.assertEqual(index_manager1.get_index('kronos'),
                     index_manager2.get_index('kronos'))

    self.put('test_concurrent_rollover',
             [{TIMESTAMP_FIELD: 0} for _ in xrange(10)])

    # Update index_manager2. It should rollover.
    index_manager2.update()
    self.assertEqual(index1, index_manager1.get_index('kronos'))
    self.assertNotEqual(index_manager1.get_index('kronos'),
                        index_manager2.get_index('kronos'))

    # This should read updated index set by index_manager2.
    index_manager1.update()
    self.assertEqual(index_manager1.get_index('kronos'),
                     index_manager2.get_index('kronos'))
Beispiel #5
0
def clear_backend(backend_name):
  from kronos.storage.router import router
  router.get_backend(backend_name)._clear()
Beispiel #6
0
def clear_backend(backend_name):
  from kronos.storage.router import router
  router.get_backend(backend_name)._clear()
Beispiel #7
0
 def setUp(self):
   super(TestCassandraBackend, self).setUp()
   self.namespace = router.get_backend('cassandra').namespaces['kronos']
   self.shards = settings.storage['cassandra']['shards_per_bucket']
   self.width_seconds = settings.storage['cassandra']['timewidth_seconds']
   self.width = epoch_time_to_kronos_time(self.width_seconds)
  def test_aliasing(self):
    settings.storage.elasticsearch.rollover_size = 10
    reload_router(kill_update_thread=True)
    index1 = router.get_backend('elasticsearch').index_manager.get_index(
      'kronos')

    time1 = datetime_to_kronos_time(datetime(2014, 1, 1, 0))
    time2 = datetime_to_kronos_time(datetime(2014, 1, 2, 0))
    time3 = datetime_to_kronos_time(datetime(2014, 1, 3, 0))
    self.put('test_aliasing', [{TIMESTAMP_FIELD: time1, 'i': i, 'j': 0}
                               for i in xrange(5)])
    self.put('test_aliasing', [{TIMESTAMP_FIELD: time2, 'i': i, 'j': 0}
                               for i in xrange(5)])

    router.get_backend('elasticsearch').index_manager.update()
    index2 = router.get_backend('elasticsearch').index_manager.get_index(
      'kronos')
    
    self.put('test_aliasing', [{TIMESTAMP_FIELD: time2, 'i': i, 'j': 1}
                               for i in xrange(5)])
    self.put('test_aliasing', [{TIMESTAMP_FIELD: time3, 'i': i, 'j': 1}
                               for i in xrange(5)])

    router.get_backend('elasticsearch').index_manager.update()
    index3 = router.get_backend('elasticsearch').index_manager.get_index(
      'kronos')

    # Has index rolled over each time?
    self.assertTrue(index1 != index2)
    self.assertTrue(index2 != index3)
    self.assertTrue(index1 != index3)

    events = self.get('test_aliasing', time1, time3)
    self.assertEqual(len(events), 20)
    self.assertEqual(events, sorted(events,
                                    key=lambda e: TimeUUID(e[ID_FIELD])))

    events = self.get('test_aliasing', time1, time2)
    self.assertEqual(len(events), 15)
    self.assertEqual(events, sorted(events,
                                    key=lambda e: TimeUUID(e[ID_FIELD])))

    events = self.get('test_aliasing', time2, time3)
    self.assertEqual(len(events), 15)
    self.assertEqual(events, sorted(events,
                                    key=lambda e: TimeUUID(e[ID_FIELD])))

    es = router.get_backend('elasticsearch').es
    aliases = es.indices.get_aliases(index=[index1, index2, index3])
    self.assertEqual(len(aliases), 2)
    self.assertTrue(index1 in aliases)
    self.assertTrue(index2 in aliases)
    self.assertEqual(set(aliases[index1]['aliases']) &
                     set(aliases[index2]['aliases']),
                     set(['kronos_test:kronos:2014.01.02']))
    self.assertEqual(set(aliases[index1]['aliases']),
                     set(['kronos_test:kronos:2014.01.01',
                          'kronos_test:kronos:2014.01.02']))
    self.assertEqual(set(aliases[index2]['aliases']),
                     set(['kronos_test:kronos:2014.01.02',
                          'kronos_test:kronos:2014.01.03']))

    self.assertEqual(es.count(index=index1)['count'], 10)
    self.assertEqual(es.count(index=index2)['count'], 10)
    self.assertEqual(es.count(index=index3,
                              ignore_unavailable=True).get('count', 0), 0)
def reload_router(kill_update_thread=False):
  router.reload()
  backend = router.get_backend('elasticsearch')
  if kill_update_thread:
    backend.index_manager.kill_rollover_worker()
Beispiel #10
0
 def setUp(self):
   super(TestCassandraBackend, self).setUp()
   self.namespace = router.get_backend('cassandra').namespaces['kronos']
   self.shards = settings.storage['cassandra']['shards_per_bucket']
   self.width_seconds = settings.storage['cassandra']['timewidth_seconds']
   self.width = epoch_time_to_kronos_time(self.width_seconds)
Beispiel #11
0
 def setUp(self):
     super(TestCassandraBackend, self).setUp()
     self.namespace = router.get_backend("cassandra").namespaces["kronos"]
     self.shards = settings.storage["cassandra"]["shards_per_bucket"]
     self.width_seconds = settings.storage["cassandra"]["timewidth_seconds"]
     self.width = epoch_time_to_kronos_time(self.width_seconds)
Beispiel #12
0
  def test_aliasing(self):
    settings.storage.elasticsearch.rollover_size = 10
    reload_router(kill_update_thread=True)
    index1 = router.get_backend('elasticsearch').index_manager.get_index(
      'kronos')

    time1 = datetime_to_kronos_time(datetime(2014, 1, 1, 0))
    time2 = datetime_to_kronos_time(datetime(2014, 1, 2, 0))
    time3 = datetime_to_kronos_time(datetime(2014, 1, 3, 0))
    self.put('test_aliasing', [{TIMESTAMP_FIELD: time1, 'i': i, 'j': 0}
                               for i in xrange(5)])
    self.put('test_aliasing', [{TIMESTAMP_FIELD: time2, 'i': i, 'j': 0}
                               for i in xrange(5)])

    router.get_backend('elasticsearch').index_manager.update()
    index2 = router.get_backend('elasticsearch').index_manager.get_index(
      'kronos')

    self.put('test_aliasing', [{TIMESTAMP_FIELD: time2, 'i': i, 'j': 1}
                               for i in xrange(5)])
    self.put('test_aliasing', [{TIMESTAMP_FIELD: time3, 'i': i, 'j': 1}
                               for i in xrange(5)])

    router.get_backend('elasticsearch').index_manager.update()
    index3 = router.get_backend('elasticsearch').index_manager.get_index(
      'kronos')

    # Has index rolled over each time?
    self.assertTrue(index1 != index2)
    self.assertTrue(index2 != index3)
    self.assertTrue(index1 != index3)

    events = self.get('test_aliasing', time1, time3)
    self.assertEqual(len(events), 20)
    self.assertEqual(events, sorted(events,
                                    key=lambda e: TimeUUID(e[ID_FIELD])))

    events = self.get('test_aliasing', time1, time2)
    self.assertEqual(len(events), 15)
    self.assertEqual(events, sorted(events,
                                    key=lambda e: TimeUUID(e[ID_FIELD])))

    events = self.get('test_aliasing', time2, time3)
    self.assertEqual(len(events), 15)
    self.assertEqual(events, sorted(events,
                                    key=lambda e: TimeUUID(e[ID_FIELD])))

    es = router.get_backend('elasticsearch').es
    aliases = es.indices.get_aliases(index=[index1, index2, index3])
    self.assertEqual(len(aliases), 2)
    self.assertTrue(index1 in aliases)
    self.assertTrue(index2 in aliases)
    self.assertEqual(set(aliases[index1]['aliases']) &
                     set(aliases[index2]['aliases']),
                     set(['kronos_test:kronos:2014.01.02']))
    self.assertEqual(set(aliases[index1]['aliases']),
                     set(['kronos_test:kronos:2014.01.01',
                          'kronos_test:kronos:2014.01.02']))
    self.assertEqual(set(aliases[index2]['aliases']),
                     set(['kronos_test:kronos:2014.01.02',
                          'kronos_test:kronos:2014.01.03']))

    self.assertEqual(es.count(index=index1)['count'], 10)
    self.assertEqual(es.count(index=index2)['count'], 10)
    self.assertEqual(es.count(index=index3,
                              ignore_unavailable=True).get('count', 0), 0)
Beispiel #13
0
def reload_router(kill_update_thread=False):
  router.reload()
  backend = router.get_backend('elasticsearch')
  if kill_update_thread:
    backend.index_manager.kill_rollover_worker()