def __init__(self, servers, bulk_amount=100, bulk_refresh_time=30): self.es_connection = Elasticsearch(servers) self.es_connection.ping() self.bulk_queue = BulkQueue(self, threshold=bulk_amount, flush_time=bulk_refresh_time) initLogger()
def test_over_threshold(self): queue = BulkQueue(self.es_client, flush_time=3, threshold=10) queue.start_timer() for num in range(0, 201): queue.add(index='events', doc_type='event', body={'keyname': 'value' + str(num)}) assert self.num_objects_saved() == 200 assert queue.size() == 1 time.sleep(4) assert self.num_objects_saved() == 201 assert queue.size() == 0 queue.stop_timer()
def test_basic_timer(self): queue = BulkQueue(self.es_client, flush_time=2) assert queue.started() is False queue.start_timer() assert queue.started() is True queue.add(index='events', doc_type='event', body={'keyname': 'valuename'}) assert queue.size() == 1 time.sleep(3) assert queue.size() == 0 queue.stop_timer() assert queue.started() is False
def test_ten_iterations(self): queue = BulkQueue(self.es_client, flush_time=3, threshold=10) queue.start_timer() total_events = 0 for num_rounds in range(0, 10): for num in range(0, 20): total_events += 1 queue.add(index='events', doc_type='event', body={'keyname': 'value' + str(num)}) assert self.num_objects_saved() == total_events assert queue.size() == 0 queue.stop_timer() assert self.num_objects_saved() == 200
def setup(self): super(TestAdd, self).setup() self.queue = BulkQueue(self.es_client, threshold=20)
def test_init_with_threshold(self): queue = BulkQueue(self.es_client, 100) assert queue.threshold == 100
def setup(self): super(TestBasicInit, self).setup() self.queue = BulkQueue(self.es_client)