def __init__(self, servers, bulk_amount=100, bulk_refresh_time=30):
self.es_connection = Elasticsearch(servers)
self.es_connection.ping()
self.bulk_queue = BulkQueue(self,
threshold=bulk_amount,
flush_time=bulk_refresh_time)
initLogger()
def test_over_threshold(self):
queue = BulkQueue(self.es_client, flush_time=3, threshold=10)
queue.start_timer()
for num in range(0, 201):
queue.add(index='events', doc_type='event', body={'keyname': 'value' + str(num)})
assert self.num_objects_saved() == 200
assert queue.size() == 1
time.sleep(4)
assert self.num_objects_saved() == 201
assert queue.size() == 0
queue.stop_timer()
def test_basic_timer(self):
queue = BulkQueue(self.es_client, flush_time=2)
assert queue.started() is False
queue.start_timer()
assert queue.started() is True
queue.add(index='events', doc_type='event', body={'keyname': 'valuename'})
assert queue.size() == 1
time.sleep(3)
assert queue.size() == 0
queue.stop_timer()
assert queue.started() is False
def test_ten_iterations(self):
queue = BulkQueue(self.es_client, flush_time=3, threshold=10)
queue.start_timer()
total_events = 0
for num_rounds in range(0, 10):
for num in range(0, 20):
total_events += 1
queue.add(index='events', doc_type='event', body={'keyname': 'value' + str(num)})
assert self.num_objects_saved() == total_events
assert queue.size() == 0
queue.stop_timer()
assert self.num_objects_saved() == 200
def setup(self):
super(TestAdd, self).setup()
self.queue = BulkQueue(self.es_client, threshold=20)
def test_init_with_threshold(self):
queue = BulkQueue(self.es_client, 100)
assert queue.threshold == 100
def setup(self):
super(TestBasicInit, self).setup()
self.queue = BulkQueue(self.es_client)