class BluefloodWorker(Worker):
def __init__(self, queue, config=None):
"""
Sets the queue and pulls info from the config
@param queue - Eventlet queue
@param config - Dictionary
"""
self.queue = queue
if config is None:
config = {}
self.client = Blueflood(
auth_url=config['auth']['auth_url'],
apikey=config['auth']['apikey'],
username=config['auth']['username'],
ingest_url=config['auth']['ingest_url']
)
self.batch_size = config.get('batch_size', 1000)
self.timeout = 300
def work(self):
data = []
logger.info("Blueflood Queue size is: %s " % self.queue.qsize())
start_work = utils.time_in_s()
for i in xrange(self.batch_size):
try:
entry = self.queue.get(block=False)
data.append(entry)
except eventlet.queue.Empty:
break
logger.info("Got %s docs" % len(data))
if data:
try:
start = utils.time_in_ms()
self.client.ingest(data)
for i in xrange(len(data)):
self.queue.task_done()
time_taken = (utils.time_in_ms() - start)
logger.info("Processed %s entries to blueflood. Took %s ms \
" % (len(data), time_taken))
except Exception:
logger.exception("Error submitting to blueflood")
# Sleep till queue is filled enough or time out is reached
while True:
work_time_taken = (utils.time_in_s() - start_work)
if (self.queue.qsize() > self.batch_size) or work_time_taken > self.timeout:
break
time.sleep(1)
def __init__(self, queue, config=None):
"""
Sets the queue and pulls info from the config
@param queue - Eventlet queue
@param config - Dictionary
"""
self.queue = queue
if config is None:
config = {}
self.client = Blueflood(
auth_url=config['auth']['auth_url'],
apikey=config['auth']['apikey'],
username=config['auth']['username']
)
self.batch_size = config.get('batch_size', 1000)
