class BluefloodWorker(Worker): def __init__(self, queue, config=None): """ Sets the queue and pulls info from the config @param queue - Eventlet queue @param config - Dictionary """ self.queue = queue if config is None: config = {} self.client = Blueflood( auth_url=config['auth']['auth_url'], apikey=config['auth']['apikey'], username=config['auth']['username'], ingest_url=config['auth']['ingest_url'] ) self.batch_size = config.get('batch_size', 1000) self.timeout = 300 def work(self): data = [] logger.info("Blueflood Queue size is: %s " % self.queue.qsize()) start_work = utils.time_in_s() for i in xrange(self.batch_size): try: entry = self.queue.get(block=False) data.append(entry) except eventlet.queue.Empty: break logger.info("Got %s docs" % len(data)) if data: try: start = utils.time_in_ms() self.client.ingest(data) for i in xrange(len(data)): self.queue.task_done() time_taken = (utils.time_in_ms() - start) logger.info("Processed %s entries to blueflood. Took %s ms \ " % (len(data), time_taken)) except Exception: logger.exception("Error submitting to blueflood") # Sleep till queue is filled enough or time out is reached while True: work_time_taken = (utils.time_in_s() - start_work) if (self.queue.qsize() > self.batch_size) or work_time_taken > self.timeout: break time.sleep(1)
def __init__(self, queue, config=None): """ Sets the queue and pulls info from the config @param queue - Eventlet queue @param config - Dictionary """ self.queue = queue if config is None: config = {} self.client = Blueflood( auth_url=config['auth']['auth_url'], apikey=config['auth']['apikey'], username=config['auth']['username'] ) self.batch_size = config.get('batch_size', 1000)