def enqueue(tag, data): data['ts'] = int(time.time()) logging.debug('[QueueRequests] ' + tag) if tag is not None: cache.lpush('queue::' + tag, json.dumps(data)) else: logging.error('invalid queue request')
def enqueue_in_error_queue(self, source_queue, thread_id, data): try: error_queue = 'error' + "::" + source_queue error_queue_data = dict(data=data, host=self.machine_name, id=thread_id, ts= int(time.time())) cache.lpush(error_queue, json.dumps(error_queue_data)) except Exception as e: logging.exception("error while enqueue in error queue: %s thread_id: %s data: %s Reason: %s", source_queue, thread_id, data, e.message)
def enqueue_in_error_queue(self, source_queue, thread_id, data): try: error_queue = 'error' + "::" + source_queue error_queue_data = dict(data=data, host=self.machine_name, id=thread_id, ts=int(time.time())) cache.lpush(error_queue, json.dumps(error_queue_data)) except Exception as e: logging.exception( "error while enqueue in error queue: %s thread_id: %s data: %s Reason: %s", source_queue, thread_id, data, e.message)
def run(self): queue = self.queue id = str(self.qid) logging.error('checking: ' + queue + ' ' + id) source_queue = 'queue::' + queue temp_queue = 'temp::' + self.machine_name + "::" + id + "::" + source_queue #To process a request which might not be done due to restart obj = cache.lindex(temp_queue, 0) if obj is not None: logging.error("Processing previous entry temp_queue=%s qid=%s", temp_queue, str(self.qid)) try: QueueProcessor.process(queue, obj) except Exception as e: logging.exception( "error while processing previous entry: %s. Reason: %s", obj, e.message) self.enqueue_in_error_queue(source_queue, id, obj) cache.lpop(temp_queue) while self.running: obj = None try: obj = cache.brpoplpush(source_queue, temp_queue, TIME_OUT) except TimeoutError: gevent.sleep(INTERVAL) except Exception as e: logging.exception( 'unable to fetch object from redis. Reason: ' + e.message) gevent.sleep(INTERVAL) if obj is not None: try: QueueProcessor.process(queue, obj) except Exception as e: logging.exception("error while processing: %s. Reason: %s", obj, e.message) self.enqueue_in_error_queue(source_queue, id, obj) cache.lpop(temp_queue) else: logging.error("[%s] No entry to process. Sleeping", self.queue) gevent.sleep(INTERVAL) gevent.sleep(0)
def run(self): queue = self.queue id = str(self.qid) logging.error('checking: ' + queue + ' ' + id) source_queue = 'queue::' + queue temp_queue = 'temp::' + self.machine_name + "::" + id + "::" + source_queue #To process a request which might not be done due to restart obj = cache.lindex(temp_queue, 0) if obj is not None: logging.error("Processing previous entry temp_queue=%s qid=%s", temp_queue, str(self.qid)) try: QueueProcessor.process(queue, obj) except Exception as e: logging.exception("error while processing previous entry: %s. Reason: %s", obj, e.message) self.enqueue_in_error_queue(source_queue, id, obj) cache.lpop(temp_queue) while self.running: obj = None try: obj = cache.brpoplpush(source_queue, temp_queue, TIME_OUT) except TimeoutError: gevent.sleep(INTERVAL) except Exception as e: logging.exception('unable to fetch object from redis. Reason: ' + e.message) gevent.sleep(INTERVAL) if obj is not None: try: QueueProcessor.process(queue, obj) except Exception as e: logging.exception("error while processing: %s. Reason: %s", obj, e.message) self.enqueue_in_error_queue(source_queue, id, obj) cache.lpop(temp_queue) else: logging.error("[%s] No entry to process. Sleeping", self.queue) gevent.sleep(INTERVAL) gevent.sleep(0)