def enqueue(tag, data):
data['ts'] = int(time.time())
logging.debug('[QueueRequests] ' + tag)
if tag is not None:
cache.lpush('queue::' + tag, json.dumps(data))
else:
logging.error('invalid queue request')
def enqueue(tag, data):
data['ts'] = int(time.time())
logging.debug('[QueueRequests] ' + tag)
if tag is not None:
cache.lpush('queue::' + tag, json.dumps(data))
else:
logging.error('invalid queue request')
def enqueue_in_error_queue(self, source_queue, thread_id, data):
try:
error_queue = 'error' + "::" + source_queue
error_queue_data = dict(data=data, host=self.machine_name, id=thread_id, ts= int(time.time()))
cache.lpush(error_queue, json.dumps(error_queue_data))
except Exception as e:
logging.exception("error while enqueue in error queue: %s thread_id: %s data: %s Reason: %s", source_queue, thread_id, data, e.message)
def enqueue_in_error_queue(self, source_queue, thread_id, data):
try:
error_queue = 'error' + "::" + source_queue
error_queue_data = dict(data=data,
host=self.machine_name,
id=thread_id,
ts=int(time.time()))
cache.lpush(error_queue, json.dumps(error_queue_data))
except Exception as e:
logging.exception(
"error while enqueue in error queue: %s thread_id: %s data: %s Reason: %s",
source_queue, thread_id, data, e.message)
def run(self):
queue = self.queue
id = str(self.qid)
logging.error('checking: ' + queue + ' ' + id)
source_queue = 'queue::' + queue
temp_queue = 'temp::' + self.machine_name + "::" + id + "::" + source_queue
#To process a request which might not be done due to restart
obj = cache.lindex(temp_queue, 0)
if obj is not None:
logging.error("Processing previous entry temp_queue=%s qid=%s",
temp_queue, str(self.qid))
try:
QueueProcessor.process(queue, obj)
except Exception as e:
logging.exception(
"error while processing previous entry: %s. Reason: %s",
obj, e.message)
self.enqueue_in_error_queue(source_queue, id, obj)
cache.lpop(temp_queue)
while self.running:
obj = None
try:
obj = cache.brpoplpush(source_queue, temp_queue, TIME_OUT)
except TimeoutError:
gevent.sleep(INTERVAL)
except Exception as e:
logging.exception(
'unable to fetch object from redis. Reason: ' + e.message)
gevent.sleep(INTERVAL)
if obj is not None:
try:
QueueProcessor.process(queue, obj)
except Exception as e:
logging.exception("error while processing: %s. Reason: %s",
obj, e.message)
self.enqueue_in_error_queue(source_queue, id, obj)
cache.lpop(temp_queue)
else:
logging.error("[%s] No entry to process. Sleeping", self.queue)
gevent.sleep(INTERVAL)
gevent.sleep(0)
def run(self):
queue = self.queue
id = str(self.qid)
logging.error('checking: ' + queue + ' ' + id)
source_queue = 'queue::' + queue
temp_queue = 'temp::' + self.machine_name + "::" + id + "::" + source_queue
#To process a request which might not be done due to restart
obj = cache.lindex(temp_queue, 0)
if obj is not None:
logging.error("Processing previous entry temp_queue=%s qid=%s", temp_queue, str(self.qid))
try:
QueueProcessor.process(queue, obj)
except Exception as e:
logging.exception("error while processing previous entry: %s. Reason: %s", obj, e.message)
self.enqueue_in_error_queue(source_queue, id, obj)
cache.lpop(temp_queue)
while self.running:
obj = None
try:
obj = cache.brpoplpush(source_queue, temp_queue, TIME_OUT)
except TimeoutError:
gevent.sleep(INTERVAL)
except Exception as e:
logging.exception('unable to fetch object from redis. Reason: ' + e.message)
gevent.sleep(INTERVAL)
if obj is not None:
try:
QueueProcessor.process(queue, obj)
except Exception as e:
logging.exception("error while processing: %s. Reason: %s", obj, e.message)
self.enqueue_in_error_queue(source_queue, id, obj)
cache.lpop(temp_queue)
else:
logging.error("[%s] No entry to process. Sleeping", self.queue)
gevent.sleep(INTERVAL)
gevent.sleep(0)
