def dequeue(self):
try:
self._logger.info('Dequeue thread started')
mq_connection = pika.BlockingConnection(
pika.ConnectionParameters(host=self._mq_hostname))
mq_channel = mq_connection.channel()
while not self._dequeue_thread_terminate.is_set():
try:
method_frame, header_frame, body = mq_channel.basic_get(
queue=self._completed_queue[0])
if body:
# Get task from the message
completed_task = Task()
completed_task.load_from_dict(json.loads(body))
self._logger.debug('Got finished task %s from queue' %
(completed_task.uid))
for pipe in self._workflow:
with pipe._stage_lock:
if not pipe._completed:
if completed_task._parent_pipeline == pipe.uid:
self._logger.debug(
'Found parent pipeline: %s' %
pipe.uid)
for stage in pipe.stages:
if completed_task._parent_stage == stage.uid:
self._logger.debug(
'Found parent stage: %s' %
(stage.uid))
#task.state = states.DONE
self._logger.debug(
'Task: %s,%s ; Stage: %s; Pipeline: %s'
%
(completed_task.uid,
completed_task.state,
pipe.stages[
pipe._current_stage].
uid, pipe.uid))
for task in stage.tasks:
if task.uid == completed_task.uid:
task.state = str(
completed_task.
state)
if task.state == states.DONE:
if stage._check_tasks_status(
):
try:
stage.state = states.DONE
self._logger.info(
'Stage %s of Pipeline %s: %s'
%
(stage.
uid,
pipe.
uid,
stage.
state)
)
pipe._increment_stage(
)
except Exception, ex:
# Rolling back stage status
self._logger.error(
'Error while updating stage '
+
'state, rolling back. Error: %s'
% ex)
stage.state = states.SCHEDULED
pipe._decrement_stage(
)
raise
if pipe._completed:
#self._workload.remove(pipe)
pipe.state = states.DONE
self._logger.info(
'Pipeline %s: %s'
%
(pipe.
uid,
pipe.
state)
)
elif completed_task.state == states.FAILED:
if self._resubmit_failed:
try:
new_task = Task(
)
new_task._replicate(
completed_task
)
pipe.stages[
pipe.
_current_stage].add_tasks(
new_task
)
except Exception, ex:
self._logger.error(
"Resubmission of task %s failed, error: %s"
%
(completed_task
.uid,
ex))
raise
else:
if stage.check_tasks_status(
):
try:
stage.state = states.DONE
self._logger.info(
'Stage %s of Pipeline %s: %s'
%
(stage
.
uid,
pipe
.
uid,
stage
.
state
))
pipe._increment_stage(
)
except Exception, ex:
# Rolling back stage status
self._logger.error(
'Error while updating stage '
+
'state, rolling back. Error: %s'
%
ex)
stage.state = states.SCHEDULED
pipe._decrement_stage(
)
raise
if pipe.completed:
#self._workload.remove(pipe)
pipe.state = states.DONE
self._logger.info(
'Pipeline %s: %s'
%
(pipe
.
uid,
pipe
.
state
))
else:
# Task is canceled
pass
# Found the task and processed it -- no more iterations needed
break
# Found the stage and processed it -- no more iterations neeeded
break
# Found the pipeline and processed it -- no more iterations neeeded
break
mq_channel.basic_ack(
delivery_tag=method_frame.delivery_tag)
if slow_run:
sleep(1)
except Exception, ex:
self._logger.error(
'Unable to receive message from completed queue: %s' %
ex)
raise
def helper(self):
# This function extracts currently tasks from the pending_queue
# and pushes it to the executed_queue. Thus mimicking an execution plugin
try:
self._logger.info('Helper process started')
# Thread should run till terminate condtion is encountered
mq_connection = pika.BlockingConnection(
pika.ConnectionParameters(host=self._mq_hostname))
mq_channel = mq_connection.channel()
while not self._helper_terminate.is_set():
try:
method_frame, header_frame, body = mq_channel.basic_get(
queue=self._pending_queue[0])
if body:
try:
task = Task()
task.load_from_dict(json.loads(body))
task.state = states.DONE
task_as_dict = json.dumps(task.to_dict())
self._logger.debug(
'Got task %s from pending_queue %s' %
(task.uid, self._pending_queue[0]))
mq_channel.basic_publish(
exchange='fork',
routing_key='',
body=task_as_dict
#properties=pika.BasicProperties(
# make message persistent
# delivery_mode = 2,
#)
)
self._logger.debug(
'Pushed task %s with state %s to completed queue %s and synchronizerq'
% (task.uid, task.state,
self._completed_queue[0]))
mq_channel.basic_ack(
delivery_tag=method_frame.delivery_tag)
except Exception, ex:
# Rolling back queue and task status
self._logger.error(
'Error while pushing task to completed queue, rolling back: %s'
% ex)
raise UnknownError(text=ex)
if slow_run:
time.sleep(1)
except Exception, ex:
self._logger.error(
'Error getting messages from pending queue: %s' % ex)
raise UnknownError(text=ex)
except KeyboardInterrupt:
self._logger.error(
'Execution interrupted by user (you probably hit Ctrl+C), ' +
'trying to cancel enqueuer thread gracefully...')
raise KeyboardInterrupt
except Exception, ex:
self._logger.error('Unknown error in helper process: %s' % ex)
print traceback.format_exc()
raise UnknownError(text=ex)
