def dequeue(self): try: self._logger.info('Dequeue thread started') mq_connection = pika.BlockingConnection( pika.ConnectionParameters(host=self._mq_hostname)) mq_channel = mq_connection.channel() while not self._dequeue_thread_terminate.is_set(): try: method_frame, header_frame, body = mq_channel.basic_get( queue=self._completed_queue[0]) if body: # Get task from the message completed_task = Task() completed_task.load_from_dict(json.loads(body)) self._logger.debug('Got finished task %s from queue' % (completed_task.uid)) for pipe in self._workflow: with pipe._stage_lock: if not pipe._completed: if completed_task._parent_pipeline == pipe.uid: self._logger.debug( 'Found parent pipeline: %s' % pipe.uid) for stage in pipe.stages: if completed_task._parent_stage == stage.uid: self._logger.debug( 'Found parent stage: %s' % (stage.uid)) #task.state = states.DONE self._logger.debug( 'Task: %s,%s ; Stage: %s; Pipeline: %s' % (completed_task.uid, completed_task.state, pipe.stages[ pipe._current_stage]. uid, pipe.uid)) for task in stage.tasks: if task.uid == completed_task.uid: task.state = str( completed_task. state) if task.state == states.DONE: if stage._check_tasks_status( ): try: stage.state = states.DONE self._logger.info( 'Stage %s of Pipeline %s: %s' % (stage. uid, pipe. uid, stage. state) ) pipe._increment_stage( ) except Exception, ex: # Rolling back stage status self._logger.error( 'Error while updating stage ' + 'state, rolling back. Error: %s' % ex) stage.state = states.SCHEDULED pipe._decrement_stage( ) raise if pipe._completed: #self._workload.remove(pipe) pipe.state = states.DONE self._logger.info( 'Pipeline %s: %s' % (pipe. uid, pipe. state) ) elif completed_task.state == states.FAILED: if self._resubmit_failed: try: new_task = Task( ) new_task._replicate( completed_task ) pipe.stages[ pipe. _current_stage].add_tasks( new_task ) except Exception, ex: self._logger.error( "Resubmission of task %s failed, error: %s" % (completed_task .uid, ex)) raise else: if stage.check_tasks_status( ): try: stage.state = states.DONE self._logger.info( 'Stage %s of Pipeline %s: %s' % (stage . uid, pipe . uid, stage . state )) pipe._increment_stage( ) except Exception, ex: # Rolling back stage status self._logger.error( 'Error while updating stage ' + 'state, rolling back. Error: %s' % ex) stage.state = states.SCHEDULED pipe._decrement_stage( ) raise if pipe.completed: #self._workload.remove(pipe) pipe.state = states.DONE self._logger.info( 'Pipeline %s: %s' % (pipe . uid, pipe . state )) else: # Task is canceled pass # Found the task and processed it -- no more iterations needed break # Found the stage and processed it -- no more iterations neeeded break # Found the pipeline and processed it -- no more iterations neeeded break mq_channel.basic_ack( delivery_tag=method_frame.delivery_tag) if slow_run: sleep(1) except Exception, ex: self._logger.error( 'Unable to receive message from completed queue: %s' % ex) raise
def helper(self): # This function extracts currently tasks from the pending_queue # and pushes it to the executed_queue. Thus mimicking an execution plugin try: self._logger.info('Helper process started') # Thread should run till terminate condtion is encountered mq_connection = pika.BlockingConnection( pika.ConnectionParameters(host=self._mq_hostname)) mq_channel = mq_connection.channel() while not self._helper_terminate.is_set(): try: method_frame, header_frame, body = mq_channel.basic_get( queue=self._pending_queue[0]) if body: try: task = Task() task.load_from_dict(json.loads(body)) task.state = states.DONE task_as_dict = json.dumps(task.to_dict()) self._logger.debug( 'Got task %s from pending_queue %s' % (task.uid, self._pending_queue[0])) mq_channel.basic_publish( exchange='fork', routing_key='', body=task_as_dict #properties=pika.BasicProperties( # make message persistent # delivery_mode = 2, #) ) self._logger.debug( 'Pushed task %s with state %s to completed queue %s and synchronizerq' % (task.uid, task.state, self._completed_queue[0])) mq_channel.basic_ack( delivery_tag=method_frame.delivery_tag) except Exception, ex: # Rolling back queue and task status self._logger.error( 'Error while pushing task to completed queue, rolling back: %s' % ex) raise UnknownError(text=ex) if slow_run: time.sleep(1) except Exception, ex: self._logger.error( 'Error getting messages from pending queue: %s' % ex) raise UnknownError(text=ex) except KeyboardInterrupt: self._logger.error( 'Execution interrupted by user (you probably hit Ctrl+C), ' + 'trying to cancel enqueuer thread gracefully...') raise KeyboardInterrupt except Exception, ex: self._logger.error('Unknown error in helper process: %s' % ex) print traceback.format_exc() raise UnknownError(text=ex)