def start(self, status_update_callback, completion_callback): uc = status_update_callback try: zmq_job = zmq_server.get_job_context(self.job_id) msg = message.RunTask.create(self.get_sender(), self.job_id, self.task_id, self.command_and_args, cwd=self.cwd, shell=self.shell) zmq_job.send_req(self.worker_node, msg._to_json(), lambda msg_data: (self)(msg_data, completion_callback)) except Exception, e: logger.exception("Task %s failed due to exception: %s(%s)" % (self.get_log_tag(), e.__class__.__name__, e)) completion_callback(self, TaskStatus.TASK_FAILED, reason="Got exception: %s" % e) return
try: job = self._get_job(job_id, api_name) pending_tasks = task_run.get_pending_task_ids() for tid in pending_tasks: t = job.get_task_by_id(tid) if t.is_running(): m.stop_task(t, TaskStatus.TASK_UNKNOWN) except Exception, e: logger.exception("Got exception when attempting to clean up from failed task run: %s" % e) m.abort_transaction() raise m.commit_transaction() # This is the main loop waiting for everything to be done iteration = 0 zmq_context_for_job = zmq_server.get_job_context(job_id) try: while task_run.has_pending_tasks(): iteration += 1 if timeout!=None and ((iteration*2)>timeout): raise TimeoutError("Tasks timed out after %d seconds" % timeout) elif (iteration % 30)==0: logger.debug("Pending tasks: %s" % task_run.get_pending_tasks_as_string()) zmq_context_for_job.poll(2000) except KeyboardInterrupt: logger.exception("Got keyboard interrupt while waiting for task completion") error_cleanup() raise except Exception, e: