def _handle_next_task(self): ''' We have to catch three ways a task can be "done" 1. Normal execution: the task runs/fails and puts a result back on the queue 2. New dependencies: the task yielded new deps that were not complete and will be rescheduled and dependencies added. 3. Child process dies: we need to catch this separately ''' while True: self._purge_children() # Deal with subprocess failures try: task_id, status, error_message, missing, new_requirements = ( self._task_result_queue.get( timeout=float(self.__wait_interval))) except Queue.Empty: return task = self._scheduled_tasks[task_id] if not task or task_id not in self._running_tasks: continue # Not a running task. Probably already removed. # Maybe it yielded something? new_deps = [] if new_requirements: new_req = [interface.load_task(task, name, params) for name, params in new_requirements] for t in new_req: self.add(t) new_deps = [t.task_id for t in new_req] self._scheduler.add_task(self._id, task_id, status=status, expl=error_message, resources=task.process_resources(), runnable=None, params=task.to_str_params(), family=task.task_family, new_deps=new_deps) if status == RUNNING: continue self._running_tasks.pop(task_id) # re-add task to reschedule missing dependencies if missing: reschedule = True # keep out of infinite loops by not rescheduling too many times for task_id in missing: self.unfulfilled_counts[task_id] += 1 if (self.unfulfilled_counts[task_id] > self.__max_reschedules): reschedule = False if reschedule: self.add(task) self.run_succeeded &= status in (DONE, SUSPENDED) return
def _handle_next_task(self): ''' We have to catch three ways a task can be "done" 1. Normal execution: the task runs/fails and puts a result back on the queue 2. New dependencies: the task yielded new deps that were not complete and will be rescheduled and dependencies added. 3. Child process dies: we need to catch this separately ''' while True: self._purge_children() # Deal with subprocess failures try: task_id, status, error_message, missing, new_requirements = ( self._task_result_queue.get( timeout=float(self.__wait_interval))) except Queue.Empty: return task = self._scheduled_tasks[task_id] if not task or task_id not in self._running_tasks: continue # Not a running task. Probably already removed. # Maybe it yielded something? new_deps = [] if new_requirements: new_req = [interface.load_task(module, name, params) for module, name, params in new_requirements] for t in new_req: self.add(t) new_deps = [t.task_id for t in new_req] self._scheduler.add_task(self._id, task_id, status=status, expl=error_message, resources=task.process_resources(), runnable=None, params=task.to_str_params(), family=task.task_family, new_deps=new_deps) if status == RUNNING: continue self._running_tasks.pop(task_id) # re-add task to reschedule missing dependencies if missing: reschedule = True # keep out of infinite loops by not rescheduling too many times for task_id in missing: self.unfulfilled_counts[task_id] += 1 if (self.unfulfilled_counts[task_id] > self.__max_reschedules): reschedule = False if reschedule: self.add(task) self.run_succeeded &= status in (DONE, SUSPENDED) return