def _run_task(self, task_id): task = self._scheduled_tasks[task_id] logger.info('[pid %s] Worker %s running %s', os.getpid(), self._id, task_id) try: # Verify that all the tasks are fulfilled! missing = [dep.task_id for dep in task.deps() if not dep.complete()] if missing: deps = 'dependency' if len(missing) == 1 else 'dependencies' # TODO: possibly try to re-add task again ad pending raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing))) task.trigger_event(Event.START, task) task.run() error_message = json.dumps(task.on_success()) logger.info('[pid %s] Worker %s done %s', os.getpid(), self._id, task_id) task.trigger_event(Event.SUCCESS, task) status = DONE except KeyboardInterrupt: raise except Exception as ex: status = FAILED logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self._id, task) error_message = task.on_failure(ex) task.trigger_event(Event.FAILURE, task, ex) subject = "Luigi: %s FAILED" % task notifications.send_error_email(subject, error_message) self._scheduler.add_task(self._id, task_id, status=status, expl=error_message, runnable=None) self.run_succeeded &= status == DONE return status
def _run_task(self, task_id): task = self._scheduled_tasks[task_id] logger.info('[pid %s] Worker %s running %s', os.getpid(), self._id, task_id) try: # Verify that all the tasks are fulfilled! ok = True for task_2 in task.deps(): if not task_2.complete(): ok = False missing_dep = task_2 if not ok: # TODO: possibly try to re-add task again ad pending raise RuntimeError('Unfulfilled dependency %r at run time!\nPrevious tasks: %r' % (missing_dep.task_id, self._previous_tasks)) task.run() error_message = json.dumps(task.on_success()) logger.info('[pid %s] Worker %s done %s', os.getpid(), self._id, task_id) task.trigger_event(Event.SUCCESS, task) status = DONE except KeyboardInterrupt: raise except Exception as ex: status = FAILED logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self._id, task) error_message = task.on_failure(ex) task.trigger_event(Event.FAILURE, task, ex) subject = "Luigi: %s FAILED" % task notifications.send_error_email(subject, error_message) self._scheduler.add_task(self._id, task_id, status=status, expl=error_message, runnable=None) return status
def run(self): logger.info('[pid %s] Worker %s running %s', os.getpid(), self.worker_id, self.task.task_id) if self.random_seed: # Need to have different random seeds if running in separate processes random.seed((os.getpid(), time.time())) try: # Verify that all the tasks are fulfilled! missing = [dep.task_id for dep in self.task.deps() if not dep.complete()] if missing: deps = 'dependency' if len(missing) == 1 else 'dependencies' raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing))) self.task.trigger_event(Event.START, self.task) t0 = time.time() try: self.task.run() finally: self.task.trigger_event(Event.PROCESSING_TIME, self.task, time.time() - t0) error_message = json.dumps(self.task.on_success()) logger.info('[pid %s] Worker %s done %s', os.getpid(), self.worker_id, self.task.task_id) self.task.trigger_event(Event.SUCCESS, self.task) status = DONE except KeyboardInterrupt: raise except Exception as ex: status = FAILED logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self.worker_id, self.task) error_message = self.task.on_failure(ex) self.task.trigger_event(Event.FAILURE, self.task, ex) subject = "Luigi: %s FAILED" % self.task notifications.send_error_email(subject, error_message) self.result_queue.put((self.task.task_id, status, error_message, missing))
def _email_complete_error(self, task, formatted_traceback): # like logger.exception but with WARNING level formatted_traceback = notifications.wrap_traceback(formatted_traceback) subject = "Luigi: {task} failed scheduling".format(task=task) message = "Will not schedule {task} or any dependencies due to error in complete() method:\n{traceback}".format( task=task, traceback=formatted_traceback) notifications.send_error_email(subject, message)
def set_status(self, new_status, config): # not sure why we have SUSPENDED, as it can never be set if new_status == SUSPENDED: new_status = PENDING if new_status == DISABLED and self.status == RUNNING: return if self.status == DISABLED: if new_status == DONE: self.re_enable() # don't allow workers to override a scheduler disable elif self.scheduler_disable_time is not None: return if new_status == FAILED and self.can_disable(): self.add_failure() if self.has_excessive_failures(): self.scheduler_disable_time = time.time() new_status = DISABLED notifications.send_error_email( 'Luigi Scheduler: DISABLED {task} due to excessive failures'.format(task=self.id), '{task} failed {failures} times in the last {window} seconds, so it is being ' 'disabled for {persist} seconds'.format( failures=config.disable_failures, task=self.id, window=config.disable_window, persist=config.disable_persist, )) elif new_status == DISABLED: self.scheduler_disable_time = None self.status = new_status
def _run_task(self, task_id): task = self.__scheduled_tasks[task_id] logger.info('[pid %s] Running %s', os.getpid(), task_id) try: # Verify that all the tasks are fulfilled! ok = True for task_2 in task.deps(): if not task_2.complete(): ok = False missing_dep = task_2 if not ok: # TODO: possibly try to re-add task again ad pending raise RuntimeError('Unfulfilled dependency %r at run time!\nPrevious tasks: %r' % (missing_dep.task_id, self._previous_tasks)) task.run() error_message = json.dumps(task.on_success()) logger.info('[pid %s] Done %s', os.getpid(), task_id) task.trigger_event(Event.SUCCESS, task) status = DONE except KeyboardInterrupt: raise except Exception as ex: status = FAILED logger.exception("[pid %s] Error while running %s", os.getpid(), task) error_message = task.on_failure(ex) task.trigger_event(Event.FAILURE, task, ex) subject = "Luigi: %s FAILED" % task notifications.send_error_email(subject, error_message) self.__scheduler.add_task(self.__id, task_id, status=status, expl=error_message, runnable=None) return status
def _email_unexpected_error(self, task, formatted_traceback): formatted_traceback = notifications.wrap_traceback(formatted_traceback) subject = "Luigi: Framework error while scheduling {task}".format( task=task) message = "Luigi framework error:\n{traceback}".format( traceback=formatted_traceback) notifications.send_error_email(subject, message)
def _email_complete_error(self, task, formatted_traceback): # like logger.exception but with WARNING level subject = "Luigi: {task} failed scheduling".format(task=task) message = "Will not schedule {task} or any dependencies due to error in complete() method:\n{traceback}".format( task=task, traceback=formatted_traceback ) notifications.send_error_email(subject, message)
def set_status(self, new_status, config): # not sure why we have SUSPENDED, as it can never be set if new_status == SUSPENDED: new_status = PENDING if new_status == DISABLED and self.status == RUNNING: return if self.status == DISABLED: if new_status == DONE: self.re_enable() # don't allow workers to override a scheduler disable elif self.scheduler_disable_time is not None: return if new_status == FAILED and self.can_disable(): self.add_failure() if self.has_excessive_failures(): self.scheduler_disable_time = time.time() new_status = DISABLED notifications.send_error_email( 'Luigi Scheduler: DISABLED {task} due to excessive failures' .format(task=self.id), '{task} failed {failures} times in the last {window} seconds, so it is being ' 'disabled for {persist} seconds'.format( failures=config.disable_failures, task=self.id, window=config.disable_window, persist=config.disable_persist, )) elif new_status == DISABLED: self.scheduler_disable_time = None self.status = new_status
def _run_task(self, task_id): task = self._scheduled_tasks[task_id] logger.info('[pid %s] Worker %s running %s', os.getpid(), self._id, task_id) try: # Verify that all the tasks are fulfilled! missing = [ dep.task_id for dep in task.deps() if not dep.complete() ] if missing: deps = 'dependency' if len(missing) == 1 else 'dependencies' raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing))) task.trigger_event(Event.START, task) task.run() error_message = json.dumps(task.on_success()) logger.info('[pid %s] Worker %s done %s', os.getpid(), self._id, task_id) task.trigger_event(Event.SUCCESS, task) status = DONE except KeyboardInterrupt: raise except Exception as ex: status = FAILED logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self._id, task) error_message = task.on_failure(ex) task.trigger_event(Event.FAILURE, task, ex) subject = "Luigi: %s FAILED" % task notifications.send_error_email(subject, error_message) self._scheduler.add_task(self._id, task_id, status=status, expl=error_message, runnable=None, params=task.to_str_params(), family=task.task_family) # re-add task to reschedule missing dependencies if missing: reschedule = True # keep out of infinite loops by not rescheduling too many times for task_id in missing: self.unfulfilled_counts[task_id] += 1 if self.unfulfilled_counts[task_id] > self.__max_reschedules: reschedule = False if reschedule: self.add(task) self.run_succeeded &= status == DONE return status
def _run_task(self, task_id): task = self._scheduled_tasks[task_id] logger.info('[pid %s] Worker %s running %s', os.getpid(), self._id, task_id) try: # Verify that all the tasks are fulfilled! missing = [dep.task_id for dep in task.deps() if not dep.complete()] if missing: deps = 'dependency' if len(missing) == 1 else 'dependencies' raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing))) task.trigger_event(Event.START, task) t0 = time.time() try: task.run() finally: task.trigger_event(Event.PROCESSING_TIME, task, time.time() - t0) error_message = json.dumps(task.on_success()) logger.info('[pid %s] Worker %s done %s', os.getpid(), self._id, task_id) task.trigger_event(Event.SUCCESS, task) status = DONE except KeyboardInterrupt: raise except Exception as ex: status = FAILED logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self._id, task) error_message = task.on_failure(ex) task.trigger_event(Event.FAILURE, task, ex) subject = "Luigi: %s FAILED" % task notifications.send_error_email(subject, error_message) self._scheduler.add_task(self._id, task_id, status=status, expl=error_message, runnable=None, params=task.to_str_params(), family=task.task_family) # re-add task to reschedule missing dependencies if missing: reschedule = True # keep out of infinite loops by not rescheduling too many times for task_id in missing: self.unfulfilled_counts[task_id] += 1 if self.unfulfilled_counts[task_id] > self.__max_reschedules: reschedule = False if reschedule: self.add(task) self.run_succeeded &= status == DONE return status
def run(self): logger.info('[pid %s] Worker %s running %s', os.getpid(), self.worker_id, self.task.task_id) if self.random_seed: # Need to have different random seeds if running in separate processes random.seed((os.getpid(), time.time())) status = FAILED error_message = '' missing = [] try: # Verify that all the tasks are fulfilled! missing = [ dep.task_id for dep in self.task.deps() if not dep.complete() ] if missing: deps = 'dependency' if len(missing) == 1 else 'dependencies' raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing))) self.task.trigger_event(Event.START, self.task) t0 = time.time() try: self.task.run() finally: self.task.trigger_event(Event.PROCESSING_TIME, self.task, time.time() - t0) error_message = json.dumps(self.task.on_success()) logger.info('[pid %s] Worker %s done %s', os.getpid(), self.worker_id, self.task.task_id) self.task.trigger_event(Event.SUCCESS, self.task) status = DONE except KeyboardInterrupt: raise except BaseException as ex: status = FAILED logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self.worker_id, self.task) error_message = notifications.wrap_traceback( self.task.on_failure(ex)) self.task.trigger_event(Event.FAILURE, self.task, ex) subject = "Luigi: %s FAILED" % self.task notifications.send_error_email(subject, error_message) finally: self.result_queue.put( (self.task.task_id, status, error_message, missing))
def run(self): logger.info('[pid %s] Worker %s running %s', os.getpid(), self.worker_id, self.task.task_id) if self.random_seed: # Need to have different random seeds if running in separate processes random.seed((os.getpid(), time.time())) status = FAILED error_message = '' try: self.task.trigger_event(Event.START, self.task) t0 = time.time() status = None try: status = DONE if self.task.complete() else FAILED logger.debug("[pid %s] Task %s has status %s" % (os.getpid(), self.task, status)) finally: self.task.trigger_event(Event.PROCESSING_TIME, self.task, time.time() - t0) error_message = json.dumps(self.task.on_success()) logger.info('[pid %s] Worker %s done %s', os.getpid(), self.worker_id, self.task.task_id) self.task.trigger_event(Event.SUCCESS, self.task) except KeyboardInterrupt: raise except BaseException as ex: status = FAILED logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self.worker_id, self.task) error_message = notifications.wrap_traceback( self.task.on_failure(ex)) self.task.trigger_event(Event.FAILURE, self.task, ex) subject = "Luigi: %s FAILED" % self.task notifications.send_error_email(subject, error_message) finally: logger.debug("Putting result into queue: %s %s %s" % (self.task.task_id, status, error_message)) self.result_queue.put( (self.task.task_id, status, error_message, [], []))
def _run_task(self, task_id): task = self._scheduled_tasks[task_id] logger.info('[pid %s] Worker %s running %s', os.getpid(), self._id, task_id) try: # Verify that all the tasks are fulfilled! missing = [dep.task_id for dep in task.deps() if not dep.complete()] if missing: deps = 'dependency' if len(missing) == 1 else 'dependencies' raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing))) task.trigger_event(Event.START, task) t0 = time.time() try: task.run() finally: task.trigger_event(Event.PROCESSING_TIME, task, time.time() - t0) error_message = json.dumps(task.on_success()) logger.info('[pid %s] Worker %s done %s', os.getpid(), self._id, task_id) task.trigger_event(Event.SUCCESS, task) status = DONE except KeyboardInterrupt: raise except Exception as ex: status = FAILED logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self._id, task) error_message = task.on_failure(ex) task.trigger_event(Event.FAILURE, task, ex) subject = "Luigi: %s FAILED" % task notifications.send_error_email(subject, error_message) self._scheduler.add_task(self._id, task_id, status=status, expl=error_message, runnable=None, params=task.to_str_params(), family=task.task_family) self.run_succeeded &= status == DONE return status
def set_status(self, task, new_status): # not sure why we have SUSPENDED, as it can never be set if new_status == SUSPENDED: new_status = PENDING if new_status == DISABLED and task.status == RUNNING: return if task.status == DISABLED: if new_status == DISABLED: task.scheduler_disable_time = None elif new_status == DONE: task.re_enable() task.status = DONE elif task.scheduler_disable_time is None: # when it is disabled by client, we allow the status change task.status = new_status return if new_status == FAILED and task.can_disable(): task.add_failure() if task.has_excessive_failures(): task.scheduler_disable_time = datetime.datetime.now() new_status = DISABLED notifications.send_error_email( 'Luigi Scheduler: DISABLED {task} due to excessive failures' .format(task=task.id), '{task} failed {failures} times in the last {window} seconds, so it is being ' 'disabled for {persist} seconds'.format( failures=self._disable_failures, task=task.id, window=self._disable_window, persist=self._disable_persist, )) elif new_status == DISABLED: task.scheduler_disable_time = None task.status = new_status
def set_status(self, task, new_status): # not sure why we have SUSPENDED, as it can never be set if new_status == SUSPENDED: new_status = PENDING if new_status == DISABLED and task.status == RUNNING: return if task.status == DISABLED: if new_status == DISABLED: task.scheduler_disable_time = None elif new_status == DONE: task.re_enable() task.status = DONE elif task.scheduler_disable_time is None: # when it is disabled by client, we allow the status change task.status = new_status return if new_status == FAILED and task.can_disable(): task.add_failure() if task.has_excessive_failures(): task.scheduler_disable_time = datetime.datetime.now() new_status = DISABLED notifications.send_error_email( 'Luigi Scheduler: DISABLED {task} due to excessive failures'.format(task=task.id), '{task} failed {failures} times in the last {window} seconds, so it is being ' 'disabled for {persist} seconds'.format( failures=self._disable_failures, task=task.id, window=self._disable_window, persist=self._disable_persist, )) elif new_status == DISABLED: task.scheduler_disable_time = None task.status = new_status
def run(self): logger.info('[pid %s] Worker %s running %s', os.getpid(), self.worker_id, self.task.task_id) if self.random_seed: # Need to have different random seeds if running in separate processes random.seed((os.getpid(), time.time())) status = FAILED error_message = '' missing = [] new_deps = [] try: # Verify that all the tasks are fulfilled! missing = [ dep.task_id for dep in self.task.deps() if not dep.complete() ] if missing: deps = 'dependency' if len(missing) == 1 else 'dependencies' raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing))) self.task.trigger_event(Event.START, self.task) t0 = time.time() status = None try: task_gen = self.task.run() if isinstance(task_gen, types.GeneratorType): # new deps next_send = None while True: try: if next_send is None: requires = task_gen.next() else: requires = task_gen.send(next_send) except StopIteration: break new_req = flatten(requires) status = (RUNNING if all( t.complete() for t in new_req) else SUSPENDED) new_deps = [(t.task_module, t.task_family, t.to_str_params()) for t in new_req] if status == RUNNING: self.result_queue.put((self.task.task_id, status, '', missing, new_deps)) next_send = getpaths(requires) new_deps = [] else: logger.info( '[pid %s] Worker %s new requirements %s', os.getpid(), self.worker_id, self.task.task_id) return finally: if status != SUSPENDED: self.task.trigger_event(Event.PROCESSING_TIME, self.task, time.time() - t0) error_message = json.dumps(self.task.on_success()) logger.info('[pid %s] Worker %s done %s', os.getpid(), self.worker_id, self.task.task_id) self.task.trigger_event(Event.SUCCESS, self.task) status = DONE except KeyboardInterrupt: raise except BaseException as ex: status = FAILED logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self.worker_id, self.task) error_message = notifications.wrap_traceback( self.task.on_failure(ex)) self.task.trigger_event(Event.FAILURE, self.task, ex) subject = "Luigi: %s FAILED" % self.task notifications.send_error_email(subject, error_message) finally: self.result_queue.put( (self.task.task_id, status, error_message, missing, new_deps))
def _email_unexpected_error(self, task, formatted_traceback): subject = "Luigi: Framework error while scheduling {task}".format(task=task) message = "Luigi framework error:\n{traceback}".format(traceback=formatted_traceback) notifications.send_error_email(subject, message)
def run(self): logger.info('[pid %s] Worker %s running %s', os.getpid(), self.worker_id, self.task.task_id) if self.random_seed: # Need to have different random seeds if running in separate processes random.seed((os.getpid(), time.time())) status = FAILED error_message = '' missing = [] new_deps = [] try: # Verify that all the tasks are fulfilled! missing = [dep.task_id for dep in self.task.deps() if not dep.complete()] if missing: deps = 'dependency' if len(missing) == 1 else 'dependencies' raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing))) self.task.trigger_event(Event.START, self.task) t0 = time.time() status = None try: task_gen = self.task.run() if isinstance(task_gen, types.GeneratorType): # new deps next_send = None while True: try: if next_send is None: requires = task_gen.next() else: requires = task_gen.send(next_send) except StopIteration: break new_req = flatten(requires) status = (RUNNING if all(t.complete() for t in new_req) else SUSPENDED) new_deps = [(t.task_family, t.to_str_params()) for t in new_req] if status == RUNNING: self.result_queue.put( (self.task.task_id, status, '', missing, new_deps)) next_send = getpaths(requires) else: logger.info( '[pid %s] Worker %s new requirements %s', os.getpid(), self.worker_id, self.task.task_id) return finally: if status != SUSPENDED: self.task.trigger_event( Event.PROCESSING_TIME, self.task, time.time() - t0) error_message = json.dumps(self.task.on_success()) logger.info('[pid %s] Worker %s done %s', os.getpid(), self.worker_id, self.task.task_id) self.task.trigger_event(Event.SUCCESS, self.task) status = DONE except KeyboardInterrupt: raise except BaseException as ex: status = FAILED logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self.worker_id, self.task) error_message = notifications.wrap_traceback(self.task.on_failure(ex)) self.task.trigger_event(Event.FAILURE, self.task, ex) subject = "Luigi: %s FAILED" % self.task notifications.send_error_email(subject, error_message) finally: self.result_queue.put( (self.task.task_id, status, error_message, missing, new_deps))