def _run_task(self, task_id):
task = self._scheduled_tasks[task_id]
logger.info('[pid %s] Worker %s running %s', os.getpid(), self._id, task_id)
try:
# Verify that all the tasks are fulfilled!
missing = [dep.task_id for dep in task.deps() if not dep.complete()]
if missing:
deps = 'dependency' if len(missing) == 1 else 'dependencies'
# TODO: possibly try to re-add task again ad pending
raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing)))
task.trigger_event(Event.START, task)
task.run()
error_message = json.dumps(task.on_success())
logger.info('[pid %s] Worker %s done %s', os.getpid(), self._id, task_id)
task.trigger_event(Event.SUCCESS, task)
status = DONE
except KeyboardInterrupt:
raise
except Exception as ex:
status = FAILED
logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self._id, task)
error_message = task.on_failure(ex)
task.trigger_event(Event.FAILURE, task, ex)
subject = "Luigi: %s FAILED" % task
notifications.send_error_email(subject, error_message)
self._scheduler.add_task(self._id, task_id, status=status,
expl=error_message, runnable=None)
self.run_succeeded &= status == DONE
return status
def _run_task(self, task_id):
task = self._scheduled_tasks[task_id]
logger.info('[pid %s] Worker %s running %s', os.getpid(), self._id, task_id)
try:
# Verify that all the tasks are fulfilled!
ok = True
for task_2 in task.deps():
if not task_2.complete():
ok = False
missing_dep = task_2
if not ok:
# TODO: possibly try to re-add task again ad pending
raise RuntimeError('Unfulfilled dependency %r at run time!\nPrevious tasks: %r' % (missing_dep.task_id, self._previous_tasks))
task.run()
error_message = json.dumps(task.on_success())
logger.info('[pid %s] Worker %s done %s', os.getpid(), self._id, task_id)
task.trigger_event(Event.SUCCESS, task)
status = DONE
except KeyboardInterrupt:
raise
except Exception as ex:
status = FAILED
logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self._id, task)
error_message = task.on_failure(ex)
task.trigger_event(Event.FAILURE, task, ex)
subject = "Luigi: %s FAILED" % task
notifications.send_error_email(subject, error_message)
self._scheduler.add_task(self._id, task_id, status=status,
expl=error_message, runnable=None)
return status
def run(self):
logger.info('[pid %s] Worker %s running %s', os.getpid(), self.worker_id, self.task.task_id)
if self.random_seed:
# Need to have different random seeds if running in separate processes
random.seed((os.getpid(), time.time()))
try:
# Verify that all the tasks are fulfilled!
missing = [dep.task_id for dep in self.task.deps() if not dep.complete()]
if missing:
deps = 'dependency' if len(missing) == 1 else 'dependencies'
raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing)))
self.task.trigger_event(Event.START, self.task)
t0 = time.time()
try:
self.task.run()
finally:
self.task.trigger_event(Event.PROCESSING_TIME, self.task, time.time() - t0)
error_message = json.dumps(self.task.on_success())
logger.info('[pid %s] Worker %s done %s', os.getpid(), self.worker_id, self.task.task_id)
self.task.trigger_event(Event.SUCCESS, self.task)
status = DONE
except KeyboardInterrupt:
raise
except Exception as ex:
status = FAILED
logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self.worker_id, self.task)
error_message = self.task.on_failure(ex)
self.task.trigger_event(Event.FAILURE, self.task, ex)
subject = "Luigi: %s FAILED" % self.task
notifications.send_error_email(subject, error_message)
self.result_queue.put((self.task.task_id, status, error_message, missing))
def _email_complete_error(self, task, formatted_traceback):
# like logger.exception but with WARNING level
formatted_traceback = notifications.wrap_traceback(formatted_traceback)
subject = "Luigi: {task} failed scheduling".format(task=task)
message = "Will not schedule {task} or any dependencies due to error in complete() method:\n{traceback}".format(
task=task, traceback=formatted_traceback)
notifications.send_error_email(subject, message)
def set_status(self, new_status, config):
# not sure why we have SUSPENDED, as it can never be set
if new_status == SUSPENDED:
new_status = PENDING
if new_status == DISABLED and self.status == RUNNING:
return
if self.status == DISABLED:
if new_status == DONE:
self.re_enable()
# don't allow workers to override a scheduler disable
elif self.scheduler_disable_time is not None:
return
if new_status == FAILED and self.can_disable():
self.add_failure()
if self.has_excessive_failures():
self.scheduler_disable_time = time.time()
new_status = DISABLED
notifications.send_error_email(
'Luigi Scheduler: DISABLED {task} due to excessive failures'.format(task=self.id),
'{task} failed {failures} times in the last {window} seconds, so it is being '
'disabled for {persist} seconds'.format(
failures=config.disable_failures,
task=self.id,
window=config.disable_window,
persist=config.disable_persist,
))
elif new_status == DISABLED:
self.scheduler_disable_time = None
self.status = new_status
def _run_task(self, task_id):
task = self.__scheduled_tasks[task_id]
logger.info('[pid %s] Running %s', os.getpid(), task_id)
try:
# Verify that all the tasks are fulfilled!
ok = True
for task_2 in task.deps():
if not task_2.complete():
ok = False
missing_dep = task_2
if not ok:
# TODO: possibly try to re-add task again ad pending
raise RuntimeError('Unfulfilled dependency %r at run time!\nPrevious tasks: %r' % (missing_dep.task_id, self._previous_tasks))
task.run()
error_message = json.dumps(task.on_success())
logger.info('[pid %s] Done %s', os.getpid(), task_id)
task.trigger_event(Event.SUCCESS, task)
status = DONE
except KeyboardInterrupt:
raise
except Exception as ex:
status = FAILED
logger.exception("[pid %s] Error while running %s", os.getpid(), task)
error_message = task.on_failure(ex)
task.trigger_event(Event.FAILURE, task, ex)
subject = "Luigi: %s FAILED" % task
notifications.send_error_email(subject, error_message)
self.__scheduler.add_task(self.__id, task_id, status=status,
expl=error_message, runnable=None)
return status
def _email_unexpected_error(self, task, formatted_traceback):
formatted_traceback = notifications.wrap_traceback(formatted_traceback)
subject = "Luigi: Framework error while scheduling {task}".format(
task=task)
message = "Luigi framework error:\n{traceback}".format(
traceback=formatted_traceback)
notifications.send_error_email(subject, message)
def _email_complete_error(self, task, formatted_traceback):
# like logger.exception but with WARNING level
subject = "Luigi: {task} failed scheduling".format(task=task)
message = "Will not schedule {task} or any dependencies due to error in complete() method:\n{traceback}".format(
task=task, traceback=formatted_traceback
)
notifications.send_error_email(subject, message)
def set_status(self, new_status, config):
# not sure why we have SUSPENDED, as it can never be set
if new_status == SUSPENDED:
new_status = PENDING
if new_status == DISABLED and self.status == RUNNING:
return
if self.status == DISABLED:
if new_status == DONE:
self.re_enable()
# don't allow workers to override a scheduler disable
elif self.scheduler_disable_time is not None:
return
if new_status == FAILED and self.can_disable():
self.add_failure()
if self.has_excessive_failures():
self.scheduler_disable_time = time.time()
new_status = DISABLED
notifications.send_error_email(
'Luigi Scheduler: DISABLED {task} due to excessive failures'
.format(task=self.id),
'{task} failed {failures} times in the last {window} seconds, so it is being '
'disabled for {persist} seconds'.format(
failures=config.disable_failures,
task=self.id,
window=config.disable_window,
persist=config.disable_persist,
))
elif new_status == DISABLED:
self.scheduler_disable_time = None
self.status = new_status
def _run_task(self, task_id):
task = self._scheduled_tasks[task_id]
logger.info('[pid %s] Worker %s running %s', os.getpid(), self._id,
task_id)
try:
# Verify that all the tasks are fulfilled!
missing = [
dep.task_id for dep in task.deps() if not dep.complete()
]
if missing:
deps = 'dependency' if len(missing) == 1 else 'dependencies'
raise RuntimeError('Unfulfilled %s at run time: %s' %
(deps, ', '.join(missing)))
task.trigger_event(Event.START, task)
task.run()
error_message = json.dumps(task.on_success())
logger.info('[pid %s] Worker %s done %s', os.getpid(),
self._id, task_id)
task.trigger_event(Event.SUCCESS, task)
status = DONE
except KeyboardInterrupt:
raise
except Exception as ex:
status = FAILED
logger.exception("[pid %s] Worker %s failed %s", os.getpid(),
self._id, task)
error_message = task.on_failure(ex)
task.trigger_event(Event.FAILURE, task, ex)
subject = "Luigi: %s FAILED" % task
notifications.send_error_email(subject, error_message)
self._scheduler.add_task(self._id,
task_id,
status=status,
expl=error_message,
runnable=None,
params=task.to_str_params(),
family=task.task_family)
# re-add task to reschedule missing dependencies
if missing:
reschedule = True
# keep out of infinite loops by not rescheduling too many times
for task_id in missing:
self.unfulfilled_counts[task_id] += 1
if self.unfulfilled_counts[task_id] > self.__max_reschedules:
reschedule = False
if reschedule:
self.add(task)
self.run_succeeded &= status == DONE
return status
def _run_task(self, task_id):
task = self._scheduled_tasks[task_id]
logger.info('[pid %s] Worker %s running %s', os.getpid(), self._id, task_id)
try:
# Verify that all the tasks are fulfilled!
missing = [dep.task_id for dep in task.deps() if not dep.complete()]
if missing:
deps = 'dependency' if len(missing) == 1 else 'dependencies'
raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing)))
task.trigger_event(Event.START, task)
t0 = time.time()
try:
task.run()
finally:
task.trigger_event(Event.PROCESSING_TIME, task, time.time() - t0)
error_message = json.dumps(task.on_success())
logger.info('[pid %s] Worker %s done %s', os.getpid(), self._id, task_id)
task.trigger_event(Event.SUCCESS, task)
status = DONE
except KeyboardInterrupt:
raise
except Exception as ex:
status = FAILED
logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self._id, task)
error_message = task.on_failure(ex)
task.trigger_event(Event.FAILURE, task, ex)
subject = "Luigi: %s FAILED" % task
notifications.send_error_email(subject, error_message)
self._scheduler.add_task(self._id, task_id, status=status,
expl=error_message, runnable=None,
params=task.to_str_params(),
family=task.task_family)
# re-add task to reschedule missing dependencies
if missing:
reschedule = True
# keep out of infinite loops by not rescheduling too many times
for task_id in missing:
self.unfulfilled_counts[task_id] += 1
if self.unfulfilled_counts[task_id] > self.__max_reschedules:
reschedule = False
if reschedule:
self.add(task)
self.run_succeeded &= status == DONE
return status
def run(self):
logger.info('[pid %s] Worker %s running %s', os.getpid(),
self.worker_id, self.task.task_id)
if self.random_seed:
# Need to have different random seeds if running in separate processes
random.seed((os.getpid(), time.time()))
status = FAILED
error_message = ''
missing = []
try:
# Verify that all the tasks are fulfilled!
missing = [
dep.task_id for dep in self.task.deps() if not dep.complete()
]
if missing:
deps = 'dependency' if len(missing) == 1 else 'dependencies'
raise RuntimeError('Unfulfilled %s at run time: %s' %
(deps, ', '.join(missing)))
self.task.trigger_event(Event.START, self.task)
t0 = time.time()
try:
self.task.run()
finally:
self.task.trigger_event(Event.PROCESSING_TIME, self.task,
time.time() - t0)
error_message = json.dumps(self.task.on_success())
logger.info('[pid %s] Worker %s done %s', os.getpid(),
self.worker_id, self.task.task_id)
self.task.trigger_event(Event.SUCCESS, self.task)
status = DONE
except KeyboardInterrupt:
raise
except BaseException as ex:
status = FAILED
logger.exception("[pid %s] Worker %s failed %s", os.getpid(),
self.worker_id, self.task)
error_message = notifications.wrap_traceback(
self.task.on_failure(ex))
self.task.trigger_event(Event.FAILURE, self.task, ex)
subject = "Luigi: %s FAILED" % self.task
notifications.send_error_email(subject, error_message)
finally:
self.result_queue.put(
(self.task.task_id, status, error_message, missing))
def run(self):
logger.info('[pid %s] Worker %s running %s', os.getpid(),
self.worker_id, self.task.task_id)
if self.random_seed:
# Need to have different random seeds if running in separate processes
random.seed((os.getpid(), time.time()))
status = FAILED
error_message = ''
try:
self.task.trigger_event(Event.START, self.task)
t0 = time.time()
status = None
try:
status = DONE if self.task.complete() else FAILED
logger.debug("[pid %s] Task %s has status %s" %
(os.getpid(), self.task, status))
finally:
self.task.trigger_event(Event.PROCESSING_TIME, self.task,
time.time() - t0)
error_message = json.dumps(self.task.on_success())
logger.info('[pid %s] Worker %s done %s', os.getpid(),
self.worker_id, self.task.task_id)
self.task.trigger_event(Event.SUCCESS, self.task)
except KeyboardInterrupt:
raise
except BaseException as ex:
status = FAILED
logger.exception("[pid %s] Worker %s failed %s", os.getpid(),
self.worker_id, self.task)
error_message = notifications.wrap_traceback(
self.task.on_failure(ex))
self.task.trigger_event(Event.FAILURE, self.task, ex)
subject = "Luigi: %s FAILED" % self.task
notifications.send_error_email(subject, error_message)
finally:
logger.debug("Putting result into queue: %s %s %s" %
(self.task.task_id, status, error_message))
self.result_queue.put(
(self.task.task_id, status, error_message, [], []))
def _run_task(self, task_id):
task = self._scheduled_tasks[task_id]
logger.info('[pid %s] Worker %s running %s', os.getpid(), self._id, task_id)
try:
# Verify that all the tasks are fulfilled!
missing = [dep.task_id for dep in task.deps() if not dep.complete()]
if missing:
deps = 'dependency' if len(missing) == 1 else 'dependencies'
raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing)))
task.trigger_event(Event.START, task)
t0 = time.time()
try:
task.run()
finally:
task.trigger_event(Event.PROCESSING_TIME, task, time.time() - t0)
error_message = json.dumps(task.on_success())
logger.info('[pid %s] Worker %s done %s', os.getpid(), self._id, task_id)
task.trigger_event(Event.SUCCESS, task)
status = DONE
except KeyboardInterrupt:
raise
except Exception as ex:
status = FAILED
logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self._id, task)
error_message = task.on_failure(ex)
task.trigger_event(Event.FAILURE, task, ex)
subject = "Luigi: %s FAILED" % task
notifications.send_error_email(subject, error_message)
self._scheduler.add_task(self._id, task_id, status=status,
expl=error_message, runnable=None,
params=task.to_str_params(),
family=task.task_family)
self.run_succeeded &= status == DONE
return status
def set_status(self, task, new_status):
# not sure why we have SUSPENDED, as it can never be set
if new_status == SUSPENDED:
new_status = PENDING
if new_status == DISABLED and task.status == RUNNING:
return
if task.status == DISABLED:
if new_status == DISABLED:
task.scheduler_disable_time = None
elif new_status == DONE:
task.re_enable()
task.status = DONE
elif task.scheduler_disable_time is None:
# when it is disabled by client, we allow the status change
task.status = new_status
return
if new_status == FAILED and task.can_disable():
task.add_failure()
if task.has_excessive_failures():
task.scheduler_disable_time = datetime.datetime.now()
new_status = DISABLED
notifications.send_error_email(
'Luigi Scheduler: DISABLED {task} due to excessive failures'
.format(task=task.id),
'{task} failed {failures} times in the last {window} seconds, so it is being '
'disabled for {persist} seconds'.format(
failures=self._disable_failures,
task=task.id,
window=self._disable_window,
persist=self._disable_persist,
))
elif new_status == DISABLED:
task.scheduler_disable_time = None
task.status = new_status
def set_status(self, task, new_status):
# not sure why we have SUSPENDED, as it can never be set
if new_status == SUSPENDED:
new_status = PENDING
if new_status == DISABLED and task.status == RUNNING:
return
if task.status == DISABLED:
if new_status == DISABLED:
task.scheduler_disable_time = None
elif new_status == DONE:
task.re_enable()
task.status = DONE
elif task.scheduler_disable_time is None:
# when it is disabled by client, we allow the status change
task.status = new_status
return
if new_status == FAILED and task.can_disable():
task.add_failure()
if task.has_excessive_failures():
task.scheduler_disable_time = datetime.datetime.now()
new_status = DISABLED
notifications.send_error_email(
'Luigi Scheduler: DISABLED {task} due to excessive failures'.format(task=task.id),
'{task} failed {failures} times in the last {window} seconds, so it is being '
'disabled for {persist} seconds'.format(
failures=self._disable_failures,
task=task.id,
window=self._disable_window,
persist=self._disable_persist,
))
elif new_status == DISABLED:
task.scheduler_disable_time = None
task.status = new_status
def run(self):
logger.info('[pid %s] Worker %s running %s', os.getpid(),
self.worker_id, self.task.task_id)
if self.random_seed:
# Need to have different random seeds if running in separate processes
random.seed((os.getpid(), time.time()))
status = FAILED
error_message = ''
missing = []
new_deps = []
try:
# Verify that all the tasks are fulfilled!
missing = [
dep.task_id for dep in self.task.deps() if not dep.complete()
]
if missing:
deps = 'dependency' if len(missing) == 1 else 'dependencies'
raise RuntimeError('Unfulfilled %s at run time: %s' %
(deps, ', '.join(missing)))
self.task.trigger_event(Event.START, self.task)
t0 = time.time()
status = None
try:
task_gen = self.task.run()
if isinstance(task_gen, types.GeneratorType): # new deps
next_send = None
while True:
try:
if next_send is None:
requires = task_gen.next()
else:
requires = task_gen.send(next_send)
except StopIteration:
break
new_req = flatten(requires)
status = (RUNNING if all(
t.complete() for t in new_req) else SUSPENDED)
new_deps = [(t.task_module, t.task_family,
t.to_str_params()) for t in new_req]
if status == RUNNING:
self.result_queue.put((self.task.task_id, status,
'', missing, new_deps))
next_send = getpaths(requires)
new_deps = []
else:
logger.info(
'[pid %s] Worker %s new requirements %s',
os.getpid(), self.worker_id, self.task.task_id)
return
finally:
if status != SUSPENDED:
self.task.trigger_event(Event.PROCESSING_TIME, self.task,
time.time() - t0)
error_message = json.dumps(self.task.on_success())
logger.info('[pid %s] Worker %s done %s', os.getpid(),
self.worker_id, self.task.task_id)
self.task.trigger_event(Event.SUCCESS, self.task)
status = DONE
except KeyboardInterrupt:
raise
except BaseException as ex:
status = FAILED
logger.exception("[pid %s] Worker %s failed %s", os.getpid(),
self.worker_id, self.task)
error_message = notifications.wrap_traceback(
self.task.on_failure(ex))
self.task.trigger_event(Event.FAILURE, self.task, ex)
subject = "Luigi: %s FAILED" % self.task
notifications.send_error_email(subject, error_message)
finally:
self.result_queue.put(
(self.task.task_id, status, error_message, missing, new_deps))
def _email_unexpected_error(self, task, formatted_traceback):
subject = "Luigi: Framework error while scheduling {task}".format(task=task)
message = "Luigi framework error:\n{traceback}".format(traceback=formatted_traceback)
notifications.send_error_email(subject, message)
def run(self):
logger.info('[pid %s] Worker %s running %s', os.getpid(), self.worker_id, self.task.task_id)
if self.random_seed:
# Need to have different random seeds if running in separate processes
random.seed((os.getpid(), time.time()))
status = FAILED
error_message = ''
missing = []
new_deps = []
try:
# Verify that all the tasks are fulfilled!
missing = [dep.task_id for dep in self.task.deps() if not dep.complete()]
if missing:
deps = 'dependency' if len(missing) == 1 else 'dependencies'
raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing)))
self.task.trigger_event(Event.START, self.task)
t0 = time.time()
status = None
try:
task_gen = self.task.run()
if isinstance(task_gen, types.GeneratorType): # new deps
next_send = None
while True:
try:
if next_send is None:
requires = task_gen.next()
else:
requires = task_gen.send(next_send)
except StopIteration:
break
new_req = flatten(requires)
status = (RUNNING if all(t.complete() for t in new_req)
else SUSPENDED)
new_deps = [(t.task_family, t.to_str_params())
for t in new_req]
if status == RUNNING:
self.result_queue.put(
(self.task.task_id, status, '', missing,
new_deps))
next_send = getpaths(requires)
else:
logger.info(
'[pid %s] Worker %s new requirements %s',
os.getpid(), self.worker_id, self.task.task_id)
return
finally:
if status != SUSPENDED:
self.task.trigger_event(
Event.PROCESSING_TIME, self.task, time.time() - t0)
error_message = json.dumps(self.task.on_success())
logger.info('[pid %s] Worker %s done %s', os.getpid(),
self.worker_id, self.task.task_id)
self.task.trigger_event(Event.SUCCESS, self.task)
status = DONE
except KeyboardInterrupt:
raise
except BaseException as ex:
status = FAILED
logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self.worker_id, self.task)
error_message = notifications.wrap_traceback(self.task.on_failure(ex))
self.task.trigger_event(Event.FAILURE, self.task, ex)
subject = "Luigi: %s FAILED" % self.task
notifications.send_error_email(subject, error_message)
finally:
self.result_queue.put(
(self.task.task_id, status, error_message, missing, new_deps))
