def _email_error(self, task, formatted_traceback, subject, headline):
formatted_subject = subject.format(task=task, host=self.host)
formatted_headline = headline.format(task=task, host=self.host)
command = subprocess.list2cmdline(sys.argv)
message = notifications.format_task_error(
formatted_headline, task, command, formatted_traceback)
notifications.send_error_email(formatted_subject, message, task.owner_email)
def _email_complete_error(self, task, formatted_traceback):
# like logger.exception but with WARNING level
subject = "Luigi: {task} failed scheduling. Host: {host}".format(task=task, host=self.host)
headline = "Will not schedule task or any dependencies due to error in complete() method"
message = notifications.format_task_error(headline, task, formatted_traceback)
notifications.send_error_email(subject, message, task.owner_email)
def run(self):
logger.info('[pid %s] Worker %s running %s', os.getpid(), self.worker_id, self.task.task_id)
if self.random_seed:
# Need to have different random seeds if running in separate processes
random.seed((os.getpid(), time.time()))
status = FAILED
expl = ''
missing = []
new_deps = []
try:
# Verify that all the tasks are fulfilled!
missing = [dep.task_id for dep in self.task.deps() if not dep.complete()]
if missing:
deps = 'dependency' if len(missing) == 1 else 'dependencies'
raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing)))
self.task.trigger_event(Event.START, self.task)
t0 = time.time()
status = None
if self.task.run == NotImplemented:
# External task
# TODO(erikbern): We should check for task completeness after non-external tasks too!
# This will resolve #814 and make things a lot more consistent
status = DONE if self.task.complete() else FAILED
else:
new_deps = self._run_get_new_deps()
status = DONE if not new_deps else SUSPENDED
if status == SUSPENDED:
logger.info(
'[pid %s] Worker %s new requirements %s',
os.getpid(), self.worker_id, self.task.task_id)
elif status == DONE:
self.task.trigger_event(
Event.PROCESSING_TIME, self.task, time.time() - t0)
expl = json.dumps(self.task.on_success())
logger.info('[pid %s] Worker %s done %s', os.getpid(),
self.worker_id, self.task.task_id)
self.task.trigger_event(Event.SUCCESS, self.task)
except KeyboardInterrupt:
raise
except BaseException as ex:
status = FAILED
logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self.worker_id, self.task)
self.task.trigger_event(Event.FAILURE, self.task, ex)
subject = "Luigi: %s FAILED" % self.task
raw_error_message = self.task.on_failure(ex)
notification_error_message = notifications.wrap_traceback(raw_error_message)
expl = json.dumps(raw_error_message)
formatted_error_message = notifications.format_task_error(subject, self.task,
formatted_exception=notification_error_message)
notifications.send_error_email(subject, formatted_error_message, self.task.owner_email)
finally:
self.result_queue.put(
(self.task.task_id, status, expl, missing, new_deps))
def _email_error(self, task, formatted_traceback, subject, headline):
formatted_subject = subject.format(task=task, host=self.host)
formatted_headline = headline.format(task=task, host=self.host)
command = subprocess.list2cmdline(sys.argv)
message = notifications.format_task_error(
formatted_headline, task, command, formatted_traceback)
notifications.send_error_email(formatted_subject, message, task.owner_email)
def _email_complete_error(self, task, formatted_traceback):
# like logger.exception but with WARNING level
subject = "Luigi: {task} failed scheduling. Host: {host}".format(task=task, host=self.host)
headline = "Will not schedule task or any dependencies due to error in complete() method"
message = notifications.format_task_error(headline, task, formatted_traceback)
notifications.send_error_email(subject, message, task.owner_email)
def _send_error_notification(self, raw_error_message):
subject = "Luigi: %s FAILED" % self.task
notification_error_message = notifications.wrap_traceback(raw_error_message)
formatted_error_message = notifications.format_task_error(
subject, self.task, formatted_exception=notification_error_message
)
notifications.send_error_email(subject, formatted_error_message, self.task.owner_email)
def run(self):
logger.info('[pid %s] Worker %s running %s', os.getpid(), self.worker_id, self.task.task_id)
if self.random_seed:
# Need to have different random seeds if running in separate processes
random.seed((os.getpid(), time.time()))
status = FAILED
expl = ''
missing = []
new_deps = []
try:
# Verify that all the tasks are fulfilled!
missing = [dep.task_id for dep in self.task.deps() if not dep.complete()]
if missing:
deps = 'dependency' if len(missing) == 1 else 'dependencies'
raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing)))
self.task.trigger_event(Event.START, self.task)
t0 = time.time()
status = None
if self.task.run == NotImplemented:
# External task
# TODO(erikbern): We should check for task completeness after non-external tasks too!
# This will resolve #814 and make things a lot more consistent
status = DONE if self.task.complete() else FAILED
else:
new_deps = self._run_get_new_deps()
status = DONE if not new_deps else SUSPENDED
if status == SUSPENDED:
logger.info(
'[pid %s] Worker %s new requirements %s',
os.getpid(), self.worker_id, self.task.task_id)
elif status == DONE:
self.task.trigger_event(
Event.PROCESSING_TIME, self.task, time.time() - t0)
expl = json.dumps(self.task.on_success())
logger.info('[pid %s] Worker %s done %s', os.getpid(),
self.worker_id, self.task.task_id)
self.task.trigger_event(Event.SUCCESS, self.task)
except KeyboardInterrupt:
raise
except BaseException as ex:
status = FAILED
logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self.worker_id, self.task)
self.task.trigger_event(Event.FAILURE, self.task, ex)
subject = "Luigi: %s FAILED" % self.task
raw_error_message = self.task.on_failure(ex)
notification_error_message = notifications.wrap_traceback(raw_error_message)
expl = json.dumps(raw_error_message)
formatted_error_message = notifications.format_task_error(subject, self.task,
formatted_exception=notification_error_message)
notifications.send_error_email(subject, formatted_error_message, self.task.owner_email)
finally:
self.result_queue.put(
(self.task.task_id, status, expl, missing, new_deps))
def _email_unexpected_error(self, task, formatted_traceback):
subject = "Luigi: Framework error while scheduling {task}. Host: {host}".format(
task=task, host=self.host)
headline = "Luigi framework error"
message = notifications.format_task_error(headline, task,
formatted_traceback)
notifications.send_error_email(subject, message, task.owner_email)
def _email_dependency_error(self, task, formatted_traceback):
subject = "Luigi: {task} failed scheduling. Host: {host}".format(
task=task, host=self.host)
headline = "Will not schedule task or any dependencies due to error in deps() method"
message = notifications.format_task_error(headline, task,
formatted_traceback)
notifications.send_error_email(subject, message, task.owner_email)
def _send_error_notification(self, raw_error_message):
subject = "Luigi: %s FAILED" % self.task
notification_error_message = notifications.wrap_traceback(
raw_error_message)
formatted_error_message = notifications.format_task_error(
subject, self.task, formatted_exception=notification_error_message)
notifications.send_error_email(subject, formatted_error_message,
self.task.owner_email)
def _email_error(self, task, formatted_traceback, subject, headline):
formatted_subject = subject.format(task=task, host=self.host)
message = notifications.format_task_error(headline, task, formatted_traceback)
notifications.send_error_email(formatted_subject, message, task.owner_email)
def _email_unexpected_error(self, task, formatted_traceback):
subject = "Luigi: Framework error while scheduling {task}. Host: {host}".format(task=task, host=self.host)
headline = "Luigi framework error"
message = notifications.format_task_error(headline, task, formatted_traceback)
notifications.send_error_email(subject, message, task.owner_email)
def _email_dependency_error(self, task, formatted_traceback):
subject = "Luigi: {task} failed scheduling. Host: {host}".format(task=task, host=self.host)
headline = "Will not schedule task or any dependencies due to error in deps() method"
message = notifications.format_task_error(headline, task, formatted_traceback)
notifications.send_error_email(subject, message, task.owner_email)
def _email_error(self, task, formatted_traceback, subject, headline):
formatted_subject = subject.format(task=task, host=self.host)
message = notifications.format_task_error(headline, task,
formatted_traceback)
notifications.send_error_email(formatted_subject, message,
task.owner_email)
