def _collect_errors(task_runs):
err = ""
upstream_failed = []
failed = []
for task_run in task_runs:
task_name = task_run.task.task_name
if task_run.task_run_state == TaskRunState.UPSTREAM_FAILED:
# we don't want to show upstream failed in the list
upstream_failed.append(task_name)
elif task_run.task_run_state in TaskRunState.direct_fail_states():
failed.append(task_name)
if upstream_failed:
err += "Task that didn't run because of failed dependency:\n\t{}\n".format(
"\n\t".join(upstream_failed))
if failed:
err += "Failed tasks are:\n\t{}".format("\n\t".join(failed))
return err
def get_error_banner(self):
# type: ()->str
err_banners = []
run = self.run
err_banners.append(self.run_banner_for_finished())
failed_task_runs = []
for task_run in run.task_runs:
if (
task_run.last_error
or task_run.task_run_state in TaskRunState.direct_fail_states()
):
failed_task_runs.append(task_run)
if len(failed_task_runs) > 1:
# clear out driver task, we don't want to print it twice
failed_task_runs = [
tr
for tr in failed_task_runs
if tr.task.task_name not in SystemTaskName.driver_and_submitter
]
for task_run in failed_task_runs:
if task_run.task_run_state == TaskRunState.CANCELLED:
msg_header = "Task has been terminated!"
else:
msg_header = "Task has failed!"
msg = task_run.task.ctrl.banner(
msg=msg_header, color="red", task_run=task_run
)
err_banners.append(msg)
err_banners.append(
self.run_banner(
"Your run has failed! See more info above.",
color="red",
show_run_info=False,
)
)
return u"\n".join(err_banners)
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True):
b = TextBanner(msg, color)
run = self.run # type: DatabandRun
ctx = run.context
task_run_env = ctx.task_run_env # type: TaskRunEnvInfo
driver_task = run.driver_task_run.task
if show_tasks_info and driver_task.is_driver:
self._add_tasks_info(b)
b.column("TRACKER URL", run.run_url, skip_if_empty=True)
if run.root_run_info.root_run_uid != run.run_uid:
b.column(
"ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True
)
b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True)
if run.scheduled_run_info:
b.column_properties(
"SCHEDULED",
[
("scheduled_job", run.scheduled_run_info.scheduled_job_uid),
("scheduled_date", run.scheduled_run_info.scheduled_date),
("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id),
],
)
if show_run_info:
b.new_line()
b.column("USER", task_run_env.user)
b.column(
"LOG",
b.f_simple_dict(
[
("local", driver_task.local_driver_log),
("remote", driver_task.remote_driver_root),
]
),
)
b.column("USER CODE VERSION", task_run_env.user_code_version)
b.column("CMD", task_run_env.cmd_line)
b.column("RUN UID", "%s" % run.run_uid)
b.column("DB", self.context.settings.core.sql_conn_repr)
b.column("ENV", run.env.name)
b.column(
"RUN",
b.f_simple_dict(
[
("TASK_EXECUTOR", run.task_executor_type),
("PARALLEL", run.parallel),
("SUBMIT_DRIVER", run.submit_driver),
("SUBMIT_TASKS", run.submit_tasks),
],
skip_if_empty=True,
),
skip_if_empty=True,
)
if task_run_env.user_data:
b.column("USER DATA", task_run_env.user_data, skip_if_empty=True)
b.new_line()
failed_task_runs = [
task_run
for task_run in run.task_runs
if task_run.task_run_state in TaskRunState.direct_fail_states()
]
if failed_task_runs:
f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs)
b.column("FAILED", f_msg)
b.new_line()
return b.getvalue()
def run_banner(self,
msg,
color="white",
show_run_info=False,
show_tasks_info=True):
b = TextBanner(msg, color)
run = self.run # type: DatabandRun
ctx = run.context
task_run_env = ctx.task_run_env # type: TaskRunEnvInfo
driver_task = run.driver_task_run.task
orchestration_mode = run.source == UpdateSource.dbnd
b.column("TRACKER URL", run.run_url, skip_if_empty=True)
if show_tasks_info and orchestration_mode and driver_task.is_driver:
self._add_tasks_info(b)
if run.root_run_info.root_run_uid != run.run_uid:
b.column("ROOT TRACKER URL",
run.root_run_info.root_run_url,
skip_if_empty=True)
b.column("ROOT UID URL",
run.root_run_info.root_run_uid,
skip_if_empty=True)
if run.scheduled_run_info:
b.column_properties(
"SCHEDULED",
[
("scheduled_job",
run.scheduled_run_info.scheduled_job_uid),
("scheduled_date", run.scheduled_run_info.scheduled_date),
("dag_run_id",
run.scheduled_run_info.scheduled_job_dag_run_id),
],
)
if show_run_info:
b.new_line()
run_params = [
("user", task_run_env.user),
("run_uid", "%s" % run.run_uid),
("env", run.env.name),
]
b.column("RUN", b.f_simple_dict(run_params))
b.column(
"LOG",
b.f_simple_dict([
("local", driver_task.local_driver_log),
("remote", driver_task.remote_driver_root),
]),
)
b.column("USER CODE VERSION", task_run_env.user_code_version)
b.column("CMD", task_run_env.cmd_line)
if orchestration_mode:
if run.context.settings.core.is_db_store_enabled():
b.column("DB", self.context.settings.core.sql_conn_repr)
if run.task_executor_type.startswith("airflow"):
assert_airflow_enabled()
from dbnd_airflow.db_utils import airflow_sql_conn_repr
b.column("Airflow DB", airflow_sql_conn_repr())
b.column(
"EXECUTE",
b.f_simple_dict(
[
("TASK_EXECUTOR", run.task_executor_type),
("PARALLEL", run.parallel),
("SUBMIT_DRIVER", run.submit_driver),
("SUBMIT_TASKS", run.submit_tasks),
],
skip_if_empty=True,
),
skip_if_empty=True,
)
if task_run_env.user_data and task_run_env.user_data != "None":
b.column("USER DATA",
task_run_env.user_data,
skip_if_empty=True)
b.new_line()
failed_task_runs = [
task_run for task_run in run.task_runs
if task_run.task_run_state in TaskRunState.direct_fail_states()
]
if failed_task_runs:
f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs)
b.column("FAILED", f_msg)
b.new_line()
return b.getvalue()
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True):
b = TextBanner(msg, color)
run = self.run # type: DatabandRun
ctx = run.context
task_run_env = ctx.task_run_env # type: TaskRunEnvInfo
b.column("TRACKER URL", run.run_url, skip_if_empty=True)
b.column("TRACKERS", CoreConfig().tracker)
if run.is_orchestration:
run_executor = run.run_executor
driver_task_run = run.driver_task_run
if (
show_tasks_info
and run_executor.run_executor_type == SystemTaskName.driver
):
self._add_tasks_info(b)
if show_run_info and driver_task_run and driver_task_run.log:
b.column(
"LOG",
b.f_simple_dict(
[
("local", driver_task_run.log.local_log_file),
("remote", driver_task_run.log.remote_log_file),
],
skip_if_empty=True,
),
)
if run.root_run_info.root_run_uid != run.run_uid:
b.column(
"ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True
)
b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True)
if run.scheduled_run_info:
b.column_properties(
"SCHEDULED",
[
("scheduled_job", run.scheduled_run_info.scheduled_job_uid),
("scheduled_date", run.scheduled_run_info.scheduled_date),
("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id),
],
)
if show_run_info:
b.new_line()
run_params = [
("user", task_run_env.user),
("run_uid", "%s" % run.run_uid),
("env", run.env.name),
]
b.column("RUN", b.f_simple_dict(run_params))
b.column("USER CODE VERSION", task_run_env.user_code_version)
b.column("CMD", task_run_env.cmd_line)
if run.is_orchestration:
run_executor = run.run_executor
b.column(
"EXECUTE",
b.f_simple_dict(
[
("TASK_EXECUTOR", run_executor.task_executor_type),
("PARALLEL", run_executor.parallel),
("SUBMIT_DRIVER", run_executor.submit_driver),
("SUBMIT_TASKS", run_executor.submit_tasks),
],
skip_if_empty=True,
),
skip_if_empty=True,
)
if task_run_env.user_data and task_run_env.user_data != "None":
b.column("USER DATA", task_run_env.user_data, skip_if_empty=True)
b.new_line()
failed_task_runs = [
task_run
for task_run in run.task_runs
if task_run.task_run_state in TaskRunState.direct_fail_states()
]
if failed_task_runs:
f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs)
b.column("FAILED", f_msg)
if run.root_task_run and run.is_orchestration:
b.column("TASK_BAND", run.root_task_run.task.task_band)
b.new_line()
return b.getvalue()