def _get_job_status_banner(self, description):
t = self.task
b = TextBanner(
"Training Job %s is running at SageMaker:" %
description.get("TrainingJobName", None),
color="yellow",
)
b.column("TASK", t.task_id)
b.column(
"JOB STATUS",
description.get("TrainingJobStatus", None) + " -> " +
description.get("SecondaryStatus", None),
)
b.column(
"JOB RESOURCES",
description["ResourceConfig"]["InstanceType"] + " x " +
str(description["ResourceConfig"]["InstanceCount"]),
)
tracker_url = current_task_run().task_tracker_url
if tracker_url:
b.column("DATABAND LOG", tracker_url)
b.column("JOB WEB UI",
self._job_url(description.get("TrainingJobName", None)))
b.column("CLOUDWATCH URL", self._logs_url())
b.new_line()
b.column("JOB ARN", description.get("TrainingJobArn", None))
b.new_section()
return b.getvalue()
def _get_batch_progresss_banner(self, batch_response):
"""
{
'id': 6,
'state': 'success',
'appId': 'application_1534487568579_0008',
'appInfo': {
'driverLogUrl': None,
'sparkUiUrl': 'http://ip-172-31-70-109.ec2.internal:20888/proxy/application_1534487568579_0008/'
},
'log': [
'\nYARN Diagnostics: '
]
}
:param batch_response:
:return:
"""
t = self.task
b = TextBanner("Spark Task %s is running at Livy:" % t.task_id,
color="yellow")
b.column("TASK", t.task_id)
b.column("JOB STATE", batch_response.get("state", None))
tracker_url = current_task_run().task_tracker_url
if tracker_url:
b.column("DATABAND LOG", tracker_url)
b.new_line()
b.column("LIVY ID", batch_response.get("id", None))
if "appId" in batch_response:
b.column("APP ID", batch_response["appId"])
app_info = batch_response["appInfo"]
b.column("DRIVER LOG", app_info["driverLogUrl"])
if "sparkUiUrl" in app_info:
spark_url = app_info["sparkUiUrl"]
b.column(
"SPARK",
colored(spark_url, on_color="on_blue", attrs=["bold"]))
b.new_section()
return b.getvalue()
def _get_step_banner(self, step):
"""
{
'id': 6,
'state': 'success',
}
"""
t = self.task
b = TextBanner("Spark Task %s is running at Emr:" % t.task_id, color="yellow")
b.column("TASK", t.task_id)
b.column("EMR STEP STATE", step["Step"]["Status"]["State"])
tracker_url = current_task_run().task_tracker_url
if tracker_url:
b.column("DATABAND LOG", tracker_url)
b.new_line()
b.column("EMR STEP ID", step["Step"]["Id"])
b.new_section()
return b.getvalue()
def run_banner(self,
msg,
color="white",
show_run_info=False,
show_tasks_info=False):
run = self.run # type: DatabandRun
if run.root_run_info.root_run_uid != run.run_uid:
msg += " -> sub-run"
b = TextBanner(msg, color)
ctx = run.context
task_run_env = ctx.task_run_env # type: TaskRunEnvInfo
if run.context.tracking_store.has_tracking_store("api",
channel_name="web"):
b.column("TRACKER URL", run.run_url, skip_if_empty=True)
b.column("TRACKERS", run.context.tracking_store.trackers_names)
if run.root_run_info.root_run_uid != run.run_uid:
b.column("ROOT TRACKER URL",
run.root_run_info.root_run_url,
skip_if_empty=True)
b.column("ROOT UID URL",
run.root_run_info.root_run_uid,
skip_if_empty=True)
if run.scheduled_run_info:
b.column_properties(
"SCHEDULED",
[
("scheduled_job",
run.scheduled_run_info.scheduled_job_uid),
("scheduled_date", run.scheduled_run_info.scheduled_date),
("dag_run_id",
run.scheduled_run_info.scheduled_job_dag_run_id),
],
)
if show_run_info:
b.new_line()
run_params = [
("user", task_run_env.user),
("run_uid", "%s" % run.run_uid),
("env", run.env.name),
("project", run.project_name) if run.project_name else None,
("user_code_version", task_run_env.user_code_version),
]
run_params = list(filter(None, run_params))
b.column("RUN", b.f_simple_dict(run_params))
b.column("CMD", task_run_env.cmd_line)
if task_run_env.user_data and task_run_env.user_data != "None":
b.column("USER DATA",
task_run_env.user_data,
skip_if_empty=True)
b.new_line()
if run.is_orchestration:
run_executor = run.run_executor
driver_task_run = run.driver_task_run
if show_run_info:
if driver_task_run and driver_task_run.log:
b.column(
"LOG",
b.f_simple_dict(
[
("local", driver_task_run.log.local_log_file),
("remote",
driver_task_run.log.remote_log_file),
],
skip_if_empty=True,
),
)
b.column(
"EXECUTE",
b.f_simple_dict(
[
("TASK_EXECUTOR", run_executor.task_executor_type),
("PARALLEL", run_executor.parallel),
("SUBMIT_DRIVER", run_executor.submit_driver),
("SUBMIT_TASKS", run_executor.submit_tasks),
],
skip_if_empty=True,
),
skip_if_empty=True,
)
if run_executor.run_executor_type == SystemTaskName.driver:
if run.root_task_run:
b.column("TASK_BAND", run.root_task_run.task.task_band)
if show_tasks_info:
self._add_tasks_info(b)
failed_task_runs = [
task_run for task_run in run.get_task_runs()
if task_run.task_run_state == TaskRunState.FAILED
]
if failed_task_runs:
f_msg = "\n".join(tr.task.task_id
for tr in failed_task_runs)
b.column("FAILED", f_msg)
b.new_line()
return b.getvalue()
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True):
b = TextBanner(msg, color)
run = self.run # type: DatabandRun
ctx = run.context
task_run_env = ctx.task_run_env # type: TaskRunEnvInfo
driver_task = run.driver_task_run.task
if show_tasks_info and driver_task.is_driver:
self._add_tasks_info(b)
b.column("TRACKER URL", run.run_url, skip_if_empty=True)
if run.root_run_info.root_run_uid != run.run_uid:
b.column(
"ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True
)
b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True)
if run.scheduled_run_info:
b.column_properties(
"SCHEDULED",
[
("scheduled_job", run.scheduled_run_info.scheduled_job_uid),
("scheduled_date", run.scheduled_run_info.scheduled_date),
("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id),
],
)
if show_run_info:
b.new_line()
b.column("USER", task_run_env.user)
b.column(
"LOG",
b.f_simple_dict(
[
("local", driver_task.local_driver_log),
("remote", driver_task.remote_driver_root),
]
),
)
b.column("USER CODE VERSION", task_run_env.user_code_version)
b.column("CMD", task_run_env.cmd_line)
b.column("RUN UID", "%s" % run.run_uid)
b.column("DB", self.context.settings.core.sql_conn_repr)
b.column("ENV", run.env.name)
b.column(
"RUN",
b.f_simple_dict(
[
("TASK_EXECUTOR", run.task_executor_type),
("PARALLEL", run.parallel),
("SUBMIT_DRIVER", run.submit_driver),
("SUBMIT_TASKS", run.submit_tasks),
],
skip_if_empty=True,
),
skip_if_empty=True,
)
if task_run_env.user_data:
b.column("USER DATA", task_run_env.user_data, skip_if_empty=True)
b.new_line()
failed_task_runs = [
task_run
for task_run in run.task_runs
if task_run.task_run_state in TaskRunState.direct_fail_states()
]
if failed_task_runs:
f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs)
b.column("FAILED", f_msg)
b.new_line()
return b.getvalue()
def run_banner(self,
msg,
color="white",
show_run_info=False,
show_tasks_info=True):
b = TextBanner(msg, color)
run = self.run # type: DatabandRun
ctx = run.context
task_run_env = ctx.task_run_env # type: TaskRunEnvInfo
driver_task = run.driver_task_run.task
orchestration_mode = run.source == UpdateSource.dbnd
b.column("TRACKER URL", run.run_url, skip_if_empty=True)
if show_tasks_info and orchestration_mode and driver_task.is_driver:
self._add_tasks_info(b)
if run.root_run_info.root_run_uid != run.run_uid:
b.column("ROOT TRACKER URL",
run.root_run_info.root_run_url,
skip_if_empty=True)
b.column("ROOT UID URL",
run.root_run_info.root_run_uid,
skip_if_empty=True)
if run.scheduled_run_info:
b.column_properties(
"SCHEDULED",
[
("scheduled_job",
run.scheduled_run_info.scheduled_job_uid),
("scheduled_date", run.scheduled_run_info.scheduled_date),
("dag_run_id",
run.scheduled_run_info.scheduled_job_dag_run_id),
],
)
if show_run_info:
b.new_line()
run_params = [
("user", task_run_env.user),
("run_uid", "%s" % run.run_uid),
("env", run.env.name),
]
b.column("RUN", b.f_simple_dict(run_params))
b.column(
"LOG",
b.f_simple_dict([
("local", driver_task.local_driver_log),
("remote", driver_task.remote_driver_root),
]),
)
b.column("USER CODE VERSION", task_run_env.user_code_version)
b.column("CMD", task_run_env.cmd_line)
if orchestration_mode:
if run.context.settings.core.is_db_store_enabled():
b.column("DB", self.context.settings.core.sql_conn_repr)
if run.task_executor_type.startswith("airflow"):
assert_airflow_enabled()
from dbnd_airflow.db_utils import airflow_sql_conn_repr
b.column("Airflow DB", airflow_sql_conn_repr())
b.column(
"EXECUTE",
b.f_simple_dict(
[
("TASK_EXECUTOR", run.task_executor_type),
("PARALLEL", run.parallel),
("SUBMIT_DRIVER", run.submit_driver),
("SUBMIT_TASKS", run.submit_tasks),
],
skip_if_empty=True,
),
skip_if_empty=True,
)
if task_run_env.user_data and task_run_env.user_data != "None":
b.column("USER DATA",
task_run_env.user_data,
skip_if_empty=True)
b.new_line()
failed_task_runs = [
task_run for task_run in run.task_runs
if task_run.task_run_state in TaskRunState.direct_fail_states()
]
if failed_task_runs:
f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs)
b.column("FAILED", f_msg)
b.new_line()
return b.getvalue()
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True):
b = TextBanner(msg, color)
run = self.run # type: DatabandRun
ctx = run.context
task_run_env = ctx.task_run_env # type: TaskRunEnvInfo
b.column("TRACKER URL", run.run_url, skip_if_empty=True)
b.column("TRACKERS", CoreConfig().tracker)
if run.is_orchestration:
run_executor = run.run_executor
driver_task_run = run.driver_task_run
if (
show_tasks_info
and run_executor.run_executor_type == SystemTaskName.driver
):
self._add_tasks_info(b)
if show_run_info and driver_task_run and driver_task_run.log:
b.column(
"LOG",
b.f_simple_dict(
[
("local", driver_task_run.log.local_log_file),
("remote", driver_task_run.log.remote_log_file),
],
skip_if_empty=True,
),
)
if run.root_run_info.root_run_uid != run.run_uid:
b.column(
"ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True
)
b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True)
if run.scheduled_run_info:
b.column_properties(
"SCHEDULED",
[
("scheduled_job", run.scheduled_run_info.scheduled_job_uid),
("scheduled_date", run.scheduled_run_info.scheduled_date),
("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id),
],
)
if show_run_info:
b.new_line()
run_params = [
("user", task_run_env.user),
("run_uid", "%s" % run.run_uid),
("env", run.env.name),
]
b.column("RUN", b.f_simple_dict(run_params))
b.column("USER CODE VERSION", task_run_env.user_code_version)
b.column("CMD", task_run_env.cmd_line)
if run.is_orchestration:
run_executor = run.run_executor
b.column(
"EXECUTE",
b.f_simple_dict(
[
("TASK_EXECUTOR", run_executor.task_executor_type),
("PARALLEL", run_executor.parallel),
("SUBMIT_DRIVER", run_executor.submit_driver),
("SUBMIT_TASKS", run_executor.submit_tasks),
],
skip_if_empty=True,
),
skip_if_empty=True,
)
if task_run_env.user_data and task_run_env.user_data != "None":
b.column("USER DATA", task_run_env.user_data, skip_if_empty=True)
b.new_line()
failed_task_runs = [
task_run
for task_run in run.task_runs
if task_run.task_run_state in TaskRunState.direct_fail_states()
]
if failed_task_runs:
f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs)
b.column("FAILED", f_msg)
if run.root_task_run and run.is_orchestration:
b.column("TASK_BAND", run.root_task_run.task.task_band)
b.new_line()
return b.getvalue()
