def banner(
self,
msg,
color=None,
verbose=False,
print_task_band=False,
task_run=None,
exc_info=None,
):
try:
b = TextBanner(msg, color)
if verbose or is_verbose():
verbosity = FormatterVerbosity.HIGH
else:
verbosity = FormatterVerbosity.NORMAL
builder = _TaskBannerBuilder(
task=self.task,
banner=b,
verbosity=verbosity,
print_task_band=print_task_band,
)
return builder.build_banner(task_run=task_run,
exc_info=exc_info).get_banner_str()
except Exception as ex:
log_exception(
"Failed to calculate banner for '%s'" % self.task_id,
ex,
non_critical=True,
)
return msg + (" ( task_id=%s)" % self.task_id)
def add_value(self):
console_value_preview_size = DescribeConfig.current(
).console_value_preview_size
value_str = (TextBanner.f_io(self.value) if self._param_kind() in [
"input", "output"
] else self.definition.to_str(self.value))
value_str = safe_string(value_str, console_value_preview_size)
# add preview
if isinstance(self.value, Target) and self.value.target_meta:
preview_value = safe_string(
self.value.target_meta.value_preview,
console_value_preview_size,
)
# we should add minimal preview
if len(preview_value) < 100:
value_str += " :='%s'" % preview_value
if value_str and "\n" in value_str:
# some simple heuristics around value
extra_padding = " " * len("\t".join(map(str, self.row)))
value_str = "".join("\n%s%s" % (extra_padding, l)
for l in value_str.split("\n"))
value_str = "-->\n" + value_str
self.row.append(value_str)
def banner(
self,
msg,
color=None,
verbose=False,
print_task_band=False,
task_run=None,
exc_info=None,
):
task_id = self.task.task_id
try:
# Saving banner for testability
self._banner = TextBanner(msg, color)
if verbose or is_verbose():
verbosity = FormatterVerbosity.HIGH
else:
verbosity = FormatterVerbosity.NORMAL
builder = _TaskBannerBuilder(
task=self.task,
banner=self._banner,
verbosity=verbosity,
print_task_band=print_task_band,
)
# different banners for tracking and orchestration
if TaskEssence.TRACKING.is_instance(self.task):
builder.build_tracking_banner(task_run=task_run,
exc_info=exc_info)
else:
if TaskEssence.CONFIG.is_instance(self.task):
builder.build_config_banner()
else:
builder.build_orchestration_banner(task_run=task_run,
exc_info=exc_info)
return self._banner.get_banner_str()
except Exception as ex:
log_exception("Failed to calculate banner for '%s'" % task_id,
ex,
non_critical=True)
return msg + (" ( task_id=%s)" % task_id)
def _qubole_banner(self, status):
b = TextBanner(
"Spark task {} is submitted to Qubole cluster labeled: {}".format(
self.task.task_id, self.qubole_config.cluster_label),
color="magenta",
)
b.column("Status", status)
b.column("URL", self.qubole_job_url)
logger.info(b.get_banner_str())
def pformat_current_config(config, as_table=False, sections=None):
# type: (DbndConfig, bool, Optional[Iterable[str]]) -> str
config_layer = config.config_layer
tb = TextBanner("Config {}".format(config_layer.name))
tb.column("LAYERS", config_layer.config_layer_path)
if as_table:
view_str = pformat_config_store_as_table(
config_store=config_layer.config, sections=sections)
else:
value_dict = config_layer.config.as_value_dict(sections=sections)
view_str = tb.f_struct(value_dict)
tb.column("CONFIG", view_str)
return tb.get_banner_str()
def pformat_all_layers(config, sections=None):
# type: (DbndConfig, Optional[Iterable[str]]) -> str
tb = TextBanner("Configs")
layers = config.config_layer.get_all_layers()
# start from the first one
for layer in reversed(layers):
layer_values = layer.layer_config.as_value_dict(sections=sections)
tb.column("LAYER {}".format(layer.config_layer_path),
tb.f_struct(layer_values))
return tb.get_banner_str()
class DoctorStatusReportBuilder(object):
def __init__(self, name):
self.name = name
msg = "Running %s Status Check:" % name
logger.info(msg)
self.text_banner = TextBanner(msg)
def log(self, key, value):
log_metric(key, value)
self.text_banner.column(key, value)
def add_sub_report(self, sub_report_str):
self.text_banner.write(sub_report_str)
self.text_banner.write("\n")
def get_status_str(self):
return self.text_banner.get_banner_str()
def add_value(self):
import dbnd # noqa: 401 import dbnd before DataFrameValueType to avoid cyclic imports
from dbnd._core.settings import DescribeConfig
from targets.values import DataFrameValueType
console_value_preview_size = (
DescribeConfig.from_databand_context().console_value_preview_size)
if self.value is None:
value_str = "@None"
else:
# We want to always use self.definition.to_str for panda's Dataframe value.
# otherwise, the value blows up the log, and is not readable.
value_str = (TextBanner.f_io(self.value)
if self._param_kind() in ["input", "output"]
and not self.definition.value_type_str
== DataFrameValueType.type_str else
self.definition.to_str(self.value))
value_str = safe_string(value_str, console_value_preview_size)
# add preview
if isinstance(self.value, Target) and self.value.target_meta:
preview_value = safe_string(self.value.target_meta.value_preview,
console_value_preview_size)
# we should add minimal preview
if len(preview_value) < 100:
value_str += " :='%s'" % preview_value
if value_str and "\n" in value_str:
# some simple heuristics around value
extra_padding = " " * len("\t".join(map(str, self.row)))
value_str = "".join("\n%s%s" % (extra_padding, l)
for l in value_str.split("\n"))
value_str = "-->\n" + value_str
self.row.append(value_str)
def print_table(header, instances_list):
banner = TextBanner(header)
banner.write(build_instances_table(instances_list))
logger.info(banner.get_banner_str())
def print_table(header, alerts_list):
banner = TextBanner(header)
banner.write(build_alerts_table(alerts_list))
logger.info(banner.get_banner_str())
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True):
b = TextBanner(msg, color)
run = self.run # type: DatabandRun
ctx = run.context
task_run_env = ctx.task_run_env # type: TaskRunEnvInfo
driver_task = run.driver_task_run.task
if show_tasks_info and driver_task.is_driver:
self._add_tasks_info(b)
b.column("TRACKER URL", run.run_url, skip_if_empty=True)
if run.root_run_info.root_run_uid != run.run_uid:
b.column(
"ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True
)
b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True)
if run.scheduled_run_info:
b.column_properties(
"SCHEDULED",
[
("scheduled_job", run.scheduled_run_info.scheduled_job_uid),
("scheduled_date", run.scheduled_run_info.scheduled_date),
("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id),
],
)
if show_run_info:
b.new_line()
b.column("USER", task_run_env.user)
b.column(
"LOG",
b.f_simple_dict(
[
("local", driver_task.local_driver_log),
("remote", driver_task.remote_driver_root),
]
),
)
b.column("USER CODE VERSION", task_run_env.user_code_version)
b.column("CMD", task_run_env.cmd_line)
b.column("RUN UID", "%s" % run.run_uid)
b.column("DB", self.context.settings.core.sql_conn_repr)
b.column("ENV", run.env.name)
b.column(
"RUN",
b.f_simple_dict(
[
("TASK_EXECUTOR", run.task_executor_type),
("PARALLEL", run.parallel),
("SUBMIT_DRIVER", run.submit_driver),
("SUBMIT_TASKS", run.submit_tasks),
],
skip_if_empty=True,
),
skip_if_empty=True,
)
if task_run_env.user_data:
b.column("USER DATA", task_run_env.user_data, skip_if_empty=True)
b.new_line()
failed_task_runs = [
task_run
for task_run in run.task_runs
if task_run.task_run_state in TaskRunState.direct_fail_states()
]
if failed_task_runs:
f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs)
b.column("FAILED", f_msg)
b.new_line()
return b.getvalue()
def run_banner(self,
msg,
color="white",
show_run_info=False,
show_tasks_info=False):
run = self.run # type: DatabandRun
if run.root_run_info.root_run_uid != run.run_uid:
msg += " -> sub-run"
b = TextBanner(msg, color)
ctx = run.context
task_run_env = ctx.task_run_env # type: TaskRunEnvInfo
if run.context.tracking_store.has_tracking_store("api",
channel_name="web"):
b.column("TRACKER URL", run.run_url, skip_if_empty=True)
b.column("TRACKERS", run.context.tracking_store.trackers_names)
if run.root_run_info.root_run_uid != run.run_uid:
b.column("ROOT TRACKER URL",
run.root_run_info.root_run_url,
skip_if_empty=True)
b.column("ROOT UID URL",
run.root_run_info.root_run_uid,
skip_if_empty=True)
if run.scheduled_run_info:
b.column_properties(
"SCHEDULED",
[
("scheduled_job",
run.scheduled_run_info.scheduled_job_uid),
("scheduled_date", run.scheduled_run_info.scheduled_date),
("dag_run_id",
run.scheduled_run_info.scheduled_job_dag_run_id),
],
)
if show_run_info:
b.new_line()
run_params = [
("user", task_run_env.user),
("run_uid", "%s" % run.run_uid),
("env", run.env.name),
("project", run.project_name) if run.project_name else None,
("user_code_version", task_run_env.user_code_version),
]
run_params = list(filter(None, run_params))
b.column("RUN", b.f_simple_dict(run_params))
b.column("CMD", task_run_env.cmd_line)
if task_run_env.user_data and task_run_env.user_data != "None":
b.column("USER DATA",
task_run_env.user_data,
skip_if_empty=True)
b.new_line()
if run.is_orchestration:
run_executor = run.run_executor
driver_task_run = run.driver_task_run
if show_run_info:
if driver_task_run and driver_task_run.log:
b.column(
"LOG",
b.f_simple_dict(
[
("local", driver_task_run.log.local_log_file),
("remote",
driver_task_run.log.remote_log_file),
],
skip_if_empty=True,
),
)
b.column(
"EXECUTE",
b.f_simple_dict(
[
("TASK_EXECUTOR", run_executor.task_executor_type),
("PARALLEL", run_executor.parallel),
("SUBMIT_DRIVER", run_executor.submit_driver),
("SUBMIT_TASKS", run_executor.submit_tasks),
],
skip_if_empty=True,
),
skip_if_empty=True,
)
if run_executor.run_executor_type == SystemTaskName.driver:
if run.root_task_run:
b.column("TASK_BAND", run.root_task_run.task.task_band)
if show_tasks_info:
self._add_tasks_info(b)
failed_task_runs = [
task_run for task_run in run.get_task_runs()
if task_run.task_run_state == TaskRunState.FAILED
]
if failed_task_runs:
f_msg = "\n".join(tr.task.task_id
for tr in failed_task_runs)
b.column("FAILED", f_msg)
b.new_line()
return b.getvalue()
def __init__(self, name):
self.name = name
msg = "Running %s Status Check:" % name
logger.info(msg)
self.text_banner = TextBanner(msg)
def run_banner(self,
msg,
color="white",
show_run_info=False,
show_tasks_info=True):
b = TextBanner(msg, color)
run = self.run # type: DatabandRun
ctx = run.context
task_run_env = ctx.task_run_env # type: TaskRunEnvInfo
driver_task = run.driver_task_run.task
orchestration_mode = run.source == UpdateSource.dbnd
b.column("TRACKER URL", run.run_url, skip_if_empty=True)
if show_tasks_info and orchestration_mode and driver_task.is_driver:
self._add_tasks_info(b)
if run.root_run_info.root_run_uid != run.run_uid:
b.column("ROOT TRACKER URL",
run.root_run_info.root_run_url,
skip_if_empty=True)
b.column("ROOT UID URL",
run.root_run_info.root_run_uid,
skip_if_empty=True)
if run.scheduled_run_info:
b.column_properties(
"SCHEDULED",
[
("scheduled_job",
run.scheduled_run_info.scheduled_job_uid),
("scheduled_date", run.scheduled_run_info.scheduled_date),
("dag_run_id",
run.scheduled_run_info.scheduled_job_dag_run_id),
],
)
if show_run_info:
b.new_line()
run_params = [
("user", task_run_env.user),
("run_uid", "%s" % run.run_uid),
("env", run.env.name),
]
b.column("RUN", b.f_simple_dict(run_params))
b.column(
"LOG",
b.f_simple_dict([
("local", driver_task.local_driver_log),
("remote", driver_task.remote_driver_root),
]),
)
b.column("USER CODE VERSION", task_run_env.user_code_version)
b.column("CMD", task_run_env.cmd_line)
if orchestration_mode:
if run.context.settings.core.is_db_store_enabled():
b.column("DB", self.context.settings.core.sql_conn_repr)
if run.task_executor_type.startswith("airflow"):
assert_airflow_enabled()
from dbnd_airflow.db_utils import airflow_sql_conn_repr
b.column("Airflow DB", airflow_sql_conn_repr())
b.column(
"EXECUTE",
b.f_simple_dict(
[
("TASK_EXECUTOR", run.task_executor_type),
("PARALLEL", run.parallel),
("SUBMIT_DRIVER", run.submit_driver),
("SUBMIT_TASKS", run.submit_tasks),
],
skip_if_empty=True,
),
skip_if_empty=True,
)
if task_run_env.user_data and task_run_env.user_data != "None":
b.column("USER DATA",
task_run_env.user_data,
skip_if_empty=True)
b.new_line()
failed_task_runs = [
task_run for task_run in run.task_runs
if task_run.task_run_state in TaskRunState.direct_fail_states()
]
if failed_task_runs:
f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs)
b.column("FAILED", f_msg)
b.new_line()
return b.getvalue()
def _get_batch_progresss_banner(self, batch_response):
"""
{
'id': 6,
'state': 'success',
'appId': 'application_1534487568579_0008',
'appInfo': {
'driverLogUrl': None,
'sparkUiUrl': 'http://ip-172-31-70-109.ec2.internal:20888/proxy/application_1534487568579_0008/'
},
'log': [
'\nYARN Diagnostics: '
]
}
:param batch_response:
:return:
"""
t = self.task
b = TextBanner("Spark Task %s is running at Livy:" % t.task_id,
color="yellow")
b.column("TASK", t.task_id)
b.column("JOB STATE", batch_response.get("state", None))
tracker_url = current_task_run().task_tracker_url
if tracker_url:
b.column("DATABAND LOG", tracker_url)
b.new_line()
b.column("LIVY ID", batch_response.get("id", None))
if "appId" in batch_response:
b.column("APP ID", batch_response["appId"])
app_info = batch_response["appInfo"]
b.column("DRIVER LOG", app_info["driverLogUrl"])
if "sparkUiUrl" in app_info:
spark_url = app_info["sparkUiUrl"]
b.column(
"SPARK",
colored(spark_url, on_color="on_blue", attrs=["bold"]))
b.new_section()
return b.getvalue()
def _get_step_banner(self, step):
"""
{
'id': 6,
'state': 'success',
}
"""
t = self.task
b = TextBanner("Spark Task %s is running at Emr:" % t.task_id,
color="yellow")
b.column("TASK", t.task_id)
b.column("EMR STEP STATE", step["Step"]["Status"]["State"])
tracker_url = current_task_run().task_tracker_url
if tracker_url:
b.column("DATABAND LOG", tracker_url)
b.new_line()
b.column("EMR STEP ID", step["Step"]["Id"])
b.new_section()
return b.getvalue()
def _handle_databricks_operator_execution(self, run_id, hook, task_id):
"""
Handles the Airflow + Databricks lifecycle logic for a Databricks operator
:param run_id: Databricks run_id
:param hook: Airflow databricks hook
:param task_id: Databand Task Id.
"""
b = TextBanner("Spark task %s is submitted to Databricks cluster:" %
task_id,
color="cyan")
url = hook.get_run_page_url(run_id)
self.task_run.set_external_resource_urls({"databricks url": url})
b.column("URL", url)
logger.info(b.get_banner_str())
while True:
b = TextBanner(
"Spark task %s is submitted to Databricks cluster:" % task_id,
color="cyan",
)
b.column("URL", url)
run_state = hook.get_run_state(run_id)
if run_state.is_terminal:
if run_state.is_successful:
b.column("Task completed successfully", task_id)
b.column("State:", run_state.life_cycle_state)
b.column("Message:", run_state.state_message)
break
else:
b.column("State", run_state.result_state)
b.column("Error Message:", run_state.state_message)
logger.info(b.get_banner_str())
raise failed_to_run_databricks_job(run_state.result_state,
run_state.state_message,
url)
else:
b.column("State:", run_state.life_cycle_state)
b.column("Message:", run_state.state_message)
time.sleep(
self.databricks_config.status_polling_interval_seconds)
logger.info(b.get_banner_str())
def _get_job_status_banner(self, description):
t = self.task
b = TextBanner(
"Training Job %s is running at SageMaker:" %
description.get("TrainingJobName", None),
color="yellow",
)
b.column("TASK", t.task_id)
b.column(
"JOB STATUS",
description.get("TrainingJobStatus", None) + " -> " +
description.get("SecondaryStatus", None),
)
b.column(
"JOB RESOURCES",
description["ResourceConfig"]["InstanceType"] + " x " +
str(description["ResourceConfig"]["InstanceCount"]),
)
tracker_url = current_task_run().task_tracker_url
if tracker_url:
b.column("DATABAND LOG", tracker_url)
b.column("JOB WEB UI",
self._job_url(description.get("TrainingJobName", None)))
b.column("CLOUDWATCH URL", self._logs_url())
b.new_line()
b.column("JOB ARN", description.get("TrainingJobArn", None))
b.new_section()
return b.getvalue()
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True):
b = TextBanner(msg, color)
run = self.run # type: DatabandRun
ctx = run.context
task_run_env = ctx.task_run_env # type: TaskRunEnvInfo
b.column("TRACKER URL", run.run_url, skip_if_empty=True)
b.column("TRACKERS", CoreConfig().tracker)
if run.is_orchestration:
run_executor = run.run_executor
driver_task_run = run.driver_task_run
if (
show_tasks_info
and run_executor.run_executor_type == SystemTaskName.driver
):
self._add_tasks_info(b)
if show_run_info and driver_task_run and driver_task_run.log:
b.column(
"LOG",
b.f_simple_dict(
[
("local", driver_task_run.log.local_log_file),
("remote", driver_task_run.log.remote_log_file),
],
skip_if_empty=True,
),
)
if run.root_run_info.root_run_uid != run.run_uid:
b.column(
"ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True
)
b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True)
if run.scheduled_run_info:
b.column_properties(
"SCHEDULED",
[
("scheduled_job", run.scheduled_run_info.scheduled_job_uid),
("scheduled_date", run.scheduled_run_info.scheduled_date),
("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id),
],
)
if show_run_info:
b.new_line()
run_params = [
("user", task_run_env.user),
("run_uid", "%s" % run.run_uid),
("env", run.env.name),
]
b.column("RUN", b.f_simple_dict(run_params))
b.column("USER CODE VERSION", task_run_env.user_code_version)
b.column("CMD", task_run_env.cmd_line)
if run.is_orchestration:
run_executor = run.run_executor
b.column(
"EXECUTE",
b.f_simple_dict(
[
("TASK_EXECUTOR", run_executor.task_executor_type),
("PARALLEL", run_executor.parallel),
("SUBMIT_DRIVER", run_executor.submit_driver),
("SUBMIT_TASKS", run_executor.submit_tasks),
],
skip_if_empty=True,
),
skip_if_empty=True,
)
if task_run_env.user_data and task_run_env.user_data != "None":
b.column("USER DATA", task_run_env.user_data, skip_if_empty=True)
b.new_line()
failed_task_runs = [
task_run
for task_run in run.task_runs
if task_run.task_run_state in TaskRunState.direct_fail_states()
]
if failed_task_runs:
f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs)
b.column("FAILED", f_msg)
if run.root_task_run and run.is_orchestration:
b.column("TASK_BAND", run.root_task_run.task.task_band)
b.new_line()
return b.getvalue()
