def banner( self, msg, color=None, verbose=False, print_task_band=False, task_run=None, exc_info=None, ): try: b = TextBanner(msg, color) if verbose or is_verbose(): verbosity = FormatterVerbosity.HIGH else: verbosity = FormatterVerbosity.NORMAL builder = _TaskBannerBuilder( task=self.task, banner=b, verbosity=verbosity, print_task_band=print_task_band, ) return builder.build_banner(task_run=task_run, exc_info=exc_info).get_banner_str() except Exception as ex: log_exception( "Failed to calculate banner for '%s'" % self.task_id, ex, non_critical=True, ) return msg + (" ( task_id=%s)" % self.task_id)
def add_value(self): console_value_preview_size = DescribeConfig.current( ).console_value_preview_size value_str = (TextBanner.f_io(self.value) if self._param_kind() in [ "input", "output" ] else self.definition.to_str(self.value)) value_str = safe_string(value_str, console_value_preview_size) # add preview if isinstance(self.value, Target) and self.value.target_meta: preview_value = safe_string( self.value.target_meta.value_preview, console_value_preview_size, ) # we should add minimal preview if len(preview_value) < 100: value_str += " :='%s'" % preview_value if value_str and "\n" in value_str: # some simple heuristics around value extra_padding = " " * len("\t".join(map(str, self.row))) value_str = "".join("\n%s%s" % (extra_padding, l) for l in value_str.split("\n")) value_str = "-->\n" + value_str self.row.append(value_str)
def banner( self, msg, color=None, verbose=False, print_task_band=False, task_run=None, exc_info=None, ): task_id = self.task.task_id try: # Saving banner for testability self._banner = TextBanner(msg, color) if verbose or is_verbose(): verbosity = FormatterVerbosity.HIGH else: verbosity = FormatterVerbosity.NORMAL builder = _TaskBannerBuilder( task=self.task, banner=self._banner, verbosity=verbosity, print_task_band=print_task_band, ) # different banners for tracking and orchestration if TaskEssence.TRACKING.is_instance(self.task): builder.build_tracking_banner(task_run=task_run, exc_info=exc_info) else: if TaskEssence.CONFIG.is_instance(self.task): builder.build_config_banner() else: builder.build_orchestration_banner(task_run=task_run, exc_info=exc_info) return self._banner.get_banner_str() except Exception as ex: log_exception("Failed to calculate banner for '%s'" % task_id, ex, non_critical=True) return msg + (" ( task_id=%s)" % task_id)
def _qubole_banner(self, status): b = TextBanner( "Spark task {} is submitted to Qubole cluster labeled: {}".format( self.task.task_id, self.qubole_config.cluster_label), color="magenta", ) b.column("Status", status) b.column("URL", self.qubole_job_url) logger.info(b.get_banner_str())
def pformat_current_config(config, as_table=False, sections=None): # type: (DbndConfig, bool, Optional[Iterable[str]]) -> str config_layer = config.config_layer tb = TextBanner("Config {}".format(config_layer.name)) tb.column("LAYERS", config_layer.config_layer_path) if as_table: view_str = pformat_config_store_as_table( config_store=config_layer.config, sections=sections) else: value_dict = config_layer.config.as_value_dict(sections=sections) view_str = tb.f_struct(value_dict) tb.column("CONFIG", view_str) return tb.get_banner_str()
def pformat_all_layers(config, sections=None): # type: (DbndConfig, Optional[Iterable[str]]) -> str tb = TextBanner("Configs") layers = config.config_layer.get_all_layers() # start from the first one for layer in reversed(layers): layer_values = layer.layer_config.as_value_dict(sections=sections) tb.column("LAYER {}".format(layer.config_layer_path), tb.f_struct(layer_values)) return tb.get_banner_str()
class DoctorStatusReportBuilder(object): def __init__(self, name): self.name = name msg = "Running %s Status Check:" % name logger.info(msg) self.text_banner = TextBanner(msg) def log(self, key, value): log_metric(key, value) self.text_banner.column(key, value) def add_sub_report(self, sub_report_str): self.text_banner.write(sub_report_str) self.text_banner.write("\n") def get_status_str(self): return self.text_banner.get_banner_str()
def add_value(self): import dbnd # noqa: 401 import dbnd before DataFrameValueType to avoid cyclic imports from dbnd._core.settings import DescribeConfig from targets.values import DataFrameValueType console_value_preview_size = ( DescribeConfig.from_databand_context().console_value_preview_size) if self.value is None: value_str = "@None" else: # We want to always use self.definition.to_str for panda's Dataframe value. # otherwise, the value blows up the log, and is not readable. value_str = (TextBanner.f_io(self.value) if self._param_kind() in ["input", "output"] and not self.definition.value_type_str == DataFrameValueType.type_str else self.definition.to_str(self.value)) value_str = safe_string(value_str, console_value_preview_size) # add preview if isinstance(self.value, Target) and self.value.target_meta: preview_value = safe_string(self.value.target_meta.value_preview, console_value_preview_size) # we should add minimal preview if len(preview_value) < 100: value_str += " :='%s'" % preview_value if value_str and "\n" in value_str: # some simple heuristics around value extra_padding = " " * len("\t".join(map(str, self.row))) value_str = "".join("\n%s%s" % (extra_padding, l) for l in value_str.split("\n")) value_str = "-->\n" + value_str self.row.append(value_str)
def print_table(header, instances_list): banner = TextBanner(header) banner.write(build_instances_table(instances_list)) logger.info(banner.get_banner_str())
def print_table(header, alerts_list): banner = TextBanner(header) banner.write(build_alerts_table(alerts_list)) logger.info(banner.get_banner_str())
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True): b = TextBanner(msg, color) run = self.run # type: DatabandRun ctx = run.context task_run_env = ctx.task_run_env # type: TaskRunEnvInfo driver_task = run.driver_task_run.task if show_tasks_info and driver_task.is_driver: self._add_tasks_info(b) b.column("TRACKER URL", run.run_url, skip_if_empty=True) if run.root_run_info.root_run_uid != run.run_uid: b.column( "ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True ) b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True) if run.scheduled_run_info: b.column_properties( "SCHEDULED", [ ("scheduled_job", run.scheduled_run_info.scheduled_job_uid), ("scheduled_date", run.scheduled_run_info.scheduled_date), ("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id), ], ) if show_run_info: b.new_line() b.column("USER", task_run_env.user) b.column( "LOG", b.f_simple_dict( [ ("local", driver_task.local_driver_log), ("remote", driver_task.remote_driver_root), ] ), ) b.column("USER CODE VERSION", task_run_env.user_code_version) b.column("CMD", task_run_env.cmd_line) b.column("RUN UID", "%s" % run.run_uid) b.column("DB", self.context.settings.core.sql_conn_repr) b.column("ENV", run.env.name) b.column( "RUN", b.f_simple_dict( [ ("TASK_EXECUTOR", run.task_executor_type), ("PARALLEL", run.parallel), ("SUBMIT_DRIVER", run.submit_driver), ("SUBMIT_TASKS", run.submit_tasks), ], skip_if_empty=True, ), skip_if_empty=True, ) if task_run_env.user_data: b.column("USER DATA", task_run_env.user_data, skip_if_empty=True) b.new_line() failed_task_runs = [ task_run for task_run in run.task_runs if task_run.task_run_state in TaskRunState.direct_fail_states() ] if failed_task_runs: f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs) b.column("FAILED", f_msg) b.new_line() return b.getvalue()
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=False): run = self.run # type: DatabandRun if run.root_run_info.root_run_uid != run.run_uid: msg += " -> sub-run" b = TextBanner(msg, color) ctx = run.context task_run_env = ctx.task_run_env # type: TaskRunEnvInfo if run.context.tracking_store.has_tracking_store("api", channel_name="web"): b.column("TRACKER URL", run.run_url, skip_if_empty=True) b.column("TRACKERS", run.context.tracking_store.trackers_names) if run.root_run_info.root_run_uid != run.run_uid: b.column("ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True) b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True) if run.scheduled_run_info: b.column_properties( "SCHEDULED", [ ("scheduled_job", run.scheduled_run_info.scheduled_job_uid), ("scheduled_date", run.scheduled_run_info.scheduled_date), ("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id), ], ) if show_run_info: b.new_line() run_params = [ ("user", task_run_env.user), ("run_uid", "%s" % run.run_uid), ("env", run.env.name), ("project", run.project_name) if run.project_name else None, ("user_code_version", task_run_env.user_code_version), ] run_params = list(filter(None, run_params)) b.column("RUN", b.f_simple_dict(run_params)) b.column("CMD", task_run_env.cmd_line) if task_run_env.user_data and task_run_env.user_data != "None": b.column("USER DATA", task_run_env.user_data, skip_if_empty=True) b.new_line() if run.is_orchestration: run_executor = run.run_executor driver_task_run = run.driver_task_run if show_run_info: if driver_task_run and driver_task_run.log: b.column( "LOG", b.f_simple_dict( [ ("local", driver_task_run.log.local_log_file), ("remote", driver_task_run.log.remote_log_file), ], skip_if_empty=True, ), ) b.column( "EXECUTE", b.f_simple_dict( [ ("TASK_EXECUTOR", run_executor.task_executor_type), ("PARALLEL", run_executor.parallel), ("SUBMIT_DRIVER", run_executor.submit_driver), ("SUBMIT_TASKS", run_executor.submit_tasks), ], skip_if_empty=True, ), skip_if_empty=True, ) if run_executor.run_executor_type == SystemTaskName.driver: if run.root_task_run: b.column("TASK_BAND", run.root_task_run.task.task_band) if show_tasks_info: self._add_tasks_info(b) failed_task_runs = [ task_run for task_run in run.get_task_runs() if task_run.task_run_state == TaskRunState.FAILED ] if failed_task_runs: f_msg = "\n".join(tr.task.task_id for tr in failed_task_runs) b.column("FAILED", f_msg) b.new_line() return b.getvalue()
def __init__(self, name): self.name = name msg = "Running %s Status Check:" % name logger.info(msg) self.text_banner = TextBanner(msg)
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True): b = TextBanner(msg, color) run = self.run # type: DatabandRun ctx = run.context task_run_env = ctx.task_run_env # type: TaskRunEnvInfo driver_task = run.driver_task_run.task orchestration_mode = run.source == UpdateSource.dbnd b.column("TRACKER URL", run.run_url, skip_if_empty=True) if show_tasks_info and orchestration_mode and driver_task.is_driver: self._add_tasks_info(b) if run.root_run_info.root_run_uid != run.run_uid: b.column("ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True) b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True) if run.scheduled_run_info: b.column_properties( "SCHEDULED", [ ("scheduled_job", run.scheduled_run_info.scheduled_job_uid), ("scheduled_date", run.scheduled_run_info.scheduled_date), ("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id), ], ) if show_run_info: b.new_line() run_params = [ ("user", task_run_env.user), ("run_uid", "%s" % run.run_uid), ("env", run.env.name), ] b.column("RUN", b.f_simple_dict(run_params)) b.column( "LOG", b.f_simple_dict([ ("local", driver_task.local_driver_log), ("remote", driver_task.remote_driver_root), ]), ) b.column("USER CODE VERSION", task_run_env.user_code_version) b.column("CMD", task_run_env.cmd_line) if orchestration_mode: if run.context.settings.core.is_db_store_enabled(): b.column("DB", self.context.settings.core.sql_conn_repr) if run.task_executor_type.startswith("airflow"): assert_airflow_enabled() from dbnd_airflow.db_utils import airflow_sql_conn_repr b.column("Airflow DB", airflow_sql_conn_repr()) b.column( "EXECUTE", b.f_simple_dict( [ ("TASK_EXECUTOR", run.task_executor_type), ("PARALLEL", run.parallel), ("SUBMIT_DRIVER", run.submit_driver), ("SUBMIT_TASKS", run.submit_tasks), ], skip_if_empty=True, ), skip_if_empty=True, ) if task_run_env.user_data and task_run_env.user_data != "None": b.column("USER DATA", task_run_env.user_data, skip_if_empty=True) b.new_line() failed_task_runs = [ task_run for task_run in run.task_runs if task_run.task_run_state in TaskRunState.direct_fail_states() ] if failed_task_runs: f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs) b.column("FAILED", f_msg) b.new_line() return b.getvalue()
def _get_batch_progresss_banner(self, batch_response): """ { 'id': 6, 'state': 'success', 'appId': 'application_1534487568579_0008', 'appInfo': { 'driverLogUrl': None, 'sparkUiUrl': 'http://ip-172-31-70-109.ec2.internal:20888/proxy/application_1534487568579_0008/' }, 'log': [ '\nYARN Diagnostics: ' ] } :param batch_response: :return: """ t = self.task b = TextBanner("Spark Task %s is running at Livy:" % t.task_id, color="yellow") b.column("TASK", t.task_id) b.column("JOB STATE", batch_response.get("state", None)) tracker_url = current_task_run().task_tracker_url if tracker_url: b.column("DATABAND LOG", tracker_url) b.new_line() b.column("LIVY ID", batch_response.get("id", None)) if "appId" in batch_response: b.column("APP ID", batch_response["appId"]) app_info = batch_response["appInfo"] b.column("DRIVER LOG", app_info["driverLogUrl"]) if "sparkUiUrl" in app_info: spark_url = app_info["sparkUiUrl"] b.column( "SPARK", colored(spark_url, on_color="on_blue", attrs=["bold"])) b.new_section() return b.getvalue()
def _get_step_banner(self, step): """ { 'id': 6, 'state': 'success', } """ t = self.task b = TextBanner("Spark Task %s is running at Emr:" % t.task_id, color="yellow") b.column("TASK", t.task_id) b.column("EMR STEP STATE", step["Step"]["Status"]["State"]) tracker_url = current_task_run().task_tracker_url if tracker_url: b.column("DATABAND LOG", tracker_url) b.new_line() b.column("EMR STEP ID", step["Step"]["Id"]) b.new_section() return b.getvalue()
def _handle_databricks_operator_execution(self, run_id, hook, task_id): """ Handles the Airflow + Databricks lifecycle logic for a Databricks operator :param run_id: Databricks run_id :param hook: Airflow databricks hook :param task_id: Databand Task Id. """ b = TextBanner("Spark task %s is submitted to Databricks cluster:" % task_id, color="cyan") url = hook.get_run_page_url(run_id) self.task_run.set_external_resource_urls({"databricks url": url}) b.column("URL", url) logger.info(b.get_banner_str()) while True: b = TextBanner( "Spark task %s is submitted to Databricks cluster:" % task_id, color="cyan", ) b.column("URL", url) run_state = hook.get_run_state(run_id) if run_state.is_terminal: if run_state.is_successful: b.column("Task completed successfully", task_id) b.column("State:", run_state.life_cycle_state) b.column("Message:", run_state.state_message) break else: b.column("State", run_state.result_state) b.column("Error Message:", run_state.state_message) logger.info(b.get_banner_str()) raise failed_to_run_databricks_job(run_state.result_state, run_state.state_message, url) else: b.column("State:", run_state.life_cycle_state) b.column("Message:", run_state.state_message) time.sleep( self.databricks_config.status_polling_interval_seconds) logger.info(b.get_banner_str())
def _get_job_status_banner(self, description): t = self.task b = TextBanner( "Training Job %s is running at SageMaker:" % description.get("TrainingJobName", None), color="yellow", ) b.column("TASK", t.task_id) b.column( "JOB STATUS", description.get("TrainingJobStatus", None) + " -> " + description.get("SecondaryStatus", None), ) b.column( "JOB RESOURCES", description["ResourceConfig"]["InstanceType"] + " x " + str(description["ResourceConfig"]["InstanceCount"]), ) tracker_url = current_task_run().task_tracker_url if tracker_url: b.column("DATABAND LOG", tracker_url) b.column("JOB WEB UI", self._job_url(description.get("TrainingJobName", None))) b.column("CLOUDWATCH URL", self._logs_url()) b.new_line() b.column("JOB ARN", description.get("TrainingJobArn", None)) b.new_section() return b.getvalue()
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True): b = TextBanner(msg, color) run = self.run # type: DatabandRun ctx = run.context task_run_env = ctx.task_run_env # type: TaskRunEnvInfo b.column("TRACKER URL", run.run_url, skip_if_empty=True) b.column("TRACKERS", CoreConfig().tracker) if run.is_orchestration: run_executor = run.run_executor driver_task_run = run.driver_task_run if ( show_tasks_info and run_executor.run_executor_type == SystemTaskName.driver ): self._add_tasks_info(b) if show_run_info and driver_task_run and driver_task_run.log: b.column( "LOG", b.f_simple_dict( [ ("local", driver_task_run.log.local_log_file), ("remote", driver_task_run.log.remote_log_file), ], skip_if_empty=True, ), ) if run.root_run_info.root_run_uid != run.run_uid: b.column( "ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True ) b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True) if run.scheduled_run_info: b.column_properties( "SCHEDULED", [ ("scheduled_job", run.scheduled_run_info.scheduled_job_uid), ("scheduled_date", run.scheduled_run_info.scheduled_date), ("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id), ], ) if show_run_info: b.new_line() run_params = [ ("user", task_run_env.user), ("run_uid", "%s" % run.run_uid), ("env", run.env.name), ] b.column("RUN", b.f_simple_dict(run_params)) b.column("USER CODE VERSION", task_run_env.user_code_version) b.column("CMD", task_run_env.cmd_line) if run.is_orchestration: run_executor = run.run_executor b.column( "EXECUTE", b.f_simple_dict( [ ("TASK_EXECUTOR", run_executor.task_executor_type), ("PARALLEL", run_executor.parallel), ("SUBMIT_DRIVER", run_executor.submit_driver), ("SUBMIT_TASKS", run_executor.submit_tasks), ], skip_if_empty=True, ), skip_if_empty=True, ) if task_run_env.user_data and task_run_env.user_data != "None": b.column("USER DATA", task_run_env.user_data, skip_if_empty=True) b.new_line() failed_task_runs = [ task_run for task_run in run.task_runs if task_run.task_run_state in TaskRunState.direct_fail_states() ] if failed_task_runs: f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs) b.column("FAILED", f_msg) if run.root_task_run and run.is_orchestration: b.column("TASK_BAND", run.root_task_run.task.task_band) b.new_line() return b.getvalue()