def _qubole_banner(self, status): b = TextBanner( "Spark task {} is submitted to Qubole cluster labeled: {}".format( self.task.task_id, self.qubole_config.cluster_label), color="magenta", ) b.column("Status", status) b.column("URL", self.qubole_job_url) logger.info(b.get_banner_str())
def pformat_all_layers(config, sections=None): # type: (DbndConfig, Optional[Iterable[str]]) -> str tb = TextBanner("Configs") layers = config.config_layer.get_all_layers() # start from the first one for layer in reversed(layers): layer_values = layer.layer_config.as_value_dict(sections=sections) tb.column("LAYER {}".format(layer.config_layer_path), tb.f_struct(layer_values)) return tb.get_banner_str()
def pformat_current_config(config, as_table=False, sections=None): # type: (DbndConfig, bool, Optional[Iterable[str]]) -> str config_layer = config.config_layer tb = TextBanner("Config {}".format(config_layer.name)) tb.column("LAYERS", config_layer.config_layer_path) if as_table: view_str = pformat_config_store_as_table( config_store=config_layer.config, sections=sections) else: value_dict = config_layer.config.as_value_dict(sections=sections) view_str = tb.f_struct(value_dict) tb.column("CONFIG", view_str) return tb.get_banner_str()
class DoctorStatusReportBuilder(object): def __init__(self, name): self.name = name msg = "Running %s Status Check:" % name logger.info(msg) self.text_banner = TextBanner(msg) def log(self, key, value): log_metric(key, value) self.text_banner.column(key, value) def add_sub_report(self, sub_report_str): self.text_banner.write(sub_report_str) self.text_banner.write("\n") def get_status_str(self): return self.text_banner.get_banner_str()
def _get_step_banner(self, step): """ { 'id': 6, 'state': 'success', } """ t = self.task b = TextBanner("Spark Task %s is running at Emr:" % t.task_id, color="yellow") b.column("TASK", t.task_id) b.column("EMR STEP STATE", step["Step"]["Status"]["State"]) tracker_url = current_task_run().task_tracker_url if tracker_url: b.column("DATABAND LOG", tracker_url) b.new_line() b.column("EMR STEP ID", step["Step"]["Id"]) b.new_section() return b.getvalue()
def _get_job_status_banner(self, description): t = self.task b = TextBanner( "Training Job %s is running at SageMaker:" % description.get("TrainingJobName", None), color="yellow", ) b.column("TASK", t.task_id) b.column( "JOB STATUS", description.get("TrainingJobStatus", None) + " -> " + description.get("SecondaryStatus", None), ) b.column( "JOB RESOURCES", description["ResourceConfig"]["InstanceType"] + " x " + str(description["ResourceConfig"]["InstanceCount"]), ) tracker_url = current_task_run().task_tracker_url if tracker_url: b.column("DATABAND LOG", tracker_url) b.column("JOB WEB UI", self._job_url(description.get("TrainingJobName", None))) b.column("CLOUDWATCH URL", self._logs_url()) b.new_line() b.column("JOB ARN", description.get("TrainingJobArn", None)) b.new_section() return b.getvalue()
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=False): run = self.run # type: DatabandRun if run.root_run_info.root_run_uid != run.run_uid: msg += " -> sub-run" b = TextBanner(msg, color) ctx = run.context task_run_env = ctx.task_run_env # type: TaskRunEnvInfo if run.context.tracking_store.has_tracking_store("api", channel_name="web"): b.column("TRACKER URL", run.run_url, skip_if_empty=True) b.column("TRACKERS", run.context.tracking_store.trackers_names) if run.root_run_info.root_run_uid != run.run_uid: b.column("ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True) b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True) if run.scheduled_run_info: b.column_properties( "SCHEDULED", [ ("scheduled_job", run.scheduled_run_info.scheduled_job_uid), ("scheduled_date", run.scheduled_run_info.scheduled_date), ("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id), ], ) if show_run_info: b.new_line() run_params = [ ("user", task_run_env.user), ("run_uid", "%s" % run.run_uid), ("env", run.env.name), ("project", run.project_name) if run.project_name else None, ("user_code_version", task_run_env.user_code_version), ] run_params = list(filter(None, run_params)) b.column("RUN", b.f_simple_dict(run_params)) b.column("CMD", task_run_env.cmd_line) if task_run_env.user_data and task_run_env.user_data != "None": b.column("USER DATA", task_run_env.user_data, skip_if_empty=True) b.new_line() if run.is_orchestration: run_executor = run.run_executor driver_task_run = run.driver_task_run if show_run_info: if driver_task_run and driver_task_run.log: b.column( "LOG", b.f_simple_dict( [ ("local", driver_task_run.log.local_log_file), ("remote", driver_task_run.log.remote_log_file), ], skip_if_empty=True, ), ) b.column( "EXECUTE", b.f_simple_dict( [ ("TASK_EXECUTOR", run_executor.task_executor_type), ("PARALLEL", run_executor.parallel), ("SUBMIT_DRIVER", run_executor.submit_driver), ("SUBMIT_TASKS", run_executor.submit_tasks), ], skip_if_empty=True, ), skip_if_empty=True, ) if run_executor.run_executor_type == SystemTaskName.driver: if run.root_task_run: b.column("TASK_BAND", run.root_task_run.task.task_band) if show_tasks_info: self._add_tasks_info(b) failed_task_runs = [ task_run for task_run in run.get_task_runs() if task_run.task_run_state == TaskRunState.FAILED ] if failed_task_runs: f_msg = "\n".join(tr.task.task_id for tr in failed_task_runs) b.column("FAILED", f_msg) b.new_line() return b.getvalue()
def _get_batch_progresss_banner(self, batch_response): """ { 'id': 6, 'state': 'success', 'appId': 'application_1534487568579_0008', 'appInfo': { 'driverLogUrl': None, 'sparkUiUrl': 'http://ip-172-31-70-109.ec2.internal:20888/proxy/application_1534487568579_0008/' }, 'log': [ '\nYARN Diagnostics: ' ] } :param batch_response: :return: """ t = self.task b = TextBanner("Spark Task %s is running at Livy:" % t.task_id, color="yellow") b.column("TASK", t.task_id) b.column("JOB STATE", batch_response.get("state", None)) tracker_url = current_task_run().task_tracker_url if tracker_url: b.column("DATABAND LOG", tracker_url) b.new_line() b.column("LIVY ID", batch_response.get("id", None)) if "appId" in batch_response: b.column("APP ID", batch_response["appId"]) app_info = batch_response["appInfo"] b.column("DRIVER LOG", app_info["driverLogUrl"]) if "sparkUiUrl" in app_info: spark_url = app_info["sparkUiUrl"] b.column( "SPARK", colored(spark_url, on_color="on_blue", attrs=["bold"])) b.new_section() return b.getvalue()
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True): b = TextBanner(msg, color) run = self.run # type: DatabandRun ctx = run.context task_run_env = ctx.task_run_env # type: TaskRunEnvInfo driver_task = run.driver_task_run.task if show_tasks_info and driver_task.is_driver: self._add_tasks_info(b) b.column("TRACKER URL", run.run_url, skip_if_empty=True) if run.root_run_info.root_run_uid != run.run_uid: b.column( "ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True ) b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True) if run.scheduled_run_info: b.column_properties( "SCHEDULED", [ ("scheduled_job", run.scheduled_run_info.scheduled_job_uid), ("scheduled_date", run.scheduled_run_info.scheduled_date), ("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id), ], ) if show_run_info: b.new_line() b.column("USER", task_run_env.user) b.column( "LOG", b.f_simple_dict( [ ("local", driver_task.local_driver_log), ("remote", driver_task.remote_driver_root), ] ), ) b.column("USER CODE VERSION", task_run_env.user_code_version) b.column("CMD", task_run_env.cmd_line) b.column("RUN UID", "%s" % run.run_uid) b.column("DB", self.context.settings.core.sql_conn_repr) b.column("ENV", run.env.name) b.column( "RUN", b.f_simple_dict( [ ("TASK_EXECUTOR", run.task_executor_type), ("PARALLEL", run.parallel), ("SUBMIT_DRIVER", run.submit_driver), ("SUBMIT_TASKS", run.submit_tasks), ], skip_if_empty=True, ), skip_if_empty=True, ) if task_run_env.user_data: b.column("USER DATA", task_run_env.user_data, skip_if_empty=True) b.new_line() failed_task_runs = [ task_run for task_run in run.task_runs if task_run.task_run_state in TaskRunState.direct_fail_states() ] if failed_task_runs: f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs) b.column("FAILED", f_msg) b.new_line() return b.getvalue()
def _handle_databricks_operator_execution(self, run_id, hook, task_id): """ Handles the Airflow + Databricks lifecycle logic for a Databricks operator :param run_id: Databricks run_id :param hook: Airflow databricks hook :param task_id: Databand Task Id. """ b = TextBanner("Spark task %s is submitted to Databricks cluster:" % task_id, color="cyan") url = hook.get_run_page_url(run_id) self.task_run.set_external_resource_urls({"databricks url": url}) b.column("URL", url) logger.info(b.get_banner_str()) while True: b = TextBanner( "Spark task %s is submitted to Databricks cluster:" % task_id, color="cyan", ) b.column("URL", url) run_state = hook.get_run_state(run_id) if run_state.is_terminal: if run_state.is_successful: b.column("Task completed successfully", task_id) b.column("State:", run_state.life_cycle_state) b.column("Message:", run_state.state_message) break else: b.column("State", run_state.result_state) b.column("Error Message:", run_state.state_message) logger.info(b.get_banner_str()) raise failed_to_run_databricks_job(run_state.result_state, run_state.state_message, url) else: b.column("State:", run_state.life_cycle_state) b.column("Message:", run_state.state_message) time.sleep( self.databricks_config.status_polling_interval_seconds) logger.info(b.get_banner_str())
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True): b = TextBanner(msg, color) run = self.run # type: DatabandRun ctx = run.context task_run_env = ctx.task_run_env # type: TaskRunEnvInfo driver_task = run.driver_task_run.task orchestration_mode = run.source == UpdateSource.dbnd b.column("TRACKER URL", run.run_url, skip_if_empty=True) if show_tasks_info and orchestration_mode and driver_task.is_driver: self._add_tasks_info(b) if run.root_run_info.root_run_uid != run.run_uid: b.column("ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True) b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True) if run.scheduled_run_info: b.column_properties( "SCHEDULED", [ ("scheduled_job", run.scheduled_run_info.scheduled_job_uid), ("scheduled_date", run.scheduled_run_info.scheduled_date), ("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id), ], ) if show_run_info: b.new_line() run_params = [ ("user", task_run_env.user), ("run_uid", "%s" % run.run_uid), ("env", run.env.name), ] b.column("RUN", b.f_simple_dict(run_params)) b.column( "LOG", b.f_simple_dict([ ("local", driver_task.local_driver_log), ("remote", driver_task.remote_driver_root), ]), ) b.column("USER CODE VERSION", task_run_env.user_code_version) b.column("CMD", task_run_env.cmd_line) if orchestration_mode: if run.context.settings.core.is_db_store_enabled(): b.column("DB", self.context.settings.core.sql_conn_repr) if run.task_executor_type.startswith("airflow"): assert_airflow_enabled() from dbnd_airflow.db_utils import airflow_sql_conn_repr b.column("Airflow DB", airflow_sql_conn_repr()) b.column( "EXECUTE", b.f_simple_dict( [ ("TASK_EXECUTOR", run.task_executor_type), ("PARALLEL", run.parallel), ("SUBMIT_DRIVER", run.submit_driver), ("SUBMIT_TASKS", run.submit_tasks), ], skip_if_empty=True, ), skip_if_empty=True, ) if task_run_env.user_data and task_run_env.user_data != "None": b.column("USER DATA", task_run_env.user_data, skip_if_empty=True) b.new_line() failed_task_runs = [ task_run for task_run in run.task_runs if task_run.task_run_state in TaskRunState.direct_fail_states() ] if failed_task_runs: f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs) b.column("FAILED", f_msg) b.new_line() return b.getvalue()
def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True): b = TextBanner(msg, color) run = self.run # type: DatabandRun ctx = run.context task_run_env = ctx.task_run_env # type: TaskRunEnvInfo b.column("TRACKER URL", run.run_url, skip_if_empty=True) b.column("TRACKERS", CoreConfig().tracker) if run.is_orchestration: run_executor = run.run_executor driver_task_run = run.driver_task_run if ( show_tasks_info and run_executor.run_executor_type == SystemTaskName.driver ): self._add_tasks_info(b) if show_run_info and driver_task_run and driver_task_run.log: b.column( "LOG", b.f_simple_dict( [ ("local", driver_task_run.log.local_log_file), ("remote", driver_task_run.log.remote_log_file), ], skip_if_empty=True, ), ) if run.root_run_info.root_run_uid != run.run_uid: b.column( "ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True ) b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True) if run.scheduled_run_info: b.column_properties( "SCHEDULED", [ ("scheduled_job", run.scheduled_run_info.scheduled_job_uid), ("scheduled_date", run.scheduled_run_info.scheduled_date), ("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id), ], ) if show_run_info: b.new_line() run_params = [ ("user", task_run_env.user), ("run_uid", "%s" % run.run_uid), ("env", run.env.name), ] b.column("RUN", b.f_simple_dict(run_params)) b.column("USER CODE VERSION", task_run_env.user_code_version) b.column("CMD", task_run_env.cmd_line) if run.is_orchestration: run_executor = run.run_executor b.column( "EXECUTE", b.f_simple_dict( [ ("TASK_EXECUTOR", run_executor.task_executor_type), ("PARALLEL", run_executor.parallel), ("SUBMIT_DRIVER", run_executor.submit_driver), ("SUBMIT_TASKS", run_executor.submit_tasks), ], skip_if_empty=True, ), skip_if_empty=True, ) if task_run_env.user_data and task_run_env.user_data != "None": b.column("USER DATA", task_run_env.user_data, skip_if_empty=True) b.new_line() failed_task_runs = [ task_run for task_run in run.task_runs if task_run.task_run_state in TaskRunState.direct_fail_states() ] if failed_task_runs: f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs) b.column("FAILED", f_msg) if run.root_task_run and run.is_orchestration: b.column("TASK_BAND", run.root_task_run.task.task_band) b.new_line() return b.getvalue()