Beispiel #1
0
    def run_banner(self,
                   msg,
                   color="white",
                   show_run_info=False,
                   show_tasks_info=False):
        run = self.run  # type: DatabandRun
        if run.root_run_info.root_run_uid != run.run_uid:
            msg += " -> sub-run"

        b = TextBanner(msg, color)
        ctx = run.context
        task_run_env = ctx.task_run_env  # type: TaskRunEnvInfo

        if run.context.tracking_store.has_tracking_store("api",
                                                         channel_name="web"):
            b.column("TRACKER URL", run.run_url, skip_if_empty=True)

        b.column("TRACKERS", run.context.tracking_store.trackers_names)

        if run.root_run_info.root_run_uid != run.run_uid:
            b.column("ROOT TRACKER URL",
                     run.root_run_info.root_run_url,
                     skip_if_empty=True)
            b.column("ROOT UID URL",
                     run.root_run_info.root_run_uid,
                     skip_if_empty=True)

        if run.scheduled_run_info:
            b.column_properties(
                "SCHEDULED",
                [
                    ("scheduled_job",
                     run.scheduled_run_info.scheduled_job_uid),
                    ("scheduled_date", run.scheduled_run_info.scheduled_date),
                    ("dag_run_id",
                     run.scheduled_run_info.scheduled_job_dag_run_id),
                ],
            )

        if show_run_info:
            b.new_line()
            run_params = [
                ("user", task_run_env.user),
                ("run_uid", "%s" % run.run_uid),
                ("env", run.env.name),
                ("project", run.project_name) if run.project_name else None,
                ("user_code_version", task_run_env.user_code_version),
            ]
            run_params = list(filter(None, run_params))
            b.column("RUN", b.f_simple_dict(run_params))
            b.column("CMD", task_run_env.cmd_line)

            if task_run_env.user_data and task_run_env.user_data != "None":
                b.column("USER DATA",
                         task_run_env.user_data,
                         skip_if_empty=True)
            b.new_line()

        if run.is_orchestration:
            run_executor = run.run_executor
            driver_task_run = run.driver_task_run
            if show_run_info:
                if driver_task_run and driver_task_run.log:
                    b.column(
                        "LOG",
                        b.f_simple_dict(
                            [
                                ("local", driver_task_run.log.local_log_file),
                                ("remote",
                                 driver_task_run.log.remote_log_file),
                            ],
                            skip_if_empty=True,
                        ),
                    )

                b.column(
                    "EXECUTE",
                    b.f_simple_dict(
                        [
                            ("TASK_EXECUTOR", run_executor.task_executor_type),
                            ("PARALLEL", run_executor.parallel),
                            ("SUBMIT_DRIVER", run_executor.submit_driver),
                            ("SUBMIT_TASKS", run_executor.submit_tasks),
                        ],
                        skip_if_empty=True,
                    ),
                    skip_if_empty=True,
                )
            if run_executor.run_executor_type == SystemTaskName.driver:
                if run.root_task_run:
                    b.column("TASK_BAND", run.root_task_run.task.task_band)

                if show_tasks_info:
                    self._add_tasks_info(b)
                failed_task_runs = [
                    task_run for task_run in run.get_task_runs()
                    if task_run.task_run_state == TaskRunState.FAILED
                ]
                if failed_task_runs:
                    f_msg = "\n".join(tr.task.task_id
                                      for tr in failed_task_runs)
                    b.column("FAILED", f_msg)
        b.new_line()

        return b.getvalue()
Beispiel #2
0
    def run_banner(self,
                   msg,
                   color="white",
                   show_run_info=False,
                   show_tasks_info=True):
        b = TextBanner(msg, color)
        run = self.run  # type: DatabandRun
        ctx = run.context
        task_run_env = ctx.task_run_env  # type: TaskRunEnvInfo
        driver_task = run.driver_task_run.task

        orchestration_mode = run.source == UpdateSource.dbnd

        b.column("TRACKER URL", run.run_url, skip_if_empty=True)
        if show_tasks_info and orchestration_mode and driver_task.is_driver:
            self._add_tasks_info(b)

        if run.root_run_info.root_run_uid != run.run_uid:
            b.column("ROOT TRACKER URL",
                     run.root_run_info.root_run_url,
                     skip_if_empty=True)
            b.column("ROOT UID URL",
                     run.root_run_info.root_run_uid,
                     skip_if_empty=True)

        if run.scheduled_run_info:
            b.column_properties(
                "SCHEDULED",
                [
                    ("scheduled_job",
                     run.scheduled_run_info.scheduled_job_uid),
                    ("scheduled_date", run.scheduled_run_info.scheduled_date),
                    ("dag_run_id",
                     run.scheduled_run_info.scheduled_job_dag_run_id),
                ],
            )

        if show_run_info:
            b.new_line()
            run_params = [
                ("user", task_run_env.user),
                ("run_uid", "%s" % run.run_uid),
                ("env", run.env.name),
            ]
            b.column("RUN", b.f_simple_dict(run_params))
            b.column(
                "LOG",
                b.f_simple_dict([
                    ("local", driver_task.local_driver_log),
                    ("remote", driver_task.remote_driver_root),
                ]),
            )
            b.column("USER CODE VERSION", task_run_env.user_code_version)
            b.column("CMD", task_run_env.cmd_line)

            if orchestration_mode:
                if run.context.settings.core.is_db_store_enabled():
                    b.column("DB", self.context.settings.core.sql_conn_repr)
                if run.task_executor_type.startswith("airflow"):
                    assert_airflow_enabled()
                    from dbnd_airflow.db_utils import airflow_sql_conn_repr

                    b.column("Airflow DB", airflow_sql_conn_repr())
                b.column(
                    "EXECUTE",
                    b.f_simple_dict(
                        [
                            ("TASK_EXECUTOR", run.task_executor_type),
                            ("PARALLEL", run.parallel),
                            ("SUBMIT_DRIVER", run.submit_driver),
                            ("SUBMIT_TASKS", run.submit_tasks),
                        ],
                        skip_if_empty=True,
                    ),
                    skip_if_empty=True,
                )
            if task_run_env.user_data and task_run_env.user_data != "None":
                b.column("USER DATA",
                         task_run_env.user_data,
                         skip_if_empty=True)
            b.new_line()

        failed_task_runs = [
            task_run for task_run in run.task_runs
            if task_run.task_run_state in TaskRunState.direct_fail_states()
        ]
        if failed_task_runs:
            f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs)
            b.column("FAILED", f_msg)

        b.new_line()

        return b.getvalue()
Beispiel #3
0
    def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True):
        b = TextBanner(msg, color)
        run = self.run  # type: DatabandRun
        ctx = run.context
        task_run_env = ctx.task_run_env  # type: TaskRunEnvInfo
        driver_task = run.driver_task_run.task
        if show_tasks_info and driver_task.is_driver:
            self._add_tasks_info(b)

        b.column("TRACKER URL", run.run_url, skip_if_empty=True)

        if run.root_run_info.root_run_uid != run.run_uid:
            b.column(
                "ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True
            )
            b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True)

        if run.scheduled_run_info:
            b.column_properties(
                "SCHEDULED",
                [
                    ("scheduled_job", run.scheduled_run_info.scheduled_job_uid),
                    ("scheduled_date", run.scheduled_run_info.scheduled_date),
                    ("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id),
                ],
            )

        if show_run_info:
            b.new_line()
            b.column("USER", task_run_env.user)
            b.column(
                "LOG",
                b.f_simple_dict(
                    [
                        ("local", driver_task.local_driver_log),
                        ("remote", driver_task.remote_driver_root),
                    ]
                ),
            )
            b.column("USER CODE VERSION", task_run_env.user_code_version)
            b.column("CMD", task_run_env.cmd_line)
            b.column("RUN UID", "%s" % run.run_uid)
            b.column("DB", self.context.settings.core.sql_conn_repr)
            b.column("ENV", run.env.name)
            b.column(
                "RUN",
                b.f_simple_dict(
                    [
                        ("TASK_EXECUTOR", run.task_executor_type),
                        ("PARALLEL", run.parallel),
                        ("SUBMIT_DRIVER", run.submit_driver),
                        ("SUBMIT_TASKS", run.submit_tasks),
                    ],
                    skip_if_empty=True,
                ),
                skip_if_empty=True,
            )
            if task_run_env.user_data:
                b.column("USER DATA", task_run_env.user_data, skip_if_empty=True)
            b.new_line()

        failed_task_runs = [
            task_run
            for task_run in run.task_runs
            if task_run.task_run_state in TaskRunState.direct_fail_states()
        ]
        if failed_task_runs:
            f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs)
            b.column("FAILED", f_msg)

        b.new_line()

        return b.getvalue()
Beispiel #4
0
    def run_banner(self, msg, color="white", show_run_info=False, show_tasks_info=True):
        b = TextBanner(msg, color)
        run = self.run  # type: DatabandRun
        ctx = run.context
        task_run_env = ctx.task_run_env  # type: TaskRunEnvInfo

        b.column("TRACKER URL", run.run_url, skip_if_empty=True)
        b.column("TRACKERS", CoreConfig().tracker)
        if run.is_orchestration:
            run_executor = run.run_executor
            driver_task_run = run.driver_task_run

            if (
                show_tasks_info
                and run_executor.run_executor_type == SystemTaskName.driver
            ):
                self._add_tasks_info(b)
            if show_run_info and driver_task_run and driver_task_run.log:
                b.column(
                    "LOG",
                    b.f_simple_dict(
                        [
                            ("local", driver_task_run.log.local_log_file),
                            ("remote", driver_task_run.log.remote_log_file),
                        ],
                        skip_if_empty=True,
                    ),
                )

        if run.root_run_info.root_run_uid != run.run_uid:
            b.column(
                "ROOT TRACKER URL", run.root_run_info.root_run_url, skip_if_empty=True
            )
            b.column("ROOT UID URL", run.root_run_info.root_run_uid, skip_if_empty=True)

        if run.scheduled_run_info:
            b.column_properties(
                "SCHEDULED",
                [
                    ("scheduled_job", run.scheduled_run_info.scheduled_job_uid),
                    ("scheduled_date", run.scheduled_run_info.scheduled_date),
                    ("dag_run_id", run.scheduled_run_info.scheduled_job_dag_run_id),
                ],
            )

        if show_run_info:
            b.new_line()
            run_params = [
                ("user", task_run_env.user),
                ("run_uid", "%s" % run.run_uid),
                ("env", run.env.name),
            ]
            b.column("RUN", b.f_simple_dict(run_params))
            b.column("USER CODE VERSION", task_run_env.user_code_version)
            b.column("CMD", task_run_env.cmd_line)

            if run.is_orchestration:
                run_executor = run.run_executor
                b.column(
                    "EXECUTE",
                    b.f_simple_dict(
                        [
                            ("TASK_EXECUTOR", run_executor.task_executor_type),
                            ("PARALLEL", run_executor.parallel),
                            ("SUBMIT_DRIVER", run_executor.submit_driver),
                            ("SUBMIT_TASKS", run_executor.submit_tasks),
                        ],
                        skip_if_empty=True,
                    ),
                    skip_if_empty=True,
                )
            if task_run_env.user_data and task_run_env.user_data != "None":
                b.column("USER DATA", task_run_env.user_data, skip_if_empty=True)
            b.new_line()

        failed_task_runs = [
            task_run
            for task_run in run.task_runs
            if task_run.task_run_state in TaskRunState.direct_fail_states()
        ]
        if failed_task_runs:
            f_msg = "\n\t".join(tr.task.task_id for tr in failed_task_runs)
            b.column("FAILED", f_msg)

        if run.root_task_run and run.is_orchestration:
            b.column("TASK_BAND", run.root_task_run.task.task_band)

        b.new_line()

        return b.getvalue()