Exemple #1
0
    def _register_dagrun(self, dagrun, execution_date):
        self.log.debug(f"self.task_dict: {self.task_dict}")
        # Register each task in the DAG
        for task_id, task in self.task_dict.items():
            t = self._now_ms()
            try:
                step = self._extract_metadata(dagrun, task)

                job_name = self._marquez_job_name(self.dag_id, task.task_id)
                run_id = self._marquez_run_id(dagrun.run_id, task.task_id)

                task_run_id = _MARQUEZ.start_task(
                    run_id,
                    job_name,
                    self.description,
                    DagUtils.to_iso_8601(self._now_ms()),
                    None,  # TODO: add parent hierarchy
                    self._get_location(task),
                    DagUtils.get_start_time(execution_date),
                    DagUtils.get_end_time(
                        execution_date,
                        self.following_schedule(execution_date)),
                    step)

                JobIdMapping.set(job_name, dagrun.run_id, task_run_id)
            except Exception as e:
                self.log.error(
                    f'Failed to record task {task_id}: {e} '
                    f'{self._timed_log_message(t)}',
                    exc_info=True)
Exemple #2
0
 def _register_dagrun(self, dagrun, execution_date, run_args):
     self.log.debug(f"self.task_dict: {self.task_dict}")
     # Register each task in the DAG
     for task_id, task in self.task_dict.items():
         t = self._now_ms()
         try:
             steps = self._extract_metadata(dagrun, task)
             [
                 _MARQUEZ.create_job(step, self._get_location(task),
                                     self.description) for step in steps
             ]
             marquez_jobrun_ids = [
                 _MARQUEZ.create_run(
                     self.new_run_id(), step, run_args,
                     DagUtils.get_start_time(execution_date),
                     DagUtils.get_end_time(
                         execution_date,
                         self.following_schedule(execution_date)))
                 for step in steps
             ]
             JobIdMapping.set(
                 self._marquez_job_name(self.dag_id, task.task_id),
                 dagrun.run_id, marquez_jobrun_ids)
         except Exception as e:
             self.log.error(
                 f'Failed to record task {task_id}: {e} '
                 f'{self._timed_log_message(t)}',
                 exc_info=True)
Exemple #3
0
    def _register_dagrun(self, dagrun: DagRun, is_external_trigger: bool, execution_date: str):
        self.log.debug(f"self.task_dict: {self.task_dict}")
        # Register each task in the DAG
        for task_id, task in self.task_dict.items():
            t = self._now_ms()
            try:
                step = self._extract_metadata(dagrun, task)

                job_name = self._marquez_job_name(self.dag_id, task.task_id)
                run_id = self._marquez_run_id(dagrun.run_id, task.task_id)

                task_run_id = _MARQUEZ.start_task(
                    run_id,
                    job_name,
                    self.description,
                    DagUtils.to_iso_8601(self._now_ms()),
                    dagrun.run_id,
                    self._get_location(task),
                    DagUtils.get_start_time(execution_date),
                    DagUtils.get_end_time(execution_date, self.following_schedule(execution_date)),
                    step,
                    {**step.run_facets, **get_custom_facets(task, is_external_trigger)}
                )

                JobIdMapping.set(
                    job_name,
                    dagrun.run_id,
                    task_run_id
                )
            except Exception as e:
                self.log.error(
                    f'Failed to record task {task_id}: {e} '
                    f'{self._timed_log_message(t)}',
                    exc_info=True)