def _register_dagrun(self, dagrun, execution_date): self.log.debug(f"self.task_dict: {self.task_dict}") # Register each task in the DAG for task_id, task in self.task_dict.items(): t = self._now_ms() try: step = self._extract_metadata(dagrun, task) job_name = self._marquez_job_name(self.dag_id, task.task_id) run_id = self._marquez_run_id(dagrun.run_id, task.task_id) task_run_id = _MARQUEZ.start_task( run_id, job_name, self.description, DagUtils.to_iso_8601(self._now_ms()), None, # TODO: add parent hierarchy self._get_location(task), DagUtils.get_start_time(execution_date), DagUtils.get_end_time( execution_date, self.following_schedule(execution_date)), step) JobIdMapping.set(job_name, dagrun.run_id, task_run_id) except Exception as e: self.log.error( f'Failed to record task {task_id}: {e} ' f'{self._timed_log_message(t)}', exc_info=True)
def _register_dagrun(self, dagrun, execution_date, run_args): self.log.debug(f"self.task_dict: {self.task_dict}") # Register each task in the DAG for task_id, task in self.task_dict.items(): t = self._now_ms() try: steps = self._extract_metadata(dagrun, task) [ _MARQUEZ.create_job(step, self._get_location(task), self.description) for step in steps ] marquez_jobrun_ids = [ _MARQUEZ.create_run( self.new_run_id(), step, run_args, DagUtils.get_start_time(execution_date), DagUtils.get_end_time( execution_date, self.following_schedule(execution_date))) for step in steps ] JobIdMapping.set( self._marquez_job_name(self.dag_id, task.task_id), dagrun.run_id, marquez_jobrun_ids) except Exception as e: self.log.error( f'Failed to record task {task_id}: {e} ' f'{self._timed_log_message(t)}', exc_info=True)
def _register_dagrun(self, dagrun: DagRun, is_external_trigger: bool, execution_date: str): self.log.debug(f"self.task_dict: {self.task_dict}") # Register each task in the DAG for task_id, task in self.task_dict.items(): t = self._now_ms() try: step = self._extract_metadata(dagrun, task) job_name = self._marquez_job_name(self.dag_id, task.task_id) run_id = self._marquez_run_id(dagrun.run_id, task.task_id) task_run_id = _MARQUEZ.start_task( run_id, job_name, self.description, DagUtils.to_iso_8601(self._now_ms()), dagrun.run_id, self._get_location(task), DagUtils.get_start_time(execution_date), DagUtils.get_end_time(execution_date, self.following_schedule(execution_date)), step, {**step.run_facets, **get_custom_facets(task, is_external_trigger)} ) JobIdMapping.set( job_name, dagrun.run_id, task_run_id ) except Exception as e: self.log.error( f'Failed to record task {task_id}: {e} ' f'{self._timed_log_message(t)}', exc_info=True)