def _register_dagrun(self, dagrun, execution_date):
self.log.debug(f"self.task_dict: {self.task_dict}")
# Register each task in the DAG
for task_id, task in self.task_dict.items():
t = self._now_ms()
try:
step = self._extract_metadata(dagrun, task)
job_name = self._marquez_job_name(self.dag_id, task.task_id)
run_id = self._marquez_run_id(dagrun.run_id, task.task_id)
task_run_id = _MARQUEZ.start_task(
run_id,
job_name,
self.description,
DagUtils.to_iso_8601(self._now_ms()),
None, # TODO: add parent hierarchy
self._get_location(task),
DagUtils.get_start_time(execution_date),
DagUtils.get_end_time(
execution_date,
self.following_schedule(execution_date)),
step)
JobIdMapping.set(job_name, dagrun.run_id, task_run_id)
except Exception as e:
self.log.error(
f'Failed to record task {task_id}: {e} '
f'{self._timed_log_message(t)}',
exc_info=True)
def _register_dagrun(self, dagrun, execution_date, run_args):
self.log.debug(f"self.task_dict: {self.task_dict}")
# Register each task in the DAG
for task_id, task in self.task_dict.items():
t = self._now_ms()
try:
steps = self._extract_metadata(dagrun, task)
[
_MARQUEZ.create_job(step, self._get_location(task),
self.description) for step in steps
]
marquez_jobrun_ids = [
_MARQUEZ.create_run(
self.new_run_id(), step, run_args,
DagUtils.get_start_time(execution_date),
DagUtils.get_end_time(
execution_date,
self.following_schedule(execution_date)))
for step in steps
]
JobIdMapping.set(
self._marquez_job_name(self.dag_id, task.task_id),
dagrun.run_id, marquez_jobrun_ids)
except Exception as e:
self.log.error(
f'Failed to record task {task_id}: {e} '
f'{self._timed_log_message(t)}',
exc_info=True)
def _register_dagrun(self, dagrun: DagRun, is_external_trigger: bool, execution_date: str):
self.log.debug(f"self.task_dict: {self.task_dict}")
# Register each task in the DAG
for task_id, task in self.task_dict.items():
t = self._now_ms()
try:
step = self._extract_metadata(dagrun, task)
job_name = self._marquez_job_name(self.dag_id, task.task_id)
run_id = self._marquez_run_id(dagrun.run_id, task.task_id)
task_run_id = _MARQUEZ.start_task(
run_id,
job_name,
self.description,
DagUtils.to_iso_8601(self._now_ms()),
dagrun.run_id,
self._get_location(task),
DagUtils.get_start_time(execution_date),
DagUtils.get_end_time(execution_date, self.following_schedule(execution_date)),
step,
{**step.run_facets, **get_custom_facets(task, is_external_trigger)}
)
JobIdMapping.set(
job_name,
dagrun.run_id,
task_run_id
)
except Exception as e:
self.log.error(
f'Failed to record task {task_id}: {e} '
f'{self._timed_log_message(t)}',
exc_info=True)