def _construct_run_with_snapshots(
self,
pipeline_name,
run_id,
environment_dict,
mode,
solid_subset,
step_keys_to_execute,
status,
tags,
root_run_id,
parent_run_id,
pipeline_snapshot,
execution_plan_snapshot,
parent_pipeline_snapshot,
):
# https://github.com/dagster-io/dagster/issues/2403
if tags and IS_AIRFLOW_INGEST_PIPELINE_STR in tags:
if AIRFLOW_EXECUTION_DATE_STR not in tags:
tags[AIRFLOW_EXECUTION_DATE_STR] = get_current_datetime_in_utc(
).isoformat()
pipeline_run = PipelineRun(
pipeline_name=pipeline_name,
run_id=run_id,
environment_dict=environment_dict,
mode=mode,
solid_subset=solid_subset,
step_keys_to_execute=step_keys_to_execute,
status=status,
tags=tags,
root_run_id=root_run_id,
parent_run_id=parent_run_id,
)
if pipeline_snapshot is not None:
from dagster.core.snap import create_pipeline_snapshot_id
if pipeline_snapshot.lineage_snapshot:
if not self._run_storage.has_pipeline_snapshot(
pipeline_snapshot.lineage_snapshot.parent_snapshot_id):
check.invariant(
create_pipeline_snapshot_id(
parent_pipeline_snapshot) ==
pipeline_snapshot.lineage_snapshot.parent_snapshot_id,
'Parent pipeline snapshot id out of sync with passed parent pipeline snapshot',
)
returned_pipeline_snapshot_id = self._run_storage.add_pipeline_snapshot(
parent_pipeline_snapshot)
check.invariant(
pipeline_snapshot.lineage_snapshot.parent_snapshot_id
== returned_pipeline_snapshot_id)
pipeline_snapshot_id = create_pipeline_snapshot_id(
pipeline_snapshot)
if not self._run_storage.has_pipeline_snapshot(
pipeline_snapshot_id):
returned_pipeline_snapshot_id = self._run_storage.add_pipeline_snapshot(
pipeline_snapshot)
check.invariant(
pipeline_snapshot_id == returned_pipeline_snapshot_id)
pipeline_run = pipeline_run.with_pipeline_snapshot_id(
pipeline_snapshot_id)
if execution_plan_snapshot is not None:
from dagster.core.snap import create_execution_plan_snapshot_id
check.invariant(execution_plan_snapshot.pipeline_snapshot_id ==
pipeline_snapshot_id)
check.invariant(
set(step_keys_to_execute) == set(
execution_plan_snapshot.step_keys_to_execute)
if step_keys_to_execute else set(
execution_plan_snapshot.step_keys_to_execute) == set(
[step.key for step in execution_plan_snapshot.steps]),
'We encode step_keys_to_execute twice in our stack, unfortunately. This check '
'ensures that they are consistent. We check that step_keys_to_execute in the plan '
'matches the step_keys_to_execute params if it is set. If it is not, this indicates '
'a full execution plan, and so we verify that.',
)
execution_plan_snapshot_id = create_execution_plan_snapshot_id(
execution_plan_snapshot)
if not self._run_storage.has_execution_plan_snapshot(
execution_plan_snapshot_id):
returned_execution_plan_snapshot_id = self._run_storage.add_execution_plan_snapshot(
execution_plan_snapshot)
check.invariant(execution_plan_snapshot_id ==
returned_execution_plan_snapshot_id)
pipeline_run = pipeline_run.with_execution_plan_snapshot_id(
execution_plan_snapshot_id)
return pipeline_run
def _construct_run_with_snapshots(
self,
pipeline_name=None,
run_id=None,
environment_dict=None,
mode=None,
solid_subset=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
):
if tags and IS_AIRFLOW_INGEST_PIPELINE_STR in tags:
if AIRFLOW_EXECUTION_DATE_STR not in tags:
tags[AIRFLOW_EXECUTION_DATE_STR] = get_current_datetime_in_utc(
).isoformat()
pipeline_run = PipelineRun(
pipeline_name=pipeline_name,
run_id=run_id,
environment_dict=environment_dict,
mode=mode,
solid_subset=solid_subset,
step_keys_to_execute=step_keys_to_execute,
status=status,
tags=tags,
root_run_id=root_run_id,
parent_run_id=parent_run_id,
)
if pipeline_snapshot is not None:
from dagster.core.snap import create_pipeline_snapshot_id
pipeline_snapshot_id = create_pipeline_snapshot_id(
pipeline_snapshot)
if not self._run_storage.has_pipeline_snapshot(
pipeline_snapshot_id):
returned_pipeline_snapshot_id = self._run_storage.add_pipeline_snapshot(
pipeline_snapshot)
check.invariant(
pipeline_snapshot_id == returned_pipeline_snapshot_id)
pipeline_run = pipeline_run.with_pipeline_snapshot_id(
pipeline_snapshot_id)
if execution_plan_snapshot is not None:
from dagster.core.snap import create_execution_plan_snapshot_id
check.invariant(execution_plan_snapshot.pipeline_snapshot_id ==
pipeline_snapshot_id)
execution_plan_snapshot_id = create_execution_plan_snapshot_id(
execution_plan_snapshot)
if not self._run_storage.has_execution_plan_snapshot(
execution_plan_snapshot_id):
returned_execution_plan_snapshot_id = self._run_storage.add_execution_plan_snapshot(
execution_plan_snapshot)
check.invariant(execution_plan_snapshot_id ==
returned_execution_plan_snapshot_id)
pipeline_run = pipeline_run.with_execution_plan_snapshot_id(
execution_plan_snapshot_id)
return pipeline_run
def get_or_create_run(
self,
pipeline_name=None,
run_id=None,
environment_dict=None,
mode=None,
selector=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
):
if tags and IS_AIRFLOW_INGEST_PIPELINE_STR in tags:
if AIRFLOW_EXECUTION_DATE_STR not in tags:
tags[AIRFLOW_EXECUTION_DATE_STR] = get_current_datetime_in_utc(
).isoformat()
pipeline_run = PipelineRun(
pipeline_name=pipeline_name,
run_id=run_id,
environment_dict=environment_dict,
mode=mode,
selector=selector,
step_keys_to_execute=step_keys_to_execute,
status=status,
tags=tags,
root_run_id=root_run_id,
parent_run_id=parent_run_id,
)
if pipeline_snapshot is not None:
from dagster.core.snap.pipeline_snapshot import create_pipeline_snapshot_id
pipeline_snapshot_id = create_pipeline_snapshot_id(
pipeline_snapshot)
if not self._run_storage.has_pipeline_snapshot(
pipeline_snapshot_id):
returned_pipeline_snapshot_id = self._run_storage.add_pipeline_snapshot(
pipeline_snapshot)
check.invariant(
pipeline_snapshot_id == returned_pipeline_snapshot_id)
pipeline_run = pipeline_run.with_pipeline_snapshot_id(
pipeline_snapshot_id)
if execution_plan_snapshot is not None:
from dagster.core.snap.execution_plan_snapshot import create_execution_plan_snapshot_id
check.invariant(execution_plan_snapshot.pipeline_snapshot_id ==
pipeline_snapshot_id)
execution_plan_snapshot_id = create_execution_plan_snapshot_id(
execution_plan_snapshot)
if not self._run_storage.has_execution_plan_snapshot(
execution_plan_snapshot_id):
returned_execution_plan_snapshot_id = self._run_storage.add_execution_plan_snapshot(
execution_plan_snapshot)
check.invariant(execution_plan_snapshot_id ==
returned_execution_plan_snapshot_id)
pipeline_run = pipeline_run.with_execution_plan_snapshot_id(
execution_plan_snapshot_id)
if self.has_run(pipeline_run.run_id):
candidate_run = self.get_run_by_id(pipeline_run.run_id)
field_diff = _check_run_equality(pipeline_run, candidate_run)
if field_diff:
raise DagsterRunConflict(
'Found conflicting existing run with same id {run_id}. Runs differ in:'
'\n{field_diff}'.format(
run_id=pipeline_run.run_id,
field_diff=_format_field_diff(field_diff),
), )
return candidate_run
return self._run_storage.add_run(pipeline_run)
