def _ensure_persisted_pipeline_snapshot(self, pipeline_snapshot, parent_pipeline_snapshot): from dagster.core.snap import create_pipeline_snapshot_id, PipelineSnapshot check.inst_param(pipeline_snapshot, "pipeline_snapshot", PipelineSnapshot) check.opt_inst_param(parent_pipeline_snapshot, "parent_pipeline_snapshot", PipelineSnapshot) if pipeline_snapshot.lineage_snapshot: if not self._run_storage.has_pipeline_snapshot( pipeline_snapshot.lineage_snapshot.parent_snapshot_id ): check.invariant( create_pipeline_snapshot_id(parent_pipeline_snapshot) == pipeline_snapshot.lineage_snapshot.parent_snapshot_id, "Parent pipeline snapshot id out of sync with passed parent pipeline snapshot", ) returned_pipeline_snapshot_id = self._run_storage.add_pipeline_snapshot( parent_pipeline_snapshot ) check.invariant( pipeline_snapshot.lineage_snapshot.parent_snapshot_id == returned_pipeline_snapshot_id ) pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot) if not self._run_storage.has_pipeline_snapshot(pipeline_snapshot_id): returned_pipeline_snapshot_id = self._run_storage.add_pipeline_snapshot( pipeline_snapshot ) check.invariant(pipeline_snapshot_id == returned_pipeline_snapshot_id) return pipeline_snapshot_id
def __init__(self, pipeline_snapshot, parent_pipeline_snapshot): self.pipeline_snapshot = check.inst_param(pipeline_snapshot, 'pipeline_snapshot', PipelineSnapshot) self.parent_pipeline_snapshot = check.opt_inst_param( parent_pipeline_snapshot, 'parent_pipeline_snapshot', PipelineSnapshot) if self.pipeline_snapshot.lineage_snapshot: check.invariant( self.parent_pipeline_snapshot is not None, 'Can not create PipelineIndex for pipeline_snapshot with lineage without parent_pipeline_snapshot', ) parent_id = create_pipeline_snapshot_id( self.parent_pipeline_snapshot) check.invariant( pipeline_snapshot.lineage_snapshot.parent_snapshot_id == parent_id, 'Mismatch in IDs between pipeline_snapshot lineage and parent_pipeline_snapshot', ) self._solid_defs_snaps_index = { sd.name: sd for sd in pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps + pipeline_snapshot.solid_definitions_snapshot. composite_solid_def_snaps } self._dagster_type_snaps_by_name_index = { dagster_type_snap.name: dagster_type_snap for dagster_type_snap in pipeline_snapshot.dagster_type_namespace_snapshot. all_dagster_type_snaps_by_key.values() if dagster_type_snap.name } self.dep_structure_index = DependencyStructureIndex( pipeline_snapshot.dep_structure_snapshot) self._comp_dep_structures = { comp_snap.name: DependencyStructureIndex(comp_snap.dep_structure_snapshot) for comp_snap in pipeline_snapshot.solid_definitions_snapshot. composite_solid_def_snaps } self.pipeline_snapshot_id = create_pipeline_snapshot_id( pipeline_snapshot)
def test_run_created_in_0_7_9_snapshot_id_change(): test_dir = file_relative_path( __file__, 'snapshot_0_7_9_shapshot_id_creation_change/sqlite') with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) # run_id = 'e297fa70-49e8-43f8-abfe-1634f02644f6' old_pipeline_snapshot_id = '88528edde2ed64da3c39cca0da8ba2f7586c1a5d' old_execution_plan_snapshot_id = '2246f8e5a10d21e15fbfa3773d7b2d0bc1fa9d3d' pipeline_snapshot = instance.get_pipeline_snapshot( old_pipeline_snapshot_id) ep_snapshot = instance.get_execution_plan_snapshot( old_execution_plan_snapshot_id) # It is the pipeline snapshot that changed # Verify that snapshot ids are not equal. This changed in 0.7.10 assert create_pipeline_snapshot_id( pipeline_snapshot) != old_pipeline_snapshot_id # We also changed execution plan schema in 0.7.11.post1 assert create_execution_plan_snapshot_id( ep_snapshot) != old_execution_plan_snapshot_id # This previously failed with a check error assert ExecutionPlanIndex(ep_snapshot, PipelineIndex(pipeline_snapshot))
def test_single_write_read_with_snapshot(self, storage): run_with_snapshot_id = "lkasjdflkjasdf" pipeline_def = PipelineDefinition(name="some_pipeline", solid_defs=[]) pipeline_snapshot = pipeline_def.get_pipeline_snapshot() pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot) run_with_snapshot = PipelineRun( run_id=run_with_snapshot_id, pipeline_name=pipeline_def.name, pipeline_snapshot_id=pipeline_snapshot_id, ) assert not storage.has_pipeline_snapshot(pipeline_snapshot_id) assert storage.add_pipeline_snapshot( pipeline_snapshot) == pipeline_snapshot_id assert serialize_pp(storage.get_pipeline_snapshot( pipeline_snapshot_id)) == serialize_pp(pipeline_snapshot) storage.add_run(run_with_snapshot) assert storage.get_run_by_id(run_with_snapshot_id) == run_with_snapshot storage.wipe() assert not storage.has_pipeline_snapshot(pipeline_snapshot_id) assert not storage.has_run(run_with_snapshot_id)
def __init__(self, pipeline_snapshot): self.pipeline_snapshot = check.inst_param(pipeline_snapshot, 'pipeline_snapshot', PipelineSnapshot) self._solid_defs_snaps_index = { sd.name: sd for sd in pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps + pipeline_snapshot.solid_definitions_snapshot. composite_solid_def_snaps } self._dagster_type_snaps_by_name_index = { dagster_type_snap.name: dagster_type_snap for dagster_type_snap in pipeline_snapshot.dagster_type_namespace_snapshot. all_dagster_type_snaps_by_key.values() if dagster_type_snap.name } self.dep_structure_index = DependencyStructureIndex( pipeline_snapshot.dep_structure_snapshot) self._comp_dep_structures = { comp_snap.name: DependencyStructureIndex(comp_snap.dep_structure_snapshot) for comp_snap in pipeline_snapshot.solid_definitions_snapshot. composite_solid_def_snaps } self.pipeline_snapshot_id = create_pipeline_snapshot_id( pipeline_snapshot)
def add_pipeline_snapshot(self, pipeline_snapshot): check.inst_param(pipeline_snapshot, 'pipeline_snapshot', PipelineSnapshot) return self._add_snapshot( snapshot_id=create_pipeline_snapshot_id(pipeline_snapshot), snapshot_obj=pipeline_snapshot, snapshot_type=SnapshotType.PIPELINE, )
def test_run_created_in_0_7_9_snapshot_id_change(): test_dir = file_relative_path( __file__, 'snapshot_0_7_9_shapshot_id_creation_change/sqlite') with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) # run_id = 'e297fa70-49e8-43f8-abfe-1634f02644f6' old_pipeline_snapshot_id = '88528edde2ed64da3c39cca0da8ba2f7586c1a5d' old_execution_plan_snapshot_id = '2246f8e5a10d21e15fbfa3773d7b2d0bc1fa9d3d' with pytest.warns( UserWarning, match=re.escape( '"input_hydration_schema_key" is deprecated and will be removed in 0.10.0, use ' '"loader_schema_key" instead.'), ): historical_pipeline = instance.get_historical_pipeline( old_pipeline_snapshot_id) pipeline_snapshot = historical_pipeline.pipeline_snapshot ep_snapshot = instance.get_execution_plan_snapshot( old_execution_plan_snapshot_id) # It is the pipeline snapshot that changed # Verify that snapshot ids are not equal. This changed in 0.7.10 assert create_pipeline_snapshot_id( pipeline_snapshot) != old_pipeline_snapshot_id # We also changed execution plan schema in 0.7.11.post1 assert create_execution_plan_snapshot_id( ep_snapshot) != old_execution_plan_snapshot_id # This previously failed with a check error assert ExternalExecutionPlan(ep_snapshot, historical_pipeline)
def test_debug_snapshot_import(self, storage): from dagster.core.execution.api import create_execution_plan from dagster.core.snap import ( snapshot_from_execution_plan, create_execution_plan_snapshot_id, ) run_id = make_new_run_id() run_to_add = TestRunStorage.build_run(pipeline_name="pipeline_name", run_id=run_id) storage.add_run(run_to_add) pipeline_def = PipelineDefinition(name="some_pipeline", solid_defs=[]) pipeline_snapshot = pipeline_def.get_pipeline_snapshot() pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot) new_pipeline_snapshot_id = f"{pipeline_snapshot_id}-new-snapshot" storage.add_snapshot(pipeline_snapshot, snapshot_id=new_pipeline_snapshot_id) assert not storage.has_snapshot(pipeline_snapshot_id) assert storage.has_snapshot(new_pipeline_snapshot_id) execution_plan = create_execution_plan(pipeline_def) ep_snapshot = snapshot_from_execution_plan(execution_plan, new_pipeline_snapshot_id) ep_snapshot_id = create_execution_plan_snapshot_id(ep_snapshot) new_ep_snapshot_id = f"{ep_snapshot_id}-new-snapshot" storage.add_snapshot(ep_snapshot, snapshot_id=new_ep_snapshot_id) assert not storage.has_snapshot(ep_snapshot_id) assert storage.has_snapshot(new_ep_snapshot_id)
def test_run_created_in_0_7_9_snapshot_id_change(): src_dir = file_relative_path( __file__, "snapshot_0_7_9_shapshot_id_creation_change/sqlite") with copy_directory(src_dir) as test_dir: instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) # run_id = 'e297fa70-49e8-43f8-abfe-1634f02644f6' old_pipeline_snapshot_id = "88528edde2ed64da3c39cca0da8ba2f7586c1a5d" old_execution_plan_snapshot_id = "2246f8e5a10d21e15fbfa3773d7b2d0bc1fa9d3d" historical_pipeline = instance.get_historical_pipeline( old_pipeline_snapshot_id) pipeline_snapshot = historical_pipeline.pipeline_snapshot ep_snapshot = instance.get_execution_plan_snapshot( old_execution_plan_snapshot_id) # It is the pipeline snapshot that changed # Verify that snapshot ids are not equal. This changed in 0.7.10 created_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot) assert created_snapshot_id != old_pipeline_snapshot_id # verify that both are accessible off of the historical pipeline assert historical_pipeline.computed_pipeline_snapshot_id == created_snapshot_id assert historical_pipeline.identifying_pipeline_snapshot_id == old_pipeline_snapshot_id # We also changed execution plan schema in 0.7.11.post1 assert create_execution_plan_snapshot_id( ep_snapshot) != old_execution_plan_snapshot_id # This previously failed with a check error assert ExternalExecutionPlan(ep_snapshot)
def add_pipeline_snapshot(self, pipeline_snapshot: PipelineSnapshot, snapshot_id: Optional[str] = None) -> str: check.inst_param(pipeline_snapshot, "pipeline_snapshot", PipelineSnapshot) check.opt_str_param(snapshot_id, "snapshot_id") if not snapshot_id: snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot) self._pipeline_snapshots[snapshot_id] = pipeline_snapshot return snapshot_id
def test_fetch_by_snapshot_id(self, storage): assert storage pipeline_def_a = PipelineDefinition(name="some_pipeline", solid_defs=[]) pipeline_def_b = PipelineDefinition(name="some_other_pipeline", solid_defs=[]) pipeline_snapshot_a = pipeline_def_a.get_pipeline_snapshot() pipeline_snapshot_b = pipeline_def_b.get_pipeline_snapshot() pipeline_snapshot_a_id = create_pipeline_snapshot_id( pipeline_snapshot_a) pipeline_snapshot_b_id = create_pipeline_snapshot_id( pipeline_snapshot_b) assert storage.add_pipeline_snapshot( pipeline_snapshot_a) == pipeline_snapshot_a_id assert storage.add_pipeline_snapshot( pipeline_snapshot_b) == pipeline_snapshot_b_id one = make_new_run_id() two = make_new_run_id() storage.add_run( TestRunStorage.build_run( run_id=one, pipeline_name="some_pipeline", pipeline_snapshot_id=pipeline_snapshot_a_id, )) storage.add_run( TestRunStorage.build_run( run_id=two, pipeline_name="some_other_pipeline", pipeline_snapshot_id=pipeline_snapshot_b_id, )) assert len(storage.get_runs()) == 2 runs_a = storage.get_runs( PipelineRunsFilter(snapshot_id=pipeline_snapshot_a_id)) assert len(runs_a) == 1 assert runs_a[0].run_id == one runs_b = storage.get_runs( PipelineRunsFilter(snapshot_id=pipeline_snapshot_b_id)) assert len(runs_b) == 1 assert runs_b[0].run_id == two
def test_empty_pipeline_snap_props(snapshot): pipeline_snapshot = PipelineSnapshot.from_pipeline_def(get_noop_pipeline()) assert pipeline_snapshot.name == 'noop_pipeline' assert pipeline_snapshot.description is None assert pipeline_snapshot.tags == {} assert pipeline_snapshot == serialize_rt(pipeline_snapshot) snapshot.assert_match(serialize_pp(pipeline_snapshot)) snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def add_pipeline_snapshot( self, pipeline_snapshot: PipelineSnapshot, snapshot_id: Optional[str] = None ) -> str: check.inst_param(pipeline_snapshot, "pipeline_snapshot", PipelineSnapshot) check.opt_str_param(snapshot_id, "snapshot_id") if not snapshot_id: snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot) return self._add_snapshot( snapshot_id=snapshot_id, snapshot_obj=pipeline_snapshot, snapshot_type=SnapshotType.PIPELINE, )
def test_add_get_snapshot(self, storage): pipeline_def = PipelineDefinition(name="some_pipeline", solid_defs=[]) pipeline_snapshot = pipeline_def.get_pipeline_snapshot() pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot) assert storage.add_pipeline_snapshot(pipeline_snapshot) == pipeline_snapshot_id fetched_pipeline_snapshot = storage.get_pipeline_snapshot(pipeline_snapshot_id) assert fetched_pipeline_snapshot assert serialize_pp(fetched_pipeline_snapshot) == serialize_pp(pipeline_snapshot) assert storage.has_pipeline_snapshot(pipeline_snapshot_id) assert not storage.has_pipeline_snapshot("nope") storage.wipe() assert not storage.has_pipeline_snapshot(pipeline_snapshot_id)
def test_create_pipeline_snapshot(): @solid def noop_solid(_): pass @pipeline def noop_pipeline(): noop_solid() with instance_for_test() as instance: result = execute_pipeline(noop_pipeline, instance=instance) assert result.success run = instance.get_run_by_id(result.run_id) assert run.pipeline_snapshot_id == create_pipeline_snapshot_id( noop_pipeline.get_pipeline_snapshot())
def test_create_noop_execution_plan_with_tags(snapshot): @solid(tags={'foo': 'bar', 'bar': 'baaz'}) def noop_solid(_): pass @pipeline def noop_pipeline(): noop_solid() execution_plan = create_execution_plan(noop_pipeline) snapshot.assert_match( serialize_pp( snapshot_from_execution_plan( execution_plan, create_pipeline_snapshot_id( noop_pipeline.get_pipeline_snapshot()))))
def test_create_noop_execution_plan(snapshot): @solid def noop_solid(_): pass @pipeline def noop_pipeline(): noop_solid() execution_plan = create_execution_plan(noop_pipeline) snapshot.assert_match( serialize_pp( snapshot_from_execution_plan( execution_plan, create_pipeline_snapshot_id( noop_pipeline.get_pipeline_snapshot()))))
def test_create_pipeline_snapshot(): @solid def noop_solid(_): pass @pipeline def noop_pipeline(): noop_solid() instance = DagsterInstance.local_temp() result = execute_pipeline(noop_pipeline, instance=instance) assert result.success run = instance.get_run_by_id(result.run_id) assert run.pipeline_snapshot_id == create_pipeline_snapshot_id( noop_pipeline.get_pipeline_snapshot())
def test_basic_dep_fan_out(snapshot): @solid def return_one(_): return 1 @solid(input_defs=[InputDefinition("value", int)]) def passthrough(_, value): return value @pipeline def single_dep_pipeline(): return_one_result = return_one() passthrough.alias("passone")(return_one_result) passthrough.alias("passtwo")(return_one_result) dep_structure_snapshot = build_dep_structure_snapshot_from_icontains_solids( single_dep_pipeline.graph ) index = DependencyStructureIndex(dep_structure_snapshot) assert index.get_invocation("return_one") assert index.get_invocation("passone") assert index.get_invocation("passtwo") assert index.get_upstream_output("passone", "value") == OutputHandleSnap("return_one", "result") assert index.get_upstream_output("passtwo", "value") == OutputHandleSnap("return_one", "result") assert set(index.get_downstream_inputs("return_one", "result")) == set( [ InputHandle("passthrough", "passone", "value"), InputHandle("passthrough", "passtwo", "value"), ] ) assert ( deserialize_json_to_dagster_namedtuple(serialize_dagster_namedtuple(dep_structure_snapshot)) == dep_structure_snapshot ) pipeline_snapshot = PipelineSnapshot.from_pipeline_def(single_dep_pipeline) assert pipeline_snapshot == serialize_rt(pipeline_snapshot) snapshot.assert_match(serialize_pp(pipeline_snapshot)) snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def __init__(self, pipeline_snapshot, parent_pipeline_snapshot, is_historical=False): self.pipeline_snapshot = check.inst_param( pipeline_snapshot, "pipeline_snapshot", PipelineSnapshot ) self.parent_pipeline_snapshot = check.opt_inst_param( parent_pipeline_snapshot, "parent_pipeline_snapshot", PipelineSnapshot ) if self.pipeline_snapshot.lineage_snapshot: check.invariant( self.parent_pipeline_snapshot is not None, "Can not create PipelineIndex for pipeline_snapshot with lineage without parent_pipeline_snapshot", ) self._node_defs_snaps_index = { sd.name: sd for sd in pipeline_snapshot.solid_definitions_snapshot.solid_def_snaps + pipeline_snapshot.solid_definitions_snapshot.composite_solid_def_snaps } self._dagster_type_snaps_by_name_index = { dagster_type_snap.name: dagster_type_snap for dagster_type_snap in pipeline_snapshot.dagster_type_namespace_snapshot.all_dagster_type_snaps_by_key.values() if dagster_type_snap.name } self.dep_structure_index = DependencyStructureIndex( pipeline_snapshot.dep_structure_snapshot ) self._comp_dep_structures = { comp_snap.name: DependencyStructureIndex(comp_snap.dep_structure_snapshot) for comp_snap in pipeline_snapshot.solid_definitions_snapshot.composite_solid_def_snaps } if is_historical: # defer the pipeline snapshot calculation for historical pipelines. This tends to be an # expensive operation, so we want to avoid it unless we need it. Also, because this is # a historical pipeline, we already have an identifying pipeline snapshot id (which may # or may not be the same as this calculated snapshot id). The identifying pipeline # snapshot id is the calculated snapshot id at the time that the run was created. self._pipeline_snapshot_id = None else: self._pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot)
def test_pipeline_snap_all_props(snapshot): @solid def noop_solid(_): pass @pipeline(description='desc', tags={'key': 'value'}) def noop_pipeline(): noop_solid() pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline) assert pipeline_snapshot.name == 'noop_pipeline' assert pipeline_snapshot.description == 'desc' assert pipeline_snapshot.tags == {'key': 'value'} assert pipeline_snapshot == serialize_rt(pipeline_snapshot) snapshot.assert_match(serialize_pp(pipeline_snapshot)) snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def test_pipeline_snap_all_props(snapshot): @solid def noop_solid(_): pass @pipeline(description="desc", tags={"key": "value"}) def noop_pipeline(): noop_solid() pipeline_snapshot = PipelineSnapshot.from_pipeline_def(noop_pipeline) assert pipeline_snapshot.name == "noop_pipeline" assert pipeline_snapshot.description == "desc" assert pipeline_snapshot.tags == {"key": "value"} assert pipeline_snapshot == serialize_rt(pipeline_snapshot) snapshot.assert_match(serialize_pp(pipeline_snapshot)) snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def test_two_invocations_deps_snap(snapshot): @solid def noop_solid(_): pass @pipeline def two_solid_pipeline(): noop_solid.alias('one')() noop_solid.alias('two')() index = DependencyStructureIndex( build_dep_structure_snapshot_from_icontains_solids(two_solid_pipeline)) assert index.get_invocation('one') assert index.get_invocation('two') pipeline_snapshot = PipelineSnapshot.from_pipeline_def(two_solid_pipeline) assert pipeline_snapshot == serialize_rt(pipeline_snapshot) snapshot.assert_match(serialize_pp(pipeline_snapshot)) snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def test_create_execution_plan_with_dep(snapshot): @solid def solid_one(_): return 1 @solid def solid_two(_, num): return num + 1 @pipeline def noop_pipeline(): solid_two(solid_one()) execution_plan = create_execution_plan(noop_pipeline) snapshot.assert_match( serialize_pp( snapshot_from_execution_plan( execution_plan, create_pipeline_snapshot_id( noop_pipeline.get_pipeline_snapshot()))))
def test_create_with_composite(snapshot): @solid(output_defs=[OutputDefinition(name='out_num', dagster_type=int)]) def return_one(_): return 1 @solid( input_defs=[InputDefinition(name='num', dagster_type=int)], output_defs=[OutputDefinition(int)], ) def add_one(_, num): return num + 1 @composite_solid( output_defs=[OutputDefinition(name='named_output', dagster_type=int)]) def comp_1(): return add_one(return_one()) @composite_solid( output_defs=[OutputDefinition(name='named_output', dagster_type=int)]) def comp_2(): return add_one(return_one()) @solid def add(_, num_one, num_two): return num_one + num_two @pipeline def do_comps(): add(num_one=comp_1(), num_two=comp_2()) execution_plan = create_execution_plan(do_comps) snapshot.assert_match( serialize_pp( snapshot_from_execution_plan( execution_plan, create_pipeline_snapshot_id( do_comps.get_pipeline_snapshot()))))
def test_basic_fan_in(snapshot): @solid(output_defs=[OutputDefinition(Nothing)]) def return_nothing(_): return None @solid(input_defs=[InputDefinition('nothing', Nothing)]) def take_nothings(_): return None @pipeline def fan_in_test(): take_nothings([ return_nothing.alias('nothing_one')(), return_nothing.alias('nothing_two')() ]) dep_structure_snapshot = build_dep_structure_snapshot_from_icontains_solids( fan_in_test) index = DependencyStructureIndex(dep_structure_snapshot) assert index.get_invocation('nothing_one') assert index.get_invocation('take_nothings') assert index.get_upstream_outputs('take_nothings', 'nothing') == [ OutputHandleSnap('nothing_one', 'result'), OutputHandleSnap('nothing_two', 'result'), ] assert (deserialize_json_to_dagster_namedtuple( serialize_dagster_namedtuple(dep_structure_snapshot)) == dep_structure_snapshot) pipeline_snapshot = PipelineSnapshot.from_pipeline_def(fan_in_test) assert pipeline_snapshot == serialize_rt(pipeline_snapshot) snapshot.assert_match(serialize_pp(pipeline_snapshot)) snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def _construct_run_with_snapshots( self, pipeline_name, run_id, environment_dict, mode, solid_subset, step_keys_to_execute, status, tags, root_run_id, parent_run_id, pipeline_snapshot, execution_plan_snapshot, parent_pipeline_snapshot, ): # https://github.com/dagster-io/dagster/issues/2403 if tags and IS_AIRFLOW_INGEST_PIPELINE_STR in tags: if AIRFLOW_EXECUTION_DATE_STR not in tags: tags[AIRFLOW_EXECUTION_DATE_STR] = get_current_datetime_in_utc( ).isoformat() pipeline_run = PipelineRun( pipeline_name=pipeline_name, run_id=run_id, environment_dict=environment_dict, mode=mode, solid_subset=solid_subset, step_keys_to_execute=step_keys_to_execute, status=status, tags=tags, root_run_id=root_run_id, parent_run_id=parent_run_id, ) if pipeline_snapshot is not None: from dagster.core.snap import create_pipeline_snapshot_id if pipeline_snapshot.lineage_snapshot: if not self._run_storage.has_pipeline_snapshot( pipeline_snapshot.lineage_snapshot.parent_snapshot_id): check.invariant( create_pipeline_snapshot_id( parent_pipeline_snapshot) == pipeline_snapshot.lineage_snapshot.parent_snapshot_id, 'Parent pipeline snapshot id out of sync with passed parent pipeline snapshot', ) returned_pipeline_snapshot_id = self._run_storage.add_pipeline_snapshot( parent_pipeline_snapshot) check.invariant( pipeline_snapshot.lineage_snapshot.parent_snapshot_id == returned_pipeline_snapshot_id) pipeline_snapshot_id = create_pipeline_snapshot_id( pipeline_snapshot) if not self._run_storage.has_pipeline_snapshot( pipeline_snapshot_id): returned_pipeline_snapshot_id = self._run_storage.add_pipeline_snapshot( pipeline_snapshot) check.invariant( pipeline_snapshot_id == returned_pipeline_snapshot_id) pipeline_run = pipeline_run.with_pipeline_snapshot_id( pipeline_snapshot_id) if execution_plan_snapshot is not None: from dagster.core.snap import create_execution_plan_snapshot_id check.invariant(execution_plan_snapshot.pipeline_snapshot_id == pipeline_snapshot_id) check.invariant( set(step_keys_to_execute) == set( execution_plan_snapshot.step_keys_to_execute) if step_keys_to_execute else set( execution_plan_snapshot.step_keys_to_execute) == set( [step.key for step in execution_plan_snapshot.steps]), 'We encode step_keys_to_execute twice in our stack, unfortunately. This check ' 'ensures that they are consistent. We check that step_keys_to_execute in the plan ' 'matches the step_keys_to_execute params if it is set. If it is not, this indicates ' 'a full execution plan, and so we verify that.', ) execution_plan_snapshot_id = create_execution_plan_snapshot_id( execution_plan_snapshot) if not self._run_storage.has_execution_plan_snapshot( execution_plan_snapshot_id): returned_execution_plan_snapshot_id = self._run_storage.add_execution_plan_snapshot( execution_plan_snapshot) check.invariant(execution_plan_snapshot_id == returned_execution_plan_snapshot_id) pipeline_run = pipeline_run.with_execution_plan_snapshot_id( execution_plan_snapshot_id) return pipeline_run
def add_pipeline_snapshot(self, pipeline_snapshot): check.inst_param(pipeline_snapshot, "pipeline_snapshot", PipelineSnapshot) pipeline_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot) self._pipeline_snapshots[pipeline_snapshot_id] = pipeline_snapshot return pipeline_snapshot_id
def _construct_run_with_snapshots( self, pipeline_name=None, run_id=None, environment_dict=None, mode=None, solid_subset=None, step_keys_to_execute=None, status=None, tags=None, root_run_id=None, parent_run_id=None, pipeline_snapshot=None, execution_plan_snapshot=None, ): if tags and IS_AIRFLOW_INGEST_PIPELINE_STR in tags: if AIRFLOW_EXECUTION_DATE_STR not in tags: tags[AIRFLOW_EXECUTION_DATE_STR] = get_current_datetime_in_utc( ).isoformat() pipeline_run = PipelineRun( pipeline_name=pipeline_name, run_id=run_id, environment_dict=environment_dict, mode=mode, solid_subset=solid_subset, step_keys_to_execute=step_keys_to_execute, status=status, tags=tags, root_run_id=root_run_id, parent_run_id=parent_run_id, ) if pipeline_snapshot is not None: from dagster.core.snap import create_pipeline_snapshot_id pipeline_snapshot_id = create_pipeline_snapshot_id( pipeline_snapshot) if not self._run_storage.has_pipeline_snapshot( pipeline_snapshot_id): returned_pipeline_snapshot_id = self._run_storage.add_pipeline_snapshot( pipeline_snapshot) check.invariant( pipeline_snapshot_id == returned_pipeline_snapshot_id) pipeline_run = pipeline_run.with_pipeline_snapshot_id( pipeline_snapshot_id) if execution_plan_snapshot is not None: from dagster.core.snap import create_execution_plan_snapshot_id check.invariant(execution_plan_snapshot.pipeline_snapshot_id == pipeline_snapshot_id) execution_plan_snapshot_id = create_execution_plan_snapshot_id( execution_plan_snapshot) if not self._run_storage.has_execution_plan_snapshot( execution_plan_snapshot_id): returned_execution_plan_snapshot_id = self._run_storage.add_execution_plan_snapshot( execution_plan_snapshot) check.invariant(execution_plan_snapshot_id == returned_execution_plan_snapshot_id) pipeline_run = pipeline_run.with_execution_plan_snapshot_id( execution_plan_snapshot_id) return pipeline_run
def pid(pipeline_def): return create_pipeline_snapshot_id( PipelineSnapshot.from_pipeline_def(pipeline_def))