def test_noop_deps_snap(): @solid def noop_solid(_): pass @pipeline def noop_pipeline(): noop_solid() invocations = build_dep_structure_snapshot_from_icontains_solids( noop_pipeline).solid_invocation_snaps assert len(invocations) == 1 assert isinstance(invocations[0], SolidInvocationSnap)
def test_basic_dep_fan_out(snapshot): @solid def return_one(_): return 1 @solid(input_defs=[InputDefinition("value", int)]) def passthrough(_, value): return value @pipeline def single_dep_pipeline(): return_one_result = return_one() passthrough.alias("passone")(return_one_result) passthrough.alias("passtwo")(return_one_result) dep_structure_snapshot = build_dep_structure_snapshot_from_icontains_solids( single_dep_pipeline.graph ) index = DependencyStructureIndex(dep_structure_snapshot) assert index.get_invocation("return_one") assert index.get_invocation("passone") assert index.get_invocation("passtwo") assert index.get_upstream_output("passone", "value") == OutputHandleSnap("return_one", "result") assert index.get_upstream_output("passtwo", "value") == OutputHandleSnap("return_one", "result") assert set(index.get_downstream_inputs("return_one", "result")) == set( [ InputHandle("passthrough", "passone", "value"), InputHandle("passthrough", "passtwo", "value"), ] ) assert ( deserialize_json_to_dagster_namedtuple(serialize_dagster_namedtuple(dep_structure_snapshot)) == dep_structure_snapshot ) pipeline_snapshot = PipelineSnapshot.from_pipeline_def(single_dep_pipeline) assert pipeline_snapshot == serialize_rt(pipeline_snapshot) snapshot.assert_match(serialize_pp(pipeline_snapshot)) snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def test_two_invocations_deps_snap(snapshot): @solid def noop_solid(_): pass @pipeline def two_solid_pipeline(): noop_solid.alias('one')() noop_solid.alias('two')() index = DependencyStructureIndex( build_dep_structure_snapshot_from_icontains_solids(two_solid_pipeline)) assert index.get_invocation('one') assert index.get_invocation('two') pipeline_snapshot = PipelineSnapshot.from_pipeline_def(two_solid_pipeline) assert pipeline_snapshot == serialize_rt(pipeline_snapshot) snapshot.assert_match(serialize_pp(pipeline_snapshot)) snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def test_multiple_non_argument_deps(): @asset def foo(): pass @asset(namespace="namespace") def bar(): pass @asset def baz(): return 1 @asset(non_argument_deps={AssetKey("foo"), AssetKey(["namespace", "bar"])}) def qux(baz): return baz job = build_assets_job("a", [foo, bar, baz, qux]) dep_structure_snapshot = build_dep_structure_snapshot_from_icontains_solids( job.graph) index = DependencyStructureIndex(dep_structure_snapshot) assert index.get_invocation("foo") assert index.get_invocation("namespace__bar") assert index.get_invocation("baz") assert index.get_upstream_outputs("qux", "foo") == [ OutputHandleSnap("foo", "result"), ] assert index.get_upstream_outputs("qux", "namespace_bar") == [ OutputHandleSnap("namespace__bar", "result") ] assert index.get_upstream_outputs( "qux", "baz") == [OutputHandleSnap("baz", "result")] result = job.execute_in_process() assert result.success assert result.output_for_node("qux") == 1
def test_basic_fan_in(snapshot): @solid(output_defs=[OutputDefinition(Nothing)]) def return_nothing(_): return None @solid(input_defs=[InputDefinition('nothing', Nothing)]) def take_nothings(_): return None @pipeline def fan_in_test(): take_nothings([ return_nothing.alias('nothing_one')(), return_nothing.alias('nothing_two')() ]) dep_structure_snapshot = build_dep_structure_snapshot_from_icontains_solids( fan_in_test) index = DependencyStructureIndex(dep_structure_snapshot) assert index.get_invocation('nothing_one') assert index.get_invocation('take_nothings') assert index.get_upstream_outputs('take_nothings', 'nothing') == [ OutputHandleSnap('nothing_one', 'result'), OutputHandleSnap('nothing_two', 'result'), ] assert (deserialize_json_to_dagster_namedtuple( serialize_dagster_namedtuple(dep_structure_snapshot)) == dep_structure_snapshot) pipeline_snapshot = PipelineSnapshot.from_pipeline_def(fan_in_test) assert pipeline_snapshot == serialize_rt(pipeline_snapshot) snapshot.assert_match(serialize_pp(pipeline_snapshot)) snapshot.assert_match(create_pipeline_snapshot_id(pipeline_snapshot))
def test_basic_dep(): @solid def return_one(_): return 1 @solid(input_defs=[InputDefinition('value', int)]) def passthrough(_, value): return value @pipeline def single_dep_pipeline(): passthrough(return_one()) index = DependencyStructureIndex( build_dep_structure_snapshot_from_icontains_solids( single_dep_pipeline)) assert index.get_invocation('return_one') assert index.get_invocation('passthrough') outputs = index.get_upstream_outputs('passthrough', 'value') assert len(outputs) == 1 assert outputs[0].solid_name == 'return_one' assert outputs[0].output_name == 'result'