def test_disconnected_graphs_adjaceny_lists(): # A <-- B # C <-- D node_a = create_root_solid("A") node_b = create_solid_with_deps("B", node_a) node_c = create_root_solid("C") node_d = create_solid_with_deps("D", node_c) forward_edges, backwards_edges = _do_construct( [node_a, node_b, node_c, node_d], {"B": {"A": DependencyDefinition("A")}, "D": {"C": DependencyDefinition("C")}}, ) assert forward_edges == {"A": {"B"}, "B": set(), "C": {"D"}, "D": set()} assert backwards_edges == {"B": {"A"}, "A": set(), "D": {"C"}, "C": set()}
def test_disconnected_graphs_adjaceny_lists(): # A <-- B # C <-- D node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) node_c = create_root_solid('C') node_d = create_solid_with_deps('D', node_c) forward_edges, backwards_edges = _do_construct( [node_a, node_b, node_c, node_d], {'B': {'A': DependencyDefinition('A')}, 'D': {'C': DependencyDefinition('C')}}, ) assert forward_edges == {'A': {'B'}, 'B': set(), 'C': {'D'}, 'D': set()} assert backwards_edges == {'B': {'A'}, 'A': set(), 'D': {'C'}, 'C': set()}
def create_diamond_solids(): a_source = define_stub_solid('A_source', [input_set('A_input')]) node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) node_c = create_solid_with_deps('C', node_a) node_d = create_solid_with_deps('D', node_b, node_c) return [node_d, node_c, node_b, node_a, a_source]
def test_composite_with_no_output_mappings(): a_source = define_stub_solid('A_source', [input_set('A_input')]) node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) node_c = create_solid_with_deps('C', node_a) node_d = create_solid_with_deps('D', node_b, node_c) @composite_solid def diamond_composite(): a = node_a(a_source()) node_d(B=node_b(a), C=node_c(a)) res = execute_solid(diamond_composite) assert res.success assert res.output_values == {} with pytest.raises( DagsterInvariantViolationError, match=re.escape( 'Output \'result\' not defined in composite solid \'diamond_composite\': no output ' 'mappings were defined. If you were expecting this output to be present, you may be ' 'missing an output_mapping from an inner solid to its enclosing composite solid.' ), ): _ = res.output_value() assert len(res.solid_result_list) == 5
def create_diamond_solids(): a_source = define_stub_solid("A_source", [input_set("A_input")]) node_a = create_root_solid("A") node_b = create_solid_with_deps("B", node_a) node_c = create_solid_with_deps("C", node_a) node_d = create_solid_with_deps("D", node_b, node_c) return [node_d, node_c, node_b, node_a, a_source]
def test_composite_io_mapping(): a_source = define_stub_solid('A_source', [input_set('A_input')]) node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) node_c = create_solid_with_deps('C', node_b) comp_a_inner = CompositeSolidDefinition( name='comp_a_inner', solid_defs=[a_source, node_a], dependencies={'A': { 'A_input': DependencyDefinition('A_source') }}, output_mappings=[OutputDefinition().mapping_from('A')], ) comp_a_outer = CompositeSolidDefinition( name='comp_a_outer', solid_defs=[comp_a_inner], output_mappings=[OutputDefinition().mapping_from('comp_a_inner')], ) comp_bc_inner = CompositeSolidDefinition( name='comp_bc_inner', solid_defs=[node_b, node_c], dependencies={'C': { 'B': DependencyDefinition('B') }}, input_mappings=[ InputDefinition(name='inner_B_in').mapping_to(solid_name='B', input_name='A') ], ) comp_bc_outer = CompositeSolidDefinition( name='comp_bc_outer', solid_defs=[comp_bc_inner], dependencies={}, input_mappings=[ InputDefinition(name='outer_B_in').mapping_to( solid_name='comp_bc_inner', input_name='inner_B_in') ], ) result = execute_pipeline( PipelineDefinition( name='wrapped_io', solid_defs=[comp_a_outer, comp_bc_outer], dependencies={ 'comp_bc_outer': { 'outer_B_in': DependencyDefinition('comp_a_outer') } }, )) assert result.success
def test_single_dep_adjacency_lists(): # A <-- B node_a = create_root_solid("A") node_b = create_solid_with_deps("B", node_a) forward_edges, backwards_edges = _do_construct( [node_a, node_b], {"B": {"A": DependencyDefinition("A")}} ) assert forward_edges == {"A": {"B"}, "B": set()} assert backwards_edges == {"B": {"A"}, "A": set()}
def test_composite_basic_execution(): a_source = define_stub_solid('A_source', [input_set('A_input')]) node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) node_c = create_solid_with_deps('C', node_a) node_d = create_solid_with_deps('D', node_b, node_c) @composite_solid def diamond_composite(): a = node_a(a_source()) node_d(B=node_b(a), C=node_c(a)) @pipeline def test_pipeline_single(): diamond_composite() result = execute_pipeline(test_pipeline_single) assert result.success @pipeline def test_pipeline_double(): diamond_composite.alias('D1')() diamond_composite.alias('D2')() result = execute_pipeline(test_pipeline_double) assert result.success @composite_solid def wrapped_composite(): diamond_composite() @pipeline def test_pipeline_mixed(): diamond_composite() wrapped_composite() result = execute_pipeline(test_pipeline_mixed) assert result.success @composite_solid def empty(): pass @pipeline def test_pipeline_empty(): empty() result = execute_pipeline(test_pipeline_empty) assert result.success nested_pipeline = nesting_composite_pipeline(6, 2) result = execute_pipeline(nested_pipeline) assert result.success
def test_single_dep_adjacency_lists(): # A <-- B node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) forward_edges, backwards_edges = _do_construct( [node_a, node_b], {'B': {'A': DependencyDefinition('A')}} ) assert forward_edges == {'A': {'B'}, 'B': set()} assert backwards_edges == {'B': {'A'}, 'A': set()}
def test_composite_basic_execution(): a_source = define_stub_solid('A_source', [input_set('A_input')]) node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) node_c = create_solid_with_deps('C', node_a) node_d = create_solid_with_deps('D', node_b, node_c) diamond_composite = CompositeSolidDefinition( name='diamond_composite', solid_defs=[a_source, node_a, node_b, node_c, node_d], dependencies={ 'A': { 'A_input': DependencyDefinition('A_source') }, 'B': { 'A': DependencyDefinition('A') }, 'C': { 'A': DependencyDefinition('A') }, 'D': { 'B': DependencyDefinition('B'), 'C': DependencyDefinition('C') }, }, ) result = execute_pipeline( PipelineDefinition(solid_defs=[diamond_composite])) assert result.success result = execute_pipeline( PipelineDefinition( solid_defs=[diamond_composite], dependencies={ SolidInvocation('diamond_composite', alias='D1'): {}, SolidInvocation('diamond_composite', alias='D2'): {}, }, )) assert result.success wrapped_composite = CompositeSolidDefinition( name='wrapped_composite', solid_defs=[diamond_composite]) result = execute_pipeline( PipelineDefinition(solid_defs=[diamond_composite, wrapped_composite])) assert result.success empty_composite = CompositeSolidDefinition(name='empty', solid_defs=[]) result = execute_pipeline(PipelineDefinition(solid_defs=[empty_composite])) assert result.success
def test_composite_basic_execution(composition_decorator): a_source = define_stub_solid("A_source", [input_set("A_input")]) node_a = create_root_solid("A") node_b = create_solid_with_deps("B", node_a) node_c = create_solid_with_deps("C", node_a) node_d = create_solid_with_deps("D", node_b, node_c) @composition_decorator def diamond_composite(): a = node_a(a_source()) node_d(B=node_b(a), C=node_c(a)) res = execute_solid(diamond_composite) assert res.success @pipeline def test_pipeline_double(): diamond_composite.alias("D1")() diamond_composite.alias("D2")() result = execute_pipeline(test_pipeline_double) assert result.success @composition_decorator def wrapped_composite(): diamond_composite() @pipeline def test_pipeline_mixed(): diamond_composite() wrapped_composite() result = execute_pipeline(test_pipeline_mixed) assert result.success @composition_decorator def empty(): pass @pipeline def test_pipeline_empty(): empty() result = execute_pipeline(test_pipeline_empty) assert result.success nested_pipeline = nesting_composite_pipeline(6, 2) result = execute_pipeline(nested_pipeline) assert result.success
def test_composite_io_mapping(): a_source = define_stub_solid("A_source", [input_set("A_input")]) node_a = create_root_solid("A") node_b = create_solid_with_deps("B", node_a) node_c = create_solid_with_deps("C", node_b) comp_a_inner = CompositeSolidDefinition( name="comp_a_inner", solid_defs=[a_source, node_a], dependencies={"A": { "A_input": DependencyDefinition("A_source") }}, output_mappings=[OutputDefinition().mapping_from("A")], ) comp_a_outer = CompositeSolidDefinition( name="comp_a_outer", solid_defs=[comp_a_inner], output_mappings=[OutputDefinition().mapping_from("comp_a_inner")], ) comp_bc_inner = CompositeSolidDefinition( name="comp_bc_inner", solid_defs=[node_b, node_c], dependencies={"C": { "B": DependencyDefinition("B") }}, input_mappings=[ InputDefinition(name="inner_B_in").mapping_to(solid_name="B", input_name="A") ], ) comp_bc_outer = CompositeSolidDefinition( name="comp_bc_outer", solid_defs=[comp_bc_inner], dependencies={}, input_mappings=[ InputDefinition(name="outer_B_in").mapping_to( solid_name="comp_bc_inner", input_name="inner_B_in") ], ) @pipeline def wrapped_io(): comp_bc_outer(comp_a_outer()) result = execute_pipeline(wrapped_io) assert result.success
def test_execute_aliased_solid_in_diamond(): a_source = define_stub_solid('A_source', [input_set('A_input')]) pipeline_def = PipelineDefinition( name='aliased_pipeline', solid_defs=[a_source, create_root_solid('A')], dependencies={ SolidInvocation('A', alias='aliased'): {'A_input': DependencyDefinition(a_source.name)} }, ) solid_result = execute_solid_within_pipeline( pipeline_def, 'aliased', inputs={'A_input': [{'a key': 'a value'}]} ) assert solid_result.success assert solid_result.result_value() == [{'a key': 'a value'}, {'aliased': 'transform_called'}]
def test_empty_adjaceny_lists(): solids = [create_root_solid('a_node')] forward_edges, backwards_edges = _do_construct(solids, {}) assert forward_edges == {'a_node': set()} assert backwards_edges == {'a_node': set()}
def aliased_pipeline(): create_root_solid("A").alias("aliased")(a_source())
def aliased_pipeline(): create_root_solid('A').alias('aliased')(a_source())