def _do_test(pipe):
result = execute_pipeline(pipe)
assert result.result_for_solid("A").output_value() == [
input_set("A_input"),
compute_called("A"),
]
assert result.result_for_solid("B").output_value() == [
input_set("A_input"),
compute_called("A"),
compute_called("B"),
]
assert result.result_for_solid("C").output_value() == [
input_set("A_input"),
compute_called("A"),
compute_called("C"),
]
assert result.result_for_solid("D").output_value() == [
input_set("A_input"),
compute_called("A"),
compute_called("C"),
compute_called("B"),
compute_called("D"),
] or result.result_for_solid("D").output_value() == [
input_set("A_input"),
compute_called("A"),
compute_called("B"),
compute_called("C"),
compute_called("D"),
]
def _do_test(pipeline):
result = execute_pipeline(pipeline)
assert result.result_for_solid('A').result_value() == [
input_set('A_input'),
transform_called('A'),
]
assert result.result_for_solid('B').result_value() == [
input_set('A_input'),
transform_called('A'),
transform_called('B'),
]
assert result.result_for_solid('C').result_value() == [
input_set('A_input'),
transform_called('A'),
transform_called('C'),
]
assert result.result_for_solid('D').result_value() == [
input_set('A_input'),
transform_called('A'),
transform_called('C'),
transform_called('B'),
transform_called('D'),
] or result.result_for_solid('D').result_value() == [
input_set('A_input'),
transform_called('A'),
transform_called('B'),
transform_called('C'),
transform_called('D'),
]
def create_diamond_solids():
a_source = define_stub_solid('A_source', [input_set('A_input')])
node_a = create_root_solid('A')
node_b = create_solid_with_deps('B', node_a)
node_c = create_solid_with_deps('C', node_a)
node_d = create_solid_with_deps('D', node_b, node_c)
return [node_d, node_c, node_b, node_a, a_source]
def test_composite_with_no_output_mappings():
a_source = define_stub_solid('A_source', [input_set('A_input')])
node_a = create_root_solid('A')
node_b = create_solid_with_deps('B', node_a)
node_c = create_solid_with_deps('C', node_a)
node_d = create_solid_with_deps('D', node_b, node_c)
@composite_solid
def diamond_composite():
a = node_a(a_source())
node_d(B=node_b(a), C=node_c(a))
res = execute_solid(diamond_composite)
assert res.success
assert res.output_values == {}
with pytest.raises(
DagsterInvariantViolationError,
match=re.escape(
'Output \'result\' not defined in composite solid \'diamond_composite\': no output '
'mappings were defined. If you were expecting this output to be present, you may be '
'missing an output_mapping from an inner solid to its enclosing composite solid.'
),
):
_ = res.output_value()
assert len(res.solid_result_list) == 5
def create_diamond_solids():
a_source = define_stub_solid("A_source", [input_set("A_input")])
node_a = create_root_solid("A")
node_b = create_solid_with_deps("B", node_a)
node_c = create_solid_with_deps("C", node_a)
node_d = create_solid_with_deps("D", node_b, node_c)
return [node_d, node_c, node_b, node_a, a_source]
def test_composite_io_mapping():
a_source = define_stub_solid('A_source', [input_set('A_input')])
node_a = create_root_solid('A')
node_b = create_solid_with_deps('B', node_a)
node_c = create_solid_with_deps('C', node_b)
comp_a_inner = CompositeSolidDefinition(
name='comp_a_inner',
solid_defs=[a_source, node_a],
dependencies={'A': {
'A_input': DependencyDefinition('A_source')
}},
output_mappings=[OutputDefinition().mapping_from('A')],
)
comp_a_outer = CompositeSolidDefinition(
name='comp_a_outer',
solid_defs=[comp_a_inner],
output_mappings=[OutputDefinition().mapping_from('comp_a_inner')],
)
comp_bc_inner = CompositeSolidDefinition(
name='comp_bc_inner',
solid_defs=[node_b, node_c],
dependencies={'C': {
'B': DependencyDefinition('B')
}},
input_mappings=[
InputDefinition(name='inner_B_in').mapping_to(solid_name='B',
input_name='A')
],
)
comp_bc_outer = CompositeSolidDefinition(
name='comp_bc_outer',
solid_defs=[comp_bc_inner],
dependencies={},
input_mappings=[
InputDefinition(name='outer_B_in').mapping_to(
solid_name='comp_bc_inner', input_name='inner_B_in')
],
)
result = execute_pipeline(
PipelineDefinition(
name='wrapped_io',
solid_defs=[comp_a_outer, comp_bc_outer],
dependencies={
'comp_bc_outer': {
'outer_B_in': DependencyDefinition('comp_a_outer')
}
},
))
assert result.success
def test_composite_basic_execution():
a_source = define_stub_solid('A_source', [input_set('A_input')])
node_a = create_root_solid('A')
node_b = create_solid_with_deps('B', node_a)
node_c = create_solid_with_deps('C', node_a)
node_d = create_solid_with_deps('D', node_b, node_c)
@composite_solid
def diamond_composite():
a = node_a(a_source())
node_d(B=node_b(a), C=node_c(a))
@pipeline
def test_pipeline_single():
diamond_composite()
result = execute_pipeline(test_pipeline_single)
assert result.success
@pipeline
def test_pipeline_double():
diamond_composite.alias('D1')()
diamond_composite.alias('D2')()
result = execute_pipeline(test_pipeline_double)
assert result.success
@composite_solid
def wrapped_composite():
diamond_composite()
@pipeline
def test_pipeline_mixed():
diamond_composite()
wrapped_composite()
result = execute_pipeline(test_pipeline_mixed)
assert result.success
@composite_solid
def empty():
pass
@pipeline
def test_pipeline_empty():
empty()
result = execute_pipeline(test_pipeline_empty)
assert result.success
nested_pipeline = nesting_composite_pipeline(6, 2)
result = execute_pipeline(nested_pipeline)
assert result.success
def test_execute_aliased_solid_in_diamond():
a_source = define_stub_solid("A_source", [input_set("A_input")])
@pipeline
def aliased_pipeline():
create_root_solid("A").alias("aliased")(a_source())
solid_result = execute_solid_within_pipeline(
aliased_pipeline, "aliased", inputs={"A_input": [{"a key": "a value"}]}
)
assert solid_result.success
assert solid_result.output_value() == [{"a key": "a value"}, {"aliased": "compute_called"}]
def test_execute_aliased_solid_in_diamond():
a_source = define_stub_solid('A_source', [input_set('A_input')])
@pipeline
def aliased_pipeline():
create_root_solid('A').alias('aliased')(a_source())
solid_result = execute_solid_within_pipeline(
aliased_pipeline, 'aliased', inputs={'A_input': [{'a key': 'a value'}]}
)
assert solid_result.success
assert solid_result.output_value() == [{'a key': 'a value'}, {'aliased': 'compute_called'}]
def test_composite_basic_execution():
a_source = define_stub_solid('A_source', [input_set('A_input')])
node_a = create_root_solid('A')
node_b = create_solid_with_deps('B', node_a)
node_c = create_solid_with_deps('C', node_a)
node_d = create_solid_with_deps('D', node_b, node_c)
diamond_composite = CompositeSolidDefinition(
name='diamond_composite',
solid_defs=[a_source, node_a, node_b, node_c, node_d],
dependencies={
'A': {
'A_input': DependencyDefinition('A_source')
},
'B': {
'A': DependencyDefinition('A')
},
'C': {
'A': DependencyDefinition('A')
},
'D': {
'B': DependencyDefinition('B'),
'C': DependencyDefinition('C')
},
},
)
result = execute_pipeline(
PipelineDefinition(solid_defs=[diamond_composite]))
assert result.success
result = execute_pipeline(
PipelineDefinition(
solid_defs=[diamond_composite],
dependencies={
SolidInvocation('diamond_composite', alias='D1'): {},
SolidInvocation('diamond_composite', alias='D2'): {},
},
))
assert result.success
wrapped_composite = CompositeSolidDefinition(
name='wrapped_composite', solid_defs=[diamond_composite])
result = execute_pipeline(
PipelineDefinition(solid_defs=[diamond_composite, wrapped_composite]))
assert result.success
empty_composite = CompositeSolidDefinition(name='empty', solid_defs=[])
result = execute_pipeline(PipelineDefinition(solid_defs=[empty_composite]))
assert result.success
def test_composite_basic_execution(composition_decorator):
a_source = define_stub_solid("A_source", [input_set("A_input")])
node_a = create_root_solid("A")
node_b = create_solid_with_deps("B", node_a)
node_c = create_solid_with_deps("C", node_a)
node_d = create_solid_with_deps("D", node_b, node_c)
@composition_decorator
def diamond_composite():
a = node_a(a_source())
node_d(B=node_b(a), C=node_c(a))
res = execute_solid(diamond_composite)
assert res.success
@pipeline
def test_pipeline_double():
diamond_composite.alias("D1")()
diamond_composite.alias("D2")()
result = execute_pipeline(test_pipeline_double)
assert result.success
@composition_decorator
def wrapped_composite():
diamond_composite()
@pipeline
def test_pipeline_mixed():
diamond_composite()
wrapped_composite()
result = execute_pipeline(test_pipeline_mixed)
assert result.success
@composition_decorator
def empty():
pass
@pipeline
def test_pipeline_empty():
empty()
result = execute_pipeline(test_pipeline_empty)
assert result.success
nested_pipeline = nesting_composite_pipeline(6, 2)
result = execute_pipeline(nested_pipeline)
assert result.success
def test_composite_io_mapping():
a_source = define_stub_solid("A_source", [input_set("A_input")])
node_a = create_root_solid("A")
node_b = create_solid_with_deps("B", node_a)
node_c = create_solid_with_deps("C", node_b)
comp_a_inner = CompositeSolidDefinition(
name="comp_a_inner",
solid_defs=[a_source, node_a],
dependencies={"A": {
"A_input": DependencyDefinition("A_source")
}},
output_mappings=[OutputDefinition().mapping_from("A")],
)
comp_a_outer = CompositeSolidDefinition(
name="comp_a_outer",
solid_defs=[comp_a_inner],
output_mappings=[OutputDefinition().mapping_from("comp_a_inner")],
)
comp_bc_inner = CompositeSolidDefinition(
name="comp_bc_inner",
solid_defs=[node_b, node_c],
dependencies={"C": {
"B": DependencyDefinition("B")
}},
input_mappings=[
InputDefinition(name="inner_B_in").mapping_to(solid_name="B",
input_name="A")
],
)
comp_bc_outer = CompositeSolidDefinition(
name="comp_bc_outer",
solid_defs=[comp_bc_inner],
dependencies={},
input_mappings=[
InputDefinition(name="outer_B_in").mapping_to(
solid_name="comp_bc_inner", input_name="inner_B_in")
],
)
@pipeline
def wrapped_io():
comp_bc_outer(comp_a_outer())
result = execute_pipeline(wrapped_io)
assert result.success
def test_execute_aliased_solid_in_diamond():
a_source = define_stub_solid('A_source', [input_set('A_input')])
pipeline_def = PipelineDefinition(
name='aliased_pipeline',
solid_defs=[a_source, create_root_solid('A')],
dependencies={
SolidInvocation('A', alias='aliased'): {'A_input': DependencyDefinition(a_source.name)}
},
)
solid_result = execute_solid_within_pipeline(
pipeline_def, 'aliased', inputs={'A_input': [{'a key': 'a value'}]}
)
assert solid_result.success
assert solid_result.result_value() == [{'a key': 'a value'}, {'aliased': 'transform_called'}]
