def dask_composite_pipeline():
return nesting_composite_pipeline(
6,
2,
mode_defs=[
ModeDefinition(executor_defs=default_executors + [dask_executor])
])
def test_composite_basic_execution():
a_source = define_stub_solid('A_source', [input_set('A_input')])
node_a = create_root_solid('A')
node_b = create_solid_with_deps('B', node_a)
node_c = create_solid_with_deps('C', node_a)
node_d = create_solid_with_deps('D', node_b, node_c)
@composite_solid
def diamond_composite():
a = node_a(a_source())
node_d(B=node_b(a), C=node_c(a))
@pipeline
def test_pipeline_single():
diamond_composite()
result = execute_pipeline(test_pipeline_single)
assert result.success
@pipeline
def test_pipeline_double():
diamond_composite.alias('D1')()
diamond_composite.alias('D2')()
result = execute_pipeline(test_pipeline_double)
assert result.success
@composite_solid
def wrapped_composite():
diamond_composite()
@pipeline
def test_pipeline_mixed():
diamond_composite()
wrapped_composite()
result = execute_pipeline(test_pipeline_mixed)
assert result.success
@composite_solid
def empty():
pass
@pipeline
def test_pipeline_empty():
empty()
result = execute_pipeline(test_pipeline_empty)
assert result.success
nested_pipeline = nesting_composite_pipeline(6, 2)
result = execute_pipeline(nested_pipeline)
assert result.success
def dask_composite_pipeline():
return nesting_composite_pipeline(
6,
2,
mode_defs=[
ModeDefinition(
resource_defs={"io_manager": fs_io_manager},
executor_defs=default_executors + [dask_executor],
)
],
)
def test_composite_basic_execution(composition_decorator):
a_source = define_stub_solid("A_source", [input_set("A_input")])
node_a = create_root_solid("A")
node_b = create_solid_with_deps("B", node_a)
node_c = create_solid_with_deps("C", node_a)
node_d = create_solid_with_deps("D", node_b, node_c)
@composition_decorator
def diamond_composite():
a = node_a(a_source())
node_d(B=node_b(a), C=node_c(a))
res = execute_solid(diamond_composite)
assert res.success
@pipeline
def test_pipeline_double():
diamond_composite.alias("D1")()
diamond_composite.alias("D2")()
result = execute_pipeline(test_pipeline_double)
assert result.success
@composition_decorator
def wrapped_composite():
diamond_composite()
@pipeline
def test_pipeline_mixed():
diamond_composite()
wrapped_composite()
result = execute_pipeline(test_pipeline_mixed)
assert result.success
@composition_decorator
def empty():
pass
@pipeline
def test_pipeline_empty():
empty()
result = execute_pipeline(test_pipeline_empty)
assert result.success
nested_pipeline = nesting_composite_pipeline(6, 2)
result = execute_pipeline(nested_pipeline)
assert result.success
def test_execute_nested_composite_solids():
nested_composite_pipeline = nesting_composite_pipeline(2, 2)
nested_composite_solid = nested_composite_pipeline.solids[0].definition
res = execute_solid(nested_composite_solid)
assert res.success
assert res.solid.name == 'wrap'
assert res.output_values == {}
with pytest.raises(
DagsterInvariantViolationError,
match=re.escape(
'Output \'result\' not defined in composite solid \'wrap\': no output mappings were '
'defined. If you were expecting this output to be present, you may be missing an '
'output_mapping from an inner solid to its enclosing composite solid.'
),
):
_ = res.output_value()
assert len(res.solid_result_list) == 2
def composite_pipeline():
return nesting_composite_pipeline(6, 2, mode_defs=celery_mode_defs)
def composite_pipeline():
return nesting_composite_pipeline(COMPOSITE_DEPTH,
2,
mode_defs=celery_mode_defs)
def define_large_pipeline_celery():
return nesting_composite_pipeline(depth=1,
num_children=6,
mode_defs=celery_mode_defs(),
name='large_pipeline_celery')
def dask_composite_pipeline():
return nesting_composite_pipeline(6, 2)
def define_large_pipeline():
return nesting_composite_pipeline(depth=1,
num_children=6,
name='large_pipeline')