def test_create_single_solid_pipeline_with_alias():
a_source = define_stub_solid('A_source', [input_set('A_input')])
stub_solid = define_stub_solid('stub', [{'a_key': 'stubbed_thing'}])
single_solid_pipeline = PipelineDefinition.create_single_solid_pipeline(
PipelineDefinition(
solids=[a_source, create_root_solid('A')],
dependencies={
SolidInstance('A', alias='aliased'): {
'A_input': DependencyDefinition(a_source.name)
},
},
),
'aliased',
{
'aliased': {
'A_input': stub_solid,
},
},
)
result = execute_pipeline(single_solid_pipeline)
assert result.success
expected = [{'a_key': 'stubbed_thing'}, {'A': 'transform_called'}]
assert result.result_for_solid('aliased').transformed_value() == expected
def test_two_root_solid_pipeline_with_empty_dependency_definition():
stub_solid_a = define_stub_solid('stub_a', [{'a key': 'a value'}])
stub_solid_b = define_stub_solid('stub_b', [{'a key': 'a value'}])
single_solid_pipeline = PipelineDefinition(solids=[stub_solid_a, stub_solid_b], dependencies={})
result = execute_pipeline(single_solid_pipeline)
assert result.success
def test_two_root_solid_pipeline_with_partial_dependency_definition():
stub_solid_a = define_stub_solid("stub_a", [{"a key": "a value"}])
stub_solid_b = define_stub_solid("stub_b", [{"a key": "a value"}])
single_dep_pipe = PipelineDefinition(
solid_defs=[stub_solid_a, stub_solid_b], dependencies={"stub_a": {}})
assert execute_pipeline(single_dep_pipe).success
def test_two_root_solid_pipeline_with_partial_dependency_definition():
stub_solid_a = define_stub_solid('stub_a', [{'a key': 'a value'}])
stub_solid_b = define_stub_solid('stub_b', [{'a key': 'a value'}])
single_dep_pipe = PipelineDefinition(
solid_defs=[stub_solid_a, stub_solid_b], dependencies={'stub_a': {}})
assert execute_pipeline(single_dep_pipe).success
def test_two_root_solid_pipeline_with_empty_dependency_definition():
stub_solid_a = define_stub_solid('stub_a', [{'a key': 'a value'}])
stub_solid_b = define_stub_solid('stub_b', [{'a key': 'a value'}])
@pipeline
def pipe():
stub_solid_a()
stub_solid_b()
assert execute_pipeline(pipe).success
def test_two_root_solid_pipeline_with_empty_dependency_definition():
stub_solid_a = define_stub_solid("stub_a", [{"a key": "a value"}])
stub_solid_b = define_stub_solid("stub_b", [{"a key": "a value"}])
@pipeline
def pipe():
stub_solid_a()
stub_solid_b()
assert execute_pipeline(pipe).success
def create_diamond_solids():
a_source = define_stub_solid("A_source", [input_set("A_input")])
node_a = create_root_solid("A")
node_b = create_solid_with_deps("B", node_a)
node_c = create_solid_with_deps("C", node_a)
node_d = create_solid_with_deps("D", node_b, node_c)
return [node_d, node_c, node_b, node_a, a_source]
def create_sum_sq_pipeline(context, expr, extra_solids=None, extra_deps=None):
check.inst_param(expr, 'expr', DagsterSqlTableExpression)
expr_solid = define_stub_solid('expr', expr)
sum_solid = create_sum_table_solid()
sum_sq_solid = create_sql_solid(
name='sum_sq_table',
inputs=[InputDefinition(sum_solid.name)],
sql_text='SELECT num1, num2, sum, sum * sum as sum_sq from {sum_table}',
)
dependencies = {
sum_solid.name: {
'num_table': DependencyDefinition('expr')
},
sum_sq_solid.name: {
sum_solid.name: DependencyDefinition(sum_solid.name)
},
}
if extra_deps:
dependencies.update(extra_deps)
return pipeline_test_def(
solids=[expr_solid, sum_solid, sum_sq_solid] +
(extra_solids if extra_solids else []),
context=context,
dependencies=dependencies,
)
def test_composite_with_no_output_mappings():
a_source = define_stub_solid('A_source', [input_set('A_input')])
node_a = create_root_solid('A')
node_b = create_solid_with_deps('B', node_a)
node_c = create_solid_with_deps('C', node_a)
node_d = create_solid_with_deps('D', node_b, node_c)
@composite_solid
def diamond_composite():
a = node_a(a_source())
node_d(B=node_b(a), C=node_c(a))
res = execute_solid(diamond_composite)
assert res.success
assert res.output_values == {}
with pytest.raises(
DagsterInvariantViolationError,
match=re.escape(
'Output \'result\' not defined in composite solid \'diamond_composite\': no output '
'mappings were defined. If you were expecting this output to be present, you may be '
'missing an output_mapping from an inner solid to its enclosing composite solid.'
),
):
_ = res.output_value()
assert len(res.solid_result_list) == 5
def create_diamond_solids():
a_source = define_stub_solid('A_source', [input_set('A_input')])
node_a = create_root_solid('A')
node_b = create_solid_with_deps('B', node_a)
node_c = create_solid_with_deps('C', node_a)
node_d = create_solid_with_deps('D', node_b, node_c)
return [node_d, node_c, node_b, node_a, a_source]
def test_execute_aliased_solid_in_diamond():
a_source = define_stub_solid('A_source', [input_set('A_input')])
pipeline_def = PipelineDefinition(
name='aliased_pipeline',
solids=[a_source, create_root_solid('A')],
dependencies={
SolidInstance('A', alias='aliased'): {
'A_input': DependencyDefinition(a_source.name)
}
},
)
solid_result = execute_solid(pipeline_def,
'aliased',
inputs={'A_input': [{
'a key': 'a value'
}]})
assert solid_result.success
assert solid_result.transformed_value() == [
{
'a key': 'a value'
},
{
'aliased': 'transform_called'
},
]
def test_singleton_pipeline():
stub_solid = define_stub_solid('stub', [{'a key': 'a value'}])
single_solid_pipeline = PipelineDefinition(solid_defs=[stub_solid],
dependencies={})
result = execute_pipeline(single_solid_pipeline)
assert result.success
def test_pipeline_init_failure():
stub_solid = define_stub_solid('stub', None)
env_config = {}
def failing_resource_fn(*args, **kwargs):
raise Exception()
pipeline_def = PipelineDefinition(
[stub_solid],
'failing_init_pipeline',
mode_definitions=[
ModeDefinition(
resources={
'failing': ResourceDefinition(
resource_fn=failing_resource_fn)
})
],
)
result = execute_pipeline(
pipeline_def,
environment_dict=env_config,
run_config=RunConfig(executor_config=InProcessExecutorConfig(
raise_on_error=False)),
)
assert result.success is False
assert len(result.event_list) == 1
event = result.event_list[0]
assert event.event_type_value == 'PIPELINE_INIT_FAILURE'
assert event.pipeline_init_failure_data
def define_pandas_input_transform_test_pipeline():
in_df = pd.DataFrame({'num': [3, 5, 7]})
return PipelineDefinition(
name='input_transform_test_pipeline',
solids=[define_stub_solid('load_df', in_df), define_pandas_input_transform_test_solid()],
dependencies={'pandas_input_transform_test': {'df': DependencyDefinition('load_df')}},
)
def test_pipeline_init_failure():
stub_solid = define_stub_solid('stub', None)
env_config = {}
def failing_resource_fn(*args, **kwargs):
raise Exception()
@pipeline(mode_defs=[
ModeDefinition(
resource_defs={
'failing': ResourceDefinition(resource_fn=failing_resource_fn)
})
])
def failing_init_pipeline():
stub_solid()
result = execute_pipeline(failing_init_pipeline,
environment_dict=dict(env_config),
raise_on_error=False)
assert result.success is False
assert len(result.event_list) == 1
event = result.event_list[0]
assert event.event_type_value == 'PIPELINE_INIT_FAILURE'
assert event.pipeline_init_failure_data
def test_singleton_pipeline():
stub_solid = define_stub_solid('stub', [{'a key': 'a value'}])
@pipeline
def single_solid_pipeline():
stub_solid()
assert execute_pipeline(single_solid_pipeline).success
def test_create_pipeline_with_bad_solids_list():
with pytest.raises(
DagsterInvalidDefinitionError,
match='"solids" arg to pipeline "a_pipeline" is not a list. Got',
):
PipelineDefinition(
name='a_pipeline', solid_defs=define_stub_solid('stub', [{'a key': 'a value'}])
)
def test_singleton_pipeline():
stub_solid = define_stub_solid("stub", [{"a key": "a value"}])
@pipeline
def single_solid_pipeline():
stub_solid()
assert execute_pipeline(single_solid_pipeline).success
def test_create_pipeline_with_bad_solids_list():
with pytest.raises(
DagsterInvalidDefinitionError,
match='"solids" arg to pipeline "a_pipeline" is not a list. Got',
):
PipelineDefinition(name="a_pipeline",
solid_defs=define_stub_solid(
"stub", [{
"a key": "a value"
}]))
def define_pandas_source_test_pipeline():
return PipelineDefinition(
name='input_transform_test_pipeline',
solids=[
define_stub_solid(
'load_num_csv', pd.read_csv(script_relative_path('data/num_prod.csv'))
),
define_pandas_source_test_solid(),
],
dependencies={'pandas_source_test': {'df': DependencyDefinition('load_num_csv')}},
)
def test_composite_io_mapping():
a_source = define_stub_solid('A_source', [input_set('A_input')])
node_a = create_root_solid('A')
node_b = create_solid_with_deps('B', node_a)
node_c = create_solid_with_deps('C', node_b)
comp_a_inner = CompositeSolidDefinition(
name='comp_a_inner',
solid_defs=[a_source, node_a],
dependencies={'A': {
'A_input': DependencyDefinition('A_source')
}},
output_mappings=[OutputDefinition().mapping_from('A')],
)
comp_a_outer = CompositeSolidDefinition(
name='comp_a_outer',
solid_defs=[comp_a_inner],
output_mappings=[OutputDefinition().mapping_from('comp_a_inner')],
)
comp_bc_inner = CompositeSolidDefinition(
name='comp_bc_inner',
solid_defs=[node_b, node_c],
dependencies={'C': {
'B': DependencyDefinition('B')
}},
input_mappings=[
InputDefinition(name='inner_B_in').mapping_to(solid_name='B',
input_name='A')
],
)
comp_bc_outer = CompositeSolidDefinition(
name='comp_bc_outer',
solid_defs=[comp_bc_inner],
dependencies={},
input_mappings=[
InputDefinition(name='outer_B_in').mapping_to(
solid_name='comp_bc_inner', input_name='inner_B_in')
],
)
result = execute_pipeline(
PipelineDefinition(
name='wrapped_io',
solid_defs=[comp_a_outer, comp_bc_outer],
dependencies={
'comp_bc_outer': {
'outer_B_in': DependencyDefinition('comp_a_outer')
}
},
))
assert result.success
def test_composite_basic_execution():
a_source = define_stub_solid('A_source', [input_set('A_input')])
node_a = create_root_solid('A')
node_b = create_solid_with_deps('B', node_a)
node_c = create_solid_with_deps('C', node_a)
node_d = create_solid_with_deps('D', node_b, node_c)
@composite_solid
def diamond_composite():
a = node_a(a_source())
node_d(B=node_b(a), C=node_c(a))
@pipeline
def test_pipeline_single():
diamond_composite()
result = execute_pipeline(test_pipeline_single)
assert result.success
@pipeline
def test_pipeline_double():
diamond_composite.alias('D1')()
diamond_composite.alias('D2')()
result = execute_pipeline(test_pipeline_double)
assert result.success
@composite_solid
def wrapped_composite():
diamond_composite()
@pipeline
def test_pipeline_mixed():
diamond_composite()
wrapped_composite()
result = execute_pipeline(test_pipeline_mixed)
assert result.success
@composite_solid
def empty():
pass
@pipeline
def test_pipeline_empty():
empty()
result = execute_pipeline(test_pipeline_empty)
assert result.success
nested_pipeline = nesting_composite_pipeline(6, 2)
result = execute_pipeline(nested_pipeline)
assert result.success
def test_singleton_pipeline():
stub_solid = define_stub_solid("stub", [{"a key": "a value"}])
# will fail if any warning is emitted
with warnings.catch_warnings():
warnings.simplefilter("error")
@pipeline
def single_solid_pipeline():
stub_solid()
assert execute_pipeline(single_solid_pipeline).success
def test_wrong_input_value():
@lambda_solid(input_defs=[InputDefinition('foo', dagster_pd.DataFrame)])
def test_wrong_input(foo):
return foo
pass_solid = define_stub_solid('pass_solid', 'not a dataframe')
@pipeline
def test_pipeline():
test_wrong_input(pass_solid())
with pytest.raises(DagsterTypeCheckError):
execute_pipeline(test_pipeline)
def test_execute_aliased_solid_in_diamond():
a_source = define_stub_solid('A_source', [input_set('A_input')])
@pipeline
def aliased_pipeline():
create_root_solid('A').alias('aliased')(a_source())
solid_result = execute_solid_within_pipeline(
aliased_pipeline, 'aliased', inputs={'A_input': [{'a key': 'a value'}]}
)
assert solid_result.success
assert solid_result.output_value() == [{'a key': 'a value'}, {'aliased': 'compute_called'}]
def test_wrong_input_value():
@lambda_solid(input_defs=[InputDefinition("foo", dagster_pd.DataFrame)])
def test_wrong_input(foo):
return foo
pass_solid = define_stub_solid("pass_solid", "not a dataframe")
@pipeline
def test_pipeline():
test_wrong_input(pass_solid())
with pytest.raises(DagsterTypeCheckDidNotPass):
execute_pipeline(test_pipeline)
def test_execute_aliased_solid_in_diamond():
a_source = define_stub_solid("A_source", [input_set("A_input")])
@pipeline
def aliased_pipeline():
create_root_solid("A").alias("aliased")(a_source())
solid_result = execute_solid_within_pipeline(
aliased_pipeline, "aliased", inputs={"A_input": [{"a key": "a value"}]}
)
assert solid_result.success
assert solid_result.output_value() == [{"a key": "a value"}, {"aliased": "compute_called"}]
def test_composite_basic_execution():
a_source = define_stub_solid('A_source', [input_set('A_input')])
node_a = create_root_solid('A')
node_b = create_solid_with_deps('B', node_a)
node_c = create_solid_with_deps('C', node_a)
node_d = create_solid_with_deps('D', node_b, node_c)
diamond_composite = CompositeSolidDefinition(
name='diamond_composite',
solid_defs=[a_source, node_a, node_b, node_c, node_d],
dependencies={
'A': {
'A_input': DependencyDefinition('A_source')
},
'B': {
'A': DependencyDefinition('A')
},
'C': {
'A': DependencyDefinition('A')
},
'D': {
'B': DependencyDefinition('B'),
'C': DependencyDefinition('C')
},
},
)
result = execute_pipeline(
PipelineDefinition(solid_defs=[diamond_composite]))
assert result.success
result = execute_pipeline(
PipelineDefinition(
solid_defs=[diamond_composite],
dependencies={
SolidInvocation('diamond_composite', alias='D1'): {},
SolidInvocation('diamond_composite', alias='D2'): {},
},
))
assert result.success
wrapped_composite = CompositeSolidDefinition(
name='wrapped_composite', solid_defs=[diamond_composite])
result = execute_pipeline(
PipelineDefinition(solid_defs=[diamond_composite, wrapped_composite]))
assert result.success
empty_composite = CompositeSolidDefinition(name='empty', solid_defs=[])
result = execute_pipeline(PipelineDefinition(solid_defs=[empty_composite]))
assert result.success
def test_composite_basic_execution(composition_decorator):
a_source = define_stub_solid("A_source", [input_set("A_input")])
node_a = create_root_solid("A")
node_b = create_solid_with_deps("B", node_a)
node_c = create_solid_with_deps("C", node_a)
node_d = create_solid_with_deps("D", node_b, node_c)
@composition_decorator
def diamond_composite():
a = node_a(a_source())
node_d(B=node_b(a), C=node_c(a))
res = execute_solid(diamond_composite)
assert res.success
@pipeline
def test_pipeline_double():
diamond_composite.alias("D1")()
diamond_composite.alias("D2")()
result = execute_pipeline(test_pipeline_double)
assert result.success
@composition_decorator
def wrapped_composite():
diamond_composite()
@pipeline
def test_pipeline_mixed():
diamond_composite()
wrapped_composite()
result = execute_pipeline(test_pipeline_mixed)
assert result.success
@composition_decorator
def empty():
pass
@pipeline
def test_pipeline_empty():
empty()
result = execute_pipeline(test_pipeline_empty)
assert result.success
nested_pipeline = nesting_composite_pipeline(6, 2)
result = execute_pipeline(nested_pipeline)
assert result.success
def test_wrong_input_value():
@lambda_solid(name="test_wrong_input", inputs=[InputDefinition('foo', dagster_pd.DataFrame)])
def df_solid(foo):
return foo
pass_solid = define_stub_solid('pass_solid', 'not a dataframe')
pipeline = PipelineDefinition(
solids=[pass_solid, df_solid],
dependencies={'test_wrong_input': {'foo': DependencyDefinition('pass_solid')}},
)
with pytest.raises(DagsterTypeError):
execute_pipeline(pipeline)
