def test_multiple_outputs_only_emit_one():
def _t_fn(*_args):
yield Output(output_name='output_one', value='foo')
solid = SolidDefinition(
name='multiple_outputs',
input_defs=[],
output_defs=[
OutputDefinition(name='output_one'),
OutputDefinition(name='output_two', is_optional=True),
],
compute_fn=_t_fn,
)
called = {}
def _compute_fn_one(*_args, **_kwargs):
called['one'] = True
downstream_one = SolidDefinition(
name='downstream_one',
input_defs=[InputDefinition('some_input')],
output_defs=[],
compute_fn=_compute_fn_one,
)
def _compute_fn_two(*_args, **_kwargs):
raise Exception('do not call me')
downstream_two = SolidDefinition(
name='downstream_two',
input_defs=[InputDefinition('some_input')],
output_defs=[],
compute_fn=_compute_fn_two,
)
pipeline = PipelineDefinition(
solid_defs=[solid, downstream_one, downstream_two],
dependencies={
'downstream_one': {
'some_input': DependencyDefinition(solid.name,
output='output_one')
},
'downstream_two': {
'some_input': DependencyDefinition(solid.name,
output='output_two')
},
},
)
result = execute_pipeline(pipeline)
assert result.success
assert called['one']
solid_result = result.result_for_solid('multiple_outputs')
assert set(solid_result.output_values.keys()) == set(['output_one'])
with pytest.raises(
DagsterInvariantViolationError,
match='not_defined not defined in solid multiple_outputs'):
solid_result.output_value('not_defined')
with pytest.raises(DagsterInvariantViolationError,
match='Did not find result output_two'):
solid_result.output_value('output_two')
with pytest.raises(
DagsterInvariantViolationError,
match=
'Try to get result for solid not_present in <<unnamed>>. No such solid.',
):
result.result_for_solid('not_present')
assert result.result_for_solid('downstream_two').skipped
def test_reexecution_fs_storage_with_solid_selection():
@solid
def return_one():
return 1
@solid
def add_one(num):
return num + 1
pipeline_def = PipelineDefinition(
solid_defs=[return_one, add_one],
name="test",
dependencies={"add_one": {
"num": DependencyDefinition("return_one")
}},
mode_defs=[default_mode_def_for_test],
)
instance = DagsterInstance.ephemeral()
# Case 1: re-execute a part of a pipeline when the original pipeline doesn't have solid selection
pipeline_result = execute_pipeline(pipeline_def, instance=instance)
assert pipeline_result.success
assert pipeline_result.result_for_solid("add_one").output_value() == 2
# This is how this is actually done in dagster_graphql.implementation.pipeline_execution_manager
reexecution_result_no_solid_selection = reexecute_pipeline(
pipeline_def,
parent_run_id=pipeline_result.run_id,
step_selection=["return_one"],
instance=instance,
)
assert reexecution_result_no_solid_selection.success
assert len(reexecution_result_no_solid_selection.solid_result_list) == 2
assert reexecution_result_no_solid_selection.result_for_solid(
"add_one").skipped
assert reexecution_result_no_solid_selection.result_for_solid(
"return_one").output_value() == 1
# Case 2: re-execute a pipeline when the original pipeline has solid selection
pipeline_result_solid_selection = execute_pipeline(
pipeline_def,
instance=instance,
solid_selection=["return_one"],
)
assert pipeline_result_solid_selection.success
assert len(pipeline_result_solid_selection.solid_result_list) == 1
with pytest.raises(DagsterInvariantViolationError):
pipeline_result_solid_selection.result_for_solid("add_one")
assert pipeline_result_solid_selection.result_for_solid(
"return_one").output_value() == 1
reexecution_result_solid_selection = reexecute_pipeline(
pipeline_def,
parent_run_id=pipeline_result_solid_selection.run_id,
instance=instance,
)
assert reexecution_result_solid_selection.success
assert len(reexecution_result_solid_selection.solid_result_list) == 1
with pytest.raises(DagsterInvariantViolationError):
pipeline_result_solid_selection.result_for_solid("add_one")
assert reexecution_result_solid_selection.result_for_solid(
"return_one").output_value() == 1
# Case 3: re-execute a pipeline partially when the original pipeline has solid selection and
# re-exeucte a step which hasn't been included in the original pipeline
with pytest.raises(
DagsterExecutionStepNotFoundError,
match="Step selection refers to unknown step: add_one",
):
reexecute_pipeline(
pipeline_def,
parent_run_id=pipeline_result_solid_selection.run_id,
step_selection=["add_one"],
instance=instance,
)
# Case 4: re-execute a pipeline partially when the original pipeline has solid selection and
# re-exeucte a step which has been included in the original pipeline
re_reexecution_result = reexecute_pipeline(
pipeline_def,
parent_run_id=reexecution_result_solid_selection.run_id,
instance=instance,
step_selection=["return_one"],
)
assert re_reexecution_result.success
assert len(re_reexecution_result.solid_result_list) == 1
assert re_reexecution_result.result_for_solid(
"return_one").output_value() == 1
# start_tag_pipeline
@pipeline
def tag_pipeline():
add_one.tag({"my_tag": "my_value"})(add_one(return_one()))
# end_tag_pipeline
# start_pipeline_definition_marker
one_plus_one_pipeline_def = PipelineDefinition(
name="one_plus_one_pipeline",
solid_defs=[return_one, add_one],
dependencies={"add_one": {"number": DependencyDefinition("return_one")}},
)
# end_pipeline_definition_marker
# start_modes_pipeline
dev_mode = ModeDefinition("dev")
staging_mode = ModeDefinition("staging")
prod_mode = ModeDefinition("prod")
@pipeline(mode_defs=[dev_mode, staging_mode, prod_mode])
def my_modes_pipeline():
my_solid()
def test_circular_dep():
with pytest.raises(DagsterInvalidDefinitionError, match='Circular reference'):
PipelineDefinition(
solids=solid_a_b_list(),
dependencies={'A': {}, 'B': {'b_input': DependencyDefinition('B')}},
)
def test_fan_in_manual():
# manually building up this guy
@composite_solid
def _target_composite_dsl(str_in, none_in):
num = emit_num()
return collect([num, str_in, none_in])
# base case works
_target_composite_manual = CompositeSolidDefinition(
name="manual_composite",
solid_defs=[emit_num, collect],
input_mappings=[
InputDefinition("str_in").mapping_to("collect", "stuff", 1),
InputDefinition("none_in").mapping_to("collect", "stuff", 2),
],
output_mappings=[OutputDefinition().mapping_from("collect")],
dependencies={
"collect": {
"stuff":
MultiDependencyDefinition([
DependencyDefinition("emit_num"),
MappedInputPlaceholder,
MappedInputPlaceholder,
])
}
},
)
with pytest.raises(
DagsterInvalidDefinitionError,
match=
"index 2 in the MultiDependencyDefinition is not a MappedInputPlaceholder",
):
_missing_placeholder = CompositeSolidDefinition(
name="manual_composite",
solid_defs=[emit_num, collect],
input_mappings=[
InputDefinition("str_in").mapping_to("collect", "stuff", 1),
InputDefinition("none_in").mapping_to("collect", "stuff", 2),
],
output_mappings=[OutputDefinition().mapping_from("collect")],
dependencies={
"collect": {
"stuff":
MultiDependencyDefinition([
DependencyDefinition("emit_num"),
MappedInputPlaceholder,
])
}
},
)
with pytest.raises(DagsterInvalidDefinitionError,
match="is not a MultiDependencyDefinition"):
_bad_target = CompositeSolidDefinition(
name="manual_composite",
solid_defs=[emit_num, collect],
input_mappings=[
InputDefinition("str_in").mapping_to("collect", "stuff", 1),
InputDefinition("none_in").mapping_to("collect", "stuff", 2),
],
output_mappings=[OutputDefinition().mapping_from("collect")],
dependencies={
"collect": {
"stuff": DependencyDefinition("emit_num")
}
},
)
with pytest.raises(
DagsterInvalidDefinitionError,
match="Unsatisfied MappedInputPlaceholder at index 3",
):
_missing_placeholder = CompositeSolidDefinition(
name="manual_composite",
solid_defs=[emit_num, collect],
input_mappings=[
InputDefinition("str_in").mapping_to("collect", "stuff", 1),
InputDefinition("none_in").mapping_to("collect", "stuff", 2),
],
output_mappings=[OutputDefinition().mapping_from("collect")],
dependencies={
"collect": {
"stuff":
MultiDependencyDefinition([
DependencyDefinition("emit_num"),
MappedInputPlaceholder,
MappedInputPlaceholder,
MappedInputPlaceholder,
])
}
},
)
def define_composites_pipeline():
@lambda_solid(inputs=[InputDefinition('num', Int)])
def add_one(num):
return num + 1
@lambda_solid(inputs=[InputDefinition('num')])
def div_two(num):
return num / 2
add_two = CompositeSolidDefinition(
'add_two',
solid_defs=[add_one],
dependencies={
SolidInvocation('add_one', 'adder_1'): {},
SolidInvocation('add_one', 'adder_2'): {
'num': DependencyDefinition('adder_1')
},
},
input_mappings=[
InputDefinition('num', Int).mapping_to('adder_1', 'num')
],
output_mappings=[OutputDefinition(Int).mapping_from('adder_2')],
)
add_four = CompositeSolidDefinition(
'add_four',
solid_defs=[add_two],
dependencies={
SolidInvocation('add_two', 'adder_1'): {},
SolidInvocation('add_two', 'adder_2'): {
'num': DependencyDefinition('adder_1')
},
},
input_mappings=[
InputDefinition('num', Int).mapping_to('adder_1', 'num')
],
output_mappings=[OutputDefinition(Int).mapping_from('adder_2')],
)
div_four = CompositeSolidDefinition(
'div_four',
solid_defs=[div_two],
dependencies={
SolidInvocation('div_two', 'div_1'): {},
SolidInvocation('div_two', 'div_2'): {
'num': DependencyDefinition('div_1')
},
},
input_mappings=[
InputDefinition('num', Int).mapping_to('div_1', 'num')
],
output_mappings=[OutputDefinition(Float).mapping_from('div_2')],
)
return PipelineDefinition(
name='composites_pipeline',
solid_defs=[add_four, div_four],
dependencies={'div_four': {
'num': DependencyDefinition('add_four')
}},
)
def test_reexecution_fs_storage_with_solid_selection():
@lambda_solid
def return_one():
return 1
@lambda_solid
def add_one(num):
return num + 1
pipeline_def = PipelineDefinition(
solid_defs=[return_one, add_one],
dependencies={'add_one': {
'num': DependencyDefinition('return_one')
}},
)
run_config = {'storage': {'filesystem': {}}}
instance = DagsterInstance.ephemeral()
# Case 1: re-execute a part of a pipeline when the original pipeline doesn't have solid selection
pipeline_result = execute_pipeline(pipeline_def,
run_config,
instance=instance)
assert pipeline_result.success
assert pipeline_result.result_for_solid('add_one').output_value() == 2
# This is how this is actually done in dagster_graphql.implementation.pipeline_execution_manager
reexecution_result_no_solid_selection = reexecute_pipeline(
pipeline_def,
parent_run_id=pipeline_result.run_id,
run_config=run_config,
step_keys_to_execute=['return_one.compute'],
instance=instance,
)
assert reexecution_result_no_solid_selection.success
assert len(reexecution_result_no_solid_selection.solid_result_list) == 2
assert reexecution_result_no_solid_selection.result_for_solid(
'add_one').skipped
assert reexecution_result_no_solid_selection.result_for_solid(
'return_one').output_value() == 1
# Case 2: re-execute a pipeline when the original pipeline has solid selection
pipeline_result_solid_selection = execute_pipeline(
pipeline_def,
run_config=run_config,
instance=instance,
solid_selection=['return_one'],
)
assert pipeline_result_solid_selection.success
assert len(pipeline_result_solid_selection.solid_result_list) == 1
with pytest.raises(DagsterInvariantViolationError):
pipeline_result_solid_selection.result_for_solid('add_one')
assert pipeline_result_solid_selection.result_for_solid(
'return_one').output_value() == 1
reexecution_result_solid_selection = reexecute_pipeline(
pipeline_def,
parent_run_id=pipeline_result_solid_selection.run_id,
run_config=run_config,
instance=instance,
)
assert reexecution_result_solid_selection.success
assert len(reexecution_result_solid_selection.solid_result_list) == 1
with pytest.raises(DagsterInvariantViolationError):
pipeline_result_solid_selection.result_for_solid('add_one')
assert reexecution_result_solid_selection.result_for_solid(
'return_one').output_value() == 1
# Case 3: re-execute a pipeline partially when the original pipeline has solid selection and
# re-exeucte a step which hasn't been included in the original pipeline
with pytest.raises(
DagsterExecutionStepNotFoundError,
match=re.escape(
'Execution plan does not contain step: add_one.compute'),
):
reexecute_pipeline(
pipeline_def,
parent_run_id=pipeline_result_solid_selection.run_id,
run_config=run_config,
step_keys_to_execute=['add_one.compute'],
instance=instance,
)
# Case 4: re-execute a pipeline partially when the original pipeline has solid selection and
# re-exeucte a step which has been included in the original pipeline
re_reexecution_result = reexecute_pipeline(
pipeline_def,
parent_run_id=reexecution_result_solid_selection.run_id,
run_config=run_config,
instance=instance,
step_keys_to_execute=['return_one.compute'],
)
assert re_reexecution_result.success
assert len(re_reexecution_result.solid_result_list) == 1
assert re_reexecution_result.result_for_solid(
'return_one').output_value() == 1
def test_solid_def():
@lambda_solid
def produce_string():
return "foo"
@solid(
input_defs=[InputDefinition("input_one", String)],
output_defs=[OutputDefinition(Any)],
config_schema={"another_field": Int},
)
def solid_one(_context, input_one):
raise Exception("should not execute")
pipeline_def = PipelineDefinition(
solid_defs=[produce_string, solid_one],
dependencies={
"solid_one": {
"input_one": DependencyDefinition("produce_string")
}
},
)
assert len(pipeline_def.solids[0].output_handles()) == 1
assert isinstance(pipeline_def.solid_named("solid_one"), Solid)
solid_one_solid = pipeline_def.solid_named("solid_one")
assert solid_one_solid.has_input("input_one")
assert isinstance(solid_one_solid.input_def_named("input_one"),
InputDefinition)
assert len(solid_one_solid.input_dict) == 1
assert len(solid_one_solid.output_dict) == 1
assert str(solid_one_solid.input_handle("input_one")) == (
"SolidInputHandle(input_name=\"'input_one'\", solid_name=\"'solid_one'\")"
)
assert repr(solid_one_solid.input_handle("input_one")) == (
"SolidInputHandle(input_name=\"'input_one'\", solid_name=\"'solid_one'\")"
)
assert str(solid_one_solid.output_handle("result")) == (
"SolidOutputHandle(output_name=\"'result'\", solid_name=\"'solid_one'\")"
)
assert repr(solid_one_solid.output_handle("result")) == (
"SolidOutputHandle(output_name=\"'result'\", solid_name=\"'solid_one'\")"
)
assert solid_one_solid.output_handle("result") == SolidOutputHandle(
solid_one_solid, solid_one_solid.output_dict["result"])
assert (len(
pipeline_def.dependency_structure.input_to_upstream_outputs_for_solid(
"solid_one")) == 1)
assert (len(
pipeline_def.dependency_structure.
output_to_downstream_inputs_for_solid("produce_string")) == 1)
assert len(pipeline_def.dependency_structure.input_handles()) == 1
assert len(pipeline_def.dependency_structure.items()) == 1
def _define_nothing_dep_pipeline():
@lambda_solid(output_def=OutputDefinition(Nothing, "complete"))
def start_nothing():
pass
@lambda_solid(input_defs=[
InputDefinition("add_complete", Nothing),
InputDefinition("yield_complete", Nothing),
])
def end_nothing():
pass
@lambda_solid(output_def=OutputDefinition(Int))
def emit_value():
return 1
@lambda_solid(
input_defs=[
InputDefinition("on_complete", Nothing),
InputDefinition("num", Int)
],
output_def=OutputDefinition(Int),
)
def add_value(num):
return 1 + num
@solid(
name="yield_values",
input_defs=[InputDefinition("on_complete", Nothing)],
output_defs=[
OutputDefinition(Int, "num_1"),
OutputDefinition(Int, "num_2"),
OutputDefinition(Nothing, "complete"),
],
)
def yield_values(_context):
yield Output(1, "num_1")
yield Output(2, "num_2")
yield Output(None, "complete")
return PipelineDefinition(
name="simple_exc",
solid_defs=[
emit_value, add_value, start_nothing, end_nothing, yield_values
],
dependencies={
"add_value": {
"on_complete": DependencyDefinition("start_nothing",
"complete"),
"num": DependencyDefinition("emit_value"),
},
"yield_values": {
"on_complete": DependencyDefinition("start_nothing",
"complete")
},
"end_nothing": {
"add_complete": DependencyDefinition("add_value"),
"yield_complete": DependencyDefinition("yield_values",
"complete"),
},
},
)
def test_templated_sql_solid_pipeline():
sum_sql_template = '''CREATE TABLE {{sum_table}} AS
SELECT num1, num2, num1 + num2 as sum FROM num_table'''
sum_sq_sql_template = '''CREATE TABLE {{sum_sq_table}} AS
SELECT num1, num2, sum, sum * sum as sum_sq FROM {{sum_table}}'''
sum_solid = create_templated_sql_transform_solid(
name='sum_table', sql=sum_sql_template, table_arguments=['sum_table'])
sum_sq_solid = create_templated_sql_transform_solid(
name='sum_sq_table',
sql=sum_sq_sql_template,
table_arguments=['sum_table', 'sum_sq_table'],
dependant_solids=[sum_solid],
)
context = in_mem_context()
pipeline = pipeline_test_def(
solids=[sum_solid, sum_sq_solid],
context=context,
dependencies={
sum_sq_solid.name: {
sum_solid.name: DependencyDefinition(sum_solid.name)
}
},
)
first_sum_table = 'first_sum_table'
first_sum_sq_table = 'first_sum_sq_table'
environment_one = {
'solids': {
'sum_table': {
'config': {
'sum_table': first_sum_table
}
},
'sum_sq_table': {
'config': {
'sum_table': first_sum_table,
'sum_sq_table': first_sum_sq_table
}
},
}
}
first_result = execute_pipeline(pipeline, environment_dict=environment_one)
assert first_result.success
assert len(first_result.solid_result_list) == 2
assert first_result.solid_result_list[0].transformed_value() == {
'sum_table': first_sum_table
}
assert first_result.solid_result_list[1].transformed_value() == {
'sum_table': first_sum_table,
'sum_sq_table': first_sum_sq_table,
}
assert _load_table(context, first_sum_table) == [(1, 2, 3), (3, 4, 7)]
assert _load_table(context, first_sum_sq_table) == [(1, 2, 3, 9),
(3, 4, 7, 49)]
# now execute subdag
pipeline_two = pipeline_test_def(
solids=[define_stub_solid('pass_value', 'TODO'), sum_sq_solid],
context=context,
dependencies={
sum_sq_solid.name: {
sum_solid.name: DependencyDefinition('pass_value')
}
},
)
second_sum_sq_table = 'second_sum_sq_table'
sum_sq_args = {
'sum_table': first_sum_table,
'sum_sq_table': second_sum_sq_table
}
environment_two = {'solids': {'sum_sq_table': {'config': sum_sq_args}}}
second_result = execute_pipeline(pipeline_two,
environment_dict=environment_two)
assert second_result.success
assert len(second_result.solid_result_list) == 2
assert _load_table(context, second_sum_sq_table) == [(1, 2, 3, 9),
(3, 4, 7, 49)]
# pylint: disable=unused-argument
from dagster import DependencyDefinition, InputDefinition, PipelineDefinition, pipeline, solid
@solid
def return_one(context):
return 1
@solid(input_defs=[InputDefinition("number", int)])
def add_one(context, number):
return number + 1
@pipeline
def one_plus_one_pipeline():
add_one(return_one())
one_plus_one_pipeline_def = PipelineDefinition(
name='one_plus_one_pipeline',
solid_defs=[return_one, add_one],
dependencies={'add_one': {
'number': DependencyDefinition('return_one')
}},
)
def test_pandas_hello_no_library():
def solid_one_transform(_context, inputs):
num_df = inputs['num_df']
num_df['sum'] = num_df['num1'] + num_df['num2']
return num_df
solid_one = single_output_transform(
name='solid_one',
inputs=[InputDefinition(name='num_df')],
transform_fn=solid_one_transform,
output=OutputDefinition(),
)
def solid_two_transform(_context, inputs):
sum_df = inputs['sum_df']
sum_df['sum_sq'] = sum_df['sum'] * sum_df['sum']
return sum_df
solid_two = single_output_transform(
name='solid_two',
inputs=[InputDefinition(name='sum_df')],
transform_fn=solid_two_transform,
output=OutputDefinition(),
)
pipeline = PipelineDefinition(
solids=[define_read_csv_solid('read_one'), solid_one, solid_two],
dependencies={
'solid_one': {
'num_df': DependencyDefinition('read_one'),
},
'solid_two': {
'sum_df': DependencyDefinition('solid_one'),
},
}
)
environment = config.Environment(
solids={
'read_one': config.Solid({
'path': script_relative_path('num.csv')
}),
}
)
execute_pipeline_result = execute_pipeline(
pipeline,
environment=environment,
)
assert execute_pipeline_result.result_for_solid('solid_two'
).transformed_value().to_dict('list') == {
'num1': [1, 3],
'num2': [2, 4],
'sum': [3, 7],
'sum_sq': [9, 49],
}
sum_sq_out_path = '/tmp/sum_sq.csv'
import os
if os.path.exists(sum_sq_out_path):
os.remove(sum_sq_out_path)
sum_sq_path_args = {'path': '/tmp/sum_sq.csv'}
environment_two = config.Environment(
solids={
'read_one': config.Solid({
'path': script_relative_path('num.csv')
}),
'write_two': config.Solid(sum_sq_path_args),
},
)
pipeline_two = PipelineDefinition(
solids=[
define_read_csv_solid('read_one'),
solid_one,
solid_two,
define_to_csv_solid('write_two'),
],
dependencies={
'solid_one': {
'num_df': DependencyDefinition('read_one'),
},
'solid_two': {
'sum_df': DependencyDefinition('solid_one'),
},
'write_two': {
'df': DependencyDefinition('solid_two'),
}
}
)
execute_pipeline(pipeline_two, environment=environment_two)
sum_sq_df = pd.read_csv('/tmp/sum_sq.csv')
assert sum_sq_df.to_dict('list') == {
'num1': [1, 3],
'num2': [2, 4],
'sum': [3, 7],
'sum_sq': [9, 49],
}
def add_one(num):
return num + 1
@lambda_solid(inputs=[InputDefinition('num')])
def div_two(num):
return num / 2
add_two = CompositeSolidDefinition(
'add_two',
solids=[add_one],
dependencies={
SolidInstance('add_one', 'adder_1'): {},
SolidInstance('add_one', 'adder_2'): {
'num': DependencyDefinition('adder_1')
},
},
input_mappings=[InputDefinition('num', Int).mapping_to('adder_1', 'num')],
output_mappings=[OutputDefinition(Int).mapping_from('adder_2')],
)
add_four = CompositeSolidDefinition(
'add_four',
solids=[add_two],
dependencies={
SolidInstance('add_two', 'adder_1'): {},
SolidInstance('add_two', 'adder_2'): {
'num': DependencyDefinition('adder_1')
},
},
def test_failure_propagation():
'''
B =========== C
// \\
A F (skipped)
\\ //
D (fails) == E (skipped)
'''
solid_a = create_root_success_solid('A')
def fail_fn(_context, inputs):
check.failed('user error')
return inputs
def success_fn(_context, inputs):
return inputs
solid_b = single_output_solid(
name='B',
input_defs=[InputDefinition(name='A')],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
solid_c = single_output_solid(
name='C',
input_defs=[InputDefinition(name='B')],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
solid_d = single_output_solid(
name='D',
input_defs=[InputDefinition(name='A')],
compute_fn=fail_fn,
output_def=OutputDefinition(),
)
solid_e = single_output_solid(
name='E',
input_defs=[InputDefinition(name='D')],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
solid_f = single_output_solid(
name='F',
input_defs=[InputDefinition(name='C'), InputDefinition(name='E')],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
pipeline_def = PipelineDefinition(
solid_defs=[solid_a, solid_b, solid_c, solid_d, solid_e, solid_f],
dependencies={
'B': {'A': DependencyDefinition(solid_a.name)},
'D': {'A': DependencyDefinition(solid_a.name)},
'C': {'B': DependencyDefinition(solid_b.name)},
'E': {'D': DependencyDefinition(solid_d.name)},
'F': {'C': DependencyDefinition(solid_c.name), 'E': DependencyDefinition(solid_e.name)},
},
)
pipeline_result = execute_pipeline(pipeline_def, raise_on_error=False)
assert pipeline_result.result_for_solid('A').success
assert pipeline_result.result_for_solid('B').success
assert pipeline_result.result_for_solid('C').success
assert not pipeline_result.result_for_solid('D').success
assert pipeline_result.result_for_solid('D').failure_data.error.cls_name == 'CheckError'
assert not pipeline_result.result_for_solid('E').success
assert pipeline_result.result_for_solid('E').skipped
assert not pipeline_result.result_for_solid('F').success
assert pipeline_result.result_for_solid('F').skipped
def execute_solid(
solid_def,
mode_def=None,
input_values=None,
environment_dict=None,
run_config=None,
raise_on_error=True,
):
'''Execute a single solid in an ephemeral pipeline.
Intended to support unit tests. Input values may be passed directly, and no pipeline need be
specified -- an ephemeral pipeline will be constructed.
Args:
solid_def (SolidDefinition): The solid to execute.
mode_def (Optional[ModeDefinition]): The mode within which to execute the solid. Use this
if, e.g., custom resources, loggers, or executors are desired.
input_values (Optional[Dict[str, Any]]): A dict of input names to input values, used to
pass inputs to the solid directly. You may also use the ``environment_dict`` to
configure any inputs that are configurable.
environment_dict (Optional[dict]): The enviroment configuration that parameterizes this
execution, as a dict.
run_config (Optional[RunConfig]): Optionally specifies additional config options for
pipeline execution.
raise_on_error (Optional[bool]): Whether or not to raise exceptions when they occur.
Defaults to ``True``, since this is the most useful behavior in test.
Returns:
Union[CompositeSolidExecutionResult, SolidExecutionResult]: The result of executing the
solid.
'''
check.inst_param(solid_def, 'solid_def', ISolidDefinition)
check.opt_inst_param(mode_def, 'mode_def', ModeDefinition)
input_values = check.opt_dict_param(input_values,
'input_values',
key_type=str)
solid_defs = [solid_def]
def create_value_solid(input_name, input_value):
@lambda_solid(name=input_name)
def input_solid():
return input_value
return input_solid
dependencies = defaultdict(dict)
for input_name, input_value in input_values.items():
dependencies[solid_def.name][input_name] = DependencyDefinition(
input_name)
solid_defs.append(create_value_solid(input_name, input_value))
result = execute_pipeline(
PipelineDefinition(
name='ephemeral_{}_solid_pipeline'.format(solid_def.name),
solid_defs=solid_defs,
dependencies=dependencies,
mode_defs=[mode_def] if mode_def else None,
),
environment_dict=environment_dict,
run_config=run_config,
raise_on_error=raise_on_error,
)
return result.result_for_handle(solid_def.name)
def test_solid_def():
@lambda_solid
def produce_string():
return 'foo'
@solid(
input_defs=[InputDefinition('input_one', String)],
output_defs=[OutputDefinition(Any)],
config={'another_field': Field(Int)},
)
def solid_one(_context, input_one):
raise Exception('should not execute')
pipeline_def = PipelineDefinition(
solid_defs=[produce_string, solid_one],
dependencies={
'solid_one': {
'input_one': DependencyDefinition('produce_string')
}
},
)
assert len(pipeline_def.solids[0].output_handles()) == 1
assert isinstance(pipeline_def.solid_named('solid_one'), Solid)
solid_one_solid = pipeline_def.solid_named('solid_one')
assert solid_one_solid.has_input('input_one')
assert isinstance(solid_one_solid.input_def_named('input_one'),
InputDefinition)
assert len(solid_one_solid.input_dict) == 1
assert len(solid_one_solid.output_dict) == 1
assert str(solid_one_solid.input_handle('input_one')) == (
'SolidInputHandle(definition_name="\'solid_one\'", input_name="\'input_one\'", '
'solid_name="\'solid_one\'")')
assert repr(solid_one_solid.input_handle('input_one')) == (
'SolidInputHandle(definition_name="\'solid_one\'", input_name="\'input_one\'", '
'solid_name="\'solid_one\'")')
assert str(solid_one_solid.output_handle('result')) == (
'SolidOutputHandle(definition_name="\'solid_one\'", output_name="\'result\'", '
'solid_name="\'solid_one\'")')
assert repr(solid_one_solid.output_handle('result')) == (
'SolidOutputHandle(definition_name="\'solid_one\'", output_name="\'result\'", '
'solid_name="\'solid_one\'")')
assert solid_one_solid.output_handle('result') == SolidOutputHandle(
solid_one_solid, solid_one_solid.output_dict['result'])
assert len(
pipeline_def.dependency_structure.deps_of_solid_with_input(
'solid_one')) == 1
assert len(
pipeline_def.dependency_structure.depended_by_of_solid(
'produce_string')) == 1
assert len(pipeline_def.dependency_structure.input_handles()) == 1
assert len(pipeline_def.dependency_structure.items()) == 1
's3': s3_resource,
'db_info': redshift_db_info_resource,
'tempfile': tempfile_resource,
},
)
process_on_time_data = CompositeSolidDefinition(
name='process_on_time_data',
solids=[s3_to_df, join_q2_data, load_data_to_database_from_spark],
dependencies={
SolidInstance('s3_to_df', alias='april_on_time_s3_to_df'): {},
SolidInstance('s3_to_df', alias='may_on_time_s3_to_df'): {},
SolidInstance('s3_to_df', alias='june_on_time_s3_to_df'): {},
SolidInstance('s3_to_df', alias='master_cord_s3_to_df'): {},
'join_q2_data': {
'april_data': DependencyDefinition('april_on_time_s3_to_df'),
'may_data': DependencyDefinition('may_on_time_s3_to_df'),
'june_data': DependencyDefinition('june_on_time_s3_to_df'),
'master_cord_data': DependencyDefinition('master_cord_s3_to_df'),
},
SolidInstance('load_data_to_database_from_spark',
alias='load_q2_on_time_data'): {
'data_frame': DependencyDefinition('join_q2_data')
},
},
)
sfo_weather_data = CompositeSolidDefinition(
name='sfo_weather_data',
solids=[
download_from_s3_to_bytes,
def test_deps_equal():
assert DependencyDefinition('foo') == DependencyDefinition('foo')
assert DependencyDefinition('foo') != DependencyDefinition('bar')
assert DependencyDefinition('foo', 'bar') == DependencyDefinition('foo', 'bar')
assert DependencyDefinition('foo', 'bar') != DependencyDefinition('foo', 'quuz')
