def test_execute_dep_solid_different_input_name():
pass_to_first = define_pass_value_solid('pass_to_first')
first_solid = single_output_solid(
'first_solid',
input_defs=[InputDefinition(name='a_thing')],
compute_fn=lambda context, inputs: inputs['a_thing'] + inputs['a_thing'],
output_def=OutputDefinition(),
)
second_solid = single_output_solid(
'second_solid',
input_defs=[InputDefinition(name='an_input')],
compute_fn=lambda context, inputs: inputs['an_input'] + inputs['an_input'],
output_def=OutputDefinition(),
)
@pipeline
def pipe():
second_solid(first_solid(pass_to_first()))
result = execute_pipeline(
pipe, environment_dict={'solids': {'pass_to_first': {'config': {'value': 'bar'}}}}
)
assert result.success
assert len(result.solid_result_list) == 3
assert result.result_for_solid('pass_to_first').output_value() == 'bar'
assert result.result_for_solid('first_solid').output_value() == 'barbar'
assert result.result_for_solid('second_solid').output_value() == 'barbarbarbar'
def test_execute_solid_with_dep_only_inputs_no_api():
did_run_dict = {}
step_one_solid = single_output_solid(
name='step_one_solid',
input_defs=[],
compute_fn=lambda context, args: _set_key_value(
did_run_dict, 'step_one', True),
output_def=OutputDefinition(),
)
step_two_solid = single_output_solid(
name='step_two_solid',
input_defs=[InputDefinition('step_one_solid')],
compute_fn=lambda context, args: _set_key_value(
did_run_dict, 'step_two', True),
output_def=OutputDefinition(),
)
@pipeline
def pipe():
step_two_solid(step_one_solid())
pipeline_result = execute_pipeline(pipe)
assert pipeline_result.success
for result in pipeline_result.solid_result_list:
assert result.success
assert did_run_dict['step_one'] is True
assert did_run_dict['step_two'] is True
def test_execute_two_solids_with_same_input_name():
input_def = InputDefinition(name='a_thing')
solid_one = single_output_solid(
'solid_one',
input_defs=[input_def],
compute_fn=lambda context, inputs: inputs['a_thing'] + inputs['a_thing'],
output_def=OutputDefinition(),
)
solid_two = single_output_solid(
'solid_two',
input_defs=[input_def],
compute_fn=lambda context, inputs: inputs['a_thing'] + inputs['a_thing'],
output_def=OutputDefinition(),
)
@pipeline
def pipe():
solid_one(define_pass_value_solid('pass_to_one')())
solid_two(define_pass_value_solid('pass_to_two')())
result = execute_pipeline(
pipe,
environment_dict={
'solids': {
'pass_to_one': {'config': {'value': 'foo'}},
'pass_to_two': {'config': {'value': 'bar'}},
}
},
)
assert result.success
assert result.result_for_solid('solid_one').output_value() == 'foofoo'
assert result.result_for_solid('solid_two').output_value() == 'barbar'
def test_failure_midstream():
'''
A
\\
C (fails) = D (skipped)
//
B
'''
solid_a = create_root_success_solid('A')
solid_b = create_root_success_solid('B')
def fail_fn(_context, inputs):
check.failed('user error')
return [inputs['A'], inputs['B'], {'C': 'compute_called'}]
def success_fn(_context, inputs):
return [inputs['C'], {'D': 'compute_called'}]
solid_c = single_output_solid(
name='C',
input_defs=[InputDefinition(name='A'),
InputDefinition(name='B')],
compute_fn=fail_fn,
output_def=OutputDefinition(),
)
solid_d = single_output_solid(
name='D',
input_defs=[InputDefinition(name='C')],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
pipeline_def = PipelineDefinition(
solid_defs=[solid_a, solid_b, solid_c, solid_d],
dependencies={
'C': {
'A': DependencyDefinition(solid_a.name),
'B': DependencyDefinition(solid_b.name)
},
'D': {
'C': DependencyDefinition(solid_c.name)
},
},
)
pipeline_result = execute_pipeline(
pipeline_def, run_config=RunConfig.nonthrowing_in_process())
assert pipeline_result.result_for_solid('A').success
assert pipeline_result.result_for_solid('B').success
assert not pipeline_result.result_for_solid('C').success
assert pipeline_result.result_for_solid(
'C').failure_data.error.cls_name == 'CheckError'
assert not pipeline_result.result_for_solid('D').success
assert pipeline_result.result_for_solid('D').skipped
def test_failure_midstream():
"""
A
\\
C (fails) = D (skipped)
//
B
"""
solid_a = create_root_success_solid("A")
solid_b = create_root_success_solid("B")
def fail_fn(_context, inputs):
check.failed("user error")
return [inputs["A"], inputs["B"], {"C": "compute_called"}]
def success_fn(_context, inputs):
return [inputs["C"], {"D": "compute_called"}]
solid_c = single_output_solid(
name="C",
input_defs=[InputDefinition(name="A"),
InputDefinition(name="B")],
compute_fn=fail_fn,
output_def=OutputDefinition(),
)
solid_d = single_output_solid(
name="D",
input_defs=[InputDefinition(name="C")],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
pipeline_def = PipelineDefinition(
solid_defs=[solid_a, solid_b, solid_c, solid_d],
dependencies={
"C": {
"A": DependencyDefinition(solid_a.name),
"B": DependencyDefinition(solid_b.name)
},
"D": {
"C": DependencyDefinition(solid_c.name)
},
},
)
pipeline_result = execute_pipeline(pipeline_def, raise_on_error=False)
assert pipeline_result.result_for_solid("A").success
assert pipeline_result.result_for_solid("B").success
assert not pipeline_result.result_for_solid("C").success
assert pipeline_result.result_for_solid(
"C").failure_data.error.cls_name == "CheckError"
assert not pipeline_result.result_for_solid("D").success
assert pipeline_result.result_for_solid("D").skipped
def test_execute_solid_with_input_same_name():
a_thing_solid = single_output_solid(
'a_thing',
input_defs=[InputDefinition(name='a_thing')],
compute_fn=lambda context, inputs: inputs['a_thing'] + inputs['a_thing'
],
output_def=OutputDefinition(),
)
@pipeline
def pipe():
pass_value = define_pass_value_solid('pass_value')
a_thing_solid(pass_value())
result = execute_pipeline(pipe,
environment_dict={
'solids': {
'pass_value': {
'config': {
'value': 'foo'
}
}
}
})
assert result.result_for_solid('a_thing').output_value() == 'foofoo'
def _dataframe_solid(name, input_defs, compute_fn):
return single_output_solid(
name=name,
input_defs=input_defs,
compute_fn=compute_fn,
output_def=OutputDefinition(DataFrame),
)
def create_root_fn_failure_solid(name):
def failed_fn(**_kwargs):
raise Exception('Compute failed')
return single_output_solid(
name=name, input_defs=[], compute_fn=failed_fn, output_def=OutputDefinition()
)
def test_execute_solid_with_input_same_name():
a_thing_solid = single_output_solid(
"a_thing",
input_defs=[InputDefinition(name="a_thing")],
compute_fn=lambda context, inputs: inputs["a_thing"] + inputs["a_thing"
],
output_def=OutputDefinition(),
)
@pipeline
def pipe():
pass_value = define_pass_value_solid("pass_value")
a_thing_solid(pass_value())
result = execute_pipeline(
pipe,
run_config={"solids": {
"pass_value": {
"config": {
"value": "foo"
}
}
}})
assert result.result_for_solid("a_thing").output_value() == "foofoo"
def test_execute_two_solids_with_same_input_name():
input_def = InputDefinition(name="a_thing")
solid_one = single_output_solid(
"solid_one",
input_defs=[input_def],
compute_fn=lambda context, inputs: inputs["a_thing"] + inputs["a_thing"
],
output_def=OutputDefinition(),
)
solid_two = single_output_solid(
"solid_two",
input_defs=[input_def],
compute_fn=lambda context, inputs: inputs["a_thing"] + inputs["a_thing"
],
output_def=OutputDefinition(),
)
@pipeline
def pipe():
solid_one(define_pass_value_solid("pass_to_one")())
solid_two(define_pass_value_solid("pass_to_two")())
result = execute_pipeline(
pipe,
run_config={
"solids": {
"pass_to_one": {
"config": {
"value": "foo"
}
},
"pass_to_two": {
"config": {
"value": "bar"
}
},
}
},
)
assert result.success
assert result.result_for_solid("solid_one").output_value() == "foofoo"
assert result.result_for_solid("solid_two").output_value() == "barbar"
def create_root_success_solid(name):
def root_fn(_context, _args):
passed_rows = []
passed_rows.append({name: 'compute_called'})
return passed_rows
return single_output_solid(
name=name, input_defs=[], compute_fn=root_fn, output_def=OutputDefinition()
)
def test_single_compute_fn_returning_result():
solid_inst = single_output_solid(
'test_return_result',
input_defs=[],
compute_fn=lambda *_args, **_kwargs: Output(None),
output_def=OutputDefinition(),
)
with pytest.raises(DagsterInvariantViolationError):
execute_solid(solid_inst)
def test_execute_dep_solid_different_input_name():
pass_to_first = define_pass_value_solid("pass_to_first")
first_solid = single_output_solid(
"first_solid",
input_defs=[InputDefinition(name="a_thing")],
compute_fn=lambda context, inputs: inputs["a_thing"] + inputs["a_thing"
],
output_def=OutputDefinition(),
)
second_solid = single_output_solid(
"second_solid",
input_defs=[InputDefinition(name="an_input")],
compute_fn=lambda context, inputs: inputs["an_input"] + inputs[
"an_input"],
output_def=OutputDefinition(),
)
@pipeline
def pipe():
second_solid(first_solid(pass_to_first()))
result = execute_pipeline(
pipe,
run_config={"solids": {
"pass_to_first": {
"config": {
"value": "bar"
}
}
}})
assert result.success
assert len(result.solid_result_list) == 3
assert result.result_for_solid("pass_to_first").output_value() == "bar"
assert result.result_for_solid("first_solid").output_value() == "barbar"
assert result.result_for_solid(
"second_solid").output_value() == "barbarbarbar"
def create_definition_based_solid():
table_input = InputDefinition('num_csv', DataFrame)
def compute_fn(_context, inputs):
num_csv = inputs['num_csv']
num_csv['sum'] = num_csv['num1'] + num_csv['num2']
return num_csv
# supports CSV and PARQUET by default
hello_world = single_output_solid(
name='hello_world',
input_defs=[table_input],
compute_fn=compute_fn,
output_def=OutputDefinition(DataFrame),
)
return hello_world
def create_definition_based_solid():
table_input = InputDefinition("num_csv", DataFrame)
def compute_fn(_context, inputs):
num_csv = inputs["num_csv"]
num_csv["sum"] = num_csv["num1"] + num_csv["num2"]
return num_csv
# supports CSV and PARQUET by default
hello_world = single_output_solid(
name="hello_world",
input_defs=[table_input],
compute_fn=compute_fn,
output_def=OutputDefinition(DataFrame),
)
return hello_world
def test_failure_propagation():
'''
B =========== C
// \\
A F (skipped)
\\ //
D (fails) == E (skipped)
'''
solid_a = create_root_success_solid('A')
def fail_fn(_context, inputs):
check.failed('user error')
return inputs
def success_fn(_context, inputs):
return inputs
solid_b = single_output_solid(
name='B',
input_defs=[InputDefinition(name='A')],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
solid_c = single_output_solid(
name='C',
input_defs=[InputDefinition(name='B')],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
solid_d = single_output_solid(
name='D',
input_defs=[InputDefinition(name='A')],
compute_fn=fail_fn,
output_def=OutputDefinition(),
)
solid_e = single_output_solid(
name='E',
input_defs=[InputDefinition(name='D')],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
solid_f = single_output_solid(
name='F',
input_defs=[InputDefinition(name='C'),
InputDefinition(name='E')],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
pipeline_def = PipelineDefinition(
solid_defs=[solid_a, solid_b, solid_c, solid_d, solid_e, solid_f],
dependencies={
'B': {
'A': DependencyDefinition(solid_a.name)
},
'D': {
'A': DependencyDefinition(solid_a.name)
},
'C': {
'B': DependencyDefinition(solid_b.name)
},
'E': {
'D': DependencyDefinition(solid_d.name)
},
'F': {
'C': DependencyDefinition(solid_c.name),
'E': DependencyDefinition(solid_e.name)
},
},
)
pipeline_result = execute_pipeline(pipeline_def, raise_on_error=False)
assert pipeline_result.result_for_solid('A').success
assert pipeline_result.result_for_solid('B').success
assert pipeline_result.result_for_solid('C').success
assert not pipeline_result.result_for_solid('D').success
assert pipeline_result.result_for_solid(
'D').failure_data.error.cls_name == 'CheckError'
assert not pipeline_result.result_for_solid('E').success
assert pipeline_result.result_for_solid('E').skipped
assert not pipeline_result.result_for_solid('F').success
assert pipeline_result.result_for_solid('F').skipped
def test_failure_propagation():
"""
B =========== C
// \\
A F (skipped)
\\ //
D (fails) == E (skipped)
"""
solid_a = create_root_success_solid("A")
def fail_fn(_context, inputs):
check.failed("user error")
return inputs
def success_fn(_context, inputs):
return inputs
solid_b = single_output_solid(
name="B",
input_defs=[InputDefinition(name="A")],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
solid_c = single_output_solid(
name="C",
input_defs=[InputDefinition(name="B")],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
solid_d = single_output_solid(
name="D",
input_defs=[InputDefinition(name="A")],
compute_fn=fail_fn,
output_def=OutputDefinition(),
)
solid_e = single_output_solid(
name="E",
input_defs=[InputDefinition(name="D")],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
solid_f = single_output_solid(
name="F",
input_defs=[InputDefinition(name="C"),
InputDefinition(name="E")],
compute_fn=success_fn,
output_def=OutputDefinition(),
)
pipeline_def = PipelineDefinition(
solid_defs=[solid_a, solid_b, solid_c, solid_d, solid_e, solid_f],
dependencies={
"B": {
"A": DependencyDefinition(solid_a.name)
},
"D": {
"A": DependencyDefinition(solid_a.name)
},
"C": {
"B": DependencyDefinition(solid_b.name)
},
"E": {
"D": DependencyDefinition(solid_d.name)
},
"F": {
"C": DependencyDefinition(solid_c.name),
"E": DependencyDefinition(solid_e.name)
},
},
)
pipeline_result = execute_pipeline(pipeline_def, raise_on_error=False)
assert pipeline_result.result_for_solid("A").success
assert pipeline_result.result_for_solid("B").success
assert pipeline_result.result_for_solid("C").success
assert not pipeline_result.result_for_solid("D").success
assert pipeline_result.result_for_solid(
"D").failure_data.error.cls_name == "CheckError"
assert not pipeline_result.result_for_solid("E").success
assert pipeline_result.result_for_solid("E").skipped
assert not pipeline_result.result_for_solid("F").success
assert pipeline_result.result_for_solid("F").skipped
