def test_pipeline():
'''just a sanity check to ensure the above pipeline works without layering on graphql'''
result = execute_pipeline(define_test_events_pipeline(),
run_config=RunConfig.nonthrowing_in_process())
assert result.result_for_solid('materialization_and_expectation').success
assert not result.result_for_solid('should_fail').success
assert result.result_for_solid('should_be_skipped').skipped
def test_error_notebook():
with pytest.raises(DagstermillError, match='Someone set up us the bomb'):
execute_pipeline(define_error_pipeline())
res = execute_pipeline(define_error_pipeline(),
run_config=RunConfig.nonthrowing_in_process())
assert not res.success
assert res.step_event_list[1].event_type.value == 'STEP_FAILURE'
def test_intro_tutorial_expectations_step_two_fails_soft():
result = execute_pipeline(
define_expectations_tutorial_pipeline(),
define_failing_environment_config(),
run_config=RunConfig.nonthrowing_in_process(),
)
assert not result.success
def test_failure_midstream():
'''
A
\\
C (fails) = D (skipped)
//
B
'''
solid_a = create_root_success_solid('A')
solid_b = create_root_success_solid('B')
def fail_fn(_context, inputs):
check.failed('user error')
return [inputs['A'], inputs['B'], {'C': 'transform_called'}]
def success_fn(_context, inputs):
return [inputs['C'], {'D': 'transform_called'}]
solid_c = single_output_transform(
name='C',
inputs=[InputDefinition(name='A'),
InputDefinition(name='B')],
compute_fn=fail_fn,
output=OutputDefinition(),
)
solid_d = single_output_transform(
name='D',
inputs=[InputDefinition(name='C')],
compute_fn=success_fn,
output=OutputDefinition(),
)
pipeline_def = PipelineDefinition(
solid_defs=[solid_a, solid_b, solid_c, solid_d],
dependencies={
'C': {
'A': DependencyDefinition(solid_a.name),
'B': DependencyDefinition(solid_b.name)
},
'D': {
'C': DependencyDefinition(solid_c.name)
},
},
)
pipeline_result = execute_pipeline(
pipeline_def, run_config=RunConfig.nonthrowing_in_process())
assert pipeline_result.result_for_solid('A').success
assert pipeline_result.result_for_solid('B').success
assert not pipeline_result.result_for_solid('C').success
assert pipeline_result.result_for_solid(
'C').failure_data.error.cls_name == 'CheckError'
assert not pipeline_result.result_for_solid('D').success
assert pipeline_result.result_for_solid('D').skipped
def test_no_repo_registration_error():
with pytest.raises(
DagstermillError,
match=
'If Dagstermill solids have outputs that require serialization strategies',
):
execute_pipeline(define_no_repo_registration_error_pipeline())
res = execute_pipeline(define_no_repo_registration_error_pipeline(),
run_config=RunConfig.nonthrowing_in_process())
assert not res.success
def test_compute_failure_pipeline():
pipeline_def = PipelineDefinition(solid_defs=[create_root_fn_failure_solid('failing')])
pipeline_result = execute_pipeline(pipeline_def, run_config=RunConfig.nonthrowing_in_process())
assert not pipeline_result.success
result_list = pipeline_result.solid_result_list
assert len(result_list) == 1
assert not result_list[0].success
assert result_list[0].failure_data
def test_error_notebook():
with pytest.raises(
DagstermillError, match='Error occurred during the execution of Dagstermill solid'
) as exc:
with exec_for_test('define_error_pipeline') as result:
pass
assert 'Someone set up us the bomb' in exc.value.original_exc_info[1].args[0]
with exec_for_test(
'define_error_pipeline', run_config=RunConfig.nonthrowing_in_process()
) as result:
assert not result.success
assert result.step_event_list[1].event_type.value == 'STEP_MATERIALIZATION'
assert result.step_event_list[2].event_type.value == 'STEP_FAILURE'
def test_no_repo_registration_error():
with pytest.raises(
DagstermillError,
match=
'Error occurred during the execution of Dagstermill solid no_repo_reg',
) as exc:
execute_pipeline(define_no_repo_registration_error_pipeline())
assert (
'If Dagstermill solids have outputs that require serialization strategies'
in exc.value.original_exc_info[1].args[0])
with exec_for_test(
define_no_repo_registration_error_pipeline(),
run_config=RunConfig.nonthrowing_in_process()) as result:
assert not result.success
def test_resources_notebook_with_exception():
result = None
with tempfile.NamedTemporaryFile() as fd:
path = fd.name
try:
with exec_for_test(
define_resource_with_exception_pipeline(),
{'resources': {
'list': {
'config': path
}
}},
run_config=RunConfig.nonthrowing_in_process(),
) as result:
assert not result.success
assert result.step_event_list[6].event_type.value == 'STEP_FAILURE'
assert (
'raise Exception()'
in result.step_event_list[6].event_specific_data.error.message)
# Expect something like:
# ['e8d636: Opened', 'e8d636: Hello, solid!', '9d438e: Opened',
# '9d438e: Hello, notebook!', '9d438e: Closed', 'e8d636: Closed']
with open(path, 'rb') as fd:
messages = pickle.load(fd)
messages = [message.split(': ') for message in messages]
resource_ids = [x[0] for x in messages]
assert len(set(resource_ids)) == 2
assert resource_ids[0] == resource_ids[1] == resource_ids[5]
assert resource_ids[2] == resource_ids[3] == resource_ids[4]
msgs = [x[1] for x in messages]
assert msgs[0] == msgs[2] == 'Opened'
assert msgs[4] == msgs[5] == 'Closed'
assert msgs[1] == 'Hello, solid!'
assert msgs[3] == 'Hello, notebook!'
finally:
if os.path.exists(path):
os.unlink(path)
def test_failure_propagation():
'''
B =========== C
// \\
A F (skipped)
\\ //
D (fails) == E (skipped)
'''
solid_a = create_root_success_solid('A')
def fail_fn(_context, inputs):
check.failed('user error')
return inputs
def success_fn(_context, inputs):
return inputs
solid_b = single_output_transform(
name='B',
inputs=[InputDefinition(name='A')],
compute_fn=success_fn,
output=OutputDefinition(),
)
solid_c = single_output_transform(
name='C',
inputs=[InputDefinition(name='B')],
compute_fn=success_fn,
output=OutputDefinition(),
)
solid_d = single_output_transform(name='D',
inputs=[InputDefinition(name='A')],
compute_fn=fail_fn,
output=OutputDefinition())
solid_e = single_output_transform(
name='E',
inputs=[InputDefinition(name='D')],
compute_fn=success_fn,
output=OutputDefinition(),
)
solid_f = single_output_transform(
name='F',
inputs=[InputDefinition(name='C'),
InputDefinition(name='E')],
compute_fn=success_fn,
output=OutputDefinition(),
)
pipeline_def = PipelineDefinition(
solid_defs=[solid_a, solid_b, solid_c, solid_d, solid_e, solid_f],
dependencies={
'B': {
'A': DependencyDefinition(solid_a.name)
},
'D': {
'A': DependencyDefinition(solid_a.name)
},
'C': {
'B': DependencyDefinition(solid_b.name)
},
'E': {
'D': DependencyDefinition(solid_d.name)
},
'F': {
'C': DependencyDefinition(solid_c.name),
'E': DependencyDefinition(solid_e.name)
},
},
)
pipeline_result = execute_pipeline(
pipeline_def, run_config=RunConfig.nonthrowing_in_process())
assert pipeline_result.result_for_solid('A').success
assert pipeline_result.result_for_solid('B').success
assert pipeline_result.result_for_solid('C').success
assert not pipeline_result.result_for_solid('D').success
assert pipeline_result.result_for_solid(
'D').failure_data.error.cls_name == 'CheckError'
assert not pipeline_result.result_for_solid('E').success
assert pipeline_result.result_for_solid('E').skipped
assert not pipeline_result.result_for_solid('F').success
assert pipeline_result.result_for_solid('F').skipped