def test_execution_plan_wrong_invalid_output_name():
pipeline_def = define_addy_pipeline()
instance = DagsterInstance.ephemeral()
old_run_id = str(uuid.uuid4())
environment_dict = env_with_fs({'solids': {'add_one': {'inputs': {'num': {'value': 3}}}}})
result = execute_pipeline(
pipeline_def,
environment_dict=environment_dict,
run_config=RunConfig(run_id=old_run_id),
instance=instance,
)
new_run_id = str(uuid.uuid4())
run_config = RunConfig(
run_id=new_run_id,
reexecution_config=ReexecutionConfig(
previous_run_id=result.run_id,
step_output_handles=[StepOutputHandle('add_one.compute', 'not_an_output')],
),
)
execution_plan = create_execution_plan(
pipeline_def, environment_dict=environment_dict, run_config=run_config
)
with pytest.raises(DagsterStepOutputNotFoundError) as exc_info:
execute_plan(
execution_plan,
environment_dict=environment_dict,
run_config=run_config,
step_keys_to_execute=['add_two.compute'],
instance=instance,
)
assert str(exc_info.value) == (
'You specified a step_output_handle in the ReexecutionConfig that does not exist: '
'Step add_one.compute does not have output not_an_output.'
)
assert exc_info.value.step_key == 'add_one.compute'
assert exc_info.value.output_name == 'not_an_output'
def test_pipeline_step_key_subset_execution_wrong_step_key_in_step_output_handles(
):
pipeline_def = define_addy_pipeline()
old_run_id = str(uuid.uuid4())
environment_dict = env_with_fs(
{'solids': {
'add_one': {
'inputs': {
'num': {
'value': 3
}
}
}
}})
instance = DagsterInstance.ephemeral()
result = execute_pipeline(
pipeline_def,
environment_dict=environment_dict,
run_config=RunConfig(run_id=old_run_id),
instance=instance,
)
assert result.success
assert result.run_id == old_run_id
new_run_id = str(uuid.uuid4())
with pytest.raises(DagsterExecutionStepNotFoundError):
execute_pipeline(
pipeline_def,
environment_dict=environment_dict,
run_config=RunConfig(
run_id=new_run_id,
reexecution_config=ReexecutionConfig(
previous_run_id=result.run_id,
step_output_handles=[
StepOutputHandle('invalid_in_step_output_handles')
],
),
step_keys_to_execute=['add_two.compute'],
),
instance=instance,
)
def test_pipeline_step_key_subset_execution_wrong_output_name_in_step_output_handles(
):
pipeline_def = define_addy_pipeline()
old_run_id = str(uuid.uuid4())
environment_dict = {
'solids': {
'add_one': {
'inputs': {
'num': {
'value': 3
}
}
}
}
}
result = execute_pipeline(
pipeline_def,
environment_dict=environment_dict,
run_config=RunConfig(storage_mode=RunStorageMode.FILESYSTEM,
run_id=old_run_id),
)
assert result.success
new_run_id = str(uuid.uuid4())
with pytest.raises(DagsterStepOutputNotFoundError):
execute_pipeline(
pipeline_def,
environment_dict=environment_dict,
run_config=RunConfig(
run_id=new_run_id,
reexecution_config=ReexecutionConfig(
previous_run_id=result.run_id,
step_output_handles=[
StepOutputHandle('add_one.compute', 'invalid_output')
],
),
storage_mode=RunStorageMode.FILESYSTEM,
step_keys_to_execute=['add_two.compute'],
),
)
def test_reexecution():
@lambda_solid
def return_one():
return 1
@lambda_solid
def add_one(num):
return num + 1
pipeline_def = PipelineDefinition(
solid_defs=[return_one, add_one],
dependencies={'add_one': {
'num': DependencyDefinition('return_one')
}},
)
pipeline_result = execute_pipeline(
pipeline_def, environment_dict={'storage': {
'filesystem': {}
}})
assert pipeline_result.success
assert pipeline_result.result_for_solid('add_one').output_value() == 2
reexecution_run_config = RunConfig(reexecution_config=ReexecutionConfig(
previous_run_id=pipeline_result.run_id,
step_output_handles=[StepOutputHandle('return_one.compute')],
))
reexecution_result = execute_pipeline(
pipeline_def,
environment_dict={'storage': {
'filesystem': {}
}},
run_config=reexecution_run_config,
)
assert reexecution_result.success
assert len(reexecution_result.solid_result_list) == 2
assert reexecution_result.result_for_solid(
'return_one').output_value() == 1
assert reexecution_result.result_for_solid('add_one').output_value() == 2
def test_execution_plan_wrong_invalid_step_key():
pipeline_def = define_addy_pipeline()
instance = DagsterInstance.ephemeral()
old_run_id = str(uuid.uuid4())
environment_dict = env_with_fs({'solids': {'add_one': {'inputs': {'num': {'value': 3}}}}})
result = execute_pipeline(
pipeline_def,
environment_dict=environment_dict,
run_config=RunConfig(run_id=old_run_id),
instance=instance,
)
new_run_id = str(uuid.uuid4())
run_config = RunConfig(
run_id=new_run_id,
reexecution_config=ReexecutionConfig(
previous_run_id=result.run_id,
step_output_handles=[StepOutputHandle('not_valid.compute')],
),
)
execution_plan = create_execution_plan(
pipeline_def, environment_dict=environment_dict, run_config=run_config
)
with pytest.raises(DagsterExecutionStepNotFoundError) as exc_info:
execute_plan(
execution_plan,
environment_dict=environment_dict,
run_config=run_config,
step_keys_to_execute=['add_two.compute'],
instance=instance,
)
assert str(exc_info.value) == (
'Step not_valid.compute was specified as a step from a previous run. ' 'It does not exist.'
)
def test_execution_plan_wrong_run_id():
pipeline_def = define_addy_pipeline()
unrun_id = str(uuid.uuid4())
environment_dict = {
'solids': {
'add_one': {
'inputs': {
'num': {
'value': 3
}
}
}
}
}
execution_plan = create_execution_plan(pipeline_def,
environment_dict=environment_dict)
with pytest.raises(DagsterRunNotFoundError) as exc_info:
execute_plan(
execution_plan,
environment_dict=environment_dict,
run_config=RunConfig(
storage_mode=RunStorageMode.FILESYSTEM,
reexecution_config=ReexecutionConfig(
previous_run_id=unrun_id,
step_output_handles=[StepOutputHandle('add_one.compute')],
),
),
)
assert str(
exc_info.value
) == 'Run id {} set as previous run id was not found in run storage'.format(
unrun_id)
assert exc_info.value.invalid_run_id == unrun_id
def test_execution_plan_reexecution_with_in_memory():
pipeline_def = define_addy_pipeline()
instance = DagsterInstance.ephemeral()
old_run_id = str(uuid.uuid4())
environment_dict = {'solids': {'add_one': {'inputs': {'num': {'value': 3}}}}}
result = execute_pipeline(
pipeline_def,
environment_dict=environment_dict,
run_config=RunConfig(run_id=old_run_id),
instance=instance,
)
assert result.success
## re-execute add_two
new_run_id = str(uuid.uuid4())
in_memory_run_config = RunConfig(
run_id=new_run_id,
reexecution_config=ReexecutionConfig(
previous_run_id=result.run_id, step_output_handles=[StepOutputHandle('add_one.compute')]
),
)
execution_plan = create_execution_plan(
pipeline_def, environment_dict=environment_dict, run_config=in_memory_run_config
)
with pytest.raises(DagsterInvariantViolationError):
execute_plan(
execution_plan,
environment_dict=environment_dict,
run_config=in_memory_run_config,
step_keys_to_execute=['add_two.compute'],
instance=instance,
)
def test_execution_plan_wrong_run_id():
pipeline_def = define_addy_pipeline()
unrun_id = str(uuid.uuid4())
environment_dict = env_with_fs(
{'solids': {
'add_one': {
'inputs': {
'num': {
'value': 3
}
}
}
}})
run_config = RunConfig(reexecution_config=ReexecutionConfig(
previous_run_id=unrun_id,
step_output_handles=[StepOutputHandle('add_one.compute')]))
execution_plan = create_execution_plan(pipeline_def,
environment_dict=environment_dict,
run_config=run_config)
with pytest.raises(DagsterRunNotFoundError) as exc_info:
execute_plan(
execution_plan,
environment_dict=environment_dict,
run_config=run_config,
instance=DagsterInstance.ephemeral(),
)
assert str(
exc_info.value
) == 'Run id {} set as previous run id was not found in instance'.format(
unrun_id)
assert exc_info.value.invalid_run_id == unrun_id