def test_no_outputs_no_inputs_config_schema():
assert create_typed_environment(no_input_no_output_pipeline())
with pytest.raises(PipelineConfigEvaluationError) as exc_info:
create_typed_environment(no_input_no_output_pipeline(), {'solids': {'return_one': {}}})
assert len(exc_info.value.errors) == 1
assert 'Error 1: Undefined field "return_one" at path root:solids' in exc_info.value.message
def test_basic_json_misnamed_output_config_schema():
with pytest.raises(PipelineConfigEvaluationError) as exc_info:
create_typed_environment(
single_int_named_output_pipeline(),
{
'solids': {
'return_named_one': {'outputs': [{'wrong_name': {'json': {'path': 'foo'}}}]}
}
},
)
assert len(exc_info.value.errors) == 1
assert 'Error 1: Undefined field "wrong_name"' in exc_info.value.message
assert 'at path root:solids:return_named_one:outputs[0]' in exc_info.value.message
def test_create_subplan_middle_step():
pipeline_def = define_two_int_pipeline()
typed_environment = create_typed_environment(pipeline_def, None)
execution_plan = create_execution_plan(pipeline_def)
with yield_context(pipeline_def, typed_environment,
ExecutionMetadata()) as context:
subplan = create_subplan(
ExecutionPlanInfo(context=context,
pipeline=pipeline_def,
environment=typed_environment),
StepBuilderState(pipeline_name=pipeline_def.name),
execution_plan,
ExecutionPlanSubsetInfo(['add_one.transform'],
{'add_one.transform': {
'num': 2
}}),
)
assert subplan
steps = subplan.topological_steps()
assert len(steps) == 2
assert steps[0].key == 'add_one.transform.input.num.value'
assert not steps[0].step_inputs
assert len(steps[0].step_outputs) == 1
assert steps[1].key == 'add_one.transform'
assert len(steps[1].step_inputs) == 1
step_input = steps[1].step_inputs[0]
assert step_input.prev_output_handle.step.key == 'add_one.transform.input.num.value'
assert step_input.prev_output_handle.output_name == VALUE_OUTPUT
assert len(steps[1].step_outputs) == 1
assert len(subplan.topological_steps()) == 2
assert [step.key for step in subplan.topological_steps()] == [
'add_one.transform.input.num.value',
'add_one.transform',
]
def test_basic_json_default_output_config_schema():
env = create_typed_environment(
single_int_output_pipeline(),
{
'solids': {
'return_one': {
'outputs': [{
'result': {
'json': {
'path': 'foo'
}
}
}]
}
}
},
)
assert env.solids['return_one']
assert env.solids['return_one'].outputs == [{
'result': {
'json': {
'path': 'foo'
}
}
}]
def run_test_pipeline(pipeline):
typed_environment = create_typed_environment(pipeline, TEST_ENVIRONMENT)
execution_metadata = ExecutionMetadata(run_id=str(uuid.uuid4()))
with yield_context(pipeline, typed_environment,
execution_metadata) as context:
execution_plan = create_execution_plan_core(
ExecutionPlanInfo(context, pipeline, typed_environment))
return execute_plan(context, execution_plan)
def run_test_pipeline(pipeline):
typed_environment = create_typed_environment(pipeline, TEST_ENVIRONMENT)
reentrant_info = ReentrantInfo(run_id=str(uuid.uuid4()))
with yield_context(pipeline, typed_environment, reentrant_info) as context:
execution_plan = create_execution_plan_core(
ExecutionPlanInfo(context, pipeline, typed_environment))
with context.value('pipeline', pipeline.display_name):
results = execute_plan(context, execution_plan)
return results
def test_no_outputs_one_input_config_schema():
assert create_typed_environment(
one_input_no_output_pipeline(),
{'solids': {'take_input_return_nothing': {'inputs': {'dummy': {'value': 'value'}}}}},
)
with pytest.raises(PipelineConfigEvaluationError) as exc_info:
create_typed_environment(
one_input_no_output_pipeline(),
{
'solids': {
'take_input_return_nothing': {
'inputs': {'dummy': {'value': 'value'}},
'outputs': {},
}
}
},
)
assert len(exc_info.value.errors) == 1
exp_msg = 'Error 1: Undefined field "outputs" at path root:solids:take_input_return_nothing'
assert exp_msg in exc_info.value.message
def test_create_subplan_source_step():
pipeline_def = define_two_int_pipeline()
typed_environment = create_typed_environment(pipeline_def, None)
execution_plan = create_execution_plan(pipeline_def)
with yield_context(pipeline_def, typed_environment) as context:
subplan = create_subplan(
ExecutionPlanInfo(context=context,
pipeline=pipeline_def,
environment=typed_environment),
execution_plan,
ExecutionPlanSubsetInfo(['return_one.transform']),
)
assert subplan
assert len(subplan.steps) == 1
assert subplan.steps[0].key == 'return_one.transform'
assert not subplan.steps[0].step_inputs
assert len(subplan.steps[0].step_outputs) == 1
assert len(subplan.topological_steps()) == 1
def execute_pipeline(self, repository_container, pipeline, pipeline_run):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
try:
return execute_reentrant_pipeline(
pipeline,
create_typed_environment(pipeline, pipeline_run.config),
throw_on_error=False,
reentrant_info=ReentrantInfo(
pipeline_run.run_id,
event_callback=pipeline_run.handle_new_event),
)
except: # pylint: disable=W0702
pipeline_run.handle_new_event(
build_synthetic_pipeline_error_record(
pipeline_run.run_id,
serializable_error_info_from_exc_info(sys.exc_info()),
pipeline.name,
))
