def test_no_outputs_no_inputs_config_schema(): assert create_typed_environment(no_input_no_output_pipeline()) with pytest.raises(PipelineConfigEvaluationError) as exc_info: create_typed_environment(no_input_no_output_pipeline(), {'solids': {'return_one': {}}}) assert len(exc_info.value.errors) == 1 assert 'Error 1: Undefined field "return_one" at path root:solids' in exc_info.value.message
def test_basic_json_misnamed_output_config_schema(): with pytest.raises(PipelineConfigEvaluationError) as exc_info: create_typed_environment( single_int_named_output_pipeline(), { 'solids': { 'return_named_one': {'outputs': [{'wrong_name': {'json': {'path': 'foo'}}}]} } }, ) assert len(exc_info.value.errors) == 1 assert 'Error 1: Undefined field "wrong_name"' in exc_info.value.message assert 'at path root:solids:return_named_one:outputs[0]' in exc_info.value.message
def test_create_subplan_middle_step(): pipeline_def = define_two_int_pipeline() typed_environment = create_typed_environment(pipeline_def, None) execution_plan = create_execution_plan(pipeline_def) with yield_context(pipeline_def, typed_environment, ExecutionMetadata()) as context: subplan = create_subplan( ExecutionPlanInfo(context=context, pipeline=pipeline_def, environment=typed_environment), StepBuilderState(pipeline_name=pipeline_def.name), execution_plan, ExecutionPlanSubsetInfo(['add_one.transform'], {'add_one.transform': { 'num': 2 }}), ) assert subplan steps = subplan.topological_steps() assert len(steps) == 2 assert steps[0].key == 'add_one.transform.input.num.value' assert not steps[0].step_inputs assert len(steps[0].step_outputs) == 1 assert steps[1].key == 'add_one.transform' assert len(steps[1].step_inputs) == 1 step_input = steps[1].step_inputs[0] assert step_input.prev_output_handle.step.key == 'add_one.transform.input.num.value' assert step_input.prev_output_handle.output_name == VALUE_OUTPUT assert len(steps[1].step_outputs) == 1 assert len(subplan.topological_steps()) == 2 assert [step.key for step in subplan.topological_steps()] == [ 'add_one.transform.input.num.value', 'add_one.transform', ]
def test_basic_json_default_output_config_schema(): env = create_typed_environment( single_int_output_pipeline(), { 'solids': { 'return_one': { 'outputs': [{ 'result': { 'json': { 'path': 'foo' } } }] } } }, ) assert env.solids['return_one'] assert env.solids['return_one'].outputs == [{ 'result': { 'json': { 'path': 'foo' } } }]
def run_test_pipeline(pipeline): typed_environment = create_typed_environment(pipeline, TEST_ENVIRONMENT) execution_metadata = ExecutionMetadata(run_id=str(uuid.uuid4())) with yield_context(pipeline, typed_environment, execution_metadata) as context: execution_plan = create_execution_plan_core( ExecutionPlanInfo(context, pipeline, typed_environment)) return execute_plan(context, execution_plan)
def run_test_pipeline(pipeline): typed_environment = create_typed_environment(pipeline, TEST_ENVIRONMENT) reentrant_info = ReentrantInfo(run_id=str(uuid.uuid4())) with yield_context(pipeline, typed_environment, reentrant_info) as context: execution_plan = create_execution_plan_core( ExecutionPlanInfo(context, pipeline, typed_environment)) with context.value('pipeline', pipeline.display_name): results = execute_plan(context, execution_plan) return results
def test_no_outputs_one_input_config_schema(): assert create_typed_environment( one_input_no_output_pipeline(), {'solids': {'take_input_return_nothing': {'inputs': {'dummy': {'value': 'value'}}}}}, ) with pytest.raises(PipelineConfigEvaluationError) as exc_info: create_typed_environment( one_input_no_output_pipeline(), { 'solids': { 'take_input_return_nothing': { 'inputs': {'dummy': {'value': 'value'}}, 'outputs': {}, } } }, ) assert len(exc_info.value.errors) == 1 exp_msg = 'Error 1: Undefined field "outputs" at path root:solids:take_input_return_nothing' assert exp_msg in exc_info.value.message
def test_create_subplan_source_step(): pipeline_def = define_two_int_pipeline() typed_environment = create_typed_environment(pipeline_def, None) execution_plan = create_execution_plan(pipeline_def) with yield_context(pipeline_def, typed_environment) as context: subplan = create_subplan( ExecutionPlanInfo(context=context, pipeline=pipeline_def, environment=typed_environment), execution_plan, ExecutionPlanSubsetInfo(['return_one.transform']), ) assert subplan assert len(subplan.steps) == 1 assert subplan.steps[0].key == 'return_one.transform' assert not subplan.steps[0].step_inputs assert len(subplan.steps[0].step_outputs) == 1 assert len(subplan.topological_steps()) == 1
def execute_pipeline(self, repository_container, pipeline, pipeline_run): check.inst_param(pipeline, 'pipeline', PipelineDefinition) try: return execute_reentrant_pipeline( pipeline, create_typed_environment(pipeline, pipeline_run.config), throw_on_error=False, reentrant_info=ReentrantInfo( pipeline_run.run_id, event_callback=pipeline_run.handle_new_event), ) except: # pylint: disable=W0702 pipeline_run.handle_new_event( build_synthetic_pipeline_error_record( pipeline_run.run_id, serializable_error_info_from_exc_info(sys.exc_info()), pipeline.name, ))