def test_external_execution_input_marshal_code_error(): pipeline = define_inty_pipeline() with pytest.raises(IOError): execute_marshalling( pipeline, ['add_one.transform'], inputs_to_marshal={'add_one.transform': { 'num': 'nope' }}, throw_on_user_error=True, ) step_events = execute_marshalling( pipeline, ['add_one.transform'], inputs_to_marshal={'add_one.transform': { 'num': 'nope' }}, throw_on_user_error=False, ) assert len(step_events) == 1 marshal_step_error = step_events[0] assert marshal_step_error.is_step_failure assert not marshal_step_error.is_successful_output assert marshal_step_error.step.kind == StepKind.UNMARSHAL_INPUT assert isinstance( marshal_step_error.failure_data.dagster_error.user_exception, IOError)
def test_external_execution_step_for_input_missing(): pipeline = define_inty_pipeline() with pytest.raises(DagsterExecutionStepNotFoundError) as exc_info: execute_marshalling(pipeline, ['add_one.transform'], inputs_to_marshal={'nope': { 'nope': 'nope' }}) assert exc_info.value.step_key == 'nope'
def test_external_execution_output_missing(): pipeline = define_inty_pipeline() with pytest.raises(DagsterInvalidSubplanOutputNotFoundError): execute_marshalling( pipeline, ['add_one.transform'], outputs_to_marshal={ 'add_one.transform': [MarshalledOutput('nope', 'nope')] }, )
def test_external_execution_output_code_error_throw_on_user_error(): pipeline = define_inty_pipeline() with pytest.raises(Exception) as exc_info: execute_marshalling( pipeline, ['user_throw_exception.transform'], execution_metadata=ExecutionMetadata(), throw_on_user_error=True, ) assert str(exc_info.value) == 'whoops'
def test_external_execution_step_for_output_missing(): pipeline = define_inty_pipeline() with pytest.raises(DagsterExecutionStepNotFoundError): execute_marshalling( pipeline, ['add_one.transform'], inputs_to_marshal={'add_one.transform': { 'num': 'nope' }}, outputs_to_marshal={'nope': [MarshalledOutput('nope', 'nope')]}, )
def test_external_execution_marshal_wrong_input_error(): pipeline = define_inty_pipeline() with pytest.raises(DagsterInvalidSubplanInputNotFoundError) as exc_info: execute_marshalling( pipeline, ['add_one.transform'], inputs_to_marshal={'add_one.transform': { 'nope': 'nope' }}, ) assert str( exc_info.value) == 'Input nope on add_one.transform does not exist.' assert exc_info.value.pipeline_name == pipeline.name assert exc_info.value.step_keys == ['add_one.transform']
def test_external_execution_unsatisfied_input_error(): pipeline = define_inty_pipeline() with pytest.raises(DagsterInvalidSubplanMissingInputError) as exc_info: execute_marshalling(pipeline, ['add_one.transform'], execution_metadata=ExecutionMetadata()) assert exc_info.value.pipeline_name == 'basic_external_plan_execution' assert exc_info.value.step_keys == ['add_one.transform'] assert exc_info.value.step.key == 'add_one.transform' assert exc_info.value.input_name == 'num' assert str(exc_info.value) == ( "You have specified a subset execution on pipeline basic_external_plan_execution with " "step_keys ['add_one.transform']. You have failed to provide the required input num for " "step add_one.transform.")
def _execute_marshalling_or_error(args, dauphin_pipeline, evaluate_value_result): check.inst_param(args, 'args', SubplanExecutionArgs) check.inst_param(dauphin_pipeline, 'dauphin_pipeline', DauphinPipeline) check.inst_param(evaluate_value_result, 'evaluate_value_result', EvaluateValueResult) try: step_events = execute_marshalling( dauphin_pipeline.get_dagster_pipeline(), step_keys=args.step_keys, inputs_to_marshal=_get_inputs_to_marshal(args), outputs_to_marshal={ se.step_key: se.marshalled_outputs for se in args.step_executions }, environment_dict=evaluate_value_result.value, execution_metadata=args.execution_metadata, throw_on_user_error=False, ) except DagsterInvalidSubplanMissingInputError as invalid_subplan_error: return EitherError( _type_of(args, 'InvalidSubplanMissingInputError')( step=DauphinExecutionStep(invalid_subplan_error.step), missing_input_name=invalid_subplan_error.input_name, )) except DagsterInvalidSubplanOutputNotFoundError as output_not_found_error: return EitherError( _type_of(args, 'StartSubplanExecutionInvalidOutputError')( step=DauphinExecutionStep(output_not_found_error.step), invalid_output_name=output_not_found_error.output_name, )) except DagsterInvalidSubplanInputNotFoundError as input_not_found_error: return EitherError( _type_of(args, 'StartSubplanExecutionInvalidInputError')( step=DauphinExecutionStep(input_not_found_error.step), invalid_input_name=input_not_found_error.input_name, )) except DagsterExecutionStepNotFoundError as step_not_found_error: return EitherError( _type_of(args, 'StartSubplanExecutionInvalidStepError')( invalid_step_key=step_not_found_error.step_key)) # https://github.com/dagster-io/dagster/issues/763 # Once this issue is resolve we should be able to eliminate this except DagsterUserCodeExecutionError as ducee: return EitherError( _type_of(args, 'PythonError')(serializable_error_info_from_exc_info( ducee.original_exc_info))) return _type_of(args, 'StartSubplanExecutionSuccess')( pipeline=dauphin_pipeline, has_failures=any(se for se in step_events if se.is_step_failure), step_events=list(map(_create_dauphin_step_event, step_events)), )
def test_external_execution_marshal_output_code_error(): pipeline = define_inty_pipeline() # guaranteed that folder does not exist hardcoded_uuid = '83fb4ace-5cab-459d-99b6-2ca9808c54a1' outputs_to_marshal = { 'add_one.transform': [ MarshalledOutput( output_name='result', marshalling_key='{uuid}/{uuid}'.format(uuid=hardcoded_uuid)) ] } with pytest.raises(IOError) as exc_info: execute_marshalling( pipeline, ['return_one.transform', 'add_one.transform'], outputs_to_marshal=outputs_to_marshal, execution_metadata=ExecutionMetadata(), throw_on_user_error=True, ) assert 'No such file or directory' in str(exc_info.value) step_events = execute_marshalling( pipeline, ['return_one.transform', 'add_one.transform'], outputs_to_marshal=outputs_to_marshal, execution_metadata=ExecutionMetadata(), throw_on_user_error=False, ) assert len(step_events) == 3 events_dict = {event.step.key: event for event in step_events} assert events_dict['return_one.transform'].is_successful_output is True assert events_dict['add_one.transform'].is_successful_output is True assert events_dict[ 'add_one.transform.marshal-output.result'].is_successful_output is False
def test_external_execution_output_code_error_no_throw_on_user_error(): pipeline = define_inty_pipeline() step_events = execute_marshalling( pipeline, ['user_throw_exception.transform'], execution_metadata=ExecutionMetadata(), throw_on_user_error=False, ) assert len(step_events) == 1 step_event = step_events[0] assert isinstance(step_event.failure_data.dagster_error, DagsterExecutionStepExecutionError) assert str( step_event.failure_data.dagster_error.user_exception) == 'whoops'
def test_basic_pipeline_external_plan_execution(): pipeline = define_inty_pipeline() with get_temp_file_names(2) as temp_files: temp_path, write_path = temp_files # pylint: disable=W0632 int_type = resolve_to_runtime_type(Int) serialize_to_file(int_type.serialization_strategy, 5, temp_path) step_events = execute_marshalling( pipeline, ['add_one.transform'], inputs_to_marshal={'add_one.transform': { 'num': temp_path }}, outputs_to_marshal={ 'add_one.transform': [MarshalledOutput('result', write_path)] }, ) assert deserialize_from_file(int_type.serialization_strategy, write_path) == 6 assert len(step_events) == 2 thunk_step_output_event = step_events[0] assert thunk_step_output_event.kind == StepKind.UNMARSHAL_INPUT transform_step_output_event = step_events[1] assert transform_step_output_event.kind == StepKind.TRANSFORM assert transform_step_output_event.is_successful_output assert transform_step_output_event.success_data.output_name == 'result' assert transform_step_output_event.success_data.value == 6