def test_implementation_error_in_execute(self): test_case = _single_successful_instruction_in_each_phase() self._check( Arrangement(test_case, act_executor_execute=execute_action_that_raises( test.ImplementationErrorTestException())), Expectation( asrt_result.matches2(PartialExeResultStatus.IMPLEMENTATION_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_exception( phase_step.ACT__EXECUTE, test.ImplementationErrorTestException) ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, (phase_step.CLEANUP__MAIN, PreviousPhase.ACT), ], ))
def test_validation_error_in_validate_post_setup(self): test_case = _single_successful_instruction_in_each_phase() self._check( Arrangement(test_case, act_executor_validate_post_setup=validate_action_that_returns( svh.new_svh_validation_error('error in act/validate-post-setup'))), Expectation( asrt_result.matches2(PartialExeResultStatus.VALIDATION_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_message( phase_step.ACT__VALIDATE_POST_SETUP, 'error in act/validate-post-setup'), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [ phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), ], ))
def test_internal_error(self): test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(PartialPhase.SETUP, test.setup_phase_instruction_that( main=do_raise(test.ImplementationErrorTestException()))) self._check( Arrangement(test_case, atc_execute=execute_action_that_returns_exit_code(5)), Expectation( asrt_result.matches2( ExecutionFailureStatus.INTERNAL_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_exception( phase_step.SETUP__MAIN, test_case.the_extra(PartialPhase.SETUP)[0].source, test.ImplementationErrorTestException), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [ phase_step.SETUP__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), ], ))
def test_hard_error_in_setup_main_step(self): py_pgm_setup = PyProgramSetup('some output to stdout', 'some output to stderr', 72) test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr( act_phase_source=py_pgm_setup.as_line_sequence()) \ .add(PartialPhase.SETUP, test.setup_phase_instruction_that( main=do_return(sh.new_sh_hard_error('hard error msg from setup')))) self._check( arr_for_py3_source(test_case), Expectation( asrt_result.matches2( PartialExeResultStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_message( phase_step.SETUP__MAIN, test_case.the_extra(PartialPhase.SETUP)[0].source, 'hard error msg from setup'), ), atc_stdout_output=asrt.equals(''), atc_stderr_output=asrt.equals(''), step_recordings= [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [phase_step.SETUP__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), ], ))
def test_internal_error(self): test_case = _single_successful_instruction_in_each_phase() self._check( Arrangement(test_case, atc_execute=execute_action_that_raises( test.ImplementationErrorTestException())), Expectation( asrt_result.matches2( ExecutionFailureStatus.INTERNAL_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_exception( phase_step.ACT__EXECUTE, test.ImplementationErrorTestException)), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [ phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_EXE_INPUT, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, (phase_step.CLEANUP__MAIN, PreviousPhase.ACT), ], ))
def test_symbol_that_does_not_meet_restriction_in_validate_symbols(self): test_case = _single_successful_instruction_in_each_phase() symbol_name = 'symbol_name' reference_to_string_symbol = data_references.reference_to__on_direct_and_indirect( symbol_name, value_restrictions.is_string()) definition_of_path_symbol = arbitrary_path_symbol_context( symbol_name).definition symbol_usages = [ definition_of_path_symbol, reference_to_string_symbol, ] self._check( Arrangement(test_case, atc_symbol_usages=do_return(symbol_usages)), Expectation( asrt_result.matches2( ExecutionFailureStatus.VALIDATION_ERROR, asrt_result.has_no_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_step( phase_step.ACT__VALIDATE_SYMBOLS), ), [ phase_step.ACT__PARSE, phase_step.SETUP__VALIDATE_SYMBOLS, phase_step.ACT__VALIDATE_SYMBOLS, ], ))
def test_internal_error_in_act_execute(self): test_case = _single_successful_instruction_in_each_phase( single_line_sequence(72, 'ignored')) self._check( Arrangement( test_case, ActorThatRunsConstantActions( execute_action=execute_action_that_raises( test.ImplementationErrorTestException()))), Expectation( asrt_result.matches2( ExecutionFailureStatus.INTERNAL_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_exception( phase_step.ACT__EXECUTE, test.ImplementationErrorTestException)), atc_stdout_output=asrt.equals(''), atc_stderr_output=asrt.equals(''), step_recordings=[phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [ phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_EXE_INPUT, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, (phase_step.CLEANUP__MAIN, PreviousPhase.ACT), ], ))
def test_implementation_error_in_setup_main_step(self): test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(PartialPhase.SETUP, test.setup_phase_instruction_that( main=do_raise(test.ImplementationErrorTestException()))) self._check( Arrangement(test_case, act_executor_execute=execute_action_that_returns_exit_code(5)), Expectation( asrt_result.matches2( PartialExeResultStatus.IMPLEMENTATION_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_exception( phase_step.SETUP__MAIN, test_case.the_extra(PartialPhase.SETUP)[0].source, test.ImplementationErrorTestException), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [phase_step.SETUP__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), ], ))
def runTest(self): conf = self.configuration defined_symbol = data_symbol_utils.string_symbol_definition('symbol_name') error_message_for_failed_restriction = 'error message' reference_with_restriction_failure = SymbolReference( defined_symbol.name, ReferenceRestrictionsOnDirectAndIndirect( direct=value_restriction_that_is_unconditionally_unsatisfied(error_message_for_failed_restriction))) test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(PartialPhase.SETUP, setup_phase_instruction_that(symbol_usages=do_return([defined_symbol]))) \ .add(conf.phase, conf.instruction_that_returns([reference_with_restriction_failure])) execute_test_case_with_recording( self, Arrangement(test_case), Expectation( asrt_result.matches2( PartialExeResultStatus.VALIDATION_ERROR, asrt_result.has_no_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_phase_and_message_assertion( conf.step, asrt.equals(error_message_for_failed_restriction)), ), conf.expected_steps_before_failing_instruction, ) )
def test_hard_error(self): test_case = _single_successful_instruction_in_each_phase() self._check( Arrangement( test_case, atc_execute=execute_action_that_returns_hard_error_with_message( 'error in execute')), Expectation( asrt_result.matches2( ExecutionFailureStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_message( phase_step.ACT__EXECUTE, 'error in execute')), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [ phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_EXE_INPUT, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, (phase_step.CLEANUP__MAIN, PreviousPhase.ACT), ], ))
def test_hard_error_in_prepare(self): test_case = _single_successful_instruction_in_each_phase() self._check( Arrangement(test_case, act_executor_prepare=prepare_action_that_returns_hard_error_with_message( 'error in act/prepare')), Expectation( asrt_result.matches2(PartialExeResultStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_message( phase_step.ACT__PREPARE, 'error in act/prepare') ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__PREPARE, (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), ], ))
def test_hard_error_exception(self): failure_message = 'HE exception in act/validate exe info' def validator_that_raises_hard_error() -> Optional[TextRenderer]: raise hard_error_ex(failure_message) test_case = _single_successful_instruction_in_each_phase() self._check( Arrangement(test_case), custom_act_execution_input_validator= validator_that_raises_hard_error, expectation=Expectation( asrt_result.matches2( ExecutionFailureStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_message( phase_step.ACT__VALIDATE_EXE_INPUT, failure_message)), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [ phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_EXE_INPUT, (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), ], ))
def test_internal_error(self): def validator_that_raises_exception() -> Optional[TextRenderer]: raise test.ImplementationErrorTestException() test_case = _single_successful_instruction_in_each_phase() self._check(Arrangement(test_case, ), custom_act_execution_input_validator= validator_that_raises_exception, expectation=Expectation( asrt_result.matches2( ExecutionFailureStatus.INTERNAL_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_exception( phase_step.ACT__VALIDATE_EXE_INPUT, test.ImplementationErrorTestException), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [ phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_EXE_INPUT, (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), ], ))
def test_hard_error_in_setup_main_step(self): test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(PartialPhase.SETUP, test.setup_phase_instruction_that( main=do_return(sh.new_sh_hard_error('hard error msg from setup')))) self._check( Arrangement(test_case), Expectation( asrt_result.matches2( PartialExeResultStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_message( phase_step.SETUP__MAIN, test_case.the_extra(PartialPhase.SETUP)[0].source, 'hard error msg from setup'), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [phase_step.SETUP__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), ], ))
def test_symbol_that_does_not_meet_restriction_in_validate_symbols(self): test_case = _single_successful_instruction_in_each_phase() symbol_name = 'symbol_name' reference_to_string_symbol = data_symbol_utils.symbol_reference(symbol_name, StringRestriction()) definition_of_path_symbol = data_symbol_utils.file_ref_symbol_definition(symbol_name) symbol_usages = [ definition_of_path_symbol, reference_to_string_symbol, ] self._check( Arrangement(test_case, act_executor_symbol_usages=do_return(symbol_usages)), Expectation( asrt_result.matches2( PartialExeResultStatus.VALIDATION_ERROR, asrt_result.has_no_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_step(phase_step.ACT__VALIDATE_SYMBOLS), ), [ phase_step.ACT__PARSE, phase_step.SETUP__VALIDATE_SYMBOLS, phase_step.ACT__VALIDATE_SYMBOLS, ], ))
def test_hard_error_in_setup_main_step(self): py_pgm_setup = PyProgramSetup('some output to stdout', 'some output to stderr', 72) test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr( act_phase_source=py_pgm_setup.as_line_sequence()) \ .add(PartialPhase.SETUP, test.setup_phase_instruction_that( main=do_return(sh.new_sh_hard_error__str('hard error msg from setup')))) self._check( arr_for_py3_source(test_case), Expectation( asrt_result.matches2( ExecutionFailureStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_message( phase_step.SETUP__MAIN, test_case.the_extra(PartialPhase.SETUP)[0].source, 'hard error msg from setup'), ), atc_stdout_output=asrt.equals(''), atc_stderr_output=asrt.equals(''), step_recordings=[phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [ phase_step.SETUP__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), ], ))
def test_implementation_error_in_act_execute(self): test_case = _single_successful_instruction_in_each_phase(single_line_sequence(72, 'ignored')) self._check( Arrangement(test_case, ActorThatRunsConstantActions( execute_action=execute_action_that_raises( test.ImplementationErrorTestException()))), Expectation( asrt_result.matches2(PartialExeResultStatus.IMPLEMENTATION_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_exception( phase_step.ACT__EXECUTE, test.ImplementationErrorTestException) ), atc_stdout_output=asrt.equals(''), atc_stderr_output=asrt.equals(''), step_recordings= [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, (phase_step.CLEANUP__MAIN, PreviousPhase.ACT), ], ))
def test_timeout_in_action_to_check(self): stdout_before_sleep = 'some output on stdout before going into sleep' stderr_before_sleep = 'some output on stderr before going into sleep' py_pgm_line_sequence = py_pgm_with_stdout_stderr_and_sleep_in_between( stdout_output_before_sleep=stdout_before_sleep, stderr_output_before_sleep=stderr_before_sleep, stdout_output_after_sleep='more stdout output after sleep', stderr_output_after_sleep='more stderr output after sleep', sleep_seconds=3, exit_code=72) test_case = _single_successful_instruction_in_each_phase( act_phase_source=py_pgm_line_sequence) self._check( arr_for_py3_source(test_case, timeout_in_seconds=1), Expectation( asrt_result.matches2(PartialExeResultStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure( phase_step.ACT__EXECUTE, asrt.anything_goes()) ), atc_stdout_output=asrt.equals(stdout_before_sleep), atc_stderr_output=asrt.equals(stderr_before_sleep), step_recordings= [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, (phase_step.CLEANUP__MAIN, PreviousPhase.ACT), ], ))
def test(self): for expected_exit_code in [0, 69]: py_pgm_setup = PyProgramSetup('the output to stdout', 'the output to stderr', expected_exit_code) with self.subTest(expected_exit_code=expected_exit_code): self._check( arr_for_py3_source(TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr( act_phase_source=py_pgm_setup.as_line_sequence() ), ), Expectation( asrt_result.matches2( PartialExeResultStatus.PASS, asrt_result.has_sds(), asrt_result.has_action_to_check_outcome_with_exit_code(py_pgm_setup.exit_code), ExpectedFailureForNoFailure(), ), atc_stdout_output=asrt.equals(py_pgm_setup.stdout_output), atc_stderr_output=asrt.equals(py_pgm_setup.stderr_output), step_recordings= [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [phase_step.SETUP__MAIN, phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, (phase_step.CLEANUP__MAIN, PreviousPhase.ACT), (phase_step.CLEANUP__MAIN, PreviousPhase.ACT), ], ))
def test_exception(self): test_case = _single_successful_instruction_in_each_phase() self._check( Arrangement(test_case, atc_validate_pre_sds=validate_action_that_raises( test.ImplementationErrorTestException())), Expectation( asrt_result.matches2( ExecutionFailureStatus.INTERNAL_ERROR, asrt_result.has_no_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_exception( phase_step.ACT__VALIDATE_PRE_SDS, test.ImplementationErrorTestException)), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + [ phase_step.SETUP__VALIDATE_PRE_SDS, phase_step.ACT__VALIDATE_PRE_SDS, ], ))
def runTest(self): conf = self.configuration test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(conf.phase, conf.instruction_that_raises(hard_error_ex('Error message from hard error exception'))) execute_test_case_with_recording( self, Arrangement(test_case), Expectation( asrt_result.matches2( ExecutionFailureStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_message( conf.step, test_case.the_extra(conf.phase)[0].source, 'Error message from hard error exception'), ), conf.expected_steps, ))
def runTest(self): conf = self.configuration test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(conf.phase, conf.instruction_that_returns(svh.new_svh_validation_error__str('validation error message'))) execute_test_case_with_recording( self, Arrangement(test_case), Expectation( asrt_result.matches2( ExecutionFailureStatus.VALIDATION_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_message( conf.step, test_case.the_extra(conf.phase)[0].source, 'validation error message'), ), conf.expected_steps, ))
def runTest(self): conf = self.configuration test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(conf.phase, conf.instruction_that_raises(test.ImplementationErrorTestException())) execute_test_case_with_recording( self, Arrangement(test_case), Expectation( asrt_result.matches2( ExecutionFailureStatus.INTERNAL_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_exception( conf.step, test_case.the_extra(conf.phase)[0].source, test.ImplementationErrorTestException), ), conf.expected_steps, ))
def test_implementation_error_in_validate_symbols(self): test_case = _single_successful_instruction_in_each_phase() self._check( Arrangement(test_case, act_executor_symbol_usages=do_raise(test.ImplementationErrorTestException())), Expectation( asrt_result.matches2(PartialExeResultStatus.IMPLEMENTATION_ERROR, asrt_result.has_no_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_exception( phase_step.ACT__VALIDATE_SYMBOLS, test.ImplementationErrorTestException) ), [ phase_step.ACT__PARSE, phase_step.SETUP__VALIDATE_SYMBOLS, phase_step.ACT__VALIDATE_SYMBOLS, ], ))
def test(self): for expected_exit_code in [0, 69]: py_pgm_setup = PyProgramSetup('the output to stdout', 'the output to stderr', expected_exit_code) with self.subTest(expected_exit_code=expected_exit_code): self._check( arr_for_py3_source( TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr( act_phase_source=py_pgm_setup.as_line_sequence()), ), Expectation( asrt_result.matches2( None, asrt_result.has_sds(), asrt_result. has_action_to_check_outcome_with_exit_code( py_pgm_setup.exit_code), ExpectedFailureForNoFailure(), ), atc_stdout_output=asrt.equals( py_pgm_setup.stdout_output), atc_stderr_output=asrt.equals( py_pgm_setup.stderr_output), step_recordings=[phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [ phase_step.SETUP__MAIN, phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_EXE_INPUT, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, (phase_step.CLEANUP__MAIN, PreviousPhase.ACT), (phase_step.CLEANUP__MAIN, PreviousPhase.ACT), ], ))
def test_hard_error_in_cleanup_main_step(self): test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(PartialPhase.CLEANUP, test.cleanup_phase_instruction_that( main=do_return(sh.new_sh_hard_error('hard error msg from CLEANUP')))) self._check( Arrangement(test_case, act_executor_execute=execute_action_that_returns_exit_code(3)), Expectation( asrt_result.matches2( PartialExeResultStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_action_to_check_outcome_with_exit_code(3), ExpectedFailureForInstructionFailure.new_with_message( phase_step.CLEANUP__MAIN, test_case.the_extra(PartialPhase.CLEANUP)[0].source, 'hard error msg from CLEANUP'), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [phase_step.SETUP__MAIN, phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, phase_step.BEFORE_ASSERT__MAIN, phase_step.BEFORE_ASSERT__MAIN, phase_step.ASSERT__MAIN, phase_step.ASSERT__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.ASSERT), ], ))
def test_reference_to_undefined_symbol_in_validate_symbols(self): test_case = _single_successful_instruction_in_each_phase() symbol_usages_with_ref_to_undefined_symbol = [symbol_reference('undefined_symbol')] self._check( Arrangement(test_case, act_executor_symbol_usages=do_return(symbol_usages_with_ref_to_undefined_symbol)), Expectation( asrt_result.matches2( PartialExeResultStatus.VALIDATION_ERROR, asrt_result.has_no_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_step(phase_step.ACT__VALIDATE_SYMBOLS), ), [ phase_step.ACT__PARSE, phase_step.SETUP__VALIDATE_SYMBOLS, phase_step.ACT__VALIDATE_SYMBOLS, ], ))
def runTest(self): conf = self.configuration test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(conf.phase, conf.instruction_that_raises(test.ImplementationErrorTestException())) execute_test_case_with_recording( self, Arrangement(test_case), Expectation( asrt_result.matches2( PartialExeResultStatus.IMPLEMENTATION_ERROR, asrt_result.has_no_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_exception( conf.step, test_case.the_extra(conf.phase)[0].source, test.ImplementationErrorTestException), ), conf.expected_steps_before_failing_instruction, ))
def test_hard_error_exception(self): test_case = _single_successful_instruction_in_each_phase() self._check( Arrangement( test_case, atc_validate_pre_sds=validate_action_that_raises( hard_error_ex('HE exception in act/validate-pre-sds'))), Expectation( asrt_result.matches2( ExecutionFailureStatus.HARD_ERROR, asrt_result.has_no_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_message( phase_step.ACT__VALIDATE_PRE_SDS, 'HE exception in act/validate-pre-sds')), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + [ phase_step.SETUP__VALIDATE_PRE_SDS, phase_step.ACT__VALIDATE_PRE_SDS, ], ))
def runTest(self): conf = self.configuration test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(conf.phase, conf.instruction_that_returns(svh.new_svh_hard_error('Error message from hard error'))) execute_test_case_with_recording( self, Arrangement(test_case), Expectation( asrt_result.matches2( PartialExeResultStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_message( conf.step, test_case.the_extra(conf.phase)[0].source, 'Error message from hard error'), ), conf.expected_steps, ))
def test_internal_error_in_validate_symbols(self): test_case = _single_successful_instruction_in_each_phase() self._check( Arrangement(test_case, atc_symbol_usages=do_raise( test.ImplementationErrorTestException())), Expectation( asrt_result.matches2( ExecutionFailureStatus.INTERNAL_ERROR, asrt_result.has_no_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_exception( phase_step.ACT__VALIDATE_SYMBOLS, test.ImplementationErrorTestException)), [ phase_step.ACT__PARSE, phase_step.SETUP__VALIDATE_SYMBOLS, phase_step.ACT__VALIDATE_SYMBOLS, ], ))
def test(self): for expected_exit_code in [0, 72]: with self.subTest(expected_exit_code=expected_exit_code): self._check( Arrangement( TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr( ), atc_execute=execute_action_that_returns_exit_code( expected_exit_code)), Expectation( asrt_result.matches2( None, asrt_result.has_sds(), asrt_result. has_action_to_check_outcome_with_exit_code( expected_exit_code), ExpectedFailureForNoFailure(), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [ phase_step.SETUP__MAIN, phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_EXE_INPUT, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, phase_step.BEFORE_ASSERT__MAIN, phase_step.BEFORE_ASSERT__MAIN, phase_step.ASSERT__MAIN, phase_step.ASSERT__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.ASSERT), (phase_step.CLEANUP__MAIN, PreviousPhase.ASSERT), ], ))
def test(self): for expected_exit_code in [0, 72]: with self.subTest(expected_exit_code=expected_exit_code): self._check( Arrangement(TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr(), act_executor_execute=execute_action_that_returns_exit_code(expected_exit_code)), Expectation( asrt_result.matches2( PartialExeResultStatus.PASS, asrt_result.has_sds(), asrt_result.has_action_to_check_outcome_with_exit_code(expected_exit_code), ExpectedFailureForNoFailure(), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [phase_step.SETUP__MAIN, phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, phase_step.BEFORE_ASSERT__MAIN, phase_step.BEFORE_ASSERT__MAIN, phase_step.ASSERT__MAIN, phase_step.ASSERT__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.ASSERT), (phase_step.CLEANUP__MAIN, PreviousPhase.ASSERT), ], ))
def test_symbol_that_does_meet_restriction_in_validate_symbols(self): test_case = _single_successful_instruction_in_each_phase() symbol_name = 'symbol_name' reference_to_string_symbol = data_symbol_utils.symbol_reference(symbol_name, StringRestriction()) definition_of_string_symbol = data_symbol_utils.string_symbol_definition(symbol_name) symbol_usages = [ definition_of_string_symbol, reference_to_string_symbol, ] self._check( Arrangement(test_case, act_executor_symbol_usages=do_return(symbol_usages), act_executor_execute=execute_action_that_returns_exit_code(128)), Expectation( asrt_result.matches2( PartialExeResultStatus.PASS, asrt_result.has_sds(), asrt_result.has_action_to_check_outcome_with_exit_code(128), ExpectedFailureForNoFailure(), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, phase_step.BEFORE_ASSERT__MAIN, phase_step.ASSERT__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.ASSERT), ], ))
def test_hard_error_in_validate_pre_sds(self): test_case = _single_successful_instruction_in_each_phase() self._check( Arrangement(test_case, act_executor_validate_pre_sds=validate_action_that_returns( svh.new_svh_hard_error('error in act/validate-pre-sds'))), Expectation( asrt_result.matches2(PartialExeResultStatus.HARD_ERROR, asrt_result.has_no_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_message( phase_step.ACT__VALIDATE_PRE_SDS, 'error in act/validate-pre-sds') ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + [ phase_step.SETUP__VALIDATE_PRE_SDS, phase_step.ACT__VALIDATE_PRE_SDS, ], ))
def runTest(self): conf = self.configuration test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(conf.phase, conf.instruction_that_returns([element_reference('undefined symbol')])) execute_test_case_with_recording( self, Arrangement(test_case), Expectation( asrt_result.matches2( PartialExeResultStatus.VALIDATION_ERROR, asrt_result.has_no_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_message_assertion( conf.step, test_case.the_extra(conf.phase)[0].source, asrt.is_instance(str)), ), conf.expected_steps_before_failing_instruction, ) )
def test_hard_error_exception(self): test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(PartialPhase.CLEANUP, test.cleanup_phase_instruction_that( main=do_raise(hard_error_ex('hard error exception msg from CLEANUP')))) self._check( Arrangement(test_case, atc_execute=execute_action_that_returns_exit_code(3)), Expectation( asrt_result.matches2( ExecutionFailureStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_action_to_check_outcome_with_exit_code(3), ExpectedFailureForInstructionFailure.new_with_message( phase_step.CLEANUP__MAIN, test_case.the_extra(PartialPhase.CLEANUP)[0].source, 'hard error exception msg from CLEANUP'), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [ phase_step.SETUP__MAIN, phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_EXE_INPUT, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, phase_step.BEFORE_ASSERT__MAIN, phase_step.BEFORE_ASSERT__MAIN, phase_step.ASSERT__MAIN, phase_step.ASSERT__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.ASSERT), ], ))
def test_fail(self): test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(PartialPhase.ASSERT, test.assert_phase_instruction_that( main=do_return(pfh.new_pfh_fail__str('fail msg from ASSERT')))) self._check( Arrangement(test_case, atc_execute=execute_action_that_returns_exit_code(5)), Expectation( asrt_result.matches2( ExecutionFailureStatus.FAIL, asrt_result.has_sds(), asrt_result.has_action_to_check_outcome_with_exit_code(5), ExpectedFailureForInstructionFailure.new_with_message( phase_step.ASSERT__MAIN, test_case.the_extra(PartialPhase.ASSERT)[0].source, 'fail msg from ASSERT'), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [ phase_step.SETUP__MAIN, phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_EXE_INPUT, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, phase_step.BEFORE_ASSERT__MAIN, phase_step.BEFORE_ASSERT__MAIN, phase_step.ASSERT__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.ASSERT), (phase_step.CLEANUP__MAIN, PreviousPhase.ASSERT), ], ))
def test_symbol_that_does_meet_restriction_in_validate_symbols(self): test_case = _single_successful_instruction_in_each_phase() symbol_name = 'symbol_name' reference_to_string_symbol = data_references.reference_to__on_direct_and_indirect( symbol_name, value_restrictions.is_string()) definition_of_string_symbol = StringConstantSymbolContext( symbol_name).definition symbol_usages = [ definition_of_string_symbol, reference_to_string_symbol, ] self._check( Arrangement( test_case, atc_symbol_usages=do_return(symbol_usages), atc_execute=execute_action_that_returns_exit_code(128)), Expectation( asrt_result.matches2( None, asrt_result.has_sds(), asrt_result.has_action_to_check_outcome_with_exit_code( 128), ExpectedFailureForNoFailure(), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [ phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_EXE_INPUT, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, phase_step.BEFORE_ASSERT__MAIN, phase_step.ASSERT__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.ASSERT), ], ))
def test_hard_error(self): test_case = _single_successful_instruction_in_each_phase() self._check( Arrangement(test_case, atc_validate_post_setup=validate_action_that_returns( svh.new_svh_hard_error__str( 'error in act/validate-post-setup'))), Expectation( asrt_result.matches2( ExecutionFailureStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_message( phase_step.ACT__VALIDATE_POST_SETUP, 'error in act/validate-post-setup')), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [ phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), ], ))
def test_timeout_in_action_to_check(self): stdout_before_sleep = 'some output on stdout before going into sleep' stderr_before_sleep = 'some output on stderr before going into sleep' py_pgm_line_sequence = py_pgm_with_stdout_stderr_and_sleep_in_between( stdout_output_before_sleep=stdout_before_sleep, stderr_output_before_sleep=stderr_before_sleep, stdout_output_after_sleep='more stdout output after sleep', stderr_output_after_sleep='more stderr output after sleep', sleep_seconds=3, exit_code=72) test_case = _single_successful_instruction_in_each_phase( act_phase_source=py_pgm_line_sequence) self._check( arr_for_py3_source(test_case, timeout_in_seconds=1), Expectation( asrt_result.matches2( ExecutionFailureStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure(phase_step.ACT__EXECUTE, asrt.anything_goes())), atc_stdout_output=asrt.equals(stdout_before_sleep), atc_stderr_output=asrt.equals(stderr_before_sleep), step_recordings=[phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__ONCE + PRE_SDS_VALIDATION_STEPS__ONCE + [ phase_step.SETUP__MAIN, phase_step.SETUP__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_POST_SETUP, phase_step.BEFORE_ASSERT__VALIDATE_POST_SETUP, phase_step.ASSERT__VALIDATE_POST_SETUP, phase_step.ACT__VALIDATE_EXE_INPUT, phase_step.ACT__PREPARE, phase_step.ACT__EXECUTE, (phase_step.CLEANUP__MAIN, PreviousPhase.ACT), ], ))
def test_hard_error_exception(self): test_case = TestCaseGeneratorWithExtraInstrsBetweenRecordingInstr() \ .add(PartialPhase.SETUP, test.setup_phase_instruction_that( main=do_raise(hard_error_ex('hard error exception msg from setup')))) self._check( Arrangement(test_case), Expectation( asrt_result.matches2( ExecutionFailureStatus.HARD_ERROR, asrt_result.has_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForInstructionFailure.new_with_message( phase_step.SETUP__MAIN, test_case.the_extra(PartialPhase.SETUP)[0].source, 'hard error exception msg from setup'), ), [phase_step.ACT__PARSE] + SYMBOL_VALIDATION_STEPS__TWICE + PRE_SDS_VALIDATION_STEPS__TWICE + [ phase_step.SETUP__MAIN, (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), (phase_step.CLEANUP__MAIN, PreviousPhase.SETUP), ], ))
def test_reference_to_undefined_symbol_in_validate_symbols(self): test_case = _single_successful_instruction_in_each_phase() symbol_usages_with_ref_to_undefined_symbol = [ data_references.reference_to__on_direct_and_indirect( 'undefined_symbol') ] self._check( Arrangement(test_case, atc_symbol_usages=do_return( symbol_usages_with_ref_to_undefined_symbol)), Expectation( asrt_result.matches2( ExecutionFailureStatus.VALIDATION_ERROR, asrt_result.has_no_sds(), asrt_result.has_no_action_to_check_outcome(), ExpectedFailureForPhaseFailure.new_with_step( phase_step.ACT__VALIDATE_SYMBOLS), ), [ phase_step.ACT__PARSE, phase_step.SETUP__VALIDATE_SYMBOLS, phase_step.ACT__VALIDATE_SYMBOLS, ], ))