def test_suite_with_only_single_sub_suite_SHOULD_not_include_root_suite_as_test_suite(self): # ARRANGE # suite_with_single_case = test_suite('suite with single case', [], [ test_case('the test case') ]) root_suite = test_suite('root suite file name', [ suite_with_single_case, ], []) suites = [ root_suite, suite_with_single_case, ] # ACT # actual = execute_with_case_processing_with_constant_result(tcp.new_executed(FULL_RESULT_PASS), root_suite, Path(), suites) # ASSERT # expected_xml = _suites_xml([ suite_xml(attributes={ 'name': 'suite with single case', 'package': '.', 'id': '1', 'tests': '1', 'errors': '0', 'failures': '0', }, test_case_elements=[successful_test_case_xml('the test case')] ), ]) expected_output = expected_output_from(expected_xml) self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code) self.assertEqual(expected_output, replace_xml_variables(actual.stdout))
def test_suite_with_only_single_sub_suite_SHOULD_not_include_root_suite_as_test_suite(self): # ARRANGE # suite_with_single_case = test_suite('suite with single case', [], [ test_case('the test case') ]) root_suite = test_suite('root suite file name', [ suite_with_single_case, ], []) suites = [ root_suite, suite_with_single_case, ] # ACT # actual = execute_with_case_processing_with_constant_result(tcp.new_executed(FULL_RESULT_PASS), root_suite, Path(), suites) # ASSERT # expected_xml = _suites_xml([ suite_xml(attributes={ 'name': 'suite with single case', 'package': '.', 'id': '1', 'tests': '1', 'errors': '0', 'failures': '0', }, test_case_elements=[successful_test_case_xml('the test case')] ), ]) expected__assertion = asrt_etree.str_as_xml_equals(expected_xml) self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code) expected__assertion.apply_with_message(self, replace_xml_variables(actual.stdout), 'suite xml on stdout')
def runTest(self): # ARRANGE # path_cases = [ NameAndValue( 'relative file name', TestCaseFileReference(Path('test.case'), Path('reference') / 'relativity')), NameAndValue( 'absolute file name', TestCaseFileReference(Path.cwd() / 'test.case', Path.cwd())), ] for path_case in path_cases: test_case_paths_expectation = asrt.matches_sequence( [equals_test_case_reference(path_case.value)]) suite_with_case = test_suite( source_file_name='ignored', sub_test_suites=[], test_cases=[path_case.value], ) suite_cases = [ NameAndValue( 'case in root suite', suite_with_case, ), NameAndValue( 'case in sub suite', test_suite(source_file_name='ignored', sub_test_suites=[suite_with_case], test_cases=[]), ), ] for suite_case in suite_cases: with self.subTest(path_case=path_case.name, suite_case=suite_case.name): suite_hierarchy_reader = ReaderThatGivesConstantSuite( suite_case.value) reporter = ProcessingReporterThatDoesNothing() path_registering_processor = TestCaseProcessorThatJustRegistersTestCaseFileReference( ) processor = sut.Processor( DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter, DepthFirstEnumerator(), lambda config: path_registering_processor) # ACT # return_value = processor.process( suite_case.value.source_file, null_output_reporting_environment()) # ASSERT # self.assertEqual( ProcessingReporterThatDoesNothing. VALID_SUITE_EXIT_CODE, return_value, 'Sanity check of result indicator') test_case_paths_expectation.apply_without_message( self, path_registering_processor.recording_media)
def runTest(self): # ARRANGE # path_cases = [ NameAndValue('relative file name', TestCaseFileReference(Path('test.case'), Path('reference') / 'relativity') ), NameAndValue('absolute file name', TestCaseFileReference(Path.cwd() / 'test.case', Path.cwd())), ] for path_case in path_cases: test_case_file_refs_expectation = asrt.matches_sequence([ equals_test_case_reference(path_case.value) ]) suite_with_case = test_suite( source_file_name='ignored', sub_test_suites=[], test_cases=[path_case.value], ) suite_cases = [ NameAndValue('case in root suite', suite_with_case, ), NameAndValue('case in sub suite', test_suite( source_file_name='ignored', sub_test_suites=[suite_with_case], test_cases=[] ), ), ] for suite_case in suite_cases: with self.subTest(path_case=path_case.name, suite_case=suite_case.name): suite_hierarchy_reader = ReaderThatGivesConstantSuite(suite_case.value) reporter = ProcessingReporterThatDoesNothing() file_ref_registering_processor = TestCaseProcessorThatJustRegistersTestCaseFileReference() processor = sut.Processor(DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter, DepthFirstEnumerator(), lambda config: file_ref_registering_processor) # ACT # return_value = processor.process(suite_case.value.source_file, null_output_files()) # ASSERT # self.assertEqual(ProcessingReporterThatDoesNothing.VALID_SUITE_EXIT_CODE, return_value, 'Sanity check of result indicator') test_case_file_refs_expectation.apply_without_message( self, file_ref_registering_processor.recording_media )
def test_single_case_with_error(self): cases = [ FULL_RESULT_HARD_ERROR, FULL_RESULT_VALIDATE, FULL_RESULT_INTERNAL_ERROR, ] for case_result in cases: with self.subTest(case_result_status=case_result.status): # ARRANGE # expected_xml = suite_xml(attributes={ 'name': 'suite with error', 'tests': '1', 'errors': '1', 'failures': '0'}, test_case_elements=[ erroneous_test_case_xml('test case file name', error_type=case_result.status.name, failure_message=error_message_for_full_result(case_result))]) expected__assertion = asrt_etree.str_as_xml_equals(expected_xml) root_suite = test_suite('suite with error', [], [test_case('test case file name')]) test_suites = [root_suite] # ACT # actual = execute_with_case_processing_with_constant_result( tcp.new_executed(case_result), root_suite, Path(), test_suites) # ASSERT # self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code) expected__assertion.apply_with_message(self, replace_xml_variables(actual.stdout), 'suite xml on stdout')
def test_empty_suite(self): # ARRANGE # expected_xml = suite_xml( attributes={ 'name': 'root file name', 'tests': '0', 'errors': '0', 'failures': '0', }, test_case_elements=[], ) expected__assertion = asrt_etree.str_as_xml_equals(expected_xml) root_suite = test_suite('root file name', [], []) test_suites = [root_suite] # ACT # actual = execute_with_case_processing_with_constant_result(FULL_RESULT_PASS, root_suite, Path(), test_suites) # ASSERT # self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code) expected__assertion.apply_with_message(self, replace_xml_variables(actual.stdout), 'suite xml on stdout')
def test_single_case_that_passes(self): cases = [ FULL_RESULT_PASS, FULL_RESULT_XFAIL, FULL_RESULT_SKIP, ] for case_result in cases: with self.subTest(case_result_status=case_result.status): # ARRANGE # expected_xml = suite_xml(attributes={ 'name': 'suite that passes', 'tests': '1', 'errors': '0', 'failures': '0', }, test_case_elements=[successful_test_case_xml('test case file name')] ) expected__assertion = asrt_etree.str_as_xml_equals(expected_xml) root_suite = test_suite('suite that passes', [], [test_case('test case file name')]) test_suites = [root_suite] # ACT # actual = execute_with_case_processing_with_constant_result( tcp.new_executed(case_result), root_suite, Path(), test_suites) # ASSERT # self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code) expected__assertion.apply_with_message(self, replace_xml_variables(actual.stdout), 'suite xml on stdout')
def test_single_case_that_passes(self): cases = [ FULL_RESULT_PASS, FULL_RESULT_XFAIL, FULL_RESULT_SKIP, ] for case_result in cases: with self.subTest(case_result_status=case_result.status): # ARRANGE # expected_xml = suite_xml(attributes={ 'name': 'suite that passes', 'tests': '1', 'errors': '0', 'failures': '0', }, test_case_elements=[successful_test_case_xml('test case file name')] ) expected_output = expected_output_from(expected_xml) root_suite = test_suite('suite that passes', [], [test_case('test case file name')]) test_suites = [root_suite] # ACT # actual = execute_with_case_processing_with_constant_result( tcp.new_executed(case_result), root_suite, Path(), test_suites) # ASSERT # self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code) self.assertEqual(expected_output, replace_xml_variables(actual.stdout))
def test_single_case_with_error_due_to_failure_to_execute(self): cases = [ (tcp.new_internal_error(error_info.of_message('error message')), tcp.Status.INTERNAL_ERROR.name), (tcp.new_access_error(AccessErrorType.FILE_ACCESS_ERROR, error_info.of_message('error message')), AccessErrorType.FILE_ACCESS_ERROR.name), ] for case_result, error_type in cases: with self.subTest(case_result_status=case_result.status): # ARRANGE # expected_xml = suite_xml(attributes={ 'name': 'suite with error', 'tests': '1', 'errors': '1', 'failures': '0'}, test_case_elements=[ erroneous_test_case_xml('test case file name', error_type=error_type, failure_message=error_message_for_error_info(case_result.error_info)) ]) expected_output = expected_output_from(expected_xml) root_suite = test_suite('suite with error', [], [test_case('test case file name')]) test_suites = [root_suite] # ACT # actual = execute_with_case_processing_with_constant_result( case_result, root_suite, Path(), test_suites) # ASSERT # self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code) self.assertEqual(expected_output, replace_xml_variables(actual.stdout))
def test_single_case_with_error(self): cases = [ FULL_RESULT_HARD_ERROR, FULL_RESULT_VALIDATE, FULL_RESULT_IMPLEMENTATION_ERROR, ] for case_result in cases: with self.subTest(case_result_status=case_result.status): # ARRANGE # expected_xml = suite_xml(attributes={ 'name': 'suite with error', 'tests': '1', 'errors': '1', 'failures': '0'}, test_case_elements=[ erroneous_test_case_xml('test case file name', error_type=case_result.status.name, failure_message=error_message_for_full_result(case_result))]) expected_output = expected_output_from(expected_xml) root_suite = test_suite('suite with error', [], [test_case('test case file name')]) test_suites = [root_suite] # ACT # actual = execute_with_case_processing_with_constant_result( tcp.new_executed(case_result), root_suite, Path(), test_suites) # ASSERT # self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code) self.assertEqual(expected_output, replace_xml_variables(actual.stdout))
def test_that_report_of_failing_tests_are_grouped_by_exit_identifiers( self): cases = [ (FULL_RESULT_FAIL, case_ev.EXECUTION__FAIL), (FULL_RESULT_HARD_ERROR, case_ev.EXECUTION__HARD_ERROR), (FULL_RESULT_VALIDATE, case_ev.EXECUTION__VALIDATION_ERROR), (FULL_RESULT_INTERNAL_ERROR, case_ev.EXECUTION__INTERNAL_ERROR), ] test_case_file_name = 'test-case-file' for case_result, expected_case_exit_value in cases: with self.subTest( case_result_status=case_result.status, expected_case_exit_value=expected_case_exit_value): # ARRANGE # root_suite = test_suite('root file name', [], [test_case(test_case_file_name)]) test_suites = [root_suite] std_output_files = StringStdOutFiles() executor = _suite_executor_for_case_processing_that_unconditionally( case_result, root_suite, std_output_files, Path()) # ACT # executor.execute_and_report(test_suites) # ASSERT # std_output_files.finish() stderr_contents = std_output_files.stderr_contents exit_ident_pos = stderr_contents.find( expected_case_exit_value.exit_identifier) self.assertNotEqual(-1, exit_ident_pos, 'stderr must contain the exit identifier') start_of_exit_ident_group = stderr_contents[exit_ident_pos:] group_elements = start_of_exit_ident_group.split() self.assertEqual( len(group_elements), 2, 'Expecting to find EXIT_IDENTIFIER followed by CASE-FILE-NAME as final contents of stderr' ) self.assertEqual( expected_case_exit_value.exit_identifier, group_elements[0], 'Expects the exit identifier as a single word') self.assertEqual( test_case_file_name, group_elements[1], 'Expects the test case file name to follow the exit identifier' )
def test_suite_execution_order_using_empty_suites(self): # ARRANGE # reporter = ExecutionTracingProcessingReporter() str_std_out_files = StringStdOutFiles() sub11 = test_suite('11', [], []) sub12 = test_suite('12', [], []) sub1 = test_suite('1', [sub11, sub12], []) sub21 = test_suite('21', [], []) sub2 = test_suite('2', [sub21], []) root = test_suite('root', [sub1, sub2], []) expected_suites = [ ExpectedSuiteReporting(sub11, []), ExpectedSuiteReporting(sub12, []), ExpectedSuiteReporting(sub1, []), ExpectedSuiteReporting(sub21, []), ExpectedSuiteReporting(sub2, []), ExpectedSuiteReporting(root, []), ] suite_hierarchy_reader = ReaderThatGivesConstantSuite(root) processor = Processor( DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter, DepthFirstEnumerator(), lambda config: TestCaseProcessorThatGivesConstantPerCase({})) # ACT # exit_code = processor.process(pathlib.Path('root-suite-file'), str_std_out_files.reporting_environment) # ASSERT # check_exit_code_and_empty_stdout( self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE, exit_code, str_std_out_files) ExpectedSuiteReporting.check_list(self, expected_suites, reporter.complete_suite_reporter)
def test_single_suite_with_test_cases_with_different_result(self): # ARRANGE # reporter = ExecutionTracingProcessingReporter() str_std_out_files = StringStdOutFiles() tc_internal_error = test_case_reference_of_source_file(Path('internal error')) tc_access_error = test_case_reference_of_source_file(Path('access error')) tc_executed = test_case_reference_of_source_file(Path('executed')) root = test_suite( 'root', [], [ tc_internal_error, tc_access_error, tc_executed, ]) test_case_processor = TestCaseProcessorThatGivesConstantPerCase({ id(tc_internal_error): new_internal_error(error_info.of_message('message')), id(tc_access_error): new_access_error( tcp.AccessErrorType.SYNTAX_ERROR, error_info.of_message('syntax error')), id(tc_executed): new_executed(FULL_RESULT_PASS), }) expected_suites = [ ExpectedSuiteReporting(root, [ (tc_internal_error, tcp.Status.INTERNAL_ERROR), (tc_access_error, tcp.Status.ACCESS_ERROR), (tc_executed, tcp.Status.EXECUTED), ]) ] suite_hierarchy_reader = ReaderThatGivesConstantSuite(root) processor = Processor(DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter, DepthFirstEnumerator(), lambda config: test_case_processor) # ACT # exit_code = processor.process(pathlib.Path('root-suite-file'), str_std_out_files.stdout_files) # ASSERT # check_exit_code_and_empty_stdout(self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE, exit_code, str_std_out_files) ExpectedSuiteReporting.check_list( self, expected_suites, reporter.complete_suite_reporter)
def test_single_suite_with_test_cases_with_different_result(self): # ARRANGE # reporter = ExecutionTracingProcessingReporter() str_std_out_files = StringStdOutFiles() tc_internal_error = test_case_reference_of_source_file( Path('internal error')) tc_access_error = test_case_reference_of_source_file( Path('access error')) tc_executed = test_case_reference_of_source_file(Path('executed')) root = test_suite('root', [], [ tc_internal_error, tc_access_error, tc_executed, ]) test_case_processor = TestCaseProcessorThatGivesConstantPerCase({ id(tc_internal_error): new_internal_error(error_info.of_message('message')), id(tc_access_error): new_access_error(tcp.AccessErrorType.SYNTAX_ERROR, error_info.of_message('syntax error')), id(tc_executed): new_executed(FULL_RESULT_PASS), }) expected_suites = [ ExpectedSuiteReporting(root, [ (tc_internal_error, tcp.Status.INTERNAL_ERROR), (tc_access_error, tcp.Status.ACCESS_ERROR), (tc_executed, tcp.Status.EXECUTED), ]) ] suite_hierarchy_reader = ReaderThatGivesConstantSuite(root) processor = Processor(DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter, DepthFirstEnumerator(), lambda config: test_case_processor) # ACT # exit_code = processor.process(pathlib.Path('root-suite-file'), str_std_out_files.reporting_environment) # ASSERT # check_exit_code_and_empty_stdout( self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE, exit_code, str_std_out_files) ExpectedSuiteReporting.check_list(self, expected_suites, reporter.complete_suite_reporter)
def test_single_sub_suite_with_test_cases_with_different_results(self): # ARRANGE # tc_pass = test_case('successful case') tc_fail = test_case('failing case') tc_error = test_case('erroneous case') root_suite = test_suite('suite file name', [], [ tc_pass, tc_fail, tc_error, ]) suites = [root_suite] test_case_processor = TestCaseProcessorThatGivesConstantPerCase({ id(tc_pass): tcp.new_executed(FULL_RESULT_PASS), id(tc_fail): tcp.new_executed(FULL_RESULT_FAIL), id(tc_error): tcp.new_executed(FULL_RESULT_HARD_ERROR), }) # ACT # actual = execute_with_case_processing_with_constant_processor(test_case_processor, root_suite, Path(), suites) # ASSERT # expected_xml = suite_xml(attributes={ 'name': 'suite file name', 'tests': '3', 'errors': '1', 'failures': '1', }, test_case_elements=[ successful_test_case_xml('successful case'), failing_test_case_xml('failing case', failure_type=FULL_RESULT_FAIL.status.name, failure_message=error_message_for_full_result(FULL_RESULT_FAIL)), erroneous_test_case_xml('erroneous case', error_type=FULL_RESULT_HARD_ERROR.status.name, failure_message=error_message_for_full_result(FULL_RESULT_HARD_ERROR)), ] ) expected__assertion = asrt_etree.str_as_xml_equals(expected_xml) self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code) expected__assertion.apply_with_message(self, replace_xml_variables(actual.stdout), 'suite xml on stdout')
def test_single_empty_suite(self): # ARRANGE # expected_exit_value = suite_ev.ALL_PASS expected_output = lines_content_with_os_linesep([ _suite_begin('root file name'), _suite_end('root file name'), expected_exit_value.exit_identifier, ]) root_suite = test_suite('root file name', [], []) test_suites = [root_suite] std_output_files = StringStdOutFiles() executor = _suite_executor_for_case_processing_that_unconditionally( FULL_RESULT_PASS, root_suite, std_output_files, Path()) # ACT # exit_code = executor.execute_and_report(test_suites) # ASSERT # std_output_files.finish() self.assertEqual(expected_exit_value.exit_code, exit_code) self.assertEqual(expected_output, std_output_files.stdout_contents)
def test_empty_suite(self): # ARRANGE # expected_xml = suite_xml(attributes={ 'name': 'root file name', 'tests': '0', 'errors': '0', 'failures': '0', }, test_case_elements=[]) expected_output = expected_output_from(expected_xml) root_suite = test_suite('root file name', [], []) test_suites = [root_suite] # ACT # actual = execute_with_case_processing_with_constant_result(FULL_RESULT_PASS, root_suite, Path(), test_suites) # ASSERT # self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code) self.assertEqual(expected_output, replace_xml_variables(actual.stdout))
def test_suite_with_single_case(self): cases = [ (FULL_RESULT_PASS, case_ev.EXECUTION__PASS, suite_ev.ALL_PASS), (FULL_RESULT_FAIL, case_ev.EXECUTION__FAIL, suite_ev.FAILED_TESTS), (FULL_RESULT_SKIP, case_ev.EXECUTION__SKIPPED, suite_ev.ALL_PASS), (FULL_RESULT_HARD_ERROR, case_ev.EXECUTION__HARD_ERROR, suite_ev.FAILED_TESTS), (FULL_RESULT_VALIDATE, case_ev.EXECUTION__VALIDATION_ERROR, suite_ev.FAILED_TESTS), (FULL_RESULT_INTERNAL_ERROR, case_ev.EXECUTION__INTERNAL_ERROR, suite_ev.FAILED_TESTS), ] for case_result, expected_case_exit_value, expected_suite_exit_value in cases: with self.subTest( case_result_status=case_result.status, expected_case_exit_value=expected_case_exit_value, expected_suite_exit_value=expected_suite_exit_value): # ARRANGE # expected_output = lines_content_with_os_linesep([ _suite_begin('root file name'), _case('test case file', expected_case_exit_value), _suite_end('root file name'), expected_suite_exit_value.exit_identifier, ]) root_suite = test_suite('root file name', [], [test_case('test case file')]) test_suites = [root_suite] std_output_files = StringStdOutFiles() executor = _suite_executor_for_case_processing_that_unconditionally( case_result, root_suite, std_output_files, Path()) # ACT # exit_code = executor.execute_and_report(test_suites) # ASSERT # std_output_files.finish() self.assertEqual(expected_suite_exit_value.exit_code, exit_code) stdout_contents_prepared_for_assertion = _replace_seconds_with_const( std_output_files.stdout_contents) self.assertEqual(expected_output, stdout_contents_prepared_for_assertion)
def test_single_sub_suite_with_test_cases_with_different_results(self): # ARRANGE # tc_pass = test_case('successful case') tc_fail = test_case('failing case') tc_error = test_case('erroneous case') root_suite = test_suite('suite file name', [], [ tc_pass, tc_fail, tc_error, ]) suites = [root_suite] test_case_processor = TestCaseProcessorThatGivesConstantPerCase({ id(tc_pass): tcp.new_executed(FULL_RESULT_PASS), id(tc_fail): tcp.new_executed(FULL_RESULT_FAIL), id(tc_error): tcp.new_executed(FULL_RESULT_HARD_ERROR), }) # ACT # actual = execute_with_case_processing_with_constant_processor(test_case_processor, root_suite, Path(), suites) # ASSERT # expected_xml = suite_xml(attributes={ 'name': 'suite file name', 'tests': '3', 'errors': '1', 'failures': '1', }, test_case_elements=[ successful_test_case_xml('successful case'), failing_test_case_xml('failing case', failure_type=FULL_RESULT_FAIL.status.name, failure_message=error_message_for_full_result(FULL_RESULT_FAIL)), erroneous_test_case_xml('erroneous case', error_type=FULL_RESULT_HARD_ERROR.status.name, failure_message=error_message_for_full_result(FULL_RESULT_HARD_ERROR)), ] ) expected_output = expected_output_from(expected_xml) self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code) self.assertEqual(expected_output, replace_xml_variables(actual.stdout))
def _check(self, result: tcp.Result): # ARRANGE # str_std_out_files = StringStdOutFiles() test_case = test_case_reference_of_source_file(Path('test-case')) root = test_suite('root', [], [test_case]) suite_hierarchy_reader = ReaderThatGivesConstantSuite(root) reporter = ExecutionTracingProcessingReporter() executor = Processor( DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter, DepthFirstEnumerator(), lambda config: TestCaseProcessorThatGivesConstant(result)) # ACT # exit_code = executor.process(pathlib.Path('root-suite-file'), str_std_out_files.reporting_environment) # ASSERT # check_exit_code_and_empty_stdout( self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE, exit_code, str_std_out_files) ExpectedSuiteReporting.check_list( self, [ExpectedSuiteReporting(root, [(test_case, result.status)])], reporter.complete_suite_reporter)
def test_single_empty_suite(self): # ARRANGE # expected_exit_value = suite_ev.ALL_PASS expected_output = lines_content_with_os_linesep([ _suite_begin('root file name'), _suite_end('root file name'), expected_exit_value.exit_identifier, ]) root_suite = test_suite('root file name', [], []) test_suites = [root_suite] std_output_files = StringStdOutFiles() executor = _suite_executor_for_case_processing_that_unconditionally(FULL_RESULT_PASS, root_suite, std_output_files, Path()) # ACT # exit_code = executor.execute_and_report(test_suites) # ASSERT # std_output_files.finish() self.assertEqual(expected_exit_value.exit_code, exit_code) self.assertEqual(expected_output, std_output_files.stdout_contents)
def test_internal_error_in_test_case_processor(self): # ARRANGE # str_std_out_files = StringStdOutFiles() test_case = test_case_reference_of_source_file(Path('test-case')) root = test_suite('root', [], [test_case]) suite_hierarchy_reader = ReaderThatGivesConstantSuite(root) reporter = ExecutionTracingProcessingReporter() processor = Processor(DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter, DepthFirstEnumerator(), lambda config: TestCaseProcessorThatRaisesUnconditionally()) # ACT # exit_code = processor.process(pathlib.Path('root-suite-file'), str_std_out_files.stdout_files) # ASSERT # check_exit_code_and_empty_stdout(self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE, exit_code, str_std_out_files) ExpectedSuiteReporting.check_list( self, [ExpectedSuiteReporting(root, [(test_case, tcp.Status.INTERNAL_ERROR)])], reporter.complete_suite_reporter)
def test_single_case_with_error_due_to_failure_to_execute(self): cases = [ (tcp.new_internal_error(error_info.of_message('error message')), tcp.Status.INTERNAL_ERROR.name), (tcp.new_access_error(AccessErrorType.FILE_ACCESS_ERROR, error_info.of_message('error message')), AccessErrorType.FILE_ACCESS_ERROR.name), ] for case_result, error_type in cases: with self.subTest(case_result_status=case_result.status): # ARRANGE # expected_xml = suite_xml(attributes={ 'name': 'suite with error', 'tests': '1', 'errors': '1', 'failures': '0'}, test_case_elements=[ erroneous_test_case_xml('test case file name', error_type=error_type, failure_message=error_message_for_error_info(case_result.error_info)) ]) expected__assertion = asrt_etree.str_as_xml_equals(expected_xml) root_suite = test_suite('suite with error', [], [test_case('test case file name')]) test_suites = [root_suite] # ACT # actual = execute_with_case_processing_with_constant_result( case_result, root_suite, Path(), test_suites) # ASSERT # self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code) expected__assertion.apply_with_message(self, replace_xml_variables(actual.stdout), 'suite xml on stdout')
def test_suite_with_single_case(self): cases = [ (FULL_RESULT_PASS, case_ev.EXECUTION__PASS, suite_ev.ALL_PASS), (FULL_RESULT_FAIL, case_ev.EXECUTION__FAIL, suite_ev.FAILED_TESTS), (FULL_RESULT_SKIP, case_ev.EXECUTION__SKIPPED, suite_ev.ALL_PASS), (FULL_RESULT_HARD_ERROR, case_ev.EXECUTION__HARD_ERROR, suite_ev.FAILED_TESTS), (FULL_RESULT_VALIDATE, case_ev.EXECUTION__VALIDATION_ERROR, suite_ev.FAILED_TESTS), (FULL_RESULT_IMPLEMENTATION_ERROR, case_ev.EXECUTION__IMPLEMENTATION_ERROR, suite_ev.FAILED_TESTS), ] for case_result, expected_case_exit_value, expected_suite_exit_value in cases: with self.subTest(case_result_status=case_result.status, expected_case_exit_value=expected_case_exit_value, expected_suite_exit_value=expected_suite_exit_value): # ARRANGE # expected_output = lines_content_with_os_linesep([ _suite_begin('root file name'), _case('test case file', expected_case_exit_value), _suite_end('root file name'), expected_suite_exit_value.exit_identifier, ]) root_suite = test_suite('root file name', [], [test_case('test case file')]) test_suites = [root_suite] std_output_files = StringStdOutFiles() executor = _suite_executor_for_case_processing_that_unconditionally(case_result, root_suite, std_output_files, Path()) # ACT # exit_code = executor.execute_and_report(test_suites) # ASSERT # std_output_files.finish() self.assertEqual(expected_suite_exit_value.exit_code, exit_code) stdout_contents_prepared_for_assertion = _replace_seconds_with_const(std_output_files.stdout_contents) self.assertEqual(expected_output, stdout_contents_prepared_for_assertion)
def test_suite_execution_order_using_empty_suites(self): # ARRANGE # reporter = ExecutionTracingProcessingReporter() str_std_out_files = StringStdOutFiles() sub11 = test_suite('11', [], []) sub12 = test_suite('12', [], []) sub1 = test_suite('1', [sub11, sub12], []) sub21 = test_suite('21', [], []) sub2 = test_suite('2', [sub21], []) root = test_suite('root', [sub1, sub2], []) expected_suites = [ ExpectedSuiteReporting(sub11, []), ExpectedSuiteReporting(sub12, []), ExpectedSuiteReporting(sub1, []), ExpectedSuiteReporting(sub21, []), ExpectedSuiteReporting(sub2, []), ExpectedSuiteReporting(root, []), ] suite_hierarchy_reader = ReaderThatGivesConstantSuite(root) processor = Processor(DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter, DepthFirstEnumerator(), lambda config: TestCaseProcessorThatGivesConstantPerCase({})) # ACT # exit_code = processor.process(pathlib.Path('root-suite-file'), str_std_out_files.stdout_files) # ASSERT # check_exit_code_and_empty_stdout(self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE, exit_code, str_std_out_files) ExpectedSuiteReporting.check_list( self, expected_suites, reporter.complete_suite_reporter)
def test_complex_suite_structure_with_test_cases(self): # ARRANGE # reporter = ExecutionTracingProcessingReporter() str_std_out_files = StringStdOutFiles() tc_internal_error_11 = test_case_reference_of_source_file( Path('internal error 11')) tc_internal_error_21 = test_case_reference_of_source_file( Path('internal error 21')) tc_access_error_1 = test_case_reference_of_source_file( Path('access error A')) tc_access_error_12 = test_case_reference_of_source_file( Path('access error 12')) tc_executed_11 = test_case_reference_of_source_file( Path('executed 11')) tc_executed_12 = test_case_reference_of_source_file( Path('executed 12')) tc_executed_1 = test_case_reference_of_source_file(Path('executed 1')) tc_executed_2 = test_case_reference_of_source_file(Path('executed 2')) tc_executed_root = test_case_reference_of_source_file( Path('executed root')) test_case_processor = TestCaseProcessorThatGivesConstantPerCase({ id(tc_internal_error_11): new_internal_error(error_info.of_message('message A')), id(tc_internal_error_21): new_internal_error(error_info.of_message('message B')), id(tc_access_error_1): new_access_error(tcp.AccessErrorType.SYNTAX_ERROR, error_info.of_message('syntax error')), id(tc_access_error_12): new_access_error(tcp.AccessErrorType.FILE_ACCESS_ERROR, error_info.of_message('file access error')), id(tc_executed_11): new_executed(FULL_RESULT_PASS), id(tc_executed_12): new_executed(FULL_RESULT_PASS), id(tc_executed_1): new_executed(FULL_RESULT_PASS), id(tc_executed_2): new_executed(FULL_RESULT_PASS), id(tc_executed_root): new_executed(FULL_RESULT_PASS), }) sub11 = test_suite('11', [], [tc_internal_error_11, tc_executed_11]) sub12 = test_suite('12', [], [tc_executed_12, tc_access_error_12]) sub1 = test_suite('1', [sub11, sub12], [tc_access_error_1, tc_executed_1]) sub21 = test_suite('21', [], [tc_internal_error_21]) sub2 = test_suite('2', [sub21], [tc_executed_2]) sub3 = test_suite('2', [], []) root = test_suite('root', [sub1, sub2, sub3], [tc_executed_root]) expected_suites = [ ExpectedSuiteReporting( sub11, [(tc_internal_error_11, tcp.Status.INTERNAL_ERROR), (tc_executed_11, tcp.Status.EXECUTED)]), ExpectedSuiteReporting( sub12, [(tc_executed_12, tcp.Status.EXECUTED), (tc_access_error_12, tcp.Status.ACCESS_ERROR)]), ExpectedSuiteReporting( sub1, [(tc_access_error_1, tcp.Status.ACCESS_ERROR), (tc_executed_1, tcp.Status.EXECUTED)]), ExpectedSuiteReporting( sub21, [(tc_internal_error_21, tcp.Status.INTERNAL_ERROR)]), ExpectedSuiteReporting(sub2, [(tc_executed_2, tcp.Status.EXECUTED)]), ExpectedSuiteReporting(sub3, []), ExpectedSuiteReporting(root, [(tc_executed_root, tcp.Status.EXECUTED)]), ] suite_hierarchy_reader = ReaderThatGivesConstantSuite(root) processor = Processor(DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter, DepthFirstEnumerator(), lambda config: test_case_processor) # ACT # exit_code = processor.process(pathlib.Path('root-suite-file'), str_std_out_files.reporting_environment) # ASSERT # check_exit_code_and_empty_stdout( self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE, exit_code, str_std_out_files) ExpectedSuiteReporting.check_list(self, expected_suites, reporter.complete_suite_reporter)
def test_complex_suite_structure_with_test_cases(self): # ARRANGE # reporter = ExecutionTracingProcessingReporter() str_std_out_files = StringStdOutFiles() tc_internal_error_11 = test_case_reference_of_source_file(Path('internal error 11')) tc_internal_error_21 = test_case_reference_of_source_file(Path('internal error 21')) tc_access_error_1 = test_case_reference_of_source_file(Path('access error A')) tc_access_error_12 = test_case_reference_of_source_file(Path('access error 12')) tc_executed_11 = test_case_reference_of_source_file(Path('executed 11')) tc_executed_12 = test_case_reference_of_source_file(Path('executed 12')) tc_executed_1 = test_case_reference_of_source_file(Path('executed 1')) tc_executed_2 = test_case_reference_of_source_file(Path('executed 2')) tc_executed_root = test_case_reference_of_source_file(Path('executed root')) test_case_processor = TestCaseProcessorThatGivesConstantPerCase({ id(tc_internal_error_11): new_internal_error(error_info.of_message('message A')), id(tc_internal_error_21): new_internal_error(error_info.of_message('message B')), id(tc_access_error_1): new_access_error( tcp.AccessErrorType.SYNTAX_ERROR, error_info.of_message('syntax error')), id(tc_access_error_12): new_access_error(tcp.AccessErrorType.FILE_ACCESS_ERROR, error_info.of_message('file access error')), id(tc_executed_11): new_executed(FULL_RESULT_PASS), id(tc_executed_12): new_executed(FULL_RESULT_PASS), id(tc_executed_1): new_executed(FULL_RESULT_PASS), id(tc_executed_2): new_executed(FULL_RESULT_PASS), id(tc_executed_root): new_executed(FULL_RESULT_PASS), }) sub11 = test_suite('11', [], [tc_internal_error_11, tc_executed_11]) sub12 = test_suite('12', [], [tc_executed_12, tc_access_error_12]) sub1 = test_suite('1', [sub11, sub12], [tc_access_error_1, tc_executed_1]) sub21 = test_suite('21', [], [tc_internal_error_21]) sub2 = test_suite('2', [sub21], [tc_executed_2]) sub3 = test_suite('2', [], []) root = test_suite('root', [sub1, sub2, sub3], [tc_executed_root]) expected_suites = [ ExpectedSuiteReporting(sub11, [(tc_internal_error_11, tcp.Status.INTERNAL_ERROR), (tc_executed_11, tcp.Status.EXECUTED)]), ExpectedSuiteReporting(sub12, [(tc_executed_12, tcp.Status.EXECUTED), (tc_access_error_12, tcp.Status.ACCESS_ERROR)]), ExpectedSuiteReporting(sub1, [(tc_access_error_1, tcp.Status.ACCESS_ERROR), (tc_executed_1, tcp.Status.EXECUTED)]), ExpectedSuiteReporting(sub21, [(tc_internal_error_21, tcp.Status.INTERNAL_ERROR)]), ExpectedSuiteReporting(sub2, [(tc_executed_2, tcp.Status.EXECUTED)]), ExpectedSuiteReporting(sub3, []), ExpectedSuiteReporting(root, [(tc_executed_root, tcp.Status.EXECUTED)]), ] suite_hierarchy_reader = ReaderThatGivesConstantSuite(root) processor = Processor(DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter, DepthFirstEnumerator(), lambda config: test_case_processor) # ACT # exit_code = processor.process(pathlib.Path('root-suite-file'), str_std_out_files.stdout_files) # ASSERT # check_exit_code_and_empty_stdout(self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE, exit_code, str_std_out_files) ExpectedSuiteReporting.check_list( self, expected_suites, reporter.complete_suite_reporter)