def check_exit_code_and_empty_stdout(put: unittest.TestCase,
expected_exit_code: int,
actual_exit_code: int,
str_std_out_files: StringStdOutFiles):
str_std_out_files.finish()
put.assertEqual(expected_exit_code, actual_exit_code, 'Exit code')
put.assertEqual('', str_std_out_files.stdout_contents, 'Output to stdout')
def check_exit_code_and_empty_stdout(put: unittest.TestCase,
expected_exit_code: int,
actual_exit_code: int,
str_std_out_files: StringStdOutFiles):
str_std_out_files.finish()
put.assertEqual(expected_exit_code,
actual_exit_code,
'Exit code')
put.assertEqual('',
str_std_out_files.stdout_contents,
'Output to stdout')
def test_that_report_of_failing_tests_are_grouped_by_exit_identifiers(
self):
cases = [
(FULL_RESULT_FAIL, case_ev.EXECUTION__FAIL),
(FULL_RESULT_HARD_ERROR, case_ev.EXECUTION__HARD_ERROR),
(FULL_RESULT_VALIDATE, case_ev.EXECUTION__VALIDATION_ERROR),
(FULL_RESULT_INTERNAL_ERROR, case_ev.EXECUTION__INTERNAL_ERROR),
]
test_case_file_name = 'test-case-file'
for case_result, expected_case_exit_value in cases:
with self.subTest(
case_result_status=case_result.status,
expected_case_exit_value=expected_case_exit_value):
# ARRANGE #
root_suite = test_suite('root file name', [],
[test_case(test_case_file_name)])
test_suites = [root_suite]
std_output_files = StringStdOutFiles()
executor = _suite_executor_for_case_processing_that_unconditionally(
case_result, root_suite, std_output_files, Path())
# ACT #
executor.execute_and_report(test_suites)
# ASSERT #
std_output_files.finish()
stderr_contents = std_output_files.stderr_contents
exit_ident_pos = stderr_contents.find(
expected_case_exit_value.exit_identifier)
self.assertNotEqual(-1, exit_ident_pos,
'stderr must contain the exit identifier')
start_of_exit_ident_group = stderr_contents[exit_ident_pos:]
group_elements = start_of_exit_ident_group.split()
self.assertEqual(
len(group_elements), 2,
'Expecting to find EXIT_IDENTIFIER followed by CASE-FILE-NAME as final contents of stderr'
)
self.assertEqual(
expected_case_exit_value.exit_identifier,
group_elements[0],
'Expects the exit identifier as a single word')
self.assertEqual(
test_case_file_name, group_elements[1],
'Expects the test case file name to follow the exit identifier'
)
def execute_with_case_processing_with_constant_processor(processor: tcp.Processor,
root_suite: structure.TestSuiteHierarchy,
root_file_path: Path,
test_suites: list) -> ExitCodeAndStdOut:
std_output_files = StringStdOutFiles()
reporter = sut.JUnitRootSuiteProcessingReporter()
execution_reporter = reporter.execution_reporter(root_suite, std_output_files.stdout_files, root_file_path)
executor = processing.SuitesExecutor(execution_reporter,
DUMMY_CASE_PROCESSING,
lambda conf: processor)
exit_code = executor.execute_and_report(test_suites)
std_output_files.finish()
return ExitCodeAndStdOut(exit_code, std_output_files.stdout_contents)
def test(self):
# ARRANGE #
reporter = sut.SimpleProgressRootSuiteProcessingReporter()
str_std_out_files = StringStdOutFiles()
exit_value = ExitValue(1, 'IDENTIFIER', ForegroundColor.BLACK)
# ACT #
reporter.report_invalid_suite(exit_value,
str_std_out_files.reporting_environment)
# ASSERT #
str_std_out_files.finish()
self.assertEqual(exit_value.exit_identifier + os.linesep,
str_std_out_files.stdout_contents, 'Output to stdout')
self.assertEqual('', str_std_out_files.stderr_contents,
'Output to stderr')
def test(self):
# ARRANGE #
reporter = sut.JUnitRootSuiteProcessingReporter()
str_std_out_files = StringStdOutFiles()
# ACT #
reporter.report_invalid_suite(ExitValue(1, 'IDENTIFIER', ForegroundColor.BLACK),
str_std_out_files.stdout_files)
# ASSERT #
str_std_out_files.finish()
self.assertEqual('',
str_std_out_files.stdout_contents,
'Output to stdout')
self.assertEqual('',
str_std_out_files.stderr_contents,
'Output to stderr')
def execute_with_case_processing_with_constant_processor(processor: tcp.Processor,
root_suite: structure.TestSuiteHierarchy,
root_file_path: Path,
test_suites: list) -> ExitCodeAndStdOut:
std_output_files = StringStdOutFiles()
reporter = sut.JUnitRootSuiteProcessingReporter()
execution_reporter = reporter.execution_reporter(root_suite,
std_output_files.reporting_environment,
root_file_path)
executor = processing.SuitesExecutor(execution_reporter,
DUMMY_CASE_PROCESSING,
lambda conf: processor)
exit_code = executor.execute_and_report(test_suites)
std_output_files.finish()
return ExitCodeAndStdOut(exit_code, std_output_files.stdout_contents)
def test(self):
# ARRANGE #
reporter = sut.JUnitRootSuiteProcessingReporter()
str_std_out_files = StringStdOutFiles()
# ACT #
reporter.report_invalid_suite(ExitValue(1, 'IDENTIFIER', ForegroundColor.BLACK),
str_std_out_files.reporting_environment)
# ASSERT #
str_std_out_files.finish()
self.assertEqual('',
str_std_out_files.stdout_contents,
'Output to stdout')
self.assertEqual('',
str_std_out_files.stderr_contents,
'Output to stderr')
def test(self):
# ARRANGE #
reporter = sut.SimpleProgressRootSuiteProcessingReporter()
str_std_out_files = StringStdOutFiles()
exit_value = ExitValue(1, 'IDENTIFIER', ForegroundColor.BLACK)
# ACT #
reporter.report_invalid_suite(exit_value,
str_std_out_files.stdout_files)
# ASSERT #
str_std_out_files.finish()
self.assertEqual(exit_value.exit_identifier + '\n',
str_std_out_files.stdout_contents,
'Output to stdout')
self.assertEqual('',
str_std_out_files.stderr_contents,
'Output to stderr')
def test_that_value_set_in_first_case_does_not_leak_to_second_case(self):
# ARRANGE #
containing_suite_file = File('test.suite', SUITE_WITH_CASES)
suite_and_case_files = DirContents([
containing_suite_file,
CASE_1_FILE,
CASE_2_FILE,
])
case_processors = [
NameAndValue('processor_that_should_not_pollute_current_process',
processors.new_processor_that_should_not_pollute_current_process),
NameAndValue('processor_that_is_allowed_to_pollute_current_process',
processors.new_processor_that_is_allowed_to_pollute_current_process),
]
with tmp_dir(suite_and_case_files) as tmp_dir_path:
suite_file_path = tmp_dir_path / containing_suite_file.file_name
for case_processor_case in case_processors:
with self.subTest(case_processor_case.name):
registry = tr.Registry()
processor = new_processor_with_no_env_vars(registry,
case_processor_case.value)
# ACT #
return_value = processor.process(suite_file_path, StringStdOutFiles().reporting_environment)
# ASSERT #
self.assertEqual(ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE,
return_value,
'Sanity check of result indicator')
self.assertFalse(registry.observation)
def test_suite_execution_order_using_empty_suites(self):
# ARRANGE #
reporter = ExecutionTracingProcessingReporter()
str_std_out_files = StringStdOutFiles()
sub11 = test_suite('11', [], [])
sub12 = test_suite('12', [], [])
sub1 = test_suite('1', [sub11, sub12], [])
sub21 = test_suite('21', [], [])
sub2 = test_suite('2', [sub21], [])
root = test_suite('root', [sub1, sub2], [])
expected_suites = [
ExpectedSuiteReporting(sub11, []),
ExpectedSuiteReporting(sub12, []),
ExpectedSuiteReporting(sub1, []),
ExpectedSuiteReporting(sub21, []),
ExpectedSuiteReporting(sub2, []),
ExpectedSuiteReporting(root, []),
]
suite_hierarchy_reader = ReaderThatGivesConstantSuite(root)
processor = Processor(
DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter,
DepthFirstEnumerator(),
lambda config: TestCaseProcessorThatGivesConstantPerCase({}))
# ACT #
exit_code = processor.process(pathlib.Path('root-suite-file'),
str_std_out_files.reporting_environment)
# ASSERT #
check_exit_code_and_empty_stdout(
self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE,
exit_code, str_std_out_files)
ExpectedSuiteReporting.check_list(self, expected_suites,
reporter.complete_suite_reporter)
def _check(self, exception_from_hierarchy_reader: SuiteReadError):
# ARRANGE #
str_std_out_files = StringStdOutFiles()
suite_root_file_path = Path('root-suite-file')
suite_hierarchy_reader = ReaderThatRaises(
exception_from_hierarchy_reader)
reporter = ExecutionTracingProcessingReporter()
processor = Processor(
DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter,
DepthFirstEnumerator(),
lambda x: TestCaseProcessorThatRaisesUnconditionally())
# ACT #
exit_code = processor.process(suite_root_file_path,
str_std_out_files.reporting_environment)
# ASSERT #
check_exit_code_and_empty_stdout(self,
exit_values.INVALID_SUITE.exit_code,
exit_code, str_std_out_files)
expected_invalid_suite_invocations = asrt.matches_sequence([
asrt.equals(exit_values.INVALID_SUITE),
])
expected_invalid_suite_invocations.apply_with_message(
self, reporter.report_invalid_suite_invocations,
'report_invalid_suite_invocations')
ExpectedSuiteReporting.check_list(self, [],
reporter.complete_suite_reporter)
def test_single_empty_suite(self):
# ARRANGE #
expected_exit_value = suite_ev.ALL_PASS
expected_output = lines_content_with_os_linesep([
_suite_begin('root file name'),
_suite_end('root file name'),
expected_exit_value.exit_identifier,
])
root_suite = test_suite('root file name', [], [])
test_suites = [root_suite]
std_output_files = StringStdOutFiles()
executor = _suite_executor_for_case_processing_that_unconditionally(
FULL_RESULT_PASS, root_suite, std_output_files, Path())
# ACT #
exit_code = executor.execute_and_report(test_suites)
# ASSERT #
std_output_files.finish()
self.assertEqual(expected_exit_value.exit_code, exit_code)
self.assertEqual(expected_output, std_output_files.stdout_contents)
def test_suite_with_single_case(self):
cases = [
(FULL_RESULT_PASS, case_ev.EXECUTION__PASS, suite_ev.ALL_PASS),
(FULL_RESULT_FAIL, case_ev.EXECUTION__FAIL, suite_ev.FAILED_TESTS),
(FULL_RESULT_SKIP, case_ev.EXECUTION__SKIPPED, suite_ev.ALL_PASS),
(FULL_RESULT_HARD_ERROR, case_ev.EXECUTION__HARD_ERROR,
suite_ev.FAILED_TESTS),
(FULL_RESULT_VALIDATE, case_ev.EXECUTION__VALIDATION_ERROR,
suite_ev.FAILED_TESTS),
(FULL_RESULT_INTERNAL_ERROR, case_ev.EXECUTION__INTERNAL_ERROR,
suite_ev.FAILED_TESTS),
]
for case_result, expected_case_exit_value, expected_suite_exit_value in cases:
with self.subTest(
case_result_status=case_result.status,
expected_case_exit_value=expected_case_exit_value,
expected_suite_exit_value=expected_suite_exit_value):
# ARRANGE #
expected_output = lines_content_with_os_linesep([
_suite_begin('root file name'),
_case('test case file', expected_case_exit_value),
_suite_end('root file name'),
expected_suite_exit_value.exit_identifier,
])
root_suite = test_suite('root file name', [],
[test_case('test case file')])
test_suites = [root_suite]
std_output_files = StringStdOutFiles()
executor = _suite_executor_for_case_processing_that_unconditionally(
case_result, root_suite, std_output_files, Path())
# ACT #
exit_code = executor.execute_and_report(test_suites)
# ASSERT #
std_output_files.finish()
self.assertEqual(expected_suite_exit_value.exit_code,
exit_code)
stdout_contents_prepared_for_assertion = _replace_seconds_with_const(
std_output_files.stdout_contents)
self.assertEqual(expected_output,
stdout_contents_prepared_for_assertion)
def test_single_empty_suite(self):
# ARRANGE #
expected_exit_value = suite_ev.ALL_PASS
expected_output = lines_content_with_os_linesep([
_suite_begin('root file name'),
_suite_end('root file name'),
expected_exit_value.exit_identifier,
])
root_suite = test_suite('root file name', [], [])
test_suites = [root_suite]
std_output_files = StringStdOutFiles()
executor = _suite_executor_for_case_processing_that_unconditionally(FULL_RESULT_PASS,
root_suite,
std_output_files,
Path())
# ACT #
exit_code = executor.execute_and_report(test_suites)
# ASSERT #
std_output_files.finish()
self.assertEqual(expected_exit_value.exit_code, exit_code)
self.assertEqual(expected_output, std_output_files.stdout_contents)
def test_suite_with_single_case(self):
cases = [
(FULL_RESULT_PASS, case_ev.EXECUTION__PASS, suite_ev.ALL_PASS),
(FULL_RESULT_FAIL, case_ev.EXECUTION__FAIL, suite_ev.FAILED_TESTS),
(FULL_RESULT_SKIP, case_ev.EXECUTION__SKIPPED, suite_ev.ALL_PASS),
(FULL_RESULT_HARD_ERROR, case_ev.EXECUTION__HARD_ERROR, suite_ev.FAILED_TESTS),
(FULL_RESULT_VALIDATE, case_ev.EXECUTION__VALIDATION_ERROR, suite_ev.FAILED_TESTS),
(FULL_RESULT_IMPLEMENTATION_ERROR, case_ev.EXECUTION__IMPLEMENTATION_ERROR, suite_ev.FAILED_TESTS),
]
for case_result, expected_case_exit_value, expected_suite_exit_value in cases:
with self.subTest(case_result_status=case_result.status,
expected_case_exit_value=expected_case_exit_value,
expected_suite_exit_value=expected_suite_exit_value):
# ARRANGE #
expected_output = lines_content_with_os_linesep([
_suite_begin('root file name'),
_case('test case file', expected_case_exit_value),
_suite_end('root file name'),
expected_suite_exit_value.exit_identifier,
])
root_suite = test_suite('root file name', [], [test_case('test case file')])
test_suites = [root_suite]
std_output_files = StringStdOutFiles()
executor = _suite_executor_for_case_processing_that_unconditionally(case_result,
root_suite,
std_output_files,
Path())
# ACT #
exit_code = executor.execute_and_report(test_suites)
# ASSERT #
std_output_files.finish()
self.assertEqual(expected_suite_exit_value.exit_code, exit_code)
stdout_contents_prepared_for_assertion = _replace_seconds_with_const(std_output_files.stdout_contents)
self.assertEqual(expected_output,
stdout_contents_prepared_for_assertion)
def test_single_suite_with_test_cases_with_different_result(self):
# ARRANGE #
reporter = ExecutionTracingProcessingReporter()
str_std_out_files = StringStdOutFiles()
tc_internal_error = test_case_reference_of_source_file(
Path('internal error'))
tc_access_error = test_case_reference_of_source_file(
Path('access error'))
tc_executed = test_case_reference_of_source_file(Path('executed'))
root = test_suite('root', [], [
tc_internal_error,
tc_access_error,
tc_executed,
])
test_case_processor = TestCaseProcessorThatGivesConstantPerCase({
id(tc_internal_error):
new_internal_error(error_info.of_message('message')),
id(tc_access_error):
new_access_error(tcp.AccessErrorType.SYNTAX_ERROR,
error_info.of_message('syntax error')),
id(tc_executed):
new_executed(FULL_RESULT_PASS),
})
expected_suites = [
ExpectedSuiteReporting(root, [
(tc_internal_error, tcp.Status.INTERNAL_ERROR),
(tc_access_error, tcp.Status.ACCESS_ERROR),
(tc_executed, tcp.Status.EXECUTED),
])
]
suite_hierarchy_reader = ReaderThatGivesConstantSuite(root)
processor = Processor(DUMMY_CASE_PROCESSING, suite_hierarchy_reader,
reporter, DepthFirstEnumerator(),
lambda config: test_case_processor)
# ACT #
exit_code = processor.process(pathlib.Path('root-suite-file'),
str_std_out_files.reporting_environment)
# ASSERT #
check_exit_code_and_empty_stdout(
self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE,
exit_code, str_std_out_files)
ExpectedSuiteReporting.check_list(self, expected_suites,
reporter.complete_suite_reporter)
def _check(self, result: tcp.Result):
# ARRANGE #
str_std_out_files = StringStdOutFiles()
test_case = test_case_reference_of_source_file(Path('test-case'))
root = test_suite('root', [], [test_case])
suite_hierarchy_reader = ReaderThatGivesConstantSuite(root)
reporter = ExecutionTracingProcessingReporter()
executor = Processor(
DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter,
DepthFirstEnumerator(),
lambda config: TestCaseProcessorThatGivesConstant(result))
# ACT #
exit_code = executor.process(pathlib.Path('root-suite-file'),
str_std_out_files.reporting_environment)
# ASSERT #
check_exit_code_and_empty_stdout(
self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE,
exit_code, str_std_out_files)
ExpectedSuiteReporting.check_list(
self, [ExpectedSuiteReporting(root, [(test_case, result.status)])],
reporter.complete_suite_reporter)
def test_complex_suite_structure_with_test_cases(self):
# ARRANGE #
reporter = ExecutionTracingProcessingReporter()
str_std_out_files = StringStdOutFiles()
tc_internal_error_11 = test_case_reference_of_source_file(
Path('internal error 11'))
tc_internal_error_21 = test_case_reference_of_source_file(
Path('internal error 21'))
tc_access_error_1 = test_case_reference_of_source_file(
Path('access error A'))
tc_access_error_12 = test_case_reference_of_source_file(
Path('access error 12'))
tc_executed_11 = test_case_reference_of_source_file(
Path('executed 11'))
tc_executed_12 = test_case_reference_of_source_file(
Path('executed 12'))
tc_executed_1 = test_case_reference_of_source_file(Path('executed 1'))
tc_executed_2 = test_case_reference_of_source_file(Path('executed 2'))
tc_executed_root = test_case_reference_of_source_file(
Path('executed root'))
test_case_processor = TestCaseProcessorThatGivesConstantPerCase({
id(tc_internal_error_11):
new_internal_error(error_info.of_message('message A')),
id(tc_internal_error_21):
new_internal_error(error_info.of_message('message B')),
id(tc_access_error_1):
new_access_error(tcp.AccessErrorType.SYNTAX_ERROR,
error_info.of_message('syntax error')),
id(tc_access_error_12):
new_access_error(tcp.AccessErrorType.FILE_ACCESS_ERROR,
error_info.of_message('file access error')),
id(tc_executed_11):
new_executed(FULL_RESULT_PASS),
id(tc_executed_12):
new_executed(FULL_RESULT_PASS),
id(tc_executed_1):
new_executed(FULL_RESULT_PASS),
id(tc_executed_2):
new_executed(FULL_RESULT_PASS),
id(tc_executed_root):
new_executed(FULL_RESULT_PASS),
})
sub11 = test_suite('11', [], [tc_internal_error_11, tc_executed_11])
sub12 = test_suite('12', [], [tc_executed_12, tc_access_error_12])
sub1 = test_suite('1', [sub11, sub12],
[tc_access_error_1, tc_executed_1])
sub21 = test_suite('21', [], [tc_internal_error_21])
sub2 = test_suite('2', [sub21], [tc_executed_2])
sub3 = test_suite('2', [], [])
root = test_suite('root', [sub1, sub2, sub3], [tc_executed_root])
expected_suites = [
ExpectedSuiteReporting(
sub11, [(tc_internal_error_11, tcp.Status.INTERNAL_ERROR),
(tc_executed_11, tcp.Status.EXECUTED)]),
ExpectedSuiteReporting(
sub12, [(tc_executed_12, tcp.Status.EXECUTED),
(tc_access_error_12, tcp.Status.ACCESS_ERROR)]),
ExpectedSuiteReporting(
sub1, [(tc_access_error_1, tcp.Status.ACCESS_ERROR),
(tc_executed_1, tcp.Status.EXECUTED)]),
ExpectedSuiteReporting(
sub21, [(tc_internal_error_21, tcp.Status.INTERNAL_ERROR)]),
ExpectedSuiteReporting(sub2,
[(tc_executed_2, tcp.Status.EXECUTED)]),
ExpectedSuiteReporting(sub3, []),
ExpectedSuiteReporting(root,
[(tc_executed_root, tcp.Status.EXECUTED)]),
]
suite_hierarchy_reader = ReaderThatGivesConstantSuite(root)
processor = Processor(DUMMY_CASE_PROCESSING, suite_hierarchy_reader,
reporter, DepthFirstEnumerator(),
lambda config: test_case_processor)
# ACT #
exit_code = processor.process(pathlib.Path('root-suite-file'),
str_std_out_files.reporting_environment)
# ASSERT #
check_exit_code_and_empty_stdout(
self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE,
exit_code, str_std_out_files)
ExpectedSuiteReporting.check_list(self, expected_suites,
reporter.complete_suite_reporter)