def test_suite_with_only_single_sub_suite_SHOULD_not_include_root_suite_as_test_suite(self):
# ARRANGE #
suite_with_single_case = test_suite('suite with single case', [], [
test_case('the test case')
])
root_suite = test_suite('root suite file name', [
suite_with_single_case,
], [])
suites = [
root_suite,
suite_with_single_case,
]
# ACT #
actual = execute_with_case_processing_with_constant_result(tcp.new_executed(FULL_RESULT_PASS),
root_suite,
Path(),
suites)
# ASSERT #
expected_xml = _suites_xml([
suite_xml(attributes={
'name': 'suite with single case',
'package': '.',
'id': '1',
'tests': '1',
'errors': '0',
'failures': '0',
},
test_case_elements=[successful_test_case_xml('the test case')]
),
])
expected_output = expected_output_from(expected_xml)
self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code)
self.assertEqual(expected_output, replace_xml_variables(actual.stdout))
def test_single_case_that_passes(self):
cases = [
FULL_RESULT_PASS,
FULL_RESULT_XFAIL,
FULL_RESULT_SKIP,
]
for case_result in cases:
with self.subTest(case_result_status=case_result.status):
# ARRANGE #
expected_xml = suite_xml(attributes={
'name': 'suite that passes',
'tests': '1',
'errors': '0',
'failures': '0',
},
test_case_elements=[successful_test_case_xml('test case file name')]
)
expected_output = expected_output_from(expected_xml)
root_suite = test_suite('suite that passes', [], [test_case('test case file name')])
test_suites = [root_suite]
# ACT #
actual = execute_with_case_processing_with_constant_result(
tcp.new_executed(case_result),
root_suite,
Path(),
test_suites)
# ASSERT #
self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code)
self.assertEqual(expected_output, replace_xml_variables(actual.stdout))
def test_single_case_with_error_due_to_failure_to_execute(self):
cases = [
(tcp.new_internal_error(error_info.of_message('error message')),
tcp.Status.INTERNAL_ERROR.name),
(tcp.new_access_error(AccessErrorType.FILE_ACCESS_ERROR,
error_info.of_message('error message')),
AccessErrorType.FILE_ACCESS_ERROR.name),
]
for case_result, error_type in cases:
with self.subTest(case_result_status=case_result.status):
# ARRANGE #
expected_xml = suite_xml(attributes={
'name': 'suite with error',
'tests': '1',
'errors': '1',
'failures': '0'},
test_case_elements=[
erroneous_test_case_xml('test case file name',
error_type=error_type,
failure_message=error_message_for_error_info(case_result.error_info))
])
expected_output = expected_output_from(expected_xml)
root_suite = test_suite('suite with error', [], [test_case('test case file name')])
test_suites = [root_suite]
# ACT #
actual = execute_with_case_processing_with_constant_result(
case_result,
root_suite,
Path(),
test_suites)
# ASSERT #
self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code)
self.assertEqual(expected_output, replace_xml_variables(actual.stdout))
def test_single_case_that_passes(self):
cases = [
FULL_RESULT_PASS,
FULL_RESULT_XFAIL,
FULL_RESULT_SKIP,
]
for case_result in cases:
with self.subTest(case_result_status=case_result.status):
# ARRANGE #
expected_xml = suite_xml(attributes={
'name': 'suite that passes',
'tests': '1',
'errors': '0',
'failures': '0',
},
test_case_elements=[successful_test_case_xml('test case file name')]
)
expected__assertion = asrt_etree.str_as_xml_equals(expected_xml)
root_suite = test_suite('suite that passes', [], [test_case('test case file name')])
test_suites = [root_suite]
# ACT #
actual = execute_with_case_processing_with_constant_result(
tcp.new_executed(case_result),
root_suite,
Path(),
test_suites)
# ASSERT #
self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code)
expected__assertion.apply_with_message(self,
replace_xml_variables(actual.stdout),
'suite xml on stdout')
def test_single_case_with_error(self):
cases = [
FULL_RESULT_HARD_ERROR,
FULL_RESULT_VALIDATE,
FULL_RESULT_IMPLEMENTATION_ERROR,
]
for case_result in cases:
with self.subTest(case_result_status=case_result.status):
# ARRANGE #
expected_xml = suite_xml(attributes={
'name': 'suite with error',
'tests': '1',
'errors': '1',
'failures': '0'},
test_case_elements=[
erroneous_test_case_xml('test case file name',
error_type=case_result.status.name,
failure_message=error_message_for_full_result(case_result))])
expected_output = expected_output_from(expected_xml)
root_suite = test_suite('suite with error', [], [test_case('test case file name')])
test_suites = [root_suite]
# ACT #
actual = execute_with_case_processing_with_constant_result(
tcp.new_executed(case_result),
root_suite,
Path(),
test_suites)
# ASSERT #
self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code)
self.assertEqual(expected_output, replace_xml_variables(actual.stdout))
def test_suite_with_only_single_sub_suite_SHOULD_not_include_root_suite_as_test_suite(self):
# ARRANGE #
suite_with_single_case = test_suite('suite with single case', [], [
test_case('the test case')
])
root_suite = test_suite('root suite file name', [
suite_with_single_case,
], [])
suites = [
root_suite,
suite_with_single_case,
]
# ACT #
actual = execute_with_case_processing_with_constant_result(tcp.new_executed(FULL_RESULT_PASS),
root_suite,
Path(),
suites)
# ASSERT #
expected_xml = _suites_xml([
suite_xml(attributes={
'name': 'suite with single case',
'package': '.',
'id': '1',
'tests': '1',
'errors': '0',
'failures': '0',
},
test_case_elements=[successful_test_case_xml('the test case')]
),
])
expected__assertion = asrt_etree.str_as_xml_equals(expected_xml)
self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code)
expected__assertion.apply_with_message(self,
replace_xml_variables(actual.stdout),
'suite xml on stdout')
def test_single_case_with_error(self):
cases = [
FULL_RESULT_HARD_ERROR,
FULL_RESULT_VALIDATE,
FULL_RESULT_INTERNAL_ERROR,
]
for case_result in cases:
with self.subTest(case_result_status=case_result.status):
# ARRANGE #
expected_xml = suite_xml(attributes={
'name': 'suite with error',
'tests': '1',
'errors': '1',
'failures': '0'},
test_case_elements=[
erroneous_test_case_xml('test case file name',
error_type=case_result.status.name,
failure_message=error_message_for_full_result(case_result))])
expected__assertion = asrt_etree.str_as_xml_equals(expected_xml)
root_suite = test_suite('suite with error', [], [test_case('test case file name')])
test_suites = [root_suite]
# ACT #
actual = execute_with_case_processing_with_constant_result(
tcp.new_executed(case_result),
root_suite,
Path(),
test_suites)
# ASSERT #
self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code)
expected__assertion.apply_with_message(self,
replace_xml_variables(actual.stdout),
'suite xml on stdout')
def test_single_sub_suite_with_test_cases_with_different_results(self):
# ARRANGE #
tc_pass = test_case('successful case')
tc_fail = test_case('failing case')
tc_error = test_case('erroneous case')
root_suite = test_suite('suite file name', [], [
tc_pass,
tc_fail,
tc_error,
])
suites = [root_suite]
test_case_processor = TestCaseProcessorThatGivesConstantPerCase({
id(tc_pass): tcp.new_executed(FULL_RESULT_PASS),
id(tc_fail): tcp.new_executed(FULL_RESULT_FAIL),
id(tc_error): tcp.new_executed(FULL_RESULT_HARD_ERROR),
})
# ACT #
actual = execute_with_case_processing_with_constant_processor(test_case_processor,
root_suite,
Path(),
suites)
# ASSERT #
expected_xml = suite_xml(attributes={
'name': 'suite file name',
'tests': '3',
'errors': '1',
'failures': '1',
},
test_case_elements=[
successful_test_case_xml('successful case'),
failing_test_case_xml('failing case',
failure_type=FULL_RESULT_FAIL.status.name,
failure_message=error_message_for_full_result(FULL_RESULT_FAIL)),
erroneous_test_case_xml('erroneous case',
error_type=FULL_RESULT_HARD_ERROR.status.name,
failure_message=error_message_for_full_result(FULL_RESULT_HARD_ERROR)),
]
)
expected__assertion = asrt_etree.str_as_xml_equals(expected_xml)
self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code)
expected__assertion.apply_with_message(self,
replace_xml_variables(actual.stdout),
'suite xml on stdout')
def test_single_sub_suite_with_test_cases_with_different_results(self):
# ARRANGE #
tc_pass = test_case('successful case')
tc_fail = test_case('failing case')
tc_error = test_case('erroneous case')
root_suite = test_suite('suite file name', [], [
tc_pass,
tc_fail,
tc_error,
])
suites = [root_suite]
test_case_processor = TestCaseProcessorThatGivesConstantPerCase({
id(tc_pass): tcp.new_executed(FULL_RESULT_PASS),
id(tc_fail): tcp.new_executed(FULL_RESULT_FAIL),
id(tc_error): tcp.new_executed(FULL_RESULT_HARD_ERROR),
})
# ACT #
actual = execute_with_case_processing_with_constant_processor(test_case_processor,
root_suite,
Path(),
suites)
# ASSERT #
expected_xml = suite_xml(attributes={
'name': 'suite file name',
'tests': '3',
'errors': '1',
'failures': '1',
},
test_case_elements=[
successful_test_case_xml('successful case'),
failing_test_case_xml('failing case',
failure_type=FULL_RESULT_FAIL.status.name,
failure_message=error_message_for_full_result(FULL_RESULT_FAIL)),
erroneous_test_case_xml('erroneous case',
error_type=FULL_RESULT_HARD_ERROR.status.name,
failure_message=error_message_for_full_result(FULL_RESULT_HARD_ERROR)),
]
)
expected_output = expected_output_from(expected_xml)
self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code)
self.assertEqual(expected_output, replace_xml_variables(actual.stdout))
def test_that_report_of_failing_tests_are_grouped_by_exit_identifiers(
self):
cases = [
(FULL_RESULT_FAIL, case_ev.EXECUTION__FAIL),
(FULL_RESULT_HARD_ERROR, case_ev.EXECUTION__HARD_ERROR),
(FULL_RESULT_VALIDATE, case_ev.EXECUTION__VALIDATION_ERROR),
(FULL_RESULT_INTERNAL_ERROR, case_ev.EXECUTION__INTERNAL_ERROR),
]
test_case_file_name = 'test-case-file'
for case_result, expected_case_exit_value in cases:
with self.subTest(
case_result_status=case_result.status,
expected_case_exit_value=expected_case_exit_value):
# ARRANGE #
root_suite = test_suite('root file name', [],
[test_case(test_case_file_name)])
test_suites = [root_suite]
std_output_files = StringStdOutFiles()
executor = _suite_executor_for_case_processing_that_unconditionally(
case_result, root_suite, std_output_files, Path())
# ACT #
executor.execute_and_report(test_suites)
# ASSERT #
std_output_files.finish()
stderr_contents = std_output_files.stderr_contents
exit_ident_pos = stderr_contents.find(
expected_case_exit_value.exit_identifier)
self.assertNotEqual(-1, exit_ident_pos,
'stderr must contain the exit identifier')
start_of_exit_ident_group = stderr_contents[exit_ident_pos:]
group_elements = start_of_exit_ident_group.split()
self.assertEqual(
len(group_elements), 2,
'Expecting to find EXIT_IDENTIFIER followed by CASE-FILE-NAME as final contents of stderr'
)
self.assertEqual(
expected_case_exit_value.exit_identifier,
group_elements[0],
'Expects the exit identifier as a single word')
self.assertEqual(
test_case_file_name, group_elements[1],
'Expects the test case file name to follow the exit identifier'
)
def test_suite_with_single_case(self):
cases = [
(FULL_RESULT_PASS, case_ev.EXECUTION__PASS, suite_ev.ALL_PASS),
(FULL_RESULT_FAIL, case_ev.EXECUTION__FAIL, suite_ev.FAILED_TESTS),
(FULL_RESULT_SKIP, case_ev.EXECUTION__SKIPPED, suite_ev.ALL_PASS),
(FULL_RESULT_HARD_ERROR, case_ev.EXECUTION__HARD_ERROR,
suite_ev.FAILED_TESTS),
(FULL_RESULT_VALIDATE, case_ev.EXECUTION__VALIDATION_ERROR,
suite_ev.FAILED_TESTS),
(FULL_RESULT_INTERNAL_ERROR, case_ev.EXECUTION__INTERNAL_ERROR,
suite_ev.FAILED_TESTS),
]
for case_result, expected_case_exit_value, expected_suite_exit_value in cases:
with self.subTest(
case_result_status=case_result.status,
expected_case_exit_value=expected_case_exit_value,
expected_suite_exit_value=expected_suite_exit_value):
# ARRANGE #
expected_output = lines_content_with_os_linesep([
_suite_begin('root file name'),
_case('test case file', expected_case_exit_value),
_suite_end('root file name'),
expected_suite_exit_value.exit_identifier,
])
root_suite = test_suite('root file name', [],
[test_case('test case file')])
test_suites = [root_suite]
std_output_files = StringStdOutFiles()
executor = _suite_executor_for_case_processing_that_unconditionally(
case_result, root_suite, std_output_files, Path())
# ACT #
exit_code = executor.execute_and_report(test_suites)
# ASSERT #
std_output_files.finish()
self.assertEqual(expected_suite_exit_value.exit_code,
exit_code)
stdout_contents_prepared_for_assertion = _replace_seconds_with_const(
std_output_files.stdout_contents)
self.assertEqual(expected_output,
stdout_contents_prepared_for_assertion)
def test_suite_with_single_case(self):
cases = [
(FULL_RESULT_PASS, case_ev.EXECUTION__PASS, suite_ev.ALL_PASS),
(FULL_RESULT_FAIL, case_ev.EXECUTION__FAIL, suite_ev.FAILED_TESTS),
(FULL_RESULT_SKIP, case_ev.EXECUTION__SKIPPED, suite_ev.ALL_PASS),
(FULL_RESULT_HARD_ERROR, case_ev.EXECUTION__HARD_ERROR, suite_ev.FAILED_TESTS),
(FULL_RESULT_VALIDATE, case_ev.EXECUTION__VALIDATION_ERROR, suite_ev.FAILED_TESTS),
(FULL_RESULT_IMPLEMENTATION_ERROR, case_ev.EXECUTION__IMPLEMENTATION_ERROR, suite_ev.FAILED_TESTS),
]
for case_result, expected_case_exit_value, expected_suite_exit_value in cases:
with self.subTest(case_result_status=case_result.status,
expected_case_exit_value=expected_case_exit_value,
expected_suite_exit_value=expected_suite_exit_value):
# ARRANGE #
expected_output = lines_content_with_os_linesep([
_suite_begin('root file name'),
_case('test case file', expected_case_exit_value),
_suite_end('root file name'),
expected_suite_exit_value.exit_identifier,
])
root_suite = test_suite('root file name', [], [test_case('test case file')])
test_suites = [root_suite]
std_output_files = StringStdOutFiles()
executor = _suite_executor_for_case_processing_that_unconditionally(case_result,
root_suite,
std_output_files,
Path())
# ACT #
exit_code = executor.execute_and_report(test_suites)
# ASSERT #
std_output_files.finish()
self.assertEqual(expected_suite_exit_value.exit_code, exit_code)
stdout_contents_prepared_for_assertion = _replace_seconds_with_const(std_output_files.stdout_contents)
self.assertEqual(expected_output,
stdout_contents_prepared_for_assertion)
def test_single_case_with_error_due_to_failure_to_execute(self):
cases = [
(tcp.new_internal_error(error_info.of_message('error message')),
tcp.Status.INTERNAL_ERROR.name),
(tcp.new_access_error(AccessErrorType.FILE_ACCESS_ERROR,
error_info.of_message('error message')),
AccessErrorType.FILE_ACCESS_ERROR.name),
]
for case_result, error_type in cases:
with self.subTest(case_result_status=case_result.status):
# ARRANGE #
expected_xml = suite_xml(attributes={
'name': 'suite with error',
'tests': '1',
'errors': '1',
'failures': '0'},
test_case_elements=[
erroneous_test_case_xml('test case file name',
error_type=error_type,
failure_message=error_message_for_error_info(case_result.error_info))
])
expected__assertion = asrt_etree.str_as_xml_equals(expected_xml)
root_suite = test_suite('suite with error', [], [test_case('test case file name')])
test_suites = [root_suite]
# ACT #
actual = execute_with_case_processing_with_constant_result(
case_result,
root_suite,
Path(),
test_suites)
# ASSERT #
self.assertEqual(sut.UNCONDITIONAL_EXIT_CODE, actual.exit_code)
expected__assertion.apply_with_message(self,
replace_xml_variables(actual.stdout),
'suite xml on stdout')