def expected_structure_based_at(self, root_path: pathlib.Path) -> ValueAssertion[TestSuiteHierarchy]:
return equals_test_suite(TestSuiteHierarchy(
self.root_suite_based_at(root_path),
[],
T_C_H_S,
[
TestSuiteHierarchy(
Path('local.suite'),
[self.root_suite_based_at(root_path)],
T_C_H_S,
[],
[test_case_reference_of_source_file(Path('from-local-suite.case'))]),
TestSuiteHierarchy(
Path('sub') / 'sub.suite',
[self.root_suite_based_at(root_path)],
T_C_H_S,
[TestSuiteHierarchy(
Path('sub') / 'sub-sub.suite',
[self.root_suite_based_at(root_path), Path('sub') / 'sub.suite'],
T_C_H_S,
[],
[test_case_reference_of_source_file(Path('sub') / 'sub-sub.case')]
)],
[test_case_reference_of_source_file(Path('sub') / 'sub.case')]),
],
[test_case_reference_of_source_file(Path('from-main-suite.case'))]
))
def test_with_multiple_errors(self):
# ARRANGE #
elapsed_time = datetime.timedelta(seconds=1)
num_test_cases = 6
rel_root = Path.cwd().resolve()
the_exit_value = ExitValue(4, 'identifier_4', ForegroundColor.RED)
errors = {the_exit_value:
[
test_case_reference_of_source_file(rel_root / Path('fip-1') / 'case-1'),
test_case_reference_of_source_file(rel_root / Path('fip-2') / 'case-2'),
],
ExitValue(12, 'longer_identifier_12', ForegroundColor.RED):
[
test_case_reference_of_source_file(rel_root / Path('fip-3') / 'case-3'),
],
}
# ACT #
actual_lines = sut.format_final_result_for_valid_suite(num_test_cases, elapsed_time,
rel_root,
errors)
# ASSERT #
self._assert_at_least_one_line_was_generated(actual_lines)
self._assert_line_is_number_of_executed_tests_line(actual_lines[0], num_test_cases)
self.assertListEqual(['',
the_exit_value.exit_identifier,
' ' + str(Path('fip-1') / Path('case-1')),
' ' + str(Path('fip-2') / Path('case-2')),
'longer_identifier_12',
' ' + str(Path('fip-3') / Path('case-3')),
],
actual_lines[1:],
'Lines after "Ran ..."')
def expected_structure_based_at(
self, root_path: pathlib.Path) -> Assertion[TestSuiteHierarchy]:
dir_with_sub_suite = Path('main-sub-dir') / 'suite-sub-dir'
return equals_test_suite(
TestSuiteHierarchy(
self.root_suite_based_at(root_path),
[],
T_C_H_S,
[
TestSuiteHierarchy(
dir_with_sub_suite / 'sub.suite',
[self.root_suite_based_at(root_path)],
T_C_H_S,
[],
[
test_case_reference_of_source_file(
dir_with_sub_suite / 'sub.case')
],
),
],
[
test_case_reference_of_source_file(
Path('main-sub-dir') / 'main.case')
],
))
def expected_structure_based_at(
self, root_path: pathlib.Path) -> Assertion[TestSuiteHierarchy]:
return equals_test_suite(
TestSuiteHierarchy(
self.root_suite_based_at(root_path), [], T_C_H_S, [
TestSuiteHierarchy(Path('local.suite'),
[self.root_suite_based_at(root_path)],
T_C_H_S, [], [
test_case_reference_of_source_file(
Path('from-local-suite.case'))
]),
TestSuiteHierarchy(
Path('sub') / 'sub.suite',
[self.root_suite_based_at(root_path)], T_C_H_S, [
TestSuiteHierarchy(
Path('sub') / 'sub-sub.suite', [
self.root_suite_based_at(root_path),
Path('sub') / 'sub.suite'
], T_C_H_S, [], [
test_case_reference_of_source_file(
Path('sub') / 'sub-sub.case')
])
], [
test_case_reference_of_source_file(
Path('sub') / 'sub.case')
]),
], [
test_case_reference_of_source_file(
Path('from-main-suite.case'))
]))
def expected_structure_based_at(
self, root_path: pathlib.Path) -> Assertion[TestSuiteHierarchy]:
return equals_test_suite(
TestSuiteHierarchy(
self.root_suite_based_at(root_path), [], T_C_H_S, [], [
test_case_reference_of_source_file(Path('1.case')),
test_case_reference_of_source_file(Path('sub') / '2.case')
]))
def expected_structure_based_at(self, root_path: pathlib.Path) -> ValueAssertion[TestSuiteHierarchy]:
return equals_test_suite(TestSuiteHierarchy(
self.root_suite_based_at(root_path),
[],
T_C_H_S,
[],
[
test_case_reference_of_source_file(Path('1.case')),
test_case_reference_of_source_file(Path('sub') / '2.case')
]))
def test_accessor_exception_from_accessor(self):
# ARRANGE #
process_error = tcp.ProcessError(
error_info.of_exception(ValueError('exception message')))
accessor = sut.AccessorFromParts(
SourceReaderThat(raises(process_error)),
PreprocessorThat(gives_constant('preprocessed source')),
ParserThat(gives_constant(TEST_CASE)),
identity_test_case_transformer())
executor = ExecutorThat(raises(PROCESS_ERROR))
processor = sut.ProcessorFromAccessorAndExecutor(accessor, executor)
# ACT #
result = processor.apply(tcp.test_case_reference_of_source_file(PATH))
# ASSERT #
self.assertIs(tcp.Status.ACCESS_ERROR, result.status)
self.assertIs(tcp.AccessErrorType.FILE_ACCESS_ERROR,
result.access_error_type)
self.assertIsNotNone(result.error_info, 'There should be ErrorInfo')
err_description = result.error_info.description
self.assertIsInstance(err_description,
error_description.ErrorDescriptionOfException)
assert isinstance(err_description,
error_description.ErrorDescriptionOfException)
self.assertEqual(err_description.exception.args[0],
'exception message')
def test_accessor_exception_from_accessor(self):
# ARRANGE #
process_error = tcp.ProcessError(
error_info.of_exception(ValueError('exception message')))
accessor = sut.AccessorFromParts(SourceReaderThat(raises(process_error)),
PreprocessorThat(gives_constant('preprocessed source')),
ParserThat(gives_constant(TEST_CASE)),
identity_test_case_transformer())
executor = ExecutorThat(raises(PROCESS_ERROR))
processor = sut.ProcessorFromAccessorAndExecutor(accessor,
executor)
# ACT #
result = processor.apply(tcp.test_case_reference_of_source_file(PATH))
# ASSERT #
self.assertIs(tcp.Status.ACCESS_ERROR,
result.status)
self.assertIs(tcp.AccessErrorType.FILE_ACCESS_ERROR,
result.access_error_type)
self.assertIsNotNone(result.error_info,
'There should be ErrorInfo')
err_description = result.error_info.description
self.assertIsInstance(err_description,
error_description.ErrorDescriptionOfException)
assert isinstance(err_description,
error_description.ErrorDescriptionOfException)
self.assertEqual(err_description.exception.args[0],
'exception message')
def _test_with_error_of_path_above_relativity_root(self,
rel_root_path: Path):
# ARRANGE #
elapsed_time = datetime.timedelta(seconds=1)
num_test_cases = 6
the_exit_identifier = 'exit_identifier_4'
errors = {
the_exit_identifier: [
test_case_reference_of_source_file(rel_root_path.parent /
Path('fip-1') / 'case-1'),
],
}
# ACT #
actual_lines = sut.format_final_result_for_valid_suite(
num_test_cases, elapsed_time, rel_root_path, errors)
# ASSERT #
self._assert_at_least_one_line_was_generated(actual_lines)
self._assert_line_is_number_of_executed_tests_line(
actual_lines[0], num_test_cases)
self.assertEqual(
['', _NUMBER_OF_ERRORS.of(1), ''], actual_lines[1:4],
'Reporting of number of unsuccessful tests (including separating lines)'
)
self.assertListEqual([
the_exit_identifier,
' ' + str(rel_root_path.parent / Path('fip-1') / Path('case-1')),
], actual_lines[4:], 'Lines after "Num unsuccessful ..."')
def test_single_suite(self):
root_suite = TestSuiteHierarchy(
pathlib.Path('root-file'), [], T_C_H_S, [],
[test_case_reference_of_source_file(pathlib.Path('case-file'))])
suite_list = DepthFirstEnumerator().apply(root_suite)
self.assertEqual(1, len(suite_list), 'Number of enumerated suites')
self.assertIs(root_suite, suite_list[0], 'Suite object')
def expected_structure_based_at(self, root_path: pathlib.Path) -> ValueAssertion[TestSuiteHierarchy]:
dir_with_sub_suite = Path('main-sub-dir') / 'suite-sub-dir'
return equals_test_suite(TestSuiteHierarchy(
self.root_suite_based_at(root_path),
[],
T_C_H_S,
[
TestSuiteHierarchy(
dir_with_sub_suite / 'sub.suite',
[self.root_suite_based_at(root_path)],
T_C_H_S,
[],
[test_case_reference_of_source_file(dir_with_sub_suite / 'sub.case')],
),
],
[test_case_reference_of_source_file(Path('main-sub-dir') / 'main.case')],
))
def expected_structure_based_at(
self, root_path: pathlib.Path) -> Assertion[TestSuiteHierarchy]:
return equals_test_suite(
TestSuiteHierarchy(self.root_suite_based_at(root_path), [],
T_C_H_S, [], [
test_case_reference_of_source_file(
Path('case-with-invalid-content.case')),
]))
def test_single_suite_with_test_cases_with_different_result(self):
# ARRANGE #
reporter = ExecutionTracingProcessingReporter()
str_std_out_files = StringStdOutFiles()
tc_internal_error = test_case_reference_of_source_file(Path('internal error'))
tc_access_error = test_case_reference_of_source_file(Path('access error'))
tc_executed = test_case_reference_of_source_file(Path('executed'))
root = test_suite(
'root',
[],
[
tc_internal_error,
tc_access_error,
tc_executed,
])
test_case_processor = TestCaseProcessorThatGivesConstantPerCase({
id(tc_internal_error): new_internal_error(error_info.of_message('message')),
id(tc_access_error): new_access_error(
tcp.AccessErrorType.SYNTAX_ERROR, error_info.of_message('syntax error')),
id(tc_executed): new_executed(FULL_RESULT_PASS),
})
expected_suites = [
ExpectedSuiteReporting(root,
[
(tc_internal_error, tcp.Status.INTERNAL_ERROR),
(tc_access_error, tcp.Status.ACCESS_ERROR),
(tc_executed, tcp.Status.EXECUTED),
])
]
suite_hierarchy_reader = ReaderThatGivesConstantSuite(root)
processor = Processor(DUMMY_CASE_PROCESSING,
suite_hierarchy_reader,
reporter,
DepthFirstEnumerator(),
lambda config: test_case_processor)
# ACT #
exit_code = processor.process(pathlib.Path('root-suite-file'), str_std_out_files.stdout_files)
# ASSERT #
check_exit_code_and_empty_stdout(self,
ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE,
exit_code,
str_std_out_files)
ExpectedSuiteReporting.check_list(
self,
expected_suites,
reporter.complete_suite_reporter)
def _check_failing_line(self,
configuration: sut.Configuration,
phases: Sequence[Phase],
invalid_line_cases: Sequence[NameAndValue[SourceAndStatus]]):
proc_cases = [
NameAndValue('not allowed to pollute current process',
sut.new_processor_that_should_not_pollute_current_process(configuration)),
NameAndValue('allowed to pollute current process',
sut.new_processor_that_is_allowed_to_pollute_current_process(configuration)),
]
for phase in phases:
for invalid_line_case in invalid_line_cases:
source_and_status = invalid_line_case.value
assert isinstance(source_and_status, SourceAndStatus) # Type info for IDE
file_with_error = fs.file_with_lines('file-with-error.src', [
source_and_status.failing_source_line,
])
erroneous_line = single_line_sequence(1, source_and_status.failing_source_line)
test_case_file = fs.file_with_lines('test.case', [
section_header(phase.section_name),
directive_for_inclusion_of_file(file_with_error.name),
])
line_that_includes_erroneous_file = single_line_sequence(2, directive_for_inclusion_of_file(
file_with_error.name))
cwd_contents = fs.DirContents([test_case_file,
file_with_error])
expected_source_location_path = SourceLocationPath(
location=SourceLocation(erroneous_line,
pathlib.Path(file_with_error.name)),
file_inclusion_chain=[SourceLocation(line_that_includes_erroneous_file,
pathlib.Path(test_case_file.name))])
for proc_case in proc_cases:
with self.subTest(phase=phase.section_name,
proc=proc_case.name,
line=erroneous_line.first_line):
processor = proc_case.value
assert isinstance(processor, Processor)
with tmp_dir_as_cwd(cwd_contents):
test_case_reference = test_case_reference_of_source_file(pathlib.Path(test_case_file.name))
# ACT #
result = processor.apply(test_case_reference)
# ASSERT #
assert isinstance(result, Result) # Type info for IDE
source_and_status.expected_result_statuses.apply_with_message(self,
result,
'result statuses')
source_location_path_expectation = equals_source_location_path(
expected_source_location_path)
source_location_path_expectation.apply_with_message(self,
result.source_location_path,
'source location path')
def expected_structure_based_at(self, root_path: pathlib.Path) -> ValueAssertion[TestSuiteHierarchy]:
return equals_test_suite(TestSuiteHierarchy(
self.root_suite_based_at(root_path),
[],
T_C_H_S,
[],
[
test_case_reference_of_source_file(Path('case-with-invalid-content.case')),
]))
def test_single_suite_with_test_cases_with_different_result(self):
# ARRANGE #
reporter = ExecutionTracingProcessingReporter()
str_std_out_files = StringStdOutFiles()
tc_internal_error = test_case_reference_of_source_file(
Path('internal error'))
tc_access_error = test_case_reference_of_source_file(
Path('access error'))
tc_executed = test_case_reference_of_source_file(Path('executed'))
root = test_suite('root', [], [
tc_internal_error,
tc_access_error,
tc_executed,
])
test_case_processor = TestCaseProcessorThatGivesConstantPerCase({
id(tc_internal_error):
new_internal_error(error_info.of_message('message')),
id(tc_access_error):
new_access_error(tcp.AccessErrorType.SYNTAX_ERROR,
error_info.of_message('syntax error')),
id(tc_executed):
new_executed(FULL_RESULT_PASS),
})
expected_suites = [
ExpectedSuiteReporting(root, [
(tc_internal_error, tcp.Status.INTERNAL_ERROR),
(tc_access_error, tcp.Status.ACCESS_ERROR),
(tc_executed, tcp.Status.EXECUTED),
])
]
suite_hierarchy_reader = ReaderThatGivesConstantSuite(root)
processor = Processor(DUMMY_CASE_PROCESSING, suite_hierarchy_reader,
reporter, DepthFirstEnumerator(),
lambda config: test_case_processor)
# ACT #
exit_code = processor.process(pathlib.Path('root-suite-file'),
str_std_out_files.reporting_environment)
# ASSERT #
check_exit_code_and_empty_stdout(
self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE,
exit_code, str_std_out_files)
ExpectedSuiteReporting.check_list(self, expected_suites,
reporter.complete_suite_reporter)
def test_implementation_exception_from_executor(self):
# ARRANGE #
accessor = sut.AccessorFromParts(
SourceReaderThat(gives_constant('source')),
PreprocessorThat(gives_constant('preprocessed source')),
ParserThat(gives_constant(TEST_CASE)),
identity_test_case_transformer())
executor = ExecutorThat(raises(RuntimeError()))
processor = sut.ProcessorFromAccessorAndExecutor(accessor, executor)
# ACT #
result = processor.apply(tcp.test_case_reference_of_source_file(PATH))
# ASSERT #
self.assertEqual(tcp.Status.INTERNAL_ERROR, result.status)
def test_single_suite(self):
root_suite = TestSuiteHierarchy(
pathlib.Path('root-file'),
[],
T_C_H_S,
[],
[test_case_reference_of_source_file(pathlib.Path('case-file'))])
suite_list = DepthFirstEnumerator().apply(root_suite)
self.assertEqual(1,
len(suite_list),
'Number of enumerated suites')
self.assertIs(root_suite,
suite_list[0],
'Suite object')
def test_implementation_exception_from_executor(self):
# ARRANGE #
accessor = sut.AccessorFromParts(SourceReaderThat(gives_constant('source')),
PreprocessorThat(gives_constant('preprocessed source')),
ParserThat(gives_constant(TEST_CASE)),
identity_test_case_transformer())
executor = ExecutorThat(raises(RuntimeError()))
processor = sut.ProcessorFromAccessorAndExecutor(accessor,
executor)
# ACT #
result = processor.apply(tcp.test_case_reference_of_source_file(PATH))
# ASSERT #
self.assertEqual(tcp.Status.INTERNAL_ERROR,
result.status)
def test_with_multiple_errors(self):
# ARRANGE #
elapsed_time = datetime.timedelta(seconds=1)
num_test_cases = 6
rel_root = Path.cwd().resolve()
the_exit_identifier = 'identifier_4'
errors = {
the_exit_identifier: [
test_case_reference_of_source_file(rel_root / Path('fip-1') /
'case-1'),
test_case_reference_of_source_file(rel_root / Path('fip-2') /
'case-2'),
],
'longer_identifier_12': [
test_case_reference_of_source_file(rel_root / Path('fip-3') /
'case-3'),
],
}
# ACT #
actual_lines = sut.format_final_result_for_valid_suite(
num_test_cases, elapsed_time, rel_root, errors)
# ASSERT #
self._assert_at_least_one_line_was_generated(actual_lines)
self._assert_line_is_number_of_executed_tests_line(
actual_lines[0], num_test_cases)
self.assertEqual(
['', _NUMBER_OF_ERRORS.of(3), ''], actual_lines[1:4],
'Reporting of number of unsuccessful tests (including separating lines)'
)
self.assertListEqual([
the_exit_identifier,
' ' + str(Path('fip-1') / Path('case-1')),
' ' + str(Path('fip-2') / Path('case-2')),
'longer_identifier_12',
' ' + str(Path('fip-3') / Path('case-3')),
], actual_lines[4:], 'Lines after "Ran ..."')
def test_successful_application(self):
# ARRANGE #
accessor = sut.AccessorFromParts(
SourceReaderThat(gives_constant('source')),
PreprocessorThat(gives_constant('preprocessed source')),
ParserThat(gives_constant(TEST_CASE)),
identity_test_case_transformer())
full_result = new_skipped()
executor = ExecutorThatReturnsIfSame(TEST_CASE, full_result)
processor = sut.ProcessorFromAccessorAndExecutor(accessor, executor)
# ACT #
result = processor.apply(tcp.test_case_reference_of_source_file(PATH))
# ASSERT #
self.assertIs(tcp.Status.EXECUTED, result.status)
self.assertIs(full_result, result.execution_result)
def expected_structure_based_at(self, root_path: pathlib.Path) -> ValueAssertion[TestSuiteHierarchy]:
return equals_test_suite(TestSuiteHierarchy(
self.root_suite_based_at(root_path),
[],
T_C_H_S,
[
TestSuiteHierarchy(root_path / '1.suite',
[self.root_suite_based_at(root_path)],
T_C_H_S,
[], []),
],
[
test_case_reference_of_source_file(root_path / '1.case'),
],
))
def test_inclusion_of_non_exiting_file_SHOULD_cause_file_access_error(
self):
# ARRANGE #
name_of_non_existing_file = 'non-existing.src'
configuration = configuration_with_no_instructions_and_no_preprocessor(
)
proc_cases = [
NameAndValue(
'not allowed to pollute current process',
sut.new_processor_that_should_not_pollute_current_process(
configuration)),
NameAndValue(
'allowed to pollute current process',
sut.new_processor_that_is_allowed_to_pollute_current_process(
configuration)),
]
for phase in phase_identifier.ALL_WITH_INSTRUCTIONS:
for proc_case in proc_cases:
with self.subTest(phase.section_name, proc=proc_case.name):
test_case_file = fs.file_with_lines(
'test.case', [
section_header(phase.section_name),
directive_for_inclusion_of_file(
name_of_non_existing_file),
])
cwd_contents = fs.DirContents([test_case_file])
processor = proc_case.value
assert isinstance(processor, Processor)
with tmp_dir_as_cwd(cwd_contents):
test_case_reference = test_case_reference_of_source_file(
pathlib.Path(test_case_file.name))
# ACT #
result = processor.apply(test_case_reference)
# ASSERT #
assert isinstance(result, Result) # Type info for IDE
self.assertEqual(Status.ACCESS_ERROR, result.status)
self.assertEqual(AccessErrorType.FILE_ACCESS_ERROR,
result.access_error_type)
source_location_path_expectation = equals_source_location_path(
source_location_path_of(
pathlib.Path(test_case_file.name),
Line(
2,
directive_for_inclusion_of_file(
name_of_non_existing_file))))
source_location_path_expectation.apply_with_message(
self, result.error_info.source_location_path,
'source location path')
def expected_structure_based_at(
self, root_path: pathlib.Path) -> Assertion[TestSuiteHierarchy]:
return equals_test_suite(
TestSuiteHierarchy(
self.root_suite_based_at(root_path),
[],
T_C_H_S,
[
TestSuiteHierarchy(root_path / '1.suite',
[self.root_suite_based_at(root_path)],
T_C_H_S, [], []),
],
[
test_case_reference_of_source_file(root_path / '1.case'),
],
))
def process(self,
std_output_files: StdOutputFiles,
settings: TestCaseExecutionSettings,
) -> int:
result_reporter = self._get_reporter(std_output_files, settings.reporting_option)
try:
processor = self._processor(settings, result_reporter)
except SuiteParseError as ex:
reporter = result_reporting.TestSuiteParseErrorReporter(std_output_files)
return reporter.report(ex)
test_case_file_ref = test_case_processing.test_case_reference_of_source_file(settings.test_case_file_path)
result = processor.apply(test_case_file_ref)
return result_reporter.report(result)
def test_successful_application(self):
# ARRANGE #
accessor = sut.AccessorFromParts(SourceReaderThat(gives_constant('source')),
PreprocessorThat(gives_constant('preprocessed source')),
ParserThat(gives_constant(TEST_CASE)),
identity_test_case_transformer())
full_result = new_skipped()
executor = ExecutorThatReturnsIfSame(TEST_CASE, full_result)
processor = sut.ProcessorFromAccessorAndExecutor(accessor,
executor)
# ACT #
result = processor.apply(tcp.test_case_reference_of_source_file(PATH))
# ASSERT #
self.assertIs(tcp.Status.EXECUTED,
result.status)
self.assertIs(full_result,
result.execution_result)
def test_inclusion_of_file_SHOULD_be_possible_in_all_phases_except_act(
self):
# ARRANGE #
name_of_recording_instruction = 'recording-instruction'
file_to_include = fs.file_with_lines('included-file.src', [
name_of_recording_instruction,
])
proc_cases = [
NameAndValue(
'not allowed to pollute current process',
sut.new_processor_that_should_not_pollute_current_process),
NameAndValue(
'allowed to pollute current process',
sut.new_processor_that_is_allowed_to_pollute_current_process),
]
for phase in phase_identifier.ALL_WITH_INSTRUCTIONS:
for proc_case in proc_cases:
with self.subTest(phase.section_name, proc=proc_case.name):
test_case_file = fs.file_with_lines(
'test.case', [
section_header(phase.section_name),
directive_for_inclusion_of_file(
file_to_include.name),
])
cwd_contents = fs.DirContents(
[test_case_file, file_to_include])
recording_output = []
configuration = configuration_with_instruction_in_each_phase_that_records_phase_name(
name_of_recording_instruction, recording_output)
processor = proc_case.value(configuration)
with tmp_dir_as_cwd(cwd_contents):
test_case_reference = test_case_reference_of_source_file(
pathlib.Path(test_case_file.name))
# ACT #
result = processor.apply(test_case_reference)
# ASSERT #
assert isinstance(result, Result) # Type info for IDE
self.assertEqual(Status.EXECUTED, result.status)
self.assertFalse(result.execution_result.is_failure)
self.assertEqual(FullExeResultStatus.PASS,
result.execution_result.status)
self.assertEqual([phase.section_name],
recording_output)
def test_inclusion_of_file_SHOULD_be_possible_in_all_phases_except_act(self):
# ARRANGE #
name_of_recording_instruction = 'recording-instruction'
file_to_include = fs.file_with_lines('included-file.src', [
name_of_recording_instruction,
])
proc_cases = [
NameAndValue('not allowed to pollute current process',
sut.new_processor_that_should_not_pollute_current_process),
NameAndValue('allowed to pollute current process',
sut.new_processor_that_is_allowed_to_pollute_current_process),
]
for phase in phase_identifier.ALL_WITH_INSTRUCTIONS:
for proc_case in proc_cases:
with self.subTest(phase.section_name,
proc=proc_case.name):
test_case_file = fs.file_with_lines('test.case', [
section_header(phase.section_name),
directive_for_inclusion_of_file(file_to_include.name),
])
cwd_contents = fs.DirContents([test_case_file,
file_to_include])
recording_output = []
configuration = configuration_with_instruction_in_each_phase_that_records_phase_name(
name_of_recording_instruction,
recording_output)
processor = proc_case.value(configuration)
with tmp_dir_as_cwd(cwd_contents):
test_case_reference = test_case_reference_of_source_file(pathlib.Path(test_case_file.name))
# ACT #
result = processor.apply(test_case_reference)
# ASSERT #
assert isinstance(result, Result) # Type info for IDE
self.assertEqual(Status.EXECUTED,
result.status)
self.assertFalse(result.execution_result.is_failure)
self.assertEqual(FullExeResultStatus.PASS,
result.execution_result.status)
self.assertEqual([phase.section_name],
recording_output)
def _check(self, result: tcp.Result):
# ARRANGE #
str_std_out_files = StringStdOutFiles()
test_case = test_case_reference_of_source_file(Path('test-case'))
root = test_suite('root', [], [test_case])
suite_hierarchy_reader = ReaderThatGivesConstantSuite(root)
reporter = ExecutionTracingProcessingReporter()
executor = Processor(
DUMMY_CASE_PROCESSING, suite_hierarchy_reader, reporter,
DepthFirstEnumerator(),
lambda config: TestCaseProcessorThatGivesConstant(result))
# ACT #
exit_code = executor.process(pathlib.Path('root-suite-file'),
str_std_out_files.reporting_environment)
# ASSERT #
check_exit_code_and_empty_stdout(
self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE,
exit_code, str_std_out_files)
ExpectedSuiteReporting.check_list(
self, [ExpectedSuiteReporting(root, [(test_case, result.status)])],
reporter.complete_suite_reporter)
def process(
self,
reporting_environment: Environment,
settings: TestCaseExecutionSettings,
) -> int:
result_reporter = self._get_reporter(reporting_environment,
settings.reporting_option)
try:
processor = self._processor(settings, result_reporter)
except SuiteParseError as ex:
reporter = result_reporting.TestSuiteParseErrorReporter(
reporting_environment)
return reporter.report(ex)
test_case_path = test_case_processing.test_case_reference_of_source_file(
settings.test_case_file_path)
result = processor.apply(test_case_path)
return result_reporter.report(result)
def test_inclusion_of_non_exiting_file_SHOULD_cause_file_access_error(self):
# ARRANGE #
name_of_non_existing_file = 'non-existing.src'
configuration = configuration_with_no_instructions_and_no_preprocessor()
proc_cases = [
NameAndValue('not allowed to pollute current process',
sut.new_processor_that_should_not_pollute_current_process(configuration)),
NameAndValue('allowed to pollute current process',
sut.new_processor_that_is_allowed_to_pollute_current_process(configuration)),
]
for phase in phase_identifier.ALL_WITH_INSTRUCTIONS:
for proc_case in proc_cases:
with self.subTest(phase.section_name,
proc=proc_case.name):
test_case_file = fs.file_with_lines('test.case', [
section_header(phase.section_name),
directive_for_inclusion_of_file(name_of_non_existing_file),
])
cwd_contents = fs.DirContents([test_case_file])
processor = proc_case.value
assert isinstance(processor, Processor)
with tmp_dir_as_cwd(cwd_contents):
test_case_reference = test_case_reference_of_source_file(pathlib.Path(test_case_file.name))
# ACT #
result = processor.apply(test_case_reference)
# ASSERT #
assert isinstance(result, Result) # Type info for IDE
self.assertEqual(Status.ACCESS_ERROR,
result.status)
self.assertEqual(AccessErrorType.FILE_ACCESS_ERROR,
result.access_error_type)
source_location_path_expectation = equals_source_location_path(
source_location_path_of(pathlib.Path(test_case_file.name),
Line(2,
directive_for_inclusion_of_file(name_of_non_existing_file))))
source_location_path_expectation.apply_with_message(self,
result.error_info.source_location_path,
'source location path')
def test_internal_error_in_test_case_processor(self):
# ARRANGE #
str_std_out_files = StringStdOutFiles()
test_case = test_case_reference_of_source_file(Path('test-case'))
root = test_suite('root', [], [test_case])
suite_hierarchy_reader = ReaderThatGivesConstantSuite(root)
reporter = ExecutionTracingProcessingReporter()
processor = Processor(DUMMY_CASE_PROCESSING,
suite_hierarchy_reader,
reporter,
DepthFirstEnumerator(),
lambda config: TestCaseProcessorThatRaisesUnconditionally())
# ACT #
exit_code = processor.process(pathlib.Path('root-suite-file'), str_std_out_files.stdout_files)
# ASSERT #
check_exit_code_and_empty_stdout(self,
ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE,
exit_code,
str_std_out_files)
ExpectedSuiteReporting.check_list(
self,
[ExpectedSuiteReporting(root, [(test_case, tcp.Status.INTERNAL_ERROR)])],
reporter.complete_suite_reporter)
def _test_case_file_ref(self) -> test_case_processing.TestCaseFileReference:
return test_case_processing.test_case_reference_of_source_file(
self.request.case_execution_settings.test_case_file_path)
def _check_failing_line(
self, configuration: sut.Configuration, phases: Sequence[Phase],
invalid_line_cases: Sequence[NameAndValue[SourceAndStatus]]):
proc_cases = [
NameAndValue(
'not allowed to pollute current process',
sut.new_processor_that_should_not_pollute_current_process(
configuration)),
NameAndValue(
'allowed to pollute current process',
sut.new_processor_that_is_allowed_to_pollute_current_process(
configuration)),
]
for phase in phases:
for invalid_line_case in invalid_line_cases:
source_and_status = invalid_line_case.value
assert isinstance(source_and_status,
SourceAndStatus) # Type info for IDE
file_with_error = fs.file_with_lines('file-with-error.src', [
source_and_status.failing_source_line,
])
erroneous_line = single_line_sequence(
1, source_and_status.failing_source_line)
test_case_file = fs.file_with_lines('test.case', [
section_header(phase.section_name),
directive_for_inclusion_of_file(file_with_error.name),
])
line_that_includes_erroneous_file = single_line_sequence(
2, directive_for_inclusion_of_file(file_with_error.name))
cwd_contents = fs.DirContents(
[test_case_file, file_with_error])
expected_source_location_path = SourceLocationPath(
location=SourceLocation(erroneous_line,
pathlib.Path(
file_with_error.name)),
file_inclusion_chain=[
SourceLocation(line_that_includes_erroneous_file,
pathlib.Path(test_case_file.name))
])
for proc_case in proc_cases:
with self.subTest(phase=phase.section_name,
proc=proc_case.name,
line=erroneous_line.first_line):
processor = proc_case.value
assert isinstance(processor, Processor)
with tmp_dir_as_cwd(cwd_contents):
test_case_reference = test_case_reference_of_source_file(
pathlib.Path(test_case_file.name))
# ACT #
result = processor.apply(test_case_reference)
# ASSERT #
assert isinstance(result,
Result) # Type info for IDE
source_and_status.expected_result_statuses.apply_with_message(
self, result, 'result statuses')
source_location_path_expectation = equals_source_location_path(
expected_source_location_path)
source_location_path_expectation.apply_with_message(
self, result.source_location_path,
'source location path')
def _test_case_path(self) -> test_case_processing.TestCaseFileReference:
return test_case_processing.test_case_reference_of_source_file(
self.execution_settings.test_case_file_path)
def test_complex_suite_structure_with_test_cases(self):
# ARRANGE #
reporter = ExecutionTracingProcessingReporter()
str_std_out_files = StringStdOutFiles()
tc_internal_error_11 = test_case_reference_of_source_file(
Path('internal error 11'))
tc_internal_error_21 = test_case_reference_of_source_file(
Path('internal error 21'))
tc_access_error_1 = test_case_reference_of_source_file(
Path('access error A'))
tc_access_error_12 = test_case_reference_of_source_file(
Path('access error 12'))
tc_executed_11 = test_case_reference_of_source_file(
Path('executed 11'))
tc_executed_12 = test_case_reference_of_source_file(
Path('executed 12'))
tc_executed_1 = test_case_reference_of_source_file(Path('executed 1'))
tc_executed_2 = test_case_reference_of_source_file(Path('executed 2'))
tc_executed_root = test_case_reference_of_source_file(
Path('executed root'))
test_case_processor = TestCaseProcessorThatGivesConstantPerCase({
id(tc_internal_error_11):
new_internal_error(error_info.of_message('message A')),
id(tc_internal_error_21):
new_internal_error(error_info.of_message('message B')),
id(tc_access_error_1):
new_access_error(tcp.AccessErrorType.SYNTAX_ERROR,
error_info.of_message('syntax error')),
id(tc_access_error_12):
new_access_error(tcp.AccessErrorType.FILE_ACCESS_ERROR,
error_info.of_message('file access error')),
id(tc_executed_11):
new_executed(FULL_RESULT_PASS),
id(tc_executed_12):
new_executed(FULL_RESULT_PASS),
id(tc_executed_1):
new_executed(FULL_RESULT_PASS),
id(tc_executed_2):
new_executed(FULL_RESULT_PASS),
id(tc_executed_root):
new_executed(FULL_RESULT_PASS),
})
sub11 = test_suite('11', [], [tc_internal_error_11, tc_executed_11])
sub12 = test_suite('12', [], [tc_executed_12, tc_access_error_12])
sub1 = test_suite('1', [sub11, sub12],
[tc_access_error_1, tc_executed_1])
sub21 = test_suite('21', [], [tc_internal_error_21])
sub2 = test_suite('2', [sub21], [tc_executed_2])
sub3 = test_suite('2', [], [])
root = test_suite('root', [sub1, sub2, sub3], [tc_executed_root])
expected_suites = [
ExpectedSuiteReporting(
sub11, [(tc_internal_error_11, tcp.Status.INTERNAL_ERROR),
(tc_executed_11, tcp.Status.EXECUTED)]),
ExpectedSuiteReporting(
sub12, [(tc_executed_12, tcp.Status.EXECUTED),
(tc_access_error_12, tcp.Status.ACCESS_ERROR)]),
ExpectedSuiteReporting(
sub1, [(tc_access_error_1, tcp.Status.ACCESS_ERROR),
(tc_executed_1, tcp.Status.EXECUTED)]),
ExpectedSuiteReporting(
sub21, [(tc_internal_error_21, tcp.Status.INTERNAL_ERROR)]),
ExpectedSuiteReporting(sub2,
[(tc_executed_2, tcp.Status.EXECUTED)]),
ExpectedSuiteReporting(sub3, []),
ExpectedSuiteReporting(root,
[(tc_executed_root, tcp.Status.EXECUTED)]),
]
suite_hierarchy_reader = ReaderThatGivesConstantSuite(root)
processor = Processor(DUMMY_CASE_PROCESSING, suite_hierarchy_reader,
reporter, DepthFirstEnumerator(),
lambda config: test_case_processor)
# ACT #
exit_code = processor.process(pathlib.Path('root-suite-file'),
str_std_out_files.reporting_environment)
# ASSERT #
check_exit_code_and_empty_stdout(
self, ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE,
exit_code, str_std_out_files)
ExpectedSuiteReporting.check_list(self, expected_suites,
reporter.complete_suite_reporter)
def test_complex_suite_structure_with_test_cases(self):
# ARRANGE #
reporter = ExecutionTracingProcessingReporter()
str_std_out_files = StringStdOutFiles()
tc_internal_error_11 = test_case_reference_of_source_file(Path('internal error 11'))
tc_internal_error_21 = test_case_reference_of_source_file(Path('internal error 21'))
tc_access_error_1 = test_case_reference_of_source_file(Path('access error A'))
tc_access_error_12 = test_case_reference_of_source_file(Path('access error 12'))
tc_executed_11 = test_case_reference_of_source_file(Path('executed 11'))
tc_executed_12 = test_case_reference_of_source_file(Path('executed 12'))
tc_executed_1 = test_case_reference_of_source_file(Path('executed 1'))
tc_executed_2 = test_case_reference_of_source_file(Path('executed 2'))
tc_executed_root = test_case_reference_of_source_file(Path('executed root'))
test_case_processor = TestCaseProcessorThatGivesConstantPerCase({
id(tc_internal_error_11): new_internal_error(error_info.of_message('message A')),
id(tc_internal_error_21): new_internal_error(error_info.of_message('message B')),
id(tc_access_error_1): new_access_error(
tcp.AccessErrorType.SYNTAX_ERROR, error_info.of_message('syntax error')),
id(tc_access_error_12): new_access_error(tcp.AccessErrorType.FILE_ACCESS_ERROR,
error_info.of_message('file access error')),
id(tc_executed_11): new_executed(FULL_RESULT_PASS),
id(tc_executed_12): new_executed(FULL_RESULT_PASS),
id(tc_executed_1): new_executed(FULL_RESULT_PASS),
id(tc_executed_2): new_executed(FULL_RESULT_PASS),
id(tc_executed_root): new_executed(FULL_RESULT_PASS),
})
sub11 = test_suite('11', [], [tc_internal_error_11,
tc_executed_11])
sub12 = test_suite('12', [], [tc_executed_12,
tc_access_error_12])
sub1 = test_suite('1', [sub11, sub12], [tc_access_error_1,
tc_executed_1])
sub21 = test_suite('21', [], [tc_internal_error_21])
sub2 = test_suite('2', [sub21], [tc_executed_2])
sub3 = test_suite('2', [], [])
root = test_suite('root', [sub1, sub2, sub3], [tc_executed_root])
expected_suites = [
ExpectedSuiteReporting(sub11, [(tc_internal_error_11, tcp.Status.INTERNAL_ERROR),
(tc_executed_11, tcp.Status.EXECUTED)]),
ExpectedSuiteReporting(sub12, [(tc_executed_12, tcp.Status.EXECUTED),
(tc_access_error_12, tcp.Status.ACCESS_ERROR)]),
ExpectedSuiteReporting(sub1, [(tc_access_error_1, tcp.Status.ACCESS_ERROR),
(tc_executed_1, tcp.Status.EXECUTED)]),
ExpectedSuiteReporting(sub21, [(tc_internal_error_21, tcp.Status.INTERNAL_ERROR)]),
ExpectedSuiteReporting(sub2, [(tc_executed_2, tcp.Status.EXECUTED)]),
ExpectedSuiteReporting(sub3, []),
ExpectedSuiteReporting(root, [(tc_executed_root, tcp.Status.EXECUTED)]),
]
suite_hierarchy_reader = ReaderThatGivesConstantSuite(root)
processor = Processor(DUMMY_CASE_PROCESSING,
suite_hierarchy_reader,
reporter,
DepthFirstEnumerator(),
lambda config: test_case_processor)
# ACT #
exit_code = processor.process(pathlib.Path('root-suite-file'), str_std_out_files.stdout_files)
# ASSERT #
check_exit_code_and_empty_stdout(self,
ExecutionTracingRootSuiteReporter.VALID_SUITE_EXIT_CODE,
exit_code,
str_std_out_files)
ExpectedSuiteReporting.check_list(
self,
expected_suites,
reporter.complete_suite_reporter)