def test_scanner_with_dash_dash_report_dir_with_non_directory():
"""
Test to make sure we get help if '--report-dir' is supplied with a file instead of a directory.
"""
# Arrange
scanner = MainlineExecutor()
temporary_work_directory, _, _ = setup_directories()
assert os.path.exists("README.md") and os.path.isfile("README.md")
readme_path = os.path.abspath("README.md")
supplied_arguments = ["--report-dir", readme_path]
expected_return_code = 2
expected_output = ""
expected_error = """usage: main.py [-h] [--version] [--add-plugin ADD_PLUGIN]
[--report-dir REPORT_DIR] [--publish-dir PUBLISH_DIR]
[--cobertura path] [--junit path] [--only-changes] [--publish]
[--quiet] [--columns DISPLAY_COLUMNS]
main.py: error: argument --report-dir: Path '{path}' is not an existing directory.
""".replace("{path}", readme_path)
# Act
execute_results = scanner.invoke_main(arguments=supplied_arguments,
cwd=temporary_work_directory.name)
# Assert
execute_results.assert_results(expected_output, expected_error,
expected_return_code)
def test_publish_with_coverage_file_as_directory():
"""
Test to make sure that publishing with a test coverage summary file that is not a file fails.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, report_directory, _ = setup_directories()
cobertura_coverage_file = os.path.join(
temporary_work_directory.name, get_coverage_file_name()
)
copyfile(
os.path.join(executor.resource_directory, get_coverage_file_name()),
cobertura_coverage_file,
)
summary_coverage_file = os.path.join(report_directory, COVERAGE_SUMMARY_FILE_NAME)
os.makedirs(summary_coverage_file)
suppplied_arguments = [PUBLISH_COMMAND_LINE_FLAG]
coverage_path = os.path.join(REPORT_DIRECTORY, COVERAGE_SUMMARY_FILE_NAME)
expected_output = ""
expected_error = f"Summary path '{coverage_path}' is not a file."
expected_return_code = 1
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
def test_scanner_with_dash_dash_columns_bad_number():
"""
Test to make sure we get help if '--columns' is supplied with a bad number.
"""
# Arrange
scanner = MainlineExecutor()
temporary_work_directory, _, _ = setup_directories()
supplied_arguments = ["--columns", "20"]
expected_return_code = 2
expected_output = ""
expected_error = """usage: main.py [-h] [--version] [--add-plugin ADD_PLUGIN]
[--report-dir REPORT_DIR] [--publish-dir PUBLISH_DIR]
[--cobertura path] [--junit path] [--only-changes] [--publish]
[--quiet] [--columns DISPLAY_COLUMNS]
main.py: error: argument --columns: Value '20' is not an integer between between 50 and 200."""
# Act
execute_results = scanner.invoke_main(arguments=supplied_arguments,
cwd=temporary_work_directory.name)
# Assert
execute_results.assert_results(expected_output, expected_error,
expected_return_code)
def test_summarize_bad_report_directory():
"""
Test the summarizing of cobertura results with a bad report directory.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, _, _ = setup_directories(create_report_directory=False)
cobertura_coverage_file = os.path.join(
temporary_work_directory.name, get_coverage_file_name()
)
copyfile(
os.path.join(executor.resource_directory, get_coverage_file_name()),
cobertura_coverage_file,
)
suppplied_arguments = [COBERTURA_COMMAND_LINE_FLAG, cobertura_coverage_file]
expected_output = ""
expected_error = """usage: main.py [-h] [--version] [--add-plugin ADD_PLUGIN]
[--report-dir REPORT_DIR] [--publish-dir PUBLISH_DIR]
[--cobertura path] [--junit path] [--only-changes] [--publish]
[--quiet] [--columns DISPLAY_COLUMNS]
main.py: error: argument --report-dir: Path 'report' does not exist."""
expected_return_code = 2
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
def test_publish_with_existing_publish_as_file():
"""
Test to make sure that publishing with a publish directory that exists as a file.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, _, publish_directory = setup_directories()
with open(
publish_directory, "w", encoding=ProjectSummarizerPlugin.DEFAULT_FILE_ENCODING
) as outfile:
outfile.write("test")
suppplied_arguments = [PUBLISH_COMMAND_LINE_FLAG]
expected_output = (
f"Publish directory '{PUBLISH_DIRECTORY}' already exists, but as a file."
)
expected_error = ""
expected_return_code = 1
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
def test_summarize_bad_test_results():
"""
Test the summarizing of junit results with a bad test-results file.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, _, _ = setup_directories()
junit_test_file = os.path.join(
temporary_work_directory.name, JUNIT_RESULTS_FILE_NAME
)
copyfile(
os.path.join(executor.resource_directory, "tests-bad.txt"), junit_test_file
)
suppplied_arguments = [JUNIT_COMMAND_LINE_FLAG, junit_test_file]
expected_output = f"Project test report file '{junit_test_file}' is not a valid test report file.\n"
expected_error = ""
expected_return_code = 1
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
def test_summarize_bad_test_coverage():
"""
Test the summarizing of cobertura results with a bad coverage file.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, _, _ = setup_directories()
cobertura_coverage_file = os.path.join(
temporary_work_directory.name, get_coverage_file_name()
)
copyfile(
os.path.join(executor.resource_directory, "coverage-bad.txt"),
cobertura_coverage_file,
)
suppplied_arguments = [COBERTURA_COMMAND_LINE_FLAG, cobertura_coverage_file]
expected_output = f"Project test coverage file '{cobertura_coverage_file}' is not a valid test coverage file.\n"
expected_error = ""
expected_return_code = 1
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
def test_summarize_cobertura_report_with_source_as_directory():
"""
Test to make sure that summarizing a test coverage file that is not a file.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, _, _ = setup_directories()
cobertura_coverage_file = os.path.join(
temporary_work_directory.name, get_coverage_file_name()
)
os.makedirs(cobertura_coverage_file)
suppplied_arguments = [COBERTURA_COMMAND_LINE_FLAG, cobertura_coverage_file]
expected_output = (
f"Project test coverage file '{cobertura_coverage_file}' is not a file.\n"
)
expected_error = ""
expected_return_code = 1
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
def test_scanner_with_dash_dash_report_dir_with_non_existant():
"""
Test to make sure we get help if '--report-dir' is supplied with a non-existant directory.
"""
# Arrange
scanner = MainlineExecutor()
temporary_work_directory, _, _ = setup_directories()
supplied_arguments = ["--report-dir", "alternate-reports"]
assert not os.path.exists("alternate-reports")
expected_return_code = 2
expected_output = ""
expected_error = """usage: main.py [-h] [--version] [--add-plugin ADD_PLUGIN]
[--report-dir REPORT_DIR] [--publish-dir PUBLISH_DIR]
[--cobertura path] [--junit path] [--only-changes] [--publish]
[--quiet] [--columns DISPLAY_COLUMNS]
main.py: error: argument --report-dir: Path 'alternate-reports' does not exist.
"""
# Act
execute_results = scanner.invoke_main(arguments=supplied_arguments,
cwd=temporary_work_directory.name)
# Assert
execute_results.assert_results(expected_output, expected_error,
expected_return_code)
def test_summarize_junit_report_with_source_as_directory():
"""
Test to make sure that summarizing a test report file that is not a file.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, _, _ = setup_directories()
junit_test_file = os.path.join(
temporary_work_directory.name, JUNIT_RESULTS_FILE_NAME
)
os.makedirs(junit_test_file)
suppplied_arguments = [JUNIT_COMMAND_LINE_FLAG, junit_test_file]
expected_output = f"Project test report file '{junit_test_file}' is not a file.\n"
expected_error = ""
expected_return_code = 1
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
def test_summarize_simple_junit_report(
create_publish_directory=False,
temporary_work_directory=None,
alternate_publish_directory=None,
):
"""
Test the summarizing of a simple junit report with no previous summary.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, report_directory, publish_directory = setup_directories(
create_publish_directory=create_publish_directory,
alternate_publish_directory=alternate_publish_directory,
temporary_work_directory=temporary_work_directory,
)
junit_test_file = os.path.join(
temporary_work_directory.name, JUNIT_RESULTS_FILE_NAME
)
copyfile(
os.path.join(executor.resource_directory, JUNIT_RESULTS_FILE_NAME),
junit_test_file,
)
summary_result_file = os.path.join(report_directory, RESULTS_SUMMARY_FILE_NAME)
suppplied_arguments = [JUNIT_COMMAND_LINE_FLAG, junit_test_file]
expected_output = """\
Test Results Summary
--------------------
CLASS NAME TOTAL TESTS FAILED TESTS SKIPPED TESTS
test.test_scenarios 3 (+3) 0 0
--- - - -
TOTALS 3 (+3) 0 0
"""
expected_error = ""
expected_return_code = 0
expected_test_results_file = compose_test_results(3)
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
execute_results.assert_resultant_file(
summary_result_file, expected_test_results_file
)
return executor, temporary_work_directory, publish_directory, junit_test_file
def test_summarize_simple_cobertura_report_with_quiet(
create_publish_directory=False, temporary_work_directory=None
):
"""
Test the summarizing of a simple cobertura report with no previous summary.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, report_directory, publish_directory = setup_directories(
create_publish_directory=create_publish_directory,
temporary_work_directory=temporary_work_directory,
)
cobertura_coverage_file = os.path.join(
temporary_work_directory.name, get_coverage_file_name()
)
copyfile(
os.path.join(executor.resource_directory, get_coverage_file_name()),
cobertura_coverage_file,
)
summary_coverage_file = os.path.join(report_directory, COVERAGE_SUMMARY_FILE_NAME)
suppplied_arguments = [
COBERTURA_COMMAND_LINE_FLAG,
cobertura_coverage_file,
"--quiet",
]
expected_output = ""
expected_error = ""
expected_return_code = 0
expected_test_coverage_file = compose_coverage_summary_file()
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
execute_results.assert_resultant_file(
summary_coverage_file, expected_test_coverage_file
)
return (
executor,
temporary_work_directory,
publish_directory,
cobertura_coverage_file,
)
def test_scanner_with_no_parameters_through_module():
"""
Test to make sure we get the simple information if no parameters are supplied,
but through the module interface.
"""
# Arrange
scanner = MainlineExecutor(use_module=True)
temporary_work_directory, _, _ = setup_directories()
supplied_arguments = []
expected_return_code = 2
expected_error = ""
expected_output = """Error: Either --publish or one of the reporting arguments mush be specified.
usage: __main.py__ [-h] [--version] [--add-plugin ADD_PLUGIN]
[--report-dir REPORT_DIR] [--publish-dir PUBLISH_DIR]
[--cobertura path] [--junit path] [--only-changes]
[--publish] [--quiet] [--columns DISPLAY_COLUMNS]
Summarize Python files.
optional arguments:
-h, --help Show this help message and exit.
--version Show program's version number and exit.
--add-plugin ADD_PLUGIN
Add a plugin file to provide additional project
summaries.
--report-dir REPORT_DIR
Directory to generate the summary reports in.
--publish-dir PUBLISH_DIR
Directory to publish the summary reports to.
--cobertura path Source file name for cobertura test coverage
reporting.
--junit path Source file name for junit test result reporting.
--only-changes Only the summary items that have changed are displayed
in the console summary.
--publish Publish the summaries to the publish directory and
exit.
--quiet The report summary files will be generated, but no
summary will be output to the console.
--columns DISPLAY_COLUMNS
Specifies the number of character columns to use in
the console summary."""
# Act
execute_results = scanner.invoke_main(arguments=supplied_arguments,
cwd=temporary_work_directory.name)
# Assert
execute_results.assert_results(expected_output, expected_error,
expected_return_code)
def test_summarize_invalid_published_summary_file():
"""
Test the summarizing of junit results with a bad report directory.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, _, publish_directory = setup_directories(
create_publish_directory=True
)
junit_test_file = os.path.join(
temporary_work_directory.name, JUNIT_RESULTS_FILE_NAME
)
copyfile(
os.path.join(executor.resource_directory, JUNIT_RESULTS_FILE_NAME),
junit_test_file,
)
summary_result_file = os.path.join(publish_directory, RESULTS_SUMMARY_FILE_NAME)
with open(
summary_result_file, "w", encoding=ProjectSummarizerPlugin.DEFAULT_FILE_ENCODING
) as outfile:
outfile.write("this is not a json file")
suppplied_arguments = [JUNIT_COMMAND_LINE_FLAG, junit_test_file]
file_name = os.path.join(PUBLISH_DIRECTORY, RESULTS_SUMMARY_FILE_NAME)
expected_output = (
f"Previous results summary file '{file_name}' is "
+ "not a valid JSON file (Expecting value: line 1 column 1 (char 0))."
)
expected_error = ""
expected_return_code = 1
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
def test_summarize_simple_cobertura_report_with_error_on_report_write():
"""
Test a summarize with an error when trying to write the summary report.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, report_directory, _ = setup_directories()
cobertura_coverage_file = os.path.join(
temporary_work_directory.name, get_coverage_file_name()
)
copyfile(
os.path.join(executor.resource_directory, get_coverage_file_name()),
cobertura_coverage_file,
)
summary_coverage_file = os.path.join(report_directory, COVERAGE_SUMMARY_FILE_NAME)
suppplied_arguments = [COBERTURA_COMMAND_LINE_FLAG, cobertura_coverage_file]
expected_output = (
f"Project test coverage summary file '{os.path.abspath(summary_coverage_file)}' "
+ "was not written (None).\n"
)
expected_error = ""
expected_return_code = 1
# Act
try:
pbo = PatchBuiltinOpen()
pbo.register_exception(os.path.abspath(summary_coverage_file), "w")
pbo.start()
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
finally:
pbo.stop()
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
def test_summarize_simple_junit_report_with_quiet(
create_publish_directory=False, temporary_work_directory=None
):
"""
Test the summarizing of a simple junit report with no previous summary with columns set.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, report_directory, publish_directory = setup_directories(
create_publish_directory=create_publish_directory,
temporary_work_directory=temporary_work_directory,
)
junit_test_file = os.path.join(
temporary_work_directory.name, JUNIT_RESULTS_FILE_NAME
)
copyfile(
os.path.join(executor.resource_directory, JUNIT_RESULTS_FILE_NAME),
junit_test_file,
)
summary_result_file = os.path.join(report_directory, RESULTS_SUMMARY_FILE_NAME)
suppplied_arguments = [JUNIT_COMMAND_LINE_FLAG, junit_test_file, "--quiet"]
expected_output = ""
expected_error = ""
expected_return_code = 0
expected_test_results_file = compose_test_results(3)
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
execute_results.assert_resultant_file(
summary_result_file, expected_test_results_file
)
return executor, temporary_work_directory, publish_directory, junit_test_file
def test_summarize_invalid_published_summary_file():
"""
Test the summarizing of cobertura results with a bad report directory.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, _, publish_directory = setup_directories(
create_publish_directory=True
)
cobertura_coverage_file = os.path.join(
temporary_work_directory.name, get_coverage_file_name()
)
copyfile(
os.path.join(executor.resource_directory, get_coverage_file_name()),
cobertura_coverage_file,
)
summary_coverage_file = os.path.join(publish_directory, COVERAGE_SUMMARY_FILE_NAME)
with open(summary_coverage_file, "w", encoding="utf-8") as outfile:
outfile.write("this is not a json file")
suppplied_arguments = [COBERTURA_COMMAND_LINE_FLAG, cobertura_coverage_file]
file_name = os.path.join(PUBLISH_DIRECTORY, COVERAGE_SUMMARY_FILE_NAME)
expected_output = (
f"Previous coverage summary file '{file_name}' is not "
+ "a valid JSON file (Expecting value: line 1 column 1 (char 0))."
)
expected_error = ""
expected_return_code = 1
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
def test_sample_1():
"""
Test the summarizing of junit results against a previous published version.
This was encountered during development, and the test case captured.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, _, publish_directory = setup_directories()
junit_test_file = os.path.join(
temporary_work_directory.name, JUNIT_RESULTS_FILE_NAME
)
summary_result_file = os.path.join(publish_directory, RESULTS_SUMMARY_FILE_NAME)
copyfile(
os.path.join(executor.resource_directory, "tests-sample-1.xml"),
junit_test_file,
)
os.makedirs(publish_directory)
previous_test_summary_contents = (
'{"projectName": "?", "reportSource": "pytest", '
+ '"measurements": ['
+ '{"name": "test.test_coverage_scenarios", "totalTests": 14, "failedTests": 0, '
+ '"errorTests": 0, "skippedTests": 0, "elapsedTimeInMilliseconds": 0}, '
+ '{"name": "test.test_scenarios", "totalTests": 23, "failedTests": 0, '
+ '"errorTests": 0, "skippedTests": 0, "elapsedTimeInMilliseconds": 0}'
+ "]}"
)
with open(
summary_result_file, "w", encoding=ProjectSummarizerPlugin.DEFAULT_FILE_ENCODING
) as outfile:
outfile.write(previous_test_summary_contents)
suppplied_arguments = [JUNIT_COMMAND_LINE_FLAG, junit_test_file]
expected_output = """
Test Results Summary
--------------------
CLASS NAME TOTAL TESTS FAILED TESTS SKIPPED TESTS
test.test_coverage_scenarios 12 (-2) 0 0
test.test_publish_scenarios 9 (+9) 0 0
test.test_results_scenarios 18 (+18) 0 0
test.test_scenarios 1 (-22) 0 0
--- -- - -
TOTALS 40 (+3) 0 0
"""
expected_error = ""
expected_return_code = 0
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
def test_summarize_simple_cobertura_report(
create_publish_directory=False, temporary_work_directory=None
):
"""
Test the summarizing of a simple cobertura report with no previous summary.
"""
# Arrange
executor = MainlineExecutor()
temporary_work_directory, report_directory, publish_directory = setup_directories(
create_publish_directory=create_publish_directory,
temporary_work_directory=temporary_work_directory,
)
cobertura_coverage_file = os.path.join(
temporary_work_directory.name, get_coverage_file_name()
)
copyfile(
os.path.join(executor.resource_directory, get_coverage_file_name()),
cobertura_coverage_file,
)
summary_coverage_file = os.path.join(report_directory, COVERAGE_SUMMARY_FILE_NAME)
suppplied_arguments = [COBERTURA_COMMAND_LINE_FLAG, cobertura_coverage_file]
expected_output = """
Test Coverage Summary
---------------------
TYPE COVERED MEASURED PERCENTAGE
Instructions -- -- -----
Lines 10 (+10) 15 (+15) 66.67 (+66.67)
Branches 2 ( +2) 4 ( +4) 50.00 (+50.00)
Complexity -- -- -----
Methods -- -- -----
Classes -- -- -----
"""
expected_error = ""
expected_return_code = 0
expected_test_coverage_file = compose_coverage_summary_file()
# Act
execute_results = executor.invoke_main(
arguments=suppplied_arguments, cwd=temporary_work_directory.name
)
# Assert
execute_results.assert_results(
expected_output, expected_error, expected_return_code
)
execute_results.assert_resultant_file(
summary_coverage_file, expected_test_coverage_file
)
return (
executor,
temporary_work_directory,
publish_directory,
cobertura_coverage_file,
)
