def export_json(self,
report: Report,
dest_directory: str,
dest_filename: str = "report.json",
exclude_image_data: bool = False) -> str:
if report.num_failures > 0:
report.outcome = Outcome.failure
exclude = None
if exclude_image_data:
# This is a complex pydantic filter meaning:
# for each result in report,
# for each png in result,
# ignore the 'base64' field.
logger.info("Stripping base64 image data from report.json")
exclude = {
"results": {
"__all__": {
"pngs": {
"__all__": {"base64"}
}
}
}
}
filename = os.path.join(dest_directory, dest_filename)
with open(filename, "w") as f:
f.write(report.json(indent=4, exclude=exclude))
logger.info(f"Report JSON saved to {filename}")
return filename
def test_browser_error_failure_reporting(run_pytest, local_html_path,
report_subdir, make_test_file):
"""
Assert that the right cleanup behavior takes place when a BrowserError
is bubbled from within a test.
"""
make_test_file(content=f"""
from webdriver_recorder.browser import BrowserError
import pytest
def test_force_failure(browser):
browser.get("file://{local_html_path}")
browser.wait_for_tag('a', 'does-not-exist', timeout=0)
""")
result = run_pytest()
try:
result.assert_outcomes(failed=1)
except ValueError:
logging.error(f"Could not parse output from pytest.")
if result.outlines:
lines = "\n".join(result.outlines)
logging.error(f"No summary found in:\n {lines}")
else:
logging.error("No STDOUT from test execution")
if result.errlines:
lines = "\n".join(result.errlines)
logging.error(f"Execution resulted in errors\n:{lines}")
raise
expected_filename = os.path.join(report_subdir, f"report.json")
report = Report.parse_file(expected_filename)
assert report.outcome == Outcome.failure
assert "BrowserError" in report.results[0].traceback
def test_call_exception(run_pytest, report_subdir, make_test_file):
make_test_file(f"""
def test_a_thing(browser):
raise RuntimeError("????")
""")
run_pytest()
report = Report.parse_file(os.path.join(report_subdir, "report.json"))
assert "RuntimeError" in report.results[0].traceback
def test_docstring_capture(run_pytest, report_subdir, make_test_file):
make_test_file(f"""
def test_a_thing(browser):
'''hello'''
pass
""")
run_pytest()
report_json = os.path.join(report_subdir, "report.json")
report = Report.parse_file(report_json)
assert report.results[0].test_description == "hello"
def test_aggregate_worker_reports(run_pytest, report_subdir, make_test_file):
make_test_file(f"""
def test_a_thing(browser):
pass
""")
run_pytest()
report_json = os.path.join(report_subdir, "report.json")
test_1_rename = os.path.join(report_subdir, "t1.result.json")
assert os.path.exists(report_json)
shutil.move(report_json, test_1_rename)
assert not os.path.exists(report_json)
assert os.path.exists(test_1_rename)
run_pytest()
assert os.path.exists(report_json)
assert not os.path.exists(test_1_rename)
report = Report.parse_file(report_json)
assert len(report.results) == 2
def test_no_outcomes(run_pytest, report_subdir, make_test_file):
make_test_file(f"""
import pytest
@pytest.fixture
def bad_fixture():
raise RuntimeError
def test_a_thing(bad_fixture, browser):
pass
""")
run_pytest()
logging.error("If you just saw an error message, you can ignore it. "
"It was on purpose, and part of a test.")
report_json = os.path.join(report_subdir, "report.json")
report = Report.parse_file(report_json)
assert report.results[0].outcome.value == "never started"
assert report.outcome == Outcome.failure
assert report.results[0].test_name == "test_a_thing"
def test_failure_reporting(run_pytest, local_html_path, report_subdir,
make_test_file):
"""
Similar to test_browser_error_failure_reporting, but with a generic AssertionError. This is what we would expect
to see under most failure circumstances.
"""
make_test_file(f"""
from webdriver_recorder.browser import BrowserError
import pytest
def test_force_failure(session_browser, report_test):
session_browser.get("file://{local_html_path}")
assert False
""")
result = run_pytest()
result.assert_outcomes(failed=1)
expected_filename = os.path.join(report_subdir, "report.json")
report = Report.parse_file(expected_filename)
assert report.outcome == Outcome.failure
assert "AssertionError" in report.results[0].traceback
help="The path to the report.json you want to load.")
parser.add_argument(
"--output-dir",
"-o",
required=False,
default=None,
help=
"The directory you want the report output to go. If not provided, it will overwrite "
"the input report artifacts.",
)
parser.add_argument(
"--template-filename",
"-t",
required=False,
default=os.path.join(here, "templates", "report.html"),
help="The name of a template to use when generating the report.",
)
return parser
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
args = get_parser().parse_args()
output_dir = args.output_dir or os.path.dirname(args.input_json)
exporter = ReportExporter(
template_dir=os.path.dirname(args.template_filename),
root_template=os.path.basename(args.template_filename))
report = Report.parse_file(args.input_json)
exporter.export_all(report, output_dir)
print(f"Exported artifacts to {output_dir}")