def process_test_data(self, test_data):
"""
Convert JUnit output into a a list of report entries.
:param test_data: JUnit test output.
:type test_data: ``list``
:return: list of sub reports.
:rtype: ``list`` of (``TestGroupReport`` or ``TestCaseReport``)
"""
result = []
for suite in test_data:
suite_name = suite.get("name", "JUnit testsuite")
suite_report = TestGroupReport(
name=suite_name,
uid=suite_name,
category=ReportCategories.TESTSUITE,
)
for case in suite.xpath("testcase"):
case_name = case.get("name", "JUnit testcase")
testcase_report = TestCaseReport(name=case_name, uid=case_name)
for error in case.xpath("error"):
testcase_report.append(
registry.serialize(
assertions.Fail("Error executing test")
)
)
testcase_report.append(
registry.serialize(
CodeLog(
error.text,
language="java",
description="stacktrace",
)
)
)
for failure in case.xpath("failure"):
message = failure.get("message", "testcase failure")
testcase_report.append(
registry.serialize(assertions.Fail(message))
)
if failure.text:
testcase_report.append(
registry.serialize(
CodeLog(
failure.text,
language="java",
description="stacktrace",
)
)
)
if not testcase_report.entries:
assertion_obj = assertions.RawAssertion(
description="Passed",
content="Testcase {} passed".format(case_name),
passed=True,
)
testcase_report.append(registry.serialize(assertion_obj))
testcase_report.runtime_status = RuntimeStatus.FINISHED
suite_report.append(testcase_report)
def process_test_data(self, test_data):
"""
XML output contains entries for skipped testcases
as well, which are not included in the report.
"""
result = []
for suite in test_data.getchildren():
suite_name = suite.attrib["name"]
suite_report = TestGroupReport(
name=suite_name,
uid=suite_name,
category=ReportCategories.TESTSUITE,
)
for testcase in suite.getchildren():
if testcase.tag != "testcase":
continue
testcase_classname = testcase.attrib["classname"]
testcase_name = testcase.attrib["name"]
testcase_prefix = testcase_classname.split(".")[-1]
testcase_report = TestCaseReport(
name="{}::{}".format(testcase_prefix, testcase_name),
uid="{}::{}".format(testcase_classname.replace(".", "::"),
testcase_name),
)
if not testcase.getchildren():
assertion_obj = RawAssertion(
description="Passed",
content="Testcase {} passed".format(testcase_name),
passed=True,
)
testcase_report.append(registry.serialize(assertion_obj))
else:
for entry in testcase.getchildren():
assertion_obj = RawAssertion(
description=entry.tag,
content=entry.text,
passed=entry.tag not in ("failure", "error"),
)
testcase_report.append(
registry.serialize(assertion_obj))
testcase_report.runtime_status = RuntimeStatus.FINISHED
suite_report.append(testcase_report)
if len(suite_report) > 0:
result.append(suite_report)
return result
def process_test_data(self, test_data):
"""
XML output contains entries for skipped testcases
as well, which are not included in the report.
"""
result = []
for suite in test_data.getchildren():
suite_name = suite.attrib["name"]
suite_report = TestGroupReport(
name=suite_name,
uid=suite_name,
category=ReportCategories.TESTSUITE,
)
suite_has_run = False
for testcase in suite.getchildren():
testcase_name = testcase.attrib["name"]
testcase_report = TestCaseReport(
name=testcase_name, uid=testcase_name
)
if not testcase.getchildren():
assertion_obj = RawAssertion(
description="Passed",
content="Testcase {} passed".format(testcase_name),
passed=True,
)
testcase_report.append(registry.serialize(assertion_obj))
else:
for entry in testcase.getchildren():
assertion_obj = RawAssertion(
description=entry.tag,
content=entry.text,
passed=entry.tag != "failure",
)
testcase_report.append(
registry.serialize(assertion_obj)
)
testcase_report.runtime_status = RuntimeStatus.FINISHED
if testcase.attrib["status"] != "notrun":
suite_report.append(testcase_report)
suite_has_run = True
if suite_has_run:
result.append(suite_report)
return result
def _process_data(self, data: Element) -> List[TestGroupReport]:
"""
Processes data read from the source file.
:param data: raw data as read by the importer
"""
# NOTE: XML output contains skipped testcases which are ignored.
result = []
for suite in data.getchildren():
suite_name = suite.attrib["name"]
suite_report = TestGroupReport(
name=suite_name,
category=ReportCategories.TESTSUITE,
)
for testcase in suite.getchildren():
if testcase.tag != "testcase":
continue
testcase_classname = testcase.attrib["classname"]
testcase_name = testcase.attrib["name"]
testcase_prefix = testcase_classname.split(".")[-1]
testcase_report = TestCaseReport(name="{}::{}".format(
testcase_prefix, testcase_name), )
if not testcase.getchildren():
assertion_obj = RawAssertion(
description="Passed",
content=f"Testcase {testcase_name} passed",
passed=True,
)
testcase_report.append(registry.serialize(assertion_obj))
else:
for entry in testcase.getchildren():
assertion_obj = RawAssertion(
description=entry.tag,
content=entry.text,
passed=entry.tag not in ("failure", "error"),
)
testcase_report.append(
registry.serialize(assertion_obj))
testcase_report.runtime_status = RuntimeStatus.FINISHED
suite_report.append(testcase_report)
if len(suite_report) > 0:
result.append(suite_report)
return result
def test_create_pdf(tmpdir):
"""PDF exporter should generate a PDF file using the report data."""
pdf_path = tmpdir.mkdir("reports").join("dummy_report.pdf").strpath
assertion_entries = [
assertions.Equal(1, 2),
assertions.Greater(2, 1),
assertions.IsFalse(True, "this should fail"),
assertions.IsTrue(True, "this should pass"),
assertions.Fail("Explicit failure"),
base.Group(
description="group description",
entries=[
assertions.NotEqual(2, 1),
assertions.Contain(1, [1, 2, 3]),
],
),
]
# Test large assertion
for _ in range(10):
assertion_entries.append(
assertions.RegexMatch("Test.*", "Testplan\n" * 500))
report = TestReport(
name="my testplan",
entries=[
TestGroupReport(
name="My Multitest",
category=ReportCategories.MULTITEST,
entries=[
TestGroupReport(
name="MySuite",
category=ReportCategories.TESTSUITE,
entries=[
TestCaseReport(
name="my_test_method",
entries=[
registry.serialize(obj)
for obj in assertion_entries
],
)
],
)
],
)
],
)
exporter = PDFExporter(
pdf_path=pdf_path,
pdf_style=styles.Style(passing="assertion-detail",
failing="assertion-detail"),
)
with log_propagation_disabled(TESTPLAN_LOGGER):
exporter.export(report)
assert os.path.exists(pdf_path)
assert os.stat(pdf_path).st_size > 0
def pytest_runtest_logreport(self, report):
"""
Hook called by pytest to report on the result of a test.
:param report: the test report for the item just tested (see pytest
documentation)
"""
if report.when == 'setup':
if report.skipped and self._current_case_report is not None:
# Status set to be SKIPPED if testcase is marked skip or xfail
# lower versioned PyTest does not support this feature
self._current_case_report.status_override = Status.SKIPPED
elif report.when == 'call':
if self._current_case_report is None:
raise RuntimeError(
'Cannot store testcase results to report: no report '
'object was created.')
# Add the assertion entry to the case report
for entry in self._current_result_obj.entries:
stdout_renderer = stdout_registry[entry]()
stdout_header = stdout_renderer.get_header(entry)
stdout_details = stdout_renderer.get_details(entry) or ''
# Add 'stdout_header' and 'stdout_details' attributes to
# serialized entries for standard output later
serialized_entry = schema_registry.serialize(entry)
serialized_entry.update(stdout_header=stdout_header,
stdout_details=stdout_details)
self._current_case_report.append(serialized_entry)
def process_test_data(self, test_data):
"""
XML output contains entries for skipped testcases
as well, which are not included in the report.
"""
result = []
for suite in test_data.getchildren():
suite_report = TestGroupReport(
name=suite.attrib['name'],
category='suite',
)
suite_has_run = False
for testcase in suite.getchildren():
testcase_report = TestCaseReport(name=testcase.attrib['name'])
for entry in testcase.getchildren():
assertion_obj = RawAssertion(description=entry.tag,
content=entry.text,
passed=entry.tag != 'failure')
testcase_report.append(registry.serialize(assertion_obj))
if testcase.attrib['status'] != 'notrun':
suite_report.append(testcase_report)
suite_has_run = True
if suite_has_run:
result.append(suite_report)
return result
def process_test_data(self, test_data):
"""
JSON output contains entries for skipped testcases
as well, which are not included in the report.
"""
result = []
for suite in test_data:
suite_report = TestGroupReport(
name=suite['name'],
category='suite',
)
suite_has_run = False
for testcase in suite['data']:
if testcase['status'] != 'skipped':
suite_has_run = True
testcase_report = TestCaseReport(name=testcase['name'])
assertion_obj = RawAssertion(
passed=testcase['status'] == 'pass',
content=testcase['error'] or testcase['duration'],
description=testcase['name'])
testcase_report.append(registry.serialize(assertion_obj))
suite_report.append(testcase_report)
if suite_has_run:
result.append(suite_report)
return result
def process_test_data(self, test_data):
"""
JSON output contains entries for skipped testcases
as well, which are not included in the report.
"""
result = []
for suite in test_data:
suite_report = TestGroupReport(name=suite["name"],
uid=suite["name"],
category="testsuite")
suite_has_run = False
for testcase in suite["data"]:
if testcase["status"] != "skipped":
suite_has_run = True
testcase_report = TestCaseReport(
name=testcase["name"],
uid=testcase["name"],
suite_related=True,
)
assertion_obj = RawAssertion(
passed=testcase["status"] == "pass",
content=testcase["error"] or testcase["duration"],
description=testcase["name"],
)
testcase_report.append(registry.serialize(assertion_obj))
testcase_report.runtime_status = RuntimeStatus.FINISHED
suite_report.append(testcase_report)
if suite_has_run:
result.append(suite_report)
return result
def process_test_data(self, test_data):
"""
XML output contains entries for skipped testcases
as well, which are not included in the report.
"""
result = []
for suite in test_data.getchildren():
suite_report = TestGroupReport(name=suite.attrib["name"],
category="testsuite")
suite_has_run = False
for testcase in suite.getchildren():
testcase_report = TestCaseReport(name=testcase.attrib["name"])
if not testcase.getchildren():
testcase_report.status_override = Status.PASSED
for entry in testcase.getchildren():
assertion_obj = RawAssertion(
description=entry.tag,
content=entry.text,
passed=entry.tag != "failure",
)
testcase_report.append(registry.serialize(assertion_obj))
if testcase.attrib["status"] != "notrun":
suite_report.append(testcase_report)
suite_has_run = True
if suite_has_run:
result.append(suite_report)
return result
def test_create_pdf(tmpdir):
"""PDF exporter should generate a PDF file using the report data."""
pdf_path = tmpdir.mkdir('reports').join('dummy_report.pdf').strpath
assertion_entries = [
assertions.Equal(1, 2),
assertions.Greater(2, 1),
assertions.IsFalse(True, 'this should fail'),
assertions.IsTrue(True, 'this should pass'),
assertions.Fail('Explicit failure'),
base.Group(
description='group description',
entries=[
assertions.NotEqual(2, 1),
assertions.Contain(1, [1, 2, 3]),
]
)
]
report = TestReport(
name='my testplan',
entries=[
TestGroupReport(
name='My Multitest',
category='multitest',
entries=[
TestGroupReport(
name='MySuite',
entries=[
TestCaseReport(
name='my_test_method',
entries=[
registry.serialize(obj)
for obj in assertion_entries
]
)
]
)
]
)
]
)
exporter = PDFExporter(
pdf_path=pdf_path,
pdf_style=styles.Style(
passing='assertion-detail',
failing='assertion-detail'
)
)
with log_propagation_disabled(TESTPLAN_LOGGER):
exporter.export(report)
assert os.path.exists(pdf_path)
assert os.stat(pdf_path).st_size > 0
def pytest_exception_interact(self, node, call, report):
"""
Hook called when an exception raised and it can be handled. This hook
is only called if the exception is not an PyTest internal exception.
:param node: PyTest Function or Module object
:param call: PyTest CallInfo object
:param report: PyTest TestReport or CollectReport object
"""
if call.when in ('memocollect', 'collect'):
# Failed to collect tests: log to console and mark the report as
# ERROR.
self._report.logger.error(format_trace(
inspect.getinnerframes(call.excinfo.tb), call.excinfo.value))
self._report.status_override = Status.ERROR
elif self._current_case_report is not None:
# Log assertion errors or exceptions in testcase report
traceback = call.excinfo.traceback[-1]
message = getattr(call.excinfo.value, 'message', None) or \
getattr(call.excinfo.value, 'msg', None) or \
getattr(call.excinfo.value, 'args', None) or ''
if isinstance(message, (tuple, list)):
message = message[0]
header = (('Assertion - Fail' if
call.excinfo.typename == 'AssertionError' else
'Exception raised') if call.when == 'call' else
'{} - Fail'.format(call.when))
details = 'File: {}{}Line: {}{}{}: {}'.format(
traceback.path.strpath,
os.linesep,
traceback.lineno + 1,
os.linesep,
call.excinfo.typename,
message
) if call.excinfo.typename == 'AssertionError' else (
report.longreprtext if hasattr(report, 'longreprtext') else
str(report.longrepr))
assertion_obj = assertions.RawAssertion(
description=header,
content=details,
passed=False
)
serialized_obj = schema_registry.serialize(assertion_obj)
self._current_case_report.append(serialized_obj)
self._current_case_report.status_override = Status.FAILED
else:
self._report.logger.error(
'Exception occured outside of a testcase: during %s',
call.when)
self._report.logger.error(format_trace(
inspect.getinnerframes(call.excinfo.tb), call.excinfo.value))
def pytest_exception_interact(self, node, call, report):
"""
Hook called when an exception raised and it can be handled. This hook
is only called if the exception is not an PyTest internal exception.
:param node: PyTest Function or Module object
:param call: PyTest CallInfo object
:param report: PyTest TestReport or CollectReport object
"""
if call.when in ("memocollect", "collect"):
# Failed to collect tests: log to console and mark the report as
# ERROR.
self._report.logger.error("".join(
traceback.format_exception(call.excinfo.type,
call.excinfo.value,
call.excinfo.tb)))
self._report.status_override = Status.ERROR
elif self._current_case_report is not None:
# Log assertion errors or exceptions in testcase report
trace = call.excinfo.traceback[-1]
message = (getattr(call.excinfo.value, "message", None)
or getattr(call.excinfo.value, "msg", None)
or getattr(call.excinfo.value, "args", None) or "")
if isinstance(message, (tuple, list)):
message = message[0]
header = (("Assertion - Fail" if call.excinfo.typename
== "AssertionError" else "Exception raised") if
call.when == "call" else "{} - Fail".format(call.when))
details = ("File: {}\nLine: {}\n{}: {}".format(
trace.path.strpath,
trace.lineno + 1,
call.excinfo.typename,
message,
) if call.excinfo.typename == "AssertionError" else
(report.longreprtext if hasattr(report, "longreprtext")
else str(report.longrepr)))
assertion_obj = assertions.RawAssertion(description=header,
content=details,
passed=False)
serialized_obj = schema_registry.serialize(assertion_obj)
self._current_case_report.append(serialized_obj)
self._current_case_report.status_override = Status.FAILED
else:
self._report.logger.error(
"Exception occured outside of a testcase: during %s",
call.when)
self._report.logger.error("".join(
traceback.format_exception(call.excinfo.type,
call.excinfo.value,
call.excinfo.tb)))
def pytest_runtest_logreport(self, report):
"""
Hook called by pytest to report on the result of a test.
:param report: the test report for the item just tested (see pytest
documentation)
"""
if report.when == "setup":
if report.skipped:
if self._current_case_report is None:
suite_name, case_name, case_params = _case_parse(
report.nodeid)
testcase_report = self.case_report(suite_name, case_name,
case_params)
else:
testcase_report = self._current_case_report
# Status set to be SKIPPED if testcase is marked skip or xfail
# lower versioned PyTest does not support this feature
testcase_report.status_override = Status.SKIPPED
elif report.when == "call":
if self._current_case_report is None:
raise RuntimeError(
"Cannot store testcase results to report: no report "
"object was created.")
if self._current_result_obj.entries:
# Add the assertion entry to the case report
for entry in self._current_result_obj.entries:
stdout_renderer = stdout_registry[entry]()
stdout_header = stdout_renderer.get_header(entry)
stdout_details = stdout_renderer.get_details(entry) or ""
# Add 'stdout_header' and 'stdout_details' attributes to
# serialized entries for standard output later
serialized_entry = schema_registry.serialize(entry)
serialized_entry.update(
stdout_header=stdout_header,
stdout_details=stdout_details,
)
self._current_case_report.append(serialized_entry)
if report.failed:
self._current_case_report.status_override = Status.FAILED
else:
self._current_case_report.pass_if_empty()
self._current_case_report.runtime_status = RuntimeStatus.FINISHED
elif report.when == "teardown":
pass
def _process_data(self, data: Element) -> List[TestGroupReport]:
"""
Processes data read from the source file.
:param data: raw data as read by the importer
"""
result = []
suites = data.getchildren() if data.tag == "testsuites" else [data]
for suite in suites:
suite_name = suite.attrib.get("name")
suite_report = TestGroupReport(
name=suite_name,
category=ReportCategories.TESTSUITE,
)
for element in suite.getchildren():
# Elements like properties, system-out, and system-err are
# skipped.
if element.tag != "testcase":
continue
case_class = element.attrib.get("classname")
case_name = element.attrib.get("name")
if case_class is None:
if case_name == suite_report.name:
path = os.path.normpath(case_name)
suite_report.name = path.rpartition(os.sep)[-1]
# We use the name "Execution" to avoid collision of
# test suite and test case.
case_report_name = "Execution"
else:
case_report_name = case_name
else:
case_report_name = (
f"{case_class.split('.')[-1]}::{case_name}")
case_report = TestCaseReport(name=case_report_name)
if not element.getchildren():
assertion = RawAssertion(
description="Passed",
content=f"Testcase {case_name} passed",
passed=True,
)
case_report.append(registry.serialize(assertion))
else:
# Upon a failure, there will be a single testcase which is
# the first child.
content, tag, desc = "", None, None
for child in element.getchildren():
tag = tag or child.tag
msg = child.attrib.get("message") or child.text
# Approach: if it is a failure/error child, then use
# the message attribute directly.
# Otherwise, for instance if it is a system-err/out,
# then build up the content step by step from it.
if not desc and child.tag in ("failure", "error"):
desc = msg
else:
content += f"[{child.tag}]\n{msg}\n"
assertion = RawAssertion(
description=desc,
content=content,
passed=tag not in ("failure", "error"),
)
case_report.append(registry.serialize(assertion))
suite_report.runtime_status = RuntimeStatus.FINISHED
suite_report.append(case_report)
if len(suite_report):
result.append(suite_report)
return result