def process_test_data(self, test_data):
"""
XML output contains entries for skipped testcases
as well, which are not included in the report.
"""
result = []
for suite in test_data.getchildren():
suite_name = suite.attrib["name"]
suite_report = TestGroupReport(
name=suite_name,
uid=suite_name,
category=ReportCategories.TESTSUITE,
)
for testcase in suite.getchildren():
if testcase.tag != "testcase":
continue
testcase_classname = testcase.attrib["classname"]
testcase_name = testcase.attrib["name"]
testcase_prefix = testcase_classname.split(".")[-1]
testcase_report = TestCaseReport(
name="{}::{}".format(testcase_prefix, testcase_name),
uid="{}::{}".format(testcase_classname.replace(".", "::"),
testcase_name),
)
if not testcase.getchildren():
assertion_obj = RawAssertion(
description="Passed",
content="Testcase {} passed".format(testcase_name),
passed=True,
)
testcase_report.append(registry.serialize(assertion_obj))
else:
for entry in testcase.getchildren():
assertion_obj = RawAssertion(
description=entry.tag,
content=entry.text,
passed=entry.tag not in ("failure", "error"),
)
testcase_report.append(
registry.serialize(assertion_obj))
testcase_report.runtime_status = RuntimeStatus.FINISHED
suite_report.append(testcase_report)
if len(suite_report) > 0:
result.append(suite_report)
return result
def process_test_data(self, test_data):
"""
XML output contains entries for skipped testcases
as well, which are not included in the report.
"""
result = []
for suite in test_data.getchildren():
suite_name = suite.attrib["name"]
suite_report = TestGroupReport(
name=suite_name,
uid=suite_name,
category=ReportCategories.TESTSUITE,
)
suite_has_run = False
for testcase in suite.getchildren():
testcase_name = testcase.attrib["name"]
testcase_report = TestCaseReport(
name=testcase_name, uid=testcase_name
)
if not testcase.getchildren():
assertion_obj = RawAssertion(
description="Passed",
content="Testcase {} passed".format(testcase_name),
passed=True,
)
testcase_report.append(registry.serialize(assertion_obj))
else:
for entry in testcase.getchildren():
assertion_obj = RawAssertion(
description=entry.tag,
content=entry.text,
passed=entry.tag != "failure",
)
testcase_report.append(
registry.serialize(assertion_obj)
)
testcase_report.runtime_status = RuntimeStatus.FINISHED
if testcase.attrib["status"] != "notrun":
suite_report.append(testcase_report)
suite_has_run = True
if suite_has_run:
result.append(suite_report)
return result
def _process_data(self, data: Element) -> List[TestGroupReport]:
"""
Processes data read from the source file.
:param data: raw data as read by the importer
"""
# NOTE: XML output contains skipped testcases which are ignored.
result = []
for suite in data.getchildren():
suite_name = suite.attrib["name"]
suite_report = TestGroupReport(
name=suite_name,
category=ReportCategories.TESTSUITE,
)
for testcase in suite.getchildren():
if testcase.tag != "testcase":
continue
testcase_classname = testcase.attrib["classname"]
testcase_name = testcase.attrib["name"]
testcase_prefix = testcase_classname.split(".")[-1]
testcase_report = TestCaseReport(name="{}::{}".format(
testcase_prefix, testcase_name), )
if not testcase.getchildren():
assertion_obj = RawAssertion(
description="Passed",
content=f"Testcase {testcase_name} passed",
passed=True,
)
testcase_report.append(registry.serialize(assertion_obj))
else:
for entry in testcase.getchildren():
assertion_obj = RawAssertion(
description=entry.tag,
content=entry.text,
passed=entry.tag not in ("failure", "error"),
)
testcase_report.append(
registry.serialize(assertion_obj))
testcase_report.runtime_status = RuntimeStatus.FINISHED
suite_report.append(testcase_report)
if len(suite_report) > 0:
result.append(suite_report)
return result
def process_test_data(self, test_data):
"""
XML output contains entries for skipped testcases
as well, which are not included in the report.
"""
result = []
for suite in test_data.getchildren():
suite_report = TestGroupReport(
name=suite.attrib['name'],
category='suite',
)
suite_has_run = False
for testcase in suite.getchildren():
testcase_report = TestCaseReport(name=testcase.attrib['name'])
for entry in testcase.getchildren():
assertion_obj = RawAssertion(description=entry.tag,
content=entry.text,
passed=entry.tag != 'failure')
testcase_report.append(registry.serialize(assertion_obj))
if testcase.attrib['status'] != 'notrun':
suite_report.append(testcase_report)
suite_has_run = True
if suite_has_run:
result.append(suite_report)
return result
def get_process_failure_report(self):
"""
When running a process fails (e.g. binary crash, timeout etc)
we can still generate dummy testsuite / testcase reports with
a certain hierarchy compatible with exporters and XUnit conventions.
"""
from testplan.testing.multitest.entries.assertions import RawAssertion
assertion_content = os.linesep.join([
'Process failure: {}'.format(self.cfg.driver),
'Exit code: {}'.format(self._test_process_retcode),
'stdout: {}'.format(self.stdout), 'stderr: {}'.format(self.stderr)
])
testcase_report = TestCaseReport(
name='failure',
entries=[
RawAssertion(description='Process failure details',
content=assertion_content,
passed=False).serialize(),
])
testcase_report.status_override = Status.ERROR
return TestGroupReport(name='ProcessFailure',
category='suite',
entries=[testcase_report])
def process_test_data(self, test_data):
"""
JSON output contains entries for skipped testcases
as well, which are not included in the report.
"""
result = []
for suite in test_data:
suite_report = TestGroupReport(
name=suite['name'],
category='suite',
)
suite_has_run = False
for testcase in suite['data']:
if testcase['status'] != 'skipped':
suite_has_run = True
testcase_report = TestCaseReport(name=testcase['name'])
assertion_obj = RawAssertion(
passed=testcase['status'] == 'pass',
content=testcase['error'] or testcase['duration'],
description=testcase['name'])
testcase_report.append(registry.serialize(assertion_obj))
suite_report.append(testcase_report)
if suite_has_run:
result.append(suite_report)
return result
def process_test_data(self, test_data):
"""
JSON output contains entries for skipped testcases
as well, which are not included in the report.
"""
result = []
for suite in test_data:
suite_report = TestGroupReport(name=suite["name"],
uid=suite["name"],
category="testsuite")
suite_has_run = False
for testcase in suite["data"]:
if testcase["status"] != "skipped":
suite_has_run = True
testcase_report = TestCaseReport(
name=testcase["name"],
uid=testcase["name"],
suite_related=True,
)
assertion_obj = RawAssertion(
passed=testcase["status"] == "pass",
content=testcase["error"] or testcase["duration"],
description=testcase["name"],
)
testcase_report.append(registry.serialize(assertion_obj))
testcase_report.runtime_status = RuntimeStatus.FINISHED
suite_report.append(testcase_report)
if suite_has_run:
result.append(suite_report)
return result
def process_test_data(self, test_data):
"""
XML output contains entries for skipped testcases
as well, which are not included in the report.
"""
result = []
for suite in test_data.getchildren():
suite_report = TestGroupReport(name=suite.attrib["name"],
category="testsuite")
suite_has_run = False
for testcase in suite.getchildren():
testcase_report = TestCaseReport(name=testcase.attrib["name"])
if not testcase.getchildren():
testcase_report.status_override = Status.PASSED
for entry in testcase.getchildren():
assertion_obj = RawAssertion(
description=entry.tag,
content=entry.text,
passed=entry.tag != "failure",
)
testcase_report.append(registry.serialize(assertion_obj))
if testcase.attrib["status"] != "notrun":
suite_report.append(testcase_report)
suite_has_run = True
if suite_has_run:
result.append(suite_report)
return result
def get_process_check_report(self, retcode, stdout, stderr):
"""
When running a process fails (e.g. binary crash, timeout etc)
we can still generate dummy testsuite / testcase reports with
a certain hierarchy compatible with exporters and XUnit conventions.
And logs of stdout & stderr can be saved as attachment.
"""
assertion_content = "\n".join(
[
"Process: {}".format(self.cfg.binary),
"Exit code: {}".format(retcode),
]
)
passed = retcode == 0 or retcode in self.cfg.ignore_exit_codes
testcase_report = TestCaseReport(
name=self._VERIFICATION_TESTCASE_NAME,
uid=self._VERIFICATION_TESTCASE_NAME,
suite_related=True,
entries=[
RawAssertion(
description="Process exit code check",
content=assertion_content,
passed=passed,
).serialize()
],
)
if stdout and os.path.isfile(stdout):
stdout_attachment = Attachment(
filepath=os.path.abspath(stdout),
description="Process stdout",
)
testcase_report.attachments.append(stdout_attachment)
testcase_report.append(stdout_attachment.serialize())
if stderr and os.path.isfile(stderr):
stderr_attachment = Attachment(
filepath=os.path.abspath(stderr),
description="Process stderr",
)
testcase_report.attachments.append(stderr_attachment)
testcase_report.append(stderr_attachment.serialize())
testcase_report.runtime_status = RuntimeStatus.FINISHED
suite_report = TestGroupReport(
name=self._VERIFICATION_SUITE_NAME,
uid=self._VERIFICATION_SUITE_NAME,
category=ReportCategories.TESTSUITE,
entries=[testcase_report],
)
return suite_report
def get_process_check_report(self, retcode, stdout, stderr):
"""
When running a process fails (e.g. binary crash, timeout etc)
we can still generate dummy testsuite / testcase reports with
a certain hierarchy compatible with exporters and XUnit conventions.
"""
assertion_content = "\n".join(
[
"Process: {}".format(self.cfg.driver),
"Exit code: {}".format(retcode),
]
)
passed = retcode == 0 or retcode in self.cfg.ignore_exit_codes
testcase_report = TestCaseReport(
name="ExitCodeCheck",
uid="ExitCodeCheck",
suite_related=True,
entries=[
RawAssertion(
description="Process exit code check",
content=assertion_content,
passed=passed,
).serialize(),
Log(
message=stdout.read(), description="Process stdout",
).serialize(),
Log(
message=stderr.read(), description="Process stderr",
).serialize(),
],
)
testcase_report.runtime_status = RuntimeStatus.FINISHED
suite_report = TestGroupReport(
name="ProcessChecks",
category=ReportCategories.TESTSUITE,
entries=[testcase_report],
)
return suite_report
def _process_data(self, data: Element) -> List[TestGroupReport]:
"""
Processes data read from the source file.
:param data: raw data as read by the importer
"""
result = []
suites = data.getchildren() if data.tag == "testsuites" else [data]
for suite in suites:
suite_name = suite.attrib.get("name")
suite_report = TestGroupReport(
name=suite_name,
category=ReportCategories.TESTSUITE,
)
for element in suite.getchildren():
# Elements like properties, system-out, and system-err are
# skipped.
if element.tag != "testcase":
continue
case_class = element.attrib.get("classname")
case_name = element.attrib.get("name")
if case_class is None:
if case_name == suite_report.name:
path = os.path.normpath(case_name)
suite_report.name = path.rpartition(os.sep)[-1]
# We use the name "Execution" to avoid collision of
# test suite and test case.
case_report_name = "Execution"
else:
case_report_name = case_name
else:
case_report_name = (
f"{case_class.split('.')[-1]}::{case_name}")
case_report = TestCaseReport(name=case_report_name)
if not element.getchildren():
assertion = RawAssertion(
description="Passed",
content=f"Testcase {case_name} passed",
passed=True,
)
case_report.append(registry.serialize(assertion))
else:
# Upon a failure, there will be a single testcase which is
# the first child.
content, tag, desc = "", None, None
for child in element.getchildren():
tag = tag or child.tag
msg = child.attrib.get("message") or child.text
# Approach: if it is a failure/error child, then use
# the message attribute directly.
# Otherwise, for instance if it is a system-err/out,
# then build up the content step by step from it.
if not desc and child.tag in ("failure", "error"):
desc = msg
else:
content += f"[{child.tag}]\n{msg}\n"
assertion = RawAssertion(
description=desc,
content=content,
passed=tag not in ("failure", "error"),
)
case_report.append(registry.serialize(assertion))
suite_report.runtime_status = RuntimeStatus.FINISHED
suite_report.append(case_report)
if len(suite_report):
result.append(suite_report)
return result
