def run_tests(self):
"""Run PyUnit and wait for it to terminate."""
suite_result = unittest.TestResult()
self.cfg.suite.run(suite_result)
# Since we can't reliably inspect the individual testcases of a PyUnit
# suite, we put all results into a single "testcase" report. This
# will only list failures and errors and not give detail on individual
# assertions like with MultiTest.
testcase_report = report_testing.TestCaseReport(
name=self._TESTCASE_NAME)
for call, error in suite_result.errors:
assertion_obj = assertions.RawAssertion(description=str(call),
content=str(error).strip(),
passed=False)
testcase_report.append(
schemas.base.registry.serialize(assertion_obj))
for call, error in suite_result.failures:
assertion_obj = assertions.RawAssertion(description=str(call),
content=str(error).strip(),
passed=False)
testcase_report.append(
schemas.base.registry.serialize(assertion_obj))
# In case of no failures or errors we need to explicitly mark the
# testsuite as passed.
if not testcase_report.entries:
log_entry = base_entries.Log("All PyUnit testcases passed",
description="PyUnit success")
testcase_report.append(schemas.base.registry.serialize(log_entry))
self.result.report.append(testcase_report)
def _run_testsuite(self, pyunit_testcase):
"""Run a single PyUnit Testcase as a suite and return a testsuite report."""
suite = unittest.defaultTestLoader.loadTestsFromTestCase(
pyunit_testcase
)
suite_result = unittest.TextTestRunner().run(suite)
# Since we can't reliably inspect the individual testcases of a PyUnit
# suite, we put all results into a single "testcase" report. This
# will only list failures and errors and not give detail on individual
# assertions like with MultiTest.
testcase_report = TestCaseReport(
name=self._TESTCASE_NAME, uid=self._TESTCASE_NAME
)
for call, error in suite_result.errors:
assertion_obj = assertions.RawAssertion(
description=str(call), content=str(error).strip(), passed=False
)
testcase_report.append(
schemas.base.registry.serialize(assertion_obj)
)
for call, error in suite_result.failures:
assertion_obj = assertions.RawAssertion(
description=str(call), content=str(error).strip(), passed=False
)
testcase_report.append(
schemas.base.registry.serialize(assertion_obj)
)
# In case of no failures or errors we need to explicitly mark the
# testsuite as passed.
if not testcase_report.entries:
log_entry = entries_base.Log(
"All PyUnit testcases passed", description="PyUnit success"
)
testcase_report.append(schemas.base.registry.serialize(log_entry))
testcase_report.runtime_status = RuntimeStatus.FINISHED
# We have to wrap the testcase report in a testsuite report.
return TestGroupReport(
name=pyunit_testcase.__name__,
uid=pyunit_testcase.__name__,
category=ReportCategories.TESTSUITE,
entries=[testcase_report],
)
def pytest_exception_interact(self, node, call, report):
"""
Hook called when an exception raised and it can be handled. This hook
is only called if the exception is not an PyTest internal exception.
:param node: PyTest Function or Module object
:param call: PyTest CallInfo object
:param report: PyTest TestReport or CollectReport object
"""
if call.when in ("memocollect", "collect"):
# Failed to collect tests: log to console and mark the report as
# ERROR.
self._report.logger.error("".join(
traceback.format_exception(call.excinfo.type,
call.excinfo.value,
call.excinfo.tb)))
self._report.status_override = Status.ERROR
elif self._current_case_report is not None:
# Log assertion errors or exceptions in testcase report
trace = call.excinfo.traceback[-1]
message = (getattr(call.excinfo.value, "message", None)
or getattr(call.excinfo.value, "msg", None)
or getattr(call.excinfo.value, "args", None) or "")
if isinstance(message, (tuple, list)):
message = message[0]
header = (("Assertion - Fail" if call.excinfo.typename
== "AssertionError" else "Exception raised") if
call.when == "call" else "{} - Fail".format(call.when))
details = ("File: {}\nLine: {}\n{}: {}".format(
str(trace.path),
trace.lineno + 1,
call.excinfo.typename,
message,
) if call.excinfo.typename == "AssertionError" else
(report.longreprtext if hasattr(report, "longreprtext")
else str(report.longrepr)))
assertion_obj = assertions.RawAssertion(description=header,
content=details,
passed=False)
serialized_obj = schema_registry.serialize(assertion_obj)
self._current_case_report.append(serialized_obj)
self._current_case_report.status_override = Status.FAILED
for capture, description in (
("caplog", "Captured Log"),
("capstdout", "Captured Stdout"),
("capstderr", "Captured Stderr"),
):
message = getattr(report, capture)
if message:
assertion_obj = entries_base.Log(message,
description=description)
serialized_obj = schema_registry.serialize(assertion_obj)
self._current_case_report.append(serialized_obj)
else:
self._report.logger.error(
"Exception occured outside of a testcase: during %s",
call.when)
self._report.logger.error("".join(
traceback.format_exception(call.excinfo.type,
call.excinfo.value,
call.excinfo.tb)))
def test_summary():
"""
Summary.summarize should group entries by
category, entry class and pass status,
leaving out non-assertion entries.
"""
asr_1 = assertions.Equal(1, 1)
asr_2 = assertions.Equal(1, 1, category='alpha')
asr_3 = assertions.Less(1, 2)
asr_4 = assertions.Less(3, 4, category='alpha')
asr_5 = assertions.Equal(1, 2)
asr_6 = assertions.Less(4, 4, category='alpha')
less_failing = [
assertions.Less(4, 4, category='alpha') for _ in range(100)
]
less_passing = [
assertions.Less(3, 4, category='alpha') for _ in range(100)
]
summary = base.Summary(
entries=[
asr_1, asr_2,
base.Log('foo'), asr_3, asr_4,
base.Log('bar'), asr_5, asr_6
] + less_failing + less_passing,
num_passing=3,
num_failing=4,
)
no_category, alpha_category = summary.entries
# Top level groups
assert isinstance(no_category, base.Group)
assert isinstance(alpha_category, base.Group)
no_category_equal, no_category_less = no_category.entries
# Assertion type groups for no category
assert isinstance(no_category_equal, base.Group)
assert isinstance(no_category_less, base.Group)
entries = no_category_equal.entries
no_category_equal_failing, no_category_equal_passing = entries
# pass/fail groups for equal assertion
assert isinstance(no_category_equal_passing, base.Group)
assert isinstance(no_category_equal_failing, base.Group)
assert no_category_equal_passing.entries == [asr_1]
assert no_category_equal_failing.entries == [asr_5]
# No failing Less assertion for None category
assert len(no_category_less.entries) == 1
no_category_less_passing = no_category_less.entries[0]
assert no_category_less_passing.entries == [asr_3]
alpha_category_equal, alpha_category_less = alpha_category.entries
assert isinstance(alpha_category_equal, base.Group)
assert isinstance(alpha_category_less, base.Group)
# No failing Equal assertion for alpha category
assert len(alpha_category_equal.entries) == 1
alpha_category_equal_passing = alpha_category_equal.entries[0]
assert isinstance(alpha_category_equal_passing, base.Group)
assert alpha_category_equal_passing.entries == [asr_2]
entries = alpha_category_less.entries
alpha_category_less_failing, alpha_category_less_passing = entries
assert isinstance(alpha_category_less_passing, base.Group)
assert isinstance(alpha_category_less_failing, base.Group)
assert len(alpha_category_less_passing.entries) == summary.num_passing
assert len(alpha_category_less_failing.entries) == summary.num_failing