def run_tests(self): """Run PyUnit and wait for it to terminate.""" suite_result = unittest.TestResult() self.cfg.suite.run(suite_result) # Since we can't reliably inspect the individual testcases of a PyUnit # suite, we put all results into a single "testcase" report. This # will only list failures and errors and not give detail on individual # assertions like with MultiTest. testcase_report = report_testing.TestCaseReport( name=self._TESTCASE_NAME) for call, error in suite_result.errors: assertion_obj = assertions.RawAssertion(description=str(call), content=str(error).strip(), passed=False) testcase_report.append( schemas.base.registry.serialize(assertion_obj)) for call, error in suite_result.failures: assertion_obj = assertions.RawAssertion(description=str(call), content=str(error).strip(), passed=False) testcase_report.append( schemas.base.registry.serialize(assertion_obj)) # In case of no failures or errors we need to explicitly mark the # testsuite as passed. if not testcase_report.entries: log_entry = base_entries.Log("All PyUnit testcases passed", description="PyUnit success") testcase_report.append(schemas.base.registry.serialize(log_entry)) self.result.report.append(testcase_report)
def _run_testsuite(self, pyunit_testcase): """Run a single PyUnit Testcase as a suite and return a testsuite report.""" suite = unittest.defaultTestLoader.loadTestsFromTestCase( pyunit_testcase ) suite_result = unittest.TextTestRunner().run(suite) # Since we can't reliably inspect the individual testcases of a PyUnit # suite, we put all results into a single "testcase" report. This # will only list failures and errors and not give detail on individual # assertions like with MultiTest. testcase_report = TestCaseReport( name=self._TESTCASE_NAME, uid=self._TESTCASE_NAME ) for call, error in suite_result.errors: assertion_obj = assertions.RawAssertion( description=str(call), content=str(error).strip(), passed=False ) testcase_report.append( schemas.base.registry.serialize(assertion_obj) ) for call, error in suite_result.failures: assertion_obj = assertions.RawAssertion( description=str(call), content=str(error).strip(), passed=False ) testcase_report.append( schemas.base.registry.serialize(assertion_obj) ) # In case of no failures or errors we need to explicitly mark the # testsuite as passed. if not testcase_report.entries: log_entry = entries_base.Log( "All PyUnit testcases passed", description="PyUnit success" ) testcase_report.append(schemas.base.registry.serialize(log_entry)) testcase_report.runtime_status = RuntimeStatus.FINISHED # We have to wrap the testcase report in a testsuite report. return TestGroupReport( name=pyunit_testcase.__name__, uid=pyunit_testcase.__name__, category=ReportCategories.TESTSUITE, entries=[testcase_report], )
def pytest_exception_interact(self, node, call, report): """ Hook called when an exception raised and it can be handled. This hook is only called if the exception is not an PyTest internal exception. :param node: PyTest Function or Module object :param call: PyTest CallInfo object :param report: PyTest TestReport or CollectReport object """ if call.when in ("memocollect", "collect"): # Failed to collect tests: log to console and mark the report as # ERROR. self._report.logger.error("".join( traceback.format_exception(call.excinfo.type, call.excinfo.value, call.excinfo.tb))) self._report.status_override = Status.ERROR elif self._current_case_report is not None: # Log assertion errors or exceptions in testcase report trace = call.excinfo.traceback[-1] message = (getattr(call.excinfo.value, "message", None) or getattr(call.excinfo.value, "msg", None) or getattr(call.excinfo.value, "args", None) or "") if isinstance(message, (tuple, list)): message = message[0] header = (("Assertion - Fail" if call.excinfo.typename == "AssertionError" else "Exception raised") if call.when == "call" else "{} - Fail".format(call.when)) details = ("File: {}\nLine: {}\n{}: {}".format( str(trace.path), trace.lineno + 1, call.excinfo.typename, message, ) if call.excinfo.typename == "AssertionError" else (report.longreprtext if hasattr(report, "longreprtext") else str(report.longrepr))) assertion_obj = assertions.RawAssertion(description=header, content=details, passed=False) serialized_obj = schema_registry.serialize(assertion_obj) self._current_case_report.append(serialized_obj) self._current_case_report.status_override = Status.FAILED for capture, description in ( ("caplog", "Captured Log"), ("capstdout", "Captured Stdout"), ("capstderr", "Captured Stderr"), ): message = getattr(report, capture) if message: assertion_obj = entries_base.Log(message, description=description) serialized_obj = schema_registry.serialize(assertion_obj) self._current_case_report.append(serialized_obj) else: self._report.logger.error( "Exception occured outside of a testcase: during %s", call.when) self._report.logger.error("".join( traceback.format_exception(call.excinfo.type, call.excinfo.value, call.excinfo.tb)))
def test_summary(): """ Summary.summarize should group entries by category, entry class and pass status, leaving out non-assertion entries. """ asr_1 = assertions.Equal(1, 1) asr_2 = assertions.Equal(1, 1, category='alpha') asr_3 = assertions.Less(1, 2) asr_4 = assertions.Less(3, 4, category='alpha') asr_5 = assertions.Equal(1, 2) asr_6 = assertions.Less(4, 4, category='alpha') less_failing = [ assertions.Less(4, 4, category='alpha') for _ in range(100) ] less_passing = [ assertions.Less(3, 4, category='alpha') for _ in range(100) ] summary = base.Summary( entries=[ asr_1, asr_2, base.Log('foo'), asr_3, asr_4, base.Log('bar'), asr_5, asr_6 ] + less_failing + less_passing, num_passing=3, num_failing=4, ) no_category, alpha_category = summary.entries # Top level groups assert isinstance(no_category, base.Group) assert isinstance(alpha_category, base.Group) no_category_equal, no_category_less = no_category.entries # Assertion type groups for no category assert isinstance(no_category_equal, base.Group) assert isinstance(no_category_less, base.Group) entries = no_category_equal.entries no_category_equal_failing, no_category_equal_passing = entries # pass/fail groups for equal assertion assert isinstance(no_category_equal_passing, base.Group) assert isinstance(no_category_equal_failing, base.Group) assert no_category_equal_passing.entries == [asr_1] assert no_category_equal_failing.entries == [asr_5] # No failing Less assertion for None category assert len(no_category_less.entries) == 1 no_category_less_passing = no_category_less.entries[0] assert no_category_less_passing.entries == [asr_3] alpha_category_equal, alpha_category_less = alpha_category.entries assert isinstance(alpha_category_equal, base.Group) assert isinstance(alpha_category_less, base.Group) # No failing Equal assertion for alpha category assert len(alpha_category_equal.entries) == 1 alpha_category_equal_passing = alpha_category_equal.entries[0] assert isinstance(alpha_category_equal_passing, base.Group) assert alpha_category_equal_passing.entries == [asr_2] entries = alpha_category_less.entries alpha_category_less_failing, alpha_category_less_passing = entries assert isinstance(alpha_category_less_passing, base.Group) assert isinstance(alpha_category_less_failing, base.Group) assert len(alpha_category_less_passing.entries) == summary.num_passing assert len(alpha_category_less_failing.entries) == summary.num_failing