def _score_test(self, test: Test, outcome: Outcome) -> Tuple[bool, bool]:
"""
Given a test and the test's outcome, determine if the test met expectations and log pertinent information
"""
# scoring outcomes
result_pass = True
sim_failed = False
try:
outcome.get()
except Exception as e:
result = remove_traceback_frames(e, ["_score_test", "get"])
else:
result = TestSuccess()
if (isinstance(result, TestSuccess) and not test.expect_fail
and not test.expect_error):
self._log_test_passed(test, None, None)
elif isinstance(result, AssertionError) and test.expect_fail:
self._log_test_passed(test, result, "failed as expected")
elif isinstance(result, TestSuccess) and test.expect_error:
self._log_test_failed(test, None,
"passed but we expected an error")
result_pass = False
elif isinstance(result, TestSuccess):
self._log_test_failed(test, None,
"passed but we expected a failure")
result_pass = False
elif isinstance(result, SimFailure):
if isinstance(result, test.expect_error):
self._log_test_passed(test, result, "errored as expected")
else:
self.log.error(
"Test error has lead to simulator shutting us down")
result_pass = False
# whether we expected it or not, the simulation has failed unrecoverably
sim_failed = True
elif test.expect_error:
if isinstance(result, test.expect_error):
self._log_test_passed(test, result, "errored as expected")
else:
self._log_test_failed(test, result,
"errored with unexpected type ")
result_pass = False
else:
self._log_test_failed(test, result, None)
result_pass = False
if _pdb_on_exception:
pdb.post_mortem(result.__traceback__)
return result_pass, sim_failed
def _score_test(self, test: Test, outcome: Outcome) -> Tuple[bool, bool]:
"""
Given a test and the test's outcome, determine if the test met expectations and log pertinent information
"""
# Helper for logging result
def _result_was():
result_was = ("{} (result was {})".format(
test.__qualname__,
type(result).__qualname__))
return result_was
# scoring outcomes
result_pass = True
sim_failed = False
try:
outcome.get()
except Exception as e:
result = remove_traceback_frames(e, ['_score_test', 'get'])
else:
result = TestSuccess()
if (isinstance(result, TestSuccess) and not test.expect_fail
and not test.expect_error):
self.log.info("Test Passed: %s" % test.__qualname__)
elif (isinstance(result, AssertionError) and test.expect_fail):
self.log.info("Test failed as expected: " + _result_was())
elif (isinstance(result, TestSuccess) and test.expect_error):
self.log.error("Test passed but we expected an error: " +
_result_was())
result_pass = False
elif isinstance(result, TestSuccess):
self.log.error("Test passed but we expected a failure: " +
_result_was())
result_pass = False
elif isinstance(result, SimFailure):
if isinstance(result, test.expect_error):
self.log.info("Test errored as expected: " + _result_was())
else:
self.log.error(
"Test error has lead to simulator shutting us "
"down",
exc_info=result)
result_pass = False
# whether we expected it or not, the simulation has failed unrecoverably
sim_failed = True
elif test.expect_error:
if isinstance(result, test.expect_error):
self.log.info("Test errored as expected: " + _result_was())
else:
self.log.error("Test errored with unexpected type: " +
_result_was(),
exc_info=result)
result_pass = False
else:
self.log.error("Test Failed: " + _result_was(), exc_info=result)
result_pass = False
if _pdb_on_exception:
pdb.post_mortem(result.__traceback__)
return result_pass, sim_failed