def initialise(self): self.ntests = 0 self.count = 1 self.skipped = 0 self.failures = 0 self.xunit = XUnitReporter() self.xunit.add_testsuite(name="all", tests=repr(self.ntests), package="all") if coverage is not None: self.log.info("Enabling coverage collection of Python code") self._cov = coverage.coverage(branch=True, omit=["*cocotb*"]) self._cov.start() self._dut = cocotb.handle.SimHandle(simulator.get_root_handle(self._root_name)) if self._dut is None: raise AttributeError("Can not find Root Handle (%s)" % root_name) # Auto discovery for module_name in self._modules: module = _my_import(module_name) if self._functions: # Specific functions specified, don't auto discover for test in self._functions.rsplit(','): if not hasattr(module, test): raise AttributeError("Test %s doesn't exist in %s" % (test, module_name)) self._queue.append(getattr(module, test)(self._dut)) self.ntests += 1 break for thing in vars(module).values(): if hasattr(thing, "im_test"): try: test = thing(self._dut) skip = test.skip except TestError: skip = True self.log.warning("Failed to initialise test %s" % thing.name) if skip: self.log.info("Skipping test %s" % thing.name) self.xunit.add_testcase(name=thing.name, classname=module_name, time="0.0") self.xunit.add_skipped() self.skipped += 1 else: self._queue.append(test) self.ntests += 1 self._queue.sort(key=lambda test: "%s.%s" % (test.module, test.funcname)) for valid_tests in self._queue: self.log.info("Found test %s.%s" % (valid_tests.module, valid_tests.funcname))
def initialise(self): self.ntests = 0 self.count = 1 self.skipped = 0 self.failures = 0 self.xunit = XUnitReporter() self.xunit.add_testsuite(name="all", tests=repr(self.ntests), package="all") # Auto discovery for module_name in self._modules: module = _my_import(module_name) if self._functions: # Specific functions specified, don't auto discover for test in self._functions.rsplit(','): if not hasattr(module, test): raise AttributeError("Test %s doesn't exist in %s" % (test, module_name)) self._queue.append(getattr(module, test)(self._dut)) self.ntests += 1 break for thing in vars(module).values(): if hasattr(thing, "im_test"): try: test = thing(self._dut) skip = test.skip except TestError: skip = True self.log.warning("Failed to initialise test %s" % thing.name) if skip: self.log.info("Skipping test %s" % thing.name) self.xunit.add_testcase(name=thing.name, classname=module_name, time="0.0") self.xunit.add_skipped() self.skipped += 1 else: self._queue.append(test) self.ntests += 1 self._queue.sort(key=lambda test: "%s.%s" % (test.module, test.funcname)) for valid_tests in self._queue: self.log.info("Found test %s.%s" % (valid_tests.module, valid_tests.funcname))
class RegressionManager(object): """Encapsulates all regression capability into a single place""" def __init__(self, root_name, modules, tests=None, seed=None, hooks=[]): """ Args: modules (list): A list of python module names to run Kwargs """ self._queue = [] self._root_name = root_name self._dut = None self._modules = modules self._functions = tests self._running_test = None self._cov = None self.log = SimLog("cocotb.regression") self._seed = seed self._hooks = hooks def initialise(self): try: self._initialise() except Exception as e: import traceback self.log.error(traceback.format_exc()) raise def _initialise(self): self.start_time = time.time() self.test_results = [] self.ntests = 0 self.count = 1 self.skipped = 0 self.failures = 0 self.xunit = XUnitReporter() suite_name = os.getenv('RESULT_TESTSUITE') if os.getenv('RESULT_TESTSUITE') else "all" package_name = os.getenv('RESULT_TESTPACKAGE') if os.getenv('RESULT_TESTPACKAGE') else "all" self.xunit.add_testsuite(name=suite_name, tests=repr(self.ntests), package=package_name) if (self._seed is not None): self.xunit.add_property(name="random_seed", value=("%d"%self._seed)) if coverage is not None: self.log.info("Enabling coverage collection of Python code") self._cov = coverage.coverage(branch=True, omit=["*cocotb*"]) self._cov.start() handle = simulator.get_root_handle(self._root_name) self._dut = cocotb.handle.SimHandle(handle) if handle else None if self._dut is None: raise AttributeError("Can not find Root Handle (%s)" % self._root_name) # Auto discovery for module_name in self._modules: try: self.log.debug("Python Path: " + ",".join(sys.path)) self.log.debug("PWD: " + os.getcwd()) module = _my_import(module_name) except Exception as E: self.log.critical("Failed to import module %s: %s", module_name, E) self.log.info("MODULE variable was \"%s\"", ".".join(self._modules)) self.log.info("Traceback: ") self.log.info(traceback.format_exc()) raise if self._functions: # Specific functions specified, don't auto discover for test in self._functions.rsplit(','): if not hasattr(module, test): raise AttributeError("Test %s doesn't exist in %s" % (test, module_name)) self._queue.append(getattr(module, test)(self._dut)) self.ntests += 1 break for thing in vars(module).values(): if hasattr(thing, "im_test"): try: test = thing(self._dut) skip = test.skip except TestError: skip = True self.log.warning("Failed to initialise test %s" % thing.name) if skip: self.log.info("Skipping test %s" % thing.name) self.xunit.add_testcase(name=thing.name, classname=module_name, time="0.0", sim_time_ns="0.0", ratio_time="0.0") self.xunit.add_skipped() self.skipped += 1 self._store_test_result(module_name, thing.name, None, 0.0, 0.0, 0.0) else: self._queue.append(test) self.ntests += 1 self._queue.sort(key=lambda test: "%s.%s" % (test.module, test.funcname)) for valid_tests in self._queue: self.log.info("Found test %s.%s" % (valid_tests.module, valid_tests.funcname)) for module_name in self._hooks: self.log.info("Loading hook from module '"+module_name+"'") module = _my_import(module_name) for thing in vars(module).values(): if hasattr(thing, "im_hook"): try: test = thing(self._dut) except TestError: self.log.warning("Failed to initialize hook %s" % thing.name) else: cocotb.scheduler.add(test) def tear_down(self): """It's the end of the world as we know it""" if self.failures: self.log.error("Failed %d out of %d tests (%d skipped)" % (self.failures, self.count - 1, self.skipped)) else: self.log.info("Passed %d tests (%d skipped)" % (self.count - 1, self.skipped)) if self._cov: self._cov.stop() self.log.info("Writing coverage data") self._cov.save() self._cov.html_report() self._log_test_summary() self._log_sim_summary() self.log.info("Shutting down...") self.xunit.write() simulator.stop_simulator() def next_test(self): """Get the next test to run""" if not self._queue: return None return self._queue.pop(0) def _add_failure(self, result): self.xunit.add_failure(stdout=repr(str(result)), stderr="\n".join(self._running_test.error_messages), message="Test failed with random_seed={}".format(self._seed)) self.failures += 1 def handle_result(self, result): """Handle a test result Dumps result to XML and schedules the next test (if any) Args: result (TestComplete exception) """ real_time = time.time() - self._running_test.start_time sim_time_ns = get_sim_time('ns') - self._running_test.start_sim_time ratio_time = sim_time_ns / real_time self.xunit.add_testcase(name=self._running_test.funcname, classname=self._running_test.module, time=repr(real_time), sim_time_ns=repr(sim_time_ns), ratio_time=repr(ratio_time)) running_test_funcname = self._running_test.funcname # Helper for logging result def _result_was(): result_was = ("%s (result was %s)" % (running_test_funcname, result.__class__.__name__)) return result_was result_pass = True if (isinstance(result, TestSuccess) and not self._running_test.expect_fail and not self._running_test.expect_error): self.log.info("Test Passed: %s" % running_test_funcname) elif (isinstance(result, TestFailure) and self._running_test.expect_fail): self.log.info("Test failed as expected: " + _result_was()) elif (isinstance(result, TestSuccess) and self._running_test.expect_error): self.log.error("Test passed but we expected an error: " + _result_was()) self._add_failure(result) result_pass = False elif isinstance(result, TestSuccess): self.log.error("Test passed but we expected a failure: " + _result_was()) self._add_failure(result) result_pass = False elif isinstance(result, TestError) and self._running_test.expect_error: self.log.info("Test errored as expected: " + _result_was()) elif isinstance(result, SimFailure): if self._running_test.expect_error: self.log.info("Test errored as expected: " + _result_was()) else: self.log.error("Test error has lead to simulator shuttting us " "down") self._add_failure(result) self._store_test_result(self._running_test.module, self._running_test.funcname, False, sim_time_ns, real_time, ratio_time) self.tear_down() return else: self.log.error("Test Failed: " + _result_was()) self._add_failure(result) result_pass = False self._store_test_result(self._running_test.module, self._running_test.funcname, result_pass, sim_time_ns, real_time, ratio_time) self.execute() def execute(self): self._running_test = cocotb.regression.next_test() if self._running_test: start = '' end = '' if self.log.colour: start = ANSI.BLUE_BG + ANSI.BLACK_FG end = ANSI.DEFAULT # Want this to stand out a little bit self.log.info("%sRunning test %d/%d:%s %s" % (start, self.count, self.ntests, end, self._running_test.funcname)) if self.count is 1: test = cocotb.scheduler.add(self._running_test) else: test = cocotb.scheduler.new_test(self._running_test) self.count += 1 else: self.tear_down() def _log_test_summary(self): TEST_FIELD = 'TEST' RESULT_FIELD = 'PASS/FAIL' SIM_FIELD = 'SIM TIME(NS)' REAL_FIELD = 'REAL TIME(S)' RATIO_FIELD = 'RATIO(NS/S)' TEST_FIELD_LEN = max(len(TEST_FIELD),len(max([x['test'] for x in self.test_results],key=len))) RESULT_FIELD_LEN = len(RESULT_FIELD) SIM_FIELD_LEN = len(SIM_FIELD) REAL_FIELD_LEN = len(REAL_FIELD) RATIO_FIELD_LEN = len(RATIO_FIELD) LINE_LEN = 3 + TEST_FIELD_LEN + 2 + RESULT_FIELD_LEN + 2 + SIM_FIELD_LEN + 2 + REAL_FIELD_LEN + 2 + RATIO_FIELD_LEN + 3 LINE_SEP = "*"*LINE_LEN+"\n" summary = "" summary += LINE_SEP summary += "** {a:<{a_len}} {b:^{b_len}} {c:>{c_len}} {d:>{d_len}} {e:>{e_len}} **\n".format(a=TEST_FIELD, a_len=TEST_FIELD_LEN, b=RESULT_FIELD, b_len=RESULT_FIELD_LEN, c=SIM_FIELD, c_len=SIM_FIELD_LEN, d=REAL_FIELD, d_len=REAL_FIELD_LEN, e=RATIO_FIELD, e_len=RATIO_FIELD_LEN) summary += LINE_SEP for result in self.test_results: hilite = '' if result['pass'] is None: pass_fail_str = "N/A" elif result['pass']: pass_fail_str = "PASS" else: pass_fail_str = "FAIL" if self.log.colour: hilite = ANSI.WHITE_FG + ANSI.RED_BG summary += "{start}** {a:<{a_len}} {b:^{b_len}} {c:>{c_len}.2f} {d:>{d_len}.2f} {e:>{e_len}.2f} **\n".format(a=result['test'], a_len=TEST_FIELD_LEN, b=pass_fail_str, b_len=RESULT_FIELD_LEN, c=result['sim'], c_len=SIM_FIELD_LEN-1, d=result['real'], d_len=REAL_FIELD_LEN-1, e=result['ratio'], e_len=RATIO_FIELD_LEN-1, start=hilite) summary += LINE_SEP self.log.info(summary) def _log_sim_summary(self): real_time = time.time() - self.start_time sim_time_ns = get_sim_time('ns') ratio_time = sim_time_ns / real_time summary = "" summary += "*************************************************************************************\n" summary += "** ERRORS : {0:<39}**\n".format(self.failures) summary += "*************************************************************************************\n" summary += "** SIM TIME : {0:<39}**\n".format('{0:.2f} NS'.format(sim_time_ns)) summary += "** REAL TIME : {0:<39}**\n".format('{0:.2f} S'.format(real_time)) summary += "** SIM / REAL TIME : {0:<39}**\n".format('{0:.2f} NS/S'.format(ratio_time)) summary += "*************************************************************************************\n" self.log.info(summary) def _store_test_result(self, module_name, test_name, result_pass, sim_time, real_time, ratio): result = { 'test' : '.'.join([module_name, test_name]), 'pass' : result_pass, 'sim' : sim_time, 'real' : real_time, 'ratio' : ratio} self.test_results.append(result)
def _initialise(self): self.start_time = time.time() self.test_results = [] self.ntests = 0 self.count = 1 self.skipped = 0 self.failures = 0 self.xunit = XUnitReporter() suite_name = os.getenv('RESULT_TESTSUITE') if os.getenv('RESULT_TESTSUITE') else "all" package_name = os.getenv('RESULT_TESTPACKAGE') if os.getenv('RESULT_TESTPACKAGE') else "all" self.xunit.add_testsuite(name=suite_name, tests=repr(self.ntests), package=package_name) if (self._seed is not None): self.xunit.add_property(name="random_seed", value=("%d"%self._seed)) if coverage is not None: self.log.info("Enabling coverage collection of Python code") self._cov = coverage.coverage(branch=True, omit=["*cocotb*"]) self._cov.start() handle = simulator.get_root_handle(self._root_name) self._dut = cocotb.handle.SimHandle(handle) if handle else None if self._dut is None: raise AttributeError("Can not find Root Handle (%s)" % self._root_name) # Auto discovery for module_name in self._modules: try: self.log.debug("Python Path: " + ",".join(sys.path)) self.log.debug("PWD: " + os.getcwd()) module = _my_import(module_name) except Exception as E: self.log.critical("Failed to import module %s: %s", module_name, E) self.log.info("MODULE variable was \"%s\"", ".".join(self._modules)) self.log.info("Traceback: ") self.log.info(traceback.format_exc()) raise if self._functions: # Specific functions specified, don't auto discover for test in self._functions.rsplit(','): if not hasattr(module, test): raise AttributeError("Test %s doesn't exist in %s" % (test, module_name)) self._queue.append(getattr(module, test)(self._dut)) self.ntests += 1 break for thing in vars(module).values(): if hasattr(thing, "im_test"): try: test = thing(self._dut) skip = test.skip except TestError: skip = True self.log.warning("Failed to initialise test %s" % thing.name) if skip: self.log.info("Skipping test %s" % thing.name) self.xunit.add_testcase(name=thing.name, classname=module_name, time="0.0", sim_time_ns="0.0", ratio_time="0.0") self.xunit.add_skipped() self.skipped += 1 self._store_test_result(module_name, thing.name, None, 0.0, 0.0, 0.0) else: self._queue.append(test) self.ntests += 1 self._queue.sort(key=lambda test: "%s.%s" % (test.module, test.funcname)) for valid_tests in self._queue: self.log.info("Found test %s.%s" % (valid_tests.module, valid_tests.funcname)) for module_name in self._hooks: self.log.info("Loading hook from module '"+module_name+"'") module = _my_import(module_name) for thing in vars(module).values(): if hasattr(thing, "im_hook"): try: test = thing(self._dut) except TestError: self.log.warning("Failed to initialize hook %s" % thing.name) else: cocotb.scheduler.add(test)
class RegressionManager(object): """Encapsulates all regression capability into a single place""" def __init__(self, dut, modules, tests=None): """ Args: modules (list): A list of python module names to run Kwargs """ self._queue = [] self._dut = dut self._modules = modules self._functions = tests self._running_test = None self.log = SimLog("cocotb.regression") def initialise(self): self.ntests = 0 self.count = 1 self.skipped = 0 self.failures = 0 self.xunit = XUnitReporter() self.xunit.add_testsuite(name="all", tests=repr(self.ntests), package="all") # Auto discovery for module_name in self._modules: module = _my_import(module_name) if self._functions: # Specific functions specified, don't auto discover for test in self._functions.rsplit(','): if not hasattr(module, test): raise AttributeError("Test %s doesn't exist in %s" % (test, module_name)) self._queue.append(getattr(module, test)(self._dut)) self.ntests += 1 break for thing in vars(module).values(): if hasattr(thing, "im_test"): try: test = thing(self._dut) skip = test.skip except TestError: skip = True self.log.warning("Failed to initialise test %s" % thing.name) if skip: self.log.info("Skipping test %s" % thing.name) self.xunit.add_testcase(name=thing.name, classname=module_name, time="0.0") self.xunit.add_skipped() self.skipped += 1 else: self._queue.append(test) self.ntests += 1 self._queue.sort(key=lambda test: "%s.%s" % (test.module, test.funcname)) for valid_tests in self._queue: self.log.info("Found test %s.%s" % (valid_tests.module, valid_tests.funcname)) def tear_down(self): """It's the end of the world as we know it""" if self.failures: self.log.error("Failed %d out of %d tests (%d skipped)" % (self.failures, self.count -1, self.skipped)) else: self.log.info("Passed %d tests (%d skipped)" % (self.count-1, self.skipped)) self.log.info("Shutting down...") self.xunit.write() simulator.stop_simulator() def next_test(self): """Get the next test to run""" if not self._queue: return None return self._queue.pop(0) def handle_result(self, result): """Handle a test result Dumps result to XML and schedules the next test (if any) Args: result (TestComplete exception) """ self.xunit.add_testcase(name =self._running_test.funcname, classname=self._running_test.module, time=repr(time.time() - self._running_test.start_time) ) if isinstance(result, TestSuccess) and not self._running_test.expect_fail and not self._running_test.expect_error: self.log.info("Test Passed: %s" % self._running_test.funcname) elif isinstance(result, TestFailure) and self._running_test.expect_fail: self.log.info("Test failed as expected: %s (result was %s)" % ( self._running_test.funcname, result.__class__.__name__)) elif isinstance(result, TestSuccess) and self._running_test.expect_error: self.log.error("Test passed but we expected an error: %s (result was %s)" % ( self._running_test.funcname, result.__class__.__name__)) self.xunit.add_failure(stdout=repr(str(result)), stderr="\n".join(self._running_test.error_messages)) self.failures += 1 elif isinstance(result, TestSuccess): self.log.error("Test passed but we expected a failure: %s (result was %s)" % ( self._running_test.funcname, result.__class__.__name__)) self.xunit.add_failure(stdout=repr(str(result)), stderr="\n".join(self._running_test.error_messages)) self.failures += 1 elif isinstance(result, TestError) and self._running_test.expect_error: self.log.info("Test errored as expected: %s (result was %s)" % ( self._running_test.funcname, result.__class__.__name__)) elif isinstance(result, SimFailure): if self._running_test.expect_error: self.log.info("Test errored as expected: %s (result was %s)" % ( self._running_test.funcname, result.__class__.__name__)) else: self.log.error("Test error has lead to simulator shuttting us down") self.failures += 1 self.tear_down() return else: self.log.error("Test Failed: %s (result was %s)" % ( self._running_test.funcname, result.__class__.__name__)) self.xunit.add_failure(stdout=repr(str(result)), stderr="\n".join(self._running_test.error_messages)) self.failures += 1 self.execute() def execute(self): self._running_test = cocotb.regression.next_test() if self._running_test: # Want this to stand out a little bit self.log.info("%sRunning test %d/%d:%s %s" % ( ANSI.BLUE_BG +ANSI.BLACK_FG, self.count, self.ntests, ANSI.DEFAULT_FG + ANSI.DEFAULT_BG, self._running_test.funcname)) if self.count is 1: test = cocotb.scheduler.add(self._running_test) else: test = cocotb.scheduler.new_test(self._running_test) self.count+=1 else: self.tear_down()
class RegressionManager(object): """Encapsulates all regression capability into a single place""" def __init__(self, root_name, modules, tests=None, seed=None, hooks=[]): """ Args: root_name (str): The name of the root handle. modules (list): A list of Python module names to run. tests (list, optional): A list of tests to run. Defaults to ``None``, meaning all discovered tests will be run. seed (int, optional): The seed for the random number generator to use. Defaults to ``None``. hooks (list, optional): A list of hook modules to import. Defaults to the empty list. """ self._queue = [] self._root_name = root_name self._dut = None self._modules = modules self._functions = tests self._running_test = None self._cov = None self.log = SimLog("cocotb.regression") self._seed = seed self._hooks = hooks def initialise(self): self.start_time = time.time() self.test_results = [] self.ntests = 0 self.count = 1 self.skipped = 0 self.failures = 0 # Setup XUnit ################### results_filename = os.getenv('COCOTB_RESULTS_FILE', "results.xml") suite_name = os.getenv('RESULT_TESTSUITE', "all") package_name = os.getenv('RESULT_TESTPACKAGE', "all") self.xunit = XUnitReporter(filename=results_filename) self.xunit.add_testsuite(name=suite_name, tests=repr(self.ntests), package=package_name) if (self._seed is not None): self.xunit.add_property(name="random_seed", value=("%d" % self._seed)) # Setup Coverage #################### if coverage is not None: self.log.info("Enabling coverage collection of Python code") self._cov = coverage.coverage(branch=True, omit=["*cocotb*"]) self._cov.start() # Setup DUT object ####################### handle = simulator.get_root_handle(self._root_name) self._dut = cocotb.handle.SimHandle(handle) if handle else None if self._dut is None: raise AttributeError("Can not find Root Handle (%s)" % self._root_name) # Test Discovery #################### for module_name in self._modules: try: self.log.debug("Python Path: " + ",".join(sys.path)) self.log.debug("PWD: " + os.getcwd()) module = _my_import(module_name) except Exception as E: self.log.critical("Failed to import module %s: %s", module_name, E) self.log.info("MODULE variable was \"%s\"", ".".join(self._modules)) self.log.info("Traceback: ") self.log.info(traceback.format_exc()) raise if self._functions: # Specific functions specified, don't auto-discover for test in self._functions.rsplit(','): try: _test = getattr(module, test) except AttributeError: self.log.error( "Requested test %s wasn't found in module %s", test, module_name) err = AttributeError("Test %s doesn't exist in %s" % (test, module_name)) _py_compat.raise_from(err, None) # discard nested traceback if not hasattr(_test, "im_test"): self.log.error( "Requested %s from module %s isn't a cocotb.test decorated coroutine", test, module_name) raise ImportError("Failed to find requested test %s" % test) self._init_test(_test) # only look in first module for all functions and don't complain if all functions are not found break # auto-discover for thing in vars(module).values(): if hasattr(thing, "im_test"): self._init_test(thing) self._queue.sort(key=lambda test: (test.stage, test._id)) for valid_tests in self._queue: self.log.info("Found test %s.%s" % (valid_tests.module, valid_tests.funcname)) # Process Hooks ################### for module_name in self._hooks: self.log.info("Loading hook from module '" + module_name + "'") module = _my_import(module_name) for thing in vars(module).values(): if hasattr(thing, "im_hook"): try: test = thing(self._dut) except Exception: self.log.warning("Failed to initialize hook %s" % thing.name, exc_info=True) else: cocotb.scheduler.add(test) def tear_down(self): # fail remaining tests while True: test = self.next_test() if test is None: break self.xunit.add_testcase(name=test.funcname, classname=test.module, time=repr(0), sim_time_ns=repr(0), ratio_time=repr(0)) result_pass, _ = self._score_test( test, cocotb.outcomes.Error(SimFailure())) self._store_test_result(test.__module__, test.__name__, result_pass, 0, 0, 0) if not result_pass: self.xunit.add_failure() self.failures += 1 # Write out final log messages if self.failures: self.log.error("Failed %d out of %d tests (%d skipped)" % (self.failures, self.count - 1, self.skipped)) else: self.log.info("Passed %d tests (%d skipped)" % (self.count - 1, self.skipped)) if len(self.test_results) > 0: self._log_test_summary() self._log_sim_summary() self.log.info("Shutting down...") # Generate output reports self.xunit.write() if self._cov: self._cov.stop() self.log.info("Writing coverage data") self._cov.save() self._cov.html_report() # Setup simulator finalization simulator.stop_simulator() def next_test(self): """Get the next test to run""" if not self._queue: return None return self._queue.pop(0) def handle_result(self, test): """Handle a test completing. Dump result to XML and schedule the next test (if any). Entered by the scheduler. Args: test: The test that completed """ assert test is self._running_test real_time = time.time() - test.start_time sim_time_ns = get_sim_time('ns') - test.start_sim_time ratio_time = self._safe_divide(sim_time_ns, real_time) self.xunit.add_testcase(name=test.funcname, classname=test.module, time=repr(real_time), sim_time_ns=repr(sim_time_ns), ratio_time=repr(ratio_time)) # score test result_pass, sim_failed = self._score_test(test, test._outcome) # stop capturing log output cocotb.log.removeHandler(test.handler) # Save results self._store_test_result(test.__module__, test.__name__, result_pass, sim_time_ns, real_time, ratio_time) if not result_pass: self.xunit.add_failure() self.failures += 1 # Fail if required if sim_failed: self.tear_down() return self.execute() def _init_test(self, test_func): """ Initializes a test. Records outcome if the initialization fails. Records skip if the test is skipped. Saves the initialized test if it successfully initializes. """ test_init_outcome = cocotb.outcomes.capture(test_func, self._dut) if isinstance(test_init_outcome, cocotb.outcomes.Error): self.log.error("Failed to initialize test %s" % test_func.name, exc_info=True) self.xunit.add_testcase(name=test_func.name, classname=test_func.__module__, time="0.0", sim_time_ns="0.0", ratio_time="0.0") result_pass, sim_failed = self._score_test(test_func, test_init_outcome) # Save results self._store_test_result(test_func.__module__, test_func.__name__, result_pass, 0.0, 0.0, 0.0) if not result_pass: self.xunit.add_failure() self.failures += 1 # Fail if required if sim_failed: self.tear_down() raise SimFailure( "Test initialization caused a simulator failure. Shutting down." ) else: test = test_init_outcome.get() if test.skip: self.log.info("Skipping test %s" % test_func.name) self.xunit.add_testcase(name=test_func.name, classname=test.module, time="0.0", sim_time_ns="0.0", ratio_time="0.0") self.xunit.add_skipped() self.skipped += 1 self._store_test_result(test.module, test_func.name, None, 0.0, 0.0, 0.0) else: self._queue.append(test) self.ntests += 1 def _score_test(self, test, outcome): """ Given a test and the test's outcome, determine if the test met expectations and log pertinent information """ # Helper for logging result def _result_was(): result_was = ("{} (result was {})".format( test.__name__, result.__class__.__name__)) return result_was # scoring outcomes result_pass = True sim_failed = False try: outcome.get() except Exception as e: result = remove_traceback_frames(e, ['_score_test', 'get']) else: result = TestSuccess() if (isinstance(result, TestSuccess) and not test.expect_fail and not test.expect_error): self.log.info("Test Passed: %s" % test.__name__) elif (isinstance(result, AssertionError) and test.expect_fail): self.log.info("Test failed as expected: " + _result_was()) elif (isinstance(result, TestSuccess) and test.expect_error): self.log.error("Test passed but we expected an error: " + _result_was()) result_pass = False elif isinstance(result, TestSuccess): self.log.error("Test passed but we expected a failure: " + _result_was()) result_pass = False elif isinstance(result, SimFailure): if isinstance(result, test.expect_error): self.log.info("Test errored as expected: " + _result_was()) else: self.log.error( "Test error has lead to simulator shutting us " "down", exc_info=result) result_pass = False # whether we expected it or not, the simulation has failed unrecoverably sim_failed = True elif test.expect_error: if isinstance(result, test.expect_error): self.log.info("Test errored as expected: " + _result_was()) else: self.log.error("Test errored with unexpected type: " + _result_was(), exc_info=result) result_pass = False else: self.log.error("Test Failed: " + _result_was(), exc_info=result) result_pass = False if _pdb_on_exception: pdb.post_mortem(result.__traceback__) return result_pass, sim_failed def execute(self): self._running_test = cocotb.regression_manager.next_test() if self._running_test: start = '' end = '' if want_color_output(): start = ANSI.COLOR_TEST end = ANSI.COLOR_DEFAULT # Want this to stand out a little bit self.log.info("%sRunning test %d/%d:%s %s" % (start, self.count, self.ntests, end, self._running_test.funcname)) # start capturing log output cocotb.log.addHandler(self._running_test.handler) cocotb.scheduler.add_test(self._running_test) self.count += 1 else: self.tear_down() def _log_test_summary(self): TEST_FIELD = 'TEST' RESULT_FIELD = 'PASS/FAIL' SIM_FIELD = 'SIM TIME(NS)' REAL_FIELD = 'REAL TIME(S)' RATIO_FIELD = 'RATIO(NS/S)' TEST_FIELD_LEN = max( len(TEST_FIELD), len(max([x['test'] for x in self.test_results], key=len))) RESULT_FIELD_LEN = len(RESULT_FIELD) SIM_FIELD_LEN = len(SIM_FIELD) REAL_FIELD_LEN = len(REAL_FIELD) RATIO_FIELD_LEN = len(RATIO_FIELD) LINE_LEN = 3 + TEST_FIELD_LEN + 2 + RESULT_FIELD_LEN + 2 + SIM_FIELD_LEN + 2 + REAL_FIELD_LEN + 2 + RATIO_FIELD_LEN + 3 LINE_SEP = "*" * LINE_LEN + "\n" summary = "" summary += LINE_SEP summary += "** {a:<{a_len}} {b:^{b_len}} {c:>{c_len}} {d:>{d_len}} {e:>{e_len}} **\n".format( a=TEST_FIELD, a_len=TEST_FIELD_LEN, b=RESULT_FIELD, b_len=RESULT_FIELD_LEN, c=SIM_FIELD, c_len=SIM_FIELD_LEN, d=REAL_FIELD, d_len=REAL_FIELD_LEN, e=RATIO_FIELD, e_len=RATIO_FIELD_LEN) summary += LINE_SEP for result in self.test_results: hilite = '' if result['pass'] is None: pass_fail_str = "N/A" elif result['pass']: pass_fail_str = "PASS" else: pass_fail_str = "FAIL" if want_color_output(): hilite = ANSI.COLOR_HILITE_SUMMARY summary += "{start}** {a:<{a_len}} {b:^{b_len}} {c:>{c_len}.2f} {d:>{d_len}.2f} {e:>{e_len}.2f} **\n".format( a=result['test'], a_len=TEST_FIELD_LEN, b=pass_fail_str, b_len=RESULT_FIELD_LEN, c=result['sim'], c_len=SIM_FIELD_LEN - 1, d=result['real'], d_len=REAL_FIELD_LEN - 1, e=result['ratio'], e_len=RATIO_FIELD_LEN - 1, start=hilite) summary += LINE_SEP self.log.info(summary) def _log_sim_summary(self): real_time = time.time() - self.start_time sim_time_ns = get_sim_time('ns') ratio_time = self._safe_divide(sim_time_ns, real_time) summary = "" summary += "*************************************************************************************\n" summary += "** ERRORS : {0:<39}**\n".format( self.failures) summary += "*************************************************************************************\n" summary += "** SIM TIME : {0:<39}**\n".format( '{0:.2f} NS'.format(sim_time_ns)) summary += "** REAL TIME : {0:<39}**\n".format( '{0:.2f} S'.format(real_time)) summary += "** SIM / REAL TIME : {0:<39}**\n".format( '{0:.2f} NS/S'.format(ratio_time)) summary += "*************************************************************************************\n" self.log.info(summary) @staticmethod def _safe_divide(a, b): try: return a / b except ZeroDivisionError: if a == 0: return float('nan') else: return float('inf') def _store_test_result(self, module_name, test_name, result_pass, sim_time, real_time, ratio): result = { 'test': '.'.join([module_name, test_name]), 'pass': result_pass, 'sim': sim_time, 'real': real_time, 'ratio': ratio } self.test_results.append(result)
def initialise(self): self.start_time = time.time() self.test_results = [] self.ntests = 0 self.count = 1 self.skipped = 0 self.failures = 0 # Setup XUnit ################### results_filename = os.getenv('COCOTB_RESULTS_FILE', "results.xml") suite_name = os.getenv('RESULT_TESTSUITE', "all") package_name = os.getenv('RESULT_TESTPACKAGE', "all") self.xunit = XUnitReporter(filename=results_filename) self.xunit.add_testsuite(name=suite_name, tests=repr(self.ntests), package=package_name) if (self._seed is not None): self.xunit.add_property(name="random_seed", value=("%d" % self._seed)) # Setup Coverage #################### if coverage is not None: self.log.info("Enabling coverage collection of Python code") self._cov = coverage.coverage(branch=True, omit=["*cocotb*"]) self._cov.start() # Setup DUT object ####################### handle = simulator.get_root_handle(self._root_name) self._dut = cocotb.handle.SimHandle(handle) if handle else None if self._dut is None: raise AttributeError("Can not find Root Handle (%s)" % self._root_name) # Test Discovery #################### for module_name in self._modules: try: self.log.debug("Python Path: " + ",".join(sys.path)) self.log.debug("PWD: " + os.getcwd()) module = _my_import(module_name) except Exception as E: self.log.critical("Failed to import module %s: %s", module_name, E) self.log.info("MODULE variable was \"%s\"", ".".join(self._modules)) self.log.info("Traceback: ") self.log.info(traceback.format_exc()) raise if self._functions: # Specific functions specified, don't auto-discover for test in self._functions.rsplit(','): try: _test = getattr(module, test) except AttributeError: self.log.error( "Requested test %s wasn't found in module %s", test, module_name) err = AttributeError("Test %s doesn't exist in %s" % (test, module_name)) _py_compat.raise_from(err, None) # discard nested traceback if not hasattr(_test, "im_test"): self.log.error( "Requested %s from module %s isn't a cocotb.test decorated coroutine", test, module_name) raise ImportError("Failed to find requested test %s" % test) self._init_test(_test) # only look in first module for all functions and don't complain if all functions are not found break # auto-discover for thing in vars(module).values(): if hasattr(thing, "im_test"): self._init_test(thing) self._queue.sort(key=lambda test: (test.stage, test._id)) for valid_tests in self._queue: self.log.info("Found test %s.%s" % (valid_tests.module, valid_tests.funcname)) # Process Hooks ################### for module_name in self._hooks: self.log.info("Loading hook from module '" + module_name + "'") module = _my_import(module_name) for thing in vars(module).values(): if hasattr(thing, "im_hook"): try: test = thing(self._dut) except Exception: self.log.warning("Failed to initialize hook %s" % thing.name, exc_info=True) else: cocotb.scheduler.add(test)
def initialise(self): self.start_time = time.time() self.test_results = [] self.ntests = 0 self.count = 1 self.skipped = 0 self.failures = 0 self.xunit = XUnitReporter() suite_name = os.getenv('RESULT_TESTSUITE') if os.getenv( 'RESULT_TESTSUITE') else "all" package_name = os.getenv('RESULT_TESTPACKAGE') if os.getenv( 'RESULT_TESTPACKAGE') else "all" self.xunit.add_testsuite(name=suite_name, tests=repr(self.ntests), package=package_name) if (self._seed is not None): self.xunit.add_property(name="random_seed", value=("%d" % self._seed)) if coverage is not None: self.log.info("Enabling coverage collection of Python code") self._cov = coverage.coverage(branch=True, omit=["*cocotb*"]) self._cov.start() handle = simulator.get_root_handle(self._root_name) self._dut = cocotb.handle.SimHandle(handle) if handle else None if self._dut is None: raise AttributeError("Can not find Root Handle (%s)" % self._root_name) # Auto discovery for module_name in self._modules: try: self.log.debug("Python Path: " + ",".join(sys.path)) self.log.debug("PWD: " + os.getcwd()) module = _my_import(module_name) except Exception as E: self.log.critical("Failed to import module %s: %s", module_name, E) self.log.info("MODULE variable was \"%s\"", ".".join(self._modules)) self.log.info("Traceback: ") self.log.info(traceback.format_exc()) raise if self._functions: # Specific functions specified, don't auto discover for test in self._functions.rsplit(','): try: _test = getattr(module, test) except AttributeError: self.log.error( "Requested test %s wasn't found in module %s", test, module_name) err = AttributeError("Test %s doesn't exist in %s" % (test, module_name)) raise_from(err, None) # discard nested traceback if not hasattr(_test, "im_test"): self.log.error( "Requested %s from module %s isn't a cocotb.test decorated coroutine", test, module_name) raise ImportError("Failed to find requested test %s" % test) self._queue.append(_test(self._dut)) self.ntests += 1 break for thing in vars(module).values(): if hasattr(thing, "im_test"): try: test = thing(self._dut) skip = test.skip except TestError: skip = True self.log.warning("Failed to initialize test %s" % thing.name) if skip: self.log.info("Skipping test %s" % thing.name) self.xunit.add_testcase(name=thing.name, classname=module_name, time="0.0", sim_time_ns="0.0", ratio_time="0.0") self.xunit.add_skipped() self.skipped += 1 self._store_test_result(module_name, thing.name, None, 0.0, 0.0, 0.0) else: self._queue.append(test) self.ntests += 1 self._queue.sort(key=lambda test: test.sort_name()) for valid_tests in self._queue: self.log.info("Found test %s.%s" % (valid_tests.module, valid_tests.funcname)) for module_name in self._hooks: self.log.info("Loading hook from module '" + module_name + "'") module = _my_import(module_name) for thing in vars(module).values(): if hasattr(thing, "im_hook"): try: test = thing(self._dut) except TestError: self.log.warning("Failed to initialize hook %s" % thing.name) else: cocotb.scheduler.add(test)
class RegressionManager(object): """Encapsulates all regression capability into a single place""" def __init__(self, root_name, modules, tests=None, seed=None, hooks=[]): """ Args: root_name (str): The name of the root handle. modules (list): A list of Python module names to run. tests (list, optional): A list of tests to run. Defaults to ``None``, meaning all discovered tests will be run. seed (int, optional): The seed for the random number generator to use. Defaults to ``None``. hooks (list, optional): A list of hook modules to import. Defaults to the empty list. """ self._queue = [] self._root_name = root_name self._dut = None self._modules = modules self._functions = tests self._running_test = None self._cov = None self.log = SimLog("cocotb.regression") self._seed = seed self._hooks = hooks def initialise(self): self.start_time = time.time() self.test_results = [] self.ntests = 0 self.count = 1 self.skipped = 0 self.failures = 0 self.xunit = XUnitReporter() suite_name = os.getenv('RESULT_TESTSUITE') if os.getenv( 'RESULT_TESTSUITE') else "all" package_name = os.getenv('RESULT_TESTPACKAGE') if os.getenv( 'RESULT_TESTPACKAGE') else "all" self.xunit.add_testsuite(name=suite_name, tests=repr(self.ntests), package=package_name) if (self._seed is not None): self.xunit.add_property(name="random_seed", value=("%d" % self._seed)) if coverage is not None: self.log.info("Enabling coverage collection of Python code") self._cov = coverage.coverage(branch=True, omit=["*cocotb*"]) self._cov.start() handle = simulator.get_root_handle(self._root_name) self._dut = cocotb.handle.SimHandle(handle) if handle else None if self._dut is None: raise AttributeError("Can not find Root Handle (%s)" % self._root_name) # Auto discovery for module_name in self._modules: try: self.log.debug("Python Path: " + ",".join(sys.path)) self.log.debug("PWD: " + os.getcwd()) module = _my_import(module_name) except Exception as E: self.log.critical("Failed to import module %s: %s", module_name, E) self.log.info("MODULE variable was \"%s\"", ".".join(self._modules)) self.log.info("Traceback: ") self.log.info(traceback.format_exc()) raise if self._functions: # Specific functions specified, don't auto discover for test in self._functions.rsplit(','): try: _test = getattr(module, test) except AttributeError: self.log.error( "Requested test %s wasn't found in module %s", test, module_name) err = AttributeError("Test %s doesn't exist in %s" % (test, module_name)) raise_from(err, None) # discard nested traceback if not hasattr(_test, "im_test"): self.log.error( "Requested %s from module %s isn't a cocotb.test decorated coroutine", test, module_name) raise ImportError("Failed to find requested test %s" % test) self._queue.append(_test(self._dut)) self.ntests += 1 break for thing in vars(module).values(): if hasattr(thing, "im_test"): try: test = thing(self._dut) skip = test.skip except TestError: skip = True self.log.warning("Failed to initialize test %s" % thing.name) if skip: self.log.info("Skipping test %s" % thing.name) self.xunit.add_testcase(name=thing.name, classname=module_name, time="0.0", sim_time_ns="0.0", ratio_time="0.0") self.xunit.add_skipped() self.skipped += 1 self._store_test_result(module_name, thing.name, None, 0.0, 0.0, 0.0) else: self._queue.append(test) self.ntests += 1 self._queue.sort(key=lambda test: test.sort_name()) for valid_tests in self._queue: self.log.info("Found test %s.%s" % (valid_tests.module, valid_tests.funcname)) for module_name in self._hooks: self.log.info("Loading hook from module '" + module_name + "'") module = _my_import(module_name) for thing in vars(module).values(): if hasattr(thing, "im_hook"): try: test = thing(self._dut) except TestError: self.log.warning("Failed to initialize hook %s" % thing.name) else: cocotb.scheduler.add(test) def tear_down(self): """It's the end of the world as we know it""" if self.failures: self.log.error("Failed %d out of %d tests (%d skipped)" % (self.failures, self.count - 1, self.skipped)) else: self.log.info("Passed %d tests (%d skipped)" % (self.count - 1, self.skipped)) if self._cov: self._cov.stop() self.log.info("Writing coverage data") self._cov.save() self._cov.html_report() if len(self.test_results) > 0: self._log_test_summary() self._log_sim_summary() self.log.info("Shutting down...") self.xunit.write() simulator.stop_simulator() def next_test(self): """Get the next test to run""" if not self._queue: return None return self._queue.pop(0) def _add_failure(self, result): self.xunit.add_failure( stdout=repr(str(result)), stderr="\n".join(self._running_test.error_messages), message="Test failed with random_seed={}".format(self._seed)) self.failures += 1 def handle_result(self, result): """Handle a test result. Dump result to XML and schedule the next test (if any). Args: result: The sub-exception of TestComplete to raise. """ real_time = time.time() - self._running_test.start_time sim_time_ns = get_sim_time('ns') - self._running_test.start_sim_time try: ratio_time = sim_time_ns / real_time except ZeroDivisionError: if round(sim_time_ns, 2) == 0: ratio_time = float('nan') else: ratio_time = float('inf') self.xunit.add_testcase(name=self._running_test.funcname, classname=self._running_test.module, time=repr(real_time), sim_time_ns=repr(sim_time_ns), ratio_time=repr(ratio_time)) running_test_funcname = self._running_test.funcname # Helper for logging result def _result_was(): result_was = ("%s (result was %s)" % (running_test_funcname, result.__class__.__name__)) return result_was result_pass = True if (isinstance(result, TestSuccess) and not self._running_test.expect_fail and not self._running_test.expect_error): self.log.info("Test Passed: %s" % running_test_funcname) elif (isinstance(result, TestFailure) and self._running_test.expect_fail): self.log.info("Test failed as expected: " + _result_was()) elif (isinstance(result, TestSuccess) and self._running_test.expect_error): self.log.error("Test passed but we expected an error: " + _result_was()) self._add_failure(result) result_pass = False elif isinstance(result, TestSuccess): self.log.error("Test passed but we expected a failure: " + _result_was()) self._add_failure(result) result_pass = False elif isinstance(result, TestError) and self._running_test.expect_error: self.log.info("Test errored as expected: " + _result_was()) elif isinstance(result, SimFailure): if self._running_test.expect_error: self.log.info("Test errored as expected: " + _result_was()) else: self.log.error("Test error has lead to simulator shutting us " "down") self._add_failure(result) self._store_test_result(self._running_test.module, self._running_test.funcname, False, sim_time_ns, real_time, ratio_time) self.tear_down() return else: self.log.error("Test Failed: " + _result_was()) self._add_failure(result) result_pass = False self._store_test_result(self._running_test.module, self._running_test.funcname, result_pass, sim_time_ns, real_time, ratio_time) self.execute() def execute(self): self._running_test = cocotb.regression_manager.next_test() if self._running_test: start = '' end = '' if self.log.colour: start = ANSI.COLOR_TEST end = ANSI.COLOR_DEFAULT # Want this to stand out a little bit self.log.info("%sRunning test %d/%d:%s %s" % (start, self.count, self.ntests, end, self._running_test.funcname)) if self.count is 1: test = cocotb.scheduler.add(self._running_test) else: test = cocotb.scheduler.new_test(self._running_test) self.count += 1 else: self.tear_down() def _log_test_summary(self): TEST_FIELD = 'TEST' RESULT_FIELD = 'PASS/FAIL' SIM_FIELD = 'SIM TIME(NS)' REAL_FIELD = 'REAL TIME(S)' RATIO_FIELD = 'RATIO(NS/S)' TEST_FIELD_LEN = max( len(TEST_FIELD), len(max([x['test'] for x in self.test_results], key=len))) RESULT_FIELD_LEN = len(RESULT_FIELD) SIM_FIELD_LEN = len(SIM_FIELD) REAL_FIELD_LEN = len(REAL_FIELD) RATIO_FIELD_LEN = len(RATIO_FIELD) LINE_LEN = 3 + TEST_FIELD_LEN + 2 + RESULT_FIELD_LEN + 2 + SIM_FIELD_LEN + 2 + REAL_FIELD_LEN + 2 + RATIO_FIELD_LEN + 3 LINE_SEP = "*" * LINE_LEN + "\n" summary = "" summary += LINE_SEP summary += "** {a:<{a_len}} {b:^{b_len}} {c:>{c_len}} {d:>{d_len}} {e:>{e_len}} **\n".format( a=TEST_FIELD, a_len=TEST_FIELD_LEN, b=RESULT_FIELD, b_len=RESULT_FIELD_LEN, c=SIM_FIELD, c_len=SIM_FIELD_LEN, d=REAL_FIELD, d_len=REAL_FIELD_LEN, e=RATIO_FIELD, e_len=RATIO_FIELD_LEN) summary += LINE_SEP for result in self.test_results: hilite = '' if result['pass'] is None: pass_fail_str = "N/A" elif result['pass']: pass_fail_str = "PASS" else: pass_fail_str = "FAIL" if self.log.colour: hilite = ANSI.COLOR_HILITE_SUMMARY summary += "{start}** {a:<{a_len}} {b:^{b_len}} {c:>{c_len}.2f} {d:>{d_len}.2f} {e:>{e_len}.2f} **\n".format( a=result['test'], a_len=TEST_FIELD_LEN, b=pass_fail_str, b_len=RESULT_FIELD_LEN, c=result['sim'], c_len=SIM_FIELD_LEN - 1, d=result['real'], d_len=REAL_FIELD_LEN - 1, e=result['ratio'], e_len=RATIO_FIELD_LEN - 1, start=hilite) summary += LINE_SEP self.log.info(summary) def _log_sim_summary(self): real_time = time.time() - self.start_time sim_time_ns = get_sim_time('ns') ratio_time = sim_time_ns / real_time summary = "" summary += "*************************************************************************************\n" summary += "** ERRORS : {0:<39}**\n".format( self.failures) summary += "*************************************************************************************\n" summary += "** SIM TIME : {0:<39}**\n".format( '{0:.2f} NS'.format(sim_time_ns)) summary += "** REAL TIME : {0:<39}**\n".format( '{0:.2f} S'.format(real_time)) summary += "** SIM / REAL TIME : {0:<39}**\n".format( '{0:.2f} NS/S'.format(ratio_time)) summary += "*************************************************************************************\n" self.log.info(summary) def _store_test_result(self, module_name, test_name, result_pass, sim_time, real_time, ratio): result = { 'test': '.'.join([module_name, test_name]), 'pass': result_pass, 'sim': sim_time, 'real': real_time, 'ratio': ratio } self.test_results.append(result)
class RegressionManager: """Encapsulates all regression capability into a single place""" def __init__(self, dut: SimHandle, tests: Iterable[Test], hooks: Iterable[Hook]): """ Args: dut (SimHandle): The root handle to pass into test functions. tests (Iterable[Test]): tests to run hooks (Iterable[Hook]): hooks to tun """ self._dut = dut self._test_task = None self._cov = None self.log = _logger self.start_time = time.time() self.test_results = [] self.count = 0 self.skipped = 0 self.failures = 0 self._tearing_down = False # Setup XUnit ################### results_filename = os.getenv('COCOTB_RESULTS_FILE', "results.xml") suite_name = os.getenv('RESULT_TESTSUITE', "all") package_name = os.getenv('RESULT_TESTPACKAGE', "all") self.xunit = XUnitReporter(filename=results_filename) self.xunit.add_testsuite(name=suite_name, package=package_name) self.xunit.add_property(name="random_seed", value=str(cocotb.RANDOM_SEED)) # Setup Coverage #################### if coverage is not None: self.log.info("Enabling coverage collection of Python code") # Exclude cocotb itself from coverage collection. cocotb_package_dir = os.path.dirname(__file__) self._cov = coverage.coverage(branch=True, omit=["{}/*".format(cocotb_package_dir)]) self._cov.start() # Test Discovery #################### self._queue = [] for test in tests: self.log.info("Found test {}.{}".format(test.__module__, test.__qualname__)) self._queue.append(test) self.ntests = len(self._queue) if not self._queue: self.log.warning("No tests were discovered") self._queue.sort(key=lambda test: (test.stage, test._id)) # Process Hooks ################### for hook in hooks: self.log.info("Found hook {}.{}".format(hook.__module__, hook.__qualname__)) self._init_hook(hook) @classmethod def from_discovery(cls, dut: SimHandle): """ Obtains the test and hook lists by discovery. See :envvar:`MODULE` and :envvar:`TESTCASE` for details on how tests are discovered. Args: dut (SimHandle): The root handle to pass into test functions. """ tests = cls._discover_tests() hooks = cls._discover_hooks() return cls(dut, tests, hooks) @staticmethod def _discover_tests() -> Iterable[Test]: """ Discovers tests in files automatically. See :envvar:`MODULE` and :envvar:`TESTCASE` for details on how tests are discovered. """ module_str = os.getenv('MODULE') test_str = os.getenv('TESTCASE') if module_str is None: raise ValueError("Environment variable MODULE, which defines the module(s) to execute, is not defined.") modules = [s.strip() for s in module_str.split(',') if s.strip()] for module_name in modules: try: _logger.debug("Python Path: " + ",".join(sys.path)) _logger.debug("PWD: " + os.getcwd()) module = _my_import(module_name) except Exception as E: _logger.critical("Failed to import module %s: %s", module_name, E) _logger.info("MODULE variable was \"%s\"", ".".join(modules)) _logger.info("Traceback: ") _logger.info(traceback.format_exc()) raise if test_str: # Specific functions specified, don't auto-discover for test_name in test_str.rsplit(','): try: test = getattr(module, test_name) except AttributeError: _logger.error("Requested test %s wasn't found in module %s", test_name, module_name) err = AttributeError("Test %s doesn't exist in %s" % (test_name, module_name)) raise err from None # discard nested traceback if not isinstance(test, Test): _logger.error("Requested %s from module %s isn't a cocotb.test decorated coroutine", test_name, module_name) raise ImportError("Failed to find requested test %s" % test_name) # If we request a test manually, it should be run even if skip=True is set. test.skip = False yield test # only look in first module for all functions and don't complain if all functions are not found break # auto-discover for thing in vars(module).values(): if isinstance(thing, Test): yield thing @staticmethod def _discover_hooks() -> Iterable[Hook]: """ Discovers hooks automatically. See :envvar:`COCOTB_HOOKS` for details on how hooks are discovered. """ hooks_str = os.getenv('COCOTB_HOOKS', '') hooks = [s.strip() for s in hooks_str.split(',') if s.strip()] for module_name in hooks: _logger.info("Loading hook from module '" + module_name + "'") module = _my_import(module_name) for thing in vars(module).values(): if hasattr(thing, "im_hook"): yield thing def _init_hook(self, hook: Hook) -> Optional[RunningTask]: try: test = hook(self._dut) except Exception: self.log.warning("Failed to initialize hook %s" % hook.name, exc_info=True) else: return cocotb.scheduler.add(test) def tear_down(self) -> None: # prevent re-entering the tear down procedure if not self._tearing_down: self._tearing_down = True else: return # fail remaining tests while True: test = self.next_test() if test is None: break self._record_result( test=test, outcome=Error(SimFailure), wall_time_s=0, sim_time_ns=0) # Write out final log messages self._log_test_summary() self._log_sim_summary() self.log.info("Shutting down...") # Generate output reports self.xunit.write() if self._cov: self._cov.stop() self.log.info("Writing coverage data") self._cov.save() self._cov.html_report() if cocotb._library_coverage is not None: # TODO: move this once we have normal shutdown behavior to _sim_event cocotb._library_coverage.stop() cocotb._library_coverage.save() # Setup simulator finalization simulator.stop_simulator() def next_test(self) -> Optional[Test]: """Get the next test to run""" if not self._queue: return None self.count += 1 return self._queue.pop(0) def handle_result(self, test: RunningTask) -> None: """Handle a test completing. Dump result to XML and schedule the next test (if any). Entered by the scheduler. Args: test: The test that completed """ assert test is self._test_task real_time = time.time() - self._test_start_time sim_time_ns = get_sim_time('ns') - self._test_start_sim_time # stop capturing log output cocotb.log.removeHandler(test.handler) self._record_result( test=self._test, outcome=self._test_task._outcome, wall_time_s=real_time, sim_time_ns=sim_time_ns) self.execute() def _init_test(self, test: Test) -> Optional[RunningTask]: """ Initializes a test. Records outcome if the initialization fails. Records skip if the test is skipped. Saves the initialized test if it successfully initializes. """ if test.skip: hilight_start = ANSI.COLOR_TEST if want_color_output() else '' hilight_end = ANSI.COLOR_DEFAULT if want_color_output() else '' # Want this to stand out a little bit self.log.info("{}Skipping test {}/{}:{} {}".format( hilight_start, self.count, self.ntests, hilight_end, test.__qualname__)) self._record_result(test, None, 0, 0) return test_init_outcome = cocotb.outcomes.capture(test, self._dut) if isinstance(test_init_outcome, cocotb.outcomes.Error): self.log.error("Failed to initialize test %s" % test.__qualname__, exc_info=test_init_outcome.error) self._record_result(test, test_init_outcome, 0, 0) return test = test_init_outcome.get() return test def _score_test(self, test: Test, outcome: Outcome) -> Tuple[bool, bool]: """ Given a test and the test's outcome, determine if the test met expectations and log pertinent information """ # Helper for logging result def _result_was(): result_was = ("{} (result was {})".format (test.__qualname__, type(result).__qualname__)) return result_was # scoring outcomes result_pass = True sim_failed = False try: outcome.get() except Exception as e: result = remove_traceback_frames(e, ['_score_test', 'get']) else: result = TestSuccess() if (isinstance(result, TestSuccess) and not test.expect_fail and not test.expect_error): self.log.info("Test Passed: %s" % test.__qualname__) elif (isinstance(result, AssertionError) and test.expect_fail): self.log.info("Test failed as expected: " + _result_was()) elif (isinstance(result, TestSuccess) and test.expect_error): self.log.error("Test passed but we expected an error: " + _result_was()) result_pass = False elif isinstance(result, TestSuccess): self.log.error("Test passed but we expected a failure: " + _result_was()) result_pass = False elif isinstance(result, SimFailure): if isinstance(result, test.expect_error): self.log.info("Test errored as expected: " + _result_was()) else: self.log.error("Test error has lead to simulator shutting us " "down", exc_info=result) result_pass = False # whether we expected it or not, the simulation has failed unrecoverably sim_failed = True elif test.expect_error: if isinstance(result, test.expect_error): self.log.info("Test errored as expected: " + _result_was()) else: self.log.error("Test errored with unexpected type: " + _result_was(), exc_info=result) result_pass = False else: self.log.error("Test Failed: " + _result_was(), exc_info=result) result_pass = False if _pdb_on_exception: pdb.post_mortem(result.__traceback__) return result_pass, sim_failed def _record_result( self, test: Test, outcome: Optional[Outcome], wall_time_s: float, sim_time_ns: float ) -> None: ratio_time = self._safe_divide(sim_time_ns, wall_time_s) self.xunit.add_testcase(name=test.__qualname__, classname=test.__module__, time=repr(wall_time_s), sim_time_ns=repr(sim_time_ns), ratio_time=repr(ratio_time)) if outcome is None: # skipped test_pass, sim_failed = None, False self.xunit.add_skipped() self.skipped += 1 else: test_pass, sim_failed = self._score_test(test, outcome) if not test_pass: self.xunit.add_failure() self.failures += 1 self.test_results.append({ 'test': '.'.join([test.__module__, test.__qualname__]), 'pass': test_pass, 'sim': sim_time_ns, 'real': wall_time_s, 'ratio': ratio_time}) if sim_failed: self.tear_down() return def execute(self) -> None: while True: self._test = self.next_test() if self._test is None: return self.tear_down() self._test_task = self._init_test(self._test) if self._test_task: return self._start_test() def _start_test(self) -> None: start = '' end = '' if want_color_output(): start = ANSI.COLOR_TEST end = ANSI.COLOR_DEFAULT # Want this to stand out a little bit self.log.info("%sRunning test %d/%d:%s %s" % (start, self.count, self.ntests, end, self._test.__qualname__)) # start capturing log output cocotb.log.addHandler(self._test_task.handler) self._test_start_time = time.time() self._test_start_sim_time = get_sim_time('ns') cocotb.scheduler.add_test(self._test_task) def _log_test_summary(self) -> None: if self.failures: self.log.error("Failed %d out of %d tests (%d skipped)" % (self.failures, self.count, self.skipped)) else: self.log.info("Passed %d tests (%d skipped)" % (self.count, self.skipped)) if len(self.test_results) == 0: return TEST_FIELD = 'TEST' RESULT_FIELD = 'PASS/FAIL' SIM_FIELD = 'SIM TIME(NS)' REAL_FIELD = 'REAL TIME(S)' RATIO_FIELD = 'RATIO(NS/S)' TEST_FIELD_LEN = max(len(TEST_FIELD), len(max([x['test'] for x in self.test_results], key=len))) RESULT_FIELD_LEN = len(RESULT_FIELD) SIM_FIELD_LEN = len(SIM_FIELD) REAL_FIELD_LEN = len(REAL_FIELD) RATIO_FIELD_LEN = len(RATIO_FIELD) header_dict = dict( a=TEST_FIELD, b=RESULT_FIELD, c=SIM_FIELD, d=REAL_FIELD, e=RATIO_FIELD, a_len=TEST_FIELD_LEN, b_len=RESULT_FIELD_LEN, c_len=SIM_FIELD_LEN, d_len=REAL_FIELD_LEN, e_len=RATIO_FIELD_LEN) LINE_LEN = 3 + TEST_FIELD_LEN + 2 + RESULT_FIELD_LEN + 2 + SIM_FIELD_LEN + 2 + \ REAL_FIELD_LEN + 2 + RATIO_FIELD_LEN + 3 LINE_SEP = "*" * LINE_LEN + "\n" summary = "" summary += LINE_SEP summary += "** {a:<{a_len}} {b:^{b_len}} {c:>{c_len}} {d:>{d_len}} {e:>{e_len}} **\n".format(**header_dict) summary += LINE_SEP test_line = "{start}** {a:<{a_len}} {b:^{b_len}} {c:>{c_len}.2f} {d:>{d_len}.2f} {e:>{e_len}.2f} **\n" for result in self.test_results: hilite = '' if result['pass'] is None: pass_fail_str = "N/A" elif result['pass']: pass_fail_str = "PASS" else: pass_fail_str = "FAIL" if want_color_output(): hilite = ANSI.COLOR_HILITE_SUMMARY test_dict = dict( a=result['test'], b=pass_fail_str, c=result['sim'], d=result['real'], e=result['ratio'], a_len=TEST_FIELD_LEN, b_len=RESULT_FIELD_LEN, c_len=SIM_FIELD_LEN - 1, d_len=REAL_FIELD_LEN - 1, e_len=RATIO_FIELD_LEN - 1, start=hilite) summary += test_line.format(**test_dict) summary += LINE_SEP self.log.info(summary) def _log_sim_summary(self) -> None: real_time = time.time() - self.start_time sim_time_ns = get_sim_time('ns') ratio_time = self._safe_divide(sim_time_ns, real_time) summary = "" summary += "*************************************************************************************\n" summary += "** ERRORS : {0:<39}**\n".format(self.failures) summary += "*************************************************************************************\n" summary += "** SIM TIME : {0:<39}**\n".format('{0:.2f} NS'.format(sim_time_ns)) summary += "** REAL TIME : {0:<39}**\n".format('{0:.2f} S'.format(real_time)) summary += "** SIM / REAL TIME : {0:<39}**\n".format('{0:.2f} NS/S'.format(ratio_time)) summary += "*************************************************************************************\n" self.log.info(summary) @staticmethod def _safe_divide(a: float, b: float) -> float: try: return a / b except ZeroDivisionError: if a == 0: return float('nan') else: return float('inf')
def __init__(self, dut: SimHandle, tests: Iterable[Test], hooks: Iterable[Hook]): """ Args: dut (SimHandle): The root handle to pass into test functions. tests (Iterable[Test]): tests to run hooks (Iterable[Hook]): hooks to tun """ self._dut = dut self._test_task = None self._cov = None self.log = _logger self.start_time = time.time() self.test_results = [] self.count = 0 self.skipped = 0 self.failures = 0 self._tearing_down = False # Setup XUnit ################### results_filename = os.getenv('COCOTB_RESULTS_FILE', "results.xml") suite_name = os.getenv('RESULT_TESTSUITE', "all") package_name = os.getenv('RESULT_TESTPACKAGE', "all") self.xunit = XUnitReporter(filename=results_filename) self.xunit.add_testsuite(name=suite_name, package=package_name) self.xunit.add_property(name="random_seed", value=str(cocotb.RANDOM_SEED)) # Setup Coverage #################### if coverage is not None: self.log.info("Enabling coverage collection of Python code") # Exclude cocotb itself from coverage collection. cocotb_package_dir = os.path.dirname(__file__) self._cov = coverage.coverage( branch=True, omit=["{}/*".format(cocotb_package_dir)]) self._cov.start() # Test Discovery #################### self._queue = [] for test in tests: self.log.info("Found test {}.{}".format(test.__module__, test.__qualname__)) self._queue.append(test) self.ntests = len(self._queue) if not self._queue: self.log.warning("No tests were discovered") self._queue.sort(key=lambda test: (test.stage, test._id)) # Process Hooks ################### for hook in hooks: self.log.info("Found hook {}.{}".format(hook.__module__, hook.__qualname__)) self._init_hook(hook)
class RegressionManager: """Encapsulates all regression capability into a single place""" def __init__(self, dut: SimHandle, tests: Iterable[Test]): """ Args: dut (SimHandle): The root handle to pass into test functions. tests (Iterable[Test]): tests to run """ self._dut = dut self._test = None self._test_task = None self._test_start_time = None self._test_start_sim_time = None self._cov = None self.log = _logger self.start_time = time.time() self.test_results = [] self.count = 0 self.passed = 0 self.skipped = 0 self.failures = 0 self._tearing_down = False # Setup XUnit ################### results_filename = os.getenv("COCOTB_RESULTS_FILE", "results.xml") suite_name = os.getenv("RESULT_TESTSUITE", "all") package_name = os.getenv("RESULT_TESTPACKAGE", "all") self.xunit = XUnitReporter(filename=results_filename) self.xunit.add_testsuite(name=suite_name, package=package_name) self.xunit.add_property(name="random_seed", value=str(cocotb.RANDOM_SEED)) # Setup Coverage #################### if coverage is not None: self.log.info("Enabling coverage collection of Python code") # Exclude cocotb itself from coverage collection. cocotb_package_dir = os.path.dirname(__file__) self._cov = coverage.coverage(branch=True, omit=[f"{cocotb_package_dir}/*"]) self._cov.start() # Test Discovery #################### self._queue = [] for test in tests: self.log.info(f"Found test {test.__module__}.{test.__qualname__}") self._queue.append(test) self.ntests = len(self._queue) if not self._queue: self.log.warning("No tests were discovered") self._queue.sort(key=lambda test: (test.stage, test._id)) @classmethod def from_discovery(cls, dut: SimHandle): """ Obtains the test list by discovery. See :envvar:`MODULE` and :envvar:`TESTCASE` for details on how tests are discovered. Args: dut (SimHandle): The root handle to pass into test functions. """ tests = cls._discover_tests() return cls(dut, tests) @staticmethod def _discover_tests() -> Iterable[Test]: """ Discovers tests in files automatically. See :envvar:`MODULE` and :envvar:`TESTCASE` for details on how tests are discovered. """ module_str = os.getenv("MODULE") test_str = os.getenv("TESTCASE") if module_str is None: raise ValueError( "Environment variable MODULE, which defines the module(s) to execute, is not defined." ) modules = [s.strip() for s in module_str.split(",") if s.strip()] tests = None if test_str: tests = [s.strip() for s in test_str.split(",") if s.strip()] for module_name in modules: try: _logger.debug("Python Path: " + ",".join(sys.path)) _logger.debug("PWD: " + os.getcwd()) module = _my_import(module_name) except Exception as E: _logger.critical("Failed to import module %s: %s", module_name, E) _logger.info('MODULE variable was "%s"', ".".join(modules)) _logger.info("Traceback: ") _logger.info(traceback.format_exc()) raise if tests is not None: not_found_tests = [] # Specific functions specified, don't auto-discover for test_name in tests: try: test = getattr(module, test_name) except AttributeError: not_found_tests.append(test_name) continue if not isinstance(test, Test): _logger.error( "Requested %s from module %s isn't a cocotb.test decorated coroutine", test_name, module_name, ) raise ImportError("Failed to find requested test %s" % test_name) # If we request a test manually, it should be run even if skip=True is set. test.skip = False yield test # Use the non-matching test names in the next module search tests = not_found_tests else: # auto-discover for thing in vars(module).values(): if isinstance(thing, Test): yield thing # If any test were not found in any module, raise an error if tests: _logger.error("Requested test(s) %s wasn't found in module(s) %s", tests, modules) raise AttributeError("Test(s) %s doesn't exist in %s" % (tests, modules)) @deprecated("This method is now private.") def tear_down(self) -> None: self._tear_down() def _tear_down(self) -> None: # prevent re-entering the tear down procedure if not self._tearing_down: self._tearing_down = True else: return # fail remaining tests while True: test = self._next_test() if test is None: break self._record_result(test=test, outcome=Error(SimFailure), wall_time_s=0, sim_time_ns=0) # Write out final log messages self._log_test_summary() # Generate output reports self.xunit.write() if self._cov: self._cov.stop() self.log.info("Writing coverage data") self._cov.save() self._cov.html_report() if cocotb._library_coverage is not None: # TODO: move this once we have normal shutdown behavior to _sim_event cocotb._library_coverage.stop() cocotb._library_coverage.save() # Setup simulator finalization simulator.stop_simulator() @deprecated("This method is now private.") def next_test(self) -> Optional[Test]: return self._next_test() def _next_test(self) -> Optional[Test]: """Get the next test to run""" if not self._queue: return None self.count += 1 return self._queue.pop(0) @deprecated("This method is now private.") def handle_result(self, test: Task) -> None: self._handle_result(test) def _handle_result(self, test: Task) -> None: """Handle a test completing. Dump result to XML and schedule the next test (if any). Entered by the scheduler. Args: test: The test that completed """ assert test is self._test_task real_time = time.time() - self._test_start_time sim_time_ns = get_sim_time("ns") - self._test_start_sim_time self._record_result( test=self._test, outcome=self._test_task._outcome, wall_time_s=real_time, sim_time_ns=sim_time_ns, ) self._execute() def _init_test(self, test: Test) -> Optional[Task]: """Initialize a test. Record outcome if the initialization fails. Record skip if the test is skipped. Save the initialized test if it successfully initializes. """ if test.skip: hilight_start = ANSI.COLOR_SKIPPED if want_color_output() else "" hilight_end = ANSI.COLOR_DEFAULT if want_color_output() else "" # Want this to stand out a little bit self.log.info("{start}skipping{end} {name} ({i}/{total})".format( start=hilight_start, i=self.count, total=self.ntests, end=hilight_end, name=test.__qualname__, )) self._record_result(test, None, 0, 0) return None test_init_outcome = cocotb.outcomes.capture(test, self._dut) if isinstance(test_init_outcome, cocotb.outcomes.Error): self.log.error( "Failed to initialize test %s" % test.__qualname__, exc_info=test_init_outcome.error, ) self._record_result(test, test_init_outcome, 0, 0) return None running_test = test_init_outcome.get() # seed random number generator based on test module, name, and RANDOM_SEED hasher = hashlib.sha1() hasher.update(test.__qualname__.encode()) hasher.update(test.__module__.encode()) seed = cocotb.RANDOM_SEED + int(hasher.hexdigest(), 16) random.seed(seed) return running_test def _score_test(self, test: Test, outcome: Outcome) -> Tuple[bool, bool]: """ Given a test and the test's outcome, determine if the test met expectations and log pertinent information """ # scoring outcomes result_pass = True sim_failed = False try: outcome.get() except Exception as e: result = remove_traceback_frames(e, ["_score_test", "get"]) else: result = TestSuccess() if (isinstance(result, TestSuccess) and not test.expect_fail and not test.expect_error): self._log_test_passed(test, None, None) elif isinstance(result, AssertionError) and test.expect_fail: self._log_test_passed(test, result, "failed as expected") elif isinstance(result, TestSuccess) and test.expect_error: self._log_test_failed(test, None, "passed but we expected an error") result_pass = False elif isinstance(result, TestSuccess): self._log_test_failed(test, None, "passed but we expected a failure") result_pass = False elif isinstance(result, SimFailure): if isinstance(result, test.expect_error): self._log_test_passed(test, result, "errored as expected") else: self.log.error( "Test error has lead to simulator shutting us down") result_pass = False # whether we expected it or not, the simulation has failed unrecoverably sim_failed = True elif test.expect_error: if isinstance(result, test.expect_error): self._log_test_passed(test, result, "errored as expected") else: self._log_test_failed(test, result, "errored with unexpected type ") result_pass = False else: self._log_test_failed(test, result, None) result_pass = False if _pdb_on_exception: pdb.post_mortem(result.__traceback__) return result_pass, sim_failed def _log_test_passed(self, test: Test, result: Optional[Exception] = None, msg: Optional[str] = None) -> None: start_hilight = ANSI.COLOR_PASSED if want_color_output() else "" stop_hilight = ANSI.COLOR_DEFAULT if want_color_output() else "" if msg is None: rest = "" else: rest = f": {msg}" if result is None: result_was = "" else: result_was = f" (result was {type(result).__qualname__})" self.log.info( f"{test} {start_hilight}passed{stop_hilight}{rest}{result_was}") def _log_test_failed(self, test: Test, result: Optional[Exception] = None, msg: Optional[str] = None) -> None: start_hilight = ANSI.COLOR_FAILED if want_color_output() else "" stop_hilight = ANSI.COLOR_DEFAULT if want_color_output() else "" if msg is None: rest = "" else: rest = f": {msg}" self.log.info(f"{test} {start_hilight}failed{stop_hilight}{rest}", exc_info=result) def _record_result( self, test: Test, outcome: Optional[Outcome], wall_time_s: float, sim_time_ns: float, ) -> None: ratio_time = self._safe_divide(sim_time_ns, wall_time_s) try: lineno = inspect.getsourcelines(test._func)[1] except OSError: lineno = 1 self.xunit.add_testcase( name=test.__qualname__, classname=test.__module__, file=inspect.getfile(test._func), lineno=repr(lineno), time=repr(wall_time_s), sim_time_ns=repr(sim_time_ns), ratio_time=repr(ratio_time), ) if outcome is None: # skipped test_pass, sim_failed = None, False self.xunit.add_skipped() self.skipped += 1 else: test_pass, sim_failed = self._score_test(test, outcome) if not test_pass: self.xunit.add_failure() self.failures += 1 else: self.passed += 1 self.test_results.append({ "test": ".".join([test.__module__, test.__qualname__]), "pass": test_pass, "sim": sim_time_ns, "real": wall_time_s, "ratio": ratio_time, }) if sim_failed: self._tear_down() return @deprecated("This method is now private.") def execute(self) -> None: self._execute() def _execute(self) -> None: while True: self._test = self._next_test() if self._test is None: return self._tear_down() self._test_task = self._init_test(self._test) if self._test_task is not None: return self._start_test() def _start_test(self) -> None: # Want this to stand out a little bit start = "" end = "" if want_color_output(): start = ANSI.COLOR_TEST end = ANSI.COLOR_DEFAULT self.log.info( "{start}running{end} {name} ({i}/{total}){description}".format( start=start, i=self.count, total=self.ntests, end=end, name=self._test.__qualname__, description=_trim(self._test.__doc__), )) self._test_start_time = time.time() self._test_start_sim_time = get_sim_time("ns") cocotb.scheduler._add_test(self._test_task) def _log_test_summary(self) -> None: real_time = time.time() - self.start_time sim_time_ns = get_sim_time("ns") ratio_time = self._safe_divide(sim_time_ns, real_time) if len(self.test_results) == 0: return TEST_FIELD = "TEST" RESULT_FIELD = "STATUS" SIM_FIELD = "SIM TIME (ns)" REAL_FIELD = "REAL TIME (s)" RATIO_FIELD = "RATIO (ns/s)" TOTAL_NAME = f"TESTS={self.ntests} PASS={self.passed} FAIL={self.failures} SKIP={self.skipped}" TEST_FIELD_LEN = max( len(TEST_FIELD), len(TOTAL_NAME), len(max([x["test"] for x in self.test_results], key=len)), ) RESULT_FIELD_LEN = len(RESULT_FIELD) SIM_FIELD_LEN = len(SIM_FIELD) REAL_FIELD_LEN = len(REAL_FIELD) RATIO_FIELD_LEN = len(RATIO_FIELD) header_dict = dict( a=TEST_FIELD, b=RESULT_FIELD, c=SIM_FIELD, d=REAL_FIELD, e=RATIO_FIELD, a_len=TEST_FIELD_LEN, b_len=RESULT_FIELD_LEN, c_len=SIM_FIELD_LEN, d_len=REAL_FIELD_LEN, e_len=RATIO_FIELD_LEN, ) LINE_LEN = (3 + TEST_FIELD_LEN + 2 + RESULT_FIELD_LEN + 2 + SIM_FIELD_LEN + 2 + REAL_FIELD_LEN + 2 + RATIO_FIELD_LEN + 3) LINE_SEP = "*" * LINE_LEN + "\n" summary = "" summary += LINE_SEP summary += "** {a:<{a_len}} {b:^{b_len}} {c:>{c_len}} {d:>{d_len}} {e:>{e_len}} **\n".format( **header_dict) summary += LINE_SEP test_line = "** {a:<{a_len}} {start}{b:^{b_len}}{end} {c:>{c_len}.2f} {d:>{d_len}.2f} {e:>{e_len}} **\n" for result in self.test_results: hilite = "" lolite = "" if result["pass"] is None: ratio = "-.--" pass_fail_str = "SKIP" if want_color_output(): hilite = ANSI.COLOR_SKIPPED lolite = ANSI.COLOR_DEFAULT elif result["pass"]: ratio = format(result["ratio"], "0.2f") pass_fail_str = "PASS" if want_color_output(): hilite = ANSI.COLOR_PASSED lolite = ANSI.COLOR_DEFAULT else: ratio = format(result["ratio"], "0.2f") pass_fail_str = "FAIL" if want_color_output(): hilite = ANSI.COLOR_FAILED lolite = ANSI.COLOR_DEFAULT test_dict = dict( a=result["test"], b=pass_fail_str, c=result["sim"], d=result["real"], e=ratio, a_len=TEST_FIELD_LEN, b_len=RESULT_FIELD_LEN, c_len=SIM_FIELD_LEN - 1, d_len=REAL_FIELD_LEN - 1, e_len=RATIO_FIELD_LEN - 1, start=hilite, end=lolite, ) summary += test_line.format(**test_dict) summary += LINE_SEP summary += test_line.format( a=TOTAL_NAME, b="", c=sim_time_ns, d=real_time, e=format(ratio_time, "0.2f"), a_len=TEST_FIELD_LEN, b_len=RESULT_FIELD_LEN, c_len=SIM_FIELD_LEN - 1, d_len=REAL_FIELD_LEN - 1, e_len=RATIO_FIELD_LEN - 1, start="", end="", ) summary += LINE_SEP self.log.info(summary) @staticmethod def _safe_divide(a: float, b: float) -> float: try: return a / b except ZeroDivisionError: if a == 0: return float("nan") else: return float("inf")
def initialise(self): self.ntests = 0 self.count = 1 self.skipped = 0 self.failures = 0 self.xunit = XUnitReporter() self.xunit.add_testsuite(name="all", tests=repr(self.ntests), package="all") if coverage is not None: self.log.info("Enabling coverage collection of Python code") self._cov = coverage.coverage(branch=True, omit=["*cocotb*"]) self._cov.start() self._dut = cocotb.handle.SimHandle( simulator.get_root_handle(self._root_name)) if self._dut is None: raise AttributeError("Can not find Root Handle (%s)" % root_name) # Auto discovery for module_name in self._modules: module = _my_import(module_name) if self._functions: # Specific functions specified, don't auto discover for test in self._functions.rsplit(','): if not hasattr(module, test): raise AttributeError("Test %s doesn't exist in %s" % (test, module_name)) self._queue.append(getattr(module, test)(self._dut)) self.ntests += 1 break for thing in vars(module).values(): if hasattr(thing, "im_test"): try: test = thing(self._dut) skip = test.skip except TestError: skip = True self.log.warning("Failed to initialise test %s" % thing.name) if skip: self.log.info("Skipping test %s" % thing.name) self.xunit.add_testcase(name=thing.name, classname=module_name, time="0.0") self.xunit.add_skipped() self.skipped += 1 else: self._queue.append(test) self.ntests += 1 self._queue.sort( key=lambda test: "%s.%s" % (test.module, test.funcname)) for valid_tests in self._queue: self.log.info("Found test %s.%s" % (valid_tests.module, valid_tests.funcname))
class RegressionManager(object): """Encapsulates all regression capability into a single place""" def __init__(self, root_name, modules, tests=None): """ Args: modules (list): A list of python module names to run Kwargs """ self._queue = [] self._root_name = root_name self._dut = None self._modules = modules self._functions = tests self._running_test = None self._cov = None self.log = SimLog("cocotb.regression") def initialise(self): self.ntests = 0 self.count = 1 self.skipped = 0 self.failures = 0 self.xunit = XUnitReporter() self.xunit.add_testsuite(name="all", tests=repr(self.ntests), package="all") if coverage is not None: self.log.info("Enabling coverage collection of Python code") self._cov = coverage.coverage(branch=True, omit=["*cocotb*"]) self._cov.start() self._dut = cocotb.handle.SimHandle( simulator.get_root_handle(self._root_name)) if self._dut is None: raise AttributeError("Can not find Root Handle (%s)" % root_name) # Auto discovery for module_name in self._modules: module = _my_import(module_name) if self._functions: # Specific functions specified, don't auto discover for test in self._functions.rsplit(','): if not hasattr(module, test): raise AttributeError("Test %s doesn't exist in %s" % (test, module_name)) self._queue.append(getattr(module, test)(self._dut)) self.ntests += 1 break for thing in vars(module).values(): if hasattr(thing, "im_test"): try: test = thing(self._dut) skip = test.skip except TestError: skip = True self.log.warning("Failed to initialise test %s" % thing.name) if skip: self.log.info("Skipping test %s" % thing.name) self.xunit.add_testcase(name=thing.name, classname=module_name, time="0.0") self.xunit.add_skipped() self.skipped += 1 else: self._queue.append(test) self.ntests += 1 self._queue.sort( key=lambda test: "%s.%s" % (test.module, test.funcname)) for valid_tests in self._queue: self.log.info("Found test %s.%s" % (valid_tests.module, valid_tests.funcname)) def tear_down(self): """It's the end of the world as we know it""" if self.failures: self.log.error("Failed %d out of %d tests (%d skipped)" % (self.failures, self.count - 1, self.skipped)) else: self.log.info("Passed %d tests (%d skipped)" % (self.count - 1, self.skipped)) if self._cov: self._cov.stop() self.log.info("Writing coverage data") self._cov.save() self._cov.html_report() self.log.info("Shutting down...") self.xunit.write() simulator.stop_simulator() def next_test(self): """Get the next test to run""" if not self._queue: return None return self._queue.pop(0) def handle_result(self, result): """Handle a test result Dumps result to XML and schedules the next test (if any) Args: result (TestComplete exception) """ self.xunit.add_testcase(name=self._running_test.funcname, classname=self._running_test.module, time=repr(time.time() - self._running_test.start_time)) if isinstance( result, TestSuccess ) and not self._running_test.expect_fail and not self._running_test.expect_error: self.log.info("Test Passed: %s" % self._running_test.funcname) elif isinstance(result, TestFailure) and self._running_test.expect_fail: self.log.info( "Test failed as expected: %s (result was %s)" % (self._running_test.funcname, result.__class__.__name__)) elif isinstance(result, TestSuccess) and self._running_test.expect_error: self.log.error( "Test passed but we expected an error: %s (result was %s)" % (self._running_test.funcname, result.__class__.__name__)) self.xunit.add_failure(stdout=repr(str(result)), stderr="\n".join( self._running_test.error_messages)) self.failures += 1 elif isinstance(result, TestSuccess): self.log.error( "Test passed but we expected a failure: %s (result was %s)" % (self._running_test.funcname, result.__class__.__name__)) self.xunit.add_failure(stdout=repr(str(result)), stderr="\n".join( self._running_test.error_messages)) self.failures += 1 elif isinstance(result, TestError) and self._running_test.expect_error: self.log.info( "Test errored as expected: %s (result was %s)" % (self._running_test.funcname, result.__class__.__name__)) elif isinstance(result, SimFailure): if self._running_test.expect_error: self.log.info( "Test errored as expected: %s (result was %s)" % (self._running_test.funcname, result.__class__.__name__)) else: self.log.error( "Test error has lead to simulator shuttting us down") self.failures += 1 self.tear_down() return else: self.log.error( "Test Failed: %s (result was %s)" % (self._running_test.funcname, result.__class__.__name__)) self.xunit.add_failure(stdout=repr(str(result)), stderr="\n".join( self._running_test.error_messages)) self.failures += 1 self.execute() def execute(self): self._running_test = cocotb.regression.next_test() if self._running_test: # Want this to stand out a little bit self.log.info("%sRunning test %d/%d:%s %s" % (ANSI.BLUE_BG + ANSI.BLACK_FG, self.count, self.ntests, ANSI.DEFAULT_FG + ANSI.DEFAULT_BG, self._running_test.funcname)) if self.count is 1: test = cocotb.scheduler.add(self._running_test) else: test = cocotb.scheduler.new_test(self._running_test) self.count += 1 else: self.tear_down()
class RegressionManager(object): """Encapsulates all regression capability into a single place""" def __init__(self, root_name, modules, tests=None): """ Args: modules (list): A list of python module names to run Kwargs """ self._queue = [] self._root_name = root_name self._dut = None self._modules = modules self._functions = tests self._running_test = None self._cov = None self.log = SimLog("cocotb.regression") def initialise(self): self.start_time = time.time() self.test_results = [] self.ntests = 0 self.count = 1 self.skipped = 0 self.failures = 0 self.xunit = XUnitReporter() self.xunit.add_testsuite(name="all", tests=repr(self.ntests), package="all") if coverage is not None: self.log.info("Enabling coverage collection of Python code") self._cov = coverage.coverage(branch=True, omit=["*cocotb*"]) self._cov.start() handle = simulator.get_root_handle(self._root_name) self._dut = cocotb.handle.SimHandle(handle) if handle else None if self._dut is None: raise AttributeError("Can not find Root Handle (%s)" % self._root_name) # Auto discovery for module_name in self._modules: try: module = _my_import(module_name) except ImportError: self.log.critical("Failed to import module %s", module_name) self.log.info("MODULE variable was \"%s\"", ",".join(self._modules)) raise if self._functions: # Specific functions specified, don't auto discover for test in self._functions.rsplit(','): if not hasattr(module, test): raise AttributeError("Test %s doesn't exist in %s" % (test, module_name)) self._queue.append(getattr(module, test)(self._dut)) self.ntests += 1 break for thing in vars(module).values(): if hasattr(thing, "im_test"): try: test = thing(self._dut) skip = test.skip except TestError: skip = True self.log.warning("Failed to initialise test %s" % thing.name) if skip: self.log.info("Skipping test %s" % thing.name) self.xunit.add_testcase(name=thing.name, classname=module_name, time="0.0", sim_time_ns="0.0", ratio_time="0.0") self.xunit.add_skipped() self.skipped += 1 self._store_test_result(module_name, thing.name, None, 0.0, 0.0, 0.0) else: self._queue.append(test) self.ntests += 1 self._queue.sort(key=lambda test: "%s.%s" % (test.module, test.funcname)) for valid_tests in self._queue: self.log.info("Found test %s.%s" % (valid_tests.module, valid_tests.funcname)) def tear_down(self): """It's the end of the world as we know it""" if self.failures: self.log.error("Failed %d out of %d tests (%d skipped)" % (self.failures, self.count - 1, self.skipped)) else: self.log.info("Passed %d tests (%d skipped)" % (self.count - 1, self.skipped)) if self._cov: self._cov.stop() self.log.info("Writing coverage data") self._cov.save() self._cov.html_report() self._log_test_summary() self._log_sim_summary() self.log.info("Shutting down...") self.xunit.write() simulator.stop_simulator() def next_test(self): """Get the next test to run""" if not self._queue: return None return self._queue.pop(0) def handle_result(self, result): """Handle a test result Dumps result to XML and schedules the next test (if any) Args: result (TestComplete exception) """ real_time = time.time() - self._running_test.start_time sim_time_ns = get_sim_time('ns') - self._running_test.start_sim_time ratio_time = sim_time_ns / real_time self.xunit.add_testcase(name=self._running_test.funcname, classname=self._running_test.module, time=repr(real_time), sim_time_ns=repr(sim_time_ns), ratio_time=repr(ratio_time)) running_test_funcname = self._running_test.funcname # Helper for logging result def _result_was(): result_was = ("%s (result was %s)" % (running_test_funcname, result.__class__.__name__)) return result_was result_pass = True if (isinstance(result, TestSuccess) and not self._running_test.expect_fail and not self._running_test.expect_error): self.log.info("Test Passed: %s" % running_test_funcname) elif (isinstance(result, TestFailure) and self._running_test.expect_fail): self.log.info("Test failed as expected: " + _result_was()) elif (isinstance(result, TestSuccess) and self._running_test.expect_error): self.log.error("Test passed but we expected an error: " + _result_was()) self.xunit.add_failure(stdout=repr(str(result)), stderr="\n".join( self._running_test.error_messages)) self.failures += 1 result_pass = False elif isinstance(result, TestSuccess): self.log.error("Test passed but we expected a failure: " + _result_was()) self.xunit.add_failure(stdout=repr(str(result)), stderr="\n".join( self._running_test.error_messages)) self.failures += 1 result_pass = False elif isinstance(result, TestError) and self._running_test.expect_error: self.log.info("Test errored as expected: " + _result_was()) elif isinstance(result, SimFailure): if self._running_test.expect_error: self.log.info("Test errored as expected: " + _result_was()) else: self.log.error("Test error has lead to simulator shuttting us " "down") self.failures += 1 self._store_test_result(self._running_test.module, self._running_test.funcname, False, sim_time_ns, real_time, ratio_time) self.tear_down() return else: self.log.error("Test Failed: " + _result_was()) self.xunit.add_failure(stdout=repr(str(result)), stderr="\n".join( self._running_test.error_messages)) self.failures += 1 result_pass = False self._store_test_result(self._running_test.module, self._running_test.funcname, result_pass, sim_time_ns, real_time, ratio_time) self.execute() def execute(self): self._running_test = cocotb.regression.next_test() if self._running_test: start = '' end = '' if self.log.colour: start = ANSI.BLUE_BG + ANSI.BLACK_FG end = ANSI.DEFAULT # Want this to stand out a little bit self.log.info("%sRunning test %d/%d:%s %s" % (start, self.count, self.ntests, end, self._running_test.funcname)) if self.count is 1: test = cocotb.scheduler.add(self._running_test) else: test = cocotb.scheduler.new_test(self._running_test) self.count += 1 else: self.tear_down() def _log_test_summary(self): TEST_FIELD = 'TEST' RESULT_FIELD = 'PASS/FAIL' SIM_FIELD = 'SIM TIME(NS)' REAL_FIELD = 'REAL TIME(S)' RATIO_FIELD = 'RATIO(NS/S)' TEST_FIELD_LEN = max(len(TEST_FIELD),len(max([x['test'] for x in self.test_results],key=len))) RESULT_FIELD_LEN = len(RESULT_FIELD) SIM_FIELD_LEN = len(SIM_FIELD) REAL_FIELD_LEN = len(REAL_FIELD) RATIO_FIELD_LEN = len(RATIO_FIELD) LINE_LEN = 3 + TEST_FIELD_LEN + 2 + RESULT_FIELD_LEN + 2 + SIM_FIELD_LEN + 2 + REAL_FIELD_LEN + 2 + RATIO_FIELD_LEN + 3 LINE_SEP = "*"*LINE_LEN+"\n" summary = "" summary += LINE_SEP summary += "** {a:<{a_len}} {b:^{b_len}} {c:>{c_len}} {d:>{d_len}} {e:>{e_len}} **\n".format(a=TEST_FIELD, a_len=TEST_FIELD_LEN, b=RESULT_FIELD, b_len=RESULT_FIELD_LEN, c=SIM_FIELD, c_len=SIM_FIELD_LEN, d=REAL_FIELD, d_len=REAL_FIELD_LEN, e=RATIO_FIELD, e_len=RATIO_FIELD_LEN) summary += LINE_SEP for result in self.test_results: hilite = '' if result['pass'] is None: pass_fail_str = "N/A" elif result['pass']: pass_fail_str = "PASS" else: pass_fail_str = "FAIL" if self.log.colour: hilite = ANSI.WHITE_FG + ANSI.RED_BG summary += "{start}** {a:<{a_len}} {b:^{b_len}} {c:>{c_len}.2f} {d:>{d_len}.2f} {e:>{e_len}.2f} **\n".format(a=result['test'], a_len=TEST_FIELD_LEN, b=pass_fail_str, b_len=RESULT_FIELD_LEN, c=result['sim'], c_len=SIM_FIELD_LEN-1, d=result['real'], d_len=REAL_FIELD_LEN-1, e=result['ratio'], e_len=RATIO_FIELD_LEN-1, start=hilite) summary += LINE_SEP self.log.info(summary) def _log_sim_summary(self): real_time = time.time() - self.start_time sim_time_ns = get_sim_time('ns') ratio_time = sim_time_ns / real_time summary = "" summary += "*************************************************************************************\n" summary += "** ERRORS : {:<39}**\n".format(self.failures) summary += "*************************************************************************************\n" summary += "** SIM TIME : {:<39}**\n".format('{:.2f} NS'.format(sim_time_ns)) summary += "** REAL TIME : {:<39}**\n".format('{:.2f} S'.format(real_time)) summary += "** SIM / REAL TIME : {:<39}**\n".format('{:.2f} NS/S'.format(ratio_time)) summary += "*************************************************************************************\n" self.log.info(summary) def _store_test_result(self, module_name, test_name, result_pass, sim_time, real_time, ratio): result = { 'test' : '.'.join([module_name, test_name]), 'pass' : result_pass, 'sim' : sim_time, 'real' : real_time, 'ratio' : ratio} self.test_results.append(result)