def setUp(self): if simulator_is("activehdl"): self.output_path = str(ROOT / "artificial_out") else: # Spaces in path intentional to verify that it is supported self.output_path = str(ROOT / "artificial _out") self.report_file = str(Path(self.output_path) / "xunit.xml") self.artificial_run_vhdl = str(ROOT / "artificial" / "vhdl" / "run.py") self.artificial_run_verilog = str(ROOT / "artificial" / "verilog" / "run.py")
def setUp(self): if simulator_is("activehdl"): self.output_path = join(dirname(__file__), "artificial_out") else: # Spaces in path intentional to verify that it is supported self.output_path = join(dirname(__file__), "artificial _out") self.report_file = join(self.output_path, "xunit.xml") self.artificial_run_vhdl = join(dirname(__file__), "artificial", "vhdl", "run.py") self.artificial_run_verilog = join(dirname(__file__), "artificial", "verilog", "run.py")
class TestVunitArtificial(unittest.TestCase): """ Acceptance test of VUnit end to end functionality using artificial test benches. """ def setUp(self): if simulator_is("activehdl"): self.output_path = str(ROOT / "artificial_out") else: # Spaces in path intentional to verify that it is supported self.output_path = str(ROOT / "artificial _out") self.report_file = str(Path(self.output_path) / "xunit.xml") self.artificial_run_vhdl = str(ROOT / "artificial" / "vhdl" / "run.py") self.artificial_run_verilog = str(ROOT / "artificial" / "verilog" / "run.py") @unittest.skipUnless( simulator_is("modelsim", "rivierapro"), "Only simulators with persistance functionality", ) def test_artificial_modelsim_unique_sim(self): self._test_artificial(args=["--unique-sim"]) def test_artificial(self): self._test_artificial() def test_artificial_elaborate_only(self): self.check(self.artificial_run_vhdl, exit_code=1, args=["--elaborate"]) elab_expected_report = [] for status, name in EXPECTED_REPORT: if name in ("lib.tb_elab_fail.all", ): status = "failed" else: status = "passed" elab_expected_report.append((status, name)) check_report(self.report_file, elab_expected_report) self.check( self.artificial_run_vhdl, exit_code=0, clean=False, args=["--elaborate", "lib.tb_pass.all"], ) check_report(self.report_file, [("passed", "lib.tb_pass.all")]) self.check( self.artificial_run_vhdl, exit_code=1, clean=False, args=["--elaborate", "lib.tb_elab_fail.all"], ) check_report(self.report_file, [("failed", "lib.tb_elab_fail.all")]) def _test_artificial(self, args=None): """ Utility function to run and check the result of all test benches using either persistent or non-persistent simulator interface mode """ self.check(self.artificial_run_vhdl, exit_code=1, args=args) check_report(self.report_file, EXPECTED_REPORT) def test_run_selected_tests_in_same_sim_test_bench_vhdl(self): self._test_run_selected_tests_in_same_sim_test_bench( self.artificial_run_vhdl) @unittest.skipUnless(simulator_is("modelsim"), "Only modelsim supports verilog") def test_run_selected_tests_in_same_sim_test_bench_verilog(self): self._test_run_selected_tests_in_same_sim_test_bench( self.artificial_run_verilog) def _test_run_selected_tests_in_same_sim_test_bench(self, run_file): """ Run selected "same_sim" test in isolation """ self.check(run_file, exit_code=0, clean=True, args=["*same_sim_some_fail*Test 1*"]) check_report(self.report_file, [("passed", "lib.tb_same_sim_some_fail.Test 1")]) self.check(run_file, exit_code=1, clean=False, args=["*same_sim_some_fail*Test 2*"]) check_report(self.report_file, [("failed", "lib.tb_same_sim_some_fail.Test 2")]) self.check(run_file, exit_code=0, clean=False, args=["*same_sim_some_fail*Test 3*"]) check_report(self.report_file, [("passed", "lib.tb_same_sim_some_fail.Test 3")]) self.check( run_file, exit_code=1, clean=False, args=[ "*same_sim_some_fail*Test 2*", "*same_sim_some_fail*Test 3*" ], ) check_report( self.report_file, [ ("failed", "lib.tb_same_sim_some_fail.Test 2"), ("skipped", "lib.tb_same_sim_some_fail.Test 3"), ], ) @unittest.skipUnless(simulator_is("modelsim"), "Only modelsim supports verilog") def test_artificial_verilog(self): self.check(self.artificial_run_verilog, exit_code=1) check_report( self.report_file, [ ("passed", "lib.tb_other_file_tests.pass"), ("failed", "lib.tb_other_file_tests.fail"), ("passed", "lib.tb_magic_paths.Test magic paths are correct"), ("passed", "lib.tb_with_define.test 1"), ("failed", "lib.tb_fail_on_warning.fail"), ("failed", "lib.tb_fail_on_fatal_and_early_finish.fatal0"), ("failed", "lib.tb_fail_on_fatal_and_early_finish.fatal1"), ("failed", "lib.tb_fail_on_fatal_and_early_finish.finish0"), ("failed", "lib.tb_fail_on_fatal_and_early_finish.finish1"), ("passed", "lib.tb_with_parameter_config.Test 0"), ("passed", "lib.tb_with_parameter_config.cfg.Test 1"), ("passed", "lib.tb_with_parameter_config.Test 2"), ("passed", "lib.tb_with_parameter_config.cfg.Test 3"), ("passed", "lib.tb_with_parameter_config.cfg.Test 4"), ("passed", "lib.tb_same_sim_all_pass.cfg.Test 1"), ("passed", "lib.tb_same_sim_all_pass.cfg.Test 2"), ("passed", "lib.tb_same_sim_all_pass.cfg.Test 3"), ("passed", "lib.tb_same_sim_some_fail.Test 1"), ("failed", "lib.tb_same_sim_some_fail.Test 2"), ("skipped", "lib.tb_same_sim_some_fail.Test 3"), ("passed", "lib.tb_with_runner.pass"), ("failed", "lib.tb_with_runner.fail"), ], ) # pylint: disable=too-many-arguments def check(self, run_file, args=None, clean=True, exit_code=0): """ Run external run file and verify exit code """ args = args if args is not None else [] new_env = environ.copy() new_env["VUNIT_VHDL_STANDARD"] = "2008" if clean: args += ["--clean"] retcode = call( [ sys.executable, run_file, "--output-path=%s" % self.output_path, "--xunit-xml=%s" % self.report_file, ] + args, env=new_env, ) self.assertEqual(retcode, exit_code) def test_exit_0_flag(self): self.check(self.artificial_run_vhdl, exit_code=1, args=["lib.tb_fail.all"]) self.check(self.artificial_run_vhdl, exit_code=0, args=["--exit-0", "lib.tb_fail.all"])
class TestExternalRunScripts(unittest.TestCase): """ Verify that example projects run correctly """ def test_vhdl_uart_example_project(self): self.check(str(ROOT / "examples" / "vhdl" / "uart" / "run.py")) @unittest.skipUnless(simulator_supports_verilog(), "Verilog") def test_verilog_uart_example_project(self): self.check(str(ROOT / "examples" / "verilog" / "uart" / "run.py")) @unittest.skipUnless(simulator_supports_verilog(), "Verilog") def test_verilog_ams_example(self): self.check( str(ROOT / "examples" / "verilog" / "verilog_ams" / "run.py")) check_report( self.report_file, [ ("passed", "lib.tb_dut.Test that pass"), ("failed", "lib.tb_dut.Test that fail"), ], ) def test_vhdl_logging_example_project(self): self.check(str(ROOT / "examples" / "vhdl" / "logging" / "run.py")) def test_vhdl_run_example_project(self): self.check(str(ROOT / "examples" / "vhdl" / "run" / "run.py"), exit_code=1) check_report( self.report_file, [ ("passed", "lib.tb_with_watchdog.Test to_string for boolean"), ("passed", "lib.tb_with_watchdog.Test that needs longer timeout"), ("passed", "lib.tb_standalone.Test to_string for boolean"), ("passed", "lib.tb_with_test_cases.Test to_string for integer"), ("passed", "lib.tb_with_test_cases.Test to_string for boolean"), ("passed", "lib.tb_with_lower_level_control.Test something"), ("passed", "lib.tb_with_lower_level_control.Test something else"), ("passed", "lib.tb_running_test_case.Test scenario A"), ("passed", "lib.tb_running_test_case.Test scenario B"), ("passed", "lib.tb_running_test_case.Test something else"), ("passed", "lib.tb_minimal.all"), ("passed", "lib.tb_magic_paths.all"), ("failed", "lib.tb_with_watchdog.Test that stalls"), ( "failed", "lib.tb_with_watchdog.Test that stalling processes can inform why they caused a timeout", ), ( "failed", "lib.tb_counting_errors.Test that fails multiple times but doesn't stop", ), ( "failed", "lib.tb_standalone.Test that fails on VUnit check procedure", ), ("failed", "lib.tb_many_ways_to_fail.Test that fails on an assert"), ( "failed", "lib.tb_many_ways_to_fail.Test that crashes on boundary problems", ), ( "failed", "lib.tb_many_ways_to_fail.Test that fails on VUnit check procedure", ), ], ) def test_vhdl_third_party_integration_example_project(self): self.check( str(ROOT / "examples" / "vhdl" / "third_party_integration" / "run.py"), exit_code=1, ) check_report( self.report_file, [ ("passed", "lib.tb_external_framework_integration.Test that pass"), ( "failed", "lib.tb_external_framework_integration.Test that stops the simulation on first error", ), ( "failed", "lib.tb_external_framework_integration.Test that doesn't stop the simulation on error", ), ], ) def test_vhdl_check_example_project(self): self.check(str(ROOT / "examples" / "vhdl" / "check" / "run.py")) @unittest.skipIf( simulator_check(lambda simclass: not simclass.supports_coverage()), "This simulator/backend does not support coverage", ) def test_vhdl_coverage_example_project(self): self.check(str(ROOT / "examples" / "vhdl" / "coverage" / "run.py")) def test_vhdl_generate_tests_example_project(self): self.check( str(ROOT / "examples" / "vhdl" / "generate_tests" / "run.py")) check_report( self.report_file, [ ("passed", "lib.tb_generated.data_width=1,sign=False.Test 1"), ("passed", "lib.tb_generated.data_width=1,sign=True.Test 1"), ("passed", "lib.tb_generated.data_width=2,sign=False.Test 1"), ("passed", "lib.tb_generated.data_width=2,sign=True.Test 1"), ("passed", "lib.tb_generated.data_width=3,sign=False.Test 1"), ("passed", "lib.tb_generated.data_width=3,sign=True.Test 1"), ("passed", "lib.tb_generated.data_width=4,sign=False.Test 1"), ("passed", "lib.tb_generated.data_width=4,sign=True.Test 1"), ("passed", "lib.tb_generated.data_width=16,sign=True.Test 2"), ], ) def test_vhdl_composite_generics_example_project(self): self.check( str(ROOT / "examples" / "vhdl" / "composite_generics" / "run.py")) check_report( self.report_file, [ ("passed", "tb_lib.tb_composite_generics.VGA.Test 1"), ("passed", "tb_lib.tb_composite_generics.tiny.Test 1"), ], ) @unittest.skipUnless(simulator_is("ghdl"), "Support complex JSON strings as generic") def test_vhdl_json4vhdl_example_project(self): self.check(str(ROOT / "examples" / "vhdl" / "json4vhdl" / "run.py")) def test_vhdl_array_example_project(self): self.check(str(ROOT / "examples" / "vhdl" / "array" / "run.py")) def test_vhdl_array_axis_vcs_example_project(self): self.check( str(ROOT / "examples" / "vhdl" / "array_axis_vcs" / "run.py")) def test_vhdl_axi_dma_example_project(self): self.check(str(ROOT / "examples" / "vhdl" / "axi_dma" / "run.py")) def test_vhdl_user_guide_example_project(self): self.check(str(ROOT / "examples" / "vhdl" / "user_guide" / "run.py"), exit_code=1) check_report( self.report_file, [ ("passed", "lib.tb_example.all"), ("passed", "lib.tb_example_many.test_pass"), ("failed", "lib.tb_example_many.test_fail"), ], ) @unittest.skipUnless(simulator_supports_verilog(), "Verilog") def test_verilog_user_guide_example_project(self): self.check(str(ROOT / "examples" / "verilog" / "user_guide" / "run.py"), exit_code=1) check_report( self.report_file, [ ("passed", "lib.tb_example_basic.all"), ("passed", "lib.tb_example.Test that a successful test case passes"), ( "failed", "lib.tb_example.Test that a failing test case actually fails", ), ( "failed", "lib.tb_example.Test that a test case that takes too long time fails with a timeout", ), ], ) def test_vhdl_com_example_project(self): self.check(str(ROOT / "examples" / "vhdl" / "com" / "run.py")) def test_array_vhdl_2008(self): self.check(str(VHDL_PATH / "array" / "run.py")) def test_data_types_vhdl_2008(self): self.check(str(VHDL_PATH / "data_types" / "run.py")) def test_data_types_vhdl_2002(self): self.check(str(VHDL_PATH / "data_types" / "run.py"), vhdl_standard="2002") def test_data_types_vhdl_93(self): self.check(str(VHDL_PATH / "data_types" / "run.py"), vhdl_standard="93") def test_random_vhdl_2008(self): self.check(str(VHDL_PATH / "random" / "run.py")) def test_check_vhdl_2008(self): self.check(str(VHDL_PATH / "check" / "run.py")) def test_check_vhdl_2002(self): self.check(str(VHDL_PATH / "check" / "run.py"), vhdl_standard="2002") def test_check_vhdl_93(self): self.check(str(VHDL_PATH / "check" / "run.py"), vhdl_standard="93") def test_logging_vhdl_2008(self): self.check(str(VHDL_PATH / "logging" / "run.py")) def test_logging_vhdl_2002(self): self.check(str(VHDL_PATH / "logging" / "run.py"), vhdl_standard="2002") def test_logging_vhdl_93(self): self.check(str(VHDL_PATH / "logging" / "run.py"), vhdl_standard="93") def test_run_vhdl_2008(self): self.check(str(VHDL_PATH / "run" / "run.py")) def test_run_vhdl_2002(self): self.check(str(VHDL_PATH / "run" / "run.py"), vhdl_standard="2002") def test_run_vhdl_93(self): self.check(str(VHDL_PATH / "run" / "run.py"), vhdl_standard="93") def test_string_ops_vhdl_2008(self): self.check(str(VHDL_PATH / "string_ops" / "run.py")) def test_string_ops_vhdl_2002(self): self.check(str(VHDL_PATH / "string_ops" / "run.py"), vhdl_standard="2002") def test_string_ops_vhdl_93(self): self.check(str(VHDL_PATH / "string_ops" / "run.py"), vhdl_standard="93") def test_dictionary_vhdl_2008(self): self.check(str(VHDL_PATH / "dictionary" / "run.py")) def test_dictionary_vhdl_2002(self): self.check(str(VHDL_PATH / "dictionary" / "run.py"), vhdl_standard="2002") def test_dictionary_vhdl_93(self): self.check(str(VHDL_PATH / "dictionary" / "run.py"), vhdl_standard="93") def test_path_vhdl_2008(self): self.check(str(VHDL_PATH / "path" / "run.py")) def test_path_vhdl_2002(self): self.check(str(VHDL_PATH / "path" / "run.py"), vhdl_standard="2002") def test_path_vhdl_93(self): self.check(str(VHDL_PATH / "path" / "run.py"), vhdl_standard="93") def test_com_vhdl_2008(self): self.check(str(VHDL_PATH / "com" / "run.py")) def setUp(self): self.output_path = str(Path(__file__).parent / "external_run_out") self.report_file = str(Path(self.output_path) / "xunit.xml") def check(self, run_file: Path, args=None, vhdl_standard="2008", exit_code=0): """ Run external run file and verify exit code """ args = args if args is not None else [] new_env = environ.copy() new_env["VUNIT_VHDL_STANDARD"] = vhdl_standard retcode = call( [ sys.executable, str(run_file), "--clean", "--output-path=%s" % self.output_path, "--xunit-xml=%s" % self.report_file, ] + args, env=new_env, ) self.assertEqual(retcode, exit_code)
def simulator_supports_verilog(): """ Returns True if simulator supports Verilog """ return simulator_is("modelsim", "incisive")