def test_verilog_user_guide_example_project(self):
self.check(join(ROOT, "examples", "verilog", "user_guide", "run.py"), exit_code=1)
check_report(self.report_file,
[("passed", "lib.tb_example_basic.all"),
("passed", "lib.tb_example.Test that a successful test case passes"),
("failed", "lib.tb_example.Test that a failing test case actually fails"),
("failed", "lib.tb_example.Test that a test case that takes too long time fails with a timeout")])
def test_vhdl_run_example_project(self):
self.check(join(ROOT, "examples", "vhdl", "run", "run.py"),
exit_code=1)
check_report(self.report_file, [
("passed", "lib.tb_with_watchdog.Test to_string for boolean"),
("passed", "lib.tb_with_watchdog.Test that needs longer timeout"),
("passed", "lib.tb_standalone.Test to_string for boolean"),
("passed", "lib.tb_with_test_cases.Test to_string for integer"),
("passed", "lib.tb_with_test_cases.Test to_string for boolean"),
("passed", "lib.tb_with_lower_level_control.Test something"),
("passed", "lib.tb_with_lower_level_control.Test something else"),
("passed", "lib.tb_running_test_case.Test scenario A"),
("passed", "lib.tb_running_test_case.Test scenario B"),
("passed", "lib.tb_running_test_case.Test something else"),
("passed", "lib.tb_minimal.all"),
("passed", "lib.tb_magic_paths.all"),
("failed", "lib.tb_with_watchdog.Test that stalls"),
("failed",
"lib.tb_counting_errors.Test that fails multiple times but doesn't stop"
),
("failed",
"lib.tb_standalone.Test that fails on VUnit check procedure"),
("failed",
"lib.tb_many_ways_to_fail.Test that fails on an assert"),
("failed",
"lib.tb_many_ways_to_fail.Test that crashes on boundary problems"
),
("failed",
"lib.tb_many_ways_to_fail.Test that fails on VUnit check procedure"
)
])
def test_vhdl_user_guide_example_project(self):
self.check(join(ROOT, "examples", "vhdl", "user_guide", "run.py"),
exit_code=1)
check_report(self.report_file,
[("passed", "lib.tb_example.all"),
("passed", "lib.tb_example_many.test_pass"),
("failed", "lib.tb_example_many.test_fail")])
def test_artificial_verilog(self):
self.check(self.artificial_run_verilog, exit_code=1)
check_report(
self.report_file,
[("passed", "lib.tb_other_file_tests.pass"),
("failed", "lib.tb_other_file_tests.fail"),
("passed", "lib.tb_magic_paths.Test magic paths are correct"),
("passed", "lib.tb_with_define.test 1"),
("failed", "lib.tb_fail_on_warning.fail"),
("failed", "lib.tb_fail_on_fatal_and_early_finish.fatal0"),
("failed", "lib.tb_fail_on_fatal_and_early_finish.fatal1"),
("failed", "lib.tb_fail_on_fatal_and_early_finish.finish0"),
("failed", "lib.tb_fail_on_fatal_and_early_finish.finish1"),
("passed", "lib.tb_with_parameter_config.Test 0"),
("passed", "lib.tb_with_parameter_config.cfg.Test 1"),
("passed", "lib.tb_with_parameter_config.Test 2"),
("passed", "lib.tb_with_parameter_config.cfg.Test 3"),
("passed", "lib.tb_with_parameter_config.cfg.Test 4"),
("passed", "lib.tb_same_sim_all_pass.cfg.Test 1"),
("passed", "lib.tb_same_sim_all_pass.cfg.Test 2"),
("passed", "lib.tb_same_sim_all_pass.cfg.Test 3"),
("passed", "lib.tb_same_sim_some_fail.Test 1"),
("failed", "lib.tb_same_sim_some_fail.Test 2"),
("skipped", "lib.tb_same_sim_some_fail.Test 3"),
("passed", "lib.tb_with_runner.pass"),
("failed", "lib.tb_with_runner.fail")])
def test_artificial_verilog(self):
self.check(self.artificial_run_verilog,
exit_code=1)
check_report(self.report_file, [
("passed", "lib.tb_other_file_tests.pass"),
("failed", "lib.tb_other_file_tests.fail"),
("passed", "lib.tb_magic_paths.Test magic paths are correct"),
("passed", "lib.tb_with_define.test 1"),
("failed", "lib.tb_fail_on_warning.fail"),
("failed", "lib.tb_fail_on_fatal_and_early_finish.fatal0"),
("failed", "lib.tb_fail_on_fatal_and_early_finish.fatal1"),
("failed", "lib.tb_fail_on_fatal_and_early_finish.finish0"),
("failed", "lib.tb_fail_on_fatal_and_early_finish.finish1"),
("passed", "lib.tb_with_parameter_config.Test 0"),
("passed", "lib.tb_with_parameter_config.Test 1"),
("passed", "lib.tb_with_parameter_config.Test 2"),
("passed", "lib.tb_with_parameter_config.Test 3"),
("passed", "lib.tb_with_parameter_config.Test 4"),
("passed", "lib.tb_same_sim_all_pass.Test 1"),
("passed", "lib.tb_same_sim_all_pass.Test 2"),
("passed", "lib.tb_same_sim_all_pass.Test 3"),
("passed", "lib.tb_same_sim_some_fail.Test 1"),
("failed", "lib.tb_same_sim_some_fail.Test 2"),
("skipped", "lib.tb_same_sim_some_fail.Test 3"),
("passed", "lib.tb_with_runner.pass"),
("failed", "lib.tb_with_runner.fail")])
def _test_artificial(self, args=None):
"""
Utility function to run and check the result of all test benches
using either persistent or non-persistent simulator interface mode
"""
self.check(self.artificial_run_vhdl, exit_code=1, args=args)
check_report(self.report_file, EXPECTED_REPORT)
def test_vhdl_third_party_integration_example_project(self):
self.check(join(ROOT, "examples", "vhdl", "third_party_integration", "run.py"), exit_code=1)
check_report(self.report_file,
[("passed", "lib.tb_external_framework_integration.Test that pass"),
("failed",
"lib.tb_external_framework_integration.Test that stops the simulation on first error"),
("failed",
"lib.tb_external_framework_integration.Test that doesn't stop the simulation on error")])
def test_verilog_ams_example(self):
self.check(join(ROOT, "examples", "verilog", "verilog_ams", "run.py"))
check_report(
self.report_file,
[
("passed", "lib.tb_dut.Test that pass"),
("failed", "lib.tb_dut.Test that fail"),
],
)
def _test_artificial(self, args=None):
"""
Utility function to run and check the result of all test benches
using either persistent or non-persistent simulator interface mode
"""
self.check(self.artificial_run_vhdl,
exit_code=1,
args=args)
check_report(self.report_file, EXPECTED_REPORT)
def test_vhdl_composite_generics_example_project(self):
self.check(
join(ROOT, "examples", "vhdl", "composite_generics", "run.py"))
check_report(
self.report_file,
[
("passed", "tb_lib.tb_composite_generics.VGA.Test 1"),
("passed", "tb_lib.tb_composite_generics.tiny.Test 1"),
],
)
def test_vhdl_osvvm_integration_example_project(self):
self.check(join(ROOT, "examples", "vhdl", "osvvm_integration", "run.py"), exit_code=1)
check_report(self.report_file,
[("passed", "lib.tb_alertlog_demo_global_with_comments.Test passing alerts"),
("passed", "lib.tb_alertlog_demo_hierarchy_with_comments.Test passing alerts"),
("passed", "lib.tb_alertlog_demo_global.Test passing alerts"),
("passed", "lib.tb_alertlog_demo_hierarchy.Test passing alerts"),
("failed", "lib.tb_alertlog_demo_global_with_comments.Test failing alerts"),
("failed", "lib.tb_alertlog_demo_hierarchy_with_comments.Test failing alerts"),
("failed", "lib.tb_alertlog_demo_global.Test failing alerts"),
("failed", "lib.tb_alertlog_demo_hierarchy.Test failing alerts")])
def test_vhdl_generate_tests_example_project(self):
self.check(join(ROOT, "examples", "vhdl", "generate_tests", "run.py"))
check_report(self.report_file,
[("passed", "lib.tb_generated.data_width=1,sign=False.Test 1"),
("passed", "lib.tb_generated.data_width=1,sign=True.Test 1"),
("passed", "lib.tb_generated.data_width=2,sign=False.Test 1"),
("passed", "lib.tb_generated.data_width=2,sign=True.Test 1"),
("passed", "lib.tb_generated.data_width=3,sign=False.Test 1"),
("passed", "lib.tb_generated.data_width=3,sign=True.Test 1"),
("passed", "lib.tb_generated.data_width=4,sign=False.Test 1"),
("passed", "lib.tb_generated.data_width=4,sign=True.Test 1"),
("passed", "lib.tb_generated.data_width=16,sign=True.Test 2")])
def _test_run_selected_tests_in_same_sim_test_bench(self, run_file):
"""
Run selected "same_sim" test in isolation
"""
self.check(run_file,
exit_code=0,
clean=True,
args=["*same_sim_some_fail*Test 1*"])
check_report(self.report_file, [
("passed", "lib.tb_same_sim_some_fail.Test 1")])
self.check(run_file,
exit_code=1,
clean=False,
args=["*same_sim_some_fail*Test 2*"])
check_report(self.report_file, [
("failed", "lib.tb_same_sim_some_fail.Test 2")])
self.check(run_file,
exit_code=0,
clean=False,
args=["*same_sim_some_fail*Test 3*"])
check_report(self.report_file, [
("passed", "lib.tb_same_sim_some_fail.Test 3")])
self.check(run_file,
exit_code=1,
clean=False,
args=["*same_sim_some_fail*Test 2*", "*same_sim_some_fail*Test 3*"])
check_report(self.report_file, [
("failed", "lib.tb_same_sim_some_fail.Test 2"),
("skipped", "lib.tb_same_sim_some_fail.Test 3")])
def _test_run_selected_tests_in_same_sim_test_bench(self, run_file):
"""
Run selected "same_sim" test in isolation
"""
self.check(run_file,
exit_code=0,
clean=True,
args=["*same_sim_some_fail*Test 1*"])
check_report(self.report_file,
[("passed", "lib.tb_same_sim_some_fail.Test 1")])
self.check(run_file,
exit_code=1,
clean=False,
args=["*same_sim_some_fail*Test 2*"])
check_report(self.report_file,
[("failed", "lib.tb_same_sim_some_fail.Test 2")])
self.check(run_file,
exit_code=0,
clean=False,
args=["*same_sim_some_fail*Test 3*"])
check_report(self.report_file,
[("passed", "lib.tb_same_sim_some_fail.Test 3")])
self.check(run_file,
exit_code=1,
clean=False,
args=[
"*same_sim_some_fail*Test 2*",
"*same_sim_some_fail*Test 3*"
])
check_report(self.report_file,
[("failed", "lib.tb_same_sim_some_fail.Test 2"),
("skipped", "lib.tb_same_sim_some_fail.Test 3")])
def test_vhdl_run_example_project(self):
self.check(join(ROOT, "examples", "vhdl", "run", "run.py"), exit_code=1)
check_report(self.report_file,
[("passed", "lib.tb_with_watchdog.Test to_string for boolean"),
("passed", "lib.tb_with_watchdog.Test that needs longer timeout"),
("passed", "lib.tb_standalone.Test to_string for boolean"),
("passed", "lib.tb_with_test_cases.Test to_string for integer"),
("passed", "lib.tb_with_test_cases.Test to_string for boolean"),
("passed", "lib.tb_with_lower_level_control.Test something"),
("passed", "lib.tb_with_lower_level_control.Test something else"),
("passed", "lib.tb_running_test_case.Test scenario A"),
("passed", "lib.tb_running_test_case.Test scenario B"),
("passed", "lib.tb_running_test_case.Test something else"),
("passed", "lib.tb_minimal.all"),
("passed", "lib.tb_magic_paths.all"),
("failed", "lib.tb_with_watchdog.Test that stalls"),
("failed", "lib.tb_counting_errors.Test that fails multiple times but doesn't stop"),
("failed", "lib.tb_standalone.Test that fails on VUnit check procedure"),
("failed", "lib.tb_many_ways_to_fail.Test that fails on an assert"),
("failed", "lib.tb_many_ways_to_fail.Test that crashes on boundary problems"),
("failed", "lib.tb_many_ways_to_fail.Test that fails on VUnit check procedure")])
def test_artificial_elaborate_only(self):
self.check(self.artificial_run_vhdl, exit_code=1, args=["--elaborate"])
elab_expected_report = []
for status, name in EXPECTED_REPORT:
if name in ("lib.tb_elab_fail.all",):
status = "failed"
else:
status = "passed"
elab_expected_report.append((status, name))
check_report(self.report_file, elab_expected_report)
self.check(
self.artificial_run_vhdl,
exit_code=0,
clean=False,
args=["--elaborate", "lib.tb_pass.all"],
)
check_report(self.report_file, [("passed", "lib.tb_pass.all")])
self.check(
self.artificial_run_vhdl,
exit_code=1,
clean=False,
args=["--elaborate", "lib.tb_elab_fail.all"],
)
check_report(self.report_file, [("failed", "lib.tb_elab_fail.all")])
def test_artificial_elaborate_only(self):
self.check(self.artificial_run_vhdl,
exit_code=1,
args=["--elaborate"])
elab_expected_report = []
for status, name in EXPECTED_REPORT:
if name in ("lib.tb_elab_fail.all",):
status = "failed"
else:
status = "passed"
elab_expected_report.append((status, name))
check_report(self.report_file, elab_expected_report)
self.check(self.artificial_run_vhdl,
exit_code=0,
clean=False,
args=["--elaborate", "lib.tb_pass.all"])
check_report(self.report_file, [
("passed", "lib.tb_pass.all")])
self.check(self.artificial_run_vhdl,
exit_code=1,
clean=False,
args=["--elaborate", "lib.tb_elab_fail.all"])
check_report(self.report_file, [
("failed", "lib.tb_elab_fail.all")])
def test_verilog_user_guide_example_project(self):
self.check(join(ROOT, "examples", "verilog", "user_guide", "run.py"), exit_code=1)
check_report(self.report_file,
[("passed", "lib.tb_example.Test that pass"),
("failed", "lib.tb_example.Test that fail"),
("failed", "lib.tb_example.Test that timeouts")])
def test_vhdl_composite_generics_example_project(self):
self.check(join(ROOT, "examples", "vhdl", "composite_generics", "run.py"))
check_report(self.report_file,
[("passed", "tb_lib.tb_composite_generics.VGA.Test 1"),
("passed", "tb_lib.tb_composite_generics.tiny.Test 1")])
def test_vhdl_user_guide_example_project(self):
self.check(join(ROOT, "examples", "vhdl", "user_guide", "run.py"), exit_code=1)
check_report(self.report_file,
[("passed", "lib.tb_example.all"),
("passed", "lib.tb_example_many.test_pass"),
("failed", "lib.tb_example_many.test_fail")])
check_report(report_file, [
('passed',
'tb_synchronization_lib.tb_synchronization.Test that an event is initially cleared'
),
('passed',
'tb_synchronization_lib.tb_synchronization.Test that an event can be set and cleared'
),
('passed',
'tb_synchronization_lib.tb_synchronization.Test that a wait on a set event does not block'
),
('passed',
'tb_synchronization_lib.tb_synchronization.Test that a cleared event blocks a wait call'
),
('passed',
'tb_synchronization_lib.tb_synchronization.Test that a wait with timeout blocks until event is set if that happens before the timeout'
),
('passed',
'tb_synchronization_lib.tb_synchronization.Test that a wait with timeout times out if timeout happens before event is set'
),
('passed',
'tb_synchronization_lib.tb_synchronization.Test that a negative timeout behaves like a zero timeout'
),
('passed',
'tb_synchronization_lib.tb_synchronization_with_other_checks.Test that an event is initially cleared using VHDL assert'
),
('passed',
'tb_synchronization_lib.tb_synchronization_with_other_checks.Test that an event is initially cleared using OSVVM'
),
('passed',
'tb_synchronization_lib.tb_synchronization_with_other_checks.Test that an event is initially cleared using UVVM'
),
('failed',
'tb_synchronization_lib.tb_synchronization.Test feature that will never be supported using VUnit'
),
('failed',
'tb_synchronization_lib.tb_synchronization_with_other_checks.Test feature that will never be supported using VHDL assert'
),
('failed',
'tb_synchronization_lib.tb_synchronization_with_other_checks.Test feature that will never be supported using OSVVM'
),
('failed',
'tb_synchronization_lib.tb_synchronization_with_other_checks.Test feature that will never be supported using UVVM'
)
])
def test_verilog_ams_example(self):
self.check(join(ROOT, "examples", "verilog", "verilog_ams", "run.py"))
check_report(self.report_file,
[("passed", "lib.tb_dut.Test that pass"),
("failed", "lib.tb_dut.Test that fail")])
