def _main(self):
"""
Base vunit main function without performing exit
"""
if self._list_only:
return self._main_list_only()
if self._compile_only:
return self._main_compile_only()
simulator_if = self._create_simulator_if()
test_cases = self._create_tests(simulator_if)
self._compile(simulator_if)
start_time = ostools.get_time()
report = TestReport(printer=self._printer)
try:
self._run_test(test_cases, report)
simulator_if.post_process(self._simulator_factory.simulator_output_path)
except KeyboardInterrupt:
print()
LOGGER.debug("_main: Caught Ctrl-C shutting down")
finally:
del test_cases
del simulator_if
report.set_real_total_time(ostools.get_time() - start_time)
self._post_process(report)
return report.all_ok()
def _main(self):
"""
Base vunit main function without performing exit
"""
if self._list_only:
return self._main_list_only()
if self._list_files_only:
return self._main_list_files_only()
if self._compile_only:
return self._main_compile_only()
simulator_if = self._create_simulator_if()
test_cases = self._create_tests(simulator_if)
self._compile(simulator_if)
start_time = ostools.get_time()
report = TestReport(printer=self._printer)
try:
self._run_test(test_cases, report)
simulator_if.post_process(self._simulator_factory.simulator_output_path)
except KeyboardInterrupt:
print()
LOGGER.debug("_main: Caught Ctrl-C shutting down")
finally:
del test_cases
del simulator_if
report.set_real_total_time(ostools.get_time() - start_time)
self._post_process(report)
return report.all_ok()
def test_can_read_output(self, tempdir):
report = TestReport()
runner = TestRunner(report, tempdir)
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
def side_effect(read_output, **kwargs): # pylint: disable=unused-argument
"""
Side effect that print output to stdout
"""
print("out1", end="")
print("out2", end="")
assert read_output() == "out1out2"
print("out3", end="")
print("out4", end="")
assert read_output() == "out1out2out3out4"
print("out5", end="")
return True
test_case.run_side_effect = side_effect
runner.run(test_list)
self.assertTrue(report.result_of("test").passed)
self.assertEqual(report.result_of("test").output, "out1out2out3out4out5")
def test_can_read_output(self, tempdir):
report = TestReport()
runner = TestRunner(report, tempdir)
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
def side_effect(read_output, **kwargs): # pylint: disable=unused-argument
"""
Side effect that print output to stdout
"""
print("out1", end="")
print("out2", end="")
assert read_output() == "out1out2"
print("out3", end="")
print("out4", end="")
assert read_output() == "out1out2out3out4"
print("out5", end="")
return True
test_case.run_side_effect = side_effect
runner.run(test_list)
self.assertTrue(report.result_of("test").passed)
self.assertEqual(report.result_of("test").output, "out1out2out3out4out5")
class TestTestRunner(unittest.TestCase):
def setUp(self):
self._tests = []
self.output_path = join(dirname(__file__), "test_runner_out")
if exists(self.output_path):
rmtree(self.output_path)
self.report = TestReport()
self.runner = TestRunner(self.report, self.output_path)
def test_runs_testcases_in_order(self):
test_case1 = self.create_test("test1", True)
test_case2 = self.create_test("test2", False)
test_list = TestList()
test_list.add_test(test_case1)
test_list.add_test(test_case2)
self.runner.run(test_list)
test_case1.run.assert_called_once_with(join(self.output_path, "test1"))
test_case2.run.assert_called_once_with(join(self.output_path, "test2"))
self.assertEqual(self._tests, ["test1", "test2"])
self.assertTrue(self.report.result_of("test1").passed)
self.assertTrue(self.report.result_of("test2").failed)
def test_handles_python_exeception(self):
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
test_case.run.side_effect = KeyError
self.runner.run(test_list)
self.assertTrue(self.report.result_of("test").failed)
def test_collects_output(self):
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
output = "Output string, <xml>, </xml>\n"
def side_effect(*args, **kwargs):
print(output, end="")
return True
test_case.run.side_effect = side_effect
self.runner.run(test_list)
self.assertTrue(self.report.result_of("test").passed)
self.assertEqual(self.report.result_of("test").output, output)
def create_test(self, name, passed):
test_case = Mock(spec_set=TestCaseMockSpec)
test_case.configure_mock(name=name)
def run_side_effect(*args, **kwargs):
self._tests.append(name)
return passed
test_case.run.side_effect = run_side_effect
return test_case
def setUp(self):
self._tests = []
self.output_path = join(dirname(__file__), "test_runner_out")
if exists(self.output_path):
rmtree(self.output_path)
self.report = TestReport()
self.runner = TestRunner(self.report, self.output_path)
def test_handles_python_exeception(self):
report = TestReport()
runner = TestRunner(report, self.output_path)
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
test_case.run.side_effect = KeyError
runner.run(test_list)
self.assertTrue(report.result_of("test").failed)
def test_handles_python_exeception(self, tempdir):
report = TestReport()
runner = TestRunner(report, tempdir)
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
def side_effect(*args, **kwargs): # pylint: disable=unused-argument
raise KeyError
test_case.run_side_effect = side_effect
runner.run(test_list)
self.assertTrue(report.result_of("test").failed)
def test_handles_python_exeception(self, tempdir):
report = TestReport()
runner = TestRunner(report, tempdir)
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
def side_effect(*args, **kwargs): # pylint: disable=unused-argument
raise KeyError
test_case.run_side_effect = side_effect
runner.run(test_list)
self.assertTrue(report.result_of("test").failed)
def _run_test(self, test_cases):
report = TestReport(printer=self._printer)
runner = TestRunner(report,
join(self._output_path, "tests"),
verbose=self._verbose)
runner.run(test_cases)
return report
def test_runs_testcases_in_order(self):
report = TestReport()
runner = TestRunner(report, self.output_path)
test_case1 = self.create_test("test1", True)
test_case2 = self.create_test("test2", False)
test_case3 = self.create_test("test3", True)
test_list = TestList()
test_list.add_test(test_case1)
test_list.add_test(test_case2)
runner.run(test_list)
test_case1.run.assert_called_once_with(
create_output_path(self.output_path, "test1"))
test_case2.run.assert_called_once_with(
create_output_path(self.output_path, "test2"))
self.assertFalse(test_case3.run.called)
self.assertEqual(self._tests, ["test1", "test2"])
self.assertTrue(report.result_of("test1").passed)
self.assertTrue(report.result_of("test2").failed)
def test_runs_testcases_in_order(self, tempdir):
report = TestReport()
runner = TestRunner(report, tempdir)
order = []
test_case1 = self.create_test("test1", True, order=order)
test_case2 = self.create_test("test2", False, order=order)
test_case3 = self.create_test("test3", True, order=order)
test_list = TestList()
test_list.add_test(test_case1)
test_list.add_test(test_case2)
test_list.add_test(test_case3)
runner.run(test_list)
self.assertEqual(test_case1.output_path, create_output_path(tempdir, "test1"))
self.assertEqual(test_case2.output_path, create_output_path(tempdir, "test2"))
self.assertEqual(test_case3.output_path, create_output_path(tempdir, "test3"))
self.assertEqual(order, ["test1", "test2", "test3"])
self.assertTrue(report.result_of("test1").passed)
self.assertTrue(report.result_of("test2").failed)
self.assertTrue(report.result_of("test3").passed)
def test_runs_testcases_in_order(self, tempdir):
report = TestReport()
runner = TestRunner(report, tempdir)
order = []
test_case1 = self.create_test("test1", True, order=order)
test_case2 = self.create_test("test2", False, order=order)
test_case3 = self.create_test("test3", True, order=order)
test_list = TestList()
test_list.add_test(test_case1)
test_list.add_test(test_case2)
test_list.add_test(test_case3)
runner.run(test_list)
self.assertEqual(test_case1.output_path, create_output_path(tempdir, "test1"))
self.assertEqual(test_case2.output_path, create_output_path(tempdir, "test2"))
self.assertEqual(test_case3.output_path, create_output_path(tempdir, "test3"))
self.assertEqual(order, ["test1", "test2", "test3"])
self.assertTrue(report.result_of("test1").passed)
self.assertTrue(report.result_of("test2").failed)
self.assertTrue(report.result_of("test3").passed)
def test_fail_fast(self):
report = TestReport()
runner = TestRunner(report, self.output_path, fail_fast=True)
test_case1 = self.create_test("test1", True)
test_case2 = self.create_test("test2", False)
test_list = TestList()
test_list.add_test(test_case1)
test_list.add_test(test_case2)
try:
runner.run(test_list)
except KeyboardInterrupt:
pass
test_case1.run.assert_called_once_with(
create_output_path(self.output_path, "test1"))
test_case2.run.assert_called_once_with(
create_output_path(self.output_path, "test2"))
self.assertEqual(self._tests, ["test1", "test2"])
self.assertTrue(report.result_of("test1").passed)
self.assertTrue(report.result_of("test2").failed)
def test_collects_output(self):
report = TestReport()
runner = TestRunner(report, self.output_path)
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
output = "Output string, <xml>, </xml>\n"
def side_effect(*args, **kwargs): # pylint: disable=unused-argument
"""
Side effect that print output to stdout
"""
print(output, end="")
return True
test_case.run.side_effect = side_effect
runner.run(test_list)
self.assertTrue(report.result_of("test").passed)
self.assertEqual(report.result_of("test").output, output)
def test_collects_output(self, tempdir):
report = TestReport()
runner = TestRunner(report, tempdir)
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
output = "Output string, <xml>, </xml>\n"
def side_effect(*args, **kwargs): # pylint: disable=unused-argument
"""
Side effect that print output to stdout
"""
print(output, end="")
return True
test_case.run_side_effect = side_effect
runner.run(test_list)
self.assertTrue(report.result_of("test").passed)
self.assertEqual(report.result_of("test").output, output)
def test_fail_fast(self, tempdir):
report = TestReport()
runner = TestRunner(report, tempdir, fail_fast=True)
order = []
test_case1 = self.create_test("test1", True, order=order)
test_case2 = self.create_test("test2", False, order=order)
test_case3 = self.create_test("test3", True, order=order)
test_list = TestList()
test_list.add_test(test_case1)
test_list.add_test(test_case2)
test_list.add_test(test_case3)
try:
runner.run(test_list)
except KeyboardInterrupt:
pass
self.assertEqual(test_case1.output_path, create_output_path(tempdir, "test1"))
self.assertEqual(test_case2.output_path, create_output_path(tempdir, "test2"))
self.assertEqual(test_case3.called, False)
self.assertEqual(order, ["test1", "test2"])
self.assertTrue(report.result_of("test1").passed)
self.assertTrue(report.result_of("test2").failed)
def test_fail_fast(self, tempdir):
report = TestReport()
runner = TestRunner(report, tempdir, fail_fast=True)
order = []
test_case1 = self.create_test("test1", True, order=order)
test_case2 = self.create_test("test2", False, order=order)
test_case3 = self.create_test("test3", True, order=order)
test_list = TestList()
test_list.add_test(test_case1)
test_list.add_test(test_case2)
test_list.add_test(test_case3)
try:
runner.run(test_list)
except KeyboardInterrupt:
pass
self.assertEqual(test_case1.output_path, create_output_path(tempdir, "test1"))
self.assertEqual(test_case2.output_path, create_output_path(tempdir, "test2"))
self.assertEqual(test_case3.called, False)
self.assertEqual(order, ["test1", "test2"])
self.assertTrue(report.result_of("test1").passed)
self.assertTrue(report.result_of("test2").failed)
def setUp(self):
self._tests = []
self.output_path = join(dirname(__file__), "test_runner_out")
renew_path(self.output_path)
self.report = TestReport()
self.runner = TestRunner(self.report, self.output_path)
class TestTestRunner(unittest.TestCase):
"""
Test the test runner
"""
def setUp(self):
self._tests = []
self.output_path = join(dirname(__file__), "test_runner_out")
renew_path(self.output_path)
self.report = TestReport()
self.runner = TestRunner(self.report, self.output_path)
def test_runs_testcases_in_order(self):
test_case1 = self.create_test("test1", True)
test_case2 = self.create_test("test2", False)
test_list = TestList()
test_list.add_test(test_case1)
test_list.add_test(test_case2)
self.runner.run(test_list)
test_case1.run.assert_called_once_with(
create_output_path(self.output_path, "test1"))
test_case2.run.assert_called_once_with(
create_output_path(self.output_path, "test2"))
self.assertEqual(self._tests, ["test1", "test2"])
self.assertTrue(self.report.result_of("test1").passed)
self.assertTrue(self.report.result_of("test2").failed)
def test_handles_python_exeception(self):
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
test_case.run.side_effect = KeyError
self.runner.run(test_list)
self.assertTrue(self.report.result_of("test").failed)
def test_collects_output(self):
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
output = "Output string, <xml>, </xml>\n"
def side_effect(*args, **kwargs): # pylint: disable=unused-argument
"""
Side effect that print output to stdout
"""
print(output, end="")
return True
test_case.run.side_effect = side_effect
self.runner.run(test_list)
self.assertTrue(self.report.result_of("test").passed)
self.assertEqual(self.report.result_of("test").output, output)
def test_create_output_path_on_linux(self):
with mock.patch("sys.platform", new="linux"):
with mock.patch("os.environ", new={}):
output_path = "output_path"
test_name = "_" * 400
test_output = create_output_path(output_path, test_name)
self.assertEqual(
test_output,
join(abspath(output_path),
test_name + "_" + hash_string(test_name)))
output_path = "output_path"
test_name = "123._-+"
test_output = create_output_path(output_path, test_name)
self.assertEqual(
test_output,
join(abspath(output_path),
test_name + "_" + hash_string(test_name)))
output_path = "output_path"
test_name = "#<>:"
safe_name = "____"
test_output = create_output_path(output_path, test_name)
self.assertEqual(
test_output,
join(abspath(output_path),
safe_name + "_" + hash_string(test_name)))
def test_create_output_path_on_windows(self):
with mock.patch("sys.platform", new="win32"):
with mock.patch("os.environ", new={}):
output_path = "output_path"
test_name = "_" * 400
test_output = create_output_path(output_path, test_name)
self.assertEqual(len(test_output), 260 - 100 + 1)
with mock.patch("os.environ",
new={"VUNIT_TEST_OUTPUT_PATH_MARGIN": "-1000"}):
output_path = "output_path"
test_name = "_" * 400
test_output = create_output_path(output_path, test_name)
self.assertEqual(
test_output,
join(abspath(output_path),
test_name + "_" + hash_string(test_name)))
with mock.patch("os.environ",
new={"VUNIT_SHORT_TEST_OUTPUT_PATHS": ""}):
output_path = "output_path"
test_name = "_" * 400
test_output = create_output_path(output_path, test_name)
self.assertEqual(
test_output,
join(abspath(output_path), hash_string(test_name)))
def create_test(self, name, passed):
"""
Utility function to create a mocked test with name
that is either passed or failed
"""
test_case = mock.Mock(spec_set=TestCaseMockSpec)
test_case.configure_mock(name=name)
def run_side_effect(*args, **kwargs): # pylint: disable=unused-argument
"""
Side effect that registers that is has been run
"""
self._tests.append(name)
return passed
test_case.run.side_effect = run_side_effect
return test_case
def setUp(self):
self._tests = []
self.output_path = join(dirname(__file__), "test_runner_out")
renew_path(self.output_path)
self.report = TestReport()
self.runner = TestRunner(self.report, self.output_path)
def run_with_compile_errors(ui, args, vhdl_standard):
def match(name, patterns):
return reduce(
lambda found_match, pattern: found_match | fnmatch(name, pattern),
patterns,
False,
)
# Run all tests in isolation to handle failure to compile
args.minimal = True
original_test_patterns = args.test_patterns
test_report = TestReport()
n_tests = 0
testbenches = (ui.library("vhdl_2008").get_test_benches() +
ui.library("vhdl_2019").get_test_benches())
total_start_time = ostools.get_time()
for tb in testbenches:
tests = tb.get_tests()
if not tests:
test_names = ["all"]
else:
test_names = [test.name for test in tests]
for test_name in test_names:
full_test_name = "%s.%s.%s" % (tb.library.name, tb.name, test_name)
if not match(full_test_name, original_test_patterns):
continue
test_start_time = ostools.get_time()
n_tests += 1
args.test_patterns = [full_test_name]
ui = VUnit.from_args(args, vhdl_standard=vhdl_standard)
vhdl_2008 = ui.add_library("vhdl_2008")
vhdl_2019 = ui.add_library("vhdl_2019")
vhdl_2008.add_source_files(join(root, "vhdl_2008", "*.vhd"))
vhdl_2019.add_source_files(join(root, "vhdl_2019", "*.vhd"))
try:
ui.main()
except SystemExit as ex:
tb_time = ostools.get_time() - test_start_time
if ex.code == 0:
test_report.add_result(full_test_name, PASSED, tb_time,
None)
else:
test_report.add_result(full_test_name, FAILED, tb_time,
None)
print("\nCompliance test completed:\n")
test_report.set_expected_num_tests(n_tests)
test_report.set_real_total_time(ostools.get_time() - total_start_time)
test_report.print_str()
class TestTestRunner(unittest.TestCase):
"""
Test the test runner
"""
def setUp(self):
self._tests = []
self.output_path = join(dirname(__file__), "test_runner_out")
renew_path(self.output_path)
self.report = TestReport()
self.runner = TestRunner(self.report, self.output_path)
def test_runs_testcases_in_order(self):
test_case1 = self.create_test("test1", True)
test_case2 = self.create_test("test2", False)
test_list = TestList()
test_list.add_test(test_case1)
test_list.add_test(test_case2)
self.runner.run(test_list)
test_case1.run.assert_called_once_with(join(self.output_path, "test1"))
test_case2.run.assert_called_once_with(join(self.output_path, "test2"))
self.assertEqual(self._tests, ["test1", "test2"])
self.assertTrue(self.report.result_of("test1").passed)
self.assertTrue(self.report.result_of("test2").failed)
def test_handles_python_exeception(self):
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
test_case.run.side_effect = KeyError
self.runner.run(test_list)
self.assertTrue(self.report.result_of("test").failed)
def test_collects_output(self):
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
output = "Output string, <xml>, </xml>\n"
def side_effect(*args, **kwargs): # pylint: disable=unused-argument
"""
Side effect that print output to stdout
"""
print(output, end="")
return True
test_case.run.side_effect = side_effect
self.runner.run(test_list)
self.assertTrue(self.report.result_of("test").passed)
self.assertEqual(self.report.result_of("test").output, output)
def create_test(self, name, passed):
"""
Utility function to create a mocked test with name
that is either passed or failed
"""
test_case = mock.Mock(spec_set=TestCaseMockSpec)
test_case.configure_mock(name=name)
def run_side_effect(*args, **kwargs): # pylint: disable=unused-argument
"""
Side effect that registers that is has been run
"""
self._tests.append(name)
return passed
test_case.run.side_effect = run_side_effect
return test_case
def _new_report(self):
return TestReport(self.printer)
class TestTestRunner(unittest.TestCase):
"""
Test the test runner
"""
def setUp(self):
self._tests = []
self.output_path = join(dirname(__file__), "test_runner_out")
renew_path(self.output_path)
self.report = TestReport()
self.runner = TestRunner(self.report, self.output_path)
def test_runs_testcases_in_order(self):
test_case1 = self.create_test("test1", True)
test_case2 = self.create_test("test2", False)
test_list = TestList()
test_list.add_test(test_case1)
test_list.add_test(test_case2)
self.runner.run(test_list)
test_case1.run.assert_called_once_with(create_output_path(self.output_path, "test1"))
test_case2.run.assert_called_once_with(create_output_path(self.output_path, "test2"))
self.assertEqual(self._tests, ["test1", "test2"])
self.assertTrue(self.report.result_of("test1").passed)
self.assertTrue(self.report.result_of("test2").failed)
def test_handles_python_exeception(self):
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
test_case.run.side_effect = KeyError
self.runner.run(test_list)
self.assertTrue(self.report.result_of("test").failed)
def test_collects_output(self):
test_case = self.create_test("test", True)
test_list = TestList()
test_list.add_test(test_case)
output = "Output string, <xml>, </xml>\n"
def side_effect(*args, **kwargs): # pylint: disable=unused-argument
"""
Side effect that print output to stdout
"""
print(output, end="")
return True
test_case.run.side_effect = side_effect
self.runner.run(test_list)
self.assertTrue(self.report.result_of("test").passed)
self.assertEqual(self.report.result_of("test").output, output)
def create_test(self, name, passed):
"""
Utility function to create a mocked test with name
that is either passed or failed
"""
test_case = mock.Mock(spec_set=TestCaseMockSpec)
test_case.configure_mock(name=name)
def run_side_effect(*args, **kwargs): # pylint: disable=unused-argument
"""
Side effect that registers that is has been run
"""
self._tests.append(name)
return passed
test_case.run.side_effect = run_side_effect
return test_case
