def schedule_tests_to_pool(plan, pool, schedule_path=None, **pool_cfg):
pool_name = pool.__name__
# Enable debug:
# from testplan.common.utils.logger import DEBUG
# TESTPLAN_LOGGER.setLevel(DEBUG)
pool = pool(name=pool_name, **pool_cfg)
plan.add_resource(pool)
if schedule_path is None:
schedule_path = fix_home_prefix(
os.path.dirname(os.path.abspath(__file__)))
uids = []
for idx in range(1, 10):
uids.append(
plan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=schedule_path,
kwargs=dict(name=idx),
resource=pool_name,
))
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
assert res.run is True
assert res.success is True
assert plan.report.passed is True
assert plan.report.status == Status.PASSED
# 1 testcase * 9 iterations
assert plan.report.counter == {"passed": 9, "total": 9, "failed": 0}
names = sorted(["MTest{}".format(x) for x in range(1, 10)])
assert sorted([entry.name for entry in plan.report.entries]) == names
assert isinstance(plan.report.serialize(), dict)
for idx in range(1, 10):
name = "MTest{}".format(idx)
assert plan.result.test_results[uids[idx - 1]].report.name == name
# All tasks assigned once
for uid in pool._task_retries_cnt:
assert pool._task_retries_cnt[uid] == 0
assert pool.added_item(uid).reassign_cnt == 0
def test_pre_post_steps():
multitest = MultiTest(name='MyMultitest',
suites=[MySuite()],
before_start=check_func_1,
after_start=check_func_2,
before_stop=check_func_3,
after_stop=check_func_4)
plan = Testplan(name='plan', parse_cmdline=False)
plan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
plan.run()
check_report(expected_report, plan.report)
def test_implicit_exporter_initialization(tmpdir):
"""
An implicit JSON should be generated if `json_path` is available
via cmdline args but no exporters were declared programmatically.
"""
json_path = tmpdir.mkdir('reports').join('report.json').strpath
with log_propagation_disabled(TESTPLAN_LOGGER):
with argv_overridden('--json', json_path):
plan = Testplan(name='plan')
multitest_1 = MultiTest(name='Primary', suites=[Alpha()])
plan.add(multitest_1)
plan.run()
assert os.path.exists(json_path)
assert os.stat(json_path).st_size > 0
def test_duplicate_testsuite_names(mockplan):
"""Custom naming function should return a valid non-empty string."""
with pytest.raises(SchemaError):
@testsuite
class MySuite(object):
def sample_test(self, env, result):
pass
multitest = MultiTest(name="MTest", suites=[MySuite(), MySuite()])
mockplan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
mockplan.run()
pytest.fail("Should fail if 2 Multitests have the same name.")
def test_command_line_listing(cmdline_args, expected_output):
multitest_x = MultiTest(name='Primary', suites=[Beta(), Alpha()])
multitest_y = MultiTest(name='Secondary', suites=[Gamma()])
with argv_overridden(*cmdline_args):
plan = Testplan(name='plan', parse_cmdline=True)
with log_propagation_disabled(TESTPLAN_LOGGER):
with captured_logging(TESTPLAN_LOGGER) as log_capture:
plan.add(multitest_x)
plan.add(multitest_y)
result = plan.run()
assert log_capture.output == expected_output
assert len(result.test_report) == 0, 'No tests should be run.'
def test_multitest_tagging(multitest_tags, expected_report):
multitest = MultiTest(name='MyMultitest',
suites=[AlphaSuite(),
BetaSuite(),
GammaSuite()],
tags=multitest_tags)
plan = Testplan(name='plan', parse_cmdline=False)
plan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
plan.run()
check_report(expected=TestReport(name='plan', entries=[expected_report]),
actual=plan.report)
def schedule_tests_to_pool(name, pool, schedule_path=None, **pool_cfg):
pool_name = pool.__name__
# Enable debug:
# from testplan.common.utils.logger import DEBUG
# TESTPLAN_LOGGER.setLevel(DEBUG)
plan = Testplan(name=name, parse_cmdline=False)
pool = pool(name=pool_name, **pool_cfg)
plan.add_resource(pool)
if schedule_path is None:
schedule_path = fix_home_prefix(
os.path.dirname(os.path.abspath(__file__)))
uids = []
for idx in range(1, 10):
uids.append(
plan.schedule(target='get_mtest',
module='func_pool_base_tasks',
path=schedule_path,
kwargs=dict(name=idx),
resource=pool_name))
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
assert res.run is True
assert res.success is True
assert plan.report.passed is True
assert plan.report.status == Status.PASSED
assert plan.report.counts.passed == 9
assert plan.report.counts.error == plan.report.counts.skipped == \
plan.report.counts.failed == plan.report.counts.incomplete == 0
names = sorted(['MTest{}'.format(x) for x in range(1, 10)])
assert sorted([entry.name for entry in plan.report.entries]) == names
assert isinstance(plan.report.serialize(), dict)
for idx in range(1, 10):
name = 'MTest{}'.format(idx)
assert plan.result.test_results[uids[idx - 1]].report.name == name
# All tasks scheduled once
for uid in pool.task_assign_cnt:
assert pool.task_assign_cnt[uid] == 1
def test_restart_worker():
pool_name = ProcessPool.__name__
plan = Testplan(name="ProcPlan", parse_cmdline=False)
pool_size = 4
retries_limit = int(pool_size / 2)
pool = ProcessPool(
name=pool_name,
size=pool_size,
task_retries_limit=retries_limit,
worker_heartbeat=2,
heartbeats_miss_limit=3,
max_active_loop_sleep=1,
)
pool_uid = plan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
plan.schedule(
target="multitest_kills_worker",
module="func_pool_base_tasks",
path=dirname,
resource=pool_name,
)
for idx in range(1, 25):
plan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name=idx),
resource=pool_name,
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
# Check that all workers are restarted
assert (len([
worker for worker in plan.resources[pool_uid]._workers
if worker._aborted is True
]) == 0)
assert res.run is False
assert res.success is False
assert plan.report.status == Status.ERROR
assert plan.report.counter[Status.ERROR] == 1
def test_testcase_trimming(listing_obj, expected_output):
multitest_x = MultiTest(name='Primary', suites=[ParametrizedSuite()])
plan = Testplan(
name='plan',
parse_cmdline=False,
test_lister=listing_obj,
)
with log_propagation_disabled(TESTPLAN_LOGGER):
with captured_logging(TESTPLAN_LOGGER) as log_capture:
plan.add(multitest_x)
assert log_capture.output == expected_output
result = plan.run()
assert len(result.test_report) == 0, 'No tests should be run.'
def test_cppunit(mockplan, binary_dir, expected_report, report_status):
binary_path = os.path.join(binary_dir, "runTests")
if not os.path.exists(binary_path):
msg = BINARY_NOT_FOUND_MESSAGE.format(binary_dir=binary_dir,
binary_path=binary_path)
pytest.skip(msg)
mockplan.add(Cppunit(name="My Cppunit", binary=binary_path))
with log_propagation_disabled(TESTPLAN_LOGGER):
assert mockplan.run().run is True
check_report(expected=expected_report, actual=mockplan.report)
assert mockplan.report.status == report_status
def test_custom_rerun_condition(mockplan):
"""Force reschedule task X times to test logic."""
pool_name = ProcessPool.__name__
uid = "custom_task_uid"
rerun_limit = 2
def custom_rerun(pool, task_result):
if task_result.task.reassign_cnt > task_result.task.rerun:
return False
return True
pool_size = 4
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0,
)
pool.set_rerun_check(custom_rerun)
pool_uid = mockplan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
uid = mockplan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name="0"),
resource=pool_name,
uid=uid,
rerun=rerun_limit,
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = mockplan.run()
assert (len([
worker for worker in mockplan.resources[pool_uid]._workers
if worker._aborted is True
]) == 0)
assert res.success is True
assert mockplan.report.status == Status.PASSED
assert pool.added_item(uid).reassign_cnt == rerun_limit
def test_multi_parts_not_successfully_executed():
"""
Any part did not run successfully then parts cannot be merged
and error will be logged.
"""
plan = TestplanMock(name="plan", merge_scheduled_parts=True)
plan.add(MockMultiTest(name="MTest", suites=[Suite1()], part=(0, 2)))
plan.add(MockMultiTest(name="MTest", suites=[Suite1()], part=(1, 2)))
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is False
assert len(plan.report.entries) == 3 # one placeholder report & 2 siblings
assert len(plan.report.entries[0].entries) == 0 # already cleared
assert plan.report.status == Status.ERROR # Testplan result
assert plan.report.entries[0].status == Status.ERROR # 1st part raised
assert "Deliberately raises" in plan.report.entries[0].logs[0]["message"]
def test_custom_reschedule_condition():
"""Force reschedule task X times to test logic."""
pool_name = ProcessPool.__name__
plan = Testplan(name="ProcPlan", parse_cmdline=False)
uid = "custom_task_uid"
max_reschedules = 2
def custom_reschedule(pool, task_result):
if pool.task_assign_cnt[uid] == max_reschedules:
return False
return True
pool_size = 4
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0,
)
pool.set_reschedule_check(custom_reschedule)
pool_uid = plan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
plan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name="0"),
resource=pool_name,
uid=uid,
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
assert (len([
worker for worker in plan.resources[pool_uid]._workers
if worker._aborted is True
]) == 0)
assert res.success is True
assert pool.task_assign_cnt[uid] == max_reschedules
assert plan.report.status == Status.PASSED
def test_pre_post_steps(mockplan):
multitest = MultiTest(
name="MyMultitest",
suites=[MySuite()],
before_start=check_func_1,
after_start=check_func_2,
before_stop=check_func_3,
after_stop=check_func_4,
)
mockplan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
mockplan.run()
check_report(expected_report, mockplan.report)
def test_kill_one_worker():
"""Kill one worker but pass after reassigning task."""
pool_name = ProcessPool.__name__
plan = Testplan(
name='ProcPlan',
parse_cmdline=False,
)
pool_size = 4
pool = ProcessPool(name=pool_name, size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2)
pool_uid = plan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
kill_uid = plan.schedule(target='multitest_kill_one_worker',
module='func_pool_base_tasks',
path=dirname,
args=('killer', os.getpid(),
pool_size), # kills 4th worker
resource=pool_name)
uids = []
for idx in range(1, 25):
uids.append(plan.schedule(target='get_mtest',
module='func_pool_base_tasks',
path=dirname, kwargs=dict(name=idx),
resource=pool_name))
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
# Check that the worker killed by test was aborted
assert len([worker for worker in plan.resources[pool_uid]._workers
if worker._aborted is True]) == 1
assert res.run is True
assert res.success is True
assert plan.report.status == Status.PASSED
# All tasks scheduled once
for uid in pool.task_assign_cnt:
if uid == kill_uid:
assert pool.task_assign_cnt[uid] == 2
else:
assert pool.task_assign_cnt[uid] == 1
def test_create_pdf(tmpdir):
"""PDF exporter should generate a PDF file using the report data."""
pdf_path = tmpdir.mkdir('reports').join('dummy_report.pdf').strpath
assertion_entries = [
assertions.Equal(1, 2),
assertions.Greater(2, 1),
assertions.IsFalse(True, 'this should fail'),
assertions.IsTrue(True, 'this should pass'),
assertions.Fail('Explicit failure'),
base.Group(description='group description',
entries=[
assertions.NotEqual(2, 1),
assertions.Contain(1, [1, 2, 3]),
])
]
report = TestReport(
name='my testplan',
entries=[
TestGroupReport(name='My Multitest',
category='multitest',
entries=[
TestGroupReport(
name='MySuite',
entries=[
TestCaseReport(
name='my_test_method',
entries=[
registry.serialize(obj)
for obj in assertion_entries
])
])
])
])
exporter = PDFExporter(pdf_path=pdf_path,
pdf_style=styles.Style(passing='assertion-detail',
failing='assertion-detail'))
with log_propagation_disabled(TESTPLAN_LOGGER):
exporter.export(report)
assert os.path.exists(pdf_path)
assert os.stat(pdf_path).st_size > 0
def test_process_pool_integration(
report_dir, fixture_dirname,
expected_report, pdf_title,
expected_plan_result, dependant_module
):
if dependant_module:
importorxfail(dependant_module)
pool = ProcessPool(name='MyPool', size=1)
pdf_path = report_dir.join('test_report_process_{}.pdf'.format(
pdf_title)).strpath
plan = Testplan(
name='plan',
parse_cmdline=False,
exporters=[
PDFExporter(pdf_path=pdf_path)
]
)
plan.add_resource(pool)
runners_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
fixture_path = os.path.join(runners_path, 'fixtures', fixture_dirname)
task = Task(
target='make_multitest',
module='suites',
path=fixture_path,
)
plan.schedule(task, resource='MyPool')
assert not os.path.exists(pdf_path)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
for log in plan.report.flattened_logs:
if all(word in log['message'] for word in ['tkinter', 'TclError']):
pytest.xfail(reason='Tkinter not installed properly')
check_report(expected=expected_report, actual=plan.report)
assert plan.result.success is expected_plan_result
assert os.path.exists(pdf_path)
assert os.stat(pdf_path).st_size > 0
def test_kill_all_workers(mockplan):
"""Kill all workers and create a failed report."""
pool_name = ProcessPool.__name__
pool_size = 4
retries_limit = 3
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0,
)
pool._task_retries_limit = retries_limit
pool_uid = mockplan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
uid = mockplan.schedule(
target="multitest_kills_worker",
module="func_pool_base_tasks",
path=dirname,
args=(os.getpid(),),
resource=pool_name,
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = mockplan.run()
# Check that the worker killed by test was aborted
assert (
len(
[
worker
for worker in mockplan.resources[pool_uid]._workers
if worker._aborted is True
]
)
== pool_size
)
assert res.success is False
# scheduled X times and killed all workers
assert pool._task_retries_cnt[uid] == retries_limit + 1
assert mockplan.report.status == Status.ERROR
def test_invalid_long_testsuite_name(mockplan):
"""Custom naming function should return a valid non-empty string."""
with pytest.raises(SchemaError):
long_string = "a" * (MAX_TESTSUITE_NAME_LENGTH + 1)
@testsuite(custom_name=long_string)
class MySuite(object):
def sample_test(self, env, result):
pass
multitest = MultiTest(name="MTest", suites=[MySuite()])
mockplan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
mockplan.run()
pytest.fail("Should fail if custom_name returns a very long string.")
def test_multitest_tagging(mockplan, multitest_tags, expected_report):
multitest = MultiTest(
name="MyMultitest",
suites=[AlphaSuite(), BetaSuite(),
GammaSuite()],
tags=multitest_tags,
)
mockplan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
mockplan.run()
check_report(
expected=TestReport(name="plan", entries=[expected_report]),
actual=mockplan.report,
)
def test_json_exporter(tmpdir):
"""
JSON Exporter should generate a json report at the given `json_path`.
"""
json_path = tmpdir.mkdir('reports').join('report.json').strpath
with log_propagation_disabled(TESTPLAN_LOGGER):
plan = Testplan(name='plan',
parse_cmdline=False,
exporters=JSONExporter(json_path=json_path))
multitest_1 = MultiTest(name='Primary', suites=[Alpha()])
multitest_2 = MultiTest(name='Secondary', suites=[Beta()])
plan.add(multitest_1)
plan.add(multitest_2)
plan.run()
assert os.path.exists(json_path)
assert os.stat(json_path).st_size > 0
def test_implicit_exporter_initialization(tmpdir):
"""
An implicit XMLExporter should be generated if `xml_dir` is available
via cmdline args but no exporters were declared programmatically.
"""
xml_dir = tmpdir.mkdir('xml')
with log_propagation_disabled(TESTPLAN_LOGGER):
with argv_overridden('--xml', xml_dir.strpath):
plan = Testplan(name='plan')
multitest_1 = MultiTest(name='Primary', suites=[Alpha()])
plan.add(multitest_1)
plan.run()
xml_path = xml_dir.join('primary.xml').strpath
assert os.path.exists(xml_path)
assert os.stat(xml_path).st_size > 0
def test_custom_reschedule_condition():
"""Force reschedule task X times to test logic."""
pool_name = ProcessPool.__name__
plan = Testplan(
name='ProcPlan',
parse_cmdline=False,
)
uid = 'custom_task_uid'
max_reschedules = 2
def custom_reschedule(pool, task_result):
if pool.task_assign_cnt[uid] == max_reschedules:
return False
return True
pool_size = 4
pool = ProcessPool(name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2)
pool.set_reschedule_check(custom_reschedule)
pool_uid = plan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
plan.schedule(target='get_mtest',
module='func_pool_base_tasks',
path=dirname,
kwargs=dict(name='0'),
resource=pool_name,
uid=uid)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
# Check that the worker killed by test was aborted
assert len([
worker for worker in plan.resources[pool_uid]._workers
if worker._aborted is True
]) == 0
assert res.success is True
assert pool.task_assign_cnt[uid] == max_reschedules
assert plan.report.status == Status.PASSED
def test_gtest(binary_dir, expected_report, report_status):
binary_path = os.path.join(binary_dir, "runTests")
if not os.path.exists(binary_path):
msg = BINARY_NOT_FOUND_MESSAGE.format(binary_dir=binary_dir,
binary_path=binary_path)
pytest.skip(msg)
plan = Testplan(name="plan", parse_cmdline=False)
plan.add(GTest(name="MyGTest", driver=binary_path))
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
check_report(expected=expected_report, actual=plan.report)
assert plan.report.status == report_status
def test_process_pool_integration(
runpath,
fixture_dirname,
expected_report,
pdf_title,
expected_plan_result,
dependant_module,
):
if dependant_module:
importorxfail(dependant_module)
pool = ProcessPool(name="MyProcessPool", size=1)
pdf_path = os.path.join(
runpath, "test_report_local_{}.pdf".format(pdf_title)
)
plan = TestplanMock(
name="plan",
exporters=[PDFExporter(pdf_path=pdf_path)],
runpath=runpath,
)
plan.add_resource(pool)
runners_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
fixture_path = os.path.join(runners_path, "fixtures", fixture_dirname)
task = Task(target="make_multitest", module="suites", path=fixture_path)
plan.schedule(task, resource="MyProcessPool")
assert not os.path.exists(pdf_path)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
for log in plan.report.flattened_logs:
if all(word in log["message"] for word in ["tkinter", "TclError"]):
pytest.xfail(reason="Tkinter not installed properly")
check_report(expected=expected_report, actual=plan.report)
assert plan.result.success is expected_plan_result
assert os.path.exists(pdf_path)
assert os.stat(pdf_path).st_size > 0
def test_process_runner(binary_path, expected_report, test_kwargs):
plan = Testplan(
name='plan',
parse_cmdline=False,
)
process_test = DummyTest(
name='MyTest',
driver=binary_path,
**test_kwargs
)
plan.add(process_test)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
check_report(expected=expected_report, actual=plan.report)
def test_implicit_exporter_initialization(tmpdir):
"""
An implicit PDFExporter should be generated if `pdf_path` is available
via cmdline args but no exporters were declared.
Multiple implicit TagFilteredPDFExporters should be initialized
if `report_tags` or `report_tags_all` arguments are passed
via cmdline, but no exporters were declared.
"""
pdf_dir = tmpdir.mkdir("reports")
pdf_path = pdf_dir.join("my_report.pdf").strpath
@testsuite
class MySuite(object):
@testcase
def test_comparison(self, env, result):
result.equal(1, 1, "equality description")
@testcase(tags="foo")
def test_membership(self, env, result):
result.contain(1, [1, 2, 3])
with log_propagation_disabled(TESTPLAN_LOGGER):
with argv_overridden(
"--pdf",
pdf_path,
"--report-tags",
"foo",
"--report-dir",
pdf_dir.strpath,
):
multitest = MultiTest(name="MyMultitest", suites=[MySuite()])
plan = TestplanMock(name="plan", parse_cmdline=True)
plan.add(multitest)
plan.run()
tag_pdf_path = pdf_dir.join("report-tags-any-foo.pdf").strpath
assert os.path.exists(pdf_path)
assert os.path.exists(tag_pdf_path)
assert os.stat(pdf_path).st_size > 0
assert os.stat(tag_pdf_path).st_size > 0
def test_hobbestest_listing(binary_dir, expected_output):
binary_path = os.path.join(binary_dir, "hobbes-test")
cmdline_args = ["--list"]
with argv_overridden(*cmdline_args):
plan = TestplanMock(name="plan", parse_cmdline=True)
with log_propagation_disabled(TESTPLAN_LOGGER):
with captured_logging(TESTPLAN_LOGGER) as log_capture:
plan.add(
HobbesTest(
name="My HobbesTest",
binary=binary_path,
tests=["Hog", "Net", "Recursives"],
))
result = plan.run()
print(log_capture.output)
assert log_capture.output == expected_output
assert len(result.test_report) == 0, "No tests should be run."
def test_hobbestest(mockplan, binary_dir, expected_report):
binary_path = os.path.join(binary_dir, "hobbes-test")
if not os.path.exists(binary_path):
msg = BINARY_NOT_FOUND_MESSAGE.format(binary_dir=binary_dir,
binary_path=binary_path)
pytest.skip(msg)
mockplan.add(
HobbesTest(
name="My HobbesTest",
binary=binary_path,
tests=["Hog", "Net", "Recursives"],
))
with log_propagation_disabled(TESTPLAN_LOGGER):
assert mockplan.run().run is True
check_report(expected=expected_report, actual=mockplan.report)
def test_duplicate_testsuite_names(mockplan):
"""Raises when duplicate suite names found."""
with pytest.raises(SchemaError):
@testsuite
class MySuite(object):
def __init__(self, val):
self.val = val
@testcase
def sample_test(self, env, result):
pass
multitest = MultiTest(name="MTest", suites=[MySuite(0), MySuite(1)])
mockplan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
mockplan.run()
pytest.fail("Duplicate test suite name found in a Multitest.")