def test_restart_worker(mockplan):
pool_name = ProcessPool.__name__
pool_size = 4
retries_limit = int(pool_size / 2)
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=3,
max_active_loop_sleep=1,
)
pool._task_retries_limit = retries_limit
pool_uid = mockplan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
mockplan.schedule(
target="multitest_kills_worker",
module="func_pool_base_tasks",
path=dirname,
args=(os.getpid(),),
resource=pool_name,
)
for idx in range(1, 25):
mockplan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name=idx),
resource=pool_name,
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = mockplan.run()
# Check that all workers are restarted
assert (
len(
[
worker
for worker in mockplan.resources[pool_uid]._workers
if worker._aborted is True
]
)
== 0
)
assert res.run is False
assert res.success is False
assert mockplan.report.status == Status.ERROR
assert mockplan.report.counter[Status.ERROR] == 1
def test_custom_rerun_condition(mockplan):
"""Force reschedule task X times to test logic."""
pool_name = ProcessPool.__name__
uid = "custom_task_uid"
rerun_limit = 2
def custom_rerun(pool, task_result):
if task_result.task.reassign_cnt > task_result.task.rerun:
return False
return True
pool_size = 4
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0,
)
pool.set_rerun_check(custom_rerun)
pool_uid = mockplan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
uid = mockplan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name="0"),
resource=pool_name,
uid=uid,
rerun=rerun_limit,
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = mockplan.run()
assert (
len(
[
worker
for worker in mockplan.resources[pool_uid]._workers
if worker._aborted is True
]
)
== 0
)
assert res.success is True
assert mockplan.report.status == Status.PASSED
assert pool.added_item(uid).reassign_cnt == rerun_limit
def test_kill_all_workers():
"""Kill all workers and create a failed report."""
pool_name = ProcessPool.__name__
plan = Testplan(
name='ProcPlan',
parse_cmdline=False,
)
pool_size = 4
pool = ProcessPool(name=pool_name,
size=pool_size,
task_retries_limit=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2)
pool_uid = plan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
uid = plan.schedule(target='multitest_kills_worker',
module='func_pool_base_tasks',
path=dirname,
resource=pool_name)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
# Check that the worker killed by test was aborted
assert len([
worker for worker in plan.resources[pool_uid]._workers
if worker._aborted is True
]) == pool_size
assert res.success is False
# scheduled X times and killed all workers
assert pool.task_assign_cnt[uid] == pool_size
assert plan.report.status == Status.ERROR
def test_schedule_from_main(mockplan):
"""
Test scheduling Tasks from __main__ - it should not be allowed for
ProcessPool.
"""
# Set up a testplan and add a ProcessPool.
pool = ProcessPool(name="ProcPool", size=2)
mockplan.add_resource(pool)
# First check that scheduling a Task with module string of '__main__'
# raises the expected ValueError.
with pytest.raises(ValueError):
mockplan.schedule(
target="target", module="__main__", resource="ProcPool"
)
# Secondly, check that scheduling a callable target with a __module__ attr
# of __main__ also raises a ValueError.
def callable_target():
raise RuntimeError
callable_target.__module__ = "__main__"
with pytest.raises(ValueError):
mockplan.schedule(target=callable_target, resource="ProcPool")
def test_kill_one_worker(mockplan):
"""Kill one worker but pass after reassigning task."""
pool_name = ProcessPool.__name__
pool_size = 4
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0,
)
pool_uid = mockplan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
kill_uid = mockplan.schedule(
target="multitest_kill_one_worker",
module="func_pool_base_tasks",
path=dirname,
args=(os.getpid(), pool_size), # kills 4th worker
resource=pool_name,
)
uids = []
for idx in range(1, 25):
uids.append(
mockplan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name=idx),
resource=pool_name,
)
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = mockplan.run()
# Check that the worker killed by test was aborted
assert (
len(
[
worker
for worker in mockplan.resources[pool_uid]._workers
if worker._aborted is True
]
)
== 1
)
assert res.run is True
assert res.success is True
assert mockplan.report.status == Status.PASSED
# All tasks scheduled once except the killed one
for idx in range(1, 25):
if uids[idx - 1] == kill_uid:
assert pool._task_retries_cnt[uids[idx - 1]] == 1
def test_custom_reschedule_condition():
"""Force reschedule task X times to test logic."""
pool_name = ProcessPool.__name__
plan = Testplan(name="ProcPlan", parse_cmdline=False)
uid = "custom_task_uid"
max_reschedules = 2
def custom_reschedule(pool, task_result):
if pool.task_assign_cnt[uid] == max_reschedules:
return False
return True
pool_size = 4
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0,
)
pool.set_reschedule_check(custom_reschedule)
pool_uid = plan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
plan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name="0"),
resource=pool_name,
uid=uid,
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
assert (len([
worker for worker in plan.resources[pool_uid]._workers
if worker._aborted is True
]) == 0)
assert res.success is True
assert pool.task_assign_cnt[uid] == max_reschedules
assert plan.report.status == Status.PASSED
def test_kill_all_workers(mockplan):
"""Kill all workers and create a failed report."""
pool_name = ProcessPool.__name__
pool_size = 4
retries_limit = 3
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0,
)
pool._task_retries_limit = retries_limit
pool_uid = mockplan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
uid = mockplan.schedule(
target="multitest_kills_worker",
module="func_pool_base_tasks",
path=dirname,
args=(os.getpid(),),
resource=pool_name,
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = mockplan.run()
# Check that the worker killed by test was aborted
assert (
len(
[
worker
for worker in mockplan.resources[pool_uid]._workers
if worker._aborted is True
]
)
== pool_size
)
assert res.success is False
# scheduled X times and killed all workers
assert pool._task_retries_cnt[uid] == retries_limit + 1
assert mockplan.report.status == Status.ERROR
def test_custom_reschedule_condition():
"""Force reschedule task X times to test logic."""
pool_name = ProcessPool.__name__
plan = Testplan(
name='ProcPlan',
parse_cmdline=False,
)
uid = 'custom_task_uid'
max_reschedules = 2
def custom_reschedule(pool, task_result):
if pool.task_assign_cnt[uid] == max_reschedules:
return False
return True
pool_size = 4
pool = ProcessPool(name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2)
pool.set_reschedule_check(custom_reschedule)
pool_uid = plan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
plan.schedule(target='get_mtest',
module='func_pool_base_tasks',
path=dirname,
kwargs=dict(name='0'),
resource=pool_name,
uid=uid)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
# Check that the worker killed by test was aborted
assert len([
worker for worker in plan.resources[pool_uid]._workers
if worker._aborted is True
]) == 0
assert res.success is True
assert pool.task_assign_cnt[uid] == max_reschedules
assert plan.report.status == Status.PASSED
def test_serialization():
"""Test serialization of test results."""
plan = Testplan(name='SerializationPlan', parse_cmdline=False)
pool = ProcessPool(name='ProcPool', size=2)
plan.add_resource(pool)
plan.schedule(target='make_serialization_mtest',
module='test_pool_process',
path=os.path.dirname(__file__),
resource='ProcPool')
res = plan.run()
assert res.success
def test_runner_timeout():
"""
Execute MultiTests in LocalRunner, ThreadPool and ProcessPool respectively.
Some of them will timeout and we'll get a report showing execution details.
"""
plan = Testplan(name='plan',
parse_cmdline=False,
runnable=MyTestRunner,
timeout=60,
abort_wait_timeout=5)
mod_path = os.path.dirname(os.path.abspath(__file__))
THREAD_POOL = 'MyThreadPool'
PROCESS_POOL = 'MyProcessPool'
thread_pool = ThreadPool(name=THREAD_POOL, size=2, worker_heartbeat=None)
proc_pool = ProcessPool(name=PROCESS_POOL, size=2, worker_heartbeat=None)
plan.add_resource(thread_pool)
plan.add_resource(proc_pool)
plan.add(func_basic_tasks.get_mtest1())
plan.add(func_basic_tasks.get_mtest2())
task3 = Task(target='get_mtest3', module='func_basic_tasks', path=mod_path)
task4 = Task(target='get_mtest4', module='func_basic_tasks', path=mod_path)
task5 = Task(target='get_mtest5', module='func_basic_tasks', path=mod_path)
task6 = Task(target='get_mtest6', module='func_basic_tasks', path=mod_path)
plan.schedule(task3, resource=THREAD_POOL)
plan.schedule(task4, resource=THREAD_POOL)
plan.schedule(task5, resource=PROCESS_POOL)
plan.schedule(task6, resource=PROCESS_POOL)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is False
assert len(plan.report.entries) == 6
assert plan.report.status == Status.ERROR
entries = plan.report.entries
assert entries[0].name == 'MTest1'
assert entries[0].status == Status.FAILED # testcase 'test_true' failed
assert entries[2].name == 'MTest3'
assert entries[2].status == Status.PASSED # testcase 'test_equal' passed
assert entries[4].name == 'MTest5'
assert entries[4].status == Status.ERROR # testcase 'test_contain' raised
assert entries[1].name == 'MTest2'
assert entries[1].status == Status.ERROR # timeout
assert ' discarding due to ' in entries[1].logs[0]['message']
assert entries[3].name == 'Task[get_mtest4]'
assert entries[3].status == Status.ERROR # timeout
assert entries[3].logs[0]['message'].startswith('_target: get_mtest4')
assert ' discarding due to ' in entries[3].logs[1]['message']
assert entries[5].name == 'Task[get_mtest6]'
assert entries[5].status == Status.ERROR # timeout
assert entries[5].logs[0]['message'].startswith('_target: get_mtest6')
assert ' discarding due to ' in entries[5].logs[1]['message']
def test_serialization(mockplan):
"""Test serialization of test results."""
pool = ProcessPool(name="ProcPool", size=2)
mockplan.add_resource(pool)
mockplan.schedule(
target="make_serialization_mtest",
module="test_pool_process",
path=os.path.dirname(__file__),
resource="ProcPool",
)
res = mockplan.run()
assert res.success
def test_kill_one_worker():
"""Kill one worker but pass after reassigning task."""
pool_name = ProcessPool.__name__
plan = Testplan(
name='ProcPlan',
parse_cmdline=False,
)
pool_size = 4
pool = ProcessPool(name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0)
pool_uid = plan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
kill_uid = plan.schedule(
target='multitest_kill_one_worker',
module='func_pool_base_tasks',
path=dirname,
args=('killer', os.getpid(), pool_size), # kills 4th worker
resource=pool_name)
uids = []
for idx in range(1, 25):
uids.append(
plan.schedule(target='get_mtest',
module='func_pool_base_tasks',
path=dirname,
kwargs=dict(name=idx),
resource=pool_name))
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
# Check that the worker killed by test was aborted
assert len([
worker for worker in plan.resources[pool_uid]._workers
if worker._aborted is True
]) == 1
assert res.run is True
assert res.success is True
assert plan.report.status == Status.PASSED
# All tasks scheduled once
for uid in pool.task_assign_cnt:
if uid == kill_uid:
assert pool.task_assign_cnt[uid] == 2
else:
assert pool.task_assign_cnt[uid] == 1
def test_task_rerun_in_process_pool(mockplan):
"""
Test 1 procedure:
- 1st run: `unstable_case` fails.
- 1st rerun: `mock_driver` raises during stop.
- 2nd rerun: all pass.
Test 2 procedure:
- 1st run: `unstable_worker` makes child process exit.
- monitor detects inactive worker, decommission the task from worker,
then re-assign it and it passes (no rerun is needed).
"""
pool_name = ProcessPool.__name__
pool = ProcessPool(name=pool_name, size=2)
mockplan.add_resource(pool)
directory = os.path.dirname(os.path.abspath(__file__))
tmp_file_1 = os.path.join(tempfile.gettempdir(), getpass.getuser(),
"{}.tmp".format(uuid.uuid4()))
tmp_file_2 = os.path.join(tempfile.gettempdir(), getpass.getuser(),
"{}.tmp".format(uuid.uuid4()))
task1 = Task(target=make_multitest_1,
path=directory,
args=(tmp_file_1, ),
rerun=2)
task2 = Task(target=make_multitest_2,
path=directory,
args=(tmp_file_2, ),
rerun=0)
uid1 = mockplan.schedule(task=task1, resource=pool_name)
uid2 = mockplan.schedule(task=task2, resource=pool_name)
assert mockplan.run().run is True
assert mockplan.report.passed is True
assert mockplan.report.counter == {"passed": 5, "total": 5, "failed": 0}
assert isinstance(mockplan.report.serialize(), dict)
assert mockplan.result.test_results[uid1].report.name == "Unstable MTest1"
assert mockplan.result.test_results[uid2].report.name == "Unstable MTest2"
assert len(mockplan.report.entries) == 4
assert mockplan.report.entries[-1].category == ReportCategories.TASK_RERUN
assert mockplan.report.entries[-2].category == ReportCategories.TASK_RERUN
assert task1.reassign_cnt == 2
assert task2.reassign_cnt == 0 # 1st run: assigned but not executed
assert pool._task_retries_cnt[uid2] == 1
_remove_existing_tmp_file(tmp_file_1)
_remove_existing_tmp_file(tmp_file_2)
def test_process_pool_integration(
report_dir, fixture_dirname,
expected_report, pdf_title,
expected_plan_result, dependant_module
):
if dependant_module:
importorxfail(dependant_module)
pool = ProcessPool(name='MyPool', size=1)
pdf_path = report_dir.join('test_report_process_{}.pdf'.format(
pdf_title)).strpath
plan = Testplan(
name='plan',
parse_cmdline=False,
exporters=[
PDFExporter(pdf_path=pdf_path)
]
)
plan.add_resource(pool)
runners_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
fixture_path = os.path.join(runners_path, 'fixtures', fixture_dirname)
task = Task(
target='make_multitest',
module='suites',
path=fixture_path,
)
plan.schedule(task, resource='MyPool')
assert not os.path.exists(pdf_path)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
for log in plan.report.flattened_logs:
if all(word in log['message'] for word in ['tkinter', 'TclError']):
pytest.xfail(reason='Tkinter not installed properly')
check_report(expected=expected_report, actual=plan.report)
assert plan.result.success is expected_plan_result
assert os.path.exists(pdf_path)
assert os.stat(pdf_path).st_size > 0
def test_process_pool_integration(
runpath,
fixture_dirname,
expected_report,
pdf_title,
expected_plan_result,
dependant_module,
):
if dependant_module:
importorxfail(dependant_module)
pool = ProcessPool(name="MyProcessPool", size=1)
pdf_path = os.path.join(
runpath, "test_report_local_{}.pdf".format(pdf_title)
)
plan = TestplanMock(
name="plan",
exporters=[PDFExporter(pdf_path=pdf_path)],
runpath=runpath,
)
plan.add_resource(pool)
runners_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
fixture_path = os.path.join(runners_path, "fixtures", fixture_dirname)
task = Task(target="make_multitest", module="suites", path=fixture_path)
plan.schedule(task, resource="MyProcessPool")
assert not os.path.exists(pdf_path)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
for log in plan.report.flattened_logs:
if all(word in log["message"] for word in ["tkinter", "TclError"]):
pytest.xfail(reason="Tkinter not installed properly")
check_report(expected=expected_report, actual=plan.report)
assert plan.result.success is expected_plan_result
assert os.path.exists(pdf_path)
assert os.stat(pdf_path).st_size > 0
def main(plan):
"""
Testplan decorated main function to add and execute MultiTests.
:return: Testplan result object.
:rtype: ``testplan.base.TestplanResult``
"""
# Add a process pool test execution resource to the plan of given size.
pool = ProcessPool(name='MyPool', size=plan.args.pool_size)
plan.add_resource(pool)
# Add a given number of similar tests to the process pool
# to be executed in parallel.
for idx in range(plan.args.tasks_num):
# All Task arguments need to be serializable.
task = Task(target='make_multitest',
module='tasks',
kwargs={'index': idx})
plan.schedule(task, resource='MyPool')
def test_restart_worker():
pool_name = ProcessPool.__name__
plan = Testplan(
name='ProcPlan',
parse_cmdline=False,
)
pool_size = 4
retries_limit = int(pool_size / 2)
pool = ProcessPool(name=pool_name,
size=pool_size,
task_retries_limit=retries_limit,
worker_heartbeat=2,
heartbeats_miss_limit=3,
max_active_loop_sleep=1)
pool_uid = plan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
plan.schedule(target='multitest_kills_worker',
module='func_pool_base_tasks',
path=dirname,
resource=pool_name)
for idx in range(1, 25):
plan.schedule(target='get_mtest',
module='func_pool_base_tasks',
path=dirname,
kwargs=dict(name=idx),
resource=pool_name)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
# Check that all workers are restarted
assert len([
worker for worker in plan.resources[pool_uid]._workers
if worker._aborted is True
]) == 0
assert res.run is False
assert res.success is False
assert plan.report.status == Status.ERROR
def test_reassign_times_limit():
"""Kill workers and reassign task up to limit times."""
pool_name = ProcessPool.__name__
plan = Testplan(name="ProcPlan", parse_cmdline=False)
pool_size = 4
retries_limit = int(pool_size / 2)
pool = ProcessPool(
name=pool_name,
size=pool_size,
task_retries_limit=retries_limit,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0,
)
pool_uid = plan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
uid = plan.schedule(
target="multitest_kills_worker",
module="func_pool_base_tasks",
path=dirname,
resource=pool_name,
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
# Check that the worker killed by test was aborted
assert (len([
worker for worker in plan.resources[pool_uid]._workers
if worker._aborted is True
]) == retries_limit)
assert res.success is False
assert pool.task_assign_cnt[uid] == retries_limit
assert plan.report.status == Status.ERROR
assert plan.report.counter["error"] == 1
def test_schedule_from_main():
"""
Test scheduling Tasks from __main__ - it should not be allowed for
ProcessPool.
"""
# Set up a testplan and add a ProcessPool.
plan = Testplan(name='ProcPlan', parse_cmdline=False)
pool = ProcessPool(name='ProcPool', size=2)
plan.add_resource(pool)
# First check that scheduling a Task with module string of '__main__'
# raises the expected ValueError.
with pytest.raises(ValueError):
plan.schedule(target='target', module='__main__', resource='ProcPool')
# Secondly, check that scheduling a callable target with a __module__ attr
# of __main__ also raises a ValueError.
def callable_target():
raise RuntimeError
callable_target.__module__ = '__main__'
with pytest.raises(ValueError):
plan.schedule(target=callable_target, resource='ProcPool')
