def test_restart_worker(mockplan):
pool_name = ProcessPool.__name__
pool_size = 4
retries_limit = int(pool_size / 2)
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=3,
max_active_loop_sleep=1,
)
pool._task_retries_limit = retries_limit
pool_uid = mockplan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
mockplan.schedule(
target="multitest_kill_workers",
module="func_pool_base_tasks",
path=dirname,
args=(os.getpid(),),
resource=pool_name,
)
for idx in range(1, 25):
mockplan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name=idx),
resource=pool_name,
)
res = mockplan.run()
# Check that all workers are restarted
assert (
len(
[
worker
for worker in mockplan.resources[pool_uid]._workers
if worker._aborted is True
]
)
== 0
)
assert res.run is False
assert res.success is False
assert mockplan.report.status == Status.ERROR
assert mockplan.report.counter[Status.ERROR] == 1
def test_schedule_from_main(mockplan):
"""
Test scheduling Tasks from __main__ - it should not be allowed for
ProcessPool.
"""
# Set up a testplan and add a ProcessPool.
pool = ProcessPool(name="ProcPool", size=2)
mockplan.add_resource(pool)
# First check that scheduling a Task with module string of '__main__'
# raises the expected ValueError.
with pytest.raises(ValueError):
mockplan.schedule(
target="target", module="__main__", resource="ProcPool"
)
# Secondly, check that scheduling a callable target with a __module__ attr
# of __main__ also raises a ValueError.
def callable_target():
raise RuntimeError
callable_target.__module__ = "__main__"
with pytest.raises(ValueError):
mockplan.schedule(target=callable_target, resource="ProcPool")
def test_disable_rerun_in_pool(mockplan):
pool_name = ProcessPool.__name__
uid = "custom_task_uid"
rerun_limit = 2
pool_size = 4
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0,
allow_task_rerun=False,
)
pool_uid = mockplan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
uid = mockplan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name="0"),
resource=pool_name,
uid=uid,
rerun=rerun_limit,
)
res = mockplan.run()
assert (
len(
[
worker
for worker in mockplan.resources[pool_uid]._workers
if worker._aborted is True
]
)
== 0
)
assert res.success is True
assert mockplan.report.status == Status.PASSED
assert pool.added_item(uid).reassign_cnt == 0
def test_kill_all_workers(mockplan):
"""Kill all workers and create a failed report."""
pool_name = ProcessPool.__name__
pool_size = 4
retries_limit = 3
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0,
)
pool._task_retries_limit = retries_limit
pool_uid = mockplan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
uid = mockplan.schedule(
target="multitest_kill_workers",
module="func_pool_base_tasks",
path=dirname,
args=(os.getpid(),),
resource=pool_name,
)
res = mockplan.run()
# Check that the worker killed by test was aborted
assert (
len(
[
worker
for worker in mockplan.resources[pool_uid]._workers
if worker._aborted is True
]
)
== pool_size
)
assert res.success is False
# scheduled X times and killed all workers
assert pool._task_retries_cnt[uid] == retries_limit + 1
assert mockplan.report.status == Status.ERROR
def test_serialization(mockplan):
"""Test serialization of test results."""
pool = ProcessPool(name="ProcPool", size=2)
mockplan.add_resource(pool)
mockplan.schedule(
target="make_serialization_mtest",
module="test_pool_process",
path=os.path.dirname(__file__),
resource="ProcPool",
)
res = mockplan.run()
assert res.success
def test_kill_one_worker(mockplan):
"""Kill one worker but pass after reassigning task."""
pool_name = ProcessPool.__name__
pool_size = 4
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0,
)
pool_uid = mockplan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
kill_uid = mockplan.schedule(
target="multitest_kill_one_worker",
module="func_pool_base_tasks",
path=dirname,
args=(os.getpid(), pool_size), # kills 4th worker
resource=pool_name,
)
uids = []
for idx in range(1, 25):
uids.append(
mockplan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name=idx),
resource=pool_name,
))
res = mockplan.run()
# Check that the worker killed by test was aborted
assert (len([
worker for worker in mockplan.resources[pool_uid]._workers
if worker._aborted is True
]) == 1)
assert res.run is True
assert res.success is True
assert mockplan.report.status == Status.PASSED
# All tasks scheduled once except the killed one
for idx in range(1, 25):
if uids[idx - 1] == kill_uid:
assert pool._task_retries_cnt[uids[idx - 1]] == 1
def main(plan):
"""
Testplan decorated main function to add and execute MultiTests.
:return: Testplan result object.
:rtype: ``testplan.base.TestplanResult``
"""
# Add a process pool test execution resource to the plan of given size.
# Also you can use thread pool or remote pool instead.
pool = ProcessPool(name="MyPool")
plan.add_resource(pool)
# Create task objects from all @task_target we could find in the modules
# that matches the name pattern under the specified path, and schedule them
# to MyPool.
plan.schedule_all(path=".",
name_pattern=r".*tasks\.py$",
resource="MyPool")
def main(plan):
"""
Testplan decorated main function to add and execute MultiTests.
:return: Testplan result object.
:rtype: ``testplan.base.TestplanResult``
"""
# Add a process pool test execution resource to the plan of given size.
pool = ProcessPool(name="MyPool", size=plan.args.pool_size)
plan.add_resource(pool)
# Add a given number of similar tests to the process pool
# to be executed in parallel.
for idx in range(plan.args.tasks_num):
# All Task arguments need to be serializable.
task = Task(target="make_multitest",
module="tasks",
kwargs={"index": idx})
plan.schedule(task, resource="MyPool")
def test_task_rerun_in_process_pool(mockplan):
"""
Test 1 procedure:
- 1st run: `unstable_case` fails.
- 1st rerun: `mock_driver` raises during stop.
- 2nd rerun: all pass.
Test 2 procedure:
- 1st run: `unstable_worker` makes child process exit.
- monitor detects inactive worker, decommission the task from worker,
then re-assign it and it passes (no rerun is needed).
"""
pool_name = ProcessPool.__name__
pool = ProcessPool(name=pool_name, size=2)
mockplan.add_resource(pool)
tmp_file_1 = os.path.join(tempfile.gettempdir(), getpass.getuser(),
"{}.tmp".format(uuid.uuid4()))
tmp_file_2 = os.path.join(tempfile.gettempdir(), getpass.getuser(),
"{}.tmp".format(uuid.uuid4()))
task1 = Task(target=make_multitest_1, args=(tmp_file_1, ), rerun=2)
task2 = Task(target=make_multitest_2, args=(tmp_file_2, ), rerun=0)
uid1 = mockplan.schedule(task=task1, resource=pool_name)
uid2 = mockplan.schedule(task=task2, resource=pool_name)
assert mockplan.run().run is True
assert mockplan.report.passed is True
assert mockplan.report.counter == {"passed": 5, "total": 5, "failed": 0}
assert isinstance(mockplan.report.serialize(), dict)
assert mockplan.result.test_results[uid1].report.name == "Unstable MTest1"
assert mockplan.result.test_results[uid2].report.name == "Unstable MTest2"
assert len(mockplan.report.entries) == 4
assert mockplan.report.entries[-1].category == ReportCategories.TASK_RERUN
assert mockplan.report.entries[-2].category == ReportCategories.TASK_RERUN
assert task1.reassign_cnt == 2
assert task2.reassign_cnt == 0 # 1st run: assigned but not executed
assert pool._task_retries_cnt[uid2] == 1
_remove_existing_tmp_file(tmp_file_1)
_remove_existing_tmp_file(tmp_file_2)
def test_process_pool_integration(
runpath,
fixture_dirname,
expected_report,
pdf_title,
expected_plan_result,
dependant_module,
):
if dependant_module:
importorxfail(dependant_module)
pool = ProcessPool(name="MyProcessPool", size=1)
pdf_path = os.path.join(runpath,
"test_report_local_{}.pdf".format(pdf_title))
plan = TestplanMock(
name="plan",
exporters=[PDFExporter(pdf_path=pdf_path)],
runpath=runpath,
)
plan.add_resource(pool)
runners_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
fixture_path = os.path.join(runners_path, "fixtures", fixture_dirname)
task = Task(target="make_multitest", module="suites", path=fixture_path)
plan.schedule(task, resource="MyProcessPool")
assert not os.path.exists(pdf_path)
assert plan.run().run is True
for log in plan.report.flattened_logs:
if all(word in log["message"] for word in ["tkinter", "TclError"]):
pytest.xfail(reason="Tkinter not installed properly")
check_report(expected=expected_report, actual=plan.report)
assert plan.result.success is expected_plan_result
assert os.path.exists(pdf_path)
assert os.stat(pdf_path).st_size > 0