def test_restart_worker(mockplan):
pool_name = ProcessPool.__name__
pool_size = 4
retries_limit = int(pool_size / 2)
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=3,
max_active_loop_sleep=1,
)
pool._task_retries_limit = retries_limit
pool_uid = mockplan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
mockplan.schedule(
target="multitest_kills_worker",
module="func_pool_base_tasks",
path=dirname,
args=(os.getpid(),),
resource=pool_name,
)
for idx in range(1, 25):
mockplan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name=idx),
resource=pool_name,
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = mockplan.run()
# Check that all workers are restarted
assert (
len(
[
worker
for worker in mockplan.resources[pool_uid]._workers
if worker._aborted is True
]
)
== 0
)
assert res.run is False
assert res.success is False
assert mockplan.report.status == Status.ERROR
assert mockplan.report.counter[Status.ERROR] == 1
def test_kill_all_workers(mockplan):
"""Kill all workers and create a failed report."""
pool_name = ProcessPool.__name__
pool_size = 4
retries_limit = 3
pool = ProcessPool(
name=pool_name,
size=pool_size,
worker_heartbeat=2,
heartbeats_miss_limit=2,
max_active_loop_sleep=1,
restart_count=0,
)
pool._task_retries_limit = retries_limit
pool_uid = mockplan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
uid = mockplan.schedule(
target="multitest_kills_worker",
module="func_pool_base_tasks",
path=dirname,
args=(os.getpid(),),
resource=pool_name,
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = mockplan.run()
# Check that the worker killed by test was aborted
assert (
len(
[
worker
for worker in mockplan.resources[pool_uid]._workers
if worker._aborted is True
]
)
== pool_size
)
assert res.success is False
# scheduled X times and killed all workers
assert pool._task_retries_cnt[uid] == retries_limit + 1
assert mockplan.report.status == Status.ERROR
