def test_restart_worker(mockplan): pool_name = ProcessPool.__name__ pool_size = 4 retries_limit = int(pool_size / 2) pool = ProcessPool( name=pool_name, size=pool_size, worker_heartbeat=2, heartbeats_miss_limit=3, max_active_loop_sleep=1, ) pool._task_retries_limit = retries_limit pool_uid = mockplan.add_resource(pool) dirname = os.path.dirname(os.path.abspath(__file__)) mockplan.schedule( target="multitest_kills_worker", module="func_pool_base_tasks", path=dirname, args=(os.getpid(),), resource=pool_name, ) for idx in range(1, 25): mockplan.schedule( target="get_mtest", module="func_pool_base_tasks", path=dirname, kwargs=dict(name=idx), resource=pool_name, ) with log_propagation_disabled(TESTPLAN_LOGGER): res = mockplan.run() # Check that all workers are restarted assert ( len( [ worker for worker in mockplan.resources[pool_uid]._workers if worker._aborted is True ] ) == 0 ) assert res.run is False assert res.success is False assert mockplan.report.status == Status.ERROR assert mockplan.report.counter[Status.ERROR] == 1
def test_custom_rerun_condition(mockplan): """Force reschedule task X times to test logic.""" pool_name = ProcessPool.__name__ uid = "custom_task_uid" rerun_limit = 2 def custom_rerun(pool, task_result): if task_result.task.reassign_cnt > task_result.task.rerun: return False return True pool_size = 4 pool = ProcessPool( name=pool_name, size=pool_size, worker_heartbeat=2, heartbeats_miss_limit=2, max_active_loop_sleep=1, restart_count=0, ) pool.set_rerun_check(custom_rerun) pool_uid = mockplan.add_resource(pool) dirname = os.path.dirname(os.path.abspath(__file__)) uid = mockplan.schedule( target="get_mtest", module="func_pool_base_tasks", path=dirname, kwargs=dict(name="0"), resource=pool_name, uid=uid, rerun=rerun_limit, ) with log_propagation_disabled(TESTPLAN_LOGGER): res = mockplan.run() assert ( len( [ worker for worker in mockplan.resources[pool_uid]._workers if worker._aborted is True ] ) == 0 ) assert res.success is True assert mockplan.report.status == Status.PASSED assert pool.added_item(uid).reassign_cnt == rerun_limit
def test_kill_all_workers(): """Kill all workers and create a failed report.""" pool_name = ProcessPool.__name__ plan = Testplan( name='ProcPlan', parse_cmdline=False, ) pool_size = 4 pool = ProcessPool(name=pool_name, size=pool_size, task_retries_limit=pool_size, worker_heartbeat=2, heartbeats_miss_limit=2) pool_uid = plan.add_resource(pool) dirname = os.path.dirname(os.path.abspath(__file__)) uid = plan.schedule(target='multitest_kills_worker', module='func_pool_base_tasks', path=dirname, resource=pool_name) with log_propagation_disabled(TESTPLAN_LOGGER): res = plan.run() # Check that the worker killed by test was aborted assert len([ worker for worker in plan.resources[pool_uid]._workers if worker._aborted is True ]) == pool_size assert res.success is False # scheduled X times and killed all workers assert pool.task_assign_cnt[uid] == pool_size assert plan.report.status == Status.ERROR
def test_schedule_from_main(mockplan): """ Test scheduling Tasks from __main__ - it should not be allowed for ProcessPool. """ # Set up a testplan and add a ProcessPool. pool = ProcessPool(name="ProcPool", size=2) mockplan.add_resource(pool) # First check that scheduling a Task with module string of '__main__' # raises the expected ValueError. with pytest.raises(ValueError): mockplan.schedule( target="target", module="__main__", resource="ProcPool" ) # Secondly, check that scheduling a callable target with a __module__ attr # of __main__ also raises a ValueError. def callable_target(): raise RuntimeError callable_target.__module__ = "__main__" with pytest.raises(ValueError): mockplan.schedule(target=callable_target, resource="ProcPool")
def test_kill_one_worker(mockplan): """Kill one worker but pass after reassigning task.""" pool_name = ProcessPool.__name__ pool_size = 4 pool = ProcessPool( name=pool_name, size=pool_size, worker_heartbeat=2, heartbeats_miss_limit=2, max_active_loop_sleep=1, restart_count=0, ) pool_uid = mockplan.add_resource(pool) dirname = os.path.dirname(os.path.abspath(__file__)) kill_uid = mockplan.schedule( target="multitest_kill_one_worker", module="func_pool_base_tasks", path=dirname, args=(os.getpid(), pool_size), # kills 4th worker resource=pool_name, ) uids = [] for idx in range(1, 25): uids.append( mockplan.schedule( target="get_mtest", module="func_pool_base_tasks", path=dirname, kwargs=dict(name=idx), resource=pool_name, ) ) with log_propagation_disabled(TESTPLAN_LOGGER): res = mockplan.run() # Check that the worker killed by test was aborted assert ( len( [ worker for worker in mockplan.resources[pool_uid]._workers if worker._aborted is True ] ) == 1 ) assert res.run is True assert res.success is True assert mockplan.report.status == Status.PASSED # All tasks scheduled once except the killed one for idx in range(1, 25): if uids[idx - 1] == kill_uid: assert pool._task_retries_cnt[uids[idx - 1]] == 1
def test_custom_reschedule_condition(): """Force reschedule task X times to test logic.""" pool_name = ProcessPool.__name__ plan = Testplan(name="ProcPlan", parse_cmdline=False) uid = "custom_task_uid" max_reschedules = 2 def custom_reschedule(pool, task_result): if pool.task_assign_cnt[uid] == max_reschedules: return False return True pool_size = 4 pool = ProcessPool( name=pool_name, size=pool_size, worker_heartbeat=2, heartbeats_miss_limit=2, max_active_loop_sleep=1, restart_count=0, ) pool.set_reschedule_check(custom_reschedule) pool_uid = plan.add_resource(pool) dirname = os.path.dirname(os.path.abspath(__file__)) plan.schedule( target="get_mtest", module="func_pool_base_tasks", path=dirname, kwargs=dict(name="0"), resource=pool_name, uid=uid, ) with log_propagation_disabled(TESTPLAN_LOGGER): res = plan.run() assert (len([ worker for worker in plan.resources[pool_uid]._workers if worker._aborted is True ]) == 0) assert res.success is True assert pool.task_assign_cnt[uid] == max_reschedules assert plan.report.status == Status.PASSED
def test_kill_all_workers(mockplan): """Kill all workers and create a failed report.""" pool_name = ProcessPool.__name__ pool_size = 4 retries_limit = 3 pool = ProcessPool( name=pool_name, size=pool_size, worker_heartbeat=2, heartbeats_miss_limit=2, max_active_loop_sleep=1, restart_count=0, ) pool._task_retries_limit = retries_limit pool_uid = mockplan.add_resource(pool) dirname = os.path.dirname(os.path.abspath(__file__)) uid = mockplan.schedule( target="multitest_kills_worker", module="func_pool_base_tasks", path=dirname, args=(os.getpid(),), resource=pool_name, ) with log_propagation_disabled(TESTPLAN_LOGGER): res = mockplan.run() # Check that the worker killed by test was aborted assert ( len( [ worker for worker in mockplan.resources[pool_uid]._workers if worker._aborted is True ] ) == pool_size ) assert res.success is False # scheduled X times and killed all workers assert pool._task_retries_cnt[uid] == retries_limit + 1 assert mockplan.report.status == Status.ERROR
def test_custom_reschedule_condition(): """Force reschedule task X times to test logic.""" pool_name = ProcessPool.__name__ plan = Testplan( name='ProcPlan', parse_cmdline=False, ) uid = 'custom_task_uid' max_reschedules = 2 def custom_reschedule(pool, task_result): if pool.task_assign_cnt[uid] == max_reschedules: return False return True pool_size = 4 pool = ProcessPool(name=pool_name, size=pool_size, worker_heartbeat=2, heartbeats_miss_limit=2) pool.set_reschedule_check(custom_reschedule) pool_uid = plan.add_resource(pool) dirname = os.path.dirname(os.path.abspath(__file__)) plan.schedule(target='get_mtest', module='func_pool_base_tasks', path=dirname, kwargs=dict(name='0'), resource=pool_name, uid=uid) with log_propagation_disabled(TESTPLAN_LOGGER): res = plan.run() # Check that the worker killed by test was aborted assert len([ worker for worker in plan.resources[pool_uid]._workers if worker._aborted is True ]) == 0 assert res.success is True assert pool.task_assign_cnt[uid] == max_reschedules assert plan.report.status == Status.PASSED
def test_serialization(): """Test serialization of test results.""" plan = Testplan(name='SerializationPlan', parse_cmdline=False) pool = ProcessPool(name='ProcPool', size=2) plan.add_resource(pool) plan.schedule(target='make_serialization_mtest', module='test_pool_process', path=os.path.dirname(__file__), resource='ProcPool') res = plan.run() assert res.success
def test_runner_timeout(): """ Execute MultiTests in LocalRunner, ThreadPool and ProcessPool respectively. Some of them will timeout and we'll get a report showing execution details. """ plan = Testplan(name='plan', parse_cmdline=False, runnable=MyTestRunner, timeout=60, abort_wait_timeout=5) mod_path = os.path.dirname(os.path.abspath(__file__)) THREAD_POOL = 'MyThreadPool' PROCESS_POOL = 'MyProcessPool' thread_pool = ThreadPool(name=THREAD_POOL, size=2, worker_heartbeat=None) proc_pool = ProcessPool(name=PROCESS_POOL, size=2, worker_heartbeat=None) plan.add_resource(thread_pool) plan.add_resource(proc_pool) plan.add(func_basic_tasks.get_mtest1()) plan.add(func_basic_tasks.get_mtest2()) task3 = Task(target='get_mtest3', module='func_basic_tasks', path=mod_path) task4 = Task(target='get_mtest4', module='func_basic_tasks', path=mod_path) task5 = Task(target='get_mtest5', module='func_basic_tasks', path=mod_path) task6 = Task(target='get_mtest6', module='func_basic_tasks', path=mod_path) plan.schedule(task3, resource=THREAD_POOL) plan.schedule(task4, resource=THREAD_POOL) plan.schedule(task5, resource=PROCESS_POOL) plan.schedule(task6, resource=PROCESS_POOL) with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is False assert len(plan.report.entries) == 6 assert plan.report.status == Status.ERROR entries = plan.report.entries assert entries[0].name == 'MTest1' assert entries[0].status == Status.FAILED # testcase 'test_true' failed assert entries[2].name == 'MTest3' assert entries[2].status == Status.PASSED # testcase 'test_equal' passed assert entries[4].name == 'MTest5' assert entries[4].status == Status.ERROR # testcase 'test_contain' raised assert entries[1].name == 'MTest2' assert entries[1].status == Status.ERROR # timeout assert ' discarding due to ' in entries[1].logs[0]['message'] assert entries[3].name == 'Task[get_mtest4]' assert entries[3].status == Status.ERROR # timeout assert entries[3].logs[0]['message'].startswith('_target: get_mtest4') assert ' discarding due to ' in entries[3].logs[1]['message'] assert entries[5].name == 'Task[get_mtest6]' assert entries[5].status == Status.ERROR # timeout assert entries[5].logs[0]['message'].startswith('_target: get_mtest6') assert ' discarding due to ' in entries[5].logs[1]['message']
def test_serialization(mockplan): """Test serialization of test results.""" pool = ProcessPool(name="ProcPool", size=2) mockplan.add_resource(pool) mockplan.schedule( target="make_serialization_mtest", module="test_pool_process", path=os.path.dirname(__file__), resource="ProcPool", ) res = mockplan.run() assert res.success
def test_kill_one_worker(): """Kill one worker but pass after reassigning task.""" pool_name = ProcessPool.__name__ plan = Testplan( name='ProcPlan', parse_cmdline=False, ) pool_size = 4 pool = ProcessPool(name=pool_name, size=pool_size, worker_heartbeat=2, heartbeats_miss_limit=2, max_active_loop_sleep=1, restart_count=0) pool_uid = plan.add_resource(pool) dirname = os.path.dirname(os.path.abspath(__file__)) kill_uid = plan.schedule( target='multitest_kill_one_worker', module='func_pool_base_tasks', path=dirname, args=('killer', os.getpid(), pool_size), # kills 4th worker resource=pool_name) uids = [] for idx in range(1, 25): uids.append( plan.schedule(target='get_mtest', module='func_pool_base_tasks', path=dirname, kwargs=dict(name=idx), resource=pool_name)) with log_propagation_disabled(TESTPLAN_LOGGER): res = plan.run() # Check that the worker killed by test was aborted assert len([ worker for worker in plan.resources[pool_uid]._workers if worker._aborted is True ]) == 1 assert res.run is True assert res.success is True assert plan.report.status == Status.PASSED # All tasks scheduled once for uid in pool.task_assign_cnt: if uid == kill_uid: assert pool.task_assign_cnt[uid] == 2 else: assert pool.task_assign_cnt[uid] == 1
def test_task_rerun_in_process_pool(mockplan): """ Test 1 procedure: - 1st run: `unstable_case` fails. - 1st rerun: `mock_driver` raises during stop. - 2nd rerun: all pass. Test 2 procedure: - 1st run: `unstable_worker` makes child process exit. - monitor detects inactive worker, decommission the task from worker, then re-assign it and it passes (no rerun is needed). """ pool_name = ProcessPool.__name__ pool = ProcessPool(name=pool_name, size=2) mockplan.add_resource(pool) directory = os.path.dirname(os.path.abspath(__file__)) tmp_file_1 = os.path.join(tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4())) tmp_file_2 = os.path.join(tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4())) task1 = Task(target=make_multitest_1, path=directory, args=(tmp_file_1, ), rerun=2) task2 = Task(target=make_multitest_2, path=directory, args=(tmp_file_2, ), rerun=0) uid1 = mockplan.schedule(task=task1, resource=pool_name) uid2 = mockplan.schedule(task=task2, resource=pool_name) assert mockplan.run().run is True assert mockplan.report.passed is True assert mockplan.report.counter == {"passed": 5, "total": 5, "failed": 0} assert isinstance(mockplan.report.serialize(), dict) assert mockplan.result.test_results[uid1].report.name == "Unstable MTest1" assert mockplan.result.test_results[uid2].report.name == "Unstable MTest2" assert len(mockplan.report.entries) == 4 assert mockplan.report.entries[-1].category == ReportCategories.TASK_RERUN assert mockplan.report.entries[-2].category == ReportCategories.TASK_RERUN assert task1.reassign_cnt == 2 assert task2.reassign_cnt == 0 # 1st run: assigned but not executed assert pool._task_retries_cnt[uid2] == 1 _remove_existing_tmp_file(tmp_file_1) _remove_existing_tmp_file(tmp_file_2)
def test_process_pool_integration( report_dir, fixture_dirname, expected_report, pdf_title, expected_plan_result, dependant_module ): if dependant_module: importorxfail(dependant_module) pool = ProcessPool(name='MyPool', size=1) pdf_path = report_dir.join('test_report_process_{}.pdf'.format( pdf_title)).strpath plan = Testplan( name='plan', parse_cmdline=False, exporters=[ PDFExporter(pdf_path=pdf_path) ] ) plan.add_resource(pool) runners_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) fixture_path = os.path.join(runners_path, 'fixtures', fixture_dirname) task = Task( target='make_multitest', module='suites', path=fixture_path, ) plan.schedule(task, resource='MyPool') assert not os.path.exists(pdf_path) with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is True for log in plan.report.flattened_logs: if all(word in log['message'] for word in ['tkinter', 'TclError']): pytest.xfail(reason='Tkinter not installed properly') check_report(expected=expected_report, actual=plan.report) assert plan.result.success is expected_plan_result assert os.path.exists(pdf_path) assert os.stat(pdf_path).st_size > 0
def test_process_pool_integration( runpath, fixture_dirname, expected_report, pdf_title, expected_plan_result, dependant_module, ): if dependant_module: importorxfail(dependant_module) pool = ProcessPool(name="MyProcessPool", size=1) pdf_path = os.path.join( runpath, "test_report_local_{}.pdf".format(pdf_title) ) plan = TestplanMock( name="plan", exporters=[PDFExporter(pdf_path=pdf_path)], runpath=runpath, ) plan.add_resource(pool) runners_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) fixture_path = os.path.join(runners_path, "fixtures", fixture_dirname) task = Task(target="make_multitest", module="suites", path=fixture_path) plan.schedule(task, resource="MyProcessPool") assert not os.path.exists(pdf_path) with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is True for log in plan.report.flattened_logs: if all(word in log["message"] for word in ["tkinter", "TclError"]): pytest.xfail(reason="Tkinter not installed properly") check_report(expected=expected_report, actual=plan.report) assert plan.result.success is expected_plan_result assert os.path.exists(pdf_path) assert os.stat(pdf_path).st_size > 0
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ # Add a process pool test execution resource to the plan of given size. pool = ProcessPool(name='MyPool', size=plan.args.pool_size) plan.add_resource(pool) # Add a given number of similar tests to the process pool # to be executed in parallel. for idx in range(plan.args.tasks_num): # All Task arguments need to be serializable. task = Task(target='make_multitest', module='tasks', kwargs={'index': idx}) plan.schedule(task, resource='MyPool')
def test_restart_worker(): pool_name = ProcessPool.__name__ plan = Testplan( name='ProcPlan', parse_cmdline=False, ) pool_size = 4 retries_limit = int(pool_size / 2) pool = ProcessPool(name=pool_name, size=pool_size, task_retries_limit=retries_limit, worker_heartbeat=2, heartbeats_miss_limit=3, max_active_loop_sleep=1) pool_uid = plan.add_resource(pool) dirname = os.path.dirname(os.path.abspath(__file__)) plan.schedule(target='multitest_kills_worker', module='func_pool_base_tasks', path=dirname, resource=pool_name) for idx in range(1, 25): plan.schedule(target='get_mtest', module='func_pool_base_tasks', path=dirname, kwargs=dict(name=idx), resource=pool_name) with log_propagation_disabled(TESTPLAN_LOGGER): res = plan.run() # Check that all workers are restarted assert len([ worker for worker in plan.resources[pool_uid]._workers if worker._aborted is True ]) == 0 assert res.run is False assert res.success is False assert plan.report.status == Status.ERROR
def test_reassign_times_limit(): """Kill workers and reassign task up to limit times.""" pool_name = ProcessPool.__name__ plan = Testplan(name="ProcPlan", parse_cmdline=False) pool_size = 4 retries_limit = int(pool_size / 2) pool = ProcessPool( name=pool_name, size=pool_size, task_retries_limit=retries_limit, worker_heartbeat=2, heartbeats_miss_limit=2, max_active_loop_sleep=1, restart_count=0, ) pool_uid = plan.add_resource(pool) dirname = os.path.dirname(os.path.abspath(__file__)) uid = plan.schedule( target="multitest_kills_worker", module="func_pool_base_tasks", path=dirname, resource=pool_name, ) with log_propagation_disabled(TESTPLAN_LOGGER): res = plan.run() # Check that the worker killed by test was aborted assert (len([ worker for worker in plan.resources[pool_uid]._workers if worker._aborted is True ]) == retries_limit) assert res.success is False assert pool.task_assign_cnt[uid] == retries_limit assert plan.report.status == Status.ERROR assert plan.report.counter["error"] == 1
def test_schedule_from_main(): """ Test scheduling Tasks from __main__ - it should not be allowed for ProcessPool. """ # Set up a testplan and add a ProcessPool. plan = Testplan(name='ProcPlan', parse_cmdline=False) pool = ProcessPool(name='ProcPool', size=2) plan.add_resource(pool) # First check that scheduling a Task with module string of '__main__' # raises the expected ValueError. with pytest.raises(ValueError): plan.schedule(target='target', module='__main__', resource='ProcPool') # Secondly, check that scheduling a callable target with a __module__ attr # of __main__ also raises a ValueError. def callable_target(): raise RuntimeError callable_target.__module__ = '__main__' with pytest.raises(ValueError): plan.schedule(target=callable_target, resource='ProcPool')