def schedule_tests_to_pool(plan, pool, **pool_cfg): pool_name = pool.__name__ pool = pool(name=pool_name, **pool_cfg) plan.add_resource(pool) dirname = os.path.dirname(os.path.abspath(__file__)) mtest1 = MultiTest(name="MTest1", suites=[MySuite()]) mtest2 = MultiTest(name="MTest2", suites=[MySuite()]) uid1 = plan.schedule(target=mtest1, resource=pool_name) uid2 = plan.schedule(Task(target=mtest2), resource=pool_name) task3 = Task(target=get_mtest, path=dirname, kwargs=dict(name=3)) uid3 = plan.schedule(task=task3, resource=pool_name) # Task schedule shortcut uid4 = plan.schedule( target="get_mtest", module="func_pool_base_tasks", path=dirname, kwargs=dict(name=4), resource=pool_name, ) uid5 = plan.schedule( Task( target="get_mtest", module="func_pool_base_tasks", path=dirname, kwargs=dict(name=5), ), resource=pool_name, ) from .func_pool_base_tasks import get_mtest_imported uid6 = plan.schedule( Task(target=get_mtest_imported, kwargs=dict(name=6)), resource=pool_name, ) uid7 = plan.schedule(Task(target=get_mtest(name=7)), resource=pool_name) # with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is True assert plan.report.passed is True assert plan.report.counter == {"passed": 7, "total": 7, "failed": 0} names = sorted(["MTest{}".format(x) for x in range(1, 8)]) assert sorted([entry.name for entry in plan.report.entries]) == names assert isinstance(plan.report.serialize(), dict) assert plan.result.test_results[uid1].report.name == "MTest1" assert plan.result.test_results[uid2].report.name == "MTest2" assert plan.result.test_results[uid3].report.name == "MTest3" assert plan.result.test_results[uid4].report.name == "MTest4" assert plan.result.test_results[uid5].report.name == "MTest5" assert plan.result.test_results[uid6].report.name == "MTest6" assert plan.result.test_results[uid7].report.name == "MTest7"
def test_mainline(self): """ Test mainline message flow between a worker and its Pool. The worker polls for a task and when one is received, the worker sends back the results of executing that task. """ pool = pools_base.Pool(name="MyPool", size=1, worker_type=ControllableWorker) # Start the pool via its context manager - this starts the Pool's main # work loop in a separate thread. with pool: assert pool.is_alive and pool.active assert pool.status.tag == pool.status.STARTED # Retrieve the only worker assigned to this pool. assert len(pool._workers) == 1 worker = pool._workers["0"] assert worker.is_alive and worker.active assert worker.status.tag == worker.status.STARTED msg_factory = communication.Message(**worker.metadata) # Send a TaskPullRequest from the worker to the Pool. The Pool # should respond with an Ack since no Tasks have been added yet. received = worker.transport.send_and_receive( msg_factory.make(msg_factory.TaskPullRequest, data=1)) assert received.cmd == communication.Message.Ack # Add a task to the pool. task1 = Task(target=Runnable(5)) pool.add(task1, uid=task1.uid()) # Send in another TaskPullRequest - the Pool should respond with # the task we just added. received = worker.transport.send_and_receive( msg_factory.make(msg_factory.TaskPullRequest, data=1)) assert received.cmd == communication.Message.TaskSending assert len(received.data) == 1 assert received.data[0] == task1 # Execute the task and send back the TaskResults. task_result = worker.execute(task1) results = [task_result] received = worker.transport.send_and_receive( msg_factory.make(msg_factory.TaskResults, data=results)) assert received.cmd == communication.Message.Ack # Check that the pool now has the results stored. assert pool._results[task1.uid()] == task_result # The Pool and its work loop should be stopped on exiting the context # manager. assert pool.status.tag == pool.status.STOPPED
def schedule_tests_to_pool(pool, **pool_cfg): pool_name = pool.__name__ plan = Testplan( name='Plan', parse_cmdline=False, ) pool = pool(name=pool_name, **pool_cfg) plan.add_resource(pool) dirname = os.path.dirname(os.path.abspath(__file__)) mtest1 = MultiTest(name='MTest1', suites=[MySuite()]) mtest2 = MultiTest(name='MTest2', suites=[MySuite()]) uid1 = plan.schedule(target=mtest1, resource=pool_name) uid2 = plan.schedule(Task(target=mtest2), resource=pool_name) task3 = Task(target=get_mtest, path=dirname) uid3 = plan.schedule(task=task3, resource=pool_name) # Task schedule shortcut uid4 = plan.schedule(target='get_mtest', module='func_pool_base_tasks', path=dirname, kwargs=dict(name=4), resource=pool_name) uid5 = plan.schedule(Task(target='get_mtest', module='func_pool_base_tasks', path=dirname, kwargs=dict(name=5)), resource=pool_name) from .func_pool_base_tasks import get_mtest_imported uid6 = plan.schedule(Task(target=get_mtest_imported, kwargs=dict(name=6)), resource=pool_name) with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is True assert plan.report.passed is True assert plan.report.counts.passed == 6 assert plan.report.counts.error == plan.report.counts.skipped == \ plan.report.counts.failed == plan.report.counts.incomplete == 0 names = sorted(['MTest{}'.format(x) for x in range(1, 7)]) assert sorted([entry.name for entry in plan.report.entries]) == names assert isinstance(plan.report.serialize(), dict) assert plan.result.test_results[uid1].report.name == 'MTest1' assert plan.result.test_results[uid2].report.name == 'MTest2' assert plan.result.test_results[uid3].report.name == 'MTest3' assert plan.result.test_results[uid4].report.name == 'MTest4' assert plan.result.test_results[uid5].report.name == 'MTest5' assert plan.result.test_results[uid6].report.name == 'MTest6'
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ import testplan workspace = os.path.abspath( os.path.join(os.path.dirname(module_abspath(testplan)), '..', '..')) # Add a remote pool test execution resource to the plan of given size. pool = RemotePool(name='MyPool', hosts={socket.gethostname(): 3}, workspace=workspace) plan.add_resource(pool) # Add a given number of similar tests to the remote pool # to be executed in parallel. for idx in range(plan.args.tasks_num): # All Task arguments need to be serializable. task = Task(target='make_multitest', module='tasks', path='.', kwargs={'index': idx}) plan.schedule(task, resource='MyPool')
def test_task_rerun_with_more_times_2(mockplan): """ Test procedure 2: (set `task_rerun_limit` to 4, a task can be scheduled 5 times in total) - 1st run: `unstable_case` fails. - 1st rerun: `unstable_case` fails. - 2nd rerun: `unstable_case` fails. - 3rd rerun: all pass. """ pool_name = ThreadPool.__name__ pool = ThreadPool(name=pool_name, size=1) mockplan.add_resource(pool) directory = os.path.dirname(os.path.abspath(__file__)) tmp_file = os.path.join(tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4())) task = Task(target=make_multitest_3, path=directory, args=(tmp_file, ), rerun=3) uid = mockplan.schedule(task=task, resource=pool_name) assert mockplan.run().run is True assert mockplan.report.passed is True assert mockplan.report.counter == {"passed": 2, "total": 2, "failed": 0} assert mockplan.result.test_results[uid].report.name == "Unstable MTest3" assert task.reassign_cnt == 3 _remove_existing_tmp_file(tmp_file)
def test_task_rerun_with_parts(): with tempfile.TemporaryDirectory() as runpath: mockplan = TestplanMock("plan", runpath=runpath, merge_scheduled_parts=True) pool_name = ThreadPool.__name__ pool = ThreadPool(name=pool_name, size=1) mockplan.add_resource(pool) uids = [] for idx in range(3): task = Task( target=make_multitest_parts, kwargs={"part_tuple": (idx, 3)}, rerun=1, ) uids.append(mockplan.schedule(task=task, resource=pool_name)) assert mockplan.run().run is True assert mockplan.report.passed is False assert mockplan.report.counter == { "passed": 2, "total": 3, "failed": 1, } # Run2 of part0 are merged with part1 and part2 assert mockplan.report.entries[0].name == "MultiTestParts" # Run1 of part0 (task_rerun) are left unmerged assert (mockplan.report.entries[1].name == "MultiTestParts - part(0/3) => Run 1")
def test_task_rerun_with_more_times(mockplan): """ Test procedure 1: (set `task_rerun_limit` to 2, a task can be scheduled 3 times in total) - 1st run: `unstable_case` fails. - 1st rerun: `unstable_case` fails. - 2nd rerun: `unstable_case` fails. """ pool_name = ThreadPool.__name__ pool = ThreadPool(name=pool_name, size=1) mockplan.add_resource(pool) tmp_file = os.path.join(tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4())) task = Task(target=make_multitest_3, args=(tmp_file, ), rerun=2) uid = mockplan.schedule(task=task, resource=pool_name) assert mockplan.run().run is True assert mockplan.report.passed is False assert mockplan.report.counter == {"passed": 1, "total": 2, "failed": 1} assert mockplan.result.test_results[uid].report.name == "Unstable MTest3" assert task.reassign_cnt == 2 # test fails, should manually remove it _remove_existing_tmp_file(tmp_file)
def test_task_rerun_in_thread_pool(mockplan): """ Test procedure: - 1st run: `unstable_case` fails. - 1st rerun: `mock_driver` raises during stop. - 2nd rerun: all pass. """ pool_name = ThreadPool.__name__ pool = ThreadPool(name=pool_name, size=2) mockplan.add_resource(pool) directory = os.path.dirname(os.path.abspath(__file__)) tmp_file = os.path.join(tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4())) task = Task(target=make_multitest_1, path=directory, args=(tmp_file, ), rerun=2) uid = mockplan.schedule(task=task, resource=pool_name) assert mockplan.run().run is True assert mockplan.report.passed is True assert mockplan.report.counter == {"passed": 3, "total": 3, "failed": 0} assert isinstance(mockplan.report.serialize(), dict) assert mockplan.result.test_results[uid].report.name == "Unstable MTest1" assert len(mockplan.report.entries) == 3 assert mockplan.report.entries[-1].category == ReportCategories.TASK_RERUN assert mockplan.report.entries[-2].category == ReportCategories.TASK_RERUN assert task.reassign_cnt == 2 _remove_existing_tmp_file(tmp_file)
def test_task_rerun_in_process_pool(mockplan): """ Test 1 procedure: - 1st run: `unstable_case` fails. - 1st rerun: `mock_driver` raises during stop. - 2nd rerun: all pass. Test 2 procedure: - 1st run: `unstable_worker` makes child process exit. - monitor detects inactive worker, decommission the task from worker, then re-assign it and it passes (no rerun is needed). """ pool_name = ProcessPool.__name__ pool = ProcessPool(name=pool_name, size=2) mockplan.add_resource(pool) directory = os.path.dirname(os.path.abspath(__file__)) tmp_file_1 = os.path.join(tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4())) tmp_file_2 = os.path.join(tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4())) task1 = Task(target=make_multitest_1, path=directory, args=(tmp_file_1, ), rerun=2) task2 = Task(target=make_multitest_2, path=directory, args=(tmp_file_2, ), rerun=0) uid1 = mockplan.schedule(task=task1, resource=pool_name) uid2 = mockplan.schedule(task=task2, resource=pool_name) assert mockplan.run().run is True assert mockplan.report.passed is True assert mockplan.report.counter == {"passed": 5, "total": 5, "failed": 0} assert isinstance(mockplan.report.serialize(), dict) assert mockplan.result.test_results[uid1].report.name == "Unstable MTest1" assert mockplan.result.test_results[uid2].report.name == "Unstable MTest2" assert len(mockplan.report.entries) == 4 assert mockplan.report.entries[-1].category == ReportCategories.TASK_RERUN assert mockplan.report.entries[-2].category == ReportCategories.TASK_RERUN assert task1.reassign_cnt == 2 assert task2.reassign_cnt == 0 # 1st run: assigned but not executed assert pool._task_retries_cnt[uid2] == 1 _remove_existing_tmp_file(tmp_file_1) _remove_existing_tmp_file(tmp_file_2)
def test_process_pool_integration( report_dir, fixture_dirname, expected_report, pdf_title, expected_plan_result, dependant_module ): if dependant_module: importorxfail(dependant_module) pool = ProcessPool(name='MyPool', size=1) pdf_path = report_dir.join('test_report_process_{}.pdf'.format( pdf_title)).strpath plan = Testplan( name='plan', parse_cmdline=False, exporters=[ PDFExporter(pdf_path=pdf_path) ] ) plan.add_resource(pool) runners_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) fixture_path = os.path.join(runners_path, 'fixtures', fixture_dirname) task = Task( target='make_multitest', module='suites', path=fixture_path, ) plan.schedule(task, resource='MyPool') assert not os.path.exists(pdf_path) with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is True for log in plan.report.flattened_logs: if all(word in log['message'] for word in ['tkinter', 'TclError']): pytest.xfail(reason='Tkinter not installed properly') check_report(expected=expected_report, actual=plan.report) assert plan.result.success is expected_plan_result assert os.path.exists(pdf_path) assert os.stat(pdf_path).st_size > 0
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ # Add a thread pool test execution resource to the plan of given size. pool = ThreadPool(name="MyPool", size=plan.args.pool_size) plan.add_resource(pool) # Add a given number of similar tests to the thread pool # to be executed in parallel. for idx in range(plan.args.tasks_num): task = Task(target="make_multitest", module="tasks", kwargs={"index": idx}) plan.schedule(task, resource="MyPool")
def test_process_pool_integration( runpath, fixture_dirname, expected_report, pdf_title, expected_plan_result, dependant_module, ): if dependant_module: importorxfail(dependant_module) pool = ProcessPool(name="MyProcessPool", size=1) pdf_path = os.path.join( runpath, "test_report_local_{}.pdf".format(pdf_title) ) plan = TestplanMock( name="plan", exporters=[PDFExporter(pdf_path=pdf_path)], runpath=runpath, ) plan.add_resource(pool) runners_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) fixture_path = os.path.join(runners_path, "fixtures", fixture_dirname) task = Task(target="make_multitest", module="suites", path=fixture_path) plan.schedule(task, resource="MyProcessPool") assert not os.path.exists(pdf_path) with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is True for log in plan.report.flattened_logs: if all(word in log["message"] for word in ["tkinter", "TclError"]): pytest.xfail(reason="Tkinter not installed properly") check_report(expected=expected_report, actual=plan.report) assert plan.result.success is expected_plan_result assert os.path.exists(pdf_path) assert os.stat(pdf_path).st_size > 0
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ # Add a thread pool test execution resource to the plan of given size. # Also you can use process pool or remote pool instead. pool = ThreadPool(name='MyPool', size=plan.args.pool_size) plan.add_resource(pool) # Add a given number of similar tests to the thread pool # to be executed in parallel. for idx in range(plan.args.parts_num): task = Task(target='make_multitest', module='tasks', kwargs={'part_tuple': (idx, plan.args.parts_num)}) plan.schedule(task, resource='MyPool')
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ # Add a process pool test execution resource to the plan of given size. pool = ProcessPool(name='MyPool', size=plan.args.pool_size) plan.add_resource(pool) # Add a given number of similar tests to the process pool # to be executed in parallel. for idx in range(plan.args.tasks_num): # All Task arguments need to be serializable. task = Task(target='make_multitest', module='tasks', kwargs={'index': idx}) plan.schedule(task, resource='MyPool')
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ # Add a thread pool test execution resource to the plan of given size. # Can also use a process pool instead. pool = ThreadPool(name="MyPool", size=plan.args.pool_size) plan.add_resource(pool) # Add a task with `rerun` argument to the thread pool tmp_file = os.path.join( tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4()) ) task = Task( target="make_multitest", module="tasks", args=(tmp_file,), rerun=2 ) plan.schedule(task, resource="MyPool")
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ workspace = os.path.dirname(__file__) # Create two temporary files locally. For demonstration, just write the # filename as the content of each. assert TEMP_DIR is not None for filename in ("file1", "file2"): make_file(filename, TEMP_DIR, content=filename) # Explicitly specify the full paths to both the local source files just # created and the destination filepaths on the remote host. push_files = [ (os.path.join(TEMP_DIR, "file1"), "/tmp/remote_example/file1"), (os.path.join(TEMP_DIR, "file2"), "/tmp/remote_example/file2"), ] # Add a remote pool test execution resource to the plan of given size. pool = RemotePool( name="MyPool", # Create 3 workers on the same remote host. hosts={REMOTE_HOST: 3}, # Allow the remote port to be overridden by the # environment. Default to 0, which will make testplan use # the default SSH port for connections. port=int(os.environ.get("TESTPLAN_REMOTE_PORT", 0)), setup_script=["/bin/bash", "setup_script.ksh"], env={"LOCAL_USER": getpass.getuser(), "LOCAL_WORKSPACE": workspace}, workspace_exclude=[".git/", ".cache/", "doc/", "test/"], # We push local files to the remote worker using the # explicit source and destination locations defined above. push=push_files, workspace=workspace, clean_remote=True, ) plan.add_resource(pool) # Add a given number of similar tests to the remote pool # to be executed in parallel. for idx in range(plan.args.tasks_num): # All Task arguments need to be serializable. task = Task( target="make_multitest", module="tasks", # We specify the full paths to files as they will be found # on the remote host. kwargs={ "index": idx, "files": [ "/tmp/remote_example/file1", "/tmp/remote_example/file2", ], }, ) plan.schedule(task, resource="MyPool")
def schedule_tests_to_pool(plan, pool, **pool_cfg): pool_name = pool.__name__ pool = pool(name=pool_name, **pool_cfg) plan.add_resource(pool) uids = [] dirname = os.path.dirname(os.path.abspath(__file__)) mtest1 = MultiTest(name="MTest1", suites=[MySuite()]) mtest2 = MultiTest(name="MTest2", suites=[MySuite()]) uids.append(plan.schedule(target=mtest1, weight=1, resource=pool_name)) uids.append( plan.schedule(Task(target=mtest2, weight=2), resource=pool_name)) task3 = Task(target=get_mtest, path=dirname, kwargs=dict(name=3), weight=3) uids.append(plan.schedule(task=task3, resource=pool_name)) # Task schedule shortcut uids.append( plan.schedule( target="get_mtest", module="func_pool_base_tasks", path=dirname, kwargs=dict(name=4), weight=4, resource=pool_name, )) uids.append( plan.schedule( Task( target="get_mtest", module="func_pool_base_tasks", path=dirname, kwargs=dict(name=5), weight=5, ), resource=pool_name, )) from .func_pool_base_tasks import get_mtest_imported uids.append( plan.schedule( Task(target=get_mtest_imported, kwargs=dict(name=6), weight=6), resource=pool_name, )) uids.append( plan.schedule(Task(target=get_mtest(name=7), weight=7), resource=pool_name)) assert plan.run().run is True assert plan.report.passed is True assert plan.report.counter == {"passed": 10, "total": 10, "failed": 0} names = ["MTest{}".format(x) for x in range(1, 8)] assert [entry.name for entry in plan.report.entries] == names assert isinstance(plan.report.serialize(), dict) assert [plan.result.test_results[uid].report.name for uid in uids] == names assert list(pool._executed_tests) == uids[::-1]
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ import testplan workspace = os.path.abspath( os.path.join( os.path.dirname(module_abspath(testplan)), '..', '..')) # Create two temporary files locally. For demonstration, just write the # filename as the content of each. assert TEMP_DIR is not None for filename in ('file1', 'file2'): make_file(filename, TEMP_DIR, content=filename) # Explicitly specify the full paths to both the local source files just # created and the destination filepaths on the remote host. push_files = [ (os.path.join(TEMP_DIR, 'file1'), '/tmp/remote_example/file1'), (os.path.join(TEMP_DIR, 'file2'), '/tmp/remote_example/file2') ] # Check if the remote host has been specified in the environment. Remote # hosts can only be Linux systems. If none is specified when running on a # Linux system we can default to using the localhost as our "remote" # worker. Whichever remote host is used must be configured to accept SSH # connections from the localhost. remote_host = os.environ.get('TESTPLAN_REMOTE_HOST') if not remote_host: if os.name == 'posix': remote_host = socket.gethostname() else: raise RuntimeError( 'You must specify a remote host via the TESTPLAN_REMOTE_HOST ' 'environment var on non-Linux systems.') # Add a remote pool test execution resource to the plan of given size. pool = RemotePool(name='MyPool', # Create 3 workers on the same remote host. hosts={remote_host: 3}, # Allow the remote port to be overridden by the # environment. Default to 0, which will make testplan use # the default SSH port for connections. port=int(os.environ.get('TESTPLAN_REMOTE_PORT', 0)), setup_script=['/bin/bash', 'setup_script.ksh'], env={'LOCAL_USER': getpass.getuser(), 'LOCAL_WORKSPACE': workspace}, workspace_exclude=['.git/', '.cache/', 'doc/', 'test/'], # We push local files to the remote worker using the # explicit source and destination locations defined above. push=push_files, workspace=workspace) plan.add_resource(pool) # Add a given number of similar tests to the remote pool # to be executed in parallel. for idx in range(plan.args.tasks_num): # All Task arguments need to be serializable. task = Task(target='make_multitest', module='tasks', path='.', # We specify the full paths to files as they will be found # on the remote host. kwargs={'index': idx, 'files': ['/tmp/remote_example/file1', '/tmp/remote_example/file2']}) plan.schedule(task, resource='MyPool')
def test_pool_basic(): dirname = os.path.dirname(os.path.abspath(__file__)) path = os.path.join(dirname, 'tasks', 'data', 'relative') task1 = Task(target=Runnable(5)) task2 = Task(target='Runnable', module='sample_tasks', path=path, args=(10,), kwargs=dict(multiplier=3)) assert task1.materialize().run() == 10 assert task2.materialize().run() == 30 pool = pools_base.Pool( name='MyPool', size=4, runpath=default_runpath) pool.add(task1, uid=task1.uid()) pool.add(task2, uid=task2.uid()) assert pool._input[task1.uid()] is task1 assert pool._input[task2.uid()] is task2 with pool: while pool.ongoing: pass assert pool.get(task1.uid()).result ==\ pool.results[task1.uid()].result == 10 assert pool.get(task2.uid()).result ==\ pool.results[task2.uid()].result == 30