def test_multi_parts_missing_parts(): """ Execute MultiTest parts with invalid parameters that not all parts of a MultiTest has been scheduled. """ plan = Testplan(name='plan', parse_cmdline=False, merge_scheduled_parts=True) pool = ThreadPool(name='MyPool', size=2) plan.add_resource(pool) for idx in range(1, 3): task = Task(target=get_mtest(part_tuple=(idx, 3))) plan.schedule(task, resource='MyPool') with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is False assert len(plan.report.entries) == 1 assert len(plan.report.entries[0].entries) == 2 assert plan.report.status == Status.ERROR # Testplan result assert plan.report.entries[0].status == Status.ERROR # Multitest assert plan.report.entries[0].entries[0].status == Status.PASSED # Suite1 assert plan.report.entries[0].entries[1].status == Status.FAILED # Suite2 assert 'not all MultiTest parts had been scheduled' in \ plan.report.entries[0].logs[0]['message']
def test_multi_parts_merged(): """Execute MultiTest parts and merge report.""" plan = Testplan(name='plan', parse_cmdline=False, merge_scheduled_parts=True) pool = ThreadPool(name='MyPool', size=2) plan.add_resource(pool) for idx in range(3): task = Task(target=get_mtest(part_tuple=(idx, 3))) plan.schedule(task, resource='MyPool') with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is True assert len(plan.report.entries) == 1 assert plan.report.entries[0].name == 'MTest' assert len(plan.report.entries[0].entries) == 2 # 2 suites assert plan.report.entries[0].entries[0].name == 'Suite1' assert plan.report.entries[0].entries[1].name == 'Suite2' assert len(plan.report.entries[0].entries[0].entries) == 1 # param group assert plan.report.entries[0].entries[0].entries[0].name == 'test_true' assert len(plan.report.entries[0].entries[1].entries) == 1 # param group assert plan.report.entries[0].entries[1].entries[0].name == 'test_false' assert len(plan.report.entries[0].entries[0].entries[0].entries) == 10 assert len(plan.report.entries[0].entries[1].entries[0].entries) == 3
def test_multi_parts_not_merged(): """Execute MultiTest parts but do not merge reports.""" plan = TestplanMock(name="plan", merge_scheduled_parts=False) pool = ThreadPool(name="MyThreadPool", size=2) plan.add_resource(pool) for idx in range(3): task = Task(target=get_mtest(part_tuple=(idx, 3))) plan.schedule(task, resource="MyThreadPool") with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is True assert len(plan.report.entries) == 3 assert plan.report.entries[0].name == "MTest - part(0/3)" assert plan.report.entries[1].name == "MTest - part(1/3)" assert plan.report.entries[2].name == "MTest - part(2/3)" assert len(plan.report.entries[0].entries) == 2 # 2 suites assert plan.report.entries[0].entries[0].name == "Suite1" assert plan.report.entries[0].entries[1].name == "Suite2" assert len(plan.report.entries[0].entries[0].entries) == 1 # param group assert plan.report.entries[0].entries[0].entries[0].name == "test_true" assert len(plan.report.entries[0].entries[1].entries) == 1 # param group assert plan.report.entries[0].entries[1].entries[0].name == "test_false" assert len(plan.report.entries[0].entries[0].entries[0].entries) == 4 assert len(plan.report.entries[0].entries[1].entries[0].entries) == 1
def test_multi_parts_merged(): """ Schedule MultiTest parts in different ways, execute them and merge reports. """ plan = TestplanMock(name="plan", merge_scheduled_parts=True) pool = ThreadPool(name="MyThreadPool", size=2) plan.add_resource(pool) def _get_mtest(): return MultiTest(name="MTest", suites=[Suite1(), Suite2()], part=(2, 3)) plan.add(target=_get_mtest()) # local_runner plan.add(Task(target=get_mtest(part_tuple=(1, 3)))) # local_runner plan.schedule(Task(target=get_mtest(part_tuple=(0, 3))), resource="MyThreadPool") with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is True assert len(plan.report.entries) == 1 assert plan.report.entries[0].name == "MTest" assert len(plan.report.entries[0].entries) == 2 # 2 suites assert plan.report.entries[0].entries[0].name == "Suite1" assert plan.report.entries[0].entries[1].name == "Suite2" assert len(plan.report.entries[0].entries[0].entries) == 1 # param group assert plan.report.entries[0].entries[0].entries[0].name == "test_true" assert len(plan.report.entries[0].entries[1].entries) == 1 # param group assert plan.report.entries[0].entries[1].entries[0].name == "test_false" assert len(plan.report.entries[0].entries[0].entries[0].entries) == 10 assert len(plan.report.entries[0].entries[1].entries[0].entries) == 3
def test_multi_parts_duplicate_part(): """ Execute MultiTest parts with a part of MultiTest has been scheduled twice and automatically be filtered out. """ plan = TestplanMock(name="plan", merge_scheduled_parts=True) pool = ThreadPool(name="MyThreadPool", size=2) plan.add_resource(pool) for idx in range(3): task = Task(target=get_mtest_with_custom_uid(part_tuple=(idx, 3))) plan.schedule(task, resource="MyThreadPool") task = Task(target=get_mtest_with_custom_uid(part_tuple=(1, 3))) plan.schedule(task, resource="MyThreadPool") assert len(plan._tests) == 4 with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is False assert len(plan.report.entries) == 5 # one placeholder report & 4 siblings assert len(plan.report.entries[0].entries) == 0 # already cleared assert plan.report.status == Status.ERROR # Testplan result assert ("duplicate MultiTest parts had been scheduled" in plan.report.entries[0].logs[0]["message"])
def test_multi_parts_incorrect_schedule(): """ Execute MultiTest parts with invalid parameters that a MultiTest has been scheduled twice and each time split into different parts. """ plan = TestplanMock(name="plan", merge_scheduled_parts=True) pool = ThreadPool(name="MyThreadPool", size=2) plan.add_resource(pool) for idx in range(3): task = Task(target=get_mtest(part_tuple=(idx, 3))) plan.schedule(task, resource="MyThreadPool") for idx in range(2): task = Task(target=get_mtest(part_tuple=(idx, 2))) plan.schedule(task, resource="MyThreadPool") assert len(plan._tests) == 5 with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is False assert len(plan.report.entries) == 6 # one placeholder report & 5 siblings assert len(plan.report.entries[0].entries) == 0 # already cleared assert plan.report.status == Status.ERROR # Testplan result assert ("invalid parameter of part provided" in plan.report.entries[0].logs[0]["message"])
def test_multi_parts_invalid_parameter_1(): """ Execute MultiTest parts with invalid parameters that a part of MultiTest has been scheduled twice. """ plan = Testplan(name="plan", parse_cmdline=False, merge_scheduled_parts=True) pool = ThreadPool(name="MyPool", size=2) plan.add_resource(pool) for idx in range(3): task = Task(target=get_mtest(part_tuple=(idx, 3))) plan.schedule(task, resource="MyPool") plan.schedule(Task(target=get_mtest(part_tuple=(1, 3))), resource="MyPool") with log_propagation_disabled(TESTPLAN_LOGGER): result = plan.run() assert result.run is False # assert plan.run().run is False assert len(plan.report.entries) == 1 assert len(plan.report.entries[0].entries) == 2 assert plan.report.status == Status.ERROR # Testplan result assert plan.report.entries[0].status == Status.ERROR # Multitest assert plan.report.entries[0].entries[0].status == Status.FAILED # Suite1 assert plan.report.entries[0].entries[1].status == Status.FAILED # Suite2 assert ("invalid parameter of part provided" in plan.report.entries[0].logs[0]["message"])
def test_runner_timeout(): """ Execute MultiTests in LocalRunner, ThreadPool and ProcessPool respectively. Some of them will timeout and we'll get a report showing execution details. """ plan = Testplan(name='plan', parse_cmdline=False, runnable=MyTestRunner, timeout=60, abort_wait_timeout=5) mod_path = os.path.dirname(os.path.abspath(__file__)) THREAD_POOL = 'MyThreadPool' PROCESS_POOL = 'MyProcessPool' thread_pool = ThreadPool(name=THREAD_POOL, size=2, worker_heartbeat=None) proc_pool = ProcessPool(name=PROCESS_POOL, size=2, worker_heartbeat=None) plan.add_resource(thread_pool) plan.add_resource(proc_pool) plan.add(func_basic_tasks.get_mtest1()) plan.add(func_basic_tasks.get_mtest2()) task3 = Task(target='get_mtest3', module='func_basic_tasks', path=mod_path) task4 = Task(target='get_mtest4', module='func_basic_tasks', path=mod_path) task5 = Task(target='get_mtest5', module='func_basic_tasks', path=mod_path) task6 = Task(target='get_mtest6', module='func_basic_tasks', path=mod_path) plan.schedule(task3, resource=THREAD_POOL) plan.schedule(task4, resource=THREAD_POOL) plan.schedule(task5, resource=PROCESS_POOL) plan.schedule(task6, resource=PROCESS_POOL) with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is False assert len(plan.report.entries) == 6 assert plan.report.status == Status.ERROR entries = plan.report.entries assert entries[0].name == 'MTest1' assert entries[0].status == Status.FAILED # testcase 'test_true' failed assert entries[2].name == 'MTest3' assert entries[2].status == Status.PASSED # testcase 'test_equal' passed assert entries[4].name == 'MTest5' assert entries[4].status == Status.ERROR # testcase 'test_contain' raised assert entries[1].name == 'MTest2' assert entries[1].status == Status.ERROR # timeout assert ' discarding due to ' in entries[1].logs[0]['message'] assert entries[3].name == 'Task[get_mtest4]' assert entries[3].status == Status.ERROR # timeout assert entries[3].logs[0]['message'].startswith('_target: get_mtest4') assert ' discarding due to ' in entries[3].logs[1]['message'] assert entries[5].name == 'Task[get_mtest6]' assert entries[5].status == Status.ERROR # timeout assert entries[5].logs[0]['message'].startswith('_target: get_mtest6') assert ' discarding due to ' in entries[5].logs[1]['message']
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ # Add a thread pool test execution resource to the plan of given size. pool = ThreadPool(name="MyPool", size=plan.args.pool_size) plan.add_resource(pool) # Add a given number of similar tests to the thread pool # to be executed in parallel. for idx in range(plan.args.tasks_num): task = Task(target="make_multitest", module="tasks", kwargs={"index": idx}) plan.schedule(task, resource="MyPool")
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ # Add a thread pool test execution resource to the plan of given size. # Also you can use process pool or remote pool instead. pool = ThreadPool(name='MyPool', size=plan.args.pool_size) plan.add_resource(pool) # Add a given number of similar tests to the thread pool # to be executed in parallel. for idx in range(plan.args.parts_num): task = Task(target='make_multitest', module='tasks', kwargs={'part_tuple': (idx, plan.args.parts_num)}) plan.schedule(task, resource='MyPool')
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ # Add a thread pool test execution resource to the plan of given size. # Can also use a process pool instead. pool = ThreadPool(name="MyPool", size=plan.args.pool_size) plan.add_resource(pool) # Add a task with `rerun` argument to the thread pool tmp_file = os.path.join( tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4()) ) task = Task( target="make_multitest", module="tasks", args=(tmp_file,), rerun=2 ) plan.schedule(task, resource="MyPool")
def test_multi_parts_missing_parts(): """ Execute MultiTest parts with invalid parameters that not all parts of a MultiTest are scheduled. """ plan = TestplanMock(name="plan", merge_scheduled_parts=True) pool = ThreadPool(name="MyThreadPool", size=2) plan.add_resource(pool) for idx in range(1, 3): task = Task(target=get_mtest(part_tuple=(idx, 3))) plan.schedule(task, resource="MyThreadPool") with log_propagation_disabled(TESTPLAN_LOGGER): assert plan.run().run is False assert len(plan.report.entries) == 3 # one placeholder report & 2 siblings assert len(plan.report.entries[0].entries) == 0 # already cleared assert plan.report.status == Status.ERROR # Testplan result assert ("not all MultiTest parts had been scheduled" in plan.report.entries[0].logs[0]["message"])
def test_timeout_on_testcases(mockplan): pool = ThreadPool(name="MyPool", size=2) mockplan.add_resource(pool) task = Task(target=get_mtest()) mockplan.schedule(task, resource="MyPool") with log_propagation_disabled(TESTPLAN_LOGGER): mockplan.run() expected_report = TestReport( name="plan", entries=[ TestGroupReport( name="MTest", category=ReportCategories.MULTITEST, entries=[ TestGroupReport( name="Suite1", description="A test suite with basic testcases.", category=ReportCategories.TESTSUITE, entries=[ TestCaseReport( name="test_normal", entries=[ { "type": "Log", "message": "Testcase will finish execution in time", } ], ), _create_testcase_report( name="test_abnormal", status_override=Status.ERROR, entries=[ { "type": "Log", "message": "Testcase will definitely timeout", } ], ), ], ), TestGroupReport( name="Suite2", description="A test suite with parameterized testcases in different exec groups.", category=ReportCategories.TESTSUITE, entries=[ TestGroupReport( name="test_timeout_1", category=ReportCategories.PARAMETRIZATION, entries=[ TestCaseReport( name="test_timeout_1__val_1", entries=[ { "type": "Log", "message": "Testcase will sleep for 1 seconds", } ], ), TestCaseReport( name="test_timeout_1__val_2", entries=[ { "type": "Log", "message": "Testcase will sleep for 2 seconds", } ], ), TestCaseReport( name="test_timeout_1__val_3", entries=[ { "type": "Log", "message": "Testcase will sleep for 3 seconds", } ], ), ], ), TestGroupReport( name="test_timeout_2", category=ReportCategories.PARAMETRIZATION, entries=[ TestCaseReport( name="test_timeout_2__val_1", entries=[ { "type": "Log", "message": "Testcase will sleep for 1 seconds", } ], ), TestCaseReport( name="test_timeout_2__val_2", entries=[ { "type": "Log", "message": "Testcase will sleep for 2 seconds", } ], ), _create_testcase_report( name="test_timeout_2__val_5", status_override=Status.ERROR, entries=[ { "type": "Log", "message": "Testcase will sleep for 5 seconds", } ], ), ], ), ], ), ], ) ], ) check_report(expected_report, mockplan.report)
def test_timeout_on_testcases(): plan = Testplan(name='plan', parse_cmdline=False) pool = ThreadPool(name='MyPool', size=2) plan.add_resource(pool) task = Task(target=get_mtest()) plan.schedule(task, resource='MyPool') with log_propagation_disabled(TESTPLAN_LOGGER): plan.run() expected_report = TestReport( name='plan', entries=[ TestGroupReport( name='MTest', category=Categories.MULTITEST, entries=[ TestGroupReport( name='Suite1', description='A test suite with basic testcases.', category=Categories.SUITE, entries=[ TestCaseReport( name='test_normal', entries=[{ 'type': 'Log', 'message': 'Testcase will finish execution in time' }]), _create_testcase_report( name='test_abnormal', status_override=Status.ERROR, entries=[{ 'type': 'Log', 'message': 'Testcase will definitely timeout' }]) ]), TestGroupReport( name='Suite2', description= 'A test suite with parameterized testcases in different exec groups.', category=Categories.SUITE, entries=[ TestGroupReport( name='test_timeout_1', category=Categories.PARAMETRIZATION, entries=[ TestCaseReport( name='test_timeout_1__val_1', entries=[{ 'type': 'Log', 'message': 'Testcase will sleep for 1 seconds' }]), TestCaseReport( name='test_timeout_1__val_2', entries=[{ 'type': 'Log', 'message': 'Testcase will sleep for 2 seconds' }]), TestCaseReport( name='test_timeout_1__val_3', entries=[{ 'type': 'Log', 'message': 'Testcase will sleep for 3 seconds' }]) ]), TestGroupReport( name='test_timeout_2', category=Categories.PARAMETRIZATION, entries=[ TestCaseReport( name='test_timeout_2__val_1', entries=[{ 'type': 'Log', 'message': 'Testcase will sleep for 1 seconds' }]), TestCaseReport( name='test_timeout_2__val_2', entries=[{ 'type': 'Log', 'message': 'Testcase will sleep for 2 seconds' }]), _create_testcase_report( name='test_timeout_2__val_5', status_override=Status.ERROR, entries=[{ 'type': 'Log', 'message': 'Testcase will sleep for 5 seconds' }]) ]) ]) ]) ]) check_report(expected_report, plan.report)