def test_multi_parts_not_merged(): """Execute MultiTest parts but do not merge reports.""" plan = TestplanMock(name="plan", merge_scheduled_parts=False) pool = ThreadPool(name="MyThreadPool", size=2) plan.add_resource(pool) for idx in range(3): task = Task(target=get_mtest(part_tuple=(idx, 3))) plan.schedule(task, resource="MyThreadPool") assert plan.run().run is True assert len(plan.report.entries) == 3 assert plan.report.entries[0].name == "MTest - part(0/3)" assert plan.report.entries[1].name == "MTest - part(1/3)" assert plan.report.entries[2].name == "MTest - part(2/3)" assert len(plan.report.entries[0].entries) == 2 # 2 suites assert plan.report.entries[0].entries[0].name == "Suite1" assert plan.report.entries[0].entries[1].name == "Suite2" assert len(plan.report.entries[0].entries[0].entries) == 1 # param group assert plan.report.entries[0].entries[0].entries[0].name == "test_true" assert len(plan.report.entries[0].entries[1].entries) == 1 # param group assert plan.report.entries[0].entries[1].entries[0].name == "test_false" assert len(plan.report.entries[0].entries[0].entries[0].entries) == 4 assert len(plan.report.entries[0].entries[1].entries[0].entries) == 1
def test_multi_parts_merged(): """ Schedule MultiTest parts in different ways, execute them and merge reports. """ plan = TestplanMock(name="plan", merge_scheduled_parts=True) pool = ThreadPool(name="MyThreadPool", size=2) plan.add_resource(pool) def _get_mtest(): return MultiTest(name="MTest", suites=[Suite1(), Suite2()], part=(2, 3)) plan.add(target=_get_mtest()) # local_runner plan.add(Task(target=get_mtest(part_tuple=(1, 3)))) # local_runner plan.schedule(Task(target=get_mtest(part_tuple=(0, 3))), resource="MyThreadPool") assert plan.run().run is True assert len(plan.report.entries) == 1 assert plan.report.entries[0].name == "MTest" assert len(plan.report.entries[0].entries) == 2 # 2 suites assert plan.report.entries[0].entries[0].name == "Suite1" assert plan.report.entries[0].entries[1].name == "Suite2" assert len(plan.report.entries[0].entries[0].entries) == 1 # param group assert plan.report.entries[0].entries[0].entries[0].name == "test_true" assert len(plan.report.entries[0].entries[1].entries) == 1 # param group assert plan.report.entries[0].entries[1].entries[0].name == "test_false" assert len(plan.report.entries[0].entries[0].entries[0].entries) == 10 assert len(plan.report.entries[0].entries[1].entries[0].entries) == 3
def test_multi_parts_duplicate_part(): """ Execute MultiTest parts with a part of MultiTest has been scheduled twice and automatically be filtered out. """ plan = TestplanMock(name="plan", merge_scheduled_parts=True) pool = ThreadPool(name="MyThreadPool", size=2) plan.add_resource(pool) for idx in range(3): task = Task(target=get_mtest_with_custom_uid(part_tuple=(idx, 3))) plan.schedule(task, resource="MyThreadPool") task = Task(target=get_mtest_with_custom_uid(part_tuple=(1, 3))) plan.schedule(task, resource="MyThreadPool") assert len(plan._tests) == 4 assert plan.run().run is False assert len(plan.report.entries) == 5 # one placeholder report & 4 siblings assert len(plan.report.entries[0].entries) == 0 # already cleared assert plan.report.status == Status.ERROR # Testplan result assert ("duplicate MultiTest parts had been scheduled" in plan.report.entries[0].logs[0]["message"])
def test_task_rerun_with_parts(): with tempfile.TemporaryDirectory() as runpath: mockplan = TestplanMock("plan", runpath=runpath, merge_scheduled_parts=True) pool_name = ThreadPool.__name__ pool = ThreadPool(name=pool_name, size=1) mockplan.add_resource(pool) uids = [] for idx in range(3): task = Task( target=make_multitest_parts, kwargs={"part_tuple": (idx, 3)}, rerun=1, ) uids.append(mockplan.schedule(task=task, resource=pool_name)) assert mockplan.run().run is True assert mockplan.report.passed is False assert mockplan.report.counter == { "passed": 2, "total": 3, "failed": 1, } # Run2 of part0 are merged with part1 and part2 assert mockplan.report.entries[0].name == "MultiTestParts" # Run1 of part0 (task_rerun) are left unmerged assert (mockplan.report.entries[1].name == "MultiTestParts - part(0/3) => Run 1")
def test_multi_parts_incorrect_schedule(): """ Execute MultiTest parts with invalid parameters that a MultiTest has been scheduled twice and each time split into different parts. """ plan = TestplanMock(name="plan", merge_scheduled_parts=True) pool = ThreadPool(name="MyThreadPool", size=2) plan.add_resource(pool) for idx in range(3): task = Task(target=get_mtest(part_tuple=(idx, 3))) plan.schedule(task, resource="MyThreadPool") for idx in range(2): task = Task(target=get_mtest(part_tuple=(idx, 2))) plan.schedule(task, resource="MyThreadPool") assert len(plan._tests) == 5 assert plan.run().run is False assert len(plan.report.entries) == 6 # one placeholder report & 5 siblings assert len(plan.report.entries[0].entries) == 0 # already cleared assert plan.report.status == Status.ERROR # Testplan result assert ("invalid parameter of part provided" in plan.report.entries[0].logs[0]["message"])
def test_task_rerun_in_thread_pool(mockplan): """ Test procedure: - 1st run: `unstable_case` fails. - 1st rerun: `mock_driver` raises during stop. - 2nd rerun: all pass. """ pool_name = ThreadPool.__name__ pool = ThreadPool(name=pool_name, size=2) mockplan.add_resource(pool) tmp_file = os.path.join(tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4())) task = Task(target=make_multitest_1, args=(tmp_file, ), rerun=3) uid = mockplan.schedule(task=task, resource=pool_name) assert mockplan.run().run is True assert mockplan.report.passed is True assert mockplan.report.counter == {"passed": 3, "total": 3, "failed": 0} assert isinstance(mockplan.report.serialize(), dict) assert mockplan.result.test_results[uid].report.name == "Unstable MTest1" assert len(mockplan.report.entries) == 3 assert mockplan.report.entries[-1].category == ReportCategories.TASK_RERUN assert mockplan.report.entries[-2].category == ReportCategories.TASK_RERUN assert task.reassign_cnt == 2 _remove_existing_tmp_file(tmp_file)
def test_task_rerun_with_more_times_2(mockplan): """ Test procedure 2: (set `task_rerun_limit` to 4, a task can be scheduled 5 times in total) - 1st run: `unstable_case` fails. - 1st rerun: `unstable_case` fails. - 2nd rerun: `unstable_case` fails. - 3rd rerun: all pass. """ pool_name = ThreadPool.__name__ pool = ThreadPool(name=pool_name, size=1) mockplan.add_resource(pool) tmp_file = os.path.join(tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4())) task = Task(target=make_multitest_3, args=(tmp_file, ), rerun=3) uid = mockplan.schedule(task=task, resource=pool_name) assert mockplan.run().run is True assert mockplan.report.passed is True assert mockplan.report.counter == {"passed": 2, "total": 2, "failed": 0} assert mockplan.result.test_results[uid].report.name == "Unstable MTest3" assert task.reassign_cnt == 3 _remove_existing_tmp_file(tmp_file)
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ # Add a thread pool test execution resource to the plan of given size. pool = ThreadPool(name="MyPool", size=plan.args.pool_size) plan.add_resource(pool) # Add a given number of similar tests to the thread pool # to be executed in parallel. for idx in range(plan.args.tasks_num): task = Task(target="make_multitest", module="tasks", kwargs={"index": idx}) plan.schedule(task, resource="MyPool")
def test_multi_parts_missing_parts(): """ Execute MultiTest parts with invalid parameters that not all parts of a MultiTest are scheduled. """ plan = TestplanMock(name="plan", merge_scheduled_parts=True) pool = ThreadPool(name="MyThreadPool", size=2) plan.add_resource(pool) for idx in range(1, 3): task = Task(target=get_mtest(part_tuple=(idx, 3))) plan.schedule(task, resource="MyThreadPool") assert plan.run().run is False assert len(plan.report.entries) == 3 # one placeholder report & 2 siblings assert len(plan.report.entries[0].entries) == 0 # already cleared assert plan.report.status == Status.ERROR # Testplan result assert ("not all MultiTest parts had been scheduled" in plan.report.entries[0].logs[0]["message"])
def main(plan): """ Testplan decorated main function to add and execute MultiTests. :return: Testplan result object. :rtype: ``testplan.base.TestplanResult`` """ # Add a thread pool test execution resource to the plan of given size. # Can also use a process pool instead. pool = ThreadPool(name="MyPool", size=plan.args.pool_size) plan.add_resource(pool) # Add a task with `rerun` argument to the thread pool tmp_file = os.path.join(tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4())) task = Task(target="make_multitest", module="tasks", args=(tmp_file, ), rerun=2) plan.schedule(task, resource="MyPool")
def test_timeout_on_case_related_methods(mockplan): pool = ThreadPool(name="MyPool", size=1) mockplan.add_resource(pool) task = Task(target=get_mtest3()) mockplan.schedule(task, resource="MyPool") mockplan.run() expected_report = TestReport( name="plan", entries=[ TestGroupReport( name="MTest3", category=ReportCategories.MULTITEST, entries=[ TestGroupReport( name="Suite5", description= "A test suite with pre/post testcase methods which may not run.", category=ReportCategories.TESTSUITE, entries=[ TestGroupReport( name="test_method", category=ReportCategories.PARAMETRIZATION, entries=[ TestCaseReport( name="test_method <val=1>", entries=[ { "type": "Log", "message": "Pre testcase method will always be OK", }, { "type": "Log", "message": "Get value 1", }, { "type": "Log", "message": "Testcase will finish in a short time", }, { "type": "Log", "message": "Post testcase method can still run", }, ], ), TestCaseReport( name="test_method <val=5>", entries=[ { "type": "Log", "message": "Pre testcase method will always be OK", }, { "type": "Log", "message": "Get value 5", }, { "type": "Log", "message": "Testcase will finish in a short time", }, ], status_override=Status.ERROR, ), ], ), ], ), ], ) ], ) check_report(expected_report, mockplan.report) assert ("`post_testcase` timeout" in mockplan.report["MTest3"]["Suite5"] ["test_method"]["test_method__val_5"].logs[0]["message"])
def test_timeout_on_suite_related_methods(mockplan): pool = ThreadPool(name="MyPool", size=1) mockplan.add_resource(pool) task = Task(target=get_mtest2()) mockplan.schedule(task, resource="MyPool") mockplan.run() expected_report = TestReport( name="plan", entries=[ TestGroupReport( name="MTest2", category=ReportCategories.MULTITEST, entries=[ TestGroupReport( name="Suite3", description= "A test suite with teardown method and it will timeout.", category=ReportCategories.TESTSUITE, entries=[ TestCaseReport( name="setup", entries=[{ "type": "Log", "message": "Setup method will sleep for 1 second", }], ), TestCaseReport( name="test_normal", entries=[{ "type": "Log", "message": "Testcase will finish execution in time", }], ), _create_testcase_report( name="teardown", status_override=Status.ERROR, entries=[], ), ], ), TestGroupReport( name="Suite4", description= "A test suite with setup method and it will timeout.", category=ReportCategories.TESTSUITE, entries=[ _create_testcase_report( name="setup", status_override=Status.ERROR, entries=[{ "type": "Log", "message": "Setup method will sleep for 5 seconds", }], ), TestCaseReport( name="teardown", entries=[{ "type": "Log", "message": "Teardown method can still run", }], ), ], ), ], ) ], ) check_report(expected_report, mockplan.report)
def test_timeout_on_testcases(mockplan): pool = ThreadPool(name="MyPool", size=2) mockplan.add_resource(pool) task = Task(target=get_mtest1()) mockplan.schedule(task, resource="MyPool") mockplan.run() expected_report = TestReport( name="plan", entries=[ TestGroupReport( name="MTest1", category=ReportCategories.MULTITEST, entries=[ TestGroupReport( name="Suite1", description="A test suite with basic testcases.", category=ReportCategories.TESTSUITE, entries=[ TestCaseReport( name="test_normal", entries=[{ "type": "Log", "message": "Testcase will finish execution in time", }], ), _create_testcase_report( name="test_abnormal", status_override=Status.ERROR, entries=[{ "type": "Log", "message": "Testcase will definitely timeout", }], ), ], ), TestGroupReport( name="Suite2", description= "A test suite with parameterized testcases in different exec groups.", category=ReportCategories.TESTSUITE, entries=[ TestGroupReport( name="test_not_timeout", category=ReportCategories.PARAMETRIZATION, entries=[ TestCaseReport( name="test_not_timeout <val=1>", entries=[{ "type": "Log", "message": "Testcase will sleep for 1 seconds", }], ), TestCaseReport( name="test_not_timeout <val=2>", entries=[{ "type": "Log", "message": "Testcase will sleep for 2 seconds", }], ), TestCaseReport( name="test_not_timeout <val=0.5>", entries=[{ "type": "Log", "message": "Testcase will sleep for 0.5 seconds", }], ), ], ), TestGroupReport( name="test_timeout", category=ReportCategories.PARAMETRIZATION, entries=[ TestCaseReport( name="test_timeout <val=1>", entries=[{ "type": "Log", "message": "Testcase will sleep for 1 seconds", }], ), TestCaseReport( name="test_timeout <val=0.5>", entries=[{ "type": "Log", "message": "Testcase will sleep for 0.5 seconds", }], ), _create_testcase_report( name="test_timeout <val=5>", status_override=Status.ERROR, entries=[{ "type": "Log", "message": "Testcase will sleep for 5 seconds", }], ), ], ), ], ), ], ) ], ) check_report(expected_report, mockplan.report)