def test_multi_parts_not_merged():
"""Execute MultiTest parts but do not merge reports."""
plan = TestplanMock(name="plan", merge_scheduled_parts=False)
pool = ThreadPool(name="MyThreadPool", size=2)
plan.add_resource(pool)
for idx in range(3):
task = Task(target=get_mtest(part_tuple=(idx, 3)))
plan.schedule(task, resource="MyThreadPool")
assert plan.run().run is True
assert len(plan.report.entries) == 3
assert plan.report.entries[0].name == "MTest - part(0/3)"
assert plan.report.entries[1].name == "MTest - part(1/3)"
assert plan.report.entries[2].name == "MTest - part(2/3)"
assert len(plan.report.entries[0].entries) == 2 # 2 suites
assert plan.report.entries[0].entries[0].name == "Suite1"
assert plan.report.entries[0].entries[1].name == "Suite2"
assert len(plan.report.entries[0].entries[0].entries) == 1 # param group
assert plan.report.entries[0].entries[0].entries[0].name == "test_true"
assert len(plan.report.entries[0].entries[1].entries) == 1 # param group
assert plan.report.entries[0].entries[1].entries[0].name == "test_false"
assert len(plan.report.entries[0].entries[0].entries[0].entries) == 4
assert len(plan.report.entries[0].entries[1].entries[0].entries) == 1
def test_multi_parts_merged():
"""
Schedule MultiTest parts in different ways, execute them and merge reports.
"""
plan = TestplanMock(name="plan", merge_scheduled_parts=True)
pool = ThreadPool(name="MyThreadPool", size=2)
plan.add_resource(pool)
def _get_mtest():
return MultiTest(name="MTest",
suites=[Suite1(), Suite2()],
part=(2, 3))
plan.add(target=_get_mtest()) # local_runner
plan.add(Task(target=get_mtest(part_tuple=(1, 3)))) # local_runner
plan.schedule(Task(target=get_mtest(part_tuple=(0, 3))),
resource="MyThreadPool")
assert plan.run().run is True
assert len(plan.report.entries) == 1
assert plan.report.entries[0].name == "MTest"
assert len(plan.report.entries[0].entries) == 2 # 2 suites
assert plan.report.entries[0].entries[0].name == "Suite1"
assert plan.report.entries[0].entries[1].name == "Suite2"
assert len(plan.report.entries[0].entries[0].entries) == 1 # param group
assert plan.report.entries[0].entries[0].entries[0].name == "test_true"
assert len(plan.report.entries[0].entries[1].entries) == 1 # param group
assert plan.report.entries[0].entries[1].entries[0].name == "test_false"
assert len(plan.report.entries[0].entries[0].entries[0].entries) == 10
assert len(plan.report.entries[0].entries[1].entries[0].entries) == 3
def test_multi_parts_duplicate_part():
"""
Execute MultiTest parts with a part of MultiTest has been
scheduled twice and automatically be filtered out.
"""
plan = TestplanMock(name="plan", merge_scheduled_parts=True)
pool = ThreadPool(name="MyThreadPool", size=2)
plan.add_resource(pool)
for idx in range(3):
task = Task(target=get_mtest_with_custom_uid(part_tuple=(idx, 3)))
plan.schedule(task, resource="MyThreadPool")
task = Task(target=get_mtest_with_custom_uid(part_tuple=(1, 3)))
plan.schedule(task, resource="MyThreadPool")
assert len(plan._tests) == 4
assert plan.run().run is False
assert len(plan.report.entries) == 5 # one placeholder report & 4 siblings
assert len(plan.report.entries[0].entries) == 0 # already cleared
assert plan.report.status == Status.ERROR # Testplan result
assert ("duplicate MultiTest parts had been scheduled"
in plan.report.entries[0].logs[0]["message"])
def test_task_rerun_with_parts():
with tempfile.TemporaryDirectory() as runpath:
mockplan = TestplanMock("plan",
runpath=runpath,
merge_scheduled_parts=True)
pool_name = ThreadPool.__name__
pool = ThreadPool(name=pool_name, size=1)
mockplan.add_resource(pool)
uids = []
for idx in range(3):
task = Task(
target=make_multitest_parts,
kwargs={"part_tuple": (idx, 3)},
rerun=1,
)
uids.append(mockplan.schedule(task=task, resource=pool_name))
assert mockplan.run().run is True
assert mockplan.report.passed is False
assert mockplan.report.counter == {
"passed": 2,
"total": 3,
"failed": 1,
}
# Run2 of part0 are merged with part1 and part2
assert mockplan.report.entries[0].name == "MultiTestParts"
# Run1 of part0 (task_rerun) are left unmerged
assert (mockplan.report.entries[1].name ==
"MultiTestParts - part(0/3) => Run 1")
def test_multi_parts_incorrect_schedule():
"""
Execute MultiTest parts with invalid parameters that a MultiTest
has been scheduled twice and each time split into different parts.
"""
plan = TestplanMock(name="plan", merge_scheduled_parts=True)
pool = ThreadPool(name="MyThreadPool", size=2)
plan.add_resource(pool)
for idx in range(3):
task = Task(target=get_mtest(part_tuple=(idx, 3)))
plan.schedule(task, resource="MyThreadPool")
for idx in range(2):
task = Task(target=get_mtest(part_tuple=(idx, 2)))
plan.schedule(task, resource="MyThreadPool")
assert len(plan._tests) == 5
assert plan.run().run is False
assert len(plan.report.entries) == 6 # one placeholder report & 5 siblings
assert len(plan.report.entries[0].entries) == 0 # already cleared
assert plan.report.status == Status.ERROR # Testplan result
assert ("invalid parameter of part provided"
in plan.report.entries[0].logs[0]["message"])
def test_task_rerun_in_thread_pool(mockplan):
"""
Test procedure:
- 1st run: `unstable_case` fails.
- 1st rerun: `mock_driver` raises during stop.
- 2nd rerun: all pass.
"""
pool_name = ThreadPool.__name__
pool = ThreadPool(name=pool_name, size=2)
mockplan.add_resource(pool)
tmp_file = os.path.join(tempfile.gettempdir(), getpass.getuser(),
"{}.tmp".format(uuid.uuid4()))
task = Task(target=make_multitest_1, args=(tmp_file, ), rerun=3)
uid = mockplan.schedule(task=task, resource=pool_name)
assert mockplan.run().run is True
assert mockplan.report.passed is True
assert mockplan.report.counter == {"passed": 3, "total": 3, "failed": 0}
assert isinstance(mockplan.report.serialize(), dict)
assert mockplan.result.test_results[uid].report.name == "Unstable MTest1"
assert len(mockplan.report.entries) == 3
assert mockplan.report.entries[-1].category == ReportCategories.TASK_RERUN
assert mockplan.report.entries[-2].category == ReportCategories.TASK_RERUN
assert task.reassign_cnt == 2
_remove_existing_tmp_file(tmp_file)
def test_task_rerun_with_more_times_2(mockplan):
"""
Test procedure 2:
(set `task_rerun_limit` to 4, a task can be scheduled 5 times in total)
- 1st run: `unstable_case` fails.
- 1st rerun: `unstable_case` fails.
- 2nd rerun: `unstable_case` fails.
- 3rd rerun: all pass.
"""
pool_name = ThreadPool.__name__
pool = ThreadPool(name=pool_name, size=1)
mockplan.add_resource(pool)
tmp_file = os.path.join(tempfile.gettempdir(), getpass.getuser(),
"{}.tmp".format(uuid.uuid4()))
task = Task(target=make_multitest_3, args=(tmp_file, ), rerun=3)
uid = mockplan.schedule(task=task, resource=pool_name)
assert mockplan.run().run is True
assert mockplan.report.passed is True
assert mockplan.report.counter == {"passed": 2, "total": 2, "failed": 0}
assert mockplan.result.test_results[uid].report.name == "Unstable MTest3"
assert task.reassign_cnt == 3
_remove_existing_tmp_file(tmp_file)
def main(plan):
"""
Testplan decorated main function to add and execute MultiTests.
:return: Testplan result object.
:rtype: ``testplan.base.TestplanResult``
"""
# Add a thread pool test execution resource to the plan of given size.
pool = ThreadPool(name="MyPool", size=plan.args.pool_size)
plan.add_resource(pool)
# Add a given number of similar tests to the thread pool
# to be executed in parallel.
for idx in range(plan.args.tasks_num):
task = Task(target="make_multitest",
module="tasks",
kwargs={"index": idx})
plan.schedule(task, resource="MyPool")
def test_multi_parts_missing_parts():
"""
Execute MultiTest parts with invalid parameters that
not all parts of a MultiTest are scheduled.
"""
plan = TestplanMock(name="plan", merge_scheduled_parts=True)
pool = ThreadPool(name="MyThreadPool", size=2)
plan.add_resource(pool)
for idx in range(1, 3):
task = Task(target=get_mtest(part_tuple=(idx, 3)))
plan.schedule(task, resource="MyThreadPool")
assert plan.run().run is False
assert len(plan.report.entries) == 3 # one placeholder report & 2 siblings
assert len(plan.report.entries[0].entries) == 0 # already cleared
assert plan.report.status == Status.ERROR # Testplan result
assert ("not all MultiTest parts had been scheduled"
in plan.report.entries[0].logs[0]["message"])
def main(plan):
"""
Testplan decorated main function to add and execute MultiTests.
:return: Testplan result object.
:rtype: ``testplan.base.TestplanResult``
"""
# Add a thread pool test execution resource to the plan of given size.
# Can also use a process pool instead.
pool = ThreadPool(name="MyPool", size=plan.args.pool_size)
plan.add_resource(pool)
# Add a task with `rerun` argument to the thread pool
tmp_file = os.path.join(tempfile.gettempdir(), getpass.getuser(),
"{}.tmp".format(uuid.uuid4()))
task = Task(target="make_multitest",
module="tasks",
args=(tmp_file, ),
rerun=2)
plan.schedule(task, resource="MyPool")
def test_timeout_on_case_related_methods(mockplan):
pool = ThreadPool(name="MyPool", size=1)
mockplan.add_resource(pool)
task = Task(target=get_mtest3())
mockplan.schedule(task, resource="MyPool")
mockplan.run()
expected_report = TestReport(
name="plan",
entries=[
TestGroupReport(
name="MTest3",
category=ReportCategories.MULTITEST,
entries=[
TestGroupReport(
name="Suite5",
description=
"A test suite with pre/post testcase methods which may not run.",
category=ReportCategories.TESTSUITE,
entries=[
TestGroupReport(
name="test_method",
category=ReportCategories.PARAMETRIZATION,
entries=[
TestCaseReport(
name="test_method <val=1>",
entries=[
{
"type":
"Log",
"message":
"Pre testcase method will always be OK",
},
{
"type": "Log",
"message": "Get value 1",
},
{
"type":
"Log",
"message":
"Testcase will finish in a short time",
},
{
"type":
"Log",
"message":
"Post testcase method can still run",
},
],
),
TestCaseReport(
name="test_method <val=5>",
entries=[
{
"type":
"Log",
"message":
"Pre testcase method will always be OK",
},
{
"type": "Log",
"message": "Get value 5",
},
{
"type":
"Log",
"message":
"Testcase will finish in a short time",
},
],
status_override=Status.ERROR,
),
],
),
],
),
],
)
],
)
check_report(expected_report, mockplan.report)
assert ("`post_testcase` timeout" in mockplan.report["MTest3"]["Suite5"]
["test_method"]["test_method__val_5"].logs[0]["message"])
def test_timeout_on_suite_related_methods(mockplan):
pool = ThreadPool(name="MyPool", size=1)
mockplan.add_resource(pool)
task = Task(target=get_mtest2())
mockplan.schedule(task, resource="MyPool")
mockplan.run()
expected_report = TestReport(
name="plan",
entries=[
TestGroupReport(
name="MTest2",
category=ReportCategories.MULTITEST,
entries=[
TestGroupReport(
name="Suite3",
description=
"A test suite with teardown method and it will timeout.",
category=ReportCategories.TESTSUITE,
entries=[
TestCaseReport(
name="setup",
entries=[{
"type":
"Log",
"message":
"Setup method will sleep for 1 second",
}],
),
TestCaseReport(
name="test_normal",
entries=[{
"type":
"Log",
"message":
"Testcase will finish execution in time",
}],
),
_create_testcase_report(
name="teardown",
status_override=Status.ERROR,
entries=[],
),
],
),
TestGroupReport(
name="Suite4",
description=
"A test suite with setup method and it will timeout.",
category=ReportCategories.TESTSUITE,
entries=[
_create_testcase_report(
name="setup",
status_override=Status.ERROR,
entries=[{
"type":
"Log",
"message":
"Setup method will sleep for 5 seconds",
}],
),
TestCaseReport(
name="teardown",
entries=[{
"type":
"Log",
"message":
"Teardown method can still run",
}],
),
],
),
],
)
],
)
check_report(expected_report, mockplan.report)
def test_timeout_on_testcases(mockplan):
pool = ThreadPool(name="MyPool", size=2)
mockplan.add_resource(pool)
task = Task(target=get_mtest1())
mockplan.schedule(task, resource="MyPool")
mockplan.run()
expected_report = TestReport(
name="plan",
entries=[
TestGroupReport(
name="MTest1",
category=ReportCategories.MULTITEST,
entries=[
TestGroupReport(
name="Suite1",
description="A test suite with basic testcases.",
category=ReportCategories.TESTSUITE,
entries=[
TestCaseReport(
name="test_normal",
entries=[{
"type":
"Log",
"message":
"Testcase will finish execution in time",
}],
),
_create_testcase_report(
name="test_abnormal",
status_override=Status.ERROR,
entries=[{
"type":
"Log",
"message":
"Testcase will definitely timeout",
}],
),
],
),
TestGroupReport(
name="Suite2",
description=
"A test suite with parameterized testcases in different exec groups.",
category=ReportCategories.TESTSUITE,
entries=[
TestGroupReport(
name="test_not_timeout",
category=ReportCategories.PARAMETRIZATION,
entries=[
TestCaseReport(
name="test_not_timeout <val=1>",
entries=[{
"type":
"Log",
"message":
"Testcase will sleep for 1 seconds",
}],
),
TestCaseReport(
name="test_not_timeout <val=2>",
entries=[{
"type":
"Log",
"message":
"Testcase will sleep for 2 seconds",
}],
),
TestCaseReport(
name="test_not_timeout <val=0.5>",
entries=[{
"type":
"Log",
"message":
"Testcase will sleep for 0.5 seconds",
}],
),
],
),
TestGroupReport(
name="test_timeout",
category=ReportCategories.PARAMETRIZATION,
entries=[
TestCaseReport(
name="test_timeout <val=1>",
entries=[{
"type":
"Log",
"message":
"Testcase will sleep for 1 seconds",
}],
),
TestCaseReport(
name="test_timeout <val=0.5>",
entries=[{
"type":
"Log",
"message":
"Testcase will sleep for 0.5 seconds",
}],
),
_create_testcase_report(
name="test_timeout <val=5>",
status_override=Status.ERROR,
entries=[{
"type":
"Log",
"message":
"Testcase will sleep for 5 seconds",
}],
),
],
),
],
),
],
)
],
)
check_report(expected_report, mockplan.report)
