def test_multi_parts_merged():
"""Execute MultiTest parts and merge report."""
plan = Testplan(name='plan',
parse_cmdline=False,
merge_scheduled_parts=True)
pool = ThreadPool(name='MyPool', size=2)
plan.add_resource(pool)
for idx in range(3):
task = Task(target=get_mtest(part_tuple=(idx, 3)))
plan.schedule(task, resource='MyPool')
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
assert len(plan.report.entries) == 1
assert plan.report.entries[0].name == 'MTest'
assert len(plan.report.entries[0].entries) == 2 # 2 suites
assert plan.report.entries[0].entries[0].name == 'Suite1'
assert plan.report.entries[0].entries[1].name == 'Suite2'
assert len(plan.report.entries[0].entries[0].entries) == 1 # param group
assert plan.report.entries[0].entries[0].entries[0].name == 'test_true'
assert len(plan.report.entries[0].entries[1].entries) == 1 # param group
assert plan.report.entries[0].entries[1].entries[0].name == 'test_false'
assert len(plan.report.entries[0].entries[0].entries[0].entries) == 10
assert len(plan.report.entries[0].entries[1].entries[0].entries) == 3
def test_multi_parts_invalid_parameter_1():
"""
Execute MultiTest parts with invalid parameters that a part of
MultiTest has been scheduled twice.
"""
plan = Testplan(name='plan',
parse_cmdline=False,
merge_scheduled_parts=True)
pool = ThreadPool(name='MyPool', size=2)
plan.add_resource(pool)
for idx in range(3):
task = Task(target=get_mtest(part_tuple=(idx, 3)))
plan.schedule(task, resource='MyPool')
plan.schedule(Task(target=get_mtest(part_tuple=(1, 3))), resource='MyPool')
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is False
assert len(plan.report.entries) == 1
assert len(plan.report.entries[0].entries) == 2
assert plan.report.status == Status.ERROR # Testplan result
assert plan.report.entries[0].status == Status.ERROR # Multitest
assert plan.report.entries[0].entries[0].status == Status.FAILED # Suite1
assert plan.report.entries[0].entries[1].status == Status.FAILED # Suite2
assert 'invalid parameter of part provided' in \
plan.report.entries[0].logs[0]['message']
def test_multi_parts_not_merged():
"""Execute MultiTest parts but do not merge report."""
plan = TestplanMock(name="plan", merge_scheduled_parts=False)
pool = ThreadPool(name="MyThreadPool", size=2)
plan.add_resource(pool)
for idx in range(3):
task = Task(target=get_mtest(part_tuple=(idx, 3)))
plan.schedule(task, resource="MyThreadPool")
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
assert len(plan.report.entries) == 3
assert plan.report.entries[0].name == "MTest - part(1/3)"
assert plan.report.entries[1].name == "MTest - part(2/3)"
assert plan.report.entries[2].name == "MTest - part(3/3)"
assert len(plan.report.entries[0].entries) == 2 # 2 suites
assert plan.report.entries[0].entries[0].name == "Suite1"
assert plan.report.entries[0].entries[1].name == "Suite2"
assert len(plan.report.entries[0].entries[0].entries) == 1 # param group
assert plan.report.entries[0].entries[0].entries[0].name == "test_true"
assert len(plan.report.entries[0].entries[1].entries) == 1 # param group
assert plan.report.entries[0].entries[1].entries[0].name == "test_false"
assert len(plan.report.entries[0].entries[0].entries[0].entries) == 4
assert len(plan.report.entries[0].entries[1].entries[0].entries) == 1
def test_task_rerun_in_thread_pool(mockplan):
"""
Test procedure:
- 1st run: `unstable_case` fails.
- 1st rerun: `mock_driver` raises during stop.
- 2nd rerun: all pass.
"""
pool_name = ThreadPool.__name__
pool = ThreadPool(name=pool_name, size=2)
mockplan.add_resource(pool)
directory = os.path.dirname(os.path.abspath(__file__))
tmp_file = os.path.join(tempfile.gettempdir(), getpass.getuser(),
"{}.tmp".format(uuid.uuid4()))
task = Task(target=make_multitest_1,
path=directory,
args=(tmp_file, ),
rerun=2)
uid = mockplan.schedule(task=task, resource=pool_name)
assert mockplan.run().run is True
assert mockplan.report.passed is True
assert mockplan.report.counter == {"passed": 3, "total": 3, "failed": 0}
assert isinstance(mockplan.report.serialize(), dict)
assert mockplan.result.test_results[uid].report.name == "Unstable MTest1"
assert len(mockplan.report.entries) == 3
assert mockplan.report.entries[-1].category == ReportCategories.TASK_RERUN
assert mockplan.report.entries[-2].category == ReportCategories.TASK_RERUN
assert task.reassign_cnt == 2
_remove_existing_tmp_file(tmp_file)
def test_multi_parts_merged():
"""
Schedule MultiTest parts in different ways, execute them and merge reports.
"""
plan = TestplanMock(name="plan", merge_scheduled_parts=True)
pool = ThreadPool(name="MyThreadPool", size=2)
plan.add_resource(pool)
def _get_mtest():
return MultiTest(name="MTest",
suites=[Suite1(), Suite2()],
part=(2, 3))
plan.add(target=_get_mtest()) # local_runner
plan.add(Task(target=get_mtest(part_tuple=(1, 3)))) # local_runner
plan.schedule(Task(target=get_mtest(part_tuple=(0, 3))),
resource="MyThreadPool")
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
assert len(plan.report.entries) == 1
assert plan.report.entries[0].name == "MTest"
assert len(plan.report.entries[0].entries) == 2 # 2 suites
assert plan.report.entries[0].entries[0].name == "Suite1"
assert plan.report.entries[0].entries[1].name == "Suite2"
assert len(plan.report.entries[0].entries[0].entries) == 1 # param group
assert plan.report.entries[0].entries[0].entries[0].name == "test_true"
assert len(plan.report.entries[0].entries[1].entries) == 1 # param group
assert plan.report.entries[0].entries[1].entries[0].name == "test_false"
assert len(plan.report.entries[0].entries[0].entries[0].entries) == 10
assert len(plan.report.entries[0].entries[1].entries[0].entries) == 3
def test_multi_parts_invalid_parameter_2():
"""
Execute MultiTest parts with invalid parameters that a MultiTest
has been scheduled twice.
"""
plan = TestplanMock(name="plan", merge_scheduled_parts=True)
pool = ThreadPool(name="MyThreadPool", size=2)
plan.add_resource(pool)
for idx in range(3):
task = Task(target=get_mtest(part_tuple=(idx, 3)))
plan.schedule(task, resource="MyThreadPool")
for idx in range(2):
task = Task(target=get_mtest(part_tuple=(idx, 2)))
plan.schedule(task, resource="MyThreadPool")
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is False
assert len(plan.report.entries) == 1
assert len(plan.report.entries[0].entries) == 2
assert plan.report.status == Status.ERROR # Testplan result
assert plan.report.entries[0].status == Status.ERROR # Multitest
assert plan.report.entries[0].entries[0].status == Status.FAILED # Suite1
assert plan.report.entries[0].entries[1].status == Status.FAILED # Suite2
assert ("invalid parameter of part provided"
in plan.report.entries[0].logs[0]["message"])
def test_task_rerun_with_more_times_2(mockplan):
"""
Test procedure 2:
(set `task_rerun_limit` to 4, a task can be scheduled 5 times in total)
- 1st run: `unstable_case` fails.
- 1st rerun: `unstable_case` fails.
- 2nd rerun: `unstable_case` fails.
- 3rd rerun: all pass.
"""
pool_name = ThreadPool.__name__
pool = ThreadPool(name=pool_name, size=1)
mockplan.add_resource(pool)
directory = os.path.dirname(os.path.abspath(__file__))
tmp_file = os.path.join(tempfile.gettempdir(), getpass.getuser(),
"{}.tmp".format(uuid.uuid4()))
task = Task(target=make_multitest_3,
path=directory,
args=(tmp_file, ),
rerun=3)
uid = mockplan.schedule(task=task, resource=pool_name)
assert mockplan.run().run is True
assert mockplan.report.passed is True
assert mockplan.report.counter == {"passed": 2, "total": 2, "failed": 0}
assert mockplan.result.test_results[uid].report.name == "Unstable MTest3"
assert task.reassign_cnt == 3
_remove_existing_tmp_file(tmp_file)
def test_multi_parts_duplicate_part():
"""
Execute MultiTest parts with a part of MultiTest has been
scheduled twice and automatically be filtered out.
"""
plan = TestplanMock(name="plan", merge_scheduled_parts=True)
pool = ThreadPool(name="MyThreadPool", size=2)
plan.add_resource(pool)
for idx in range(3):
task = Task(target=get_mtest_with_custom_uid(part_tuple=(idx, 3)))
plan.schedule(task, resource="MyThreadPool")
task = Task(target=get_mtest_with_custom_uid(part_tuple=(1, 3)))
plan.schedule(task, resource="MyThreadPool")
assert len(plan._tests) == 4
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is False
assert len(plan.report.entries) == 5 # one placeholder report & 4 siblings
assert len(plan.report.entries[0].entries) == 0 # already cleared
assert plan.report.status == Status.ERROR # Testplan result
assert ("duplicate MultiTest parts had been scheduled"
in plan.report.entries[0].logs[0]["message"])
def test_multi_parts_incorrect_schedule():
"""
Execute MultiTest parts with invalid parameters that a MultiTest
has been scheduled twice and each time split into different parts.
"""
plan = TestplanMock(name="plan", merge_scheduled_parts=True)
pool = ThreadPool(name="MyThreadPool", size=2)
plan.add_resource(pool)
for idx in range(3):
task = Task(target=get_mtest(part_tuple=(idx, 3)))
plan.schedule(task, resource="MyThreadPool")
for idx in range(2):
task = Task(target=get_mtest(part_tuple=(idx, 2)))
plan.schedule(task, resource="MyThreadPool")
assert len(plan._tests) == 5
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is False
assert len(plan.report.entries) == 6 # one placeholder report & 5 siblings
assert len(plan.report.entries[0].entries) == 0 # already cleared
assert plan.report.status == Status.ERROR # Testplan result
assert ("invalid parameter of part provided"
in plan.report.entries[0].logs[0]["message"])
def test_multi_parts_missing_parts():
"""
Execute MultiTest parts with invalid parameters that not all
parts of a MultiTest has been scheduled.
"""
plan = Testplan(name="plan",
parse_cmdline=False,
merge_scheduled_parts=True)
pool = ThreadPool(name="MyPool", size=2)
plan.add_resource(pool)
for idx in range(1, 3):
task = Task(target=get_mtest(part_tuple=(idx, 3)))
plan.schedule(task, resource="MyPool")
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is False
assert len(plan.report.entries) == 1
assert len(plan.report.entries[0].entries) == 2
assert plan.report.status == Status.ERROR # Testplan result
assert plan.report.entries[0].status == Status.ERROR # Multitest
assert plan.report.entries[0].entries[0].status == Status.PASSED # Suite1
assert plan.report.entries[0].entries[1].status == Status.FAILED # Suite2
assert ("not all MultiTest parts had been scheduled"
in plan.report.entries[0].logs[0]["message"])
def test_runner_timeout():
"""
Execute MultiTests in LocalRunner, ThreadPool and ProcessPool respectively.
Some of them will timeout and we'll get a report showing execution details.
"""
plan = Testplan(name='plan',
parse_cmdline=False,
runnable=MyTestRunner,
timeout=60,
abort_wait_timeout=5)
mod_path = os.path.dirname(os.path.abspath(__file__))
THREAD_POOL = 'MyThreadPool'
PROCESS_POOL = 'MyProcessPool'
thread_pool = ThreadPool(name=THREAD_POOL, size=2, worker_heartbeat=None)
proc_pool = ProcessPool(name=PROCESS_POOL, size=2, worker_heartbeat=None)
plan.add_resource(thread_pool)
plan.add_resource(proc_pool)
plan.add(func_basic_tasks.get_mtest1())
plan.add(func_basic_tasks.get_mtest2())
task3 = Task(target='get_mtest3', module='func_basic_tasks', path=mod_path)
task4 = Task(target='get_mtest4', module='func_basic_tasks', path=mod_path)
task5 = Task(target='get_mtest5', module='func_basic_tasks', path=mod_path)
task6 = Task(target='get_mtest6', module='func_basic_tasks', path=mod_path)
plan.schedule(task3, resource=THREAD_POOL)
plan.schedule(task4, resource=THREAD_POOL)
plan.schedule(task5, resource=PROCESS_POOL)
plan.schedule(task6, resource=PROCESS_POOL)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is False
assert len(plan.report.entries) == 6
assert plan.report.status == Status.ERROR
entries = plan.report.entries
assert entries[0].name == 'MTest1'
assert entries[0].status == Status.FAILED # testcase 'test_true' failed
assert entries[2].name == 'MTest3'
assert entries[2].status == Status.PASSED # testcase 'test_equal' passed
assert entries[4].name == 'MTest5'
assert entries[4].status == Status.ERROR # testcase 'test_contain' raised
assert entries[1].name == 'MTest2'
assert entries[1].status == Status.ERROR # timeout
assert ' discarding due to ' in entries[1].logs[0]['message']
assert entries[3].name == 'Task[get_mtest4]'
assert entries[3].status == Status.ERROR # timeout
assert entries[3].logs[0]['message'].startswith('_target: get_mtest4')
assert ' discarding due to ' in entries[3].logs[1]['message']
assert entries[5].name == 'Task[get_mtest6]'
assert entries[5].status == Status.ERROR # timeout
assert entries[5].logs[0]['message'].startswith('_target: get_mtest6')
assert ' discarding due to ' in entries[5].logs[1]['message']
def main(plan):
"""
Testplan decorated main function to add and execute MultiTests.
:return: Testplan result object.
:rtype: ``testplan.base.TestplanResult``
"""
# Add a thread pool test execution resource to the plan of given size.
pool = ThreadPool(name="MyPool", size=plan.args.pool_size)
plan.add_resource(pool)
# Add a given number of similar tests to the thread pool
# to be executed in parallel.
for idx in range(plan.args.tasks_num):
task = Task(target="make_multitest",
module="tasks",
kwargs={"index": idx})
plan.schedule(task, resource="MyPool")
def main(plan):
"""
Testplan decorated main function to add and execute MultiTests.
:return: Testplan result object.
:rtype: ``testplan.base.TestplanResult``
"""
# Add a thread pool test execution resource to the plan of given size.
# Also you can use process pool or remote pool instead.
pool = ThreadPool(name='MyPool', size=plan.args.pool_size)
plan.add_resource(pool)
# Add a given number of similar tests to the thread pool
# to be executed in parallel.
for idx in range(plan.args.parts_num):
task = Task(target='make_multitest',
module='tasks',
kwargs={'part_tuple': (idx, plan.args.parts_num)})
plan.schedule(task, resource='MyPool')
def main(plan):
"""
Testplan decorated main function to add and execute MultiTests.
:return: Testplan result object.
:rtype: ``testplan.base.TestplanResult``
"""
# Add a thread pool test execution resource to the plan of given size.
# Can also use a process pool instead.
pool = ThreadPool(name="MyPool", size=plan.args.pool_size)
plan.add_resource(pool)
# Add a task with `rerun` argument to the thread pool
tmp_file = os.path.join(
tempfile.gettempdir(), getpass.getuser(), "{}.tmp".format(uuid.uuid4())
)
task = Task(
target="make_multitest", module="tasks", args=(tmp_file,), rerun=2
)
plan.schedule(task, resource="MyPool")
def test_multi_parts_missing_parts():
"""
Execute MultiTest parts with invalid parameters that
not all parts of a MultiTest are scheduled.
"""
plan = TestplanMock(name="plan", merge_scheduled_parts=True)
pool = ThreadPool(name="MyThreadPool", size=2)
plan.add_resource(pool)
for idx in range(1, 3):
task = Task(target=get_mtest(part_tuple=(idx, 3)))
plan.schedule(task, resource="MyThreadPool")
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is False
assert len(plan.report.entries) == 3 # one placeholder report & 2 siblings
assert len(plan.report.entries[0].entries) == 0 # already cleared
assert plan.report.status == Status.ERROR # Testplan result
assert ("not all MultiTest parts had been scheduled"
in plan.report.entries[0].logs[0]["message"])
def test_timeout_on_testcases(mockplan):
pool = ThreadPool(name="MyPool", size=2)
mockplan.add_resource(pool)
task = Task(target=get_mtest())
mockplan.schedule(task, resource="MyPool")
with log_propagation_disabled(TESTPLAN_LOGGER):
mockplan.run()
expected_report = TestReport(
name="plan",
entries=[
TestGroupReport(
name="MTest",
category=ReportCategories.MULTITEST,
entries=[
TestGroupReport(
name="Suite1",
description="A test suite with basic testcases.",
category=ReportCategories.TESTSUITE,
entries=[
TestCaseReport(
name="test_normal",
entries=[
{
"type": "Log",
"message": "Testcase will finish execution in time",
}
],
),
_create_testcase_report(
name="test_abnormal",
status_override=Status.ERROR,
entries=[
{
"type": "Log",
"message": "Testcase will definitely timeout",
}
],
),
],
),
TestGroupReport(
name="Suite2",
description="A test suite with parameterized testcases in different exec groups.",
category=ReportCategories.TESTSUITE,
entries=[
TestGroupReport(
name="test_timeout_1",
category=ReportCategories.PARAMETRIZATION,
entries=[
TestCaseReport(
name="test_timeout_1__val_1",
entries=[
{
"type": "Log",
"message": "Testcase will sleep for 1 seconds",
}
],
),
TestCaseReport(
name="test_timeout_1__val_2",
entries=[
{
"type": "Log",
"message": "Testcase will sleep for 2 seconds",
}
],
),
TestCaseReport(
name="test_timeout_1__val_3",
entries=[
{
"type": "Log",
"message": "Testcase will sleep for 3 seconds",
}
],
),
],
),
TestGroupReport(
name="test_timeout_2",
category=ReportCategories.PARAMETRIZATION,
entries=[
TestCaseReport(
name="test_timeout_2__val_1",
entries=[
{
"type": "Log",
"message": "Testcase will sleep for 1 seconds",
}
],
),
TestCaseReport(
name="test_timeout_2__val_2",
entries=[
{
"type": "Log",
"message": "Testcase will sleep for 2 seconds",
}
],
),
_create_testcase_report(
name="test_timeout_2__val_5",
status_override=Status.ERROR,
entries=[
{
"type": "Log",
"message": "Testcase will sleep for 5 seconds",
}
],
),
],
),
],
),
],
)
],
)
check_report(expected_report, mockplan.report)
def test_timeout_on_testcases():
plan = Testplan(name='plan', parse_cmdline=False)
pool = ThreadPool(name='MyPool', size=2)
plan.add_resource(pool)
task = Task(target=get_mtest())
plan.schedule(task, resource='MyPool')
with log_propagation_disabled(TESTPLAN_LOGGER):
plan.run()
expected_report = TestReport(
name='plan',
entries=[
TestGroupReport(
name='MTest',
category=Categories.MULTITEST,
entries=[
TestGroupReport(
name='Suite1',
description='A test suite with basic testcases.',
category=Categories.SUITE,
entries=[
TestCaseReport(
name='test_normal',
entries=[{
'type':
'Log',
'message':
'Testcase will finish execution in time'
}]),
_create_testcase_report(
name='test_abnormal',
status_override=Status.ERROR,
entries=[{
'type':
'Log',
'message':
'Testcase will definitely timeout'
}])
]),
TestGroupReport(
name='Suite2',
description=
'A test suite with parameterized testcases in different exec groups.',
category=Categories.SUITE,
entries=[
TestGroupReport(
name='test_timeout_1',
category=Categories.PARAMETRIZATION,
entries=[
TestCaseReport(
name='test_timeout_1__val_1',
entries=[{
'type':
'Log',
'message':
'Testcase will sleep for 1 seconds'
}]),
TestCaseReport(
name='test_timeout_1__val_2',
entries=[{
'type':
'Log',
'message':
'Testcase will sleep for 2 seconds'
}]),
TestCaseReport(
name='test_timeout_1__val_3',
entries=[{
'type':
'Log',
'message':
'Testcase will sleep for 3 seconds'
}])
]),
TestGroupReport(
name='test_timeout_2',
category=Categories.PARAMETRIZATION,
entries=[
TestCaseReport(
name='test_timeout_2__val_1',
entries=[{
'type':
'Log',
'message':
'Testcase will sleep for 1 seconds'
}]),
TestCaseReport(
name='test_timeout_2__val_2',
entries=[{
'type':
'Log',
'message':
'Testcase will sleep for 2 seconds'
}]),
_create_testcase_report(
name='test_timeout_2__val_5',
status_override=Status.ERROR,
entries=[{
'type':
'Log',
'message':
'Testcase will sleep for 5 seconds'
}])
])
])
])
])
check_report(expected_report, plan.report)