def test_multitest_drivers():
"""TODO."""
for idx, opts in enumerate(
(dict(name='Mtest', suites=[MySuite()],
runpath=runpath_maker), dict(name='Mtest', suites=[MySuite()]))):
server = TCPServer(name='server')
client = TCPClient(name='client',
host=context(server.cfg.name, '{{host}}'),
port=context(server.cfg.name, '{{port}}'))
opts.update(environment=[server, client],
initial_context={'test_key': 'test_value'})
mtest = MultiTest(**opts)
assert server.status.tag == ResourceStatus.NONE
assert client.status.tag == ResourceStatus.NONE
mtest.run()
res = mtest.result
assert res.run is True
if idx == 0:
assert mtest.runpath == runpath_maker(None)
else:
assert mtest.runpath == default_runpath(mtest)
assert server.runpath == os.path.join(mtest.runpath, server.uid())
assert client.runpath == os.path.join(mtest.runpath, client.uid())
assert server.status.tag == ResourceStatus.STOPPED
assert client.status.tag == ResourceStatus.STOPPED
def test_assertion_extra_attribute(mockplan):
"""Test that required extra attribute correctly recorded in report."""
mtest = MultiTest(name="AssertionExtraAttribute",
suites=[AssertionExtraAttribute()])
mtest.cfg.parent = mockplan.cfg
mtest.run()
expected = [
{
"color": "red",
"bgcolor": "white"
},
{
"123": "foo",
"456": "bar",
"789": "baz"
},
{},
{
"border": "1",
"margin": "2"
},
]
assertions = [
entry for entry in mtest.report.flatten()
if isinstance(entry, dict) and "custom_style" in entry
]
for idx, custom_style in enumerate(expected):
assert custom_style == assertions[idx]["custom_style"]
def test_multitest_drivers_in_testplan():
"""TODO."""
for idx, opts in enumerate((dict(name='MyPlan',
parse_cmdline=False,
runpath=runpath_maker),
dict(name='MyPlan', parse_cmdline=False))):
plan = Testplan(**opts)
server = TCPServer(name='server')
client = TCPClient(name='client',
host=context(server.cfg.name, '{{host}}'),
port=context(server.cfg.name, '{{port}}'))
mtest = MultiTest(name='Mtest',
suites=[MySuite()],
environment=[server, client],
initial_context={'test_key': 'test_value'})
plan.add(mtest)
assert server.status.tag == ResourceStatus.NONE
assert client.status.tag == ResourceStatus.NONE
with log_propagation_disabled(TESTPLAN_LOGGER):
plan.run()
res = plan.result
assert res.run is True
if idx == 0:
assert plan.runpath == runpath_maker(None)
else:
assert plan.runpath == default_runpath(plan._runnable)
assert mtest.runpath == os.path.join(plan.runpath, mtest.uid())
assert server.runpath == os.path.join(mtest.runpath, server.uid())
assert client.runpath == os.path.join(mtest.runpath, client.uid())
assert server.status.tag == ResourceStatus.STOPPED
assert client.status.tag == ResourceStatus.STOPPED
def test_parametrized_group_marking_multitest(mockplan, flag):
"""
Tests, at MultiTest-level, if marking works as expected
for parametrized testcases.
"""
test = MultiTest(
name="ParametrizedGroupMarking",
suites=[ParametrizedGroupMarking()],
testcase_report_target=flag,
)
test.cfg.parent = mockplan.cfg
test.run()
assertions = {
entry["description"]: entry
for entry in test.report.flatten()
if isinstance(entry, dict) and entry["meta_type"] == "assertion"
}
expected = {
"A0":
get_line_no(ParametrizedGroupMarking.case, 2),
"B0":
get_line_no(ParametrizedGroupMarking.case, 3) if flag else get_line_no(
helper, 1),
"C0":
get_line_no(intermediary, 2),
}
def test_multitest_drivers_in_testplan(runpath):
"""TODO."""
for idx, opts in enumerate((dict(name="MyPlan",
runpath=runpath), dict(name="MyPlan"))):
plan = TestplanMock(**opts)
server = TCPServer(name="server")
client = TCPClient(
name="client",
host=context(server.cfg.name, "{{host}}"),
port=context(server.cfg.name, "{{port}}"),
)
mtest = MultiTest(
name="Mtest",
suites=[MySuite()],
environment=[server, client],
initial_context={"test_key": "test_value"},
)
plan.add(mtest)
assert server.status.tag == ResourceStatus.NONE
assert client.status.tag == ResourceStatus.NONE
plan.run()
res = plan.result
assert res.run is True
assert res.report.passed
if idx == 0:
assert plan.runpath == runpath
assert mtest.runpath == os.path.join(plan.runpath,
slugify(mtest.uid()))
assert server.runpath == os.path.join(mtest.runpath, server.uid())
assert client.runpath == os.path.join(mtest.runpath, client.uid())
assert server.status.tag == ResourceStatus.STOPPED
assert client.status.tag == ResourceStatus.STOPPED
def test_multitest_drivers(runpath):
"""TODO."""
for idx, opts in enumerate((
dict(name="Mtest", suites=[MySuite()], runpath=runpath),
dict(name="Mtest", suites=[MySuite()]),
)):
server = TCPServer(name="server")
client = TCPClient(
name="client",
host=context(server.cfg.name, "{{host}}"),
port=context(server.cfg.name, "{{port}}"),
)
opts.update(
environment=[server, client],
initial_context={"test_key": "test_value"},
stdout_style=defaults.STDOUT_STYLE,
test_filter=Filter(),
test_sorter=NoopSorter(),
)
mtest = MultiTest(**opts)
assert server.status.tag == ResourceStatus.NONE
assert client.status.tag == ResourceStatus.NONE
res = mtest.run()
assert res.run is True
assert res.report.passed
if idx == 0:
assert mtest.runpath == runpath
else:
assert mtest.runpath == default_runpath(mtest)
assert server.runpath == os.path.join(mtest.runpath, server.uid())
assert client.runpath == os.path.join(mtest.runpath, client.uid())
assert server.status.tag == ResourceStatus.STOPPED
assert client.status.tag == ResourceStatus.STOPPED
def main(plan):
multi_test_1 = MultiTest(name='Primary', suites=[Alpha(), Beta()])
multi_test_2 = MultiTest(name='Secondary', suites=[Gamma()])
multi_test_3 = MultiTest(name='Other', suites=[Delta()])
plan.add(multi_test_1)
plan.add(multi_test_2)
plan.add(multi_test_3)
def main(plan):
multi_test_1 = MultiTest(name='Primary',
suites=[Alpha(), Beta()],
test_filter=Pattern('*:*:test_1'))
multi_test_2 = MultiTest(name='Secondary', suites=[Gamma()])
plan.add(multi_test_1)
plan.add(multi_test_2)
def main(plan):
multi_test_1 = MultiTest(name="Primary",
suites=[Alpha(), Beta()],
tags={"color": "white"})
multi_test_2 = MultiTest(name="Secondary", suites=[Gamma()])
plan.add(multi_test_1)
plan.add(multi_test_2)
def main(plan):
multi_test_1 = MultiTest(name='Primary',
suites=[Alpha(), Beta()],
tags={'color': 'white'})
multi_test_2 = MultiTest(name='Secondary', suites=[Gamma()])
plan.add(multi_test_1)
plan.add(multi_test_2)
def schedule_tests_to_pool(plan, pool, **pool_cfg):
pool_name = pool.__name__
pool = pool(name=pool_name, **pool_cfg)
plan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
mtest1 = MultiTest(name="MTest1", suites=[MySuite()])
mtest2 = MultiTest(name="MTest2", suites=[MySuite()])
uid1 = plan.schedule(target=mtest1, resource=pool_name)
uid2 = plan.schedule(Task(target=mtest2), resource=pool_name)
task3 = Task(target=get_mtest, path=dirname, kwargs=dict(name=3))
uid3 = plan.schedule(task=task3, resource=pool_name)
# Task schedule shortcut
uid4 = plan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name=4),
resource=pool_name,
)
uid5 = plan.schedule(
Task(
target="get_mtest",
module="func_pool_base_tasks",
path=dirname,
kwargs=dict(name=5),
),
resource=pool_name,
)
from .func_pool_base_tasks import get_mtest_imported
uid6 = plan.schedule(
Task(target=get_mtest_imported, kwargs=dict(name=6)),
resource=pool_name,
)
uid7 = plan.schedule(Task(target=get_mtest(name=7)), resource=pool_name)
# with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
assert plan.report.passed is True
assert plan.report.counter == {"passed": 7, "total": 7, "failed": 0}
names = sorted(["MTest{}".format(x) for x in range(1, 8)])
assert sorted([entry.name for entry in plan.report.entries]) == names
assert isinstance(plan.report.serialize(), dict)
assert plan.result.test_results[uid1].report.name == "MTest1"
assert plan.result.test_results[uid2].report.name == "MTest2"
assert plan.result.test_results[uid3].report.name == "MTest3"
assert plan.result.test_results[uid4].report.name == "MTest4"
assert plan.result.test_results[uid5].report.name == "MTest5"
assert plan.result.test_results[uid6].report.name == "MTest6"
assert plan.result.test_results[uid7].report.name == "MTest7"
def main(plan):
multi_test_1 = MultiTest(name='Primary', suites=[AlphaSuite()])
multi_test_2 = MultiTest(
name='Secondary',
suites=[BetaSuite()],
# Just print out assertion names / descriptions but not the details
stdout_style=Style(passing='assertion', failing='assertion'))
plan.add(multi_test_1)
plan.add(multi_test_2)
def main(plan):
multi_test_1 = MultiTest(name='Primary',
test_sorter=ShuffleSorter('all'),
suites=[Alpha(), Beta()])
multi_test_2 = MultiTest(name='Secondary', suites=[Zeta(), Gamma()])
plan.add(multi_test_1)
plan.add(multi_test_2)
def main(plan):
multi_test_1 = MultiTest(
name="Primary",
suites=[Alpha(), Beta()],
test_filter=Pattern("*:*:test_1"),
)
multi_test_2 = MultiTest(name="Secondary", suites=[Gamma()])
plan.add(multi_test_1)
plan.add(multi_test_2)
def test_programmatic_filtering(filter_obj, report_ctx):
multitest_x = MultiTest(name="XXX", suites=[Alpha(), Beta()])
multitest_y = MultiTest(name="YYY", suites=[Gamma()])
plan = TestplanMock(name="plan", test_filter=filter_obj)
plan.add(multitest_x)
plan.add(multitest_y)
plan.run()
test_report = plan.report
check_report_context(test_report, report_ctx)
def schedule_tests_to_pool(pool, **pool_cfg):
pool_name = pool.__name__
plan = Testplan(
name='Plan',
parse_cmdline=False,
)
pool = pool(name=pool_name, **pool_cfg)
plan.add_resource(pool)
dirname = os.path.dirname(os.path.abspath(__file__))
mtest1 = MultiTest(name='MTest1', suites=[MySuite()])
mtest2 = MultiTest(name='MTest2', suites=[MySuite()])
uid1 = plan.schedule(target=mtest1, resource=pool_name)
uid2 = plan.schedule(Task(target=mtest2), resource=pool_name)
task3 = Task(target=get_mtest, path=dirname)
uid3 = plan.schedule(task=task3, resource=pool_name)
# Task schedule shortcut
uid4 = plan.schedule(target='get_mtest',
module='func_pool_base_tasks',
path=dirname,
kwargs=dict(name=4),
resource=pool_name)
uid5 = plan.schedule(Task(target='get_mtest',
module='func_pool_base_tasks',
path=dirname,
kwargs=dict(name=5)),
resource=pool_name)
from .func_pool_base_tasks import get_mtest_imported
uid6 = plan.schedule(Task(target=get_mtest_imported, kwargs=dict(name=6)),
resource=pool_name)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
assert plan.report.passed is True
assert plan.report.counts.passed == 6
assert plan.report.counts.error == plan.report.counts.skipped == \
plan.report.counts.failed == plan.report.counts.incomplete == 0
names = sorted(['MTest{}'.format(x) for x in range(1, 7)])
assert sorted([entry.name for entry in plan.report.entries]) == names
assert isinstance(plan.report.serialize(), dict)
assert plan.result.test_results[uid1].report.name == 'MTest1'
assert plan.result.test_results[uid2].report.name == 'MTest2'
assert plan.result.test_results[uid3].report.name == 'MTest3'
assert plan.result.test_results[uid4].report.name == 'MTest4'
assert plan.result.test_results[uid5].report.name == 'MTest5'
assert plan.result.test_results[uid6].report.name == 'MTest6'
def main(plan):
multi_test_1 = MultiTest(
name="Primary",
test_sorter=ShuffleSorter("all"),
suites=[Alpha(), Beta()],
)
multi_test_2 = MultiTest(name="Secondary", suites=[Zeta(), Gamma()])
plan.add(multi_test_1)
plan.add(multi_test_2)
def test_programmatic_filtering(filter_obj, report_ctx):
multitest_x = MultiTest(name="XXX", suites=[Alpha(), Beta()])
multitest_y = MultiTest(name="YYY", suites=[Gamma()])
plan = TestplanMock(name="plan", test_filter=filter_obj)
plan.add(multitest_x)
plan.add(multitest_y)
with log_propagation_disabled(TESTPLAN_LOGGER):
plan.run()
test_report = plan.report
check_report_context(test_report, report_ctx)
def test_programmatic_filtering(filter_obj, report_ctx):
multitest_x = MultiTest(name='XXX', suites=[Alpha(), Beta()])
multitest_y = MultiTest(name='YYY', suites=[Gamma()])
plan = Testplan(name='plan', parse_cmdline=False, test_filter=filter_obj)
plan.add(multitest_x)
plan.add(multitest_y)
with log_propagation_disabled(TESTPLAN_LOGGER):
plan.run()
test_report = plan.report
check_report_context(test_report, report_ctx)
def test_assertion_orders():
mtest = MultiTest(name='AssertionsOrder', suites=[AssertionOrder()])
mtest.run()
expected = [
'AssertionFirst1', 'AssertionFirst2', 'AssertionSecond',
'AssertionMain1', 'AssertionMain2', 'Report passed so far.'
]
assertions = (
entry for entry in mtest.report.flatten()
if isinstance(entry, dict) and entry['meta_type'] == 'assertion')
for idx, entry in enumerate(assertions):
assert entry['description'] == expected[idx]
def test_command_line_listing(runpath, cmdline_args, expected_output):
multitest_x = MultiTest(name="Primary", suites=[Beta(), Alpha()])
multitest_y = MultiTest(name="Secondary", suites=[Gamma()])
with argv_overridden(*cmdline_args):
plan = TestplanMock(name="plan", parse_cmdline=True, runpath=runpath)
with captured_logging(TESTPLAN_LOGGER) as log_capture:
plan.add(multitest_x)
plan.add(multitest_y)
result = plan.run()
assert log_capture.output == expected_output
assert len(result.test_report) == 0, "No tests should be run."
def test_command_line_filtering(cmdline_args, report_ctx):
multitest_x = MultiTest(name='XXX', suites=[Alpha(), Beta()])
multitest_y = MultiTest(name='YYY', suites=[Gamma()])
with argv_overridden(*cmdline_args):
plan = Testplan(name='plan', parse_cmdline=True)
plan.add(multitest_x)
plan.add(multitest_y)
with log_propagation_disabled(TESTPLAN_LOGGER):
plan.run()
test_report = plan.report
check_report_context(test_report, report_ctx)
def main(plan):
plan.add(MultiTest(
name="Dict Assertions Test",
suites=[
DictSuite(),
],
))
def main():
# CREATE AN INTERACTIVE TESTPLAN
plan = TestplanMock(name="MyPlan",
interactive_port=0,
interactive_block=False)
with open("basic_suite_template.txt") as fobj:
template = fobj.read()
# WRITE A FAILING TESTCASE
with open("basic_suite_with_value.py", "w") as fobj:
fobj.write(template.format(VALUE=3))
atexit.register(os.remove, "basic_suite_with_value.py")
from basic_suite_with_value import SuiteTemplate
# TRIGGER RUN OF INTERACTIVE EXECUTIONER
plan.run()
# ADD A TEST
plan.add(MultiTest(name="Test1", suites=[SuiteTemplate()]))
# RUN THE TESTS
plan.i.run_all_tests()
# EXPECTED 1 != 3 FAILURE
serialized = plan.i.test_case_report(
test_uid="Test1",
suite_uid="SuiteTemplate",
case_uid="basic_case",
serialized=True,
)
assert (compare(
serialized["entries"][0]["entries"][0]["entries"][0],
FAILED_CASE_REPORT,
)[0] is True)
assert plan.i.report().passed is False
# APPLY A CODE CHANGE - FIX
with open("basic_suite_with_value.py", "w") as fobj:
fobj.write(template.format(VALUE=1))
# SEND RELOAD CODE
plan.i.reload()
# RUN TESTS AGAIN
plan.i.run_all_tests()
# EXPECTED 1 == 1 SUCCESS
serialized = plan.i.test_case_report(
test_uid="Test1",
suite_uid="SuiteTemplate",
case_uid="basic_case",
serialized=True,
)
assert (compare(
serialized["entries"][0]["entries"][0]["entries"][0],
PASSED_CASE_REPORT,
)[0] is True)
assert plan.i.report().passed is True
def test_testcase_related_with_inivalid_arguments_in_suite_object(mockplan):
"""
``pre_testcase`` and ``post_testcase`` methods should have argument
like [self, name, env, result] or [self, name, env, result, kwargs].
"""
@testsuite
class MySuite(object):
def pre_testcase(self, name, env, result, kwargs): # valid
result.dict.log(kwargs)
def post_testcase(self, name, env, result, my_arg): # invalid
pass
@testcase
def sample_test(self, env, result):
pass
@testcase(parameters=(("foo", "bar"), ))
def param_test(self, env, result, x, y):
pass
mockplan.add(
MultiTest(name="MyMultitest", suites=[MySuite()], stop_on_error=False))
mockplan.run()
multitest_report = mockplan.result.report["MyMultitest"]
case_report = multitest_report["MySuite"]["sample_test"]
param_case_report = multitest_report["MySuite"]["param_test"].entries[0]
assert multitest_report.status == Status.ERROR
assert "MethodSignatureMismatch" in case_report.logs[0]["message"]
assert len(case_report.entries[0]["flattened_dict"]) == 0
assert "MethodSignatureMismatch" in param_case_report.logs[0]["message"]
assert len(param_case_report.entries[0]["flattened_dict"]) == 2
def make_multitest_3(tmp_file):
return MultiTest(
name="Unstable MTest3",
description="MultiTest that passes until iteration 4",
suites=[UnstableSuite3(tmp_file=tmp_file, max_retries=4)],
environment=[],
)
def make_multitest_2(tmp_file):
return MultiTest(
name="Unstable MTest2",
description="MultiTest that fails in iteration 0, and passes on 1",
suites=[UnstableSuite2(tmp_file=tmp_file, max_retries=1)],
environment=[],
)
def make_multitest_1(tmp_file):
return MultiTest(
name="Unstable MTest1",
description="MultiTest that fails in iteration 0 & 1, and passes on 2",
suites=[UnstableSuite1(tmp_file=tmp_file, max_retries=2)],
environment=[MockDriver(name="mock_driver")],
)
def main(plan):
plan.add(MultiTest(
name="XML Assertions Test",
suites=[
XMLSuite(),
],
))
def main(plan):
multi_test_1 = MultiTest(name="Primary",
suites=[Alpha(), Beta(),
Epsilon()])
plan.add(multi_test_1)
