def test_report_json_binary_serialization(
dummy_test_plan_report_with_binary_asserts,
):
"""JSON Serialized & deserialized reports should be equal."""
test_plan_schema = TestReportSchema(strict=True)
data = test_plan_schema.dumps(
dummy_test_plan_report_with_binary_asserts
).data
j = json.loads(data)
bkey = EntriesField._BYTES_KEY
# passing assertion
hx_1_1 = j["entries"][1]["entries"][0]["entries"][1]["first"][bkey]
hx_1_2 = j["entries"][1]["entries"][0]["entries"][1]["second"][bkey]
assert ["0xF2"] == hx_1_1 == hx_1_2
# failing assertion
hx_2_1 = j["entries"][1]["entries"][0]["entries"][2]["first"][bkey]
hx_2_2 = j["entries"][1]["entries"][0]["entries"][2]["second"][bkey]
assert ["0x00", "0xB1", "0xC1"] == hx_2_1
assert ["0x00", "0xB2", "0xC2"] == hx_2_2
deserialized_report = test_plan_schema.loads(data).data
check_report(
actual=deserialized_report,
expected=dummy_test_plan_report_with_binary_asserts,
)
def test_parametrization_tagging():
"""
Parametrization report group should include tags
generated by `tag_func` and native suite tags in `tag_index` attribute.
"""
@testsuite(tags='foo')
class DummySuite(object):
@testcase(parameters=('red', 'blue', 'green'),
tags='alpha',
tag_func=lambda kwargs: {'color': kwargs['color']})
def dummy_test(self, env, result, color):
pass
all_tags_index = {
'simple': {'foo', 'alpha'},
'color': {'red', 'blue', 'green'}
}
parametrization_group = TestGroupReport(
name='dummy_test',
category=Categories.PARAMETRIZATION,
tags={'simple': {'alpha'}},
entries=[
TestCaseReport(
name='dummy_test__color_red',
tags={'color': {'red'}},
),
TestCaseReport(
name='dummy_test__color_blue',
tags={'color': {'blue'}},
),
TestCaseReport(
name='dummy_test__color_green',
tags={'color': {'green'}},
),
])
multitest = MultiTest(name='MyMultitest', suites=[DummySuite()])
plan = Testplan(name='plan', parse_cmdline=False)
plan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
plan.run()
expected_report = TestReport(
name='plan',
entries=[
TestGroupReport(name='MyMultitest',
category=Categories.MULTITEST,
entries=[
TestGroupReport(
name='DummySuite',
tags={'simple': {'foo'}},
category=Categories.SUITE,
entries=[parametrization_group])
])
])
check_report(expected_report, plan.report)
def check_parametrization(suite_kls, parametrization_group):
multitest = MultiTest(name="MyMultitest", suites=[suite_kls()])
plan = Testplan(name="plan", parse_cmdline=False)
plan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
plan.run()
expected_report = TestReport(
name="plan",
entries=[
TestGroupReport(
name="MyMultitest",
category=ReportCategories.MULTITEST,
entries=[
TestGroupReport(
name="MySuite",
category=ReportCategories.TESTSUITE,
entries=[parametrization_group],
)
],
)
],
)
check_report(expected_report, plan.report)
def test_no_pre_post_steps(mockplan):
"""
Runs a MultiTest w/o pre/post steps, expects no pre/post step report.
"""
multitest = MultiTest(
name="MyMultiTest",
suites=[MySuite()],
)
mockplan.add(multitest)
mockplan.run()
expected_report = TestReport(
name="plan",
entries=[
TestGroupReport(
name="MyMultiTest",
category=ReportCategories.MULTITEST,
entries=[
TestGroupReport(
name="MySuite",
category=ReportCategories.TESTSUITE,
entries=[
TestCaseReport(name="test_one"),
TestCaseReport(
name="teardown",
entries=[{"type": "Attachment"}],
),
],
),
],
),
],
)
check_report(expected_report, mockplan.report)
def test_process_runner(mockplan, binary_path, expected_report, test_kwargs):
process_test = DummyTest(name="MyTest", binary=binary_path, **test_kwargs)
mockplan.add(process_test)
assert mockplan.run().run is True
check_report(expected=expected_report, actual=mockplan.report)
def test_parametrization_tagging():
"""
Parametrization report group should include tags
generated by `tag_func` and native suite tags in `tag_index` attribute.
"""
@testsuite(tags="foo")
class DummySuite(object):
@testcase(
parameters=("red", "blue", "green"),
tags="alpha",
tag_func=lambda kwargs: {"color": kwargs["color"]},
)
def dummy_test(self, env, result, color):
pass
parametrization_group = TestGroupReport(
name="dummy_test",
category=ReportCategories.PARAMETRIZATION,
tags={"simple": {"alpha"}},
entries=[
TestCaseReport(
name="dummy_test__color_red", tags={"color": {"red"}}
),
TestCaseReport(
name="dummy_test__color_blue", tags={"color": {"blue"}}
),
TestCaseReport(
name="dummy_test__color_green", tags={"color": {"green"}}
),
],
)
multitest = MultiTest(name="MyMultitest", suites=[DummySuite()])
plan = Testplan(name="plan", parse_cmdline=False)
plan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
plan.run()
expected_report = TestReport(
name="plan",
entries=[
TestGroupReport(
name="MyMultitest",
category=ReportCategories.MULTITEST,
entries=[
TestGroupReport(
name="DummySuite",
tags={"simple": {"foo"}},
category=ReportCategories.TESTSUITE,
entries=[parametrization_group],
)
],
)
],
)
check_report(expected_report, plan.report)
def check_parametrization(mockplan, suite_kls, report_list, tag_dict=None):
tag_dict = tag_dict or {}
multitest = MultiTest(name="MyMultitest", suites=[suite_kls()])
mockplan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
mockplan.run()
expected_report = TestReport(
name="plan",
entries=[
TestGroupReport(
name="MyMultitest",
category=ReportCategories.MULTITEST,
entries=[
TestGroupReport(
name="MySuite",
tags=tag_dict,
category=ReportCategories.TESTSUITE,
entries=report_list,
)
],
)
],
)
check_report(expected_report, mockplan.report)
def test_local_pool_integration(
report_dir,
multitest_maker,
expected_report,
pdf_title,
expected_plan_result,
dependant_module,
):
if dependant_module:
importorxfail(dependant_module)
pdf_path = report_dir.join(
"test_report_local_{}.pdf".format(pdf_title)).strpath
plan = Testplan(
name="plan",
parse_cmdline=False,
exporters=[PDFExporter.with_config(pdf_path=pdf_path)],
)
plan.add(multitest_maker())
assert not os.path.exists(pdf_path)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
for log in plan.report.flattened_logs:
if all(word in log["message"] for word in ["tkinter", "TclError"]):
pytest.xfail(reason="Tkinter not installed properly")
check_report(expected=expected_report, actual=plan.report)
assert plan.result.success is expected_plan_result
assert os.path.exists(pdf_path)
assert os.stat(pdf_path).st_size > 0
def test_pre_post_steps(mockplan):
multitest = MultiTest(
name="MyMultitest",
suites=[MySuite()],
before_start=check_func_1,
after_start=check_func_2,
before_stop=check_func_3,
after_stop=check_func_4,
)
mockplan.add(multitest)
mockplan.run()
expected_report = TestReport(
name="plan",
entries=[
TestGroupReport(
name="MyMultitest",
category=ReportCategories.MULTITEST,
entries=[
TestGroupReport(
name="Before/After Step Checks",
category=ReportCategories.TESTSUITE,
entries=[
TestCaseReport(
name="before_start - check_func_1",
entries=[{"type": "Equal", "passed": True}],
),
TestCaseReport(name="after_start - check_func_2"),
TestCaseReport(name="before_stop - check_func_3"),
TestCaseReport(
name="after_stop - check_func_4",
entries=[
{"type": "Equal", "passed": False},
{"type": "Attachment"},
],
),
],
),
TestGroupReport(
name="MySuite",
category=ReportCategories.TESTSUITE,
entries=[
TestCaseReport(name="test_one"),
TestCaseReport(
name="teardown",
entries=[{"type": "Attachment"}],
),
],
),
],
)
],
)
check_report(expected_report, mockplan.report)
assert len(mockplan.report.attachments) == 2
def test_process_runner(mockplan, binary_path, expected_report, test_kwargs):
process_test = DummyTest(name="MyTest", binary=binary_path, **test_kwargs)
mockplan.add(process_test)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert mockplan.run().run is True
check_report(expected=expected_report, actual=mockplan.report)
def test_dry_run():
"""Test the "dry_run" method which generates an empty report skeleton."""
mtest = multitest.MultiTest(name="MTest",
description="Basic multitest.",
suites=[Suite()],
**MTEST_DEFAULT_PARAMS)
result = mtest.dry_run()
report_skeleton = result.report
# Comparing the reports to spot any inconsistencies.
testing.check_report(expected=EXPECTED_REPORT_SKELETON,
actual=report_skeleton)
def test_import(params):
input_path = params.input_path
importer = CPPUnitResultImporter(
input_path,
name=params.expected_report.name,
description=params.expected_report.description,
)
result = importer.import_result()
check_report(
expected=params.expected_report,
actual=result.as_test_report(),
)
def test_process_runner(binary_path, expected_report, test_kwargs):
plan = Testplan(name="plan", parse_cmdline=False)
process_test = DummyTest(name="MyTest", driver=binary_path, **test_kwargs)
plan.add(process_test)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
check_report(expected=expected_report, actual=plan.report)
def test_merge():
mt_report = TestGroupReport(
name="MyMultiTest",
category="multitest",
uid=0,
tags={"color": {"green"}},
)
mt_report.merge(mt_report_alpha, strict=False)
mt_report.merge(mt_report_beta_1, strict=False)
mt_report.merge(mt_report_beta_2, strict=False)
mt_report.merge(mt_report_gamma, strict=False)
check_report(actual=mt_report, expected=expected_report)
def test_merge():
mt_report = TestGroupReport(
name='MyMultiTest',
category='multitest',
uid=0,
tags={'color': {'green'}},
)
mt_report.merge(mt_report_alpha, strict=False)
mt_report.merge(mt_report_beta_1, strict=False)
mt_report.merge(mt_report_beta_2, strict=False)
mt_report.merge(mt_report_gamma, strict=False)
check_report(actual=mt_report, expected=expected_report)
def test_multitest_tagging(mockplan, multitest_tags, expected_report):
multitest = MultiTest(
name="MyMultitest",
suites=[AlphaSuite(), BetaSuite(),
GammaSuite()],
tags=multitest_tags,
)
mockplan.add(multitest)
mockplan.run()
check_report(
expected=TestReport(name="plan", entries=[expected_report]),
actual=mockplan.report,
)
def test_report_json_binary_serialization(
dummy_test_plan_report_with_binary_asserts, ):
"""JSON Serialized & deserialized reports should be equal."""
test_plan_schema = TestReportSchema(strict=True)
data = test_plan_schema.dumps(
dummy_test_plan_report_with_binary_asserts).data
j = json.loads(data)
bkey = EntriesField._BYTES_KEY
# passing assertion
hx_1_1 = get_path(j, "entries.1.entries.0.entries.1.first")[bkey]
hx_1_2 = get_path(j, "entries.1.entries.0.entries.1.second")[bkey]
assert ["0xF2"] == hx_1_1 == hx_1_2
# failing assertion
hx_2_1 = get_path(j, "entries.1.entries.0.entries.2.first")[bkey]
hx_2_2 = get_path(j, "entries.1.entries.0.entries.2.second")[bkey]
assert ["0x00", "0xB1", "0xC1"] == hx_2_1
assert ["0x00", "0xB2", "0xC2"] == hx_2_2
# dict.match the schema for that producing list of tuples
KEY_INDEX = 1
FIRST_INDEX = 3
SECOND_INDEX = 4
comps = get_path(j, "entries.1.entries.0.entries.3.comparison")
assert comps[0][KEY_INDEX][bkey] == EntriesField._binary_to_hex_list(
b"binarykey\xB1")
assert comps[1][FIRST_INDEX][1][bkey] == EntriesField._binary_to_hex_list(
b"binary value\xB1")
assert comps[1][SECOND_INDEX][1][bkey] == EntriesField._binary_to_hex_list(
b"binary value\xB1")
assert comps[3][FIRST_INDEX][1][bkey] == EntriesField._binary_to_hex_list(
b"binary\xB1")
assert comps[3][SECOND_INDEX][1][bkey] == EntriesField._binary_to_hex_list(
b"binary\xB1")
assert comps[7][FIRST_INDEX][1][bkey] == EntriesField._binary_to_hex_list(
b"binary\xB1")
assert comps[7][SECOND_INDEX][1][bkey] == EntriesField._binary_to_hex_list(
b"binary\xB1")
deserialized_report = test_plan_schema.loads(data).data
check_report(
actual=deserialized_report,
expected=dummy_test_plan_report_with_binary_asserts,
)
def test_pre_post_steps(mockplan):
multitest = MultiTest(
name="MyMultitest",
suites=[MySuite()],
before_start=check_func_1,
after_start=check_func_2,
before_stop=check_func_3,
after_stop=check_func_4,
)
mockplan.add(multitest)
mockplan.run()
check_report(expected_report, mockplan.report)
assert len(mockplan.report.attachments) == 2
def test_pre_post_steps():
multitest = MultiTest(name='MyMultitest',
suites=[MySuite()],
before_start=check_func_1,
after_start=check_func_2,
before_stop=check_func_3,
after_stop=check_func_4)
plan = Testplan(name='plan', parse_cmdline=False)
plan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
plan.run()
check_report(expected_report, plan.report)
def test_multitest_tagging(multitest_tags, expected_report):
multitest = MultiTest(name='MyMultitest',
suites=[AlphaSuite(),
BetaSuite(),
GammaSuite()],
tags=multitest_tags)
plan = Testplan(name='plan', parse_cmdline=False)
plan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
plan.run()
check_report(expected=TestReport(name='plan', entries=[expected_report]),
actual=plan.report)
def test_cppunit(mockplan, binary_dir, expected_report, report_status):
binary_path = os.path.join(binary_dir, "runTests")
if not os.path.exists(binary_path):
msg = BINARY_NOT_FOUND_MESSAGE.format(binary_dir=binary_dir,
binary_path=binary_path)
pytest.skip(msg)
mockplan.add(Cppunit(name="My Cppunit", binary=binary_path))
assert mockplan.run().run is True
check_report(expected=expected_report, actual=mockplan.report)
assert mockplan.report.status == report_status
def test_gtest(mockplan, binary_dir, expected_report, report_status):
binary_path = os.path.join(binary_dir, "runTests")
if not os.path.exists(binary_path):
msg = BINARY_NOT_FOUND_MESSAGE.format(binary_dir=binary_dir,
binary_path=binary_path)
pytest.skip(msg)
mockplan.add(GTest(name="My GTest", binary=binary_path))
with log_propagation_disabled(TESTPLAN_LOGGER):
assert mockplan.run().run is True
check_report(expected=expected_report, actual=mockplan.report)
assert mockplan.report.status == report_status
def test_pre_post_steps(mockplan):
multitest = MultiTest(
name="MyMultitest",
suites=[MySuite()],
before_start=check_func_1,
after_start=check_func_2,
before_stop=check_func_3,
after_stop=check_func_4,
)
mockplan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
mockplan.run()
check_report(expected_report, mockplan.report)
def test_process_pool_integration(
report_dir, fixture_dirname,
expected_report, pdf_title,
expected_plan_result, dependant_module
):
if dependant_module:
importorxfail(dependant_module)
pool = ProcessPool(name='MyPool', size=1)
pdf_path = report_dir.join('test_report_process_{}.pdf'.format(
pdf_title)).strpath
plan = Testplan(
name='plan',
parse_cmdline=False,
exporters=[
PDFExporter(pdf_path=pdf_path)
]
)
plan.add_resource(pool)
runners_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
fixture_path = os.path.join(runners_path, 'fixtures', fixture_dirname)
task = Task(
target='make_multitest',
module='suites',
path=fixture_path,
)
plan.schedule(task, resource='MyPool')
assert not os.path.exists(pdf_path)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
for log in plan.report.flattened_logs:
if all(word in log['message'] for word in ['tkinter', 'TclError']):
pytest.xfail(reason='Tkinter not installed properly')
check_report(expected=expected_report, actual=plan.report)
assert plan.result.success is expected_plan_result
assert os.path.exists(pdf_path)
assert os.stat(pdf_path).st_size > 0
def test_empty_pre_post_steps(mockplan):
"""
Runs a MultiTest without an empty after_start, expects it present and passing.
"""
multitest = MultiTest(
name="MyMultiTest",
suites=[MySuite()],
after_start=check_func_2,
)
mockplan.add(multitest)
mockplan.run()
expected_report = TestReport(
name="plan",
entries=[
TestGroupReport(
name="MyMultiTest",
category=ReportCategories.MULTITEST,
entries=[
TestGroupReport(
name="Before/After Step Checks",
category=ReportCategories.TESTSUITE,
entries=[
TestCaseReport(name="after_start - check_func_2"),
],
),
TestGroupReport(
name="MySuite",
category=ReportCategories.TESTSUITE,
entries=[
TestCaseReport(name="test_one"),
TestCaseReport(
name="teardown",
entries=[{"type": "Attachment"}],
),
],
),
],
),
],
)
assert mockplan.report["MyMultiTest"].status == Status.PASSED
assert mockplan.report.status == Status.PASSED
check_report(expected_report, mockplan.report)
def test_multitest_tagging(mockplan, multitest_tags, expected_report):
multitest = MultiTest(
name="MyMultitest",
suites=[AlphaSuite(), BetaSuite(),
GammaSuite()],
tags=multitest_tags,
)
mockplan.add(multitest)
with log_propagation_disabled(TESTPLAN_LOGGER):
mockplan.run()
check_report(
expected=TestReport(name="plan", entries=[expected_report]),
actual=mockplan.report,
)
def test_process_pool_integration(
runpath,
fixture_dirname,
expected_report,
pdf_title,
expected_plan_result,
dependant_module,
):
if dependant_module:
importorxfail(dependant_module)
pool = ProcessPool(name="MyProcessPool", size=1)
pdf_path = os.path.join(
runpath, "test_report_local_{}.pdf".format(pdf_title)
)
plan = TestplanMock(
name="plan",
exporters=[PDFExporter(pdf_path=pdf_path)],
runpath=runpath,
)
plan.add_resource(pool)
runners_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
fixture_path = os.path.join(runners_path, "fixtures", fixture_dirname)
task = Task(target="make_multitest", module="suites", path=fixture_path)
plan.schedule(task, resource="MyProcessPool")
assert not os.path.exists(pdf_path)
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
for log in plan.report.flattened_logs:
if all(word in log["message"] for word in ["tkinter", "TclError"]):
pytest.xfail(reason="Tkinter not installed properly")
check_report(expected=expected_report, actual=plan.report)
assert plan.result.success is expected_plan_result
assert os.path.exists(pdf_path)
assert os.stat(pdf_path).st_size > 0
def test_hobbestest(mockplan, binary_dir, expected_report):
binary_path = os.path.join(binary_dir, "hobbes-test")
if not os.path.exists(binary_path):
msg = BINARY_NOT_FOUND_MESSAGE.format(binary_dir=binary_dir,
binary_path=binary_path)
pytest.skip(msg)
mockplan.add(
HobbesTest(
name="My HobbesTest",
binary=binary_path,
tests=["Hog", "Net", "Recursives"],
))
assert mockplan.run().run is True
check_report(expected=expected_report, actual=mockplan.report)
def test_run_test(mockplan):
mockplan.add(
junit.JUnit(
name="My Junit",
description="Junit example test",
binary=JUNIT_FAKE_BIN,
junit_args=["test"],
results_dir=REPORT_PATH,
proc_cwd=CURRENT_PATH,
))
assert mockplan.run().run is True
report = mockplan.report
assert report.status == testplan.report.Status.FAILED
mt_report = report.entries[0]
assert len(mt_report.entries) == 3
check_report(expect_report, mt_report)
def test_gtest(binary_dir, expected_report):
binary_path = os.path.join(binary_dir, 'runTests')
if not os.path.exists(binary_path):
msg = BINARY_NOT_FOUND_MESSAGE.format(binary_dir=binary_dir,
binary_path=binary_path)
pytest.skip(msg)
plan = Testplan(
name='plan',
parse_cmdline=False,
)
plan.add(GTest(name='MyGTest', driver=binary_path))
with log_propagation_disabled(TESTPLAN_LOGGER):
assert plan.run().run is True
check_report(expected=expected_report, actual=plan.report)
