def test_num_failures_by_type(capfd):
# Test that the number of failures by status type is correctly calculated.
# Set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# Run some tests with different statuses: 3 passes, 1 timeout
logger.suite_start(["t1", "t2", "t3", "t4"], run_info={}, time=123)
logger.test_start("t1")
logger.test_end("t1", status="PASS", expected="PASS")
logger.test_start("t2")
logger.test_end("t2", status="PASS", expected="PASS")
logger.test_start("t3")
logger.test_end("t3", status="PASS", expected="FAIL")
logger.test_start("t4")
logger.test_end("t4", status="TIMEOUT", expected="CRASH")
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
num_failures_by_type = json.load(output)["num_failures_by_type"]
# We expect 3 passes and 1 timeout, nothing else.
assert sorted(num_failures_by_type.keys()) == ["PASS", "TIMEOUT"]
assert num_failures_by_type["PASS"] == 3
assert num_failures_by_type["TIMEOUT"] == 1
def test_subtest_failure(capfd):
# Tests that a test fails if a subtest fails
# Set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
formatter = ChromiumFormatter()
logger.add_handler(handlers.StreamHandler(output, formatter))
# Run a test with some subtest failures.
logger.suite_start(["t1"], run_info={}, time=123)
logger.test_start("t1")
logger.test_status("t1",
status="FAIL",
subtest="t1_a",
message="t1_a_message")
logger.test_status("t1",
status="PASS",
subtest="t1_b",
message="t1_b_message")
logger.test_status("t1",
status="TIMEOUT",
subtest="t1_c",
message="t1_c_message")
# Make sure the test name was added to the set of tests with subtest fails
assert "t1" in formatter.tests_with_subtest_fails
# The test status is reported as a pass here because the harness was able to
# run the test to completion.
logger.test_end("t1", status="PASS", expected="PASS")
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
test_obj = output_json["tests"]["t1"]
t1_artifacts = test_obj["artifacts"]
assert t1_artifacts["log"] == [
"[t1_a] [FAIL expected PASS] t1_a_message",
"[t1_b] [PASS expected PASS] t1_b_message",
"[t1_c] [TIMEOUT expected PASS] t1_c_message",
"[] [FAIL expected PASS] ",
]
assert t1_artifacts["wpt_subtest_failure"] == ["true"]
# The status of the test in the output is a failure because subtests failed,
# despite the harness reporting that the test passed. But the harness status
# is logged as an artifact.
assert t1_artifacts["wpt_actual_status"] == ["PASS"]
assert test_obj["actual"] == "FAIL"
assert test_obj["expected"] == "PASS"
# Also ensure that the formatter cleaned up its internal state
assert "t1" not in formatter.tests_with_subtest_fails
def test_chromium_required_fields(capfd):
# Test that the test results contain a handful of required fields.
# Set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# output a bunch of stuff
logger.suite_start(["test-id-1"], run_info={}, time=123)
logger.test_start("test-id-1")
logger.test_end("test-id-1", status="PASS", expected="PASS")
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_obj = json.load(output)
# Check for existence of required fields
assert "interrupted" in output_obj
assert "path_delimiter" in output_obj
assert "version" in output_obj
assert "num_failures_by_type" in output_obj
assert "tests" in output_obj
test_obj = output_obj["tests"]["test-id-1"]
assert "actual" in test_obj
assert "expected" in test_obj
def test_precondition_failed(capfd):
# Check that a failed precondition gets properly handled.
# set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# Run a test with a precondition failure
logger.suite_start(["t1"], run_info={}, time=123)
logger.test_start("t1")
logger.test_end("t1", status="PRECONDITION_FAILED", expected="OK")
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
test_obj = output_json["tests"]["t1"]
# The precondition failure should map to FAIL status, but we should also
# have an artifact containing the original PRECONDITION_FAILED status.
assert test_obj["actual"] == "FAIL"
assert test_obj["artifacts"]["wpt_actual_status"] == [
"PRECONDITION_FAILED"
]
# ...this is an unexpected regression because we expected a pass but failed
assert test_obj["is_regression"] is True
assert test_obj["is_unexpected"] is True
def test_unexpected_test_fail(capfd):
# Check that an unexpected test-level failure is marked as unexpected and
# as a regression.
# Set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# Run some tests with different statuses: 3 passes, 1 timeout
logger.suite_start(["t1"], run_info={}, time=123)
logger.test_start("t1")
logger.test_end("t1", status="ERROR", expected="OK")
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
test_obj = output_json["tests"]["t1"]
# The test's actual and expected status should be mapped, ERROR->FAIL and
# OK->PASS
assert test_obj["actual"] == "FAIL"
assert test_obj["expected"] == "PASS"
# ..and this test should be a regression and unexpected
assert test_obj["is_regression"] is True
assert test_obj["is_unexpected"] is True
def test_known_intermittent_empty(capfd):
# If the known_intermittent list is empty, we want to ensure we don't append
# any extraneous characters to the output.
# set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# Run a test and include an empty known_intermittent list
logger.suite_start(["t1"], run_info={}, time=123)
logger.test_start("t1")
logger.test_end("t1", status="OK", expected="OK", known_intermittent=[])
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
test_obj = output_json["tests"]["t1"]
# Both actual and expected statuses get mapped to Pass. No extra whitespace
# anywhere.
assert test_obj["actual"] == "PASS"
assert test_obj["expected"] == "PASS"
def test_expected_test_fail(logger, capfd):
# Check that an expected test-level failure is treated as a Pass
# Set up the handler.
output = StringIO()
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# Run some tests with different statuses: 3 passes, 1 timeout
logger.suite_start(["t1"], run_info={}, time=123)
logger.test_start("t1")
logger.test_end("t1", status="ERROR", expected="ERROR")
logger.suite_end()
logger.shutdown()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
test_obj = output_json["tests"]["t1"]
# The test's actual and expected status should map from "ERROR" to "FAIL"
assert test_obj["actual"] == "FAIL"
assert test_obj["expected"] == "FAIL"
# ..and this test should not be a regression nor unexpected
assert "is_regression" not in test_obj
assert "is_unexpected" not in test_obj
def test_known_intermittent_duplicate(capfd):
# We don't want to have duplicate statuses in the final "expected" field.
# Set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# There are two duplications in this input:
# 1. known_intermittent already contains expected;
# 2. both statuses in known_intermittent map to FAIL in Chromium.
# In the end, we should only get one FAIL in Chromium "expected".
logger.suite_start(["t1"], run_info={}, time=123)
logger.test_start("t1")
logger.test_end("t1",
status="ERROR",
expected="ERROR",
known_intermittent=["FAIL", "ERROR"])
logger.suite_end()
# Check nothing got output to stdout/stderr.
# (Note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# Check the actual output of the formatter.
output.seek(0)
output_json = json.load(output)
test_obj = output_json["tests"]["t1"]
assert test_obj["actual"] == "FAIL"
# No duplicate "FAIL" in "expected".
assert test_obj["expected"] == "FAIL"
def test_flaky_test_unexpected(capfd):
# Check that a flaky test with multiple possible statuses is seen as
# unexpected if its actual status is NOT one of the possible ones.
# set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# Run a test that is known to be flaky
logger.suite_start(["t1"], run_info={}, time=123)
logger.test_start("t1")
logger.test_end("t1", status="ERROR", expected="OK", known_intermittent=["TIMEOUT"])
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
test_obj = output_json["tests"]["t1"]
# The test's statuses are all mapped, changing ERROR->FAIL and OK->PASS
assert test_obj["actual"] == "FAIL"
# All the possible statuses are concatenated together into expected.
assert test_obj["expected"] == "PASS TIMEOUT"
# ...this is a regression and unexpected because the actual status is not
# one of the expected ones
assert test_obj["is_regression"] is True
assert test_obj["is_unexpected"] is True
def test_subtest_messages(capfd):
# Tests accumulation of test output
# Set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# Run two tests with subtest messages. The subtest name should be included
# in the output. We should also tolerate missing messages.
logger.suite_start(["t1", "t2"], run_info={}, time=123)
logger.test_start("t1")
logger.test_status("t1",
status="FAIL",
subtest="t1_a",
message="t1_a_message")
logger.test_status("t1",
status="PASS",
subtest="t1_b",
message="t1_b_message")
logger.test_end("t1", status="PASS", expected="PASS")
logger.test_start("t2")
# Subtests with empty messages should not be ignored.
logger.test_status("t2", status="PASS", subtest="t2_a")
# A test-level message will also be appended
logger.test_end("t2",
status="TIMEOUT",
expected="PASS",
message="t2_message")
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
t1_artifacts = output_json["tests"]["t1"]["artifacts"]
assert t1_artifacts["wpt_actual_metadata"] == [
"[t1]\n expected: PASS\n",
" [t1_a]\n expected: FAIL\n",
" [t1_b]\n expected: PASS\n",
]
assert t1_artifacts["wpt_log"] == [
"t1_a: t1_a_message\n",
"t1_b: t1_b_message\n",
]
assert t1_artifacts["wpt_subtest_failure"] == ["true"]
t2_artifacts = output_json["tests"]["t2"]["artifacts"]
assert t2_artifacts["wpt_actual_metadata"] == [
"[t2]\n expected: TIMEOUT\n",
" [t2_a]\n expected: PASS\n",
]
assert t2_artifacts["wpt_log"] == ["Harness: t2_message\n"]
assert "wpt_subtest_failure" not in t2_artifacts.keys()
def test_expected_subtest_failure(capfd):
# Tests that an expected subtest failure does not cause the test to fail
# Set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
formatter = ChromiumFormatter()
logger.add_handler(handlers.StreamHandler(output, formatter))
# Run a test with some expected subtest failures.
logger.suite_start(["t1"], run_info={}, time=123)
logger.test_start("t1")
logger.test_status("t1",
status="FAIL",
expected="FAIL",
subtest="t1_a",
message="t1_a_message")
logger.test_status("t1",
status="PASS",
subtest="t1_b",
message="t1_b_message")
logger.test_status("t1",
status="TIMEOUT",
expected="TIMEOUT",
subtest="t1_c",
message="t1_c_message")
# The subtest failures are all expected so this test should not be added to
# the set of tests with subtest failures.
assert "t1" not in formatter.tests_with_subtest_fails
# The test status is reported as a pass here because the harness was able to
# run the test to completion.
logger.test_end("t1", status="OK", expected="OK")
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
test_obj = output_json["tests"]["t1"]
t1_log = test_obj["artifacts"]["log"]
print("Lpz t1log=%s" % t1_log)
assert t1_log == [
"[t1]\n expected: OK\n",
" [t1_a]\n expected: FAIL\n message: t1_a_message\n",
" [t1_b]\n expected: PASS\n message: t1_b_message\n",
" [t1_c]\n expected: TIMEOUT\n message: t1_c_message\n",
]
# The status of the test in the output is a pass because the subtest
# failures were all expected.
assert test_obj["actual"] == "PASS"
assert test_obj["expected"] == "PASS"
def test_process_output_crashing_test(capfd):
"""Test that chromedriver logs are preserved for crashing tests"""
# Set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
logger.suite_start(["t1", "t2", "t3"], run_info={}, time=123)
logger.test_start("t1")
logger.process_output(100, "This message should be recorded",
"/some/path/to/chromedriver --some-flag")
logger.process_output(101, "This message should not be recorded",
"/some/other/process --another-flag")
logger.process_output(100, "This message should also be recorded",
"/some/path/to/chromedriver --some-flag")
logger.test_end("t1", status="CRASH", expected="CRASH")
logger.test_start("t2")
logger.process_output(100, "Another message for the second test",
"/some/path/to/chromedriver --some-flag")
logger.test_end("t2", status="CRASH", expected="PASS")
logger.test_start("t3")
logger.process_output(100, "This test fails",
"/some/path/to/chromedriver --some-flag")
logger.process_output(100, "But the output should not be captured",
"/some/path/to/chromedriver --some-flag")
logger.process_output(100, "Because it does not crash",
"/some/path/to/chromedriver --some-flag")
logger.test_end("t3", status="FAIL", expected="PASS")
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
test_obj = output_json["tests"]["t1"]
assert test_obj["artifacts"]["wpt_crash_log"] == [
"This message should be recorded",
"This message should also be recorded"
]
test_obj = output_json["tests"]["t2"]
assert test_obj["artifacts"]["wpt_crash_log"] == [
"Another message for the second test"
]
test_obj = output_json["tests"]["t3"]
assert "wpt_crash_log" not in test_obj["artifacts"]
def test_unexpected_subtest_pass(capfd):
# A subtest that unexpectedly passes is considered a failure condition.
# Set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
formatter = ChromiumFormatter()
logger.add_handler(handlers.StreamHandler(output, formatter))
# Run a test with a subtest that is expected to fail but passes.
logger.suite_start(["t1"], run_info={}, time=123)
logger.test_start("t1")
logger.test_status("t1",
status="PASS",
expected="FAIL",
subtest="t1_a",
message="t1_a_message")
# Since the subtest behaviour is unexpected, it's considered a failure, so
# the test should be added to the set of tests with subtest failures.
assert "t1" in formatter.tests_with_subtest_fails
# The test status is reported as a pass here because the harness was able to
# run the test to completion.
logger.test_end("t1", status="PASS", expected="PASS")
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
test_obj = output_json["tests"]["t1"]
t1_artifacts = test_obj["artifacts"]
assert t1_artifacts["wpt_actual_metadata"] == [
"[t1]\n expected: PASS\n",
" [t1_a]\n expected: PASS\n",
]
assert t1_artifacts["wpt_log"] == [
"t1_a: t1_a_message\n",
]
assert t1_artifacts["wpt_subtest_failure"] == ["true"]
# Since the subtest status is unexpected, we fail the test. But we report
# wpt_actual_status as an artifact
assert t1_artifacts["wpt_actual_status"] == ["PASS"]
assert test_obj["actual"] == "FAIL"
assert test_obj["expected"] == "PASS"
# Also ensure that the formatter cleaned up its internal state
assert "t1" not in formatter.tests_with_subtest_fails
def test_subtest_messages(capfd):
# Tests accumulation of test output
# Set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# Run two tests with subtest messages. The subtest name should be included
# in the output. We should also tolerate missing messages.
logger.suite_start(["t1", "t2"], run_info={}, time=123)
logger.test_start("t1")
logger.test_status("t1",
status="FAIL",
subtest="t1_a",
message="t1_a_message")
logger.test_status("t1",
status="PASS",
subtest="t1_b",
message="t1_b_message")
logger.test_end("t1", status="PASS", expected="PASS")
logger.test_start("t2")
# Currently, subtests with empty messages will be ignored
logger.test_status("t2", status="PASS", subtest="t2_a")
# A test-level message will also be appended
logger.test_end("t2",
status="TIMEOUT",
expected="PASS",
message="t2_message")
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
t1_log = output_json["tests"]["t1"]["artifacts"]["log"]
assert t1_log == "[FAIL] t1_a: t1_a_message\n" \
"[PASS] t1_b: t1_b_message\n"
t2_log = output_json["tests"]["t2"]["artifacts"]["log"]
assert t2_log == "[TIMEOUT] t2_message\n"
def test_repeated_test_statuses(logger, capfd):
# Check that the logger outputs all statuses from multiple runs of a test.
# Set up the handler.
output = StringIO()
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# Run a test suite for the first time.
logger.suite_start(["t1"], run_info={}, time=123)
logger.test_start("t1")
logger.test_end("t1",
status="PASS",
expected="PASS",
known_intermittent=[])
logger.suite_end()
# Run the test suite for the second time.
logger.suite_start(["t1"], run_info={}, time=456)
logger.test_start("t1")
logger.test_end("t1",
status="FAIL",
expected="PASS",
known_intermittent=[])
logger.suite_end()
# Write the final results.
logger.shutdown()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
status_totals = output_json["num_failures_by_type"]
assert status_totals["PASS"] == 1
# A missing result type is the same as being present and set to zero (0).
assert status_totals.get("FAIL", 0) == 0
# The actual statuses are accumulated in a ordered space-separated list.
test_obj = output_json["tests"]["t1"]
assert test_obj["actual"] == "PASS FAIL"
assert test_obj["expected"] == "PASS"
def test_reftest_screenshots(capfd):
# reftest_screenshots, if present, should be plumbed into artifacts.
# Set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# Run a reftest with reftest_screenshots.
logger.suite_start(["t1"], run_info={}, time=123)
logger.test_start("t1")
logger.test_end("t1",
status="FAIL",
expected="PASS",
extra={
"reftest_screenshots": [
{
"url": "foo.html",
"hash": "HASH1",
"screenshot": "DATA1"
},
"!=",
{
"url": "foo-ref.html",
"hash": "HASH2",
"screenshot": "DATA2"
},
]
})
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
test_obj = output_json["tests"]["t1"]
assert test_obj["artifacts"]["screenshots"] == [
"foo.html: DATA1",
"foo-ref.html: DATA2",
]
def test_time_per_test(logger, capfd):
# Test that the formatter measures time per test correctly.
# Set up the handler.
output = StringIO()
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
logger.suite_start(["test-id-1", "test-id-2"], run_info={}, time=50)
logger.test_start("test-id-1", time=100)
logger.test_start("test-id-2", time=200)
logger.test_end("test-id-1", status="PASS", expected="PASS", time=300)
logger.test_end("test-id-2", status="PASS", expected="PASS", time=199)
logger.suite_end()
logger.suite_start(["test-id-1"], run_info={}, time=400)
logger.test_start("test-id-1", time=500)
logger.test_end("test-id-1", status="PASS", expected="PASS", time=600)
logger.suite_end()
# Write the final results.
logger.shutdown()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_obj = json.load(output)
test1_obj = output_obj["tests"]["test-id-1"]
test2_obj = output_obj["tests"]["test-id-2"]
# Test 1 run 1: 300ms - 100ms = 0.2s
# Test 1 run 2: 600ms - 500ms = 0.1s
assert test1_obj["time"] == pytest.approx(0.2)
assert len(test1_obj["times"]) == 2
assert test1_obj["times"][0] == pytest.approx(0.2)
assert test1_obj["times"][1] == pytest.approx(0.1)
assert "time" not in test2_obj
assert "times" not in test2_obj
def test_repeated_runs(logger, capfd):
# Check that the logger outputs one report after multiple test suite runs.
# Set up the handler.
output = StringIO()
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# Run a test suite for the first time.
logger.suite_start(["t1"], run_info={}, time=123)
logger.test_start("t1")
logger.test_end("t1",
status="PASS",
expected="PASS",
known_intermittent=[])
logger.suite_end()
# Run the test suite for the second time.
logger.suite_start(["t1"], run_info={}, time=456)
logger.test_start("t1")
logger.test_end("t1",
status="FAIL",
expected="PASS",
known_intermittent=[])
logger.suite_end()
logger.shutdown()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
# The latest status (FAIL) replaces the earlier one (PASS).
test_obj = output_json["tests"]["t1"]
assert test_obj["actual"] == "FAIL"
assert test_obj["expected"] == "PASS"
def test_flaky_test_detection(logger, capfd):
# Check that the logger detects flakiness for a test run multiple times.
# Set up the handler.
output = StringIO()
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
logger.suite_start(["t1", "t2"], run_info={})
logger.test_start("t1")
logger.test_start("t2")
logger.test_end("t1", status="FAIL", expected="PASS")
logger.test_end("t2", status="FAIL", expected="FAIL")
logger.suite_end()
logger.suite_start(["t1", "t2"], run_info={})
logger.test_start("t1")
logger.test_start("t2")
logger.test_end("t1", status="PASS", expected="PASS")
logger.test_end("t2", status="FAIL", expected="FAIL")
logger.suite_end()
# Write the final results.
logger.shutdown()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_json = json.load(output)
# We consider a test flaky if it runs multiple times and produces more than
# one kind of result.
test1_obj = output_json["tests"]["t1"]
test2_obj = output_json["tests"]["t2"]
assert test1_obj["is_flaky"] is True
assert "is_flaky" not in test2_obj
def test_chromium_test_name_trie(capfd):
# Ensure test names are broken into directories and stored in a trie with
# test results at the leaves.
# Set up the handler.
output = StringIO()
logger = structuredlog.StructuredLogger("test_a")
logger.add_handler(handlers.StreamHandler(output, ChromiumFormatter()))
# output a bunch of stuff
logger.suite_start(["/foo/bar/test-id-1", "/foo/test-id-2"],
run_info={},
time=123)
logger.test_start("/foo/bar/test-id-1")
logger.test_end("/foo/bar/test-id-1", status="TIMEOUT", expected="FAIL")
logger.test_start("/foo/test-id-2")
logger.test_end("/foo/test-id-2", status="ERROR", expected="TIMEOUT")
logger.suite_end()
# check nothing got output to stdout/stderr
# (note that mozlog outputs exceptions during handling to stderr!)
captured = capfd.readouterr()
assert captured.out == ""
assert captured.err == ""
# check the actual output of the formatter
output.seek(0)
output_obj = json.load(output)
# Ensure that the test names are broken up by directory name and that the
# results are stored at the leaves.
test_obj = output_obj["tests"]["foo"]["bar"]["test-id-1"]
assert test_obj["actual"] == "TIMEOUT"
assert test_obj["expected"] == "FAIL"
test_obj = output_obj["tests"]["foo"]["test-id-2"]
# The ERROR status is mapped to FAIL for Chromium
assert test_obj["actual"] == "FAIL"
assert test_obj["expected"] == "TIMEOUT"
