def test_output_fail(runtests):
formatter = pytest.importorskip("output").ReftestFormatter()
status, lines = runtests("reftest-fail.list")
assert status == 0
buf = StringIO()
tbpl_status, log_level, summary = get_mozharness_status(
lines, status, formatter=formatter, buf=buf
)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
test_status = filter_action("test_status", lines)
assert len(test_status) == 3
assert all(t["status"] == "FAIL" for t in test_status)
assert all("reftest_screenshots" in t["extra"] for t in test_status)
test_end = filter_action("test_end", lines)
assert len(test_end) == 3
assert all(t["status"] == "OK" for t in test_end)
# ensure screenshots were printed
formatted = buf.getvalue()
assert "REFTEST IMAGE 1" in formatted
assert "REFTEST IMAGE 2" in formatted
def test_output_assertion(flavor, runFailures, runtests, test_name):
extra_opts = {}
results = {
"status": 0,
"tbpl_status": TBPL_WARNING,
"log_level": WARNING,
"lines": 1,
"assertions": 1,
}
status, lines = runtests(test_name("assertion"), **extra_opts)
# TODO: mochitest should return non-zero here
assert status == results["status"]
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == results["tbpl_status"]
assert log_level == results["log_level"]
test_end = filter_action("test_end", lines)
assert len(test_end) == results["lines"]
# TODO: this should be ASSERT, but moving the assertion check before
# the test_end action caused a bunch of failures.
assert test_end[0]["status"] == "OK"
assertions = filter_action("assertion_count", lines)
assert len(assertions) == results["assertions"]
assert assertions[0]["count"] == results["assertions"]
def test_output_fail(runtests):
formatter = pytest.importorskip('output').ReftestFormatter()
status, lines = runtests('reftest-fail.list')
assert status == 0
buf = StringIO()
tbpl_status, log_level, summary = get_mozharness_status(
lines, status, formatter=formatter, buf=buf)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
test_status = filter_action('test_status', lines)
assert len(test_status) == 3
assert all(t['status'] == 'FAIL' for t in test_status)
assert all('reftest_screenshots' in t['extra'] for t in test_status)
test_end = filter_action('test_end', lines)
assert len(test_end) == 3
assert all(t['status'] == 'OK' for t in test_end)
# ensure screenshots were printed
formatted = buf.getvalue()
assert 'REFTEST IMAGE 1' in formatted
assert 'REFTEST IMAGE 2' in formatted
def test_output_asan(runtests):
status, lines = runtests('reftest-crash.list', environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"])
assert status == 0
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_FAILURE
assert log_level == ERROR
crash = filter_action('crash', lines)
assert len(crash) == 0
process_output = filter_action('process_output', lines)
assert any('ERROR: AddressSanitizer' in l['data'] for l in process_output)
def test_output_asan(runtests):
status, lines = runtests('test_crash.html', environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"])
assert status == 1
tbpl_status, log_level = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_FAILURE
assert log_level == ERROR
crash = filter_action('crash', lines)
assert len(crash) == 0
process_output = filter_action('process_output', lines)
assert any('ERROR: AddressSanitizer' in l['data'] for l in process_output)
def test_output_asan(runtests):
status, lines = runtests("test_crash.html",
environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"])
assert status == 1
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_FAILURE
assert log_level == ERROR
crash = filter_action("crash", lines)
assert len(crash) == 0
process_output = filter_action("process_output", lines)
assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output)
def test_output_asan(runtests):
status, lines = runtests('test_crash.html',
environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"])
# TODO: mochitest should return non-zero here
assert status == 0
tbpl_status, log_level = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_FAILURE
assert log_level == ERROR
crash = filter_action('crash', lines)
assert len(crash) == 0
process_output = filter_action('process_output', lines)
assert any('ERROR: AddressSanitizer' in l['data'] for l in process_output)
def test_output_pass(runtests):
status, lines = runtests("reftest-pass.list")
assert status == 0
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_SUCCESS
assert log_level in (INFO, WARNING)
test_status = filter_action("test_status", lines)
assert len(test_status) == 3
assert all(t["status"] == "PASS" for t in test_status)
test_end = filter_action("test_end", lines)
assert len(test_end) == 3
assert all(t["status"] == "OK" for t in test_end)
def test_output_pass(runtests):
status, lines = runtests('reftest-pass.list')
assert status == 0
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_SUCCESS
assert log_level in (INFO, WARNING)
test_status = filter_action('test_status', lines)
assert len(test_status) == 3
assert all(t['status'] == 'PASS' for t in test_status)
test_end = filter_action('test_end', lines)
assert len(test_end) == 3
assert all(t['status'] == 'OK' for t in test_end)
def test_output_crash(runtests):
status, lines = runtests('test_crash.html', environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"])
assert status == 1
tbpl_status, log_level = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_FAILURE
assert log_level == ERROR
crash = filter_action('crash', lines)
assert len(crash) == 1
assert crash[0]['action'] == 'crash'
assert crash[0]['signature']
assert crash[0]['minidump_path']
lines = filter_action('test_end', lines)
assert len(lines) == 0
def test_output_leak(monkeypatch, runtests):
# Monkeypatch mozleak so we always process a failing leak log
# instead of the actual one.
import mozleak
old_process_leak_log = mozleak.process_leak_log
def process_leak_log(*args, **kwargs):
return old_process_leak_log(
os.path.join(here, "files", "leaks.log"), *args[1:], **kwargs
)
monkeypatch.setattr("mozleak.process_leak_log", process_leak_log)
status, lines = runtests("reftest-pass.list")
assert status == 0
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
leaks = filter_action("mozleak_total", lines)
assert len(leaks) == 1
assert leaks[0]["process"] == "default"
assert leaks[0]["bytes"] == 19915
def test_output_leak(flavor, runFailures, runtests, test_name):
extra_opts = {}
results = {"status": 0, "tbpl_status": TBPL_WARNING, "log_level": WARNING}
status, lines = runtests(test_name("leak"), **extra_opts)
# TODO: mochitest should return non-zero here
assert status == results["status"]
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == results["tbpl_status"]
assert log_level == results["log_level"]
leak_totals = filter_action("mozleak_total", lines)
found_leaks = False
for lt in leak_totals:
if lt["bytes"] == 0:
# No leaks in this process.
assert len(lt["objects"]) == 0
continue
assert not found_leaks, "Only one process should have leaked"
found_leaks = True
assert lt["process"] == "tab"
assert lt["bytes"] == 1
assert lt["objects"] == ["IntentionallyLeakedObject"]
assert found_leaks, "At least one process should have leaked"
def test_output_crash(runtests):
status, lines = runtests('reftest-crash.list', environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"])
assert status == 1
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_FAILURE
assert log_level == ERROR
crash = filter_action('crash', lines)
assert len(crash) == 1
assert crash[0]['action'] == 'crash'
assert crash[0]['signature']
assert crash[0]['minidump_path']
lines = filter_action('test_end', lines)
assert len(lines) == 0
def test_output_crash(runtests):
status, lines = runtests("test_crash.html",
environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"])
assert status == 1
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_FAILURE
assert log_level == ERROR
crash = filter_action("crash", lines)
assert len(crash) == 1
assert crash[0]["action"] == "crash"
assert crash[0]["signature"]
assert crash[0]["minidump_path"]
lines = filter_action("test_end", lines)
assert len(lines) == 0
def test_output_assertion(runtests):
status, lines = runtests('test_assertion.html')
# TODO: mochitest should return non-zero here
assert status == 0
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
test_end = filter_action('test_end', lines)
assert len(test_end) == 1
# TODO: this should be ASSERT, but moving the assertion check before
# the test_end action caused a bunch of failures.
assert test_end[0]['status'] == 'OK'
assertions = filter_action('assertion_count', lines)
assert len(assertions) == 1
assert assertions[0]['count'] == 1
def test_output_assertion(runtests):
status, lines = runtests('test_assertion.html')
# TODO: mochitest should return non-zero here
assert status == 0
tbpl_status, log_level = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
test_end = filter_action('test_end', lines)
assert len(test_end) == 1
# TODO: this should be ASSERT, but moving the assertion check before
# the test_end action caused a bunch of failures.
assert test_end[0]['status'] == 'OK'
assertions = filter_action('assertion_count', lines)
assert len(assertions) == 1
assert assertions[0]['count'] == 1
def test_output_assertion(runtests):
status, lines = runtests("reftest-assert.list")
assert status == 0
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
test_status = filter_action("test_status", lines)
assert len(test_status) == 1
assert test_status[0]["status"] == "PASS"
test_end = filter_action("test_end", lines)
assert len(test_end) == 1
assert test_end[0]["status"] == "OK"
assertions = filter_action("assertion_count", lines)
assert len(assertions) == 1
assert assertions[0]["count"] == 1
def test_output_assertion(runtests):
status, lines = runtests('reftest-assert.list')
assert status == 0
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
test_status = filter_action('test_status', lines)
assert len(test_status) == 1
assert test_status[0]['status'] == 'PASS'
test_end = filter_action('test_end', lines)
assert len(test_end) == 1
assert test_end[0]['status'] == 'OK'
assertions = filter_action('assertion_count', lines)
assert len(assertions) == 1
assert assertions[0]['count'] == 1
def test_output_fail(runtests):
status, lines = runtests('reftest-fail.list')
assert status == 0
tbpl_status, log_level = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
test_end = filter_action('test_end', lines)
assert len(test_end) == 3
assert all(t['status'] == 'FAIL' for t in test_end)
def test_output_pass(runtests):
status, lines = runtests('test_pass.html')
assert status == 0
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_SUCCESS
assert log_level in (INFO, WARNING)
lines = filter_action('test_status', lines)
assert len(lines) == 1
assert lines[0]['status'] == 'PASS'
def test_output_pass(runtests):
status, lines = runtests('test_pass.html')
assert status == 0
tbpl_status, log_level = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_SUCCESS
assert log_level in (INFO, WARNING)
lines = filter_action('test_status', lines)
assert len(lines) == 1
assert lines[0]['status'] == 'PASS'
def test_output_fail(runtests):
status, lines = runtests('test_fail.html')
assert status == 1
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
lines = filter_action('test_status', lines)
assert len(lines) == 1
assert lines[0]['status'] == 'FAIL'
def test_output_fail(runtests):
status, lines = runtests('test_fail.html')
assert status == 1
tbpl_status, log_level = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
lines = filter_action('test_status', lines)
assert len(lines) == 1
assert lines[0]['status'] == 'FAIL'
def test_output_asan(flavor, runFailures, runtests, test_name):
extra_opts = {}
results = {
"status": 1,
"tbpl_status": TBPL_FAILURE,
"log_level": ERROR,
"lines": 0
}
status, lines = runtests(test_name("crash"),
environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"],
**extra_opts)
assert status == results["status"]
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == results["tbpl_status"]
assert log_level == results["log_level"]
crash = filter_action("crash", lines)
assert len(crash) == results["lines"]
process_output = filter_action("process_output", lines)
assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output)
def test_output_crash(flavor, runFailures, runtests, test_name):
extra_opts = {}
results = {
"status": 0 if runFailures else 1,
"tbpl_status": TBPL_FAILURE,
"log_level": ERROR,
"lines": 1 if runFailures else 0,
}
if runFailures:
extra_opts["runFailures"] = runFailures
extra_opts["crashAsPass"] = True
extra_opts["timeoutAsPass"] = True
# bug 1443327 - we do not set MOZ_CRASHREPORTER_SHUTDOWN for browser-chrome
# the error regex's don't pick this up as a failure
if flavor == "browser-chrome":
results["tbpl_status"] = TBPL_SUCCESS
results["log_level"] = (INFO, WARNING)
status, lines = runtests(test_name("crash"),
environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"],
**extra_opts)
assert status == results["status"]
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == results["tbpl_status"]
assert log_level in results["log_level"]
if not runFailures:
crash = filter_action("crash", lines)
assert len(crash) == 1
assert crash[0]["action"] == "crash"
assert crash[0]["signature"]
assert crash[0]["minidump_path"]
lines = filter_action("test_end", lines)
assert len(lines) == results["lines"]
def test_output_leak(monkeypatch, runtests):
# Monkeypatch mozleak so we always process a failing leak log
# instead of the actual one.
import mozleak
old_process_leak_log = mozleak.process_leak_log
def process_leak_log(*args, **kwargs):
return old_process_leak_log(os.path.join(here, 'files', 'leaks.log'), *args[1:], **kwargs)
monkeypatch.setattr('mozleak.process_leak_log', process_leak_log)
status, lines = runtests('reftest-pass.list')
assert status == 0
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_FAILURE
assert log_level == ERROR
errors = filter_action('log', lines)
errors = [e for e in errors if e['level'] == 'ERROR']
assert len(errors) == 1
assert 'leakcheck' in errors[0]['message']
def test_output_leak(monkeypatch, runtests):
# Monkeypatch mozleak so we always process a failing leak log
# instead of the actual one.
import mozleak
old_process_leak_log = mozleak.process_leak_log
def process_leak_log(*args, **kwargs):
return old_process_leak_log(os.path.join(here, 'files', 'leaks.log'), *args[1:], **kwargs)
monkeypatch.setattr('mozleak.process_leak_log', process_leak_log)
status, lines = runtests('test_pass.html')
# TODO: mochitest should return non-zero here
assert status == 0
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
leaks = filter_action('mozleak_total', lines)
assert len(leaks) == 1
assert leaks[0]['process'] == "default"
assert leaks[0]['bytes'] == 19915
def test_output_leak(monkeypatch, runtests):
# Monkeypatch mozleak so we always process a failing leak log
# instead of the actual one.
import mozleak
old_process_leak_log = mozleak.process_leak_log
def process_leak_log(*args, **kwargs):
return old_process_leak_log(os.path.join(here, 'files', 'leaks.log'), *args[1:], **kwargs)
monkeypatch.setattr('mozleak.process_leak_log', process_leak_log)
status, lines = runtests('test_pass.html')
# TODO: mochitest should return non-zero here
assert status == 0
tbpl_status, log_level = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_FAILURE
assert log_level == ERROR
errors = filter_action('log', lines)
errors = [e for e in errors if e['level'] == 'ERROR']
assert len(errors) == 1
assert 'leakcheck' in errors[0]['message']
def test_output_fail(runtests):
from runtests import build_obj
status, lines = runtests('test_fail.html')
assert status == 1
tbpl_status, log_level = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
lines = filter_action('test_status', lines)
# If we are running with a build_obj, the failed status will be
# logged a second time at the end of the run.
if build_obj:
assert len(lines) == 2
else:
assert len(lines) == 1
assert lines[0]['status'] == 'FAIL'
if build_obj:
assert set(lines[0].keys()) == set(lines[1].keys())
assert set(lines[0].values()) == set(lines[1].values())
def test_output_fail(flavor, runFailures, runtests, test_name):
extra_opts = {}
results = {
"status": 0 if runFailures else 1,
"tbpl_status": TBPL_SUCCESS if runFailures else TBPL_WARNING,
"log_level": (INFO, WARNING),
"lines": 1,
"line_status": "PASS" if runFailures else "FAIL",
}
if runFailures:
extra_opts["runFailures"] = runFailures
extra_opts["crashAsPass"] = True
extra_opts["timeoutAsPass"] = True
status, lines = runtests(test_name("fail"), **extra_opts)
assert status == results["status"]
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == results["tbpl_status"]
assert log_level in results["log_level"]
lines = filter_action("test_status", lines)
assert len(lines) == results["lines"]
assert lines[0]["status"] == results["line_status"]
def test_output_leak(runtests):
status, lines = runtests('test_leak.html')
# TODO: mochitest should return non-zero here
assert status == 0
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
leak_totals = filter_action('mozleak_total', lines)
found_leaks = False
for lt in leak_totals:
if lt['bytes'] == 0:
# No leaks in this process.
assert len(lt['objects']) == 0
continue
assert not found_leaks, "Only one process should have leaked"
found_leaks = True
assert lt['process'] == "tab"
assert lt['bytes'] == 1
assert lt['objects'] == ['IntentionallyLeakedObject']
assert found_leaks, "At least one process should have leaked"
