def test_check_stability_iterations():
logger = StructuredLogger("test-stability")
logger.add_handler(StreamHandler(sys.stdout, TbplFormatter()))
kwargs = {"verify_log_full": False}
def mock_run_tests(**kwargs):
repeats = kwargs.get("repeat", 1)
for _ in range(repeats):
logger.suite_start(tests=[], name="test")
for _ in range(kwargs.get("rerun", 1)):
logger.test_start("/example/test.html")
logger.test_status("/example/test.html",
subtest="test1",
status="PASS")
logger.test_end("/example/test.html", status="OK")
logger.suite_end()
status = wptrunner.TestStatus()
status.total_tests = 1
status.repeated_runs = repeats
status.expected_repeated_runs = repeats
return (None, status)
# Don't actually load wptrunner, because that will end up starting a browser
# which we don't want to do in this test.
with mock.patch("wptrunner.stability.wptrunner.run_tests") as mock_run:
mock_run.side_effect = mock_run_tests
assert stability.check_stability(logger,
repeat_loop=10,
repeat_restart=5,
chaos_mode=False,
output_results=False,
**kwargs) is None
def run_step(logger, iterations, restart_after_iteration, kwargs_extras,
**kwargs):
import wptrunner
kwargs = copy.deepcopy(kwargs)
if restart_after_iteration:
kwargs["repeat"] = iterations
else:
kwargs["rerun"] = iterations
kwargs["pause_after_test"] = False
kwargs.update(kwargs_extras)
handler = LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()), "WARNING"),
["log", "process_output"])
# There is a public API for this in the next mozlog
initial_handlers = logger._state.handlers
logger._state.handlers = []
with open("raw.log", "wb") as log:
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
logger.add_handler(handler)
logger.add_handler(StreamHandler(log, JSONFormatter()))
wptrunner.run_tests(**kwargs)
logger._state.handlers = initial_handlers
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, iterations)
return results, inconsistent, iterations
def run(venv, logger, **kwargs):
kwargs["pause_after_test"] = False
if kwargs["repeat"] == 1:
kwargs["repeat"] = 10
handler = LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()), "WARNING"),
["log", "process_output"])
# There is a public API for this in the next mozlog
initial_handlers = logger._state.handlers
logger._state.handlers = []
with open("raw.log", "wb") as log:
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
logger.add_handler(handler)
logger.add_handler(StreamHandler(log, JSONFormatter()))
wptrunner.run_tests(**kwargs)
logger._state.handlers = initial_handlers
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, kwargs["repeat"])
return kwargs["repeat"], results, inconsistent
def main():
retcode = 0
parser = get_parser()
args = parser.parse_args()
if not os.path.exists(args.root):
logger.critical("Root directory %s does not exist" % args.root)
return 1
os.chdir(args.root)
if args.gh_token:
gh_handler = setup_github_logging(args)
else:
logger.warning("Can't log to GitHub")
gh_handler = None
print >> sys.stderr, "travis_fold:start:browser_setup"
logger.info("# %s #" % args.browser.title())
browser_cls = {"firefox": Firefox, "chrome": Chrome}.get(args.browser)
if browser_cls is None:
logger.critical("Unrecognised browser %s" % args.browser)
return 1
fetch_wpt_master()
head_sha1 = get_sha1()
logger.info("Testing revision %s" % head_sha1)
# For now just pass the whole list of changed files to wptrunner and
# assume that it will run everything that's actually a test
files_changed = get_files_changed()
if not files_changed:
logger.info("No files changed")
return 0
build_manifest()
install_wptrunner()
do_delayed_imports()
logger.debug("Files changed:\n%s" % "".join(" * %s\n" % item
for item in files_changed))
browser = browser_cls(args.gh_token)
browser.install()
browser.install_webdriver()
kwargs = wptrunner_args(args.root, files_changed, args.iterations, browser)
print >> sys.stderr, "travis_fold:end:browser_setup"
print >> sys.stderr, "travis_fold:start:running_tests"
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs, {"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()),
"WARNING"), ["log", "process_output"]))
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
print >> sys.stderr, "travis_fold:end:running_tests"
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
retcode = 2
else:
logger.info("All results were stable\n")
print >> sys.stderr, "travis_fold:start:full_results"
write_results(results, args.iterations)
print >> sys.stderr, "travis_fold:end:full_results"
else:
logger.info("No tests run.")
try:
if gh_handler:
gh_handler.send()
except Exception:
logger.error(traceback.format_exc())
return retcode
files_changed.extend(affected_testfiles)
kwargs = wptrunner_args(args.root, files_changed, args.iterations,
browser)
browser.prepare_environment()
with TravisFold("running_tests"):
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs, {"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()),
"WARNING"), ["log", "process_output"]))
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(results, args.iterations, args.comment_pr)
def get_formatter(self):
return TbplFormatter(summary_on_shutdown=True)
def get_formatter(self):
return TbplFormatter()
browser.prepare_environment()
with TravisFold("running_tests"):
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs,
{"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(
StreamHandler(
sys.stdout,
TbplFormatter()
),
"WARNING"),
["log", "process_output"]))
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
retcode = 2
else:
logger.info("All results were stable\n")