def run(venv, logger, **kwargs):
kwargs["pause_after_test"] = False
if kwargs["repeat"] == 1:
kwargs["repeat"] = 10
handler = LogActionFilter(
LogLevelFilter(
StreamHandler(
sys.stdout,
TbplFormatter()
),
"WARNING"),
["log", "process_output"])
# There is a public API for this in the next mozlog
initial_handlers = logger._state.handlers
logger._state.handlers = []
with open("raw.log", "wb") as log:
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
logger.add_handler(handler)
logger.add_handler(StreamHandler(log, JSONFormatter()))
wptrunner.run_tests(**kwargs)
logger._state.handlers = initial_handlers
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, kwargs["repeat"])
return kwargs["repeat"], results, inconsistent
def main():
retcode = 0
parser = get_parser()
args = parser.parse_args()
gh_handler = setup_github_logging(args)
logger.info("Testing in **%s**" % args.browser.title())
browser_cls = {"firefox": Firefox,
"chrome": Chrome}.get(args.browser)
if browser_cls is None:
logger.critical("Unrecognised browser %s" % args.browser)
return 2
# For now just pass the whole list of changed files to wptrunner and
# assume that it will run everything that's actually a test
files_changed = get_files_changed(args.root)
if not files_changed:
return 0
logger.info("Files changed:\n%s" % "".join(" * %s\n" % item for item in files_changed))
browser = browser_cls()
kwargs = wptrunner_args(args.root,
files_changed,
args.iterations,
browser)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs,
{"mach": sys.stdout,
"raw": log})
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
if results:
if inconsistent:
write_inconsistent(inconsistent)
retcode = 1
else:
logger.info("All results were stable\n")
write_results(results, args.iterations)
else:
logger.info("No tests run.")
try:
if gh_handler:
gh_handler.send()
except Exception:
logger.error(traceback.format_exc())
return retcode
def run_tests(paths=None, **kwargs):
if paths is None:
paths = {}
set_defaults(paths, kwargs)
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
success = wptrunner.run_tests(**kwargs)
return 0 if success else 1
def run_tests(self, **kwargs):
from wptrunner import wptrunner
self.setup_kwargs(kwargs)
logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
result = wptrunner.run_tests(**kwargs)
return int(not result)
def run_tests(paths=None, **kwargs):
if paths is None:
paths = {}
set_defaults(paths, kwargs)
mozlog.commandline.log_formatters["servo"] = \
(grouping_formatter.GroupingFormatter, "A grouping output formatter")
wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
success = wptrunner.run_tests(**kwargs)
return 0 if success else 1
def run_tests(**kwargs):
set_defaults(kwargs)
mozlog.commandline.log_formatters["servo"] = \
(grouping_formatter.GroupingFormatter, "A grouping output formatter")
use_mach_logging = False
if len(kwargs["test_list"]) == 1:
file_ext = os.path.splitext(kwargs["test_list"][0])[1].lower()
if file_ext in [".htm", ".html", ".js", ".xhtml", ".xht", ".py"]:
use_mach_logging = True
if use_mach_logging:
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
else:
wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
success = wptrunner.run_tests(**kwargs)
return 0 if success else 1
def run_tests(**kwargs):
set_defaults(kwargs)
mozlog.commandline.log_formatters["servo"] = \
(grouping_formatter.GroupingFormatter, "A grouping output formatter")
use_mach_logging = False
if len(kwargs["test_list"]) == 1:
file_ext = os.path.splitext(kwargs["test_list"][0])[1].lower()
if file_ext in [".htm", ".html", ".js", ".xhtml", ".xht", ".py"]:
use_mach_logging = True
if use_mach_logging:
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
else:
wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
success = wptrunner.run_tests(**kwargs)
return 0 if success else 1
def main():
retcode = 0
parser = get_parser()
args = parser.parse_args()
if not os.path.exists(args.root):
logger.critical("Root directory %s does not exist" % args.root)
return 1
os.chdir(args.root)
if args.gh_token:
gh_handler = setup_github_logging(args)
else:
logger.warning("Can't log to GitHub")
gh_handler = None
print >> sys.stderr, "travis_fold:start:browser_setup"
logger.info("# %s #" % args.browser.title())
browser_cls = {"firefox": Firefox, "chrome": Chrome}.get(args.browser)
if browser_cls is None:
logger.critical("Unrecognised browser %s" % args.browser)
return 1
fetch_wpt_master()
head_sha1 = get_sha1()
logger.info("Testing revision %s" % head_sha1)
# For now just pass the whole list of changed files to wptrunner and
# assume that it will run everything that's actually a test
files_changed = get_files_changed()
if not files_changed:
logger.info("No files changed")
return 0
build_manifest()
install_wptrunner()
do_delayed_imports()
logger.debug("Files changed:\n%s" % "".join(" * %s\n" % item
for item in files_changed))
browser = browser_cls(args.gh_token)
browser.install()
browser.install_webdriver()
kwargs = wptrunner_args(args.root, files_changed, args.iterations, browser)
print >> sys.stderr, "travis_fold:end:browser_setup"
print >> sys.stderr, "travis_fold:start:running_tests"
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs, {"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()),
"WARNING"), ["log", "process_output"]))
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
print >> sys.stderr, "travis_fold:end:running_tests"
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
retcode = 2
else:
logger.info("All results were stable\n")
print >> sys.stderr, "travis_fold:start:full_results"
write_results(results, args.iterations)
print >> sys.stderr, "travis_fold:end:full_results"
else:
logger.info("No tests run.")
try:
if gh_handler:
gh_handler.send()
except Exception:
logger.error(traceback.format_exc())
return retcode
browser)
browser.prepare_environment()
with TravisFold("running_tests"):
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs, {"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()),
"WARNING"), ["log", "process_output"]))
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(results, args.iterations, args.comment_pr)
else:
logger.info("No tests run.")
def run_tests(product, kwargs):
kwargs["test_paths"]["/_test/"] = {"tests_path": os.path.join(here, "testdata"),
"metadata_path": os.path.join(here, "metadata")}
wptrunner.run_tests(**kwargs)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs,
{"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(
StreamHandler(
sys.stdout,
TbplFormatter()
),
"WARNING"),
["log", "process_output"]))
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(results, args.iterations, args.comment_pr)
else:
logger.info("No tests run.")
def run_tests(product, kwargs):
kwargs["test_paths"]["/_test/"] = {"tests_path": os.path.join(here, "testdata"),
"metadata_path": os.path.join(here, "metadata")}
wptrunner.run_tests(**kwargs)
def main():
retcode = 0
parser = get_parser()
args = parser.parse_args()
if not os.path.exists(args.root):
logger.critical("Root directory %s does not exist" % args.root)
return 1
os.chdir(args.root)
if args.gh_token:
gh_handler = setup_github_logging(args)
else:
logger.warning("Can't log to GitHub")
gh_handler = None
with TravisFold("browser_setup"):
logger.info("# %s #" % args.browser.title())
browser_cls = {"firefox": Firefox, "chrome": Chrome}.get(args.browser)
if browser_cls is None:
logger.critical("Unrecognised browser %s" % args.browser)
return 1
fetch_wpt_master()
head_sha1 = get_sha1()
logger.info("Testing revision %s" % head_sha1)
# For now just pass the whole list of changed files to wptrunner and
# assume that it will run everything that's actually a test
files_changed = get_files_changed()
if not files_changed:
logger.info("No files changed")
return 0
build_manifest()
install_wptrunner()
do_delayed_imports()
logger.debug("Files changed:\n%s" % "".join(" * %s\n" % item for item in files_changed))
affected_testfiles = get_affected_testfiles(files_changed)
logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in affected_testfiles))
files_changed.extend(affected_testfiles)
browser = browser_cls(args.gh_token)
browser.install()
browser.install_webdriver()
kwargs = wptrunner_args(args.root, files_changed, args.iterations, browser)
with TravisFold("running_tests"):
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs, {"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()), "WARNING"), ["log", "process_output"]
)
)
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(results, args.iterations, args.comment_pr)
else:
logger.info("No tests run.")
try:
if gh_handler:
gh_handler.send()
except Exception:
logger.error(traceback.format_exc())
return retcode
def run_tests(**kwargs):
wptrunner.setup_logging(kwargs, {"raw": sys.stdout})
return wptrunner.run_tests(**kwargs)
def run_tests(**kwargs):
if not os.path.isfile(wptsubdir("metadata", "MANIFEST.json")):
raise Exception("Manifest not found. Please use --update-manifest in WPTARGS to create one")
wptrunner.setup_logging(kwargs, {"raw": sys.stdout})
return wptrunner.run_tests(**kwargs)
def run_tests(**kwargs):
wptrunner.setup_logging(kwargs, {"raw": sys.stdout})
return wptrunner.run_tests(**kwargs)
