def setup_wptrunner(venv, prompt=True, install=False, **kwargs):
from wptrunner import wptrunner, wptcommandline
global logger
kwargs = utils.Kwargs(kwargs.iteritems())
product_parts = kwargs["product"].split(":")
kwargs["product"] = product_parts[0]
sub_product = product_parts[1:]
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
logger = wptrunner.logger
check_environ(kwargs["product"])
args_general(kwargs)
if kwargs["product"] not in product_setup:
raise WptrunError("Unsupported product %s" % kwargs["product"])
setup_cls = product_setup[kwargs["product"]](venv, prompt, sub_product)
if install:
logger.info("Installing browser")
kwargs["binary"] = setup_cls.install(venv)
setup_cls.setup(kwargs)
wptcommandline.check_args(kwargs)
wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner")
venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt"))
return kwargs
def run_tests(paths=None, **kwargs):
if paths is None:
paths = {}
set_defaults(paths, kwargs)
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
success = wptrunner.run_tests(**kwargs)
return 0 if success else 1
def run_tests(paths=None, **kwargs):
if paths is None:
paths = {}
set_defaults(paths, kwargs)
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
success = wptrunner.run_tests(**kwargs)
return 0 if success else 1
def run_tests(**kwargs):
if not os.path.isfile(wptsubdir("metadata", "MANIFEST.json")):
raise Exception(
"Manifest not found. Please use --update-manifest in WPTARGS to create one"
)
wptrunner.setup_logging(kwargs, {"raw": sys.stdout})
return wptrunner.run_tests(**kwargs)
def setup_wptrunner(venv, product, tests, wptrunner_args, prompt=True,):
global logger
wptparser = wptcommandline.create_parser()
kwargs = utils.Kwargs(vars(wptparser.parse_args(wptrunner_args)).iteritems())
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
logger = wptrunner.logger
kwargs["product"] = product
kwargs["test_list"] = tests
check_environ(product)
args_general(kwargs)
if product not in product_setup:
exit("Unsupported product %s" % product)
product_setup[product](venv, kwargs, prompt)
wptcommandline.check_args(kwargs)
wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner")
venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt"))
return kwargs
def setup_wptrunner(venv, prompt=True, install=False, **kwargs):
from wptrunner import wptrunner, wptcommandline
global logger
kwargs = utils.Kwargs(kwargs.iteritems())
product_parts = kwargs["product"].split(":")
kwargs["product"] = product_parts[0]
sub_product = product_parts[1:]
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
logger = wptrunner.logger
check_environ(kwargs["product"])
args_general(kwargs)
if kwargs["product"] not in product_setup:
raise WptrunError("Unsupported product %s" % kwargs["product"])
setup_cls = product_setup[kwargs["product"]](venv, prompt, sub_product)
if install:
logger.info("Installing browser")
kwargs["binary"] = setup_cls.install(venv)
setup_cls.setup(kwargs)
wptcommandline.check_args(kwargs)
wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner")
venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt"))
return kwargs
def setup_wptrunner(venv, prompt=True, install_browser=False, **kwargs):
from wptrunner import wptrunner, wptcommandline
import mozlog
global logger
kwargs = utils.Kwargs(kwargs.iteritems())
product_parts = kwargs["product"].split(":")
kwargs["product"] = product_parts[0]
sub_product = product_parts[1:]
# Use the grouped formatter by default where mozlog 3.9+ is installed
if hasattr(mozlog.formatters, "GroupingFormatter"):
default_formatter = "grouped"
else:
default_formatter = "mach"
wptrunner.setup_logging(kwargs, {default_formatter: sys.stdout})
logger = wptrunner.logger
check_environ(kwargs["product"])
args_general(kwargs)
if kwargs["product"] not in product_setup:
raise WptrunError("Unsupported product %s" % kwargs["product"])
setup_cls = product_setup[kwargs["product"]](venv, prompt, sub_product)
setup_cls.install_requirements()
if install_browser and not kwargs["channel"]:
logger.info("--install-browser is given but --channel is not set, default to nightly channel")
kwargs["channel"] = "nightly"
if kwargs["channel"]:
channel = install.get_channel(kwargs["product"], kwargs["channel"])
if channel is not None:
if channel != kwargs["channel"]:
logger.info("Interpreting channel '%s' as '%s'" % (kwargs["channel"],
channel))
kwargs["browser_channel"] = channel
else:
logger.info("Valid channels for %s not known; using argument unmodified" % kwargs["product"])
del kwargs["channel"]
if install_browser:
logger.info("Installing browser")
kwargs["binary"] = setup_cls.install(venv, channel=channel)
setup_cls.setup(kwargs)
wptcommandline.check_args(kwargs)
wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner")
venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt"))
kwargs['browser_version'] = setup_cls.browser.version(binary=kwargs.get("binary"),
webdriver_binary=kwargs.get("webdriver_binary"))
return kwargs
def setup_wptrunner(venv, prompt=True, install_browser=False, **kwargs):
from wptrunner import wptrunner, wptcommandline
import mozlog
global logger
kwargs = utils.Kwargs(kwargs.iteritems())
product_parts = kwargs["product"].split(":")
kwargs["product"] = product_parts[0]
sub_product = product_parts[1:]
# Use the grouped formatter by default where mozlog 3.9+ is installed
if hasattr(mozlog.formatters, "GroupingFormatter"):
default_formatter = "grouped"
else:
default_formatter = "mach"
wptrunner.setup_logging(kwargs, {default_formatter: sys.stdout})
logger = wptrunner.logger
check_environ(kwargs["product"])
args_general(kwargs)
if kwargs["product"] not in product_setup:
raise WptrunError("Unsupported product %s" % kwargs["product"])
setup_cls = product_setup[kwargs["product"]](venv, prompt, sub_product)
setup_cls.install_requirements()
if install_browser and not kwargs["channel"]:
logger.info("--install-browser is given but --channel is not set, default to nightly channel")
kwargs["channel"] = "nightly"
if kwargs["channel"]:
channel = install.get_channel(kwargs["product"], kwargs["channel"])
if channel != kwargs["channel"]:
logger.info("Interpreting channel '%s' as '%s'" % (kwargs["channel"],
channel))
kwargs["browser_channel"] = channel
del kwargs["channel"]
if install_browser:
logger.info("Installing browser")
kwargs["binary"] = setup_cls.install(venv, channel=channel)
setup_cls.setup(kwargs)
wptcommandline.check_args(kwargs)
wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner")
venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt"))
kwargs['browser_version'] = setup_cls.browser.version(kwargs.get("binary"))
return kwargs
def run_tests(paths=None, **kwargs):
if paths is None:
paths = {}
set_defaults(paths, kwargs)
mozlog.commandline.log_formatters["servo"] = \
(grouping_formatter.GroupingFormatter, "A grouping output formatter")
wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
success = wptrunner.run_tests(**kwargs)
return 0 if success else 1
def run_tests(paths=None, **kwargs):
if paths is None:
paths = {}
set_defaults(paths, kwargs)
mozlog.commandline.log_formatters["servo"] = \
(grouping_formatter.GroupingFormatter, "A grouping output formatter")
wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
success = wptrunner.run_tests(**kwargs)
return 0 if success else 1
def main():
retcode = 0
parser = get_parser()
args = parser.parse_args()
gh_handler = setup_github_logging(args)
logger.info("Testing in **%s**" % args.browser.title())
browser_cls = {"firefox": Firefox,
"chrome": Chrome}.get(args.browser)
if browser_cls is None:
logger.critical("Unrecognised browser %s" % args.browser)
return 2
# For now just pass the whole list of changed files to wptrunner and
# assume that it will run everything that's actually a test
files_changed = get_files_changed(args.root)
if not files_changed:
return 0
logger.info("Files changed:\n%s" % "".join(" * %s\n" % item for item in files_changed))
browser = browser_cls()
kwargs = wptrunner_args(args.root,
files_changed,
args.iterations,
browser)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs,
{"mach": sys.stdout,
"raw": log})
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
if results:
if inconsistent:
write_inconsistent(inconsistent)
retcode = 1
else:
logger.info("All results were stable\n")
write_results(results, args.iterations)
else:
logger.info("No tests run.")
try:
if gh_handler:
gh_handler.send()
except Exception:
logger.error(traceback.format_exc())
return retcode
def setup_logging(kwargs):
import mozlog
from wptrunner import wptrunner
global logger
# Use the grouped formatter by default where mozlog 3.9+ is installed
if hasattr(mozlog.formatters, "GroupingFormatter"):
default_formatter = "grouped"
else:
default_formatter = "mach"
wptrunner.setup_logging(kwargs, {default_formatter: sys.stdout})
logger = wptrunner.logger
def setup_logging(kwargs):
import mozlog
from wptrunner import wptrunner
global logger
# Use the grouped formatter by default where mozlog 3.9+ is installed
if hasattr(mozlog.formatters, "GroupingFormatter"):
default_formatter = "grouped"
else:
default_formatter = "mach"
wptrunner.setup_logging(kwargs, {default_formatter: sys.stdout})
logger = wptrunner.logger
def setup_wptrunner(venv, prompt=True, install_browser=False, **kwargs):
from wptrunner import wptrunner, wptcommandline
global logger
kwargs = utils.Kwargs(kwargs.iteritems())
product_parts = kwargs["product"].split(":")
kwargs["product"] = product_parts[0]
sub_product = product_parts[1:]
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
logger = wptrunner.logger
check_environ(kwargs["product"])
args_general(kwargs)
if kwargs["product"] not in product_setup:
raise WptrunError("Unsupported product %s" % kwargs["product"])
setup_cls = product_setup[kwargs["product"]](venv, prompt, sub_product)
setup_cls.install_requirements()
if install_browser and not kwargs["channel"]:
logger.info(
"--install-browser is given but --channel is not set, default to nightly channel"
)
kwargs["channel"] = "nightly"
if kwargs["channel"]:
channel = install.get_channel(kwargs["product"], kwargs["channel"])
if channel != kwargs["channel"]:
logger.info("Interpreting channel '%s' as '%s'" %
(kwargs["channel"], channel))
kwargs["browser_channel"] = channel
del kwargs["channel"]
if install_browser:
logger.info("Installing browser")
kwargs["binary"] = setup_cls.install(venv, channel=channel)
setup_cls.setup(kwargs)
wptcommandline.check_args(kwargs)
wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner")
venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt"))
kwargs['browser_version'] = setup_cls.browser.version(kwargs.get("binary"))
return kwargs
def setup_wptrunner(venv, prompt=True, install_browser=False, **kwargs):
from wptrunner import wptrunner, wptcommandline
global logger
kwargs = utils.Kwargs(kwargs.iteritems())
product_parts = kwargs["product"].split(":")
kwargs["product"] = product_parts[0]
sub_product = product_parts[1:]
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
logger = wptrunner.logger
check_environ(kwargs["product"])
args_general(kwargs)
if kwargs["product"] not in product_setup:
raise WptrunError("Unsupported product %s" % kwargs["product"])
setup_cls = product_setup[kwargs["product"]](venv, prompt, sub_product)
setup_cls.install_requirements()
if install_browser and not kwargs["channel"]:
kwargs["channel"] = "nightly"
if kwargs["channel"]:
channel = install.get_channel(kwargs["product"], kwargs["channel"])
if channel != kwargs["channel"]:
logger.info("Interpreting channel '%s' as '%s'" % (kwargs["channel"],
channel))
kwargs["browser_channel"] = channel
del kwargs["channel"]
if install_browser:
logger.info("Installing browser")
kwargs["binary"] = setup_cls.install(venv, channel=channel)
setup_cls.setup(kwargs)
wptcommandline.check_args(kwargs)
wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner")
venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt"))
kwargs['browser_version'] = setup_cls.browser.version(kwargs.get("binary"))
return kwargs
def setup_logging(kwargs, default_config=None):
import mozlog
from wptrunner import wptrunner
global logger
# Use the grouped formatter by default where mozlog 3.9+ is installed
if default_config is None:
if hasattr(mozlog.formatters, "GroupingFormatter"):
default_formatter = "grouped"
else:
default_formatter = "mach"
default_config = {default_formatter: sys.stdout}
wptrunner.setup_logging(kwargs, default_config)
logger = wptrunner.logger
return logger
def setup_logging(kwargs, default_config=None):
import mozlog
from wptrunner import wptrunner
global logger
# Use the grouped formatter by default where mozlog 3.9+ is installed
if default_config is None:
if hasattr(mozlog.formatters, "GroupingFormatter"):
default_formatter = "grouped"
else:
default_formatter = "mach"
default_config = {default_formatter: sys.stdout}
wptrunner.setup_logging(kwargs, default_config)
logger = wptrunner.logger
return logger
def run_tests(self, **kwargs):
from wptrunner import wptrunner
self.setup_kwargs(kwargs)
logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
result = wptrunner.run_tests(**kwargs)
return int(not result)
def run_tests(self, **kwargs):
from wptrunner import wptrunner
self.setup_kwargs(kwargs)
logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
result = wptrunner.run_tests(**kwargs)
return int(not result)
def run_tests(**kwargs):
set_defaults(kwargs)
mozlog.commandline.log_formatters["servo"] = \
(grouping_formatter.GroupingFormatter, "A grouping output formatter")
use_mach_logging = False
if len(kwargs["test_list"]) == 1:
file_ext = os.path.splitext(kwargs["test_list"][0])[1].lower()
if file_ext in [".htm", ".html", ".js", ".xhtml", ".xht", ".py"]:
use_mach_logging = True
if use_mach_logging:
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
else:
wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
success = wptrunner.run_tests(**kwargs)
return 0 if success else 1
def run_tests(**kwargs):
set_defaults(kwargs)
mozlog.commandline.log_formatters["servo"] = \
(grouping_formatter.GroupingFormatter, "A grouping output formatter")
use_mach_logging = False
if len(kwargs["test_list"]) == 1:
file_ext = os.path.splitext(kwargs["test_list"][0])[1].lower()
if file_ext in [".htm", ".html", ".js", ".xhtml", ".xht", ".py"]:
use_mach_logging = True
if use_mach_logging:
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
else:
wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
success = wptrunner.run_tests(**kwargs)
return 0 if success else 1
def run(self, **kwargs):
from wptrunner import wptrunner
if kwargs["product"] in ["firefox", None]:
kwargs = self.setup.kwargs_firefox(kwargs)
elif kwargs["product"] in ("chrome", "edge", "servo"):
kwargs = self.setup.kwargs_wptrun(kwargs)
else:
raise ValueError("Unknown product %s" % kwargs["product"])
logger = wptrunner.setup_logging(kwargs, {self.setup.default_log_type: sys.stdout})
result = wptrunner.start(**kwargs)
return int(not result)
def run_tests(self, **kwargs):
from wptrunner import wptrunner
if kwargs["product"] in ["firefox", None]:
self.setup_kwargs_firefox(kwargs)
elif kwargs["product"] in ("chrome", "edge", "servo"):
self.setup_kwargs_wptrun(kwargs)
else:
raise ValueError("Unknown product %s" % kwargs["product"])
logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
result = wptrunner.run_tests(**kwargs)
return int(not result)
def run_tests(**kwargs):
wptrunner.setup_logging(kwargs, {"raw": sys.stdout})
return wptrunner.run_tests(**kwargs)
logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in affected_testfiles))
files_changed.extend(affected_testfiles)
kwargs = wptrunner_args(args.root,
files_changed,
args.iterations,
browser)
browser.prepare_environment()
with TravisFold("running_tests"):
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs,
{"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(
StreamHandler(
sys.stdout,
TbplFormatter()
),
"WARNING"),
["log", "process_output"]))
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
def run_tests(**kwargs):
if not os.path.isfile(wptsubdir("metadata", "MANIFEST.json")):
raise Exception("Manifest not found. Please use --update-manifest in WPTARGS to create one")
wptrunner.setup_logging(kwargs, {"raw": sys.stdout})
return wptrunner.run_tests(**kwargs)
affected_testfiles = get_affected_testfiles(files_changed, skip_tests)
logger.debug("Affected tests:\n%s" %
"".join(" * %s\n" % item for item in affected_testfiles))
files_changed.extend(affected_testfiles)
kwargs = wptrunner_args(args.root, files_changed, args.iterations,
browser)
browser.prepare_environment()
with TravisFold("running_tests"):
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs, {"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()),
"WARNING"), ["log", "process_output"]))
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
def setup_logging(self, **kwargs):
from wptrunner import wptrunner
return wptrunner.setup_logging(
kwargs, {self.setup.default_log_type: sys.stdout})
def run_tests(**kwargs):
wptrunner.setup_logging(kwargs, {"raw": sys.stdout})
return wptrunner.run_tests(**kwargs)
def main():
retcode = 0
parser = get_parser()
args = parser.parse_args()
if not os.path.exists(args.root):
logger.critical("Root directory %s does not exist" % args.root)
return 1
os.chdir(args.root)
if args.gh_token:
gh_handler = setup_github_logging(args)
else:
logger.warning("Can't log to GitHub")
gh_handler = None
print >> sys.stderr, "travis_fold:start:browser_setup"
logger.info("# %s #" % args.browser.title())
browser_cls = {"firefox": Firefox, "chrome": Chrome}.get(args.browser)
if browser_cls is None:
logger.critical("Unrecognised browser %s" % args.browser)
return 1
fetch_wpt_master()
head_sha1 = get_sha1()
logger.info("Testing revision %s" % head_sha1)
# For now just pass the whole list of changed files to wptrunner and
# assume that it will run everything that's actually a test
files_changed = get_files_changed()
if not files_changed:
logger.info("No files changed")
return 0
build_manifest()
install_wptrunner()
do_delayed_imports()
logger.debug("Files changed:\n%s" % "".join(" * %s\n" % item
for item in files_changed))
browser = browser_cls(args.gh_token)
browser.install()
browser.install_webdriver()
kwargs = wptrunner_args(args.root, files_changed, args.iterations, browser)
print >> sys.stderr, "travis_fold:end:browser_setup"
print >> sys.stderr, "travis_fold:start:running_tests"
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs, {"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()),
"WARNING"), ["log", "process_output"]))
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
print >> sys.stderr, "travis_fold:end:running_tests"
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
retcode = 2
else:
logger.info("All results were stable\n")
print >> sys.stderr, "travis_fold:start:full_results"
write_results(results, args.iterations)
print >> sys.stderr, "travis_fold:end:full_results"
else:
logger.info("No tests run.")
try:
if gh_handler:
gh_handler.send()
except Exception:
logger.error(traceback.format_exc())
return retcode
def main():
retcode = 0
parser = get_parser()
args = parser.parse_args()
if not os.path.exists(args.root):
logger.critical("Root directory %s does not exist" % args.root)
return 1
os.chdir(args.root)
if args.gh_token:
gh_handler = setup_github_logging(args)
else:
logger.warning("Can't log to GitHub")
gh_handler = None
with TravisFold("browser_setup"):
logger.info("# %s #" % args.browser.title())
browser_cls = {"firefox": Firefox, "chrome": Chrome}.get(args.browser)
if browser_cls is None:
logger.critical("Unrecognised browser %s" % args.browser)
return 1
fetch_wpt_master()
head_sha1 = get_sha1()
logger.info("Testing revision %s" % head_sha1)
# For now just pass the whole list of changed files to wptrunner and
# assume that it will run everything that's actually a test
files_changed = get_files_changed()
if not files_changed:
logger.info("No files changed")
return 0
build_manifest()
install_wptrunner()
do_delayed_imports()
logger.debug("Files changed:\n%s" % "".join(" * %s\n" % item for item in files_changed))
affected_testfiles = get_affected_testfiles(files_changed)
logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in affected_testfiles))
files_changed.extend(affected_testfiles)
browser = browser_cls(args.gh_token)
browser.install()
browser.install_webdriver()
kwargs = wptrunner_args(args.root, files_changed, args.iterations, browser)
with TravisFold("running_tests"):
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs, {"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()), "WARNING"), ["log", "process_output"]
)
)
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(results, args.iterations, args.comment_pr)
else:
logger.info("No tests run.")
try:
if gh_handler:
gh_handler.send()
except Exception:
logger.error(traceback.format_exc())
return retcode
