def test_setup_wptrunner(venv, product):
parser = run.create_parser()
kwargs = vars(parser.parse_args(["--channel=nightly", product]))
kwargs["prompt"] = False
# Hack to get a real existing path
kwargs["binary"] = sys.argv[0]
kwargs["webdriver_binary"] = sys.argv[0]
if kwargs["product"] == "sauce":
kwargs["product"] = "sauce:firefox:63"
run.setup_wptrunner(venv, **kwargs)
def test_setup_wptrunner(venv, logger, product):
parser = run.create_parser()
kwargs = vars(parser.parse_args(["--channel=nightly", product]))
kwargs["prompt"] = False
# Hack to get a real existing path
kwargs["binary"] = sys.argv[0]
kwargs["webdriver_binary"] = sys.argv[0]
if kwargs["product"] == "sauce":
kwargs["product"] = "sauce:firefox:63"
run.setup_wptrunner(venv, **kwargs)
def test_setup_wptrunner(venv, logger, product):
if product == "firefox_android":
pytest.skip("Android emulator doesn't work on docker")
parser = run.create_parser()
kwargs = vars(parser.parse_args(["--channel=nightly", product]))
kwargs["prompt"] = False
# Hack to get a real existing path
kwargs["binary"] = sys.argv[0]
kwargs["webdriver_binary"] = sys.argv[0]
if kwargs["product"] == "sauce":
kwargs["product"] = "sauce:firefox:63"
run.setup_wptrunner(venv, **kwargs)
def kwargs_wptrun(self, kwargs):
from wptrunner import wptcommandline
here = os.path.join(self.topsrcdir, 'testing', 'web-platform')
kwargs["tests_root"] = os.path.join(here, "tests")
sys.path.insert(0, kwargs["tests_root"])
if kwargs["metadata_root"] is None:
metadir = os.path.join(here, "products", kwargs["product"])
if not os.path.exists(metadir):
os.makedirs(metadir)
kwargs["metadata_root"] = metadir
src_manifest = os.path.join(here, "meta", "MANIFEST.json")
dest_manifest = os.path.join(kwargs["metadata_root"], "MANIFEST.json")
if not os.path.exists(dest_manifest) and os.path.exists(src_manifest):
with open(src_manifest) as src, open(dest_manifest, "w") as dest:
dest.write(src.read())
from tools.wpt import run
try:
kwargs = run.setup_wptrunner(
run.virtualenv.Virtualenv(
self.virtualenv_manager.virtualenv_root), **kwargs)
except run.WptrunError as e:
print(e.message, file=sys.stderr)
sys.exit(1)
return kwargs
def kwargs_wptrun(self, kwargs):
from wptrunner import wptcommandline
if kwargs["metadata_root"] is None:
metadir = os.path.join(self._here, "products", kwargs["product"])
if not os.path.exists(metadir):
os.makedirs(metadir)
kwargs["metadata_root"] = metadir
src_manifest = os.path.join(self._here, "meta", "MANIFEST.json")
dest_manifest = os.path.join(kwargs["metadata_root"], "MANIFEST.json")
if not os.path.exists(dest_manifest) and os.path.exists(src_manifest):
with open(src_manifest) as src, open(dest_manifest, "w") as dest:
dest.write(src.read())
from tools.wpt import run
# Add additional kwargs consumed by the run frontend. Currently we don't
# have a way to set these through mach
kwargs["channel"] = None
kwargs["prompt"] = True
kwargs["install_browser"] = False
try:
kwargs = run.setup_wptrunner(
run.virtualenv.Virtualenv(
self.virtualenv_manager.virtualenv_root), **kwargs)
except run.WptrunError as e:
print(e.message, file=sys.stderr)
sys.exit(1)
return kwargs
def kwargs_wptrun(self, kwargs):
from wptrunner import wptcommandline
if kwargs["metadata_root"] is None:
metadir = os.path.join(self._here, "products", kwargs["product"])
if not os.path.exists(metadir):
os.makedirs(metadir)
kwargs["metadata_root"] = metadir
src_manifest = os.path.join(self._here, "meta", "MANIFEST.json")
dest_manifest = os.path.join(kwargs["metadata_root"], "MANIFEST.json")
if not os.path.exists(dest_manifest) and os.path.exists(src_manifest):
with open(src_manifest) as src, open(dest_manifest, "w") as dest:
dest.write(src.read())
from tools.wpt import run
# Add additional kwargs consumed by the run frontend. Currently we don't
# have a way to set these through mach
kwargs["channel"] = None
kwargs["prompt"] = True
kwargs["install_browser"] = False
kwargs["install_webdriver"] = None
kwargs["affected"] = None
# Install the deps
# We do this explicitly to avoid calling pip with options that aren't
# supported in the in-tree version
wptrunner_path = os.path.join(self._here, "tests", "tools",
"wptrunner")
browser_cls = run.product_setup[kwargs["product"]].browser_cls
requirements = ["requirements.txt"]
if hasattr(browser_cls, "requirements"):
requirements.append(browser_cls.requirements)
for filename in requirements:
path = os.path.join(wptrunner_path, filename)
if os.path.exists(path):
self.virtualenv_manager.install_pip_requirements(
path, require_hashes=False)
venv = run.virtualenv.Virtualenv(
self.virtualenv_manager.virtualenv_root,
skip_virtualenv_setup=True)
try:
kwargs = run.setup_wptrunner(venv, **kwargs)
except run.WptrunError as e:
print(e.message, file=sys.stderr)
sys.exit(1)
return kwargs
def kwargs_wptrun(self, kwargs):
"""Setup kwargs for wpt-run which is only used for non-gecko browser products"""
from tools.wpt import run
kwargs = self.kwargs_common(kwargs)
# Add additional kwargs consumed by the run frontend. Currently we don't
# have a way to set these through mach
kwargs["channel"] = None
kwargs["prompt"] = True
kwargs["install_browser"] = False
kwargs["install_webdriver"] = None
kwargs["affected"] = None
# Install the deps
# We do this explicitly to avoid calling pip with options that aren't
# supported in the in-tree version
wptrunner_path = os.path.join(self._here, "tests", "tools", "wptrunner")
browser_cls = run.product_setup[kwargs["product"]].browser_cls
requirements = ["requirements.txt"]
if hasattr(browser_cls, "requirements"):
requirements.append(browser_cls.requirements)
for filename in requirements:
path = os.path.join(wptrunner_path, filename)
if os.path.exists(path):
self.virtualenv_manager.install_pip_requirements(
path, require_hashes=False
)
venv = run.virtualenv.Virtualenv(
self.virtualenv_manager.virtualenv_root, skip_virtualenv_setup=True
)
try:
kwargs = run.setup_wptrunner(venv, **kwargs)
except run.WptrunError as e:
print(e.message, file=sys.stderr)
sys.exit(1)
# This is kind of a hack; override the metadata paths so we don't use
# gecko metadata for non-gecko products
for key, value in list(iteritems(kwargs["test_paths"])):
meta_suffix = key.strip("/")
meta_dir = os.path.join(
self._here, "products", kwargs["product"], meta_suffix
)
value["metadata_path"] = meta_dir
if not os.path.exists(meta_dir):
os.makedirs(meta_dir)
return kwargs
def run(venv, wpt_args, **kwargs):
global logger
do_delayed_imports()
retcode = 0
parser = get_parser()
wpt_args = create_parser().parse_args(wpt_args)
with open(kwargs["config_file"], 'r') as config_fp:
config = SafeConfigParser()
config.readfp(config_fp)
skip_tests = config.get("file detection", "skip_tests").split()
ignore_changes = set(config.get("file detection", "ignore_changes").split())
results_url = config.get("file detection", "results_url")
if kwargs["output_bytes"] is not None:
replace_streams(kwargs["output_bytes"],
"Log reached capacity (%s bytes); output disabled." % kwargs["output_bytes"])
wpt_args.metadata_root = kwargs["metadata_root"]
try:
os.makedirs(wpt_args.metadata_root)
except OSError:
pass
setup_logging()
browser_name = wpt_args.product.split(":")[0]
if browser_name == "sauce" and not wpt_args.sauce_key:
logger.warning("Cannot run tests on Sauce Labs. No access key.")
return retcode
pr_number = pr()
with TravisFold("browser_setup"):
logger.info(markdown.format_comment_title(wpt_args.product))
if pr is not None:
deepen_checkout(kwargs["user"])
# Ensure we have a branch called "master"
fetch_wpt(kwargs["user"], "master:master")
head_sha1 = get_sha1()
logger.info("Testing web-platform-tests at revision %s" % head_sha1)
wpt_kwargs = Kwargs(vars(wpt_args))
if not wpt_kwargs["test_list"]:
manifest_path = os.path.join(wpt_kwargs["metadata_root"], "MANIFEST.json")
tests_changed, files_affected = get_changed_files(manifest_path, kwargs["rev"],
ignore_changes, skip_tests)
if not (tests_changed or files_affected):
logger.info("No tests changed")
return 0
if tests_changed:
logger.debug("Tests changed:\n%s" % "".join(" * %s\n" % item for item in tests_changed))
if files_affected:
logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in files_affected))
wpt_kwargs["test_list"] = list(tests_changed | files_affected)
set_default_args(wpt_kwargs)
do_delayed_imports()
wpt_kwargs["stability"] = True
wpt_kwargs["prompt"] = False
wpt_kwargs["install_browser"] = True
wpt_kwargs["install"] = wpt_kwargs["product"].split(":")[0] == "firefox"
wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs)
logger.info("Using binary %s" % wpt_kwargs["binary"])
with TravisFold("running_tests"):
logger.info("Starting tests")
wpt_logger = wptrunner.logger
iterations, results, inconsistent = run(venv, wpt_logger, **wpt_kwargs)
if results:
if inconsistent:
write_inconsistent(logger.error, inconsistent, iterations)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(logger.info, results, iterations,
pr_number=pr_number,
use_details=True)
if pr_number:
post_results(results, iterations=iterations, url=results_url,
product=wpt_args.product, pr_number=pr_number,
status="failed" if inconsistent else "passed")
else:
logger.info("No tests run.")
return retcode
def run(venv, wpt_args, **kwargs):
global logger
do_delayed_imports()
retcode = 0
parser = get_parser()
wpt_args = create_parser().parse_args(wpt_args)
with open(kwargs["config_file"], 'r') as config_fp:
config = SafeConfigParser()
config.readfp(config_fp)
skip_tests = config.get("file detection", "skip_tests").split()
ignore_changes = set(config.get("file detection", "ignore_changes").split())
if kwargs["output_bytes"] is not None:
replace_streams(kwargs["output_bytes"],
"Log reached capacity (%s bytes); output disabled." % kwargs["output_bytes"])
wpt_args.metadata_root = kwargs["metadata_root"]
try:
os.makedirs(wpt_args.metadata_root)
except OSError:
pass
logger = logging.getLogger(os.path.splitext(__file__)[0])
setup_logging()
browser_name = wpt_args.product.split(":")[0]
if browser_name == "sauce" and not wpt_args.sauce_key:
logger.warning("Cannot run tests on Sauce Labs. No access key.")
return retcode
pr_number = pr()
with TravisFold("browser_setup"):
logger.info(markdown.format_comment_title(wpt_args.product))
if pr is not None:
deepen_checkout(kwargs["user"])
# Ensure we have a branch called "master"
fetch_wpt(kwargs["user"], "master:master")
head_sha1 = get_sha1()
logger.info("Testing web-platform-tests at revision %s" % head_sha1)
if not kwargs["rev"]:
branch_point = testfiles.branch_point()
revish = "%s..HEAD" % branch_point
else:
revish = kwargs["rev"]
files_changed, files_ignored = testfiles.files_changed(revish, ignore_changes)
if files_ignored:
logger.info("Ignoring %s changed files:\n%s" % (len(files_ignored),
"".join(" * %s\n" % item for item in files_ignored)))
tests_changed, files_affected = testfiles.affected_testfiles(files_changed, skip_tests,
manifest_path=os.path.join(
wpt_args.metadata_root,
"MANIFEST.json"))
if not (tests_changed or files_affected):
logger.info("No tests changed")
return 0
wpt_kwargs = Kwargs(vars(wpt_args))
wpt_kwargs["test_list"] = list(tests_changed | files_affected)
set_default_args(wpt_kwargs)
do_delayed_imports()
wpt_kwargs["stability"] = True
wpt_kwargs["prompt"] = False
wpt_kwargs["install_browser"] = True
wpt_kwargs["install"] = wpt_kwargs["product"].split(":")[0] == "firefox"
wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs)
logger.info("Using binary %s" % wpt_kwargs["binary"])
if tests_changed:
logger.debug("Tests changed:\n%s" % "".join(" * %s\n" % item for item in tests_changed))
if files_affected:
logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in files_affected))
with TravisFold("running_tests"):
logger.info("Starting tests")
wpt_logger = wptrunner.logger
iterations, results, inconsistent = run(venv, wpt_logger, **wpt_kwargs)
if results:
if inconsistent:
write_inconsistent(logger.error, inconsistent, iterations)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(logger.info, results, iterations,
pr_number=kwargs["comment_pr"],
use_details=True)
else:
logger.info("No tests run.")
return retcode
def run(venv, wpt_args, **kwargs):
do_delayed_imports()
setup_logging()
retcode = 0
wpt_args = create_parser().parse_args(wpt_args)
with open(kwargs["config_file"], 'r') as config_fp:
config = SafeConfigParser()
config.readfp(config_fp)
ignore_changes = set(
config.get("file detection", "ignore_changes").split())
if kwargs["output_bytes"] is not None:
replace_streams(
kwargs["output_bytes"],
"Log reached capacity (%s bytes); output disabled." %
kwargs["output_bytes"])
wpt_args.metadata_root = kwargs["metadata_root"]
try:
os.makedirs(wpt_args.metadata_root)
except OSError:
pass
pr_number = pr()
with TravisFold("browser_setup"):
logger.info(markdown.format_comment_title(wpt_args.product))
if pr is not None:
deepen_checkout(kwargs["user"])
# Ensure we have a branch called "master"
fetch_wpt(kwargs["user"], "master:master")
head_sha1 = get_sha1()
logger.info("Testing web-platform-tests at revision %s" % head_sha1)
wpt_kwargs = Kwargs(vars(wpt_args))
if not wpt_kwargs["test_list"]:
manifest_path = os.path.join(wpt_kwargs["metadata_root"],
"MANIFEST.json")
tests_changed, files_affected = get_changed_files(
manifest_path, kwargs["rev"], ignore_changes)
if not (tests_changed or files_affected):
logger.info("No tests changed")
return 0
if tests_changed:
logger.debug("Tests changed:\n%s" %
"".join(" * %s\n" % item
for item in tests_changed))
if files_affected:
logger.debug("Affected tests:\n%s" %
"".join(" * %s\n" % item
for item in files_affected))
wpt_kwargs["test_list"] = list(tests_changed | files_affected)
do_delayed_imports()
wpt_kwargs["prompt"] = False
wpt_kwargs["install_browser"] = wpt_kwargs["product"].split(
":")[0] == "firefox"
wpt_kwargs["pause_after_test"] = False
wpt_kwargs["verify_log_full"] = False
if wpt_kwargs["repeat"] == 1:
wpt_kwargs["repeat"] = 10
wpt_kwargs["headless"] = False
wpt_kwargs["log_tbpl"] = [sys.stdout]
wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs)
logger.info("Using binary %s" % wpt_kwargs["binary"])
with TravisFold("running_tests"):
logger.info("Starting tests")
wpt_logger = wptrunner.logger
results, inconsistent, slow, iterations = run_step(
wpt_logger, wpt_kwargs["repeat"], True, {}, **wpt_kwargs)
if results:
if inconsistent:
write_inconsistent(logger.error, inconsistent, iterations)
retcode = 2
elif slow:
write_slow_tests(logger.error, slow)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(logger.info,
results,
iterations,
pr_number=pr_number,
use_details=True)
else:
logger.info("No tests run.")
# Be conservative and only return errors when we know for sure tests are changed.
if tests_changed:
retcode = 3
return retcode
def run(venv, wpt_args, **kwargs):
do_delayed_imports()
retcode = 0
wpt_args = create_parser().parse_args(wpt_args)
with open(kwargs["config_file"], 'r') as config_fp:
config = SafeConfigParser()
config.readfp(config_fp)
skip_tests = config.get("file detection", "skip_tests").split()
ignore_changes = set(config.get("file detection", "ignore_changes").split())
results_url = config.get("file detection", "results_url")
if kwargs["output_bytes"] is not None:
replace_streams(kwargs["output_bytes"],
"Log reached capacity (%s bytes); output disabled." % kwargs["output_bytes"])
wpt_args.metadata_root = kwargs["metadata_root"]
try:
os.makedirs(wpt_args.metadata_root)
except OSError:
pass
setup_logging()
browser_name = wpt_args.product.split(":")[0]
pr_number = pr()
with TravisFold("browser_setup"):
logger.info(markdown.format_comment_title(wpt_args.product))
if pr is not None:
deepen_checkout(kwargs["user"])
# Ensure we have a branch called "master"
fetch_wpt(kwargs["user"], "master:master")
head_sha1 = get_sha1()
logger.info("Testing web-platform-tests at revision %s" % head_sha1)
wpt_kwargs = Kwargs(vars(wpt_args))
if not wpt_kwargs["test_list"]:
manifest_path = os.path.join(wpt_kwargs["metadata_root"], "MANIFEST.json")
tests_changed, files_affected = get_changed_files(manifest_path, kwargs["rev"],
ignore_changes, skip_tests)
if not (tests_changed or files_affected):
logger.info("No tests changed")
return 0
if tests_changed:
logger.debug("Tests changed:\n%s" % "".join(" * %s\n" % item for item in tests_changed))
if files_affected:
logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in files_affected))
wpt_kwargs["test_list"] = list(tests_changed | files_affected)
do_delayed_imports()
wpt_kwargs["stability"] = True
wpt_kwargs["prompt"] = False
wpt_kwargs["install_browser"] = True
wpt_kwargs["install"] = wpt_kwargs["product"].split(":")[0] == "firefox"
wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs)
logger.info("Using binary %s" % wpt_kwargs["binary"])
with TravisFold("running_tests"):
logger.info("Starting tests")
wpt_logger = wptrunner.logger
iterations, results, inconsistent = stability_run(venv, wpt_logger, **wpt_kwargs)
if results:
if inconsistent:
write_inconsistent(logger.error, inconsistent, iterations)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(logger.info, results, iterations,
pr_number=pr_number,
use_details=True)
if pr_number:
post_results(results, iterations=iterations, url=results_url,
product=wpt_args.product, pr_number=pr_number,
status="failed" if inconsistent else "passed")
else:
logger.info("No tests run.")
# Be conservative and only return errors when we know for sure tests are changed.
if tests_changed:
retcode = 3
return retcode
