def setup_wptrunner(venv, product, tests, wptrunner_args, prompt=True,):
global logger
wptparser = wptcommandline.create_parser()
kwargs = utils.Kwargs(vars(wptparser.parse_args(wptrunner_args)).iteritems())
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
logger = wptrunner.logger
kwargs["product"] = product
kwargs["test_list"] = tests
check_environ(product)
args_general(kwargs)
if product not in product_setup:
exit("Unsupported product %s" % product)
product_setup[product](venv, kwargs, prompt)
wptcommandline.check_args(kwargs)
wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner")
venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt"))
return kwargs
def run(config, args):
logger = structuredlog.StructuredLogger("web-platform-tests")
logger.add_handler(ResultHandler(logger=logger, verbose=args.verbose))
setup_wptrunner_logging(logger)
parser = wptcommandline.create_parser()
logger.suite_start(tests=[])
for product, product_settings in config["products"].iteritems():
if args.product and product not in args.product:
continue
settings = test_settings()
settings.update(config["general"])
settings.update(product_settings)
settings["product"] = product
set_from_args(settings, args)
kwargs = vars(parser.parse_args(settings_to_argv(settings)))
wptcommandline.check_args(kwargs)
logger.send_message("wptrunner-test", "set-product", product)
run_tests(product, kwargs)
logger.send_message("wptrunner-test", "set-product", None)
logger.suite_end()
def create_parser_wpt():
parser = wptcommandline.create_parser()
parser.add_argument('--release', default=False, action="store_true",
help="Run with a release build of servo")
parser.add_argument('--chaos', default=False, action="store_true",
help="Run under chaos mode in rr until a failure is captured")
return parser
def main():
parser = wptcommandline.create_parser()
parser.add_argument('--update-manifest', dest='update_manifest', action='store_true')
args = parser.parse_args()
if args.update_manifest:
return update_manifest()
kwargs = set_defaults(args)
return run_tests(**kwargs)
def create_parser_wpt():
parser = wptcommandline.create_parser()
parser.add_argument('--release', default=False, action="store_true",
help="Run with a release build of servo")
parser.add_argument('--chaos', default=False, action="store_true",
help="Run under chaos mode in rr until a failure is captured")
parser.add_argument('--pref', default=[], action="append", dest="prefs",
help="Pass preferences to servo")
parser.add_argument('--always-succeed', default=False, action="store_true",
help="Always yield exit code of zero")
return parser
def create_parser():
from wptrunner import wptcommandline
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("product", action="store",
help="Browser to run tests in")
parser.add_argument("--yes", "-y", dest="prompt", action="store_false", default=True,
help="Don't prompt before installing components")
parser.add_argument("--stability", action="store_true",
help="Stability check tests")
parser.add_argument("--install-browser", action="store_true",
help="Install the latest development version of the browser")
parser._add_container_actions(wptcommandline.create_parser())
return parser
def wptrunner_args(root, files_changed, iterations, browser):
"""Derive and return arguments for wpt-runner."""
parser = wptcommandline.create_parser([browser.product])
args = vars(parser.parse_args([]))
args.update(browser.wptrunner_args(root))
args.update({
"tests_root": wpt_root,
"metadata_root": wpt_root,
"repeat": iterations,
"config": "%s//wptrunner.default.ini" % (wptrunner_root),
"test_list": files_changed,
"restart_on_unexpected": False,
"pause_after_test": False
})
wptcommandline.check_args(args)
return args
def wptrunner_args(root, files_changed, iterations, browser):
parser = wptcommandline.create_parser([browser.product])
args = vars(parser.parse_args([]))
wpt_root = os.path.join(root, "w3c", "web-platform-tests")
args.update(browser.wptrunner_args(root))
args.update({
"tests_root": wpt_root,
"metadata_root": wpt_root,
"repeat": iterations,
"config": "%s/w3c/wptrunner/wptrunner.default.ini" % root,
"test_list": files_changed,
"restart_on_unexpected": False,
"pause_after_test": False
})
wptcommandline.check_args(args)
return args
def create_parser():
from wptrunner import wptcommandline
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("product", action="store",
help="Browser to run tests in")
parser.add_argument("--yes", "-y", dest="prompt", action="store_false", default=True,
help="Don't prompt before installing components")
parser.add_argument("--install-browser", action="store_true",
help="Install the browser")
parser.add_argument("--channel", action="store",
choices=install.channel_by_name.keys(),
default=None, help='Name of browser release channel.'
'"stable" and "release" are synonyms for the latest browser stable release,'
'"nightly", "dev", "experimental", and "preview" are all synonyms for '
'the latest available development release. For WebDriver installs, '
'we attempt to select an appropriate, compatible, version for the '
'latest browser release on the selected channel.')
parser._add_container_actions(wptcommandline.create_parser())
return parser
def create_parser():
from wptrunner import wptcommandline
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("product",
action="store",
help="Browser to run tests in")
parser.add_argument("--yes",
"-y",
dest="prompt",
action="store_false",
default=True,
help="Don't prompt before installing components")
parser.add_argument("--stability",
action="store_true",
help="Stability check tests")
parser.add_argument(
"--install-browser",
action="store_true",
help="Install the latest development version of the browser")
parser._add_container_actions(wptcommandline.create_parser())
return parser
def create_parser_wpt():
import mozlog.commandline
parser = wptcommandline.create_parser()
parser.add_argument('--release',
default=False,
action="store_true",
help="Run with a release build of servo")
parser.add_argument(
'--rr-chaos',
default=False,
action="store_true",
help="Run under chaos mode in rr until a failure is captured")
parser.add_argument('--pref',
default=[],
action="append",
dest="prefs",
help="Pass preferences to servo")
parser.add_argument('--layout-2020',
default=False,
action="store_true",
help="Use expected results for the 2020 layout engine")
parser.add_argument('--log-servojson',
action="append",
type=mozlog.commandline.log_file,
help="Servo's JSON logger of unexpected results")
parser.add_argument('--always-succeed',
default=False,
action="store_true",
help="Always yield exit code of zero")
parser.add_argument(
'--no-default-test-types',
default=False,
action="store_true",
help=
"Run all of the test types provided by wptrunner or specified explicitly by --test-types"
),
return parser
def create_parser():
from wptrunner import wptcommandline
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("product",
action="store",
help="Browser to run tests in")
parser.add_argument("--affected",
action="store",
default=None,
help="Run affected tests since revish")
parser.add_argument("--yes",
"-y",
dest="prompt",
action="store_false",
default=True,
help="Don't prompt before installing components")
parser.add_argument(
"--install-browser",
action="store_true",
help=
"Install the browser from the release channel specified by --channel "
"(or the nightly channel by default).")
parser.add_argument(
"--channel",
action="store",
choices=install.channel_by_name.keys(),
default=None,
help='Name of browser release channel. '
'"stable" and "release" are synonyms for the latest browser stable '
'release, "nightly", "dev", "experimental", and "preview" are all '
'synonyms for the latest available development release. (For WebDriver '
'installs, we attempt to select an appropriate, compatible version for '
'the latest browser release on the selected channel.) '
'This flag overrides --browser-channel.')
parser._add_container_actions(wptcommandline.create_parser())
return parser
class MachCommands(MachCommandBase):
@Command("web-platform-tests",
category="testing",
conditions=[conditions.is_firefox],
parser=wptcommandline.create_parser(["firefox"]))
def run_web_platform_tests(self, **params):
self.setup()
wpt_runner = self._spawn(WebPlatformTestsRunner)
if params["list_test_groups"]:
return wpt_runner.list_test_groups(**params)
else:
return wpt_runner.run_tests(**params)
@Command("web-platform-tests-update",
category="testing",
conditions=[conditions.is_firefox],
parser=wptcommandline.create_parser_update())
def update_web_platform_tests(self, **params):
self.setup()
self.virtualenv_manager.install_pip_package('html5lib==0.99')
wpt_updater = self._spawn(WebPlatformTestsUpdater)
return wpt_updater.run_update(**params)
def setup(self):
self._activate_virtualenv()
self.virtualenv_manager.install_pip_package('py==1.4.14')
@Command("web-platform-tests-reduce",
category="testing",
conditions=[conditions.is_firefox],
parser=wptcommandline.create_parser_reduce(["firefox"]))
def unstable_web_platform_tests(self, **params):
self.setup()
wpt_reduce = self._spawn(WebPlatformTestsReduce)
return wpt_reduce.run_reduce(**params)
def create_parser():
from wptrunner import wptcommandline
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("product", action="store",
help="Browser to run tests in")
parser.add_argument("--affected", action="store", default=None,
help="Run affected tests since revish")
parser.add_argument("--yes", "-y", dest="prompt", action="store_false", default=True,
help="Don't prompt before installing components")
parser.add_argument("--install-browser", action="store_true",
help="Install the browser from the release channel specified by --channel "
"(or the nightly channel by default).")
parser.add_argument("--channel", action="store",
choices=install.channel_by_name.keys(),
default=None, help='Name of browser release channel. '
'"stable" and "release" are synonyms for the latest browser stable '
'release, "nightly", "dev", "experimental", and "preview" are all '
'synonyms for the latest available development release. (For WebDriver '
'installs, we attempt to select an appropriate, compatible version for '
'the latest browser release on the selected channel.) '
'This flag overrides --browser-channel.')
parser._add_container_actions(wptcommandline.create_parser())
return parser
def main():
parser = wptcommandline.create_parser()
kwargs = vars(parser.parse_args())
return update_tests(**kwargs)
class MachCommands(CommandBase):
DEFAULT_RENDER_MODE = "cpu"
HELP_RENDER_MODE = "Value can be 'cpu', 'gpu' or 'both' (default " + DEFAULT_RENDER_MODE + ")"
def __init__(self, context):
CommandBase.__init__(self, context)
if not hasattr(self.context, "built_tests"):
self.context.built_tests = False
def ensure_built_tests(self):
if self.context.built_tests:
return
returncode = Registrar.dispatch('build-tests', context=self.context)
if returncode:
sys.exit(returncode)
self.context.built_tests = True
def find_test(self, prefix):
target_contents = os.listdir(path.join(self.get_target_dir(), "debug"))
for filename in target_contents:
if filename.startswith(prefix + "-"):
filepath = path.join(self.get_target_dir(), "debug", filename)
if path.isfile(filepath) and os.access(filepath, os.X_OK):
return filepath
def run_test(self, prefix, args=[]):
t = self.find_test(prefix)
if t:
return subprocess.call([t] + args, env=self.build_env())
@Command('test', description='Run all Servo tests', category='testing')
@CommandArgument('params',
default=None,
nargs="...",
help="Optionally select test based on "
"test file directory")
@CommandArgument('--render-mode',
'-rm',
default=DEFAULT_RENDER_MODE,
help="The render mode to be used on all tests. " +
HELP_RENDER_MODE)
@CommandArgument('--release',
default=False,
action="store_true",
help="Run with a release build of servo")
def test(self, params, render_mode=DEFAULT_RENDER_MODE, release=False):
suites = OrderedDict([
("tidy", {}),
("ref", {
"kwargs": {
"kind": render_mode
},
"path": path.abspath(path.join("tests", "ref")),
"include_arg": "name"
}),
("wpt", {
"kwargs": {
"release": release
},
"path":
path.abspath(path.join("tests", "wpt", "web-platform-tests")),
"include_arg":
"include"
}),
("css", {
"kwargs": {
"release": release
},
"path": path.abspath(path.join("tests", "wpt", "css-tests")),
"include_arg": "include"
}),
("unit", {}),
])
suites_by_prefix = {
v["path"]: k
for k, v in suites.iteritems() if "path" in v
}
selected_suites = OrderedDict()
if params is None:
params = suites.keys()
for arg in params:
found = False
if arg in suites and arg not in selected_suites:
selected_suites[arg] = []
found = True
elif os.path.exists(path.abspath(arg)):
abs_path = path.abspath(arg)
for prefix, suite in suites_by_prefix.iteritems():
if abs_path.startswith(prefix):
if suite not in selected_suites:
selected_suites[suite] = []
selected_suites[suite].append(arg)
found = True
break
if not found:
print("%s is not a valid test path or suite name" % arg)
return 1
test_start = time()
for suite, tests in selected_suites.iteritems():
props = suites[suite]
kwargs = props.get("kwargs", {})
if tests:
kwargs[props["include_arg"]] = tests
Registrar.dispatch("test-%s" % suite,
context=self.context,
**kwargs)
elapsed = time() - test_start
print("Tests completed in %0.2fs" % elapsed)
@Command('test-unit', description='Run unit tests', category='testing')
@CommandArgument('--package',
'-p',
default=None,
help="Specific package to test")
@CommandArgument('test_name',
nargs=argparse.REMAINDER,
help="Only run tests that match this pattern")
def test_unit(self, test_name=None, package=None):
if test_name is None:
test_name = []
self.ensure_bootstrapped()
if package:
packages = [package]
else:
packages = os.listdir(
path.join(self.context.topdir, "tests", "unit"))
for crate in packages:
result = subprocess.call(
["cargo", "test", "-p",
"%s_tests" % crate] + test_name,
env=self.build_env(),
cwd=self.servo_crate())
if result != 0:
return result
@Command('test-ref',
description='Run the reference tests',
category='testing')
@CommandArgument('--kind',
'-k',
default=DEFAULT_RENDER_MODE,
help=HELP_RENDER_MODE)
@CommandArgument('--name',
default=None,
help="Only run tests that match this pattern. If the "
"path to the ref test directory is included, it "
"will automatically be trimmed out.")
@CommandArgument(
'servo_params',
default=None,
nargs=argparse.REMAINDER,
help="Command-line arguments to be passed through to Servo")
def test_ref(self, kind=DEFAULT_RENDER_MODE, name=None, servo_params=None):
self.ensure_bootstrapped()
self.ensure_built_tests()
assert kind is not None, 'kind cannot be None, see help'
kinds = ["cpu", "gpu"] if kind == 'both' else [kind]
test_path = path.join(self.context.topdir, "tests", "ref")
error = False
test_start = time()
for k in kinds:
print("Running %s reftests..." % k)
test_args = [k, test_path]
if name is not None:
maybe_path = path.normpath(name)
ref_path = path.join("tests", "ref")
# Check to see if we were passed something leading with the
# path to the ref test directory, and trim it so that reftest
# knows how to filter it.
if ref_path in maybe_path:
test_args.append(path.relpath(maybe_path, ref_path))
else:
test_args.append(name)
if servo_params is not None:
test_args += ["--"] + servo_params
ret = self.run_test("reftest", test_args)
error = error or ret != 0
elapsed = time() - test_start
print("Reference tests completed in %0.2fs" % elapsed)
if error:
return 1
@Command('test-content',
description='Run the content tests',
category='testing')
def test_content(self):
print("Content tests have been replaced by web-platform-tests under "
"tests/wpt/mozilla/.")
return 0
@Command('test-tidy',
description='Run the source code tidiness check',
category='testing')
def test_tidy(self):
return tidy.scan()
@Command('test-wpt-failure',
description='Run the web platform tests',
category='testing')
def test_wpt_failure(self):
self.ensure_bootstrapped()
return not subprocess.call([
"bash",
path.join("tests", "wpt", "run.sh"), "--no-pause-after-test",
"--include", "infrastructure/failing-test.html"
],
env=self.build_env())
@Command('test-wpt',
description='Run the web platform tests',
category='testing',
parser=wptcommandline.create_parser)
@CommandArgument('--release',
default=False,
action="store_true",
help="Run with a release build of servo")
def test_wpt(self, **kwargs):
self.ensure_bootstrapped()
self.ensure_wpt_virtualenv()
hosts_file_path = path.join('tests', 'wpt', 'hosts')
os.environ["hosts_file_path"] = hosts_file_path
run_file = path.abspath(path.join("tests", "wpt", "run_wpt.py"))
run_globals = {"__file__": run_file}
execfile(run_file, run_globals)
return run_globals["run_tests"](**kwargs)
@Command('update-wpt',
description='Update the web platform tests',
category='testing',
parser=updatecommandline.create_parser())
def update_wpt(self, **kwargs):
self.ensure_bootstrapped()
self.ensure_wpt_virtualenv()
run_file = path.abspath(path.join("tests", "wpt", "update.py"))
run_globals = {"__file__": run_file}
execfile(run_file, run_globals)
return run_globals["update_tests"](**kwargs)
@Command('test-jquery',
description='Run the jQuery test suite',
category='testing')
@CommandArgument('--release',
'-r',
action='store_true',
help='Run the release build')
@CommandArgument('--dev',
'-d',
action='store_true',
help='Run the dev build')
def test_jquery(self, release, dev):
return self.jquery_test_runner("test", release, dev)
@Command('update-jquery',
description='Update the jQuery test suite expected results',
category='testing')
@CommandArgument('--release',
'-r',
action='store_true',
help='Run the release build')
@CommandArgument('--dev',
'-d',
action='store_true',
help='Run the dev build')
def update_jquery(self, release, dev):
return self.jquery_test_runner("update", release, dev)
@Command('test-css',
description='Run the web platform tests',
category='testing',
parser=wptcommandline.create_parser())
@CommandArgument('--release',
default=False,
action="store_true",
help="Run with a release build of servo")
def test_css(self, **kwargs):
self.ensure_bootstrapped()
self.ensure_wpt_virtualenv()
run_file = path.abspath(path.join("tests", "wpt", "run_css.py"))
run_globals = {"__file__": run_file}
execfile(run_file, run_globals)
return run_globals["run_tests"](**kwargs)
@Command('update-css',
description='Update the web platform tests',
category='testing',
parser=updatecommandline.create_parser())
def update_css(self, **kwargs):
self.ensure_bootstrapped()
self.ensure_wpt_virtualenv()
run_file = path.abspath(path.join("tests", "wpt", "update_css.py"))
run_globals = {"__file__": run_file}
execfile(run_file, run_globals)
return run_globals["update_tests"](**kwargs)
def ensure_wpt_virtualenv(self):
virtualenv_path = path.join("tests", "wpt", "_virtualenv")
python = self.get_exec("python2", "python")
if not os.path.exists(virtualenv_path):
virtualenv = self.get_exec("virtualenv2", "virtualenv")
subprocess.check_call([virtualenv, "-p", python, virtualenv_path])
activate_path = path.join(virtualenv_path, "bin", "activate_this.py")
execfile(activate_path, dict(__file__=activate_path))
try:
import wptrunner # noqa
from wptrunner.browsers import servo # noqa
except ImportError:
subprocess.check_call([
"pip", "install", "-r",
path.join("tests", "wpt", "harness", "requirements.txt")
])
subprocess.check_call([
"pip", "install", "-r",
path.join("tests", "wpt", "harness", "requirements_servo.txt")
])
try:
import blessings
except ImportError:
subprocess.check_call(["pip", "install", "blessings"])
# This is an unfortunate hack. Because mozlog gets imported by wptcommandline
# before the virtualenv is initalised it doesn't see the blessings module so we don't
# get coloured output. Setting the blessings global explicitly fixes that.
from mozlog.structured.formatters import machformatter
import blessings # noqa
machformatter.blessings = blessings
def get_exec(self, name, default=None):
path = find_executable(name)
if not path:
return default
return path
def jquery_test_runner(self, cmd, release, dev):
self.ensure_bootstrapped()
base_dir = path.abspath(path.join("tests", "jquery"))
jquery_dir = path.join(base_dir, "jquery")
run_file = path.join(base_dir, "run_jquery.py")
# Clone the jQuery repository if it doesn't exist
if not os.path.isdir(jquery_dir):
subprocess.check_call([
"git", "clone", "-b", "servo", "--depth", "1",
"https://github.com/servo/jquery", jquery_dir
])
# Run pull in case the jQuery repo was updated since last test run
subprocess.check_call(["git", "-C", jquery_dir, "pull"])
# Check that a release servo build exists
bin_path = path.abspath(self.get_binary_path(release, dev))
return subprocess.check_call([run_file, cmd, bin_path, base_dir])
def create_parser_wpt():
from wptrunner import wptcommandline
return wptcommandline.create_parser(["firefox", "chrome", "edge", "servo"])
class MachCommands(CommandBase):
DEFAULT_RENDER_MODE = "cpu"
HELP_RENDER_MODE = "Value can be 'cpu', 'gpu' or 'both' (default " + DEFAULT_RENDER_MODE + ")"
def __init__(self, context):
CommandBase.__init__(self, context)
if not hasattr(self.context, "built_tests"):
self.context.built_tests = False
def ensure_built_tests(self, release=False):
if self.context.built_tests:
return
returncode = Registrar.dispatch(
'build-tests', context=self.context, release=release)
if returncode:
sys.exit(returncode)
self.context.built_tests = True
def find_test(self, prefix, release=False):
build_mode = "release" if release else "debug"
target_contents = os.listdir(path.join(
self.get_target_dir(), build_mode))
for filename in target_contents:
if filename.startswith(prefix + "-"):
filepath = path.join(
self.get_target_dir(), build_mode, filename)
if path.isfile(filepath) and os.access(filepath, os.X_OK):
return filepath
def run_test(self, prefix, args=[], release=False):
t = self.find_test(prefix, release=release)
if t:
return subprocess.call([t] + args, env=self.build_env())
@Command('test',
description='Run all Servo tests',
category='testing')
@CommandArgument('params', default=None, nargs="...",
help="Optionally select test based on "
"test file directory")
@CommandArgument('--render-mode', '-rm', default=DEFAULT_RENDER_MODE,
help="The render mode to be used on all tests. " +
HELP_RENDER_MODE)
@CommandArgument('--release', default=False, action="store_true",
help="Run with a release build of servo")
def test(self, params, render_mode=DEFAULT_RENDER_MODE, release=False):
suites = OrderedDict([
("tidy", {}),
("ref", {"kwargs": {"kind": render_mode},
"paths": [path.abspath(path.join("tests", "ref"))],
"include_arg": "include"}),
("wpt", {"kwargs": {"release": release},
"paths": [path.abspath(path.join("tests", "wpt", "web-platform-tests")),
path.abspath(path.join("tests", "wpt", "mozilla"))],
"include_arg": "include"}),
("css", {"kwargs": {"release": release},
"paths": [path.abspath(path.join("tests", "wpt", "css-tests"))],
"include_arg": "include"}),
("unit", {"kwargs": {},
"paths": [path.abspath(path.join("tests", "unit"))],
"include_arg": "test_name"})
])
suites_by_prefix = {path: k for k, v in suites.iteritems() if "paths" in v for path in v["paths"]}
selected_suites = OrderedDict()
if params is None:
params = suites.keys()
for arg in params:
found = False
if arg in suites and arg not in selected_suites:
selected_suites[arg] = []
found = True
elif os.path.exists(path.abspath(arg)):
abs_path = path.abspath(arg)
for prefix, suite in suites_by_prefix.iteritems():
if abs_path.startswith(prefix):
if suite not in selected_suites:
selected_suites[suite] = []
selected_suites[suite].append(arg)
found = True
break
if not found:
print("%s is not a valid test path or suite name" % arg)
return 1
test_start = time()
for suite, tests in selected_suites.iteritems():
props = suites[suite]
kwargs = props.get("kwargs", {})
if tests:
kwargs[props["include_arg"]] = tests
Registrar.dispatch("test-%s" % suite, context=self.context, **kwargs)
elapsed = time() - test_start
print("Tests completed in %0.2fs" % elapsed)
@Command('test-unit',
description='Run unit tests',
category='testing')
@CommandArgument('--package', '-p', default=None, help="Specific package to test")
@CommandArgument('test_name', nargs=argparse.REMAINDER,
help="Only run tests that match this pattern or file path")
def test_unit(self, test_name=None, package=None):
if test_name is None:
test_name = []
self.ensure_bootstrapped()
if package:
packages = {package}
else:
packages = set()
test_patterns = []
for test in test_name:
# add package if 'tests/unit/<package>'
match = re.search("tests/unit/(\\w+)/?$", test)
if match:
packages.add(match.group(1))
# add package & test if '<package>/<test>', 'tests/unit/<package>/<test>.rs', or similar
elif re.search("\\w/\\w", test):
tokens = test.split("/")
packages.add(tokens[-2])
test_prefix = tokens[-1]
if test_prefix.endswith(".rs"):
test_prefix = test_prefix[:-3]
test_prefix += "::"
test_patterns.append(test_prefix)
# add test as-is otherwise
else:
test_patterns.append(test)
if not packages:
packages = set(os.listdir(path.join(self.context.topdir, "tests", "unit")))
args = ["cargo", "test"]
for crate in packages:
args += ["-p", "%s_tests" % crate]
args += test_patterns
result = subprocess.call(args, env=self.build_env(), cwd=self.servo_crate())
if result != 0:
return result
@Command('test-ref',
description='Run the reference tests',
category='testing')
@CommandArgument('--kind', '-k', default=DEFAULT_RENDER_MODE,
help=HELP_RENDER_MODE)
@CommandArgument('--release', '-r', action='store_true',
help='Run with a release build of Servo')
@CommandArgument('--include', default=None, nargs='+',
help="Only run tests that match this pattern. If the "
"path to the ref test directory is included, it "
"will automatically be trimmed out.")
@CommandArgument(
'servo_params', default=None, nargs=argparse.REMAINDER,
help="Command-line arguments to be passed through to Servo")
def test_ref(self, kind=DEFAULT_RENDER_MODE, include=None, servo_params=None,
release=False):
self.ensure_bootstrapped()
self.ensure_built_tests(release=release)
assert kind is not None, 'kind cannot be None, see help'
kinds = ["cpu", "gpu"] if kind == 'both' else [kind]
test_path = path.join(self.context.topdir, "tests", "ref")
error = False
test_start = time()
for k in kinds:
print("Running %s reftests..." % k)
test_args = [k, test_path]
if include is not None:
ref_path = path.join("tests", "ref")
for name in include:
# Check to see if we were passed something leading with the
# path to the ref test directory, and trim it so that reftest
# knows how to filter it.
maybe_path = path.normpath(name)
if ref_path in maybe_path:
test_args.append(path.relpath(maybe_path, ref_path))
else:
test_args.append(name)
if servo_params is not None:
test_args += ["--"] + servo_params
ret = self.run_test("reftest", test_args, release=release)
error = error or ret != 0
elapsed = time() - test_start
print("Reference tests completed in %0.2fs" % elapsed)
if error:
return 1
@Command('test-content',
description='Run the content tests',
category='testing')
def test_content(self):
print("Content tests have been replaced by web-platform-tests under "
"tests/wpt/mozilla/.")
return 0
@Command('test-tidy',
description='Run the source code tidiness check',
category='testing')
def test_tidy(self):
return tidy.scan()
@Command('test-wpt-failure',
description='Run the web platform tests',
category='testing')
def test_wpt_failure(self):
self.ensure_bootstrapped()
return not subprocess.call([
"bash",
path.join("tests", "wpt", "run.sh"),
"--no-pause-after-test",
"--include",
"infrastructure/failing-test.html"
], env=self.build_env())
@Command('test-wpt',
description='Run the web platform tests',
category='testing',
parser=wptcommandline.create_parser)
@CommandArgument('--release', default=False, action="store_true",
help="Run with a release build of servo")
def test_wpt(self, **kwargs):
self.ensure_bootstrapped()
hosts_file_path = path.join(self.context.topdir, 'tests', 'wpt', 'hosts')
os.environ["hosts_file_path"] = hosts_file_path
kwargs["debug"] = not kwargs["release"]
run_file = path.abspath(path.join(self.context.topdir, "tests", "wpt", "run_wpt.py"))
run_globals = {"__file__": run_file}
execfile(run_file, run_globals)
return run_globals["run_tests"](**kwargs)
@Command('update-wpt',
description='Update the web platform tests',
category='testing',
parser=updatecommandline.create_parser())
def update_wpt(self, **kwargs):
self.ensure_bootstrapped()
run_file = path.abspath(path.join("tests", "wpt", "update.py"))
run_globals = {"__file__": run_file}
execfile(run_file, run_globals)
return run_globals["update_tests"](**kwargs)
@Command('test-jquery',
description='Run the jQuery test suite',
category='testing')
@CommandArgument('--release', '-r', action='store_true',
help='Run the release build')
@CommandArgument('--dev', '-d', action='store_true',
help='Run the dev build')
def test_jquery(self, release, dev):
return self.jquery_test_runner("test", release, dev)
@Command('test-dromaeo',
description='Run the Dromaeo test suite',
category='testing')
@CommandArgument('tests', default=["recommended"], nargs="...",
help="Specific tests to run")
@CommandArgument('--release', '-r', action='store_true',
help='Run the release build')
@CommandArgument('--dev', '-d', action='store_true',
help='Run the dev build')
def test_dromaeo(self, tests, release, dev):
return self.dromaeo_test_runner(tests, release, dev)
@Command('update-jquery',
description='Update the jQuery test suite expected results',
category='testing')
@CommandArgument('--release', '-r', action='store_true',
help='Run the release build')
@CommandArgument('--dev', '-d', action='store_true',
help='Run the dev build')
def update_jquery(self, release, dev):
return self.jquery_test_runner("update", release, dev)
@Command('test-css',
description='Run the web platform tests',
category='testing',
parser=wptcommandline.create_parser())
@CommandArgument('--release', default=False, action="store_true",
help="Run with a release build of servo")
def test_css(self, **kwargs):
self.ensure_bootstrapped()
run_file = path.abspath(path.join("tests", "wpt", "run_css.py"))
run_globals = {"__file__": run_file}
execfile(run_file, run_globals)
return run_globals["run_tests"](**kwargs)
@Command('update-css',
description='Update the web platform tests',
category='testing',
parser=updatecommandline.create_parser())
def update_css(self, **kwargs):
self.ensure_bootstrapped()
run_file = path.abspath(path.join("tests", "wpt", "update_css.py"))
run_globals = {"__file__": run_file}
execfile(run_file, run_globals)
return run_globals["update_tests"](**kwargs)
@Command('compare_dromaeo',
description='compare outputs of two runs of ./mach test-dromaeo command',
category='testing')
@CommandArgument('params', default=None, nargs="...",
help=" filepaths of output files of two runs of dromaeo test ")
def compare_dromaeo(self, params):
prev_op_filename = params[0]
cur_op_filename = params[1]
result = {'Test': [], 'Prev_Time': [], 'Cur_Time': [], 'Difference(%)': []}
with open(prev_op_filename, 'r') as prev_op, open(cur_op_filename, 'r') as cur_op:
l1 = prev_op.readline()
l2 = cur_op.readline()
while ((l1.find('[dromaeo] Saving...') and l2.find('[dromaeo] Saving...'))):
l1 = prev_op.readline()
l2 = cur_op.readline()
reach = 3
while (reach > 0):
l1 = prev_op.readline()
l2 = cur_op.readline()
reach -= 1
while True:
l1 = prev_op.readline()
l2 = cur_op.readline()
if not l1:
break
result['Test'].append(str(l1).split('|')[0].strip())
result['Prev_Time'].append(float(str(l1).split('|')[1].strip()))
result['Cur_Time'].append(float(str(l2).split('|')[1].strip()))
a = float(str(l1).split('|')[1].strip())
b = float(str(l2).split('|')[1].strip())
result['Difference(%)'].append(((b - a) / a) * 100)
width_col1 = max([len(x) for x in result['Test']])
width_col2 = max([len(str(x)) for x in result['Prev_Time']])
width_col3 = max([len(str(x)) for x in result['Cur_Time']])
width_col4 = max([len(str(x)) for x in result['Difference(%)']])
for p, q, r, s in zip(['Test'], ['First Run'], ['Second Run'], ['Difference(%)']):
print ("\033[1m" + "{}|{}|{}|{}".format(p.ljust(width_col1), q.ljust(width_col2), r.ljust(width_col3),
s.ljust(width_col4)) + "\033[0m" + "\n" + "--------------------------------------------------"
+ "-------------------------------------------------------------------------")
for a1, b1, c1, d1 in zip(result['Test'], result['Prev_Time'], result['Cur_Time'], result['Difference(%)']):
if d1 > 0:
print ("\033[91m" + "{}|{}|{}|{}".format(a1.ljust(width_col1),
str(b1).ljust(width_col2), str(c1).ljust(width_col3), str(d1).ljust(width_col4)) + "\033[0m")
elif d1 < 0:
print ("\033[92m" + "{}|{}|{}|{}".format(a1.ljust(width_col1),
str(b1).ljust(width_col2), str(c1).ljust(width_col3), str(d1).ljust(width_col4)) + "\033[0m")
else:
print ("{}|{}|{}|{}".format(a1.ljust(width_col1), str(b1).ljust(width_col2),
str(c1).ljust(width_col3), str(d1).ljust(width_col4)))
def jquery_test_runner(self, cmd, release, dev):
self.ensure_bootstrapped()
base_dir = path.abspath(path.join("tests", "jquery"))
jquery_dir = path.join(base_dir, "jquery")
run_file = path.join(base_dir, "run_jquery.py")
# Clone the jQuery repository if it doesn't exist
if not os.path.isdir(jquery_dir):
subprocess.check_call(
["git", "clone", "-b", "servo", "--depth", "1", "https://github.com/servo/jquery", jquery_dir])
# Run pull in case the jQuery repo was updated since last test run
subprocess.check_call(
["git", "-C", jquery_dir, "pull"])
# Check that a release servo build exists
bin_path = path.abspath(self.get_binary_path(release, dev))
return subprocess.check_call(
[run_file, cmd, bin_path, base_dir])
def dromaeo_test_runner(self, tests, release, dev):
self.ensure_bootstrapped()
base_dir = path.abspath(path.join("tests", "dromaeo"))
dromaeo_dir = path.join(base_dir, "dromaeo")
run_file = path.join(base_dir, "run_dromaeo.py")
# Clone the Dromaeo repository if it doesn't exist
if not os.path.isdir(dromaeo_dir):
subprocess.check_call(
["git", "clone", "-b", "servo", "--depth", "1", "https://github.com/notriddle/dromaeo", dromaeo_dir])
# Run pull in case the Dromaeo repo was updated since last test run
subprocess.check_call(
["git", "-C", dromaeo_dir, "pull"])
# Compile test suite
subprocess.check_call(
["make", "-C", dromaeo_dir, "web"])
# Check that a release servo build exists
bin_path = path.abspath(self.get_binary_path(release, dev))
return subprocess.check_call(
[run_file, "|".join(tests), bin_path, base_dir])
def main(paths=None):
parser = wptcommandline.create_parser()
kwargs = vars(parser.parse_args())
return run_tests(paths, **kwargs)
def create_parser_wpt():
parser = wptcommandline.create_parser()
parser.add_argument('--release', default=False, action="store_true",
help="Run with a release build of servo")
return parser
def create_parser_wpt():
from wptrunner import wptcommandline
return wptcommandline.create_parser(["firefox"])
def main():
parser = wptcommandline.create_parser()
kwargs = vars(parser.parse_args())
return update_tests(**kwargs)
def create_parser_wpt():
from wptrunner import wptcommandline
return wptcommandline.create_parser(["firefox"])
class MachCommands(CommandBase):
def __init__(self, context):
CommandBase.__init__(self, context)
if not hasattr(self.context, "built_tests"):
self.context.built_tests = False
def ensure_built_tests(self):
if self.context.built_tests:
return
returncode = Registrar.dispatch('build-tests', context=self.context)
if returncode:
sys.exit(returncode)
self.context.built_tests = True
def find_test(self, prefix):
target_contents = os.listdir(
path.join(self.context.topdir, "components", "servo", "target",
"debug"))
for filename in target_contents:
if filename.startswith(prefix + "-"):
filepath = path.join(self.context.topdir, "components",
"servo", "target", "debug", filename)
if path.isfile(filepath) and os.access(filepath, os.X_OK):
return filepath
def run_test(self, prefix, args=[]):
t = self.find_test(prefix)
if t:
return subprocess.call([t] + args, env=self.build_env())
def infer_test_by_dir(self, params):
maybe_path = path.normpath(params[0])
mach_command = path.join(self.context.topdir, "mach")
args = None
if not path.exists(maybe_path):
print("%s is not a valid file or directory" % maybe_path)
return 1
test_dirs = [
# path, mach test command, optional flag for path argument
(path.join("tests", "wpt"), "test-wpt", None),
(path.join("tests", "ref"), "test-ref", ["--name"]),
]
for test_dir, test_name, path_flag in test_dirs:
if not path_flag:
path_flag = []
if test_dir in maybe_path:
args = ([mach_command, test_name] + path_flag + [maybe_path] +
params[1:])
break
else:
print("%s is not a valid test file or directory" % maybe_path)
return 1
return subprocess.call(args, env=self.build_env())
@Command('test', description='Run all Servo tests', category='testing')
@CommandArgument('params',
default=None,
nargs="...",
help="Optionally select test based on "
"test file directory")
def test(self, params):
if params:
return self.infer_test_by_dir(params)
test_start = time()
for t in ["tidy", "ref", "wpt", "css", "unit"]:
Registrar.dispatch("test-%s" % t, context=self.context)
elapsed = time() - test_start
print("Tests completed in %0.2fs" % elapsed)
@Command('test-unit', description='Run unit tests', category='testing')
@CommandArgument('test_name',
nargs=argparse.REMAINDER,
help="Only run tests that match this pattern")
def test_unit(self, test_name=None, component=None, package=None):
if test_name is None:
test_name = []
self.ensure_bootstrapped()
return 0 != subprocess.call(
["cargo", "test", "-p", "unit_tests"] + test_name,
env=self.build_env(),
cwd=self.servo_crate())
@Command('test-ref',
description='Run the reference tests',
category='testing')
@CommandArgument('--kind',
'-k',
default=None,
help="'cpu' or 'gpu' (default both)")
@CommandArgument('--name',
default=None,
help="Only run tests that match this pattern. If the "
"path to the ref test directory is included, it "
"will automatically be trimmed out.")
@CommandArgument(
'servo_params',
default=None,
nargs=argparse.REMAINDER,
help="Command-line arguments to be passed through to Servo")
def test_ref(self, kind=None, name=None, servo_params=None):
self.ensure_bootstrapped()
self.ensure_built_tests()
kinds = ["cpu", "gpu"] if kind is None else [kind]
test_path = path.join(self.context.topdir, "tests", "ref")
error = False
test_start = time()
for k in kinds:
print("Running %s reftests..." % k)
test_args = [k, test_path]
if name is not None:
maybe_path = path.normpath(name)
ref_path = path.join("tests", "ref")
# Check to see if we were passed something leading with the
# path to the ref test directory, and trim it so that reftest
# knows how to filter it.
if ref_path in maybe_path:
test_args.append(path.relpath(maybe_path, ref_path))
else:
test_args.append(name)
if servo_params is not None:
test_args += ["--"] + servo_params
ret = self.run_test("reftest", test_args)
error = error or ret != 0
elapsed = time() - test_start
print("Reference tests completed in %0.2fs" % elapsed)
if error:
return 1
@Command('test-content',
description='Run the content tests',
category='testing')
def test_content(self):
print("Content tests have been replaced by web-platform-tests under "
"tests/wpt/mozilla/.")
return 0
@Command('test-tidy',
description='Run the source code tidiness check',
category='testing')
def test_tidy(self):
return tidy.scan()
@Command('test-wpt-failure',
description='Run the web platform tests',
category='testing')
def test_wpt_failure(self):
self.ensure_bootstrapped()
return not subprocess.call([
"bash",
path.join("tests", "wpt", "run.sh"), "--no-pause-after-test",
"--include", "infrastructure/failing-test.html"
],
env=self.build_env())
@Command('test-wpt',
description='Run the web platform tests',
category='testing',
parser=wptcommandline.create_parser())
@CommandArgument('--release',
default=False,
action="store_true",
help="Run with a release build of servo")
def test_wpt(self, **kwargs):
self.ensure_bootstrapped()
self.ensure_wpt_virtualenv()
hosts_file_path = path.join('tests', 'wpt', 'hosts')
os.environ["hosts_file_path"] = hosts_file_path
run_file = path.abspath(path.join("tests", "wpt", "run_wpt.py"))
run_globals = {"__file__": run_file}
execfile(run_file, run_globals)
return run_globals["run_tests"](**kwargs)
@Command('update-wpt',
description='Update the web platform tests',
category='testing',
parser=updatecommandline.create_parser())
def update_wpt(self, **kwargs):
self.ensure_bootstrapped()
self.ensure_wpt_virtualenv()
run_file = path.abspath(path.join("tests", "wpt", "update.py"))
run_globals = {"__file__": run_file}
execfile(run_file, run_globals)
return run_globals["update_tests"](**kwargs)
@Command('test-css',
description='Run the web platform tests',
category='testing',
parser=wptcommandline.create_parser())
@CommandArgument('--release',
default=False,
action="store_true",
help="Run with a release build of servo")
def test_css(self, **kwargs):
self.ensure_bootstrapped()
self.ensure_wpt_virtualenv()
run_file = path.abspath(path.join("tests", "wpt", "run_css.py"))
run_globals = {"__file__": run_file}
execfile(run_file, run_globals)
return run_globals["run_tests"](**kwargs)
@Command('update-css',
description='Update the web platform tests',
category='testing',
parser=updatecommandline.create_parser())
def update_css(self, **kwargs):
self.ensure_bootstrapped()
self.ensure_wpt_virtualenv()
run_file = path.abspath(path.join("tests", "wpt", "update_css.py"))
run_globals = {"__file__": run_file}
execfile(run_file, run_globals)
return run_globals["update_tests"](**kwargs)
def ensure_wpt_virtualenv(self):
virtualenv_path = path.join("tests", "wpt", "_virtualenv")
python = self.get_exec("python2", "python")
if not os.path.exists(virtualenv_path):
virtualenv = self.get_exec("virtualenv2", "virtualenv")
subprocess.check_call([virtualenv, "-p", python, virtualenv_path])
activate_path = path.join(virtualenv_path, "bin", "activate_this.py")
execfile(activate_path, dict(__file__=activate_path))
try:
import wptrunner
from wptrunner.browsers import servo
except ImportError:
subprocess.check_call([
"pip", "install", "-r",
path.join("tests", "wpt", "harness", "requirements.txt")
])
subprocess.check_call([
"pip", "install", "-r",
path.join("tests", "wpt", "harness", "requirements_servo.txt")
])
try:
import blessings
except ImportError:
subprocess.check_call(["pip", "install", "blessings"])
# This is an unfortunate hack. Because mozlog gets imported by wptcommandline
# before the virtualenv is initalised it doesn't see the blessings module so we don't
# get coloured output. Setting the blessings global explicitly fixes that.
from mozlog.structured.formatters import machformatter
import blessings
machformatter.blessings = blessings
def get_exec(self, name, default=None):
path = find_executable(name)
if not path:
return default
return path
def main():
parser = wptcommandline.create_parser()
args = parser.parse_args()
kwargs = set_defaults(args)
return run_tests(**kwargs)
def create_parser_wpt():
from wptrunner import wptcommandline
return wptcommandline.create_parser(["firefox", "firefox_android", "chrome", "edge", "servo"])
def load_wpt_tests(xul_tester, requested_paths, excluded_paths, update_manifest=True):
"""Return a list of `RefTestCase` objects for the jsshell testharness.js
tests filtered by the given paths and debug-ness."""
repo_root = abspath(os.path.join(here, "..", "..", ".."))
wp = os.path.join(repo_root, "testing", "web-platform")
wpt = os.path.join(wp, "tests")
sys_paths = [
"python/mozterm",
"python/mozboot",
"testing/mozbase/mozcrash",
"testing/mozbase/mozdevice",
"testing/mozbase/mozfile",
"testing/mozbase/mozinfo",
"testing/mozbase/mozleak",
"testing/mozbase/mozlog",
"testing/mozbase/mozprocess",
"testing/mozbase/mozprofile",
"testing/mozbase/mozrunner",
"testing/mozbase/mozversion",
"testing/web-platform/",
"testing/web-platform/tests/tools",
"testing/web-platform/tests/tools/third_party/html5lib",
"testing/web-platform/tests/tools/third_party/webencodings",
"testing/web-platform/tests/tools/wptrunner",
"testing/web-platform/tests/tools/wptserve",
"third_party/python/requests",
]
abs_sys_paths = [os.path.join(repo_root, path) for path in sys_paths]
failed = False
for path in abs_sys_paths:
if not os.path.isdir(path):
failed = True
print("Could not add '%s' to the path")
if failed:
return []
sys.path[0:0] = abs_sys_paths
import manifestupdate
from wptrunner import products, testloader, wptcommandline, wpttest, wptlogging
manifest_root = tempfile.gettempdir()
path_split = os.path.dirname(xul_tester.js_bin).split(os.path.sep)
if path_split[-2:] == ["dist", "bin"]:
maybe_root = os.path.join(*path_split[:-2])
if os.path.exists(os.path.join(maybe_root, "_tests")):
# Assume this is a gecko objdir.
manifest_root = maybe_root
logger = wptlogging.setup({}, {})
test_manifests = manifestupdate.run(repo_root, manifest_root, logger,
update=update_manifest)
kwargs = vars(wptcommandline.create_parser().parse_args([]))
kwargs.update({
"config": os.path.join(manifest_root, "_tests", "web-platform", "wptrunner.local.ini"),
"gecko_e10s": False,
"verify": False,
"wasm": xul_tester.test("wasmIsSupported()"),
})
wptcommandline.set_from_config(kwargs)
def filter_jsshell_tests(it):
for item_type, path, tests in it:
tests = set(item for item in tests if item.jsshell)
if tests:
yield item_type, path, tests
run_info_extras = products.load_product(kwargs["config"], "firefox")[-1](**kwargs)
run_info = wpttest.get_run_info(kwargs["test_paths"]["/"]["metadata_path"],
"firefox",
debug=xul_tester.test("isDebugBuild"),
extras=run_info_extras)
release_or_beta = xul_tester.test("getBuildConfiguration().release_or_beta")
run_info["release_or_beta"] = release_or_beta
run_info["nightly_build"] = not release_or_beta
path_filter = testloader.TestFilter(test_manifests,
include=requested_paths,
exclude=excluded_paths)
loader = testloader.TestLoader(test_manifests,
["testharness"],
run_info,
manifest_filters=[path_filter, filter_jsshell_tests])
extra_helper_paths = [
os.path.join(here, "web-platform-test-shims.js"),
os.path.join(wpt, "resources", "testharness.js"),
os.path.join(here, "testharnessreport.js"),
]
def resolve(test_path, script):
if script.startswith("/"):
return os.path.join(wpt, script[1:])
return os.path.join(wpt, os.path.dirname(test_path), script)
tests = []
for test in loader.tests["testharness"]:
test_path = os.path.relpath(test.path, wpt)
scripts = [resolve(test_path, s) for s in test.scripts]
extra_helper_paths_for_test = extra_helper_paths + scripts
# We must create at least one test with the default options, along with
# one test for each option given in a test-also annotation.
options = [None]
for m in test.itermeta():
if m.has_key("test-also"): # NOQA: W601
options += m.get("test-also").split()
for option in options:
test_case = RefTestCase(
wpt,
test_path,
extra_helper_paths=extra_helper_paths_for_test[:],
wpt=test
)
if option:
test_case.options.append(option)
tests.append(test_case)
return tests
def __call__(self, parser, namespace, values, option_string=None):
from wptrunner import wptcommandline
wptparser = wptcommandline.create_parser()
wptparser.usage = parser.usage
wptparser.print_help()
parser.exit()
def load_wpt_tests(xul_tester, requested_paths, excluded_paths):
"""Return a list of `RefTestCase` objects for the jsshell testharness.js
tests filtered by the given paths and debug-ness."""
repo_root = abspath(os.path.join(here, "..", "..", ".."))
wp = os.path.join(repo_root, "testing", "web-platform")
wpt = os.path.join(wp, "tests")
sys_paths = [
"python/mozterm",
"python/mozboot",
"testing/mozbase/mozcrash",
"testing/mozbase/mozdevice",
"testing/mozbase/mozfile",
"testing/mozbase/mozinfo",
"testing/mozbase/mozleak",
"testing/mozbase/mozlog",
"testing/mozbase/mozprocess",
"testing/mozbase/mozprofile",
"testing/mozbase/mozrunner",
"testing/web-platform/",
"testing/web-platform/tests/tools",
"testing/web-platform/tests/tools/third_party/html5lib",
"testing/web-platform/tests/tools/third_party/webencodings",
"testing/web-platform/tests/tools/wptrunner",
"testing/web-platform/tests/tools/wptserve",
"third_party/python/requests",
]
abs_sys_paths = [os.path.join(repo_root, path) for path in sys_paths]
failed = False
for path in abs_sys_paths:
if not os.path.isdir(path):
failed = True
print("Could not add '%s' to the path")
if failed:
return []
sys.path[0:0] = abs_sys_paths
import manifestupdate
from wptrunner import products, testloader, wptcommandline, wpttest, wptlogging
manifest_root = tempfile.gettempdir()
path_split = os.path.dirname(xul_tester.js_bin).split(os.path.sep)
if path_split[-2:] == ["dist", "bin"]:
maybe_root = os.path.join(*path_split[:-2])
if os.path.exists(os.path.join(maybe_root, "_tests")):
# Assume this is a gecko objdir.
manifest_root = maybe_root
logger = wptlogging.setup({}, {})
manifestupdate.run(repo_root, manifest_root, logger)
kwargs = vars(wptcommandline.create_parser().parse_args([]))
kwargs.update({
"config": os.path.join(manifest_root, "_tests", "web-platform", "wptrunner.local.ini"),
"gecko_e10s": False,
"verify": False,
"wasm": xul_tester.test("wasmIsSupported()"),
})
wptcommandline.set_from_config(kwargs)
test_paths = kwargs["test_paths"]
def filter_jsshell_tests(it):
for test in it:
if test[1].get("jsshell"):
yield test
test_manifests = testloader.ManifestLoader(test_paths, types=["testharness"],
meta_filters=[filter_jsshell_tests]).load()
run_info_extras = products.load_product(kwargs["config"], "firefox")[-1](**kwargs)
run_info = wpttest.get_run_info(kwargs["test_paths"]["/"]["metadata_path"],
"firefox",
debug=xul_tester.test("isDebugBuild"),
extras=run_info_extras)
path_filter = testloader.TestFilter(test_manifests,
include=requested_paths,
exclude=excluded_paths)
loader = testloader.TestLoader(test_manifests,
["testharness"],
run_info,
manifest_filters=[path_filter])
extra_helper_paths = [
os.path.join(here, "web-platform-test-shims.js"),
os.path.join(wpt, "resources", "testharness.js"),
os.path.join(here, "testharnessreport.js"),
]
def resolve(test_path, script):
if script.startswith("/"):
return os.path.join(wpt, script[1:])
return os.path.join(wpt, os.path.dirname(test_path), script)
return [
RefTestCase(
wpt,
test_path,
extra_helper_paths=extra_helper_paths + [resolve(test_path, s) for s in test.scripts],
wpt=test
)
for test_path, test in (
(os.path.relpath(test.path, wpt), test) for test in loader.tests["testharness"]
)
]
def main(paths=None):
parser = wptcommandline.create_parser()
kwargs = vars(parser.parse_args())
return run_tests(paths, **kwargs)
def main():
ensure_manifest()
parser = wptcommandline.create_parser(False)
args = parser.parse_args()
kwargs = set_defaults(args)
return run_tests(**kwargs)
def __call__(self, parser, namespace, values, option_string=None):
from wptrunner import wptcommandline
wptparser = wptcommandline.create_parser()
wptparser.usage = parser.usage
wptparser.print_help()
parser.exit()
def create_parser_wpt():
parser = wptcommandline.create_parser()
parser.add_argument('--release', default=False, action="store_true",
help="Run with a release build of servo")
return parser
def main():
parser = wptcommandline.create_parser()
args = parser.parse_args()
kwargs = set_defaults(args)
return run_tests(**kwargs)
def load_wpt_tests(xul_tester, requested_paths, excluded_paths):
"""Return a list of `RefTestCase` objects for the jsshell testharness.js
tests filtered by the given paths and debug-ness."""
repo_root = abspath(os.path.join(here, "..", "..", ".."))
wp = os.path.join(repo_root, "testing", "web-platform")
wpt = os.path.join(wp, "tests")
sys_paths = [
"python/mozterm",
"python/mozboot",
"testing/mozbase/mozcrash",
"testing/mozbase/mozdevice",
"testing/mozbase/mozfile",
"testing/mozbase/mozinfo",
"testing/mozbase/mozleak",
"testing/mozbase/mozlog",
"testing/mozbase/mozprocess",
"testing/mozbase/mozprofile",
"testing/mozbase/mozrunner",
"testing/web-platform/",
"testing/web-platform/tests/tools",
"testing/web-platform/tests/tools/third_party/html5lib",
"testing/web-platform/tests/tools/third_party/webencodings",
"testing/web-platform/tests/tools/wptrunner",
"testing/web-platform/tests/tools/wptserve",
"third_party/python/requests",
]
abs_sys_paths = [os.path.join(repo_root, path) for path in sys_paths]
failed = False
for path in abs_sys_paths:
if not os.path.isdir(path):
failed = True
print("Could not add '%s' to the path")
if failed:
return []
sys.path[0:0] = abs_sys_paths
import manifestupdate
from wptrunner import products, testloader, wptcommandline, wpttest, wptlogging
manifest_root = tempfile.gettempdir()
path_split = os.path.dirname(xul_tester.js_bin).split(os.path.sep)
if path_split[-2:] == ["dist", "bin"]:
maybe_root = os.path.join(*path_split[:-2])
if os.path.exists(os.path.join(maybe_root, "_tests")):
# Assume this is a gecko objdir.
manifest_root = maybe_root
logger = wptlogging.setup({}, {})
manifestupdate.run(repo_root, manifest_root, logger)
kwargs = vars(wptcommandline.create_parser().parse_args([]))
kwargs.update({
"config": os.path.join(manifest_root, "_tests", "web-platform", "wptrunner.local.ini"),
"gecko_e10s": False,
"verify": False,
"wasm": xul_tester.test("wasmIsSupported()"),
})
wptcommandline.set_from_config(kwargs)
test_paths = kwargs["test_paths"]
def filter_jsshell_tests(it):
for test in it:
if test[1].get("jsshell"):
yield test
test_manifests = testloader.ManifestLoader(test_paths, types=["testharness"],
meta_filters=[filter_jsshell_tests]).load()
run_info_extras = products.load_product(kwargs["config"], "firefox")[-1](**kwargs)
run_info = wpttest.get_run_info(kwargs["test_paths"]["/"]["metadata_path"],
"firefox",
debug=xul_tester.test("isDebugBuild"),
extras=run_info_extras)
path_filter = testloader.TestFilter(test_manifests,
include=requested_paths,
exclude=excluded_paths)
loader = testloader.TestLoader(test_manifests,
["testharness"],
run_info,
manifest_filters=[path_filter])
extra_helper_paths = [
os.path.join(here, "web-platform-test-shims.js"),
os.path.join(wpt, "resources", "testharness.js"),
os.path.join(here, "testharnessreport.js"),
]
def resolve(test_path, script):
if script.startswith("/"):
return os.path.join(wpt, script[1:])
return os.path.join(wpt, os.path.dirname(test_path), script)
return [
RefTestCase(
wpt,
test_path,
extra_helper_paths=extra_helper_paths + [resolve(test_path, s) for s in test.scripts],
wpt=test
)
for test_path, test in (
(os.path.relpath(test.path, wpt), test) for test in loader.tests["testharness"]
)
]
