def setup_wptrunner(venv, product, tests, wptrunner_args, prompt=True,): global logger wptparser = wptcommandline.create_parser() kwargs = utils.Kwargs(vars(wptparser.parse_args(wptrunner_args)).iteritems()) wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) logger = wptrunner.logger kwargs["product"] = product kwargs["test_list"] = tests check_environ(product) args_general(kwargs) if product not in product_setup: exit("Unsupported product %s" % product) product_setup[product](venv, kwargs, prompt) wptcommandline.check_args(kwargs) wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner") venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt")) return kwargs
def run(config, args): logger = structuredlog.StructuredLogger("web-platform-tests") logger.add_handler(ResultHandler(logger=logger, verbose=args.verbose)) setup_wptrunner_logging(logger) parser = wptcommandline.create_parser() logger.suite_start(tests=[]) for product, product_settings in config["products"].iteritems(): if args.product and product not in args.product: continue settings = test_settings() settings.update(config["general"]) settings.update(product_settings) settings["product"] = product set_from_args(settings, args) kwargs = vars(parser.parse_args(settings_to_argv(settings))) wptcommandline.check_args(kwargs) logger.send_message("wptrunner-test", "set-product", product) run_tests(product, kwargs) logger.send_message("wptrunner-test", "set-product", None) logger.suite_end()
def create_parser_wpt(): parser = wptcommandline.create_parser() parser.add_argument('--release', default=False, action="store_true", help="Run with a release build of servo") parser.add_argument('--chaos', default=False, action="store_true", help="Run under chaos mode in rr until a failure is captured") return parser
def main(): parser = wptcommandline.create_parser() parser.add_argument('--update-manifest', dest='update_manifest', action='store_true') args = parser.parse_args() if args.update_manifest: return update_manifest() kwargs = set_defaults(args) return run_tests(**kwargs)
def create_parser_wpt(): parser = wptcommandline.create_parser() parser.add_argument('--release', default=False, action="store_true", help="Run with a release build of servo") parser.add_argument('--chaos', default=False, action="store_true", help="Run under chaos mode in rr until a failure is captured") parser.add_argument('--pref', default=[], action="append", dest="prefs", help="Pass preferences to servo") parser.add_argument('--always-succeed', default=False, action="store_true", help="Always yield exit code of zero") return parser
def create_parser(): from wptrunner import wptcommandline parser = argparse.ArgumentParser(add_help=False) parser.add_argument("product", action="store", help="Browser to run tests in") parser.add_argument("--yes", "-y", dest="prompt", action="store_false", default=True, help="Don't prompt before installing components") parser.add_argument("--stability", action="store_true", help="Stability check tests") parser.add_argument("--install-browser", action="store_true", help="Install the latest development version of the browser") parser._add_container_actions(wptcommandline.create_parser()) return parser
def wptrunner_args(root, files_changed, iterations, browser): """Derive and return arguments for wpt-runner.""" parser = wptcommandline.create_parser([browser.product]) args = vars(parser.parse_args([])) args.update(browser.wptrunner_args(root)) args.update({ "tests_root": wpt_root, "metadata_root": wpt_root, "repeat": iterations, "config": "%s//wptrunner.default.ini" % (wptrunner_root), "test_list": files_changed, "restart_on_unexpected": False, "pause_after_test": False }) wptcommandline.check_args(args) return args
def wptrunner_args(root, files_changed, iterations, browser): parser = wptcommandline.create_parser([browser.product]) args = vars(parser.parse_args([])) wpt_root = os.path.join(root, "w3c", "web-platform-tests") args.update(browser.wptrunner_args(root)) args.update({ "tests_root": wpt_root, "metadata_root": wpt_root, "repeat": iterations, "config": "%s/w3c/wptrunner/wptrunner.default.ini" % root, "test_list": files_changed, "restart_on_unexpected": False, "pause_after_test": False }) wptcommandline.check_args(args) return args
def create_parser(): from wptrunner import wptcommandline parser = argparse.ArgumentParser(add_help=False) parser.add_argument("product", action="store", help="Browser to run tests in") parser.add_argument("--yes", "-y", dest="prompt", action="store_false", default=True, help="Don't prompt before installing components") parser.add_argument("--install-browser", action="store_true", help="Install the browser") parser.add_argument("--channel", action="store", choices=install.channel_by_name.keys(), default=None, help='Name of browser release channel.' '"stable" and "release" are synonyms for the latest browser stable release,' '"nightly", "dev", "experimental", and "preview" are all synonyms for ' 'the latest available development release. For WebDriver installs, ' 'we attempt to select an appropriate, compatible, version for the ' 'latest browser release on the selected channel.') parser._add_container_actions(wptcommandline.create_parser()) return parser
def create_parser(): from wptrunner import wptcommandline parser = argparse.ArgumentParser(add_help=False) parser.add_argument("product", action="store", help="Browser to run tests in") parser.add_argument("--yes", "-y", dest="prompt", action="store_false", default=True, help="Don't prompt before installing components") parser.add_argument("--stability", action="store_true", help="Stability check tests") parser.add_argument( "--install-browser", action="store_true", help="Install the latest development version of the browser") parser._add_container_actions(wptcommandline.create_parser()) return parser
def create_parser_wpt(): import mozlog.commandline parser = wptcommandline.create_parser() parser.add_argument('--release', default=False, action="store_true", help="Run with a release build of servo") parser.add_argument( '--rr-chaos', default=False, action="store_true", help="Run under chaos mode in rr until a failure is captured") parser.add_argument('--pref', default=[], action="append", dest="prefs", help="Pass preferences to servo") parser.add_argument('--layout-2020', default=False, action="store_true", help="Use expected results for the 2020 layout engine") parser.add_argument('--log-servojson', action="append", type=mozlog.commandline.log_file, help="Servo's JSON logger of unexpected results") parser.add_argument('--always-succeed', default=False, action="store_true", help="Always yield exit code of zero") parser.add_argument( '--no-default-test-types', default=False, action="store_true", help= "Run all of the test types provided by wptrunner or specified explicitly by --test-types" ), return parser
def create_parser(): from wptrunner import wptcommandline parser = argparse.ArgumentParser(add_help=False) parser.add_argument("product", action="store", help="Browser to run tests in") parser.add_argument("--affected", action="store", default=None, help="Run affected tests since revish") parser.add_argument("--yes", "-y", dest="prompt", action="store_false", default=True, help="Don't prompt before installing components") parser.add_argument( "--install-browser", action="store_true", help= "Install the browser from the release channel specified by --channel " "(or the nightly channel by default).") parser.add_argument( "--channel", action="store", choices=install.channel_by_name.keys(), default=None, help='Name of browser release channel. ' '"stable" and "release" are synonyms for the latest browser stable ' 'release, "nightly", "dev", "experimental", and "preview" are all ' 'synonyms for the latest available development release. (For WebDriver ' 'installs, we attempt to select an appropriate, compatible version for ' 'the latest browser release on the selected channel.) ' 'This flag overrides --browser-channel.') parser._add_container_actions(wptcommandline.create_parser()) return parser
class MachCommands(MachCommandBase): @Command("web-platform-tests", category="testing", conditions=[conditions.is_firefox], parser=wptcommandline.create_parser(["firefox"])) def run_web_platform_tests(self, **params): self.setup() wpt_runner = self._spawn(WebPlatformTestsRunner) if params["list_test_groups"]: return wpt_runner.list_test_groups(**params) else: return wpt_runner.run_tests(**params) @Command("web-platform-tests-update", category="testing", conditions=[conditions.is_firefox], parser=wptcommandline.create_parser_update()) def update_web_platform_tests(self, **params): self.setup() self.virtualenv_manager.install_pip_package('html5lib==0.99') wpt_updater = self._spawn(WebPlatformTestsUpdater) return wpt_updater.run_update(**params) def setup(self): self._activate_virtualenv() self.virtualenv_manager.install_pip_package('py==1.4.14') @Command("web-platform-tests-reduce", category="testing", conditions=[conditions.is_firefox], parser=wptcommandline.create_parser_reduce(["firefox"])) def unstable_web_platform_tests(self, **params): self.setup() wpt_reduce = self._spawn(WebPlatformTestsReduce) return wpt_reduce.run_reduce(**params)
def create_parser(): from wptrunner import wptcommandline parser = argparse.ArgumentParser(add_help=False) parser.add_argument("product", action="store", help="Browser to run tests in") parser.add_argument("--affected", action="store", default=None, help="Run affected tests since revish") parser.add_argument("--yes", "-y", dest="prompt", action="store_false", default=True, help="Don't prompt before installing components") parser.add_argument("--install-browser", action="store_true", help="Install the browser from the release channel specified by --channel " "(or the nightly channel by default).") parser.add_argument("--channel", action="store", choices=install.channel_by_name.keys(), default=None, help='Name of browser release channel. ' '"stable" and "release" are synonyms for the latest browser stable ' 'release, "nightly", "dev", "experimental", and "preview" are all ' 'synonyms for the latest available development release. (For WebDriver ' 'installs, we attempt to select an appropriate, compatible version for ' 'the latest browser release on the selected channel.) ' 'This flag overrides --browser-channel.') parser._add_container_actions(wptcommandline.create_parser()) return parser
def main(): parser = wptcommandline.create_parser() kwargs = vars(parser.parse_args()) return update_tests(**kwargs)
class MachCommands(CommandBase): DEFAULT_RENDER_MODE = "cpu" HELP_RENDER_MODE = "Value can be 'cpu', 'gpu' or 'both' (default " + DEFAULT_RENDER_MODE + ")" def __init__(self, context): CommandBase.__init__(self, context) if not hasattr(self.context, "built_tests"): self.context.built_tests = False def ensure_built_tests(self): if self.context.built_tests: return returncode = Registrar.dispatch('build-tests', context=self.context) if returncode: sys.exit(returncode) self.context.built_tests = True def find_test(self, prefix): target_contents = os.listdir(path.join(self.get_target_dir(), "debug")) for filename in target_contents: if filename.startswith(prefix + "-"): filepath = path.join(self.get_target_dir(), "debug", filename) if path.isfile(filepath) and os.access(filepath, os.X_OK): return filepath def run_test(self, prefix, args=[]): t = self.find_test(prefix) if t: return subprocess.call([t] + args, env=self.build_env()) @Command('test', description='Run all Servo tests', category='testing') @CommandArgument('params', default=None, nargs="...", help="Optionally select test based on " "test file directory") @CommandArgument('--render-mode', '-rm', default=DEFAULT_RENDER_MODE, help="The render mode to be used on all tests. " + HELP_RENDER_MODE) @CommandArgument('--release', default=False, action="store_true", help="Run with a release build of servo") def test(self, params, render_mode=DEFAULT_RENDER_MODE, release=False): suites = OrderedDict([ ("tidy", {}), ("ref", { "kwargs": { "kind": render_mode }, "path": path.abspath(path.join("tests", "ref")), "include_arg": "name" }), ("wpt", { "kwargs": { "release": release }, "path": path.abspath(path.join("tests", "wpt", "web-platform-tests")), "include_arg": "include" }), ("css", { "kwargs": { "release": release }, "path": path.abspath(path.join("tests", "wpt", "css-tests")), "include_arg": "include" }), ("unit", {}), ]) suites_by_prefix = { v["path"]: k for k, v in suites.iteritems() if "path" in v } selected_suites = OrderedDict() if params is None: params = suites.keys() for arg in params: found = False if arg in suites and arg not in selected_suites: selected_suites[arg] = [] found = True elif os.path.exists(path.abspath(arg)): abs_path = path.abspath(arg) for prefix, suite in suites_by_prefix.iteritems(): if abs_path.startswith(prefix): if suite not in selected_suites: selected_suites[suite] = [] selected_suites[suite].append(arg) found = True break if not found: print("%s is not a valid test path or suite name" % arg) return 1 test_start = time() for suite, tests in selected_suites.iteritems(): props = suites[suite] kwargs = props.get("kwargs", {}) if tests: kwargs[props["include_arg"]] = tests Registrar.dispatch("test-%s" % suite, context=self.context, **kwargs) elapsed = time() - test_start print("Tests completed in %0.2fs" % elapsed) @Command('test-unit', description='Run unit tests', category='testing') @CommandArgument('--package', '-p', default=None, help="Specific package to test") @CommandArgument('test_name', nargs=argparse.REMAINDER, help="Only run tests that match this pattern") def test_unit(self, test_name=None, package=None): if test_name is None: test_name = [] self.ensure_bootstrapped() if package: packages = [package] else: packages = os.listdir( path.join(self.context.topdir, "tests", "unit")) for crate in packages: result = subprocess.call( ["cargo", "test", "-p", "%s_tests" % crate] + test_name, env=self.build_env(), cwd=self.servo_crate()) if result != 0: return result @Command('test-ref', description='Run the reference tests', category='testing') @CommandArgument('--kind', '-k', default=DEFAULT_RENDER_MODE, help=HELP_RENDER_MODE) @CommandArgument('--name', default=None, help="Only run tests that match this pattern. If the " "path to the ref test directory is included, it " "will automatically be trimmed out.") @CommandArgument( 'servo_params', default=None, nargs=argparse.REMAINDER, help="Command-line arguments to be passed through to Servo") def test_ref(self, kind=DEFAULT_RENDER_MODE, name=None, servo_params=None): self.ensure_bootstrapped() self.ensure_built_tests() assert kind is not None, 'kind cannot be None, see help' kinds = ["cpu", "gpu"] if kind == 'both' else [kind] test_path = path.join(self.context.topdir, "tests", "ref") error = False test_start = time() for k in kinds: print("Running %s reftests..." % k) test_args = [k, test_path] if name is not None: maybe_path = path.normpath(name) ref_path = path.join("tests", "ref") # Check to see if we were passed something leading with the # path to the ref test directory, and trim it so that reftest # knows how to filter it. if ref_path in maybe_path: test_args.append(path.relpath(maybe_path, ref_path)) else: test_args.append(name) if servo_params is not None: test_args += ["--"] + servo_params ret = self.run_test("reftest", test_args) error = error or ret != 0 elapsed = time() - test_start print("Reference tests completed in %0.2fs" % elapsed) if error: return 1 @Command('test-content', description='Run the content tests', category='testing') def test_content(self): print("Content tests have been replaced by web-platform-tests under " "tests/wpt/mozilla/.") return 0 @Command('test-tidy', description='Run the source code tidiness check', category='testing') def test_tidy(self): return tidy.scan() @Command('test-wpt-failure', description='Run the web platform tests', category='testing') def test_wpt_failure(self): self.ensure_bootstrapped() return not subprocess.call([ "bash", path.join("tests", "wpt", "run.sh"), "--no-pause-after-test", "--include", "infrastructure/failing-test.html" ], env=self.build_env()) @Command('test-wpt', description='Run the web platform tests', category='testing', parser=wptcommandline.create_parser) @CommandArgument('--release', default=False, action="store_true", help="Run with a release build of servo") def test_wpt(self, **kwargs): self.ensure_bootstrapped() self.ensure_wpt_virtualenv() hosts_file_path = path.join('tests', 'wpt', 'hosts') os.environ["hosts_file_path"] = hosts_file_path run_file = path.abspath(path.join("tests", "wpt", "run_wpt.py")) run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["run_tests"](**kwargs) @Command('update-wpt', description='Update the web platform tests', category='testing', parser=updatecommandline.create_parser()) def update_wpt(self, **kwargs): self.ensure_bootstrapped() self.ensure_wpt_virtualenv() run_file = path.abspath(path.join("tests", "wpt", "update.py")) run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["update_tests"](**kwargs) @Command('test-jquery', description='Run the jQuery test suite', category='testing') @CommandArgument('--release', '-r', action='store_true', help='Run the release build') @CommandArgument('--dev', '-d', action='store_true', help='Run the dev build') def test_jquery(self, release, dev): return self.jquery_test_runner("test", release, dev) @Command('update-jquery', description='Update the jQuery test suite expected results', category='testing') @CommandArgument('--release', '-r', action='store_true', help='Run the release build') @CommandArgument('--dev', '-d', action='store_true', help='Run the dev build') def update_jquery(self, release, dev): return self.jquery_test_runner("update", release, dev) @Command('test-css', description='Run the web platform tests', category='testing', parser=wptcommandline.create_parser()) @CommandArgument('--release', default=False, action="store_true", help="Run with a release build of servo") def test_css(self, **kwargs): self.ensure_bootstrapped() self.ensure_wpt_virtualenv() run_file = path.abspath(path.join("tests", "wpt", "run_css.py")) run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["run_tests"](**kwargs) @Command('update-css', description='Update the web platform tests', category='testing', parser=updatecommandline.create_parser()) def update_css(self, **kwargs): self.ensure_bootstrapped() self.ensure_wpt_virtualenv() run_file = path.abspath(path.join("tests", "wpt", "update_css.py")) run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["update_tests"](**kwargs) def ensure_wpt_virtualenv(self): virtualenv_path = path.join("tests", "wpt", "_virtualenv") python = self.get_exec("python2", "python") if not os.path.exists(virtualenv_path): virtualenv = self.get_exec("virtualenv2", "virtualenv") subprocess.check_call([virtualenv, "-p", python, virtualenv_path]) activate_path = path.join(virtualenv_path, "bin", "activate_this.py") execfile(activate_path, dict(__file__=activate_path)) try: import wptrunner # noqa from wptrunner.browsers import servo # noqa except ImportError: subprocess.check_call([ "pip", "install", "-r", path.join("tests", "wpt", "harness", "requirements.txt") ]) subprocess.check_call([ "pip", "install", "-r", path.join("tests", "wpt", "harness", "requirements_servo.txt") ]) try: import blessings except ImportError: subprocess.check_call(["pip", "install", "blessings"]) # This is an unfortunate hack. Because mozlog gets imported by wptcommandline # before the virtualenv is initalised it doesn't see the blessings module so we don't # get coloured output. Setting the blessings global explicitly fixes that. from mozlog.structured.formatters import machformatter import blessings # noqa machformatter.blessings = blessings def get_exec(self, name, default=None): path = find_executable(name) if not path: return default return path def jquery_test_runner(self, cmd, release, dev): self.ensure_bootstrapped() base_dir = path.abspath(path.join("tests", "jquery")) jquery_dir = path.join(base_dir, "jquery") run_file = path.join(base_dir, "run_jquery.py") # Clone the jQuery repository if it doesn't exist if not os.path.isdir(jquery_dir): subprocess.check_call([ "git", "clone", "-b", "servo", "--depth", "1", "https://github.com/servo/jquery", jquery_dir ]) # Run pull in case the jQuery repo was updated since last test run subprocess.check_call(["git", "-C", jquery_dir, "pull"]) # Check that a release servo build exists bin_path = path.abspath(self.get_binary_path(release, dev)) return subprocess.check_call([run_file, cmd, bin_path, base_dir])
def create_parser_wpt(): from wptrunner import wptcommandline return wptcommandline.create_parser(["firefox", "chrome", "edge", "servo"])
class MachCommands(CommandBase): DEFAULT_RENDER_MODE = "cpu" HELP_RENDER_MODE = "Value can be 'cpu', 'gpu' or 'both' (default " + DEFAULT_RENDER_MODE + ")" def __init__(self, context): CommandBase.__init__(self, context) if not hasattr(self.context, "built_tests"): self.context.built_tests = False def ensure_built_tests(self, release=False): if self.context.built_tests: return returncode = Registrar.dispatch( 'build-tests', context=self.context, release=release) if returncode: sys.exit(returncode) self.context.built_tests = True def find_test(self, prefix, release=False): build_mode = "release" if release else "debug" target_contents = os.listdir(path.join( self.get_target_dir(), build_mode)) for filename in target_contents: if filename.startswith(prefix + "-"): filepath = path.join( self.get_target_dir(), build_mode, filename) if path.isfile(filepath) and os.access(filepath, os.X_OK): return filepath def run_test(self, prefix, args=[], release=False): t = self.find_test(prefix, release=release) if t: return subprocess.call([t] + args, env=self.build_env()) @Command('test', description='Run all Servo tests', category='testing') @CommandArgument('params', default=None, nargs="...", help="Optionally select test based on " "test file directory") @CommandArgument('--render-mode', '-rm', default=DEFAULT_RENDER_MODE, help="The render mode to be used on all tests. " + HELP_RENDER_MODE) @CommandArgument('--release', default=False, action="store_true", help="Run with a release build of servo") def test(self, params, render_mode=DEFAULT_RENDER_MODE, release=False): suites = OrderedDict([ ("tidy", {}), ("ref", {"kwargs": {"kind": render_mode}, "paths": [path.abspath(path.join("tests", "ref"))], "include_arg": "include"}), ("wpt", {"kwargs": {"release": release}, "paths": [path.abspath(path.join("tests", "wpt", "web-platform-tests")), path.abspath(path.join("tests", "wpt", "mozilla"))], "include_arg": "include"}), ("css", {"kwargs": {"release": release}, "paths": [path.abspath(path.join("tests", "wpt", "css-tests"))], "include_arg": "include"}), ("unit", {"kwargs": {}, "paths": [path.abspath(path.join("tests", "unit"))], "include_arg": "test_name"}) ]) suites_by_prefix = {path: k for k, v in suites.iteritems() if "paths" in v for path in v["paths"]} selected_suites = OrderedDict() if params is None: params = suites.keys() for arg in params: found = False if arg in suites and arg not in selected_suites: selected_suites[arg] = [] found = True elif os.path.exists(path.abspath(arg)): abs_path = path.abspath(arg) for prefix, suite in suites_by_prefix.iteritems(): if abs_path.startswith(prefix): if suite not in selected_suites: selected_suites[suite] = [] selected_suites[suite].append(arg) found = True break if not found: print("%s is not a valid test path or suite name" % arg) return 1 test_start = time() for suite, tests in selected_suites.iteritems(): props = suites[suite] kwargs = props.get("kwargs", {}) if tests: kwargs[props["include_arg"]] = tests Registrar.dispatch("test-%s" % suite, context=self.context, **kwargs) elapsed = time() - test_start print("Tests completed in %0.2fs" % elapsed) @Command('test-unit', description='Run unit tests', category='testing') @CommandArgument('--package', '-p', default=None, help="Specific package to test") @CommandArgument('test_name', nargs=argparse.REMAINDER, help="Only run tests that match this pattern or file path") def test_unit(self, test_name=None, package=None): if test_name is None: test_name = [] self.ensure_bootstrapped() if package: packages = {package} else: packages = set() test_patterns = [] for test in test_name: # add package if 'tests/unit/<package>' match = re.search("tests/unit/(\\w+)/?$", test) if match: packages.add(match.group(1)) # add package & test if '<package>/<test>', 'tests/unit/<package>/<test>.rs', or similar elif re.search("\\w/\\w", test): tokens = test.split("/") packages.add(tokens[-2]) test_prefix = tokens[-1] if test_prefix.endswith(".rs"): test_prefix = test_prefix[:-3] test_prefix += "::" test_patterns.append(test_prefix) # add test as-is otherwise else: test_patterns.append(test) if not packages: packages = set(os.listdir(path.join(self.context.topdir, "tests", "unit"))) args = ["cargo", "test"] for crate in packages: args += ["-p", "%s_tests" % crate] args += test_patterns result = subprocess.call(args, env=self.build_env(), cwd=self.servo_crate()) if result != 0: return result @Command('test-ref', description='Run the reference tests', category='testing') @CommandArgument('--kind', '-k', default=DEFAULT_RENDER_MODE, help=HELP_RENDER_MODE) @CommandArgument('--release', '-r', action='store_true', help='Run with a release build of Servo') @CommandArgument('--include', default=None, nargs='+', help="Only run tests that match this pattern. If the " "path to the ref test directory is included, it " "will automatically be trimmed out.") @CommandArgument( 'servo_params', default=None, nargs=argparse.REMAINDER, help="Command-line arguments to be passed through to Servo") def test_ref(self, kind=DEFAULT_RENDER_MODE, include=None, servo_params=None, release=False): self.ensure_bootstrapped() self.ensure_built_tests(release=release) assert kind is not None, 'kind cannot be None, see help' kinds = ["cpu", "gpu"] if kind == 'both' else [kind] test_path = path.join(self.context.topdir, "tests", "ref") error = False test_start = time() for k in kinds: print("Running %s reftests..." % k) test_args = [k, test_path] if include is not None: ref_path = path.join("tests", "ref") for name in include: # Check to see if we were passed something leading with the # path to the ref test directory, and trim it so that reftest # knows how to filter it. maybe_path = path.normpath(name) if ref_path in maybe_path: test_args.append(path.relpath(maybe_path, ref_path)) else: test_args.append(name) if servo_params is not None: test_args += ["--"] + servo_params ret = self.run_test("reftest", test_args, release=release) error = error or ret != 0 elapsed = time() - test_start print("Reference tests completed in %0.2fs" % elapsed) if error: return 1 @Command('test-content', description='Run the content tests', category='testing') def test_content(self): print("Content tests have been replaced by web-platform-tests under " "tests/wpt/mozilla/.") return 0 @Command('test-tidy', description='Run the source code tidiness check', category='testing') def test_tidy(self): return tidy.scan() @Command('test-wpt-failure', description='Run the web platform tests', category='testing') def test_wpt_failure(self): self.ensure_bootstrapped() return not subprocess.call([ "bash", path.join("tests", "wpt", "run.sh"), "--no-pause-after-test", "--include", "infrastructure/failing-test.html" ], env=self.build_env()) @Command('test-wpt', description='Run the web platform tests', category='testing', parser=wptcommandline.create_parser) @CommandArgument('--release', default=False, action="store_true", help="Run with a release build of servo") def test_wpt(self, **kwargs): self.ensure_bootstrapped() hosts_file_path = path.join(self.context.topdir, 'tests', 'wpt', 'hosts') os.environ["hosts_file_path"] = hosts_file_path kwargs["debug"] = not kwargs["release"] run_file = path.abspath(path.join(self.context.topdir, "tests", "wpt", "run_wpt.py")) run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["run_tests"](**kwargs) @Command('update-wpt', description='Update the web platform tests', category='testing', parser=updatecommandline.create_parser()) def update_wpt(self, **kwargs): self.ensure_bootstrapped() run_file = path.abspath(path.join("tests", "wpt", "update.py")) run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["update_tests"](**kwargs) @Command('test-jquery', description='Run the jQuery test suite', category='testing') @CommandArgument('--release', '-r', action='store_true', help='Run the release build') @CommandArgument('--dev', '-d', action='store_true', help='Run the dev build') def test_jquery(self, release, dev): return self.jquery_test_runner("test", release, dev) @Command('test-dromaeo', description='Run the Dromaeo test suite', category='testing') @CommandArgument('tests', default=["recommended"], nargs="...", help="Specific tests to run") @CommandArgument('--release', '-r', action='store_true', help='Run the release build') @CommandArgument('--dev', '-d', action='store_true', help='Run the dev build') def test_dromaeo(self, tests, release, dev): return self.dromaeo_test_runner(tests, release, dev) @Command('update-jquery', description='Update the jQuery test suite expected results', category='testing') @CommandArgument('--release', '-r', action='store_true', help='Run the release build') @CommandArgument('--dev', '-d', action='store_true', help='Run the dev build') def update_jquery(self, release, dev): return self.jquery_test_runner("update", release, dev) @Command('test-css', description='Run the web platform tests', category='testing', parser=wptcommandline.create_parser()) @CommandArgument('--release', default=False, action="store_true", help="Run with a release build of servo") def test_css(self, **kwargs): self.ensure_bootstrapped() run_file = path.abspath(path.join("tests", "wpt", "run_css.py")) run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["run_tests"](**kwargs) @Command('update-css', description='Update the web platform tests', category='testing', parser=updatecommandline.create_parser()) def update_css(self, **kwargs): self.ensure_bootstrapped() run_file = path.abspath(path.join("tests", "wpt", "update_css.py")) run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["update_tests"](**kwargs) @Command('compare_dromaeo', description='compare outputs of two runs of ./mach test-dromaeo command', category='testing') @CommandArgument('params', default=None, nargs="...", help=" filepaths of output files of two runs of dromaeo test ") def compare_dromaeo(self, params): prev_op_filename = params[0] cur_op_filename = params[1] result = {'Test': [], 'Prev_Time': [], 'Cur_Time': [], 'Difference(%)': []} with open(prev_op_filename, 'r') as prev_op, open(cur_op_filename, 'r') as cur_op: l1 = prev_op.readline() l2 = cur_op.readline() while ((l1.find('[dromaeo] Saving...') and l2.find('[dromaeo] Saving...'))): l1 = prev_op.readline() l2 = cur_op.readline() reach = 3 while (reach > 0): l1 = prev_op.readline() l2 = cur_op.readline() reach -= 1 while True: l1 = prev_op.readline() l2 = cur_op.readline() if not l1: break result['Test'].append(str(l1).split('|')[0].strip()) result['Prev_Time'].append(float(str(l1).split('|')[1].strip())) result['Cur_Time'].append(float(str(l2).split('|')[1].strip())) a = float(str(l1).split('|')[1].strip()) b = float(str(l2).split('|')[1].strip()) result['Difference(%)'].append(((b - a) / a) * 100) width_col1 = max([len(x) for x in result['Test']]) width_col2 = max([len(str(x)) for x in result['Prev_Time']]) width_col3 = max([len(str(x)) for x in result['Cur_Time']]) width_col4 = max([len(str(x)) for x in result['Difference(%)']]) for p, q, r, s in zip(['Test'], ['First Run'], ['Second Run'], ['Difference(%)']): print ("\033[1m" + "{}|{}|{}|{}".format(p.ljust(width_col1), q.ljust(width_col2), r.ljust(width_col3), s.ljust(width_col4)) + "\033[0m" + "\n" + "--------------------------------------------------" + "-------------------------------------------------------------------------") for a1, b1, c1, d1 in zip(result['Test'], result['Prev_Time'], result['Cur_Time'], result['Difference(%)']): if d1 > 0: print ("\033[91m" + "{}|{}|{}|{}".format(a1.ljust(width_col1), str(b1).ljust(width_col2), str(c1).ljust(width_col3), str(d1).ljust(width_col4)) + "\033[0m") elif d1 < 0: print ("\033[92m" + "{}|{}|{}|{}".format(a1.ljust(width_col1), str(b1).ljust(width_col2), str(c1).ljust(width_col3), str(d1).ljust(width_col4)) + "\033[0m") else: print ("{}|{}|{}|{}".format(a1.ljust(width_col1), str(b1).ljust(width_col2), str(c1).ljust(width_col3), str(d1).ljust(width_col4))) def jquery_test_runner(self, cmd, release, dev): self.ensure_bootstrapped() base_dir = path.abspath(path.join("tests", "jquery")) jquery_dir = path.join(base_dir, "jquery") run_file = path.join(base_dir, "run_jquery.py") # Clone the jQuery repository if it doesn't exist if not os.path.isdir(jquery_dir): subprocess.check_call( ["git", "clone", "-b", "servo", "--depth", "1", "https://github.com/servo/jquery", jquery_dir]) # Run pull in case the jQuery repo was updated since last test run subprocess.check_call( ["git", "-C", jquery_dir, "pull"]) # Check that a release servo build exists bin_path = path.abspath(self.get_binary_path(release, dev)) return subprocess.check_call( [run_file, cmd, bin_path, base_dir]) def dromaeo_test_runner(self, tests, release, dev): self.ensure_bootstrapped() base_dir = path.abspath(path.join("tests", "dromaeo")) dromaeo_dir = path.join(base_dir, "dromaeo") run_file = path.join(base_dir, "run_dromaeo.py") # Clone the Dromaeo repository if it doesn't exist if not os.path.isdir(dromaeo_dir): subprocess.check_call( ["git", "clone", "-b", "servo", "--depth", "1", "https://github.com/notriddle/dromaeo", dromaeo_dir]) # Run pull in case the Dromaeo repo was updated since last test run subprocess.check_call( ["git", "-C", dromaeo_dir, "pull"]) # Compile test suite subprocess.check_call( ["make", "-C", dromaeo_dir, "web"]) # Check that a release servo build exists bin_path = path.abspath(self.get_binary_path(release, dev)) return subprocess.check_call( [run_file, "|".join(tests), bin_path, base_dir])
def main(paths=None): parser = wptcommandline.create_parser() kwargs = vars(parser.parse_args()) return run_tests(paths, **kwargs)
def create_parser_wpt(): parser = wptcommandline.create_parser() parser.add_argument('--release', default=False, action="store_true", help="Run with a release build of servo") return parser
def create_parser_wpt(): from wptrunner import wptcommandline return wptcommandline.create_parser(["firefox"])
class MachCommands(CommandBase): def __init__(self, context): CommandBase.__init__(self, context) if not hasattr(self.context, "built_tests"): self.context.built_tests = False def ensure_built_tests(self): if self.context.built_tests: return returncode = Registrar.dispatch('build-tests', context=self.context) if returncode: sys.exit(returncode) self.context.built_tests = True def find_test(self, prefix): target_contents = os.listdir( path.join(self.context.topdir, "components", "servo", "target", "debug")) for filename in target_contents: if filename.startswith(prefix + "-"): filepath = path.join(self.context.topdir, "components", "servo", "target", "debug", filename) if path.isfile(filepath) and os.access(filepath, os.X_OK): return filepath def run_test(self, prefix, args=[]): t = self.find_test(prefix) if t: return subprocess.call([t] + args, env=self.build_env()) def infer_test_by_dir(self, params): maybe_path = path.normpath(params[0]) mach_command = path.join(self.context.topdir, "mach") args = None if not path.exists(maybe_path): print("%s is not a valid file or directory" % maybe_path) return 1 test_dirs = [ # path, mach test command, optional flag for path argument (path.join("tests", "wpt"), "test-wpt", None), (path.join("tests", "ref"), "test-ref", ["--name"]), ] for test_dir, test_name, path_flag in test_dirs: if not path_flag: path_flag = [] if test_dir in maybe_path: args = ([mach_command, test_name] + path_flag + [maybe_path] + params[1:]) break else: print("%s is not a valid test file or directory" % maybe_path) return 1 return subprocess.call(args, env=self.build_env()) @Command('test', description='Run all Servo tests', category='testing') @CommandArgument('params', default=None, nargs="...", help="Optionally select test based on " "test file directory") def test(self, params): if params: return self.infer_test_by_dir(params) test_start = time() for t in ["tidy", "ref", "wpt", "css", "unit"]: Registrar.dispatch("test-%s" % t, context=self.context) elapsed = time() - test_start print("Tests completed in %0.2fs" % elapsed) @Command('test-unit', description='Run unit tests', category='testing') @CommandArgument('test_name', nargs=argparse.REMAINDER, help="Only run tests that match this pattern") def test_unit(self, test_name=None, component=None, package=None): if test_name is None: test_name = [] self.ensure_bootstrapped() return 0 != subprocess.call( ["cargo", "test", "-p", "unit_tests"] + test_name, env=self.build_env(), cwd=self.servo_crate()) @Command('test-ref', description='Run the reference tests', category='testing') @CommandArgument('--kind', '-k', default=None, help="'cpu' or 'gpu' (default both)") @CommandArgument('--name', default=None, help="Only run tests that match this pattern. If the " "path to the ref test directory is included, it " "will automatically be trimmed out.") @CommandArgument( 'servo_params', default=None, nargs=argparse.REMAINDER, help="Command-line arguments to be passed through to Servo") def test_ref(self, kind=None, name=None, servo_params=None): self.ensure_bootstrapped() self.ensure_built_tests() kinds = ["cpu", "gpu"] if kind is None else [kind] test_path = path.join(self.context.topdir, "tests", "ref") error = False test_start = time() for k in kinds: print("Running %s reftests..." % k) test_args = [k, test_path] if name is not None: maybe_path = path.normpath(name) ref_path = path.join("tests", "ref") # Check to see if we were passed something leading with the # path to the ref test directory, and trim it so that reftest # knows how to filter it. if ref_path in maybe_path: test_args.append(path.relpath(maybe_path, ref_path)) else: test_args.append(name) if servo_params is not None: test_args += ["--"] + servo_params ret = self.run_test("reftest", test_args) error = error or ret != 0 elapsed = time() - test_start print("Reference tests completed in %0.2fs" % elapsed) if error: return 1 @Command('test-content', description='Run the content tests', category='testing') def test_content(self): print("Content tests have been replaced by web-platform-tests under " "tests/wpt/mozilla/.") return 0 @Command('test-tidy', description='Run the source code tidiness check', category='testing') def test_tidy(self): return tidy.scan() @Command('test-wpt-failure', description='Run the web platform tests', category='testing') def test_wpt_failure(self): self.ensure_bootstrapped() return not subprocess.call([ "bash", path.join("tests", "wpt", "run.sh"), "--no-pause-after-test", "--include", "infrastructure/failing-test.html" ], env=self.build_env()) @Command('test-wpt', description='Run the web platform tests', category='testing', parser=wptcommandline.create_parser()) @CommandArgument('--release', default=False, action="store_true", help="Run with a release build of servo") def test_wpt(self, **kwargs): self.ensure_bootstrapped() self.ensure_wpt_virtualenv() hosts_file_path = path.join('tests', 'wpt', 'hosts') os.environ["hosts_file_path"] = hosts_file_path run_file = path.abspath(path.join("tests", "wpt", "run_wpt.py")) run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["run_tests"](**kwargs) @Command('update-wpt', description='Update the web platform tests', category='testing', parser=updatecommandline.create_parser()) def update_wpt(self, **kwargs): self.ensure_bootstrapped() self.ensure_wpt_virtualenv() run_file = path.abspath(path.join("tests", "wpt", "update.py")) run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["update_tests"](**kwargs) @Command('test-css', description='Run the web platform tests', category='testing', parser=wptcommandline.create_parser()) @CommandArgument('--release', default=False, action="store_true", help="Run with a release build of servo") def test_css(self, **kwargs): self.ensure_bootstrapped() self.ensure_wpt_virtualenv() run_file = path.abspath(path.join("tests", "wpt", "run_css.py")) run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["run_tests"](**kwargs) @Command('update-css', description='Update the web platform tests', category='testing', parser=updatecommandline.create_parser()) def update_css(self, **kwargs): self.ensure_bootstrapped() self.ensure_wpt_virtualenv() run_file = path.abspath(path.join("tests", "wpt", "update_css.py")) run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["update_tests"](**kwargs) def ensure_wpt_virtualenv(self): virtualenv_path = path.join("tests", "wpt", "_virtualenv") python = self.get_exec("python2", "python") if not os.path.exists(virtualenv_path): virtualenv = self.get_exec("virtualenv2", "virtualenv") subprocess.check_call([virtualenv, "-p", python, virtualenv_path]) activate_path = path.join(virtualenv_path, "bin", "activate_this.py") execfile(activate_path, dict(__file__=activate_path)) try: import wptrunner from wptrunner.browsers import servo except ImportError: subprocess.check_call([ "pip", "install", "-r", path.join("tests", "wpt", "harness", "requirements.txt") ]) subprocess.check_call([ "pip", "install", "-r", path.join("tests", "wpt", "harness", "requirements_servo.txt") ]) try: import blessings except ImportError: subprocess.check_call(["pip", "install", "blessings"]) # This is an unfortunate hack. Because mozlog gets imported by wptcommandline # before the virtualenv is initalised it doesn't see the blessings module so we don't # get coloured output. Setting the blessings global explicitly fixes that. from mozlog.structured.formatters import machformatter import blessings machformatter.blessings = blessings def get_exec(self, name, default=None): path = find_executable(name) if not path: return default return path
def main(): parser = wptcommandline.create_parser() args = parser.parse_args() kwargs = set_defaults(args) return run_tests(**kwargs)
def create_parser_wpt(): from wptrunner import wptcommandline return wptcommandline.create_parser(["firefox", "firefox_android", "chrome", "edge", "servo"])
def load_wpt_tests(xul_tester, requested_paths, excluded_paths, update_manifest=True): """Return a list of `RefTestCase` objects for the jsshell testharness.js tests filtered by the given paths and debug-ness.""" repo_root = abspath(os.path.join(here, "..", "..", "..")) wp = os.path.join(repo_root, "testing", "web-platform") wpt = os.path.join(wp, "tests") sys_paths = [ "python/mozterm", "python/mozboot", "testing/mozbase/mozcrash", "testing/mozbase/mozdevice", "testing/mozbase/mozfile", "testing/mozbase/mozinfo", "testing/mozbase/mozleak", "testing/mozbase/mozlog", "testing/mozbase/mozprocess", "testing/mozbase/mozprofile", "testing/mozbase/mozrunner", "testing/mozbase/mozversion", "testing/web-platform/", "testing/web-platform/tests/tools", "testing/web-platform/tests/tools/third_party/html5lib", "testing/web-platform/tests/tools/third_party/webencodings", "testing/web-platform/tests/tools/wptrunner", "testing/web-platform/tests/tools/wptserve", "third_party/python/requests", ] abs_sys_paths = [os.path.join(repo_root, path) for path in sys_paths] failed = False for path in abs_sys_paths: if not os.path.isdir(path): failed = True print("Could not add '%s' to the path") if failed: return [] sys.path[0:0] = abs_sys_paths import manifestupdate from wptrunner import products, testloader, wptcommandline, wpttest, wptlogging manifest_root = tempfile.gettempdir() path_split = os.path.dirname(xul_tester.js_bin).split(os.path.sep) if path_split[-2:] == ["dist", "bin"]: maybe_root = os.path.join(*path_split[:-2]) if os.path.exists(os.path.join(maybe_root, "_tests")): # Assume this is a gecko objdir. manifest_root = maybe_root logger = wptlogging.setup({}, {}) test_manifests = manifestupdate.run(repo_root, manifest_root, logger, update=update_manifest) kwargs = vars(wptcommandline.create_parser().parse_args([])) kwargs.update({ "config": os.path.join(manifest_root, "_tests", "web-platform", "wptrunner.local.ini"), "gecko_e10s": False, "verify": False, "wasm": xul_tester.test("wasmIsSupported()"), }) wptcommandline.set_from_config(kwargs) def filter_jsshell_tests(it): for item_type, path, tests in it: tests = set(item for item in tests if item.jsshell) if tests: yield item_type, path, tests run_info_extras = products.load_product(kwargs["config"], "firefox")[-1](**kwargs) run_info = wpttest.get_run_info(kwargs["test_paths"]["/"]["metadata_path"], "firefox", debug=xul_tester.test("isDebugBuild"), extras=run_info_extras) release_or_beta = xul_tester.test("getBuildConfiguration().release_or_beta") run_info["release_or_beta"] = release_or_beta run_info["nightly_build"] = not release_or_beta path_filter = testloader.TestFilter(test_manifests, include=requested_paths, exclude=excluded_paths) loader = testloader.TestLoader(test_manifests, ["testharness"], run_info, manifest_filters=[path_filter, filter_jsshell_tests]) extra_helper_paths = [ os.path.join(here, "web-platform-test-shims.js"), os.path.join(wpt, "resources", "testharness.js"), os.path.join(here, "testharnessreport.js"), ] def resolve(test_path, script): if script.startswith("/"): return os.path.join(wpt, script[1:]) return os.path.join(wpt, os.path.dirname(test_path), script) tests = [] for test in loader.tests["testharness"]: test_path = os.path.relpath(test.path, wpt) scripts = [resolve(test_path, s) for s in test.scripts] extra_helper_paths_for_test = extra_helper_paths + scripts # We must create at least one test with the default options, along with # one test for each option given in a test-also annotation. options = [None] for m in test.itermeta(): if m.has_key("test-also"): # NOQA: W601 options += m.get("test-also").split() for option in options: test_case = RefTestCase( wpt, test_path, extra_helper_paths=extra_helper_paths_for_test[:], wpt=test ) if option: test_case.options.append(option) tests.append(test_case) return tests
def __call__(self, parser, namespace, values, option_string=None): from wptrunner import wptcommandline wptparser = wptcommandline.create_parser() wptparser.usage = parser.usage wptparser.print_help() parser.exit()
def load_wpt_tests(xul_tester, requested_paths, excluded_paths): """Return a list of `RefTestCase` objects for the jsshell testharness.js tests filtered by the given paths and debug-ness.""" repo_root = abspath(os.path.join(here, "..", "..", "..")) wp = os.path.join(repo_root, "testing", "web-platform") wpt = os.path.join(wp, "tests") sys_paths = [ "python/mozterm", "python/mozboot", "testing/mozbase/mozcrash", "testing/mozbase/mozdevice", "testing/mozbase/mozfile", "testing/mozbase/mozinfo", "testing/mozbase/mozleak", "testing/mozbase/mozlog", "testing/mozbase/mozprocess", "testing/mozbase/mozprofile", "testing/mozbase/mozrunner", "testing/web-platform/", "testing/web-platform/tests/tools", "testing/web-platform/tests/tools/third_party/html5lib", "testing/web-platform/tests/tools/third_party/webencodings", "testing/web-platform/tests/tools/wptrunner", "testing/web-platform/tests/tools/wptserve", "third_party/python/requests", ] abs_sys_paths = [os.path.join(repo_root, path) for path in sys_paths] failed = False for path in abs_sys_paths: if not os.path.isdir(path): failed = True print("Could not add '%s' to the path") if failed: return [] sys.path[0:0] = abs_sys_paths import manifestupdate from wptrunner import products, testloader, wptcommandline, wpttest, wptlogging manifest_root = tempfile.gettempdir() path_split = os.path.dirname(xul_tester.js_bin).split(os.path.sep) if path_split[-2:] == ["dist", "bin"]: maybe_root = os.path.join(*path_split[:-2]) if os.path.exists(os.path.join(maybe_root, "_tests")): # Assume this is a gecko objdir. manifest_root = maybe_root logger = wptlogging.setup({}, {}) manifestupdate.run(repo_root, manifest_root, logger) kwargs = vars(wptcommandline.create_parser().parse_args([])) kwargs.update({ "config": os.path.join(manifest_root, "_tests", "web-platform", "wptrunner.local.ini"), "gecko_e10s": False, "verify": False, "wasm": xul_tester.test("wasmIsSupported()"), }) wptcommandline.set_from_config(kwargs) test_paths = kwargs["test_paths"] def filter_jsshell_tests(it): for test in it: if test[1].get("jsshell"): yield test test_manifests = testloader.ManifestLoader(test_paths, types=["testharness"], meta_filters=[filter_jsshell_tests]).load() run_info_extras = products.load_product(kwargs["config"], "firefox")[-1](**kwargs) run_info = wpttest.get_run_info(kwargs["test_paths"]["/"]["metadata_path"], "firefox", debug=xul_tester.test("isDebugBuild"), extras=run_info_extras) path_filter = testloader.TestFilter(test_manifests, include=requested_paths, exclude=excluded_paths) loader = testloader.TestLoader(test_manifests, ["testharness"], run_info, manifest_filters=[path_filter]) extra_helper_paths = [ os.path.join(here, "web-platform-test-shims.js"), os.path.join(wpt, "resources", "testharness.js"), os.path.join(here, "testharnessreport.js"), ] def resolve(test_path, script): if script.startswith("/"): return os.path.join(wpt, script[1:]) return os.path.join(wpt, os.path.dirname(test_path), script) return [ RefTestCase( wpt, test_path, extra_helper_paths=extra_helper_paths + [resolve(test_path, s) for s in test.scripts], wpt=test ) for test_path, test in ( (os.path.relpath(test.path, wpt), test) for test in loader.tests["testharness"] ) ]
def main(): ensure_manifest() parser = wptcommandline.create_parser(False) args = parser.parse_args() kwargs = set_defaults(args) return run_tests(**kwargs)