def test_setup_wptrunner(venv, product): parser = run.create_parser() kwargs = vars(parser.parse_args(["--channel=nightly", product])) kwargs["prompt"] = False # Hack to get a real existing path kwargs["binary"] = sys.argv[0] kwargs["webdriver_binary"] = sys.argv[0] if kwargs["product"] == "sauce": kwargs["product"] = "sauce:firefox:63" run.setup_wptrunner(venv, **kwargs)
def test_setup_wptrunner(venv, logger, product): parser = run.create_parser() kwargs = vars(parser.parse_args(["--channel=nightly", product])) kwargs["prompt"] = False # Hack to get a real existing path kwargs["binary"] = sys.argv[0] kwargs["webdriver_binary"] = sys.argv[0] if kwargs["product"] == "sauce": kwargs["product"] = "sauce:firefox:63" run.setup_wptrunner(venv, **kwargs)
def test_setup_wptrunner(venv, logger, product): if product == "firefox_android": pytest.skip("Android emulator doesn't work on docker") parser = run.create_parser() kwargs = vars(parser.parse_args(["--channel=nightly", product])) kwargs["prompt"] = False # Hack to get a real existing path kwargs["binary"] = sys.argv[0] kwargs["webdriver_binary"] = sys.argv[0] if kwargs["product"] == "sauce": kwargs["product"] = "sauce:firefox:63" run.setup_wptrunner(venv, **kwargs)
def kwargs_wptrun(self, kwargs): from wptrunner import wptcommandline here = os.path.join(self.topsrcdir, 'testing', 'web-platform') kwargs["tests_root"] = os.path.join(here, "tests") sys.path.insert(0, kwargs["tests_root"]) if kwargs["metadata_root"] is None: metadir = os.path.join(here, "products", kwargs["product"]) if not os.path.exists(metadir): os.makedirs(metadir) kwargs["metadata_root"] = metadir src_manifest = os.path.join(here, "meta", "MANIFEST.json") dest_manifest = os.path.join(kwargs["metadata_root"], "MANIFEST.json") if not os.path.exists(dest_manifest) and os.path.exists(src_manifest): with open(src_manifest) as src, open(dest_manifest, "w") as dest: dest.write(src.read()) from tools.wpt import run try: kwargs = run.setup_wptrunner( run.virtualenv.Virtualenv( self.virtualenv_manager.virtualenv_root), **kwargs) except run.WptrunError as e: print(e.message, file=sys.stderr) sys.exit(1) return kwargs
def kwargs_wptrun(self, kwargs): from wptrunner import wptcommandline if kwargs["metadata_root"] is None: metadir = os.path.join(self._here, "products", kwargs["product"]) if not os.path.exists(metadir): os.makedirs(metadir) kwargs["metadata_root"] = metadir src_manifest = os.path.join(self._here, "meta", "MANIFEST.json") dest_manifest = os.path.join(kwargs["metadata_root"], "MANIFEST.json") if not os.path.exists(dest_manifest) and os.path.exists(src_manifest): with open(src_manifest) as src, open(dest_manifest, "w") as dest: dest.write(src.read()) from tools.wpt import run # Add additional kwargs consumed by the run frontend. Currently we don't # have a way to set these through mach kwargs["channel"] = None kwargs["prompt"] = True kwargs["install_browser"] = False try: kwargs = run.setup_wptrunner( run.virtualenv.Virtualenv( self.virtualenv_manager.virtualenv_root), **kwargs) except run.WptrunError as e: print(e.message, file=sys.stderr) sys.exit(1) return kwargs
def kwargs_wptrun(self, kwargs): from wptrunner import wptcommandline if kwargs["metadata_root"] is None: metadir = os.path.join(self._here, "products", kwargs["product"]) if not os.path.exists(metadir): os.makedirs(metadir) kwargs["metadata_root"] = metadir src_manifest = os.path.join(self._here, "meta", "MANIFEST.json") dest_manifest = os.path.join(kwargs["metadata_root"], "MANIFEST.json") if not os.path.exists(dest_manifest) and os.path.exists(src_manifest): with open(src_manifest) as src, open(dest_manifest, "w") as dest: dest.write(src.read()) from tools.wpt import run # Add additional kwargs consumed by the run frontend. Currently we don't # have a way to set these through mach kwargs["channel"] = None kwargs["prompt"] = True kwargs["install_browser"] = False kwargs["install_webdriver"] = None kwargs["affected"] = None # Install the deps # We do this explicitly to avoid calling pip with options that aren't # supported in the in-tree version wptrunner_path = os.path.join(self._here, "tests", "tools", "wptrunner") browser_cls = run.product_setup[kwargs["product"]].browser_cls requirements = ["requirements.txt"] if hasattr(browser_cls, "requirements"): requirements.append(browser_cls.requirements) for filename in requirements: path = os.path.join(wptrunner_path, filename) if os.path.exists(path): self.virtualenv_manager.install_pip_requirements( path, require_hashes=False) venv = run.virtualenv.Virtualenv( self.virtualenv_manager.virtualenv_root, skip_virtualenv_setup=True) try: kwargs = run.setup_wptrunner(venv, **kwargs) except run.WptrunError as e: print(e.message, file=sys.stderr) sys.exit(1) return kwargs
def kwargs_wptrun(self, kwargs): """Setup kwargs for wpt-run which is only used for non-gecko browser products""" from tools.wpt import run kwargs = self.kwargs_common(kwargs) # Add additional kwargs consumed by the run frontend. Currently we don't # have a way to set these through mach kwargs["channel"] = None kwargs["prompt"] = True kwargs["install_browser"] = False kwargs["install_webdriver"] = None kwargs["affected"] = None # Install the deps # We do this explicitly to avoid calling pip with options that aren't # supported in the in-tree version wptrunner_path = os.path.join(self._here, "tests", "tools", "wptrunner") browser_cls = run.product_setup[kwargs["product"]].browser_cls requirements = ["requirements.txt"] if hasattr(browser_cls, "requirements"): requirements.append(browser_cls.requirements) for filename in requirements: path = os.path.join(wptrunner_path, filename) if os.path.exists(path): self.virtualenv_manager.install_pip_requirements( path, require_hashes=False ) venv = run.virtualenv.Virtualenv( self.virtualenv_manager.virtualenv_root, skip_virtualenv_setup=True ) try: kwargs = run.setup_wptrunner(venv, **kwargs) except run.WptrunError as e: print(e.message, file=sys.stderr) sys.exit(1) # This is kind of a hack; override the metadata paths so we don't use # gecko metadata for non-gecko products for key, value in list(iteritems(kwargs["test_paths"])): meta_suffix = key.strip("/") meta_dir = os.path.join( self._here, "products", kwargs["product"], meta_suffix ) value["metadata_path"] = meta_dir if not os.path.exists(meta_dir): os.makedirs(meta_dir) return kwargs
def run(venv, wpt_args, **kwargs): global logger do_delayed_imports() retcode = 0 parser = get_parser() wpt_args = create_parser().parse_args(wpt_args) with open(kwargs["config_file"], 'r') as config_fp: config = SafeConfigParser() config.readfp(config_fp) skip_tests = config.get("file detection", "skip_tests").split() ignore_changes = set(config.get("file detection", "ignore_changes").split()) results_url = config.get("file detection", "results_url") if kwargs["output_bytes"] is not None: replace_streams(kwargs["output_bytes"], "Log reached capacity (%s bytes); output disabled." % kwargs["output_bytes"]) wpt_args.metadata_root = kwargs["metadata_root"] try: os.makedirs(wpt_args.metadata_root) except OSError: pass setup_logging() browser_name = wpt_args.product.split(":")[0] if browser_name == "sauce" and not wpt_args.sauce_key: logger.warning("Cannot run tests on Sauce Labs. No access key.") return retcode pr_number = pr() with TravisFold("browser_setup"): logger.info(markdown.format_comment_title(wpt_args.product)) if pr is not None: deepen_checkout(kwargs["user"]) # Ensure we have a branch called "master" fetch_wpt(kwargs["user"], "master:master") head_sha1 = get_sha1() logger.info("Testing web-platform-tests at revision %s" % head_sha1) wpt_kwargs = Kwargs(vars(wpt_args)) if not wpt_kwargs["test_list"]: manifest_path = os.path.join(wpt_kwargs["metadata_root"], "MANIFEST.json") tests_changed, files_affected = get_changed_files(manifest_path, kwargs["rev"], ignore_changes, skip_tests) if not (tests_changed or files_affected): logger.info("No tests changed") return 0 if tests_changed: logger.debug("Tests changed:\n%s" % "".join(" * %s\n" % item for item in tests_changed)) if files_affected: logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in files_affected)) wpt_kwargs["test_list"] = list(tests_changed | files_affected) set_default_args(wpt_kwargs) do_delayed_imports() wpt_kwargs["stability"] = True wpt_kwargs["prompt"] = False wpt_kwargs["install_browser"] = True wpt_kwargs["install"] = wpt_kwargs["product"].split(":")[0] == "firefox" wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs) logger.info("Using binary %s" % wpt_kwargs["binary"]) with TravisFold("running_tests"): logger.info("Starting tests") wpt_logger = wptrunner.logger iterations, results, inconsistent = run(venv, wpt_logger, **wpt_kwargs) if results: if inconsistent: write_inconsistent(logger.error, inconsistent, iterations) retcode = 2 else: logger.info("All results were stable\n") with TravisFold("full_results"): write_results(logger.info, results, iterations, pr_number=pr_number, use_details=True) if pr_number: post_results(results, iterations=iterations, url=results_url, product=wpt_args.product, pr_number=pr_number, status="failed" if inconsistent else "passed") else: logger.info("No tests run.") return retcode
def run(venv, wpt_args, **kwargs): global logger do_delayed_imports() retcode = 0 parser = get_parser() wpt_args = create_parser().parse_args(wpt_args) with open(kwargs["config_file"], 'r') as config_fp: config = SafeConfigParser() config.readfp(config_fp) skip_tests = config.get("file detection", "skip_tests").split() ignore_changes = set(config.get("file detection", "ignore_changes").split()) if kwargs["output_bytes"] is not None: replace_streams(kwargs["output_bytes"], "Log reached capacity (%s bytes); output disabled." % kwargs["output_bytes"]) wpt_args.metadata_root = kwargs["metadata_root"] try: os.makedirs(wpt_args.metadata_root) except OSError: pass logger = logging.getLogger(os.path.splitext(__file__)[0]) setup_logging() browser_name = wpt_args.product.split(":")[0] if browser_name == "sauce" and not wpt_args.sauce_key: logger.warning("Cannot run tests on Sauce Labs. No access key.") return retcode pr_number = pr() with TravisFold("browser_setup"): logger.info(markdown.format_comment_title(wpt_args.product)) if pr is not None: deepen_checkout(kwargs["user"]) # Ensure we have a branch called "master" fetch_wpt(kwargs["user"], "master:master") head_sha1 = get_sha1() logger.info("Testing web-platform-tests at revision %s" % head_sha1) if not kwargs["rev"]: branch_point = testfiles.branch_point() revish = "%s..HEAD" % branch_point else: revish = kwargs["rev"] files_changed, files_ignored = testfiles.files_changed(revish, ignore_changes) if files_ignored: logger.info("Ignoring %s changed files:\n%s" % (len(files_ignored), "".join(" * %s\n" % item for item in files_ignored))) tests_changed, files_affected = testfiles.affected_testfiles(files_changed, skip_tests, manifest_path=os.path.join( wpt_args.metadata_root, "MANIFEST.json")) if not (tests_changed or files_affected): logger.info("No tests changed") return 0 wpt_kwargs = Kwargs(vars(wpt_args)) wpt_kwargs["test_list"] = list(tests_changed | files_affected) set_default_args(wpt_kwargs) do_delayed_imports() wpt_kwargs["stability"] = True wpt_kwargs["prompt"] = False wpt_kwargs["install_browser"] = True wpt_kwargs["install"] = wpt_kwargs["product"].split(":")[0] == "firefox" wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs) logger.info("Using binary %s" % wpt_kwargs["binary"]) if tests_changed: logger.debug("Tests changed:\n%s" % "".join(" * %s\n" % item for item in tests_changed)) if files_affected: logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in files_affected)) with TravisFold("running_tests"): logger.info("Starting tests") wpt_logger = wptrunner.logger iterations, results, inconsistent = run(venv, wpt_logger, **wpt_kwargs) if results: if inconsistent: write_inconsistent(logger.error, inconsistent, iterations) retcode = 2 else: logger.info("All results were stable\n") with TravisFold("full_results"): write_results(logger.info, results, iterations, pr_number=kwargs["comment_pr"], use_details=True) else: logger.info("No tests run.") return retcode
def run(venv, wpt_args, **kwargs): do_delayed_imports() setup_logging() retcode = 0 wpt_args = create_parser().parse_args(wpt_args) with open(kwargs["config_file"], 'r') as config_fp: config = SafeConfigParser() config.readfp(config_fp) ignore_changes = set( config.get("file detection", "ignore_changes").split()) if kwargs["output_bytes"] is not None: replace_streams( kwargs["output_bytes"], "Log reached capacity (%s bytes); output disabled." % kwargs["output_bytes"]) wpt_args.metadata_root = kwargs["metadata_root"] try: os.makedirs(wpt_args.metadata_root) except OSError: pass pr_number = pr() with TravisFold("browser_setup"): logger.info(markdown.format_comment_title(wpt_args.product)) if pr is not None: deepen_checkout(kwargs["user"]) # Ensure we have a branch called "master" fetch_wpt(kwargs["user"], "master:master") head_sha1 = get_sha1() logger.info("Testing web-platform-tests at revision %s" % head_sha1) wpt_kwargs = Kwargs(vars(wpt_args)) if not wpt_kwargs["test_list"]: manifest_path = os.path.join(wpt_kwargs["metadata_root"], "MANIFEST.json") tests_changed, files_affected = get_changed_files( manifest_path, kwargs["rev"], ignore_changes) if not (tests_changed or files_affected): logger.info("No tests changed") return 0 if tests_changed: logger.debug("Tests changed:\n%s" % "".join(" * %s\n" % item for item in tests_changed)) if files_affected: logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in files_affected)) wpt_kwargs["test_list"] = list(tests_changed | files_affected) do_delayed_imports() wpt_kwargs["prompt"] = False wpt_kwargs["install_browser"] = wpt_kwargs["product"].split( ":")[0] == "firefox" wpt_kwargs["pause_after_test"] = False wpt_kwargs["verify_log_full"] = False if wpt_kwargs["repeat"] == 1: wpt_kwargs["repeat"] = 10 wpt_kwargs["headless"] = False wpt_kwargs["log_tbpl"] = [sys.stdout] wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs) logger.info("Using binary %s" % wpt_kwargs["binary"]) with TravisFold("running_tests"): logger.info("Starting tests") wpt_logger = wptrunner.logger results, inconsistent, slow, iterations = run_step( wpt_logger, wpt_kwargs["repeat"], True, {}, **wpt_kwargs) if results: if inconsistent: write_inconsistent(logger.error, inconsistent, iterations) retcode = 2 elif slow: write_slow_tests(logger.error, slow) retcode = 2 else: logger.info("All results were stable\n") with TravisFold("full_results"): write_results(logger.info, results, iterations, pr_number=pr_number, use_details=True) else: logger.info("No tests run.") # Be conservative and only return errors when we know for sure tests are changed. if tests_changed: retcode = 3 return retcode
def run(venv, wpt_args, **kwargs): do_delayed_imports() retcode = 0 wpt_args = create_parser().parse_args(wpt_args) with open(kwargs["config_file"], 'r') as config_fp: config = SafeConfigParser() config.readfp(config_fp) skip_tests = config.get("file detection", "skip_tests").split() ignore_changes = set(config.get("file detection", "ignore_changes").split()) results_url = config.get("file detection", "results_url") if kwargs["output_bytes"] is not None: replace_streams(kwargs["output_bytes"], "Log reached capacity (%s bytes); output disabled." % kwargs["output_bytes"]) wpt_args.metadata_root = kwargs["metadata_root"] try: os.makedirs(wpt_args.metadata_root) except OSError: pass setup_logging() browser_name = wpt_args.product.split(":")[0] pr_number = pr() with TravisFold("browser_setup"): logger.info(markdown.format_comment_title(wpt_args.product)) if pr is not None: deepen_checkout(kwargs["user"]) # Ensure we have a branch called "master" fetch_wpt(kwargs["user"], "master:master") head_sha1 = get_sha1() logger.info("Testing web-platform-tests at revision %s" % head_sha1) wpt_kwargs = Kwargs(vars(wpt_args)) if not wpt_kwargs["test_list"]: manifest_path = os.path.join(wpt_kwargs["metadata_root"], "MANIFEST.json") tests_changed, files_affected = get_changed_files(manifest_path, kwargs["rev"], ignore_changes, skip_tests) if not (tests_changed or files_affected): logger.info("No tests changed") return 0 if tests_changed: logger.debug("Tests changed:\n%s" % "".join(" * %s\n" % item for item in tests_changed)) if files_affected: logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in files_affected)) wpt_kwargs["test_list"] = list(tests_changed | files_affected) do_delayed_imports() wpt_kwargs["stability"] = True wpt_kwargs["prompt"] = False wpt_kwargs["install_browser"] = True wpt_kwargs["install"] = wpt_kwargs["product"].split(":")[0] == "firefox" wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs) logger.info("Using binary %s" % wpt_kwargs["binary"]) with TravisFold("running_tests"): logger.info("Starting tests") wpt_logger = wptrunner.logger iterations, results, inconsistent = stability_run(venv, wpt_logger, **wpt_kwargs) if results: if inconsistent: write_inconsistent(logger.error, inconsistent, iterations) retcode = 2 else: logger.info("All results were stable\n") with TravisFold("full_results"): write_results(logger.info, results, iterations, pr_number=pr_number, use_details=True) if pr_number: post_results(results, iterations=iterations, url=results_url, product=wpt_args.product, pr_number=pr_number, status="failed" if inconsistent else "passed") else: logger.info("No tests run.") # Be conservative and only return errors when we know for sure tests are changed. if tests_changed: retcode = 3 return retcode