def setup_wptrunner(venv, prompt=True, install=False, **kwargs): from wptrunner import wptrunner, wptcommandline global logger kwargs = utils.Kwargs(kwargs.iteritems()) product_parts = kwargs["product"].split(":") kwargs["product"] = product_parts[0] sub_product = product_parts[1:] wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) logger = wptrunner.logger check_environ(kwargs["product"]) args_general(kwargs) if kwargs["product"] not in product_setup: raise WptrunError("Unsupported product %s" % kwargs["product"]) setup_cls = product_setup[kwargs["product"]](venv, prompt, sub_product) if install: logger.info("Installing browser") kwargs["binary"] = setup_cls.install(venv) setup_cls.setup(kwargs) wptcommandline.check_args(kwargs) wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner") venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt")) return kwargs
def run_tests(paths=None, **kwargs): if paths is None: paths = {} set_defaults(paths, kwargs) wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) success = wptrunner.run_tests(**kwargs) return 0 if success else 1
def run_tests(**kwargs): if not os.path.isfile(wptsubdir("metadata", "MANIFEST.json")): raise Exception( "Manifest not found. Please use --update-manifest in WPTARGS to create one" ) wptrunner.setup_logging(kwargs, {"raw": sys.stdout}) return wptrunner.run_tests(**kwargs)
def setup_wptrunner(venv, product, tests, wptrunner_args, prompt=True,): global logger wptparser = wptcommandline.create_parser() kwargs = utils.Kwargs(vars(wptparser.parse_args(wptrunner_args)).iteritems()) wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) logger = wptrunner.logger kwargs["product"] = product kwargs["test_list"] = tests check_environ(product) args_general(kwargs) if product not in product_setup: exit("Unsupported product %s" % product) product_setup[product](venv, kwargs, prompt) wptcommandline.check_args(kwargs) wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner") venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt")) return kwargs
def setup_wptrunner(venv, prompt=True, install_browser=False, **kwargs): from wptrunner import wptrunner, wptcommandline import mozlog global logger kwargs = utils.Kwargs(kwargs.iteritems()) product_parts = kwargs["product"].split(":") kwargs["product"] = product_parts[0] sub_product = product_parts[1:] # Use the grouped formatter by default where mozlog 3.9+ is installed if hasattr(mozlog.formatters, "GroupingFormatter"): default_formatter = "grouped" else: default_formatter = "mach" wptrunner.setup_logging(kwargs, {default_formatter: sys.stdout}) logger = wptrunner.logger check_environ(kwargs["product"]) args_general(kwargs) if kwargs["product"] not in product_setup: raise WptrunError("Unsupported product %s" % kwargs["product"]) setup_cls = product_setup[kwargs["product"]](venv, prompt, sub_product) setup_cls.install_requirements() if install_browser and not kwargs["channel"]: logger.info("--install-browser is given but --channel is not set, default to nightly channel") kwargs["channel"] = "nightly" if kwargs["channel"]: channel = install.get_channel(kwargs["product"], kwargs["channel"]) if channel is not None: if channel != kwargs["channel"]: logger.info("Interpreting channel '%s' as '%s'" % (kwargs["channel"], channel)) kwargs["browser_channel"] = channel else: logger.info("Valid channels for %s not known; using argument unmodified" % kwargs["product"]) del kwargs["channel"] if install_browser: logger.info("Installing browser") kwargs["binary"] = setup_cls.install(venv, channel=channel) setup_cls.setup(kwargs) wptcommandline.check_args(kwargs) wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner") venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt")) kwargs['browser_version'] = setup_cls.browser.version(binary=kwargs.get("binary"), webdriver_binary=kwargs.get("webdriver_binary")) return kwargs
def setup_wptrunner(venv, prompt=True, install_browser=False, **kwargs): from wptrunner import wptrunner, wptcommandline import mozlog global logger kwargs = utils.Kwargs(kwargs.iteritems()) product_parts = kwargs["product"].split(":") kwargs["product"] = product_parts[0] sub_product = product_parts[1:] # Use the grouped formatter by default where mozlog 3.9+ is installed if hasattr(mozlog.formatters, "GroupingFormatter"): default_formatter = "grouped" else: default_formatter = "mach" wptrunner.setup_logging(kwargs, {default_formatter: sys.stdout}) logger = wptrunner.logger check_environ(kwargs["product"]) args_general(kwargs) if kwargs["product"] not in product_setup: raise WptrunError("Unsupported product %s" % kwargs["product"]) setup_cls = product_setup[kwargs["product"]](venv, prompt, sub_product) setup_cls.install_requirements() if install_browser and not kwargs["channel"]: logger.info("--install-browser is given but --channel is not set, default to nightly channel") kwargs["channel"] = "nightly" if kwargs["channel"]: channel = install.get_channel(kwargs["product"], kwargs["channel"]) if channel != kwargs["channel"]: logger.info("Interpreting channel '%s' as '%s'" % (kwargs["channel"], channel)) kwargs["browser_channel"] = channel del kwargs["channel"] if install_browser: logger.info("Installing browser") kwargs["binary"] = setup_cls.install(venv, channel=channel) setup_cls.setup(kwargs) wptcommandline.check_args(kwargs) wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner") venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt")) kwargs['browser_version'] = setup_cls.browser.version(kwargs.get("binary")) return kwargs
def run_tests(paths=None, **kwargs): if paths is None: paths = {} set_defaults(paths, kwargs) mozlog.commandline.log_formatters["servo"] = \ (grouping_formatter.GroupingFormatter, "A grouping output formatter") wptrunner.setup_logging(kwargs, {"servo": sys.stdout}) success = wptrunner.run_tests(**kwargs) return 0 if success else 1
def main(): retcode = 0 parser = get_parser() args = parser.parse_args() gh_handler = setup_github_logging(args) logger.info("Testing in **%s**" % args.browser.title()) browser_cls = {"firefox": Firefox, "chrome": Chrome}.get(args.browser) if browser_cls is None: logger.critical("Unrecognised browser %s" % args.browser) return 2 # For now just pass the whole list of changed files to wptrunner and # assume that it will run everything that's actually a test files_changed = get_files_changed(args.root) if not files_changed: return 0 logger.info("Files changed:\n%s" % "".join(" * %s\n" % item for item in files_changed)) browser = browser_cls() kwargs = wptrunner_args(args.root, files_changed, args.iterations, browser) with open("raw.log", "wb") as log: wptrunner.setup_logging(kwargs, {"mach": sys.stdout, "raw": log}) wptrunner.run_tests(**kwargs) with open("raw.log", "rb") as log: results, inconsistent = process_results(log, args.iterations) if results: if inconsistent: write_inconsistent(inconsistent) retcode = 1 else: logger.info("All results were stable\n") write_results(results, args.iterations) else: logger.info("No tests run.") try: if gh_handler: gh_handler.send() except Exception: logger.error(traceback.format_exc()) return retcode
def setup_logging(kwargs): import mozlog from wptrunner import wptrunner global logger # Use the grouped formatter by default where mozlog 3.9+ is installed if hasattr(mozlog.formatters, "GroupingFormatter"): default_formatter = "grouped" else: default_formatter = "mach" wptrunner.setup_logging(kwargs, {default_formatter: sys.stdout}) logger = wptrunner.logger
def setup_wptrunner(venv, prompt=True, install_browser=False, **kwargs): from wptrunner import wptrunner, wptcommandline global logger kwargs = utils.Kwargs(kwargs.iteritems()) product_parts = kwargs["product"].split(":") kwargs["product"] = product_parts[0] sub_product = product_parts[1:] wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) logger = wptrunner.logger check_environ(kwargs["product"]) args_general(kwargs) if kwargs["product"] not in product_setup: raise WptrunError("Unsupported product %s" % kwargs["product"]) setup_cls = product_setup[kwargs["product"]](venv, prompt, sub_product) setup_cls.install_requirements() if install_browser and not kwargs["channel"]: logger.info( "--install-browser is given but --channel is not set, default to nightly channel" ) kwargs["channel"] = "nightly" if kwargs["channel"]: channel = install.get_channel(kwargs["product"], kwargs["channel"]) if channel != kwargs["channel"]: logger.info("Interpreting channel '%s' as '%s'" % (kwargs["channel"], channel)) kwargs["browser_channel"] = channel del kwargs["channel"] if install_browser: logger.info("Installing browser") kwargs["binary"] = setup_cls.install(venv, channel=channel) setup_cls.setup(kwargs) wptcommandline.check_args(kwargs) wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner") venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt")) kwargs['browser_version'] = setup_cls.browser.version(kwargs.get("binary")) return kwargs
def setup_wptrunner(venv, prompt=True, install_browser=False, **kwargs): from wptrunner import wptrunner, wptcommandline global logger kwargs = utils.Kwargs(kwargs.iteritems()) product_parts = kwargs["product"].split(":") kwargs["product"] = product_parts[0] sub_product = product_parts[1:] wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) logger = wptrunner.logger check_environ(kwargs["product"]) args_general(kwargs) if kwargs["product"] not in product_setup: raise WptrunError("Unsupported product %s" % kwargs["product"]) setup_cls = product_setup[kwargs["product"]](venv, prompt, sub_product) setup_cls.install_requirements() if install_browser and not kwargs["channel"]: kwargs["channel"] = "nightly" if kwargs["channel"]: channel = install.get_channel(kwargs["product"], kwargs["channel"]) if channel != kwargs["channel"]: logger.info("Interpreting channel '%s' as '%s'" % (kwargs["channel"], channel)) kwargs["browser_channel"] = channel del kwargs["channel"] if install_browser: logger.info("Installing browser") kwargs["binary"] = setup_cls.install(venv, channel=channel) setup_cls.setup(kwargs) wptcommandline.check_args(kwargs) wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner") venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt")) kwargs['browser_version'] = setup_cls.browser.version(kwargs.get("binary")) return kwargs
def setup_logging(kwargs, default_config=None): import mozlog from wptrunner import wptrunner global logger # Use the grouped formatter by default where mozlog 3.9+ is installed if default_config is None: if hasattr(mozlog.formatters, "GroupingFormatter"): default_formatter = "grouped" else: default_formatter = "mach" default_config = {default_formatter: sys.stdout} wptrunner.setup_logging(kwargs, default_config) logger = wptrunner.logger return logger
def run_tests(self, **kwargs): from wptrunner import wptrunner self.setup_kwargs(kwargs) logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) result = wptrunner.run_tests(**kwargs) return int(not result)
def run_tests(**kwargs): set_defaults(kwargs) mozlog.commandline.log_formatters["servo"] = \ (grouping_formatter.GroupingFormatter, "A grouping output formatter") use_mach_logging = False if len(kwargs["test_list"]) == 1: file_ext = os.path.splitext(kwargs["test_list"][0])[1].lower() if file_ext in [".htm", ".html", ".js", ".xhtml", ".xht", ".py"]: use_mach_logging = True if use_mach_logging: wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) else: wptrunner.setup_logging(kwargs, {"servo": sys.stdout}) success = wptrunner.run_tests(**kwargs) return 0 if success else 1
def run(self, **kwargs): from wptrunner import wptrunner if kwargs["product"] in ["firefox", None]: kwargs = self.setup.kwargs_firefox(kwargs) elif kwargs["product"] in ("chrome", "edge", "servo"): kwargs = self.setup.kwargs_wptrun(kwargs) else: raise ValueError("Unknown product %s" % kwargs["product"]) logger = wptrunner.setup_logging(kwargs, {self.setup.default_log_type: sys.stdout}) result = wptrunner.start(**kwargs) return int(not result)
def run_tests(self, **kwargs): from wptrunner import wptrunner if kwargs["product"] in ["firefox", None]: self.setup_kwargs_firefox(kwargs) elif kwargs["product"] in ("chrome", "edge", "servo"): self.setup_kwargs_wptrun(kwargs) else: raise ValueError("Unknown product %s" % kwargs["product"]) logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) result = wptrunner.run_tests(**kwargs) return int(not result)
def run_tests(**kwargs): wptrunner.setup_logging(kwargs, {"raw": sys.stdout}) return wptrunner.run_tests(**kwargs)
logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in affected_testfiles)) files_changed.extend(affected_testfiles) kwargs = wptrunner_args(args.root, files_changed, args.iterations, browser) browser.prepare_environment() with TravisFold("running_tests"): logger.info("Starting %i test iterations" % args.iterations) with open("raw.log", "wb") as log: wptrunner.setup_logging(kwargs, {"raw": log}) # Setup logging for wptrunner that keeps process output and # warning+ level logs only wptrunner.logger.add_handler( LogActionFilter( LogLevelFilter( StreamHandler( sys.stdout, TbplFormatter() ), "WARNING"), ["log", "process_output"])) wptrunner.run_tests(**kwargs) with open("raw.log", "rb") as log:
def run_tests(**kwargs): if not os.path.isfile(wptsubdir("metadata", "MANIFEST.json")): raise Exception("Manifest not found. Please use --update-manifest in WPTARGS to create one") wptrunner.setup_logging(kwargs, {"raw": sys.stdout}) return wptrunner.run_tests(**kwargs)
affected_testfiles = get_affected_testfiles(files_changed, skip_tests) logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in affected_testfiles)) files_changed.extend(affected_testfiles) kwargs = wptrunner_args(args.root, files_changed, args.iterations, browser) browser.prepare_environment() with TravisFold("running_tests"): logger.info("Starting %i test iterations" % args.iterations) with open("raw.log", "wb") as log: wptrunner.setup_logging(kwargs, {"raw": log}) # Setup logging for wptrunner that keeps process output and # warning+ level logs only wptrunner.logger.add_handler( LogActionFilter( LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()), "WARNING"), ["log", "process_output"])) wptrunner.run_tests(**kwargs) with open("raw.log", "rb") as log: results, inconsistent = process_results(log, args.iterations) if results: if inconsistent: write_inconsistent(inconsistent, args.iterations)
def setup_logging(self, **kwargs): from wptrunner import wptrunner return wptrunner.setup_logging( kwargs, {self.setup.default_log_type: sys.stdout})
def main(): retcode = 0 parser = get_parser() args = parser.parse_args() if not os.path.exists(args.root): logger.critical("Root directory %s does not exist" % args.root) return 1 os.chdir(args.root) if args.gh_token: gh_handler = setup_github_logging(args) else: logger.warning("Can't log to GitHub") gh_handler = None print >> sys.stderr, "travis_fold:start:browser_setup" logger.info("# %s #" % args.browser.title()) browser_cls = {"firefox": Firefox, "chrome": Chrome}.get(args.browser) if browser_cls is None: logger.critical("Unrecognised browser %s" % args.browser) return 1 fetch_wpt_master() head_sha1 = get_sha1() logger.info("Testing revision %s" % head_sha1) # For now just pass the whole list of changed files to wptrunner and # assume that it will run everything that's actually a test files_changed = get_files_changed() if not files_changed: logger.info("No files changed") return 0 build_manifest() install_wptrunner() do_delayed_imports() logger.debug("Files changed:\n%s" % "".join(" * %s\n" % item for item in files_changed)) browser = browser_cls(args.gh_token) browser.install() browser.install_webdriver() kwargs = wptrunner_args(args.root, files_changed, args.iterations, browser) print >> sys.stderr, "travis_fold:end:browser_setup" print >> sys.stderr, "travis_fold:start:running_tests" logger.info("Starting %i test iterations" % args.iterations) with open("raw.log", "wb") as log: wptrunner.setup_logging(kwargs, {"raw": log}) # Setup logging for wptrunner that keeps process output and # warning+ level logs only wptrunner.logger.add_handler( LogActionFilter( LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()), "WARNING"), ["log", "process_output"])) wptrunner.run_tests(**kwargs) with open("raw.log", "rb") as log: results, inconsistent = process_results(log, args.iterations) print >> sys.stderr, "travis_fold:end:running_tests" if results: if inconsistent: write_inconsistent(inconsistent, args.iterations) retcode = 2 else: logger.info("All results were stable\n") print >> sys.stderr, "travis_fold:start:full_results" write_results(results, args.iterations) print >> sys.stderr, "travis_fold:end:full_results" else: logger.info("No tests run.") try: if gh_handler: gh_handler.send() except Exception: logger.error(traceback.format_exc()) return retcode
def main(): retcode = 0 parser = get_parser() args = parser.parse_args() if not os.path.exists(args.root): logger.critical("Root directory %s does not exist" % args.root) return 1 os.chdir(args.root) if args.gh_token: gh_handler = setup_github_logging(args) else: logger.warning("Can't log to GitHub") gh_handler = None with TravisFold("browser_setup"): logger.info("# %s #" % args.browser.title()) browser_cls = {"firefox": Firefox, "chrome": Chrome}.get(args.browser) if browser_cls is None: logger.critical("Unrecognised browser %s" % args.browser) return 1 fetch_wpt_master() head_sha1 = get_sha1() logger.info("Testing revision %s" % head_sha1) # For now just pass the whole list of changed files to wptrunner and # assume that it will run everything that's actually a test files_changed = get_files_changed() if not files_changed: logger.info("No files changed") return 0 build_manifest() install_wptrunner() do_delayed_imports() logger.debug("Files changed:\n%s" % "".join(" * %s\n" % item for item in files_changed)) affected_testfiles = get_affected_testfiles(files_changed) logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in affected_testfiles)) files_changed.extend(affected_testfiles) browser = browser_cls(args.gh_token) browser.install() browser.install_webdriver() kwargs = wptrunner_args(args.root, files_changed, args.iterations, browser) with TravisFold("running_tests"): logger.info("Starting %i test iterations" % args.iterations) with open("raw.log", "wb") as log: wptrunner.setup_logging(kwargs, {"raw": log}) # Setup logging for wptrunner that keeps process output and # warning+ level logs only wptrunner.logger.add_handler( LogActionFilter( LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()), "WARNING"), ["log", "process_output"] ) ) wptrunner.run_tests(**kwargs) with open("raw.log", "rb") as log: results, inconsistent = process_results(log, args.iterations) if results: if inconsistent: write_inconsistent(inconsistent, args.iterations) retcode = 2 else: logger.info("All results were stable\n") with TravisFold("full_results"): write_results(results, args.iterations, args.comment_pr) else: logger.info("No tests run.") try: if gh_handler: gh_handler.send() except Exception: logger.error(traceback.format_exc()) return retcode