class FirefoxBrowser(Browser): used_ports = set() init_timeout = 60 shutdown_timeout = 60 def __init__(self, logger, binary, prefs_root, test_type, extra_prefs=None, debug_info=None, symbols_path=None, stackwalk_binary=None, certutil_binary=None, ca_certificate_path=None, e10s=False, stackfix_dir=None, binary_args=None, timeout_multiplier=None, leak_check=False, stylo_threads=1, chaos_mode_flags=None): Browser.__init__(self, logger) self.binary = binary self.prefs_root = prefs_root self.test_type = test_type self.extra_prefs = extra_prefs self.marionette_port = None self.runner = None self.debug_info = debug_info self.profile = None self.symbols_path = symbols_path self.stackwalk_binary = stackwalk_binary self.ca_certificate_path = ca_certificate_path self.certutil_binary = certutil_binary self.e10s = e10s self.binary_args = binary_args if stackfix_dir: self.stack_fixer = get_stack_fixer_function( stackfix_dir, self.symbols_path) else: self.stack_fixer = None if timeout_multiplier: self.init_timeout = self.init_timeout * timeout_multiplier self.leak_report_file = None self.leak_check = leak_check self.stylo_threads = stylo_threads self.chaos_mode_flags = chaos_mode_flags def settings(self, test): return {"check_leaks": self.leak_check and not test.leaks} def start(self, **kwargs): if self.marionette_port is None: self.marionette_port = get_free_port(2828, exclude=self.used_ports) self.used_ports.add(self.marionette_port) env = os.environ.copy() env["MOZ_CRASHREPORTER"] = "1" env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env["STYLO_THREADS"] = str(self.stylo_threads) if self.chaos_mode_flags is not None: env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags) locations = ServerLocations( filename=os.path.join(here, "server-locations.txt")) preferences = self.load_prefs() self.profile = FirefoxProfile(locations=locations, preferences=preferences) self.profile.set_preferences({ "marionette.port": self.marionette_port, "dom.disable_open_during_load": False, "network.dns.localDomains": ",".join(hostnames), "network.proxy.type": 0, "places.history.enabled": False, "dom.send_after_paint_to_content": True, "network.preload": True }) if self.e10s: self.profile.set_preferences( {"browser.tabs.remote.autostart": True}) if self.test_type == "reftest": self.profile.set_preferences( {"layout.interruptible-reflow.enabled": False}) if self.leak_check and kwargs.get("check_leaks", True): self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log") if os.path.exists(self.leak_report_file): os.remove(self.leak_report_file) env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file else: self.leak_report_file = None # Bug 1262954: winxp + e10s, disable hwaccel if (self.e10s and platform.system() in ("Windows", "Microsoft") and '5.1' in platform.version()): self.profile.set_preferences( {"layers.acceleration.disabled": True}) if self.ca_certificate_path is not None: self.setup_ssl() debug_args, cmd = browser_command( self.binary, self.binary_args if self.binary_args else [] + [cmd_arg("marionette"), "about:blank"], self.debug_info) self.runner = FirefoxRunner( profile=self.profile, binary=cmd[0], cmdargs=cmd[1:], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) self.logger.debug("Firefox Started") def load_prefs(self): prefs = Preferences() prefs_path = os.path.join(self.prefs_root, "prefs_general.js") if os.path.exists(prefs_path): prefs.add(Preferences.read_prefs(prefs_path)) else: self.logger.warning("Failed to find base prefs file in %s" % prefs_path) # Add any custom preferences prefs.add(self.extra_prefs, cast=True) return prefs() def stop(self, force=False): if self.runner is not None and self.runner.is_running(): try: # For Firefox we assume that stopping the runner prompts the # browser to shut down. This allows the leak log to be written for clean, stop_f in [ (True, lambda: self.runner.wait(self.shutdown_timeout)), (False, lambda: self.runner.stop(signal.SIGTERM)), (False, lambda: self.runner.stop(signal.SIGKILL)) ]: if not force or not clean: retcode = stop_f() if retcode is not None: self.logger.info( "Browser exited with return code %s" % retcode) break except OSError: # This can happen on Windows if the process is already dead pass self.logger.debug("stopped") def process_leaks(self): self.logger.debug("PROCESS LEAKS %s" % self.leak_report_file) if self.leak_report_file is None: return mozleak.process_leak_log( self.leak_report_file, leak_thresholds={ "default": 0, "tab": 10000, # See dependencies of bug 1051230. # GMP rarely gets a log, but when it does, it leaks a little. "geckomediaplugin": 20000, }, ignore_missing_leaks=["geckomediaplugin"], log=self.logger, stack_fixer=self.stack_fixer) def pid(self): if self.runner.process_handler is None: return None try: return self.runner.process_handler.pid except AttributeError: return None def on_output(self, line): """Write a line of output from the firefox process to the log""" data = line.decode("utf8", "replace") if self.stack_fixer: data = self.stack_fixer(data) self.logger.process_output(self.pid(), data, command=" ".join(self.runner.command)) def is_alive(self): if self.runner: return self.runner.is_running() return False def cleanup(self): self.stop() self.process_leaks() def executor_browser(self): assert self.marionette_port is not None return ExecutorBrowser, {"marionette_port": self.marionette_port} def check_for_crashes(self): dump_dir = os.path.join(self.profile.profile, "minidumps") return bool( mozcrash.check_for_crashes(dump_dir, symbols_path=self.symbols_path, stackwalk_binary=self.stackwalk_binary, quiet=True)) def log_crash(self, process, test): dump_dir = os.path.join(self.profile.profile, "minidumps") mozcrash.log_crashes(self.logger, dump_dir, symbols_path=self.symbols_path, stackwalk_binary=self.stackwalk_binary, process=process, test=test) def setup_ssl(self): """Create a certificate database to use in the test profile. This is configured to trust the CA Certificate that has signed the web-platform.test server certificate.""" if self.certutil_binary is None: self.logger.info( "--certutil-binary not supplied; Firefox will not check certificates" ) return self.logger.info("Setting up ssl") # Make sure the certutil libraries from the source tree are loaded when using a # local copy of certutil # TODO: Maybe only set this if certutil won't launch? env = os.environ.copy() certutil_dir = os.path.dirname(self.binary) if mozinfo.isMac: env_var = "DYLD_LIBRARY_PATH" elif mozinfo.isUnix: env_var = "LD_LIBRARY_PATH" else: env_var = "PATH" env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]]) if env_var in env else certutil_dir).encode( sys.getfilesystemencoding() or 'utf-8', 'replace') def certutil(*args): cmd = [self.certutil_binary] + list(args) self.logger.process_output( "certutil", subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT), " ".join(cmd)) pw_path = os.path.join(self.profile.profile, ".crtdbpw") with open(pw_path, "w") as f: # Use empty password for certificate db f.write("\n") cert_db_path = self.profile.profile # Create a new certificate db certutil("-N", "-d", cert_db_path, "-f", pw_path) # Add the CA certificate to the database and mark as trusted to issue server certs certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,", "-n", "web-platform-tests", "-i", self.ca_certificate_path) # List all certs in the database certutil("-L", "-d", cert_db_path)
class FirefoxBrowser(Browser): used_ports = set() init_timeout = 60 def __init__(self, logger, binary, prefs_root, debug_info=None, symbols_path=None, stackwalk_binary=None, certutil_binary=None, ca_certificate_path=None, e10s=False): Browser.__init__(self, logger) self.binary = binary self.prefs_root = prefs_root self.marionette_port = None self.runner = None self.debug_info = debug_info self.profile = None self.symbols_path = symbols_path self.stackwalk_binary = stackwalk_binary self.ca_certificate_path = ca_certificate_path self.certutil_binary = certutil_binary self.e10s = e10s def start(self): self.marionette_port = get_free_port(2828, exclude=self.used_ports) self.used_ports.add(self.marionette_port) env = os.environ.copy() env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" locations = ServerLocations(filename=os.path.join(here, "server-locations.txt")) preferences = self.load_prefs() self.profile = FirefoxProfile(locations=locations, preferences=preferences) self.profile.set_preferences({"marionette.defaultPrefs.enabled": True, "marionette.defaultPrefs.port": self.marionette_port, "dom.disable_open_during_load": False, "network.dns.localDomains": ",".join(hostnames), "network.proxy.type": 0, "places.history.enabled": False}) if self.e10s: self.profile.set_preferences({"browser.tabs.remote.autostart": True}) # Bug 1262954: winxp + e10s, disable hwaccel if (self.e10s and platform.system() in ("Windows", "Microsoft") and '5.1' in platform.version()): self.profile.set_preferences({"layers.acceleration.disabled": True}) if self.ca_certificate_path is not None: self.setup_ssl() debug_args, cmd = browser_command(self.binary, [cmd_arg("marionette"), "about:blank"], self.debug_info) self.runner = FirefoxRunner(profile=self.profile, binary=cmd[0], cmdargs=cmd[1:], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) self.logger.debug("Firefox Started") def load_prefs(self): prefs_path = os.path.join(self.prefs_root, "prefs_general.js") if os.path.exists(prefs_path): preferences = Preferences.read_prefs(prefs_path) else: self.logger.warning("Failed to find base prefs file in %s" % prefs_path) preferences = [] return preferences def stop(self): self.logger.debug("Stopping browser") if self.runner is not None: try: self.runner.stop() except OSError: # This can happen on Windows if the process is already dead pass def pid(self): if self.runner.process_handler is None: return None try: return self.runner.process_handler.pid except AttributeError: return None def on_output(self, line): """Write a line of output from the firefox process to the log""" self.logger.process_output(self.pid(), line.decode("utf8", "replace"), command=" ".join(self.runner.command)) def is_alive(self): if self.runner: return self.runner.is_running() return False def cleanup(self): self.stop() def executor_browser(self): assert self.marionette_port is not None return ExecutorBrowser, {"marionette_port": self.marionette_port} def log_crash(self, process, test): dump_dir = os.path.join(self.profile.profile, "minidumps") mozcrash.log_crashes(self.logger, dump_dir, symbols_path=self.symbols_path, stackwalk_binary=self.stackwalk_binary, process=process, test=test) def setup_ssl(self): """Create a certificate database to use in the test profile. This is configured to trust the CA Certificate that has signed the web-platform.test server certificate.""" self.logger.info("Setting up ssl") # Make sure the certutil libraries from the source tree are loaded when using a # local copy of certutil # TODO: Maybe only set this if certutil won't launch? env = os.environ.copy() certutil_dir = os.path.dirname(self.binary) if mozinfo.isMac: env_var = "DYLD_LIBRARY_PATH" elif mozinfo.isUnix: env_var = "LD_LIBRARY_PATH" else: env_var = "PATH" env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]]) if env_var in env else certutil_dir).encode( sys.getfilesystemencoding() or 'utf-8', 'replace') def certutil(*args): cmd = [self.certutil_binary] + list(args) self.logger.process_output("certutil", subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT), " ".join(cmd)) pw_path = os.path.join(self.profile.profile, ".crtdbpw") with open(pw_path, "w") as f: # Use empty password for certificate db f.write("\n") cert_db_path = self.profile.profile # Create a new certificate db certutil("-N", "-d", cert_db_path, "-f", pw_path) # Add the CA certificate to the database and mark as trusted to issue server certs certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,", "-n", "web-platform-tests", "-i", self.ca_certificate_path) # List all certs in the database certutil("-L", "-d", cert_db_path)
class FirefoxBrowser(Browser): used_ports = set() init_timeout = 60 shutdown_timeout = 60 def __init__(self, logger, binary, prefs_root, test_type, extra_prefs=None, debug_info=None, symbols_path=None, stackwalk_binary=None, certutil_binary=None, ca_certificate_path=None, e10s=False, stackfix_dir=None, binary_args=None, timeout_multiplier=None, leak_check=False, stylo_threads=1, chaos_mode_flags=None, config=None): Browser.__init__(self, logger) self.binary = binary self.prefs_root = prefs_root self.test_type = test_type self.extra_prefs = extra_prefs self.marionette_port = None self.runner = None self.debug_info = debug_info self.profile = None self.symbols_path = symbols_path self.stackwalk_binary = stackwalk_binary self.ca_certificate_path = ca_certificate_path self.certutil_binary = certutil_binary self.e10s = e10s self.binary_args = binary_args self.config = config if stackfix_dir: self.stack_fixer = get_stack_fixer_function(stackfix_dir, self.symbols_path) else: self.stack_fixer = None if timeout_multiplier: self.init_timeout = self.init_timeout * timeout_multiplier self.leak_report_file = None self.leak_check = leak_check self.stylo_threads = stylo_threads self.chaos_mode_flags = chaos_mode_flags def settings(self, test): return {"check_leaks": self.leak_check and not test.leaks} def start(self, **kwargs): if self.marionette_port is None: self.marionette_port = get_free_port(2828, exclude=self.used_ports) self.used_ports.add(self.marionette_port) env = os.environ.copy() env["MOZ_CRASHREPORTER"] = "1" env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env["STYLO_THREADS"] = str(self.stylo_threads) if self.chaos_mode_flags is not None: env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags) preferences = self.load_prefs() self.profile = FirefoxProfile(preferences=preferences) self.profile.set_preferences({"marionette.port": self.marionette_port, "dom.disable_open_during_load": False, "network.dns.localDomains": ",".join(self.config.domains_set), "network.proxy.type": 0, "places.history.enabled": False, "dom.send_after_paint_to_content": True, "network.preload": True}) if self.e10s: self.profile.set_preferences({"browser.tabs.remote.autostart": True}) if self.test_type == "reftest": self.profile.set_preferences({"layout.interruptible-reflow.enabled": False}) if self.leak_check and kwargs.get("check_leaks", True): self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log") if os.path.exists(self.leak_report_file): os.remove(self.leak_report_file) env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file else: self.leak_report_file = None # Bug 1262954: winxp + e10s, disable hwaccel if (self.e10s and platform.system() in ("Windows", "Microsoft") and '5.1' in platform.version()): self.profile.set_preferences({"layers.acceleration.disabled": True}) if self.ca_certificate_path is not None: self.setup_ssl() debug_args, cmd = browser_command(self.binary, self.binary_args if self.binary_args else [] + [cmd_arg("marionette"), "about:blank"], self.debug_info) self.runner = FirefoxRunner(profile=self.profile, binary=cmd[0], cmdargs=cmd[1:], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) self.logger.debug("Firefox Started") def load_prefs(self): prefs = Preferences() pref_paths = [] prefs_general = os.path.join(self.prefs_root, 'prefs_general.js') if os.path.isfile(prefs_general): # Old preference file used in Firefox 60 and earlier (remove when no longer supported) pref_paths.append(prefs_general) profiles = os.path.join(self.prefs_root, 'profiles.json') if os.path.isfile(profiles): with open(profiles, 'r') as fh: for name in json.load(fh)['web-platform-tests']: pref_paths.append(os.path.join(self.prefs_root, name, 'user.js')) for path in pref_paths: if os.path.exists(path): prefs.add(Preferences.read_prefs(path)) else: self.logger.warning("Failed to find base prefs file in %s" % path) # Add any custom preferences prefs.add(self.extra_prefs, cast=True) return prefs() def stop(self, force=False): if self.runner is not None and self.runner.is_running(): try: # For Firefox we assume that stopping the runner prompts the # browser to shut down. This allows the leak log to be written for clean, stop_f in [(True, lambda: self.runner.wait(self.shutdown_timeout)), (False, lambda: self.runner.stop(signal.SIGTERM)), (False, lambda: self.runner.stop(signal.SIGKILL))]: if not force or not clean: retcode = stop_f() if retcode is not None: self.logger.info("Browser exited with return code %s" % retcode) break except OSError: # This can happen on Windows if the process is already dead pass self.logger.debug("stopped") def process_leaks(self): self.logger.debug("PROCESS LEAKS %s" % self.leak_report_file) if self.leak_report_file is None: return mozleak.process_leak_log( self.leak_report_file, leak_thresholds={ "default": 0, "tab": 10000, # See dependencies of bug 1051230. # GMP rarely gets a log, but when it does, it leaks a little. "geckomediaplugin": 20000, }, ignore_missing_leaks=["geckomediaplugin"], log=self.logger, stack_fixer=self.stack_fixer ) def pid(self): if self.runner.process_handler is None: return None try: return self.runner.process_handler.pid except AttributeError: return None def on_output(self, line): """Write a line of output from the firefox process to the log""" if "GLib-GObject-CRITICAL" in line: return if line: data = line.decode("utf8", "replace") if self.stack_fixer: data = self.stack_fixer(data) self.logger.process_output(self.pid(), data, command=" ".join(self.runner.command)) def is_alive(self): if self.runner: return self.runner.is_running() return False def cleanup(self): self.stop() self.process_leaks() def executor_browser(self): assert self.marionette_port is not None return ExecutorBrowser, {"marionette_port": self.marionette_port} def check_for_crashes(self): dump_dir = os.path.join(self.profile.profile, "minidumps") return bool(mozcrash.check_for_crashes(dump_dir, symbols_path=self.symbols_path, stackwalk_binary=self.stackwalk_binary, quiet=True)) def log_crash(self, process, test): dump_dir = os.path.join(self.profile.profile, "minidumps") mozcrash.log_crashes(self.logger, dump_dir, symbols_path=self.symbols_path, stackwalk_binary=self.stackwalk_binary, process=process, test=test) def setup_ssl(self): """Create a certificate database to use in the test profile. This is configured to trust the CA Certificate that has signed the web-platform.test server certificate.""" if self.certutil_binary is None: self.logger.info("--certutil-binary not supplied; Firefox will not check certificates") return self.logger.info("Setting up ssl") # Make sure the certutil libraries from the source tree are loaded when using a # local copy of certutil # TODO: Maybe only set this if certutil won't launch? env = os.environ.copy() certutil_dir = os.path.dirname(self.binary) if mozinfo.isMac: env_var = "DYLD_LIBRARY_PATH" elif mozinfo.isUnix: env_var = "LD_LIBRARY_PATH" else: env_var = "PATH" env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]]) if env_var in env else certutil_dir).encode( sys.getfilesystemencoding() or 'utf-8', 'replace') def certutil(*args): cmd = [self.certutil_binary] + list(args) self.logger.process_output("certutil", subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT), " ".join(cmd)) pw_path = os.path.join(self.profile.profile, ".crtdbpw") with open(pw_path, "w") as f: # Use empty password for certificate db f.write("\n") cert_db_path = self.profile.profile # Create a new certificate db certutil("-N", "-d", cert_db_path, "-f", pw_path) # Add the CA certificate to the database and mark as trusted to issue server certs certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,", "-n", "web-platform-tests", "-i", self.ca_certificate_path) # List all certs in the database certutil("-L", "-d", cert_db_path)
class FirefoxBrowser(Browser): used_ports = set() init_timeout = 70 shutdown_timeout = 70 def __init__(self, logger, binary, prefs_root, test_type, extra_prefs=None, debug_info=None, symbols_path=None, stackwalk_binary=None, certutil_binary=None, ca_certificate_path=None, e10s=False, lsan_dir=None, stackfix_dir=None, binary_args=None, timeout_multiplier=None, leak_check=False, asan=False, stylo_threads=1, chaos_mode_flags=None, config=None, browser_channel="nightly", headless=None, **kwargs): Browser.__init__(self, logger) self.binary = binary self.prefs_root = prefs_root self.test_type = test_type self.extra_prefs = extra_prefs self.marionette_port = None self.runner = None self.debug_info = debug_info self.profile = None self.symbols_path = symbols_path self.stackwalk_binary = stackwalk_binary self.ca_certificate_path = ca_certificate_path self.certutil_binary = certutil_binary self.e10s = e10s self.binary_args = binary_args self.config = config if stackfix_dir: self.stack_fixer = get_stack_fixer_function( stackfix_dir, self.symbols_path) else: self.stack_fixer = None if timeout_multiplier: self.init_timeout = self.init_timeout * timeout_multiplier self.asan = asan self.lsan_dir = lsan_dir self.lsan_allowed = None self.lsan_max_stack_depth = None self.mozleak_allowed = None self.mozleak_thresholds = None self.leak_check = leak_check self.leak_report_file = None self.lsan_handler = None self.stylo_threads = stylo_threads self.chaos_mode_flags = chaos_mode_flags self.browser_channel = browser_channel self.headless = headless def settings(self, test): return { "check_leaks": self.leak_check and not test.leaks, "lsan_allowed": test.lsan_allowed, "lsan_max_stack_depth": test.lsan_max_stack_depth, "mozleak_allowed": self.leak_check and test.mozleak_allowed, "mozleak_thresholds": self.leak_check and test.mozleak_threshold } def start(self, group_metadata=None, **kwargs): if group_metadata is None: group_metadata = {} self.group_metadata = group_metadata self.lsan_allowed = kwargs.get("lsan_allowed") self.lsan_max_stack_depth = kwargs.get("lsan_max_stack_depth") self.mozleak_allowed = kwargs.get("mozleak_allowed") self.mozleak_thresholds = kwargs.get("mozleak_thresholds") if self.marionette_port is None: self.marionette_port = get_free_port(2828, exclude=self.used_ports) self.used_ports.add(self.marionette_port) if self.asan: self.lsan_handler = mozleak.LSANLeaks( self.logger, scope=group_metadata.get("scope", "/"), allowed=self.lsan_allowed, maxNumRecordedFrames=self.lsan_max_stack_depth) env = test_environment(xrePath=os.path.dirname(self.binary), debugger=self.debug_info is not None, log=self.logger, lsanPath=self.lsan_dir) env["STYLO_THREADS"] = str(self.stylo_threads) if self.chaos_mode_flags is not None: env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags) if self.headless: env["MOZ_HEADLESS"] = "1" preferences = self.load_prefs() self.profile = FirefoxProfile(preferences=preferences) self.profile.set_preferences({ "marionette.port": self.marionette_port, "network.dns.localDomains": ",".join(self.config.domains_set), "dom.file.createInChild": True, # TODO: Remove preferences once Firefox 64 is stable (Bug 905404) "network.proxy.type": 0, "places.history.enabled": False, "network.preload": True, }) if self.e10s: self.profile.set_preferences( {"browser.tabs.remote.autostart": True}) if self.test_type == "reftest": self.profile.set_preferences( {"layout.interruptible-reflow.enabled": False}) if self.leak_check: self.leak_report_file = os.path.join( self.profile.profile, "runtests_leaks_%s.log" % os.getpid()) if os.path.exists(self.leak_report_file): os.remove(self.leak_report_file) env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file else: self.leak_report_file = None # Bug 1262954: winxp + e10s, disable hwaccel if (self.e10s and platform.system() in ("Windows", "Microsoft") and '5.1' in platform.version()): self.profile.set_preferences( {"layers.acceleration.disabled": True}) if self.ca_certificate_path is not None: self.setup_ssl() args = self.binary_args[:] if self.binary_args else [] args += [cmd_arg("marionette"), "about:blank"] debug_args, cmd = browser_command(self.binary, args, self.debug_info) self.runner = FirefoxRunner( profile=self.profile, binary=cmd[0], cmdargs=cmd[1:], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) self.logger.debug("Firefox Started") def load_prefs(self): prefs = Preferences() pref_paths = [] profiles = os.path.join(self.prefs_root, 'profiles.json') if os.path.isfile(profiles): with open(profiles, 'r') as fh: for name in json.load(fh)['web-platform-tests']: if self.browser_channel in (None, 'nightly'): pref_paths.append( os.path.join(self.prefs_root, name, 'user.js')) elif name != 'unittest-features': pref_paths.append( os.path.join(self.prefs_root, name, 'user.js')) else: # Old preference files used before the creation of profiles.json (remove when no longer supported) legacy_pref_paths = ( os.path.join( self.prefs_root, 'prefs_general.js'), # Used in Firefox 60 and below os.path.join(self.prefs_root, 'common', 'user.js'), # Used in Firefox 61 ) for path in legacy_pref_paths: if os.path.isfile(path): pref_paths.append(path) for path in pref_paths: if os.path.exists(path): prefs.add(Preferences.read_prefs(path)) else: self.logger.warning("Failed to find base prefs file in %s" % path) # Add any custom preferences prefs.add(self.extra_prefs, cast=True) return prefs() def stop(self, force=False): if self.runner is not None and self.runner.is_running(): try: # For Firefox we assume that stopping the runner prompts the # browser to shut down. This allows the leak log to be written for clean, stop_f in [ (True, lambda: self.runner.wait(self.shutdown_timeout)), (False, lambda: self.runner.stop(signal.SIGTERM)), (False, lambda: self.runner.stop(signal.SIGKILL)) ]: if not force or not clean: retcode = stop_f() if retcode is not None: self.logger.info( "Browser exited with return code %s" % retcode) break except OSError: # This can happen on Windows if the process is already dead pass self.process_leaks() self.logger.debug("stopped") def process_leaks(self): self.logger.info("PROCESS LEAKS %s" % self.leak_report_file) if self.lsan_handler: self.lsan_handler.process() if self.leak_report_file is not None: mozleak.process_leak_log(self.leak_report_file, leak_thresholds=self.mozleak_thresholds, ignore_missing_leaks=["gmplugin"], log=self.logger, stack_fixer=self.stack_fixer, scope=self.group_metadata.get("scope"), allowed=self.mozleak_allowed) def pid(self): if self.runner.process_handler is None: return None try: return self.runner.process_handler.pid except AttributeError: return None def on_output(self, line): """Write a line of output from the firefox process to the log""" if "GLib-GObject-CRITICAL" in line: return if line: data = line.decode("utf8", "replace") if self.stack_fixer: data = self.stack_fixer(data) if self.lsan_handler: data = self.lsan_handler.log(data) if data is not None: self.logger.process_output(self.pid(), data, command=" ".join( self.runner.command)) def is_alive(self): if self.runner: return self.runner.is_running() return False def cleanup(self, force=False): self.stop(force) def executor_browser(self): assert self.marionette_port is not None return ExecutorBrowser, {"marionette_port": self.marionette_port} def check_crash(self, process, test): dump_dir = os.path.join(self.profile.profile, "minidumps") return bool( mozcrash.log_crashes(self.logger, dump_dir, symbols_path=self.symbols_path, stackwalk_binary=self.stackwalk_binary, process=process, test=test)) def setup_ssl(self): """Create a certificate database to use in the test profile. This is configured to trust the CA Certificate that has signed the web-platform.test server certificate.""" if self.certutil_binary is None: self.logger.info( "--certutil-binary not supplied; Firefox will not check certificates" ) return self.logger.info("Setting up ssl") # Make sure the certutil libraries from the source tree are loaded when using a # local copy of certutil # TODO: Maybe only set this if certutil won't launch? env = os.environ.copy() certutil_dir = os.path.dirname(self.binary or self.certutil_binary) if mozinfo.isMac: env_var = "DYLD_LIBRARY_PATH" elif mozinfo.isUnix: env_var = "LD_LIBRARY_PATH" else: env_var = "PATH" env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]]) if env_var in env else certutil_dir).encode( sys.getfilesystemencoding() or 'utf-8', 'replace') def certutil(*args): cmd = [self.certutil_binary] + list(args) self.logger.process_output( "certutil", subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT), " ".join(cmd)) pw_path = os.path.join(self.profile.profile, ".crtdbpw") with open(pw_path, "w") as f: # Use empty password for certificate db f.write("\n") cert_db_path = self.profile.profile # Create a new certificate db certutil("-N", "-d", cert_db_path, "-f", pw_path) # Add the CA certificate to the database and mark as trusted to issue server certs certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,", "-n", "web-platform-tests", "-i", self.ca_certificate_path) # List all certs in the database certutil("-L", "-d", cert_db_path)
class FirefoxBrowser(Browser): used_ports = set() def __init__(self, logger, binary, prefs_root, debug_args=None, interactive=None, symbols_path=None, stackwalk_binary=None): Browser.__init__(self, logger) self.binary = binary self.prefs_root = prefs_root self.marionette_port = None self.used_ports.add(self.marionette_port) self.runner = None self.debug_args = debug_args self.interactive = interactive self.profile = None self.symbols_path = symbols_path self.stackwalk_binary = stackwalk_binary def start(self): self.marionette_port = get_free_port(2828, exclude=self.used_ports) env = os.environ.copy() env["MOZ_CRASHREPORTER"] = "1" env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" locations = ServerLocations( filename=os.path.join(here, "server-locations.txt")) preferences = self.load_prefs() self.profile = FirefoxProfile(locations=locations, proxy=True, preferences=preferences) self.profile.set_preferences({ "marionette.defaultPrefs.enabled": True, "marionette.defaultPrefs.port": self.marionette_port, "dom.disable_open_during_load": False }) self.runner = FirefoxRunner( profile=self.profile, binary=self.binary, cmdargs=[cmd_arg("marionette"), "about:blank"], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=self.debug_args, interactive=self.interactive) self.logger.debug("Firefox Started") def load_prefs(self): prefs_path = os.path.join(self.prefs_root, "prefs_general.js") if os.path.exists(prefs_path): preferences = Preferences.read_prefs(prefs_path) else: self.logger.warning("Failed to find base prefs file in %s" % prefs_path) preferences = [] return preferences def stop(self): self.logger.debug("Stopping browser") if self.runner is not None: try: self.runner.stop() except OSError: # This can happen on Windows if the process is already dead pass def pid(self): if self.runner.process_handler is None: return None try: return self.runner.process_handler.pid except AttributeError: return None def on_output(self, line): """Write a line of output from the firefox process to the log""" self.logger.process_output(self.pid(), line.decode("utf8", "replace"), command=" ".join(self.runner.command)) def is_alive(self): return self.runner.is_running() def cleanup(self): self.stop() def executor_browser(self): assert self.marionette_port is not None return ExecutorBrowser, {"marionette_port": self.marionette_port} def log_crash(self, logger, process, test): dump_dir = os.path.join(self.profile.profile, "minidumps") mozcrash.log_crashes(logger, dump_dir, symbols_path=self.symbols_path, stackwalk_binary=self.stackwalk_binary, process=process, test=test)
class FirefoxBrowser(Browser): used_ports = set() def __init__(self, logger, binary, prefs_root, debug_args=None, interactive=None, symbols_path=None, stackwalk_binary=None): Browser.__init__(self, logger) self.binary = binary self.prefs_root = prefs_root self.marionette_port = None self.used_ports.add(self.marionette_port) self.runner = None self.debug_args = debug_args self.interactive = interactive self.profile = None self.symbols_path = symbols_path self.stackwalk_binary = stackwalk_binary def start(self): self.marionette_port = get_free_port(2828, exclude=self.used_ports) env = os.environ.copy() env["MOZ_CRASHREPORTER"] = "1" env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" locations = ServerLocations(filename=os.path.join(here, "server-locations.txt")) preferences = self.load_prefs() ports = {"http": "8000", "https": "8443", "ws": "8888"} self.profile = FirefoxProfile(locations=locations, proxy=ports, preferences=preferences) self.profile.set_preferences({"marionette.defaultPrefs.enabled": True, "marionette.defaultPrefs.port": self.marionette_port, "dom.disable_open_during_load": False}) self.runner = FirefoxRunner(profile=self.profile, binary=self.binary, cmdargs=[cmd_arg("marionette"), "about:blank"], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=self.debug_args, interactive=self.interactive) self.logger.debug("Firefox Started") def load_prefs(self): prefs_path = os.path.join(self.prefs_root, "prefs_general.js") if os.path.exists(prefs_path): preferences = Preferences.read_prefs(prefs_path) else: self.logger.warning("Failed to find base prefs file in %s" % prefs_path) preferences = [] return preferences def stop(self): self.logger.debug("Stopping browser") if self.runner is not None: try: self.runner.stop() except OSError: # This can happen on Windows if the process is already dead pass def pid(self): if self.runner.process_handler is None: return None try: return self.runner.process_handler.pid except AttributeError: return None def on_output(self, line): """Write a line of output from the firefox process to the log""" self.logger.process_output(self.pid(), line.decode("utf8", "replace"), command=" ".join(self.runner.command)) def is_alive(self): if self.runner: return self.runner.is_running() return False def cleanup(self): self.stop() def executor_browser(self): assert self.marionette_port is not None return ExecutorBrowser, {"marionette_port": self.marionette_port} def log_crash(self, process, test): dump_dir = os.path.join(self.profile.profile, "minidumps") mozcrash.log_crashes(self.logger, dump_dir, symbols_path=self.symbols_path, stackwalk_binary=self.stackwalk_binary, process=process, test=test)
class FirefoxBrowser(Browser): used_ports = set() def __init__(self, logger, binary, prefs_root, debug_args=None, interactive=None, symbols_path=None, stackwalk_binary=None, certutil_binary=None, ca_certificate_path=None): Browser.__init__(self, logger) self.binary = binary self.prefs_root = prefs_root self.marionette_port = None self.used_ports.add(self.marionette_port) self.runner = None self.debug_args = debug_args self.interactive = interactive self.profile = None self.symbols_path = symbols_path self.stackwalk_binary = stackwalk_binary self.ca_certificate_path = ca_certificate_path self.certutil_binary = certutil_binary def start(self): self.marionette_port = get_free_port(2828, exclude=self.used_ports) env = os.environ.copy() env["MOZ_CRASHREPORTER"] = "1" env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" locations = ServerLocations(filename=os.path.join(here, "server-locations.txt")) preferences = self.load_prefs() ports = {"http": "8000", "https": "8443", "ws": "8888"} self.profile = FirefoxProfile(locations=locations, proxy=ports, preferences=preferences) self.profile.set_preferences({"marionette.defaultPrefs.enabled": True, "marionette.defaultPrefs.port": self.marionette_port, "dom.disable_open_during_load": False}) if self.ca_certificate_path is not None: self.setup_ssl() self.runner = FirefoxRunner(profile=self.profile, binary=self.binary, cmdargs=[cmd_arg("marionette"), "about:blank"], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=self.debug_args, interactive=self.interactive) self.logger.debug("Firefox Started") def load_prefs(self): prefs_path = os.path.join(self.prefs_root, "prefs_general.js") if os.path.exists(prefs_path): preferences = Preferences.read_prefs(prefs_path) else: self.logger.warning("Failed to find base prefs file in %s" % prefs_path) preferences = [] return preferences def stop(self): self.logger.debug("Stopping browser") if self.runner is not None: try: self.runner.stop() except OSError: # This can happen on Windows if the process is already dead pass def pid(self): if self.runner.process_handler is None: return None try: return self.runner.process_handler.pid except AttributeError: return None def on_output(self, line): """Write a line of output from the firefox process to the log""" self.logger.process_output(self.pid(), line.decode("utf8", "replace"), command=" ".join(self.runner.command)) def is_alive(self): if self.runner: return self.runner.is_running() return False def cleanup(self): self.stop() def executor_browser(self): assert self.marionette_port is not None return ExecutorBrowser, {"marionette_port": self.marionette_port} def log_crash(self, process, test): dump_dir = os.path.join(self.profile.profile, "minidumps") mozcrash.log_crashes(self.logger, dump_dir, symbols_path=self.symbols_path, stackwalk_binary=self.stackwalk_binary, process=process, test=test) def setup_ssl(self): """Create a certificate database to use in the test profile. This is configured to trust the CA Certificate that has signed the web-platform.test server certificate.""" self.logger.info("Setting up ssl") # Make sure the certutil libraries from the source tree are loaded when using a # local copy of certutil # TODO: Maybe only set this if certutil won't launch? env = os.environ.copy() certutil_dir = os.path.dirname(self.binary) env["LD_LIBRARY_PATH"] = certutil_dir env["PATH"] = os.path.pathsep.join([certutil_dir, env["PATH"]]) def certutil(*args): cmd = [self.certutil_binary] + list(args) self.logger.process_output("certutil", subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT), " ".join(cmd)) pw_path = os.path.join(self.profile.profile, ".crtdbpw") with open(pw_path, "w") as f: # Use empty password for certificate db f.write("\n") cert_db_path = self.profile.profile # Create a new certificate db certutil("-N", "-d", cert_db_path, "-f", pw_path) # Add the CA certificate to the database and mark as trusted to issue server certs certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,", "-n", "web-platform-tests", "-i", self.ca_certificate_path) # List all certs in the database certutil("-L", "-d", cert_db_path)
class FirefoxBrowser(Browser): init_timeout = 70 shutdown_timeout = 70 def __init__(self, logger, binary, prefs_root, test_type, extra_prefs=None, debug_info=None, symbols_path=None, stackwalk_binary=None, certutil_binary=None, ca_certificate_path=None, e10s=False, lsan_dir=None, stackfix_dir=None, binary_args=None, timeout_multiplier=None, leak_check=False, asan=False, stylo_threads=1, chaos_mode_flags=None, config=None, browser_channel="nightly", headless=None, **kwargs): Browser.__init__(self, logger) self.binary = binary self.prefs_root = prefs_root self.test_type = test_type self.extra_prefs = extra_prefs self.marionette_port = None self.runner = None self.debug_info = debug_info self.profile = None self.symbols_path = symbols_path self.stackwalk_binary = stackwalk_binary self.ca_certificate_path = ca_certificate_path self.certutil_binary = certutil_binary self.e10s = e10s self.binary_args = binary_args self.config = config if stackfix_dir: self.stack_fixer = get_stack_fixer_function(stackfix_dir, self.symbols_path) else: self.stack_fixer = None if timeout_multiplier: self.init_timeout = self.init_timeout * timeout_multiplier self.asan = asan self.lsan_dir = lsan_dir self.lsan_allowed = None self.lsan_max_stack_depth = None self.mozleak_allowed = None self.mozleak_thresholds = None self.leak_check = leak_check self.leak_report_file = None self.lsan_handler = None self.stylo_threads = stylo_threads self.chaos_mode_flags = chaos_mode_flags self.browser_channel = browser_channel self.headless = headless def settings(self, test): return {"check_leaks": self.leak_check and not test.leaks, "lsan_allowed": test.lsan_allowed, "lsan_max_stack_depth": test.lsan_max_stack_depth, "mozleak_allowed": self.leak_check and test.mozleak_allowed, "mozleak_thresholds": self.leak_check and test.mozleak_threshold} def start(self, group_metadata=None, **kwargs): if group_metadata is None: group_metadata = {} self.group_metadata = group_metadata self.lsan_allowed = kwargs.get("lsan_allowed") self.lsan_max_stack_depth = kwargs.get("lsan_max_stack_depth") self.mozleak_allowed = kwargs.get("mozleak_allowed") self.mozleak_thresholds = kwargs.get("mozleak_thresholds") if self.marionette_port is None: self.marionette_port = get_free_port() if self.asan: self.lsan_handler = mozleak.LSANLeaks(self.logger, scope=group_metadata.get("scope", "/"), allowed=self.lsan_allowed, maxNumRecordedFrames=self.lsan_max_stack_depth) env = test_environment(xrePath=os.path.dirname(self.binary), debugger=self.debug_info is not None, log=self.logger, lsanPath=self.lsan_dir) env["STYLO_THREADS"] = str(self.stylo_threads) if self.chaos_mode_flags is not None: env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags) if self.headless: env["MOZ_HEADLESS"] = "1" preferences = self.load_prefs() self.profile = FirefoxProfile(preferences=preferences) self.profile.set_preferences({ "marionette.port": self.marionette_port, "network.dns.localDomains": ",".join(self.config.domains_set), "dom.file.createInChild": True, # TODO: Remove preferences once Firefox 64 is stable (Bug 905404) "network.proxy.type": 0, "places.history.enabled": False, "network.preload": True, }) if self.e10s: self.profile.set_preferences({"browser.tabs.remote.autostart": True}) if self.test_type == "reftest": self.profile.set_preferences({"layout.interruptible-reflow.enabled": False}) if self.leak_check: self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks_%s.log" % os.getpid()) if os.path.exists(self.leak_report_file): os.remove(self.leak_report_file) env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file else: self.leak_report_file = None # Bug 1262954: winxp + e10s, disable hwaccel if (self.e10s and platform.system() in ("Windows", "Microsoft") and '5.1' in platform.version()): self.profile.set_preferences({"layers.acceleration.disabled": True}) if self.ca_certificate_path is not None: self.setup_ssl() args = self.binary_args[:] if self.binary_args else [] args += [cmd_arg("marionette"), "about:blank"] debug_args, cmd = browser_command(self.binary, args, self.debug_info) self.runner = FirefoxRunner(profile=self.profile, binary=cmd[0], cmdargs=cmd[1:], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) self.logger.debug("Firefox Started") def load_prefs(self): prefs = Preferences() pref_paths = [] profiles = os.path.join(self.prefs_root, 'profiles.json') if os.path.isfile(profiles): with open(profiles, 'r') as fh: for name in json.load(fh)['web-platform-tests']: if self.browser_channel in (None, 'nightly'): pref_paths.append(os.path.join(self.prefs_root, name, 'user.js')) elif name != 'unittest-features': pref_paths.append(os.path.join(self.prefs_root, name, 'user.js')) else: # Old preference files used before the creation of profiles.json (remove when no longer supported) legacy_pref_paths = ( os.path.join(self.prefs_root, 'prefs_general.js'), # Used in Firefox 60 and below os.path.join(self.prefs_root, 'common', 'user.js'), # Used in Firefox 61 ) for path in legacy_pref_paths: if os.path.isfile(path): pref_paths.append(path) for path in pref_paths: if os.path.exists(path): prefs.add(Preferences.read_prefs(path)) else: self.logger.warning("Failed to find base prefs file in %s" % path) # Add any custom preferences prefs.add(self.extra_prefs, cast=True) return prefs() def stop(self, force=False): if self.runner is not None and self.runner.is_running(): try: # For Firefox we assume that stopping the runner prompts the # browser to shut down. This allows the leak log to be written for clean, stop_f in [(True, lambda: self.runner.wait(self.shutdown_timeout)), (False, lambda: self.runner.stop(signal.SIGTERM)), (False, lambda: self.runner.stop(signal.SIGKILL))]: if not force or not clean: retcode = stop_f() if retcode is not None: self.logger.info("Browser exited with return code %s" % retcode) break except OSError: # This can happen on Windows if the process is already dead pass self.process_leaks() self.logger.debug("stopped") def process_leaks(self): self.logger.info("PROCESS LEAKS %s" % self.leak_report_file) if self.lsan_handler: self.lsan_handler.process() if self.leak_report_file is not None: mozleak.process_leak_log( self.leak_report_file, leak_thresholds=self.mozleak_thresholds, ignore_missing_leaks=["gmplugin"], log=self.logger, stack_fixer=self.stack_fixer, scope=self.group_metadata.get("scope"), allowed=self.mozleak_allowed ) def pid(self): if self.runner.process_handler is None: return None try: return self.runner.process_handler.pid except AttributeError: return None def on_output(self, line): """Write a line of output from the firefox process to the log""" if "GLib-GObject-CRITICAL" in line: return if line: data = line.decode("utf8", "replace") if self.stack_fixer: data = self.stack_fixer(data) if self.lsan_handler: data = self.lsan_handler.log(data) if data is not None: self.logger.process_output(self.pid(), data, command=" ".join(self.runner.command)) def is_alive(self): if self.runner: return self.runner.is_running() return False def cleanup(self, force=False): self.stop(force) def executor_browser(self): assert self.marionette_port is not None return ExecutorBrowser, {"marionette_port": self.marionette_port} def check_crash(self, process, test): dump_dir = os.path.join(self.profile.profile, "minidumps") return bool(mozcrash.log_crashes(self.logger, dump_dir, symbols_path=self.symbols_path, stackwalk_binary=self.stackwalk_binary, process=process, test=test)) def setup_ssl(self): """Create a certificate database to use in the test profile. This is configured to trust the CA Certificate that has signed the web-platform.test server certificate.""" if self.certutil_binary is None: self.logger.info("--certutil-binary not supplied; Firefox will not check certificates") return self.logger.info("Setting up ssl") # Make sure the certutil libraries from the source tree are loaded when using a # local copy of certutil # TODO: Maybe only set this if certutil won't launch? env = os.environ.copy() certutil_dir = os.path.dirname(self.binary or self.certutil_binary) if mozinfo.isMac: env_var = "DYLD_LIBRARY_PATH" elif mozinfo.isUnix: env_var = "LD_LIBRARY_PATH" else: env_var = "PATH" env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]]) if env_var in env else certutil_dir).encode( sys.getfilesystemencoding() or 'utf-8', 'replace') def certutil(*args): cmd = [self.certutil_binary] + list(args) self.logger.process_output("certutil", subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT), " ".join(cmd)) pw_path = os.path.join(self.profile.profile, ".crtdbpw") with open(pw_path, "w") as f: # Use empty password for certificate db f.write("\n") cert_db_path = self.profile.profile # Create a new certificate db certutil("-N", "-d", cert_db_path, "-f", pw_path) # Add the CA certificate to the database and mark as trusted to issue server certs certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,", "-n", "web-platform-tests", "-i", self.ca_certificate_path) # List all certs in the database certutil("-L", "-d", cert_db_path)