class PuppetTarget(Target): __slots__ = ("use_rr", "use_valgrind", "_puppet", "_remove_prefs") def __init__(self, binary, extension, launch_timeout, log_limit, memory_limit, **kwds): super().__init__(binary, extension, launch_timeout, log_limit, memory_limit) self.use_rr = kwds.pop("rr", False) self.use_valgrind = kwds.pop("valgrind", False) self._remove_prefs = False # create Puppet object self._puppet = FFPuppet(use_rr=self.use_rr, use_valgrind=self.use_valgrind, use_xvfb=kwds.pop("xvfb", False)) if kwds: LOG.warning("PuppetTarget ignoring unsupported arguments: %s", ", ".join(kwds)) def _abort_hung_proc(self): # send SIGABRT to the busiest process with self._lock: proc_usage = self._puppet.cpu_usage() for pid, cpu in sorted(proc_usage, reverse=True, key=lambda x: x[1]): LOG.debug("sending SIGABRT to pid: %r, cpu: %0.2f%%", pid, cpu) try: kill(pid, SIGABRT) except OSError: LOG.warning("Failed to send SIGABRT to pid %d", pid) break def add_abort_token(self, token): self._puppet.add_abort_token(token) def cleanup(self): # prevent parallel calls to FFPuppet.close() and/or FFPuppet.clean_up() with self._lock: self._puppet.clean_up() if self._remove_prefs and self._prefs and isfile(self._prefs): unlink(self._prefs) def close(self): # prevent parallel calls to FFPuppet.close() and/or FFPuppet.clean_up() with self._lock: self._puppet.close() @property def closed(self): return self._puppet.reason is not None def is_idle(self, threshold): for _, cpu in self._puppet.cpu_usage(): if cpu >= threshold: return False return True def create_report(self): logs = mkdtemp(prefix="logs_", dir=grz_tmp("logs")) self.save_logs(logs) return Report(logs, self.binary) @property def monitor(self): if self._monitor is None: class _PuppetMonitor(TargetMonitor): # pylint: disable=no-self-argument,protected-access def clone_log(_, log_id, offset=0): return self._puppet.clone_log(log_id, offset=offset) def is_running(_): return self._puppet.is_running() def is_healthy(_): return self._puppet.is_healthy() @property def launches(_): return self._puppet.launches def log_length(_, log_id): return self._puppet.log_length(log_id) self._monitor = _PuppetMonitor() return self._monitor def detect_failure(self, ignored, was_timeout): status = self.RESULT_NONE is_healthy = self._puppet.is_healthy() # check if there has been a crash, hang, etc... if not is_healthy or was_timeout: if self._puppet.is_running(): LOG.debug("terminating browser...") if was_timeout and "timeout" not in ignored and system( ) == "Linux": self._abort_hung_proc() # give the process a moment to start dump self._puppet.wait(timeout=1) self.close() # if something has happened figure out what if not is_healthy: if self._puppet.reason == FFPuppet.RC_CLOSED: LOG.debug("target.close() was called") elif self._puppet.reason == FFPuppet.RC_EXITED: LOG.debug("target closed itself") elif (self._puppet.reason == FFPuppet.RC_WORKER and "memory" in ignored and "ffp_worker_memory_usage" in self._puppet.available_logs()): status = self.RESULT_IGNORED LOG.debug("memory limit exceeded") elif (self._puppet.reason == FFPuppet.RC_WORKER and "log-limit" in ignored and "ffp_worker_log_size" in self._puppet.available_logs()): status = self.RESULT_IGNORED LOG.debug("log size limit exceeded") else: LOG.debug("failure detected, ffpuppet reason %r", self._puppet.reason) status = self.RESULT_FAILURE elif was_timeout: LOG.debug("timeout detected") status = self.RESULT_IGNORED if "timeout" in ignored else self.RESULT_FAILURE return status def dump_coverage(self, timeout=15): assert SIGUSR1 is not None pid = self._puppet.get_pid() if pid is None or not self._puppet.is_healthy(): LOG.debug("Skipping coverage dump (target is not in a good state)") return # If at this point, the browser is in a good state, i.e. no crashes # or hangs, so signal the browser to dump coverage. try: for child in Process(pid).children(recursive=True): LOG.debug("Sending SIGUSR1 to %d (child)", child.pid) try: kill(child.pid, SIGUSR1) except OSError: LOG.warning("Failed to send SIGUSR1 to pid %d", child.pid) except (AccessDenied, NoSuchProcess): # pragma: no cover pass LOG.debug("Sending SIGUSR1 to %d (parent)", pid) try: kill(pid, SIGUSR1) except OSError: LOG.warning("Failed to send SIGUSR1 to pid %d", pid) start_time = time() gcda_found = False delay = 0.1 # wait for processes to write .gcno files # this should typically take less than 1 second while True: for proc in process_iter(attrs=["pid", "ppid", "open_files"]): # check if proc is the target or child process if pid in (proc.info["pid"], proc.info["ppid"]): if proc.info["open_files"] is None: continue if any( x.path.endswith(".gcda") for x in proc.info["open_files"]): gcda_found = True # get the pid of the process that has the file open gcda_open = proc.info["pid"] break else: gcda_open = None elapsed = time() - start_time if gcda_found: if gcda_open is None: # success LOG.debug("gcda dump took %0.2fs", elapsed) break if elapsed >= timeout: # timeout failure LOG.warning("gcda file open by pid %d after %0.2fs", gcda_open, elapsed) try: kill(gcda_open, SIGABRT) except OSError: pass sleep(1) self.close() break if delay < 1.0: # increase delay to a maximum of 1 second delay = min(1.0, delay + 0.1) elif elapsed >= 3: # assume we missed the process writing .gcno files LOG.warning("No gcda files seen after %0.2fs", elapsed) break if not self._puppet.is_healthy(): LOG.warning("Browser failure during dump_coverage()") break sleep(delay) def launch(self, location, env_mod=None): # setup environment env_mod = dict(env_mod or []) # do not allow network connections to non local endpoints env_mod["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env_mod["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" try: self._puppet.launch(self.binary, launch_timeout=self.launch_timeout, location=location, log_limit=self.log_limit, memory_limit=self.memory_limit, prefs_js=self.prefs, extension=self.extension, env_mod=env_mod) except LaunchError as exc: LOG.error("FFPuppet LaunchError: %s", str(exc)) self.close() if isinstance(exc, BrowserTimeoutError): raise TargetLaunchTimeout(str(exc)) from None raise TargetLaunchError(str(exc), self.create_report()) from None def log_size(self): return self._puppet.log_length("stderr") + self._puppet.log_length( "stdout") @property def prefs(self): if self._prefs is None: # generate temporary prefs.js for prefs_template in PrefPicker.templates(): if prefs_template.endswith("browser-fuzzing.yml"): LOG.debug("using prefpicker template %r", prefs_template) tmp_fd, self._prefs = mkstemp(prefix="prefs_", suffix=".js", dir=grz_tmp()) close(tmp_fd) PrefPicker.load_template(prefs_template).create_prefsjs( self._prefs) LOG.debug("generated prefs.js %r", self._prefs) self._remove_prefs = True break else: # pragma: no cover raise TargetError("Failed to generate prefs.js") return self._prefs @prefs.setter def prefs(self, prefs_file): if self._remove_prefs and self._prefs and isfile(self._prefs): unlink(self._prefs) if prefs_file is None: self._prefs = None self._remove_prefs = True elif isfile(prefs_file): self._prefs = abspath(prefs_file) self._remove_prefs = False else: raise TargetError("Missing prefs.js file %r" % (prefs_file, )) def save_logs(self, *args, **kwargs): self._puppet.save_logs(*args, **kwargs)
class PuppetTarget(Target): SUPPORTED_ASSETS = ( # file containing line separated list of tokens to scan stderr/out for "abort-tokens", # xpi or directory containing the unpacked extension "extension", # LSan suppression list file "lsan-suppressions", # prefs.js file to use "prefs", # TSan suppression list file "tsan-suppressions", # UBSan suppression list file "ubsan-suppressions", ) TRACKED_ENVVARS = ( "ASAN_OPTIONS", "LSAN_OPTIONS", "TSAN_OPTIONS", "UBSAN_OPTIONS", "GNOME_ACCESSIBILITY", "MOZ_CHAOSMODE", "XPCOM_DEBUG_BREAK", ) __slots__ = ("use_valgrind", "_extension", "_prefs", "_puppet") def __init__(self, binary, launch_timeout, log_limit, memory_limit, **kwds): super().__init__( binary, launch_timeout, log_limit, memory_limit, assets=kwds.pop("assets", None), ) # TODO: clean up handling debuggers debugger = Debugger.NONE if kwds.pop("pernosco", False): debugger = Debugger.PERNOSCO if kwds.pop("rr", False): debugger = Debugger.RR if kwds.pop("valgrind", False): self.use_valgrind = True debugger = Debugger.VALGRIND self._extension = None self._prefs = None # create Puppet object self._puppet = FFPuppet( debugger=debugger, use_xvfb=kwds.pop("xvfb", False), working_path=grz_tmp("target"), ) if kwds: LOG.warning( "PuppetTarget ignoring unsupported arguments: %s", ", ".join(kwds) ) def _cleanup(self): # prevent parallel calls to FFPuppet.close() and/or FFPuppet.clean_up() with self._lock: self._puppet.clean_up() def close(self, force_close=False): # prevent parallel calls to FFPuppet.close() and/or FFPuppet.clean_up() with self._lock: self._puppet.close(force_close=force_close) @property def closed(self): return self._puppet.reason is not None def create_report(self, is_hang=False): logs = mkdtemp(prefix="logs_", dir=grz_tmp("logs")) self.save_logs(logs) return Report(logs, self.binary, is_hang=is_hang) def filtered_environ(self): # remove context specific entries from environment filtered = dict(self.environ) opts = SanitizerOptions() # iterate over *SAN_OPTIONS entries for san in (x for x in filtered if x.endswith("SAN_OPTIONS")): opts.load_options(filtered[san]) # remove entries specific to the current environment opts.pop("external_symbolizer_path") opts.pop("log_path") opts.pop("strip_path_prefix") opts.pop("suppressions") filtered[san] = opts.options # remove empty entries return {k: v for k, v in filtered.items() if v} def is_idle(self, threshold): for _, cpu in self._puppet.cpu_usage(): if cpu >= threshold: return False return True @property def monitor(self): if self._monitor is None: class _PuppetMonitor(TargetMonitor): # pylint: disable=no-self-argument,protected-access def clone_log(_, log_id, offset=0): return self._puppet.clone_log(log_id, offset=offset) def is_running(_): return self._puppet.is_running() def is_healthy(_): return self._puppet.is_healthy() @property def launches(_): return self._puppet.launches def log_length(_, log_id): return self._puppet.log_length(log_id) self._monitor = _PuppetMonitor() return self._monitor def check_result(self, ignored): result = Result.NONE # check if there has been a crash, hangs will appear as SIGABRT if not self._puppet.is_healthy(): self.close() # something has happened figure out what if self._puppet.reason == Reason.CLOSED: LOG.debug("target.close() was called") elif self._puppet.reason == Reason.EXITED: LOG.debug("target closed itself") elif ( self._puppet.reason == Reason.WORKER and "memory" in ignored and "ffp_worker_memory_usage" in self._puppet.available_logs() ): result = Result.IGNORED LOG.debug("memory limit exceeded") elif ( self._puppet.reason == Reason.WORKER and "log-limit" in ignored and "ffp_worker_log_size" in self._puppet.available_logs() ): result = Result.IGNORED LOG.debug("log size limit exceeded") else: # crash or hang (forced SIGABRT) has been detected LOG.debug("result detected (%s)", self._puppet.reason.name) result = Result.FOUND return result def handle_hang(self, ignore_idle=True): was_idle = False if self._puppet.is_healthy(): proc_usage = sorted(self._puppet.cpu_usage(), key=lambda x: x[1]) if proc_usage: pid, cpu = proc_usage.pop() if ignore_idle and cpu < 15: # don't send SIGABRT if process is idle LOG.debug("ignoring idle hang (%0.1f%%)", cpu) was_idle = True elif system() == "Linux": # sending SIGABRT is only supported on Linux for now # TODO: add/test on other OSs LOG.debug("sending SIGABRT to %r (%0.1f%%)", pid, cpu) try: kill(pid, SIGABRT) except OSError: LOG.warning("Failed to send SIGABRT to pid %d", pid) self._puppet.wait(timeout=10) # always call close() since this function should only/always # be called when there has been a timeout self.close() return was_idle def dump_coverage(self, timeout=15): assert SIGUSR1 is not None pid = self._puppet.get_pid() if pid is None or not self._puppet.is_healthy(): LOG.debug("Skipping coverage dump (target is not in a good state)") return # If at this point, the browser is in a good state, i.e. no crashes # or hangs, so signal the browser to dump coverage. try: for child in Process(pid).children(recursive=True): LOG.debug("Sending SIGUSR1 to %d (child)", child.pid) try: kill(child.pid, SIGUSR1) except OSError: LOG.warning("Failed to send SIGUSR1 to pid %d", child.pid) except (AccessDenied, NoSuchProcess): # pragma: no cover pass LOG.debug("Sending SIGUSR1 to %d (parent)", pid) try: kill(pid, SIGUSR1) except OSError: LOG.warning("Failed to send SIGUSR1 to pid %d", pid) start_time = time() gcda_found = False delay = 0.1 # wait for processes to write .gcno files # this should typically take less than 1 second while True: for proc in process_iter(attrs=["pid", "ppid", "open_files"]): # check if proc is the target or child process if pid in (proc.info["pid"], proc.info["ppid"]): if proc.info["open_files"] is None: continue if any(x.path.endswith(".gcda") for x in proc.info["open_files"]): gcda_found = True # get the pid of the process that has the file open gcda_open = proc.info["pid"] break else: gcda_open = None elapsed = time() - start_time if gcda_found: if gcda_open is None: # success LOG.debug("gcda dump took %0.2fs", elapsed) break if elapsed >= timeout: # timeout failure LOG.warning( "gcda file open by pid %d after %0.2fs", gcda_open, elapsed ) try: kill(gcda_open, SIGABRT) except OSError: pass sleep(1) self.close() break if delay < 1.0: # increase delay to a maximum of 1 second delay = min(1.0, delay + 0.1) elif elapsed >= 3: # assume we missed the process writing .gcno files LOG.warning("No gcda files seen after %0.2fs", elapsed) break if not self._puppet.is_healthy(): LOG.warning("Browser failure during dump_coverage()") break sleep(delay) def launch(self, location): # setup environment env_mod = dict(self.environ) # do not allow network connections to non local endpoints env_mod["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env_mod["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" try: self._puppet.launch( self.binary, launch_timeout=self.launch_timeout, location=location, log_limit=self.log_limit, memory_limit=self.memory_limit, prefs_js=self._prefs, extension=self._extension, env_mod=env_mod, ) except LaunchError as exc: LOG.error("FFPuppet LaunchError: %s", str(exc)) self.close() if isinstance(exc, BrowserTimeoutError): raise TargetLaunchTimeout(str(exc)) from None raise TargetLaunchError(str(exc), self.create_report()) from None def log_size(self): return self._puppet.log_length("stderr") + self._puppet.log_length("stdout") def process_assets(self): self._extension = self.assets.get("extension") self._prefs = self.assets.get("prefs") # generate temporary prefs.js with prefpicker if self._prefs is None: LOG.debug("using prefpicker to generate prefs.js") with TemporaryDirectory(dir=grz_tmp("target")) as tmp_path: prefs = Path(tmp_path) / "prefs.js" template = PrefPicker.lookup_template("browser-fuzzing.yml") PrefPicker.load_template(template).create_prefsjs(prefs) self._prefs = self.assets.add("prefs", str(prefs), copy=False) abort_tokens = self.assets.get("abort-tokens") if abort_tokens: LOG.debug("loading 'abort tokens' from %r", abort_tokens) with open(abort_tokens, "r") as in_fp: for line in in_fp: line = line.strip() if line: self._puppet.add_abort_token(line) # configure sanitizer suppressions opts = SanitizerOptions() for sanitizer in ("lsan", "tsan", "ubsan"): asset = "%s-suppressions" % (sanitizer,) # load existing sanitizer options from environment var_name = "%s_OPTIONS" % (sanitizer.upper(),) opts.load_options(self.environ.get(var_name, "")) if self.assets.get(asset): # use suppression file if provided as asset opts.add("suppressions", repr(self.assets.get(asset)), overwrite=True) elif opts.get("suppressions"): supp_file = opts.pop("suppressions") if SanitizerOptions.is_quoted(supp_file): supp_file = supp_file[1:-1] if isfile(supp_file): # use environment specified suppression file LOG.debug("using %r from environment", asset) opts.add( "suppressions", repr(self.assets.add(asset, supp_file)), overwrite=True, ) else: LOG.warning("Missing %s suppressions file %r", sanitizer, supp_file) else: # nothing to do continue # update sanitized *SAN_OPTIONS LOG.debug("updating suppressions in %r", var_name) self.environ[var_name] = opts.options def save_logs(self, *args, **kwargs): self._puppet.save_logs(*args, **kwargs)
class PuppetTarget(Target): def __init__(self, binary, extension, launch_timeout, log_limit, memory_limit, prefs, relaunch, **kwds): super(PuppetTarget, self).__init__(binary, extension, launch_timeout, log_limit, memory_limit, prefs, relaunch) self.use_rr = kwds.pop("rr", False) self.use_valgrind = kwds.pop("valgrind", False) use_xvfb = kwds.pop("xvfb", False) if kwds: log.warning("PuppetTarget ignoring unsupported arguments: %s", ", ".join(kwds)) # create Puppet object self._puppet = FFPuppet(use_rr=self.use_rr, use_valgrind=self.use_valgrind, use_xvfb=use_xvfb) def _abort_hung_proc(self): # send SIGABRT to the busiest process with self._lock: proc_usage = self._puppet.cpu_usage() for pid, cpu in sorted(proc_usage, reverse=True, key=lambda x: x[1]): log.debug("sending SIGABRT to pid: %r, cpu: %0.2f%%", pid, cpu) os.kill(pid, signal.SIGABRT) break def add_abort_token(self, token): self._puppet.add_abort_token(token) def cleanup(self): # prevent parallel calls to FFPuppet.clean_up() with self._lock: self._puppet.clean_up() def close(self): # prevent parallel calls to FFPuppet.close() with self._lock: self._puppet.close() @property def closed(self): return self._puppet.reason is not None @property def monitor(self): if self._monitor is None: class _PuppetMonitor(TargetMonitor): # pylint: disable=no-self-argument,protected-access def clone_log(_, log_id, offset=0): return self._puppet.clone_log(log_id, offset=offset) def is_running(_): return self._puppet.is_running() def is_healthy(_): return self._puppet.is_healthy() @property def launches(_): return self._puppet.launches def log_length(_, log_id): return self._puppet.log_length(log_id) self._monitor = _PuppetMonitor() return self._monitor def poll_for_idle(self, threshold, interval): # return POLL_IDLE if cpu usage of target is below threshold for interval seconds start_time = time.time() while time.time() - start_time < interval: for _, cpu in self._puppet.cpu_usage(): if cpu >= threshold: return self.POLL_BUSY if not self._puppet.is_running(): break else: log.info("Process utilized <= %d%% CPU for %ds", threshold, interval) return self.POLL_IDLE def detect_failure(self, ignored, was_timeout): status = self.RESULT_NONE if self.expect_close and not was_timeout: # give the browser a moment to close if needed self._puppet.wait(timeout=30) is_healthy = self._puppet.is_healthy() # check if there has been a crash, hang, etc... if not is_healthy or was_timeout: if self._puppet.is_running(): log.debug("terminating browser...") if was_timeout and "timeout" not in ignored: self._abort_hung_proc() # give the process a moment to start dump self._puppet.wait(timeout=1) self.close() # if something has happened figure out what if not is_healthy: if self._puppet.reason == FFPuppet.RC_CLOSED: log.info("target.close() was called") elif self._puppet.reason == FFPuppet.RC_EXITED: log.info("Target closed itself") elif (self._puppet.reason == FFPuppet.RC_WORKER and "memory" in ignored and "ffp_worker_memory_usage" in self._puppet.available_logs()): status = self.RESULT_IGNORED log.info("Memory limit exceeded") elif (self._puppet.reason == FFPuppet.RC_WORKER and "log-limit" in ignored and "ffp_worker_log_size" in self._puppet.available_logs()): status = self.RESULT_IGNORED log.info("Log size limit exceeded") else: log.debug("failure detected, ffpuppet return code: %r", self._puppet.reason) status = self.RESULT_FAILURE elif was_timeout: log.info("Timeout detected") status = self.RESULT_IGNORED if "timeout" in ignored else self.RESULT_FAILURE return status def dump_coverage(self): # If at this point, the browser is running, i.e. we did neither # relaunch nor crash/timeout, then we need to signal the browser # to dump coverage before attempting a new test that potentially # crashes. # Note: This is not required if we closed or are going to close # the browser (relaunch or done with all iterations) because the # SIGTERM will also trigger coverage to be synced out. pid = self._puppet.get_pid() if pid is None or not self._puppet.is_running(): log.debug("Could not dump coverage because process is not running") return try: for child in psutil.Process(pid).children(recursive=True): log.debug("Sending SIGUSR1 to %d", child.pid) os.kill(child.pid, signal.SIGUSR1) except (psutil.AccessDenied, psutil.NoSuchProcess): pass log.debug("Sending SIGUSR1 to %d", pid) os.kill(pid, signal.SIGUSR1) def launch(self, location, env_mod=None): if not self.prefs: raise TargetError("A prefs.js file is required") self.rl_countdown = self.rl_reset env_mod = dict(env_mod or []) # do not allow network connections to non local endpoints env_mod["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env_mod["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" try: self._puppet.launch(self.binary, launch_timeout=self.launch_timeout, location=location, log_limit=self.log_limit, memory_limit=self.memory_limit, prefs_js=self.prefs, extension=self.extension, env_mod=env_mod) except LaunchError as exc: log.error("FFPuppet Error: %s", str(exc)) self.close() if isinstance(exc, BrowserTimeoutError): raise TargetLaunchTimeout(str(exc)) raise TargetLaunchError(str(exc)) def log_size(self): return self._puppet.log_length("stderr") + self._puppet.log_length( "stdout") def save_logs(self, *args, **kwargs): self._puppet.save_logs(*args, **kwargs)