class PuppetTarget(Target):
__slots__ = ("use_rr", "use_valgrind", "_puppet", "_remove_prefs")
def __init__(self, binary, extension, launch_timeout, log_limit,
memory_limit, **kwds):
super().__init__(binary, extension, launch_timeout, log_limit,
memory_limit)
self.use_rr = kwds.pop("rr", False)
self.use_valgrind = kwds.pop("valgrind", False)
self._remove_prefs = False
# create Puppet object
self._puppet = FFPuppet(use_rr=self.use_rr,
use_valgrind=self.use_valgrind,
use_xvfb=kwds.pop("xvfb", False))
if kwds:
LOG.warning("PuppetTarget ignoring unsupported arguments: %s",
", ".join(kwds))
def _abort_hung_proc(self):
# send SIGABRT to the busiest process
with self._lock:
proc_usage = self._puppet.cpu_usage()
for pid, cpu in sorted(proc_usage, reverse=True, key=lambda x: x[1]):
LOG.debug("sending SIGABRT to pid: %r, cpu: %0.2f%%", pid, cpu)
try:
kill(pid, SIGABRT)
except OSError:
LOG.warning("Failed to send SIGABRT to pid %d", pid)
break
def add_abort_token(self, token):
self._puppet.add_abort_token(token)
def cleanup(self):
# prevent parallel calls to FFPuppet.close() and/or FFPuppet.clean_up()
with self._lock:
self._puppet.clean_up()
if self._remove_prefs and self._prefs and isfile(self._prefs):
unlink(self._prefs)
def close(self):
# prevent parallel calls to FFPuppet.close() and/or FFPuppet.clean_up()
with self._lock:
self._puppet.close()
@property
def closed(self):
return self._puppet.reason is not None
def is_idle(self, threshold):
for _, cpu in self._puppet.cpu_usage():
if cpu >= threshold:
return False
return True
def create_report(self):
logs = mkdtemp(prefix="logs_", dir=grz_tmp("logs"))
self.save_logs(logs)
return Report(logs, self.binary)
@property
def monitor(self):
if self._monitor is None:
class _PuppetMonitor(TargetMonitor):
# pylint: disable=no-self-argument,protected-access
def clone_log(_, log_id, offset=0):
return self._puppet.clone_log(log_id, offset=offset)
def is_running(_):
return self._puppet.is_running()
def is_healthy(_):
return self._puppet.is_healthy()
@property
def launches(_):
return self._puppet.launches
def log_length(_, log_id):
return self._puppet.log_length(log_id)
self._monitor = _PuppetMonitor()
return self._monitor
def detect_failure(self, ignored, was_timeout):
status = self.RESULT_NONE
is_healthy = self._puppet.is_healthy()
# check if there has been a crash, hang, etc...
if not is_healthy or was_timeout:
if self._puppet.is_running():
LOG.debug("terminating browser...")
if was_timeout and "timeout" not in ignored and system(
) == "Linux":
self._abort_hung_proc()
# give the process a moment to start dump
self._puppet.wait(timeout=1)
self.close()
# if something has happened figure out what
if not is_healthy:
if self._puppet.reason == FFPuppet.RC_CLOSED:
LOG.debug("target.close() was called")
elif self._puppet.reason == FFPuppet.RC_EXITED:
LOG.debug("target closed itself")
elif (self._puppet.reason == FFPuppet.RC_WORKER
and "memory" in ignored and "ffp_worker_memory_usage"
in self._puppet.available_logs()):
status = self.RESULT_IGNORED
LOG.debug("memory limit exceeded")
elif (self._puppet.reason == FFPuppet.RC_WORKER
and "log-limit" in ignored
and "ffp_worker_log_size" in self._puppet.available_logs()):
status = self.RESULT_IGNORED
LOG.debug("log size limit exceeded")
else:
LOG.debug("failure detected, ffpuppet reason %r",
self._puppet.reason)
status = self.RESULT_FAILURE
elif was_timeout:
LOG.debug("timeout detected")
status = self.RESULT_IGNORED if "timeout" in ignored else self.RESULT_FAILURE
return status
def dump_coverage(self, timeout=15):
assert SIGUSR1 is not None
pid = self._puppet.get_pid()
if pid is None or not self._puppet.is_healthy():
LOG.debug("Skipping coverage dump (target is not in a good state)")
return
# If at this point, the browser is in a good state, i.e. no crashes
# or hangs, so signal the browser to dump coverage.
try:
for child in Process(pid).children(recursive=True):
LOG.debug("Sending SIGUSR1 to %d (child)", child.pid)
try:
kill(child.pid, SIGUSR1)
except OSError:
LOG.warning("Failed to send SIGUSR1 to pid %d", child.pid)
except (AccessDenied, NoSuchProcess): # pragma: no cover
pass
LOG.debug("Sending SIGUSR1 to %d (parent)", pid)
try:
kill(pid, SIGUSR1)
except OSError:
LOG.warning("Failed to send SIGUSR1 to pid %d", pid)
start_time = time()
gcda_found = False
delay = 0.1
# wait for processes to write .gcno files
# this should typically take less than 1 second
while True:
for proc in process_iter(attrs=["pid", "ppid", "open_files"]):
# check if proc is the target or child process
if pid in (proc.info["pid"], proc.info["ppid"]):
if proc.info["open_files"] is None:
continue
if any(
x.path.endswith(".gcda")
for x in proc.info["open_files"]):
gcda_found = True
# get the pid of the process that has the file open
gcda_open = proc.info["pid"]
break
else:
gcda_open = None
elapsed = time() - start_time
if gcda_found:
if gcda_open is None:
# success
LOG.debug("gcda dump took %0.2fs", elapsed)
break
if elapsed >= timeout:
# timeout failure
LOG.warning("gcda file open by pid %d after %0.2fs",
gcda_open, elapsed)
try:
kill(gcda_open, SIGABRT)
except OSError:
pass
sleep(1)
self.close()
break
if delay < 1.0:
# increase delay to a maximum of 1 second
delay = min(1.0, delay + 0.1)
elif elapsed >= 3:
# assume we missed the process writing .gcno files
LOG.warning("No gcda files seen after %0.2fs", elapsed)
break
if not self._puppet.is_healthy():
LOG.warning("Browser failure during dump_coverage()")
break
sleep(delay)
def launch(self, location, env_mod=None):
# setup environment
env_mod = dict(env_mod or [])
# do not allow network connections to non local endpoints
env_mod["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env_mod["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
try:
self._puppet.launch(self.binary,
launch_timeout=self.launch_timeout,
location=location,
log_limit=self.log_limit,
memory_limit=self.memory_limit,
prefs_js=self.prefs,
extension=self.extension,
env_mod=env_mod)
except LaunchError as exc:
LOG.error("FFPuppet LaunchError: %s", str(exc))
self.close()
if isinstance(exc, BrowserTimeoutError):
raise TargetLaunchTimeout(str(exc)) from None
raise TargetLaunchError(str(exc), self.create_report()) from None
def log_size(self):
return self._puppet.log_length("stderr") + self._puppet.log_length(
"stdout")
@property
def prefs(self):
if self._prefs is None:
# generate temporary prefs.js
for prefs_template in PrefPicker.templates():
if prefs_template.endswith("browser-fuzzing.yml"):
LOG.debug("using prefpicker template %r", prefs_template)
tmp_fd, self._prefs = mkstemp(prefix="prefs_",
suffix=".js",
dir=grz_tmp())
close(tmp_fd)
PrefPicker.load_template(prefs_template).create_prefsjs(
self._prefs)
LOG.debug("generated prefs.js %r", self._prefs)
self._remove_prefs = True
break
else: # pragma: no cover
raise TargetError("Failed to generate prefs.js")
return self._prefs
@prefs.setter
def prefs(self, prefs_file):
if self._remove_prefs and self._prefs and isfile(self._prefs):
unlink(self._prefs)
if prefs_file is None:
self._prefs = None
self._remove_prefs = True
elif isfile(prefs_file):
self._prefs = abspath(prefs_file)
self._remove_prefs = False
else:
raise TargetError("Missing prefs.js file %r" % (prefs_file, ))
def save_logs(self, *args, **kwargs):
self._puppet.save_logs(*args, **kwargs)
class PuppetTarget(Target):
SUPPORTED_ASSETS = (
# file containing line separated list of tokens to scan stderr/out for
"abort-tokens",
# xpi or directory containing the unpacked extension
"extension",
# LSan suppression list file
"lsan-suppressions",
# prefs.js file to use
"prefs",
# TSan suppression list file
"tsan-suppressions",
# UBSan suppression list file
"ubsan-suppressions",
)
TRACKED_ENVVARS = (
"ASAN_OPTIONS",
"LSAN_OPTIONS",
"TSAN_OPTIONS",
"UBSAN_OPTIONS",
"GNOME_ACCESSIBILITY",
"MOZ_CHAOSMODE",
"XPCOM_DEBUG_BREAK",
)
__slots__ = ("use_valgrind", "_extension", "_prefs", "_puppet")
def __init__(self, binary, launch_timeout, log_limit, memory_limit, **kwds):
super().__init__(
binary,
launch_timeout,
log_limit,
memory_limit,
assets=kwds.pop("assets", None),
)
# TODO: clean up handling debuggers
debugger = Debugger.NONE
if kwds.pop("pernosco", False):
debugger = Debugger.PERNOSCO
if kwds.pop("rr", False):
debugger = Debugger.RR
if kwds.pop("valgrind", False):
self.use_valgrind = True
debugger = Debugger.VALGRIND
self._extension = None
self._prefs = None
# create Puppet object
self._puppet = FFPuppet(
debugger=debugger,
use_xvfb=kwds.pop("xvfb", False),
working_path=grz_tmp("target"),
)
if kwds:
LOG.warning(
"PuppetTarget ignoring unsupported arguments: %s", ", ".join(kwds)
)
def _cleanup(self):
# prevent parallel calls to FFPuppet.close() and/or FFPuppet.clean_up()
with self._lock:
self._puppet.clean_up()
def close(self, force_close=False):
# prevent parallel calls to FFPuppet.close() and/or FFPuppet.clean_up()
with self._lock:
self._puppet.close(force_close=force_close)
@property
def closed(self):
return self._puppet.reason is not None
def create_report(self, is_hang=False):
logs = mkdtemp(prefix="logs_", dir=grz_tmp("logs"))
self.save_logs(logs)
return Report(logs, self.binary, is_hang=is_hang)
def filtered_environ(self):
# remove context specific entries from environment
filtered = dict(self.environ)
opts = SanitizerOptions()
# iterate over *SAN_OPTIONS entries
for san in (x for x in filtered if x.endswith("SAN_OPTIONS")):
opts.load_options(filtered[san])
# remove entries specific to the current environment
opts.pop("external_symbolizer_path")
opts.pop("log_path")
opts.pop("strip_path_prefix")
opts.pop("suppressions")
filtered[san] = opts.options
# remove empty entries
return {k: v for k, v in filtered.items() if v}
def is_idle(self, threshold):
for _, cpu in self._puppet.cpu_usage():
if cpu >= threshold:
return False
return True
@property
def monitor(self):
if self._monitor is None:
class _PuppetMonitor(TargetMonitor):
# pylint: disable=no-self-argument,protected-access
def clone_log(_, log_id, offset=0):
return self._puppet.clone_log(log_id, offset=offset)
def is_running(_):
return self._puppet.is_running()
def is_healthy(_):
return self._puppet.is_healthy()
@property
def launches(_):
return self._puppet.launches
def log_length(_, log_id):
return self._puppet.log_length(log_id)
self._monitor = _PuppetMonitor()
return self._monitor
def check_result(self, ignored):
result = Result.NONE
# check if there has been a crash, hangs will appear as SIGABRT
if not self._puppet.is_healthy():
self.close()
# something has happened figure out what
if self._puppet.reason == Reason.CLOSED:
LOG.debug("target.close() was called")
elif self._puppet.reason == Reason.EXITED:
LOG.debug("target closed itself")
elif (
self._puppet.reason == Reason.WORKER
and "memory" in ignored
and "ffp_worker_memory_usage" in self._puppet.available_logs()
):
result = Result.IGNORED
LOG.debug("memory limit exceeded")
elif (
self._puppet.reason == Reason.WORKER
and "log-limit" in ignored
and "ffp_worker_log_size" in self._puppet.available_logs()
):
result = Result.IGNORED
LOG.debug("log size limit exceeded")
else:
# crash or hang (forced SIGABRT) has been detected
LOG.debug("result detected (%s)", self._puppet.reason.name)
result = Result.FOUND
return result
def handle_hang(self, ignore_idle=True):
was_idle = False
if self._puppet.is_healthy():
proc_usage = sorted(self._puppet.cpu_usage(), key=lambda x: x[1])
if proc_usage:
pid, cpu = proc_usage.pop()
if ignore_idle and cpu < 15:
# don't send SIGABRT if process is idle
LOG.debug("ignoring idle hang (%0.1f%%)", cpu)
was_idle = True
elif system() == "Linux":
# sending SIGABRT is only supported on Linux for now
# TODO: add/test on other OSs
LOG.debug("sending SIGABRT to %r (%0.1f%%)", pid, cpu)
try:
kill(pid, SIGABRT)
except OSError:
LOG.warning("Failed to send SIGABRT to pid %d", pid)
self._puppet.wait(timeout=10)
# always call close() since this function should only/always
# be called when there has been a timeout
self.close()
return was_idle
def dump_coverage(self, timeout=15):
assert SIGUSR1 is not None
pid = self._puppet.get_pid()
if pid is None or not self._puppet.is_healthy():
LOG.debug("Skipping coverage dump (target is not in a good state)")
return
# If at this point, the browser is in a good state, i.e. no crashes
# or hangs, so signal the browser to dump coverage.
try:
for child in Process(pid).children(recursive=True):
LOG.debug("Sending SIGUSR1 to %d (child)", child.pid)
try:
kill(child.pid, SIGUSR1)
except OSError:
LOG.warning("Failed to send SIGUSR1 to pid %d", child.pid)
except (AccessDenied, NoSuchProcess): # pragma: no cover
pass
LOG.debug("Sending SIGUSR1 to %d (parent)", pid)
try:
kill(pid, SIGUSR1)
except OSError:
LOG.warning("Failed to send SIGUSR1 to pid %d", pid)
start_time = time()
gcda_found = False
delay = 0.1
# wait for processes to write .gcno files
# this should typically take less than 1 second
while True:
for proc in process_iter(attrs=["pid", "ppid", "open_files"]):
# check if proc is the target or child process
if pid in (proc.info["pid"], proc.info["ppid"]):
if proc.info["open_files"] is None:
continue
if any(x.path.endswith(".gcda") for x in proc.info["open_files"]):
gcda_found = True
# get the pid of the process that has the file open
gcda_open = proc.info["pid"]
break
else:
gcda_open = None
elapsed = time() - start_time
if gcda_found:
if gcda_open is None:
# success
LOG.debug("gcda dump took %0.2fs", elapsed)
break
if elapsed >= timeout:
# timeout failure
LOG.warning(
"gcda file open by pid %d after %0.2fs", gcda_open, elapsed
)
try:
kill(gcda_open, SIGABRT)
except OSError:
pass
sleep(1)
self.close()
break
if delay < 1.0:
# increase delay to a maximum of 1 second
delay = min(1.0, delay + 0.1)
elif elapsed >= 3:
# assume we missed the process writing .gcno files
LOG.warning("No gcda files seen after %0.2fs", elapsed)
break
if not self._puppet.is_healthy():
LOG.warning("Browser failure during dump_coverage()")
break
sleep(delay)
def launch(self, location):
# setup environment
env_mod = dict(self.environ)
# do not allow network connections to non local endpoints
env_mod["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env_mod["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
try:
self._puppet.launch(
self.binary,
launch_timeout=self.launch_timeout,
location=location,
log_limit=self.log_limit,
memory_limit=self.memory_limit,
prefs_js=self._prefs,
extension=self._extension,
env_mod=env_mod,
)
except LaunchError as exc:
LOG.error("FFPuppet LaunchError: %s", str(exc))
self.close()
if isinstance(exc, BrowserTimeoutError):
raise TargetLaunchTimeout(str(exc)) from None
raise TargetLaunchError(str(exc), self.create_report()) from None
def log_size(self):
return self._puppet.log_length("stderr") + self._puppet.log_length("stdout")
def process_assets(self):
self._extension = self.assets.get("extension")
self._prefs = self.assets.get("prefs")
# generate temporary prefs.js with prefpicker
if self._prefs is None:
LOG.debug("using prefpicker to generate prefs.js")
with TemporaryDirectory(dir=grz_tmp("target")) as tmp_path:
prefs = Path(tmp_path) / "prefs.js"
template = PrefPicker.lookup_template("browser-fuzzing.yml")
PrefPicker.load_template(template).create_prefsjs(prefs)
self._prefs = self.assets.add("prefs", str(prefs), copy=False)
abort_tokens = self.assets.get("abort-tokens")
if abort_tokens:
LOG.debug("loading 'abort tokens' from %r", abort_tokens)
with open(abort_tokens, "r") as in_fp:
for line in in_fp:
line = line.strip()
if line:
self._puppet.add_abort_token(line)
# configure sanitizer suppressions
opts = SanitizerOptions()
for sanitizer in ("lsan", "tsan", "ubsan"):
asset = "%s-suppressions" % (sanitizer,)
# load existing sanitizer options from environment
var_name = "%s_OPTIONS" % (sanitizer.upper(),)
opts.load_options(self.environ.get(var_name, ""))
if self.assets.get(asset):
# use suppression file if provided as asset
opts.add("suppressions", repr(self.assets.get(asset)), overwrite=True)
elif opts.get("suppressions"):
supp_file = opts.pop("suppressions")
if SanitizerOptions.is_quoted(supp_file):
supp_file = supp_file[1:-1]
if isfile(supp_file):
# use environment specified suppression file
LOG.debug("using %r from environment", asset)
opts.add(
"suppressions",
repr(self.assets.add(asset, supp_file)),
overwrite=True,
)
else:
LOG.warning("Missing %s suppressions file %r", sanitizer, supp_file)
else:
# nothing to do
continue
# update sanitized *SAN_OPTIONS
LOG.debug("updating suppressions in %r", var_name)
self.environ[var_name] = opts.options
def save_logs(self, *args, **kwargs):
self._puppet.save_logs(*args, **kwargs)
class PuppetTarget(Target):
def __init__(self, binary, extension, launch_timeout, log_limit,
memory_limit, prefs, relaunch, **kwds):
super(PuppetTarget,
self).__init__(binary, extension, launch_timeout, log_limit,
memory_limit, prefs, relaunch)
self.use_rr = kwds.pop("rr", False)
self.use_valgrind = kwds.pop("valgrind", False)
use_xvfb = kwds.pop("xvfb", False)
if kwds:
log.warning("PuppetTarget ignoring unsupported arguments: %s",
", ".join(kwds))
# create Puppet object
self._puppet = FFPuppet(use_rr=self.use_rr,
use_valgrind=self.use_valgrind,
use_xvfb=use_xvfb)
def _abort_hung_proc(self):
# send SIGABRT to the busiest process
with self._lock:
proc_usage = self._puppet.cpu_usage()
for pid, cpu in sorted(proc_usage, reverse=True, key=lambda x: x[1]):
log.debug("sending SIGABRT to pid: %r, cpu: %0.2f%%", pid, cpu)
os.kill(pid, signal.SIGABRT)
break
def add_abort_token(self, token):
self._puppet.add_abort_token(token)
def cleanup(self):
# prevent parallel calls to FFPuppet.clean_up()
with self._lock:
self._puppet.clean_up()
def close(self):
# prevent parallel calls to FFPuppet.close()
with self._lock:
self._puppet.close()
@property
def closed(self):
return self._puppet.reason is not None
@property
def monitor(self):
if self._monitor is None:
class _PuppetMonitor(TargetMonitor):
# pylint: disable=no-self-argument,protected-access
def clone_log(_, log_id, offset=0):
return self._puppet.clone_log(log_id, offset=offset)
def is_running(_):
return self._puppet.is_running()
def is_healthy(_):
return self._puppet.is_healthy()
@property
def launches(_):
return self._puppet.launches
def log_length(_, log_id):
return self._puppet.log_length(log_id)
self._monitor = _PuppetMonitor()
return self._monitor
def poll_for_idle(self, threshold, interval):
# return POLL_IDLE if cpu usage of target is below threshold for interval seconds
start_time = time.time()
while time.time() - start_time < interval:
for _, cpu in self._puppet.cpu_usage():
if cpu >= threshold:
return self.POLL_BUSY
if not self._puppet.is_running():
break
else:
log.info("Process utilized <= %d%% CPU for %ds", threshold,
interval)
return self.POLL_IDLE
def detect_failure(self, ignored, was_timeout):
status = self.RESULT_NONE
if self.expect_close and not was_timeout:
# give the browser a moment to close if needed
self._puppet.wait(timeout=30)
is_healthy = self._puppet.is_healthy()
# check if there has been a crash, hang, etc...
if not is_healthy or was_timeout:
if self._puppet.is_running():
log.debug("terminating browser...")
if was_timeout and "timeout" not in ignored:
self._abort_hung_proc()
# give the process a moment to start dump
self._puppet.wait(timeout=1)
self.close()
# if something has happened figure out what
if not is_healthy:
if self._puppet.reason == FFPuppet.RC_CLOSED:
log.info("target.close() was called")
elif self._puppet.reason == FFPuppet.RC_EXITED:
log.info("Target closed itself")
elif (self._puppet.reason == FFPuppet.RC_WORKER
and "memory" in ignored and "ffp_worker_memory_usage"
in self._puppet.available_logs()):
status = self.RESULT_IGNORED
log.info("Memory limit exceeded")
elif (self._puppet.reason == FFPuppet.RC_WORKER
and "log-limit" in ignored
and "ffp_worker_log_size" in self._puppet.available_logs()):
status = self.RESULT_IGNORED
log.info("Log size limit exceeded")
else:
log.debug("failure detected, ffpuppet return code: %r",
self._puppet.reason)
status = self.RESULT_FAILURE
elif was_timeout:
log.info("Timeout detected")
status = self.RESULT_IGNORED if "timeout" in ignored else self.RESULT_FAILURE
return status
def dump_coverage(self):
# If at this point, the browser is running, i.e. we did neither
# relaunch nor crash/timeout, then we need to signal the browser
# to dump coverage before attempting a new test that potentially
# crashes.
# Note: This is not required if we closed or are going to close
# the browser (relaunch or done with all iterations) because the
# SIGTERM will also trigger coverage to be synced out.
pid = self._puppet.get_pid()
if pid is None or not self._puppet.is_running():
log.debug("Could not dump coverage because process is not running")
return
try:
for child in psutil.Process(pid).children(recursive=True):
log.debug("Sending SIGUSR1 to %d", child.pid)
os.kill(child.pid, signal.SIGUSR1)
except (psutil.AccessDenied, psutil.NoSuchProcess):
pass
log.debug("Sending SIGUSR1 to %d", pid)
os.kill(pid, signal.SIGUSR1)
def launch(self, location, env_mod=None):
if not self.prefs:
raise TargetError("A prefs.js file is required")
self.rl_countdown = self.rl_reset
env_mod = dict(env_mod or [])
# do not allow network connections to non local endpoints
env_mod["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env_mod["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
try:
self._puppet.launch(self.binary,
launch_timeout=self.launch_timeout,
location=location,
log_limit=self.log_limit,
memory_limit=self.memory_limit,
prefs_js=self.prefs,
extension=self.extension,
env_mod=env_mod)
except LaunchError as exc:
log.error("FFPuppet Error: %s", str(exc))
self.close()
if isinstance(exc, BrowserTimeoutError):
raise TargetLaunchTimeout(str(exc))
raise TargetLaunchError(str(exc))
def log_size(self):
return self._puppet.log_length("stderr") + self._puppet.log_length(
"stdout")
def save_logs(self, *args, **kwargs):
self._puppet.save_logs(*args, **kwargs)
