def _setup_fuzzer_and_device(self):
""" Build a Fuzzer object based on the QEMU values.
Call this only after setup_qemu_values()"""
fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
if (not fuchsia_pkey_path or not fuchsia_portnum or
not fuchsia_resources_dir):
raise fuchsia.errors.FuchsiaConfigError(
('FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
'not set'))
fuchsia_resources_dir_plus_build = os.path.join(fuchsia_resources_dir,
self.FUCHSIA_BUILD_REL_PATH)
self.host = Host.from_dir(fuchsia_resources_dir_plus_build)
self.device = Device(self.host, 'localhost', fuchsia_portnum)
self.device.set_ssh_option('StrictHostKeyChecking no')
self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
self.device.set_ssh_identity(fuchsia_pkey_path)
# Fuchsia fuzzer names have the format {package_name}/{binary_name}.
package, target = environment.get_value('FUZZ_TARGET').split('/')
test_data_dir = os.path.join(fuchsia_resources_dir_plus_build,
self.FUZZER_TEST_DATA_REL_PATH, package,
target)
self.fuzzer = Fuzzer(
self.device, package, target, output=test_data_dir, foreground=True)
def __init__(self, executable_path, default_args=None):
fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
if (not fuchsia_pkey_path or not fuchsia_portnum
or not fuchsia_resources_dir):
raise fuchsia.errors.FuchsiaConfigError((
'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
'not set'))
fuchsia_resources_dir_plus_build = os.path.join(
fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH)
self.host = Host.from_dir(fuchsia_resources_dir_plus_build)
self.device = Device(self.host, 'localhost', fuchsia_portnum)
self.device.set_ssh_option('StrictHostKeyChecking no')
self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
self.device.set_ssh_identity(fuchsia_pkey_path)
# Fuchsia fuzzer names have the format {package_name}/{binary_name}.
package, target = environment.get_value('FUZZ_TARGET').split('/')
test_data_dir = os.path.join(fuchsia_resources_dir_plus_build,
self.FUZZER_TEST_DATA_REL_PATH, package,
target)
self.fuzzer = Fuzzer(self.device,
package,
target,
output=test_data_dir,
foreground=True)
super(FuchsiaQemuLibFuzzerRunner,
self).__init__(executable_path=executable_path,
default_args=default_args)
def _setup_device_and_fuzzer(self):
"""Build a Device and Fuzzer object based on QEMU's settings."""
# These environment variables are set when start_qemu is run.
# We need them in order to ssh / otherwise communicate with the VM.
fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
if (not fuchsia_pkey_path or not fuchsia_portnum
or not fuchsia_resources_dir):
raise fuchsia.errors.FuchsiaConfigError((
'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
'not set'))
# Fuzzer objects communicate with the VM via a Device object,
# which we set up here.
fuchsia_resources_dir_plus_build = os.path.join(
fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH)
self.host = Host.from_dir(fuchsia_resources_dir_plus_build)
self.device = Device(self.host, 'localhost', fuchsia_portnum)
self.device.set_ssh_option('StrictHostKeyChecking no')
self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
self.device.set_ssh_identity(fuchsia_pkey_path)
# Fuchsia fuzzer names have the format {package_name}/{binary_name}.
package, target = self.executable_path.split('/')
test_data_dir = os.path.join(fuchsia_resources_dir_plus_build,
self.FUZZER_TEST_DATA_REL_PATH, package,
target)
# Finally, we set up the Fuzzer object itself, which will run our fuzzer!
self.fuzzer = Fuzzer(self.device,
package,
target,
output=test_data_dir,
foreground=True)
def __init__(self, executable_path, default_args=None):
fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
if (not fuchsia_pkey_path or not fuchsia_portnum
or not fuchsia_resources_dir):
raise fuchsia.errors.FuchsiaConfigError((
'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
'not set'))
self.host = Host.from_dir(
os.path.join(fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH))
self.device = Device(self.host, 'localhost', fuchsia_portnum)
# Fuchsia fuzzer names have the format {package_name}/{binary_name}.
# TODO(ochang): Properly handle fuzzers with '/' in the binary name.
package, target = environment.get_value('FUZZ_TARGET').split('/')
self.fuzzer = Fuzzer(self.device, package, target)
self.device.set_ssh_option('StrictHostKeyChecking no')
self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
self.device.set_ssh_identity(fuchsia_pkey_path)
super(FuchsiaQemuLibFuzzerRunner,
self).__init__(executable_path=executable_path,
default_args=default_args)
class FuchsiaQemuLibFuzzerRunner(new_process.ProcessRunner, LibFuzzerCommon):
"""libFuzzer runner (when Fuchsia is the target platform)."""
FUCHSIA_BUILD_REL_PATH = os.path.join('build', 'out', 'default')
SSH_RETRIES = 3
SSH_WAIT = 2
FUZZER_TEST_DATA_REL_PATH = os.path.join('test_data', 'fuzzing')
def __init__(self, executable_path, default_args=None):
fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
if (not fuchsia_pkey_path or not fuchsia_portnum
or not fuchsia_resources_dir):
raise fuchsia.errors.FuchsiaConfigError((
'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
'not set'))
fuchsia_resources_dir_plus_build = os.path.join(
fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH)
self.host = Host.from_dir(fuchsia_resources_dir_plus_build)
self.device = Device(self.host, 'localhost', fuchsia_portnum)
self.device.set_ssh_option('StrictHostKeyChecking no')
self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
self.device.set_ssh_identity(fuchsia_pkey_path)
# Fuchsia fuzzer names have the format {package_name}/{binary_name}.
package, target = environment.get_value('FUZZ_TARGET').split('/')
test_data_dir = os.path.join(fuchsia_resources_dir_plus_build,
self.FUZZER_TEST_DATA_REL_PATH, package,
target)
self.fuzzer = Fuzzer(self.device,
package,
target,
output=test_data_dir,
foreground=True)
super(FuchsiaQemuLibFuzzerRunner,
self).__init__(executable_path=executable_path,
default_args=default_args)
def get_command(self, additional_args=None):
# TODO(flowerhack): Update this to dynamically pick a result from "fuzz
# list" and then run that fuzzer.
return self.ssh_command('ls')
def fetch_and_process_logs_and_crash(self):
"""Fetch symbolized logs and crashes."""
# Fetch the symbolized log.
for logname in os.listdir(self.fuzzer.results_output()):
if logname == os.path.basename(self.fuzzer.logfile):
self.device.dlog(self.fuzzer.logfile)
# Clusterfuzz assumes that the Libfuzzer output points to an absolute path,
# where it can find the crash file.
# This doesn't work in our case due to how Fuchsia is run.
# So, we make a new file, change the appropriate line with a regex to point
# to the true location. Apologies for the hackery.
crash_location_regex = r'(.*)(Test unit written to )(data/.*)'
_, new_file_handle_path = tempfile.mkstemp()
with open(new_file_handle_path, 'w') as new_file:
with open(self.fuzzer.logfile) as old_file:
for line in old_file:
line_match = re.match(crash_location_regex, line)
if line_match:
# We now know the name of our crash file.
crash_name = line_match.group(3).replace('data/', '')
# Save the crash locally.
self.device.fetch(self.fuzzer.data_path(crash_name),
self.fuzzer.results_output())
# Then update the crash report to point to that file.
crash_testcase_file_path = os.path.join(
self.fuzzer.results_output(), crash_name)
line = re.sub(crash_location_regex,
r'\1\2' + crash_testcase_file_path, line)
new_file.write(line)
os.remove(self.fuzzer.logfile)
os.rename(new_file_handle_path, self.fuzzer.logfile)
def fuzz(self,
corpus_directories,
fuzz_timeout,
artifact_prefix=None,
additional_args=None,
extra_env=None):
"""LibFuzzerCommon.fuzz override."""
self._test_qemu_ssh()
self.fuzzer.start([])
self.fetch_and_process_logs_and_crash()
with open(self.fuzzer.logfile) as logfile:
symbolized_output = logfile.read()
# TODO(flowerhack): Would be nice if we could figure out a way to make
# the "fuzzer start" code return its own ProcessResult. For now, we simply
# craft one by hand here.
fuzzer_process_result = new_process.ProcessResult()
fuzzer_process_result.return_code = 0
fuzzer_process_result.output = symbolized_output
fuzzer_process_result.time_executed = 0
fuzzer_process_result.command = self.fuzzer.last_fuzz_cmd
return fuzzer_process_result
def run_single_testcase(self,
testcase_path,
timeout=None,
additional_args=None):
# TODO(flowerhack): Fill out this command.
pass
def ssh_command(self, *args):
return ['ssh'] + self.ssh_root + list(args)
@retry.wrap(retries=SSH_RETRIES, delay=SSH_WAIT, function='_test_qemu_ssh')
def _test_qemu_ssh(self):
"""Tests that a VM is up and can be successfully SSH'd into.
Raises an exception if no success after MAX_SSH_RETRIES."""
ssh_test_process = new_process.ProcessRunner(
'ssh',
self.device.get_ssh_cmd(
['ssh', 'localhost', 'echo running on fuchsia!'])[1:])
result = ssh_test_process.run_and_wait()
if result.return_code or result.timed_out:
raise fuchsia.errors.FuchsiaConnectionError(
'Failed to establish initial SSH connection: ' +
str(result.return_code) + " , " + str(result.command) + " , " +
str(result.output))
return result
def create(self):
"""Configures a QEMU process which can subsequently be `run`.
Assumes that initial_qemu_setup was already called exactly once.
"""
qemu_vars = _fetch_qemu_vars()
# Get a free port for the VM, so we can SSH in later.
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.bind(('localhost', 0))
_, port = tcp.getsockname()
tcp.close()
# Fuzzing jobs that SSH into the QEMU VM need access to this env var.
environment.set_value('FUCHSIA_PORTNUM', port)
environment.set_value('FUCHSIA_RESOURCES_DIR',
qemu_vars['fuchsia_resources_dir'])
# yapf: disable
qemu_args = [
'-m', '3072',
'-nographic',
'-kernel', qemu_vars['kernel_path'],
'-initrd', qemu_vars['initrd_path'],
'-smp', '4',
'-drive',
('file=' + qemu_vars['drive_path'] + ',format=raw,if=none,'
'id=blobstore'),
'-device', 'virtio-blk-pci,drive=blobstore',
'-monitor', 'none',
'-append', 'kernel.serial=legacy TERM=dumb',
'-machine', 'q35',
'-display', 'none',
'-netdev',
('user,id=net0,net=192.168.3.0/24,dhcpstart=192.168.3.9,'
'host=192.168.3.2,hostfwd=tcp::') + str(port) + '-:22',
'-device', 'e1000,netdev=net0,mac=52:54:00:63:5e:7b',
'-L', qemu_vars['sharefiles_path']
]
# yapf: enable
# Detecting KVM is tricky, so use an environment variable to determine
# whether to turn it on or not.
if environment.get_value('FUCHSIA_USE_KVM'):
# In builds before fxrev.dev/375343, a bug prevents booting with newer
# versions of KVM. On some of these older builds,
# `kernel.x86.disable-spec-mitigations` also doesn't work as
# expected, so we work around this by selecting a CPU type where the
# speculation mitigation will not applied.
if environment.get_value('APP_REVISION') < 20200414210423:
qemu_args.extend(['-cpu', 'Opteron_G5,+invtsc'])
else:
qemu_args.extend(['-cpu', 'host,migratable=no,+invtsc'])
qemu_args.append('-enable-kvm')
else:
# Can't use host CPU since we don't necessarily have KVM on the machine.
# Emulate a Haswell CPU with a few feature toggles. This mirrors the most
# common configuration for Fuchsia VMs when using in-tree tools.
qemu_args.extend(['-cpu', 'Haswell,+smap,-check,-fsgsbase'])
# Get the list of fuzzers for ClusterFuzz to choose from.
host = Host.from_dir(
os.path.join(qemu_vars['fuchsia_resources_dir'], 'build', 'out',
'default'))
Device(host, 'localhost', str(port))
Fuzzer.filter(host.fuzzers, '')
# Fuzzing jobs that SSH into the QEMU VM need access to this env var.
environment.set_value('FUCHSIA_PKEY_PATH', qemu_vars['pkey_path'])
logs.log('Ready to run QEMU. Command: ' + qemu_vars['qemu_path'] +
' ' + ' '.join(shlex.quote(arg) for arg in qemu_args))
self.process_runner = new_process.ProcessRunner(
qemu_vars['qemu_path'], qemu_args)
def setup_qemu_values(initial_setup=True):
"""Sets up and runs a QEMU VM in the background.
Returns a process.Popen object.
Does not block the calling process, and teardown must be handled by the
caller (use .kill()).
Fuchsia fuzzers assume a QEMU VM is running; call this routine prior to
beginning Fuchsia fuzzing tasks.
This initialization routine assumes the following layout for
fuchsia_resources_dir:
* /qemu-for-fuchsia/*
* /.ssh/*
* target/x64/fvm.blk
* target/x64/fuchsia.zbi
* target/x64/multiboot.bin
* build/out/default/fuzzers.json
* build/out/default/ids.txt
* build/out/default.zircon/tools/*
* build/zircon/prebuilt/downloads/symbolize
* build/buildtools/linux-x64/clang/bin/llvm-symbolizer"""
# First download the Fuchsia resources locally.
fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
if not fuchsia_resources_dir:
raise errors.FuchsiaConfigError('Could not find FUCHSIA_RESOURCES_DIR')
# Then, save paths for necessary commands later.
qemu_path = os.path.join(fuchsia_resources_dir, 'qemu-for-fuchsia', 'bin',
'qemu-system-x86_64')
os.chmod(qemu_path, 0o550)
kernel_path = os.path.join(fuchsia_resources_dir, 'target', 'x64',
'multiboot.bin')
os.chmod(kernel_path, 0o644)
pkey_path = os.path.join(fuchsia_resources_dir, '.ssh', 'pkey')
os.chmod(pkey_path, 0o400)
sharefiles_path = os.path.join(fuchsia_resources_dir, 'qemu-for-fuchsia',
'share', 'qemu')
drive_path = os.path.join(fuchsia_resources_dir, 'target', 'x64', 'fvm.blk')
os.chmod(drive_path, 0o644)
fuchsia_zbi = os.path.join(fuchsia_resources_dir, 'target', 'x64',
'fuchsia.zbi')
initrd_path = os.path.join(fuchsia_resources_dir, 'fuchsia-ssh.zbi')
# Perform some more initiailization steps.
# Only do these the first time you run QEMU after downloading the build.
if initial_setup:
extend_fvm(fuchsia_resources_dir, drive_path)
add_keys_to_zbi(fuchsia_resources_dir, initrd_path, fuchsia_zbi)
# Get a free port for the VM, so we can SSH in later.
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.bind(('localhost', 0))
_, port = tcp.getsockname()
tcp.close()
# Fuzzing jobs that SSH into the QEMU VM need access to this env var.
environment.set_value('FUCHSIA_PORTNUM', port)
environment.set_value('FUCHSIA_RESOURCES_DIR', fuchsia_resources_dir)
# yapf: disable
qemu_args = [
'-m', '2048',
'-nographic',
'-kernel', kernel_path,
'-initrd', initrd_path,
'-smp', '4',
'-drive', 'file=' + drive_path + ',format=raw,if=none,id=blobstore',
'-device', 'virtio-blk-pci,drive=blobstore',
'-monitor', 'none',
'-append', '"kernel.serial=legacy TERM=dumb"',
'-machine', 'q35',
'-display', 'none',
'-netdev',
('user,id=net0,net=192.168.3.0/24,dhcpstart=192.168.3.9,'
'host=192.168.3.2,hostfwd=tcp::') + str(port) + '-:22',
'-device', 'e1000,netdev=net0,mac=52:54:00:63:5e:7b',
'-L', sharefiles_path
]
# yapf: enable
# Detecing KVM is tricky, so use an environment variable to determine whether
# to turn it on or not.
if environment.get_value('FUCHSIA_USE_KVM'):
qemu_args.extend(['-cpu', 'host,migratable=no'])
qemu_args.append('-enable-kvm')
else:
# Can't use host CPU since we don't necessarily have KVM on the machine.
# Emulate a Haswell CPU with a few feature toggles. This mirrors the most
# common configuration for Fuchsia VMs when using in-tree tools.
qemu_args.extend(['-cpu', 'Haswell,+smap,-check,-fsgsbase'])
# Get the list of fuzzers for ClusterFuzz to choose from.
host = Host.from_dir(
os.path.join(fuchsia_resources_dir, 'build', 'out', 'default'))
Device(host, 'localhost', str(port))
Fuzzer.filter(host.fuzzers, '')
# Fuzzing jobs that SSH into the QEMU VM need access to this env var.
environment.set_value('FUCHSIA_PKEY_PATH', pkey_path)
return qemu_path, qemu_args
class FuchsiaQemuLibFuzzerRunner(new_process.ProcessRunner, LibFuzzerCommon):
"""libFuzzer runner (when Fuchsia is the target platform)."""
FUCHSIA_BUILD_REL_PATH = os.path.join('build', 'out', 'default')
SSH_RETRIES = 3
SSH_WAIT = 2
def __init__(self, executable_path, default_args=None):
fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
if (not fuchsia_pkey_path or not fuchsia_portnum
or not fuchsia_resources_dir):
raise fuchsia.errors.FuchsiaConfigError((
'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
'not set'))
self.host = Host.from_dir(
os.path.join(fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH))
self.device = Device(self.host, 'localhost', fuchsia_portnum)
# Fuchsia fuzzer names have the format {package_name}/{binary_name}.
# TODO(ochang): Properly handle fuzzers with '/' in the binary name.
package, target = environment.get_value('FUZZ_TARGET').split('/')
self.fuzzer = Fuzzer(self.device, package, target)
self.device.set_ssh_option('StrictHostKeyChecking no')
self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
self.device.set_ssh_identity(fuchsia_pkey_path)
super(FuchsiaQemuLibFuzzerRunner,
self).__init__(executable_path=executable_path,
default_args=default_args)
def get_command(self, additional_args=None):
# TODO(flowerhack): Update this to dynamically pick a result from "fuzz
# list" and then run that fuzzer.
return self.ssh_command('ls')
def fuzz(self,
corpus_directories,
fuzz_timeout,
artifact_prefix=None,
additional_args=None,
extra_env=None):
"""LibFuzzerCommon.fuzz override."""
self._test_qemu_ssh()
self.fuzzer.run([])
# TODO(flowerhack): Modify fuzzer.run() to return a ProcessResult, rather
# than artisinally handcrafting one here.
fuzzer_process_result = new_process.ProcessResult()
fuzzer_process_result.return_code = 0
fuzzer_process_result.output = ''
fuzzer_process_result.time_executed = 0
fuzzer_process_result.command = self.fuzzer.last_fuzz_cmd
return fuzzer_process_result
def run_single_testcase(self,
testcase_path,
timeout=None,
additional_args=None):
# TODO(flowerhack): Fill out this command.
pass
def ssh_command(self, *args):
return ['ssh'] + self.ssh_root + list(args)
@retry.wrap(retries=SSH_RETRIES, delay=SSH_WAIT, function='_test_qemu_ssh')
def _test_qemu_ssh(self):
"""Tests that a VM is up and can be successfully SSH'd into.
Raises an exception if no success after MAX_SSH_RETRIES."""
ssh_test_process = new_process.ProcessRunner(
'ssh',
self.device.get_ssh_cmd(
['ssh', 'localhost', 'echo running on fuchsia!'])[1:])
result = ssh_test_process.run_and_wait()
if result.return_code or result.timed_out:
raise fuchsia.errors.FuchsiaConnectionError(
'Failed to establish initial SSH connection: ' +
str(result.return_code) + " , " + str(result.command) + " , " +
str(result.output))
return result
def create(self):
"""Configures a QEMU process which can subsequently be `run`.
Assumes that initial_qemu_setup was already called exactly once.
"""
qemu_vars = _fetch_qemu_vars()
# Get a free port for the VM, so we can SSH in later.
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.bind(("localhost", 0))
_, port = tcp.getsockname()
tcp.close()
# Fuzzing jobs that SSH into the QEMU VM need access to this env var.
environment.set_value("FUCHSIA_PORTNUM", port)
environment.set_value("FUCHSIA_RESOURCES_DIR",
qemu_vars["fuchsia_resources_dir"])
# yapf: disable
qemu_args = [
'-m', '3072',
'-nographic',
'-kernel', qemu_vars['kernel_path'],
'-initrd', qemu_vars['initrd_path'],
'-smp', '4',
'-drive',
('file=' + qemu_vars['drive_path'] + ',format=raw,if=none,'
'id=blobstore'),
'-device', 'virtio-blk-pci,drive=blobstore',
'-monitor', 'none',
'-append', 'kernel.serial=legacy TERM=dumb',
'-machine', 'q35',
'-display', 'none',
'-netdev',
('user,id=net0,net=192.168.3.0/24,dhcpstart=192.168.3.9,'
'host=192.168.3.2,hostfwd=tcp::') + str(port) + '-:22',
'-device', 'e1000,netdev=net0,mac=52:54:00:63:5e:7b',
'-L', qemu_vars['sharefiles_path']
]
# yapf: enable
# Detecing KVM is tricky, so use an environment variable to determine
# whether to turn it on or not.
if environment.get_value("FUCHSIA_USE_KVM"):
qemu_args.extend(["-cpu", "host,migratable=no"])
qemu_args.append("-enable-kvm")
else:
# Can't use host CPU since we don't necessarily have KVM on the machine.
# Emulate a Haswell CPU with a few feature toggles. This mirrors the most
# common configuration for Fuchsia VMs when using in-tree tools.
qemu_args.extend(["-cpu", "Haswell,+smap,-check,-fsgsbase"])
# Get the list of fuzzers for ClusterFuzz to choose from.
host = Host.from_dir(
os.path.join(qemu_vars["fuchsia_resources_dir"], "build", "out",
"default"))
Device(host, "localhost", str(port))
Fuzzer.filter(host.fuzzers, "")
# Fuzzing jobs that SSH into the QEMU VM need access to this env var.
environment.set_value("FUCHSIA_PKEY_PATH", qemu_vars["pkey_path"])
logs.log("Ready to run QEMU. Command: " + qemu_vars["qemu_path"] +
" " + str(qemu_args))
self.process_runner = new_process.ProcessRunner(
qemu_vars["qemu_path"], qemu_args)
class FuchsiaQemuLibFuzzerRunner(new_process.ProcessRunner, LibFuzzerCommon):
"""libFuzzer runner (when Fuchsia is the target platform)."""
FUCHSIA_BUILD_REL_PATH = os.path.join('build', 'out', 'default')
SSH_RETRIES = 3
SSH_WAIT = 3
FUZZER_TEST_DATA_REL_PATH = os.path.join('test_data', 'fuzzing')
def _setup_device_and_fuzzer(self):
"""Build a Device and Fuzzer object based on QEMU's settings."""
# These environment variables are set when start_qemu is run.
# We need them in order to ssh / otherwise communicate with the VM.
fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
if (not fuchsia_pkey_path or not fuchsia_portnum
or not fuchsia_resources_dir):
raise fuchsia.errors.FuchsiaConfigError((
'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
'not set'))
# Fuzzer objects communicate with the VM via a Device object,
# which we set up here.
fuchsia_resources_dir_plus_build = os.path.join(
fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH)
self.host = Host.from_dir(fuchsia_resources_dir_plus_build)
self.device = Device(self.host, 'localhost', fuchsia_portnum)
self.device.set_ssh_option('StrictHostKeyChecking no')
self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
self.device.set_ssh_identity(fuchsia_pkey_path)
# Fuchsia fuzzer names have the format {package_name}/{binary_name}.
package, target = self.executable_path.split('/')
test_data_dir = os.path.join(fuchsia_resources_dir_plus_build,
self.FUZZER_TEST_DATA_REL_PATH, package,
target)
# Finally, we set up the Fuzzer object itself, which will run our fuzzer!
sanitizer = environment.get_memory_tool_name(
environment.get_value('JOB_NAME')).lower()
self.fuzzer = Fuzzer(self.device,
package,
target,
output=test_data_dir,
foreground=True,
sanitizer=sanitizer)
def __init__(self, executable_path, default_args=None):
# We always assume QEMU is running on __init__, since build_manager sets
# it up initially. If this isn't the case, _test_ssh will detect and
# restart QEMU anyway.
super(FuchsiaQemuLibFuzzerRunner,
self).__init__(executable_path=executable_path,
default_args=default_args)
self._setup_device_and_fuzzer()
def process_logs_and_crash(self, artifact_prefix):
"""Fetch symbolized logs and crashes."""
if not artifact_prefix:
return
# Clusterfuzz assumes that the Libfuzzer output points to an absolute path,
# where it can find the crash file.
# This doesn't work in our case due to how Fuchsia is run.
# So, we make a new file, change the appropriate line with a regex to point
# to the true location. Apologies for the hackery.
crash_location_regex = r'(.*)(Test unit written to )(data/.*)'
_, processed_log_path = tempfile.mkstemp()
with open(processed_log_path, 'w') as new_file:
with open(self.fuzzer.logfile) as old_file:
for line in old_file:
line_match = re.match(crash_location_regex, line)
if line_match:
# We now know the name of our crash file.
crash_name = line_match.group(3).replace('data/', '')
# Save the crash locally.
self.device.fetch(self.fuzzer.data_path(crash_name),
artifact_prefix)
# Then update the crash report to point to that file.
crash_testcase_file_path = os.path.join(
artifact_prefix, crash_name)
line = re.sub(crash_location_regex,
r'\1\2' + crash_testcase_file_path, line)
new_file.write(line)
os.remove(self.fuzzer.logfile)
shutil.move(processed_log_path, self.fuzzer.logfile)
def _test_ssh(self):
"""Test the ssh connection."""
# Test the connection. If this works, proceed.
# - If we fail, restart QEMU and test the connection again.
# - If that fails, throw the error; we can't seem to recover.
try:
self._test_qemu_ssh()
except fuchsia.errors.FuchsiaConnectionError:
self._restart_qemu()
self._test_qemu_ssh()
@retry.wrap(retries=SSH_RETRIES, delay=SSH_WAIT, function='_test_qemu_ssh')
def _test_qemu_ssh(self):
"""Tests that a VM is up and can be successfully SSH'd into.
Raises an exception if no success after MAX_SSH_RETRIES."""
ssh_test_process = new_process.ProcessRunner(
'ssh',
self.device.get_ssh_cmd(
['ssh', 'localhost', 'echo running on fuchsia!'])[1:])
result = ssh_test_process.run_and_wait()
if result.return_code or result.timed_out:
raise fuchsia.errors.FuchsiaConnectionError(
'Failed to establish initial SSH connection: ' +
str(result.return_code) + " , " + str(result.command) + " , " +
str(result.output))
return result
def _restart_qemu(self):
"""Restart QEMU."""
logs.log_warn('Connection to fuzzing VM lost. Restarting.')
stop_qemu()
start_qemu()
self._setup_device_and_fuzzer()
def _corpus_target_subdir(self, relpath):
""" Returns the absolute path of the corpus subdirectory on the target,
given "relpath", the name of the specific corpus. """
return os.path.join(self._corpus_directories_target(), relpath)
def _corpus_directories_libfuzzer(self, corpus_directories):
""" Returns the corpus directory paths expected by libfuzzer itself. """
corpus_directories_libfuzzer = []
for corpus_dir in corpus_directories:
corpus_directories_libfuzzer.append(
os.path.join('data', 'corpus', os.path.basename(corpus_dir)))
return corpus_directories_libfuzzer
def _new_corpus_dir_host(self, corpus_directories):
""" Returns the path of the 'new' corpus directory on the host. """
return corpus_directories[0]
def _new_corpus_dir_target(self, corpus_directories):
""" Returns the path of the 'new' corpus directory on the target. """
new_corpus_dir_host = self._new_corpus_dir_host(corpus_directories)
return self.fuzzer.data_path(
os.path.join('corpus', os.path.basename(new_corpus_dir_host)))
def _corpus_directories_target(self):
""" Returns the path of the root corpus directory on the target. """
return self.fuzzer.data_path('corpus')
def _push_corpora_from_host_to_target(self, corpus_directories):
# Push corpus directories to the device.
self._clear_all_target_corpora()
logs.log('Push corpora from host to target.')
for corpus_dir in corpus_directories:
# Appending '/*' indicates we want all the *files* in the corpus_dir's
self.fuzzer.device.store(
corpus_dir + '/*',
self._corpus_target_subdir(os.path.basename(corpus_dir)))
def _pull_new_corpus_from_target_to_host(self, corpus_directories):
# Appending '/*' indicates we want all the *files* in the target's
# directory, rather than the directory itself.
logs.log('Fuzzer ran; pull down corpus')
files_in_new_corpus_dir_target = self._new_corpus_dir_target(
corpus_directories) + "/*"
self.fuzzer.device.fetch(files_in_new_corpus_dir_target,
self._new_corpus_dir_host(corpus_directories))
def _clear_all_target_corpora(self):
""" Clears out all the corpora on the target. """
logs.log('Clearing corpora on target')
self.fuzzer.device.ssh(
['rm', '-rf', self._corpus_directories_target()])
def fuzz(self,
corpus_directories,
fuzz_timeout,
artifact_prefix=None,
additional_args=None,
extra_env=None):
"""LibFuzzerCommon.fuzz override."""
additional_args = copy.copy(additional_args)
if additional_args is None:
additional_args = []
self._test_ssh()
self._push_corpora_from_host_to_target(corpus_directories)
max_total_time = self.get_max_total_time(fuzz_timeout)
if any(arg.startswith(constants.FORK_FLAG) for arg in additional_args):
max_total_time -= self.LIBFUZZER_FORK_MODE_CLEAN_EXIT_TIME
assert max_total_time > 0
additional_args.extend([
'%s%d' % (constants.MAX_TOTAL_TIME_FLAG, max_total_time),
constants.PRINT_FINAL_STATS_ARGUMENT,
])
# Run the fuzzer.
# TODO: actually we want new_corpus_relative_dir_target for *each* corpus
return_code = self.fuzzer.start(
self._corpus_directories_libfuzzer(corpus_directories) +
additional_args)
self.fuzzer.monitor(return_code)
self.process_logs_and_crash(artifact_prefix)
with open(self.fuzzer.logfile) as logfile:
symbolized_output = logfile.read()
self._pull_new_corpus_from_target_to_host(corpus_directories)
self._clear_all_target_corpora()
# TODO(flowerhack): Would be nice if we could figure out a way to make
# the "fuzzer start" code return its own ProcessResult. For now, we simply
# craft one by hand here.
fuzzer_process_result = new_process.ProcessResult()
fuzzer_process_result.return_code = 0
fuzzer_process_result.output = symbolized_output
fuzzer_process_result.time_executed = 0
fuzzer_process_result.command = self.fuzzer.last_fuzz_cmd
return fuzzer_process_result
def merge(self,
corpus_directories,
merge_timeout,
artifact_prefix=None,
tmp_dir=None,
additional_args=None):
# TODO(flowerhack): Integrate some notion of a merge timeout.
self._push_corpora_from_host_to_target(corpus_directories)
# Run merge.
_, _ = self.fuzzer.merge(
self._corpus_directories_libfuzzer(corpus_directories) +
additional_args)
self._pull_new_corpus_from_target_to_host(corpus_directories)
self._clear_all_target_corpora()
merge_result = new_process.ProcessResult()
merge_result.return_code = 0
merge_result.timed_out = False
merge_result.output = ''
merge_result.time_executed = 0
merge_result.command = ''
return merge_result
def run_single_testcase(self,
testcase_path,
timeout=None,
additional_args=None):
"""Run a single testcase."""
self._test_ssh()
# We need to push the testcase to the device and pass in the name.
testcase_path_name = os.path.basename(os.path.normpath(testcase_path))
self.device.store(testcase_path, self.fuzzer.data_path())
return_code = self.fuzzer.start(
['repro', 'data/' + testcase_path_name] + additional_args)
self.fuzzer.monitor(return_code)
with open(self.fuzzer.logfile) as logfile:
symbolized_output = logfile.read()
fuzzer_process_result = new_process.ProcessResult()
fuzzer_process_result.return_code = 0
fuzzer_process_result.output = symbolized_output
fuzzer_process_result.time_executed = 0
fuzzer_process_result.command = self.fuzzer.last_fuzz_cmd
return fuzzer_process_result
def minimize_crash(self,
testcase_path,
output_path,
timeout,
artifact_prefix=None,
additional_args=None):
return new_process.ProcessResult()
def ssh_command(self, *args):
return ['ssh'] + self.ssh_root + list(args)
class FuchsiaQemuLibFuzzerRunner(new_process.ProcessRunner, LibFuzzerCommon):
"""libFuzzer runner (when Fuchsia is the target platform)."""
FUCHSIA_BUILD_REL_PATH = os.path.join('build', 'out', 'default')
SSH_RETRIES = 3
SSH_WAIT = 3
FUZZER_TEST_DATA_REL_PATH = os.path.join('test_data', 'fuzzing')
def _setup_device_and_fuzzer(self):
"""Build a Device and Fuzzer object based on QEMU's settings."""
# These environment variables are set when start_qemu is run.
# We need them in order to ssh / otherwise communicate with the VM.
fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
if (not fuchsia_pkey_path or not fuchsia_portnum
or not fuchsia_resources_dir):
raise fuchsia.errors.FuchsiaConfigError((
'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
'not set'))
# Fuzzer objects communicate with the VM via a Device object,
# which we set up here.
fuchsia_resources_dir_plus_build = os.path.join(
fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH)
self.host = Host.from_dir(fuchsia_resources_dir_plus_build)
self.device = Device(self.host, 'localhost', fuchsia_portnum)
self.device.set_ssh_option('StrictHostKeyChecking no')
self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
self.device.set_ssh_identity(fuchsia_pkey_path)
# Fuchsia fuzzer names have the format {package_name}/{binary_name}.
package, target = self.executable_path.split('/')
test_data_dir = os.path.join(fuchsia_resources_dir_plus_build,
self.FUZZER_TEST_DATA_REL_PATH, package,
target)
# Finally, we set up the Fuzzer object itself, which will run our fuzzer!
self.fuzzer = Fuzzer(self.device,
package,
target,
output=test_data_dir,
foreground=True)
def __init__(self, executable_path, default_args=None):
# We always assume QEMU is running on __init__, since build_manager sets
# it up initially. If this isn't the case, _test_ssh will detect and
# restart QEMU anyway.
super(FuchsiaQemuLibFuzzerRunner,
self).__init__(executable_path=executable_path,
default_args=default_args)
self._setup_device_and_fuzzer()
def get_command(self, additional_args=None):
# TODO(flowerhack): Update this to dynamically pick a result from "fuzz
# list" and then run that fuzzer.
return self.ssh_command('ls')
def process_logs_and_crash(self, artifact_prefix):
"""Fetch symbolized logs and crashes."""
if not artifact_prefix:
return
# Clusterfuzz assumes that the Libfuzzer output points to an absolute path,
# where it can find the crash file.
# This doesn't work in our case due to how Fuchsia is run.
# So, we make a new file, change the appropriate line with a regex to point
# to the true location. Apologies for the hackery.
crash_location_regex = r'(.*)(Test unit written to )(data/.*)'
_, processed_log_path = tempfile.mkstemp()
with open(processed_log_path, 'w') as new_file:
with open(self.fuzzer.logfile) as old_file:
for line in old_file:
line_match = re.match(crash_location_regex, line)
if line_match:
# We now know the name of our crash file.
crash_name = line_match.group(3).replace('data/', '')
# Save the crash locally.
self.device.fetch(self.fuzzer.data_path(crash_name),
artifact_prefix)
# Then update the crash report to point to that file.
crash_testcase_file_path = os.path.join(
artifact_prefix, crash_name)
line = re.sub(crash_location_regex,
r'\1\2' + crash_testcase_file_path, line)
new_file.write(line)
os.remove(self.fuzzer.logfile)
shutil.move(processed_log_path, self.fuzzer.logfile)
def _test_ssh(self):
"""Test the ssh connection."""
# Test the connection. If this works, proceed.
# - If we fail, restart QEMU and test the connection again.
# - If that fails, throw the error; we can't seem to recover.
try:
self._test_qemu_ssh()
except fuchsia.errors.FuchsiaConnectionError:
self._restart_qemu()
self._test_qemu_ssh()
@retry.wrap(retries=SSH_RETRIES, delay=SSH_WAIT, function='_test_qemu_ssh')
def _test_qemu_ssh(self):
"""Tests that a VM is up and can be successfully SSH'd into.
Raises an exception if no success after MAX_SSH_RETRIES."""
ssh_test_process = new_process.ProcessRunner(
'ssh',
self.device.get_ssh_cmd(
['ssh', 'localhost', 'echo running on fuchsia!'])[1:])
result = ssh_test_process.run_and_wait()
if result.return_code or result.timed_out:
raise fuchsia.errors.FuchsiaConnectionError(
'Failed to establish initial SSH connection: ' +
str(result.return_code) + " , " + str(result.command) + " , " +
str(result.output))
return result
def _restart_qemu(self):
"""Restart QEMU."""
process_handler.terminate_processes_matching_names(
'qemu_system-x86_64')
start_qemu()
self._setup_device_and_fuzzer()
def fuzz(self,
corpus_directories,
fuzz_timeout,
artifact_prefix=None,
additional_args=None,
extra_env=None):
"""LibFuzzerCommon.fuzz override."""
self._test_ssh()
#TODO(flowerhack): Pass libfuzzer args (additional_args) here
return_code = self.fuzzer.start(additional_args)
self.fuzzer.monitor(return_code)
self.process_logs_and_crash(artifact_prefix)
with open(self.fuzzer.logfile) as logfile:
symbolized_output = logfile.read()
# TODO(flowerhack): Would be nice if we could figure out a way to make
# the "fuzzer start" code return its own ProcessResult. For now, we simply
# craft one by hand here.
fuzzer_process_result = new_process.ProcessResult()
fuzzer_process_result.return_code = 0
fuzzer_process_result.output = symbolized_output
fuzzer_process_result.time_executed = 0
fuzzer_process_result.command = self.fuzzer.last_fuzz_cmd
return fuzzer_process_result
def run_single_testcase(self,
testcase_path,
timeout=None,
additional_args=None):
"""Run a single testcase."""
self._test_ssh()
# We need to push the testcase to the device and pass in the name.
testcase_path_name = os.path.basename(os.path.normpath(testcase_path))
self.device.store(testcase_path, self.fuzzer.data_path())
# TODO(flowerhack): Pass libfuzzer args (additional_args) here
return_code = self.fuzzer.start(
['repro', 'data/' + testcase_path_name] + additional_args)
self.fuzzer.monitor(return_code)
with open(self.fuzzer.logfile) as logfile:
symbolized_output = logfile.read()
fuzzer_process_result = new_process.ProcessResult()
fuzzer_process_result.return_code = 0
fuzzer_process_result.output = symbolized_output
fuzzer_process_result.time_executed = 0
fuzzer_process_result.command = self.fuzzer.last_fuzz_cmd
return fuzzer_process_result
def minimize_crash(self,
testcase_path,
output_path,
timeout,
artifact_prefix=None,
additional_args=None):
return new_process.ProcessResult()
def ssh_command(self, *args):
return ['ssh'] + self.ssh_root + list(args)