def initialize_resources_dir():
"""Download Fuchsia QEMU resources from GCS bucket."""
resources_dir = environment.get_value('RESOURCES_DIR')
if not resources_dir:
raise errors.FuchsiaConfigError('Could not find RESOURCES_DIR')
fuchsia_resources_dir = os.path.join(resources_dir, 'fuchsia')
shell.create_directory(fuchsia_resources_dir, recreate=True)
fuchsia_resources_url = environment.get_value('FUCHSIA_RESOURCES_URL')
if not fuchsia_resources_url:
raise errors.FuchsiaConfigError(
'Could not find path for remote'
'Fuchsia resources bucket (FUCHSIA_RESOURCES_URL')
gsutil_command_arguments = [
'-m', 'cp', '-r', fuchsia_resources_url, fuchsia_resources_dir
]
logs.log("Beginning Fuchsia SDK download.")
result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments)
if result.return_code or result.timed_out:
raise errors.FuchsiaSdkError('Failed to download Fuchsia'
'resources: ' + result.output)
logs.log("Fuchsia SDK download complete.")
return fuchsia_resources_dir
def initialize_resources_dir():
"""Download Fuchsia QEMU resources from GCS bucket."""
# This module depends on multiprocessing, which is not available in
# appengine, and since appengine *imports* this file (but does not run this
# function!), we import it here.
from google_cloud_utils import gsutil
resources_dir = environment.get_value('RESOURCES_DIR')
if not resources_dir:
raise errors.FuchsiaConfigError('Could not find RESOURCES_DIR')
fuchsia_resources_dir = os.path.join(resources_dir, 'fuchsia')
shell.create_directory(
fuchsia_resources_dir, create_intermediates=True, recreate=True)
# Bucket for QEMU resources.
fuchsia_resources_url = environment.get_value('FUCHSIA_RESOURCES_URL')
if not fuchsia_resources_url:
raise errors.FuchsiaConfigError(
'Could not find path for remote'
'Fuchsia resources bucket (FUCHSIA_RESOURCES_URL')
gsutil_command_arguments = [
'-m', 'cp', '-r', fuchsia_resources_url, fuchsia_resources_dir
]
logs.log("Beginning Fuchsia SDK download.")
result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments)
if result.return_code or result.timed_out:
raise errors.FuchsiaSdkError('Failed to download Fuchsia '
'resources: ' + result.output)
logs.log("Fuchsia SDK download complete.")
# Bucket for build resources. Necessary for fuzzer selection.
logs.log("Fetching Fuchsia build.")
fuchsia_build_url = environment.get_value('FUCHSIA_BUILD_URL')
if not fuchsia_build_url:
raise errors.FuchsiaConfigError('Could not find path for remote'
'Fuchsia build bucket (FUCHSIA BUILD URL')
gsutil_command_arguments = [
'-m', 'cp', '-r', fuchsia_build_url, fuchsia_resources_dir
]
logs.log("Beginning Fuchsia build download.")
result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments)
if result.return_code or result.timed_out:
raise errors.FuchsiaSdkError('Failed to download Fuchsia '
'resources: ' + result.output)
return fuchsia_resources_dir
def initialize_resources_dir():
"""Download Fuchsia QEMU resources from GCS bucket."""
# This module depends on multiprocessing, which is not available in
# appengine, and since appengine *imports* this file (but does not run this
# function!), we import it here.
from google_cloud_utils import gsutil
resources_dir = environment.get_value('RESOURCES_DIR')
if not resources_dir:
raise errors.FuchsiaConfigError('Could not find RESOURCES_DIR')
fuchsia_resources_dir = os.path.join(resources_dir, 'fuchsia')
shell.create_directory(fuchsia_resources_dir,
create_intermediates=True,
recreate=True)
# Bucket for QEMU resources.
fuchsia_resources_url = environment.get_value('FUCHSIA_BUILD_URL')
if not fuchsia_resources_url:
raise errors.FuchsiaConfigError(
'Could not find path for remote'
'Fuchsia resources bucket (FUCHSIA_BUILD_URL')
gsutil_command_arguments = [
'-m', 'cp', '-r', fuchsia_resources_url, fuchsia_resources_dir
]
logs.log("Fetching Fuchsia build.")
result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments)
if result.return_code or result.timed_out:
raise errors.FuchsiaSdkError('Failed to download Fuchsia '
'resources: ' + result.output)
# Chmod the symbolizers so they can be used easily.
symbolizer_path = os.path.join(fuchsia_resources_dir, 'build', 'zircon',
'prebuilt', 'downloads', 'symbolize',
'linux-x64', 'symbolize')
llvm_symbolizer_path = os.path.join(fuchsia_resources_dir, 'build',
'buildtools', 'linux-x64', 'clang',
'bin', 'llvm-symbolizer')
os.chmod(symbolizer_path, 0o111)
os.chmod(llvm_symbolizer_path, 0o111)
logs.log("Fuchsia build download complete.")
return fuchsia_resources_dir
def _fetch_qemu_vars():
"""
Returns a dictionary with variables necessary for configuring and running
Fuchsia via QEMU.
This function assumes the following layout for
fuchsia_resources_dir:
* /qemu-for-fuchsia/*
* /.ssh/*
* target/x64/fvm.blk
* target/x64/fuchsia.zbi
* target/x64/multiboot.bin
* build/out/default/fuzzers.json
* build/out/default/ids.txt
* build/out/default.zircon/tools/*
* build/zircon/prebuilt/downloads/symbolize
* build/buildtools/linux-x64/clang/bin/llvm-symbolizer
If these locations change in the build, they should be changed here as well.
"""
qemu_vars = {}
# First, ensure we have a resources directory.
fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
if not fuchsia_resources_dir:
raise errors.FuchsiaConfigError('Could not find FUCHSIA_RESOURCES_DIR')
# Then, save and chmod the associated paths.
qemu_vars['fuchsia_resources_dir'] = fuchsia_resources_dir
qemu_vars['qemu_path'] = os.path.join(fuchsia_resources_dir,
'qemu-for-fuchsia', 'bin',
'qemu-system-x86_64')
os.chmod(qemu_vars['qemu_path'], 0o550)
qemu_vars['kernel_path'] = os.path.join(fuchsia_resources_dir, 'target',
'x64', 'multiboot.bin')
os.chmod(qemu_vars['kernel_path'], 0o644)
qemu_vars['pkey_path'] = os.path.join(fuchsia_resources_dir, '.ssh',
'pkey')
os.chmod(qemu_vars['pkey_path'], 0o400)
qemu_vars['sharefiles_path'] = os.path.join(fuchsia_resources_dir,
'qemu-for-fuchsia', 'share',
'qemu')
qemu_vars['drive_path'] = os.path.join(fuchsia_resources_dir, 'target',
'x64', 'fvm-extended.blk')
qemu_vars['orig_drive_path'] = os.path.join(fuchsia_resources_dir,
'target', 'x64', 'fvm.blk')
os.chmod(qemu_vars['orig_drive_path'], 0o644)
qemu_vars['fuchsia_zbi'] = os.path.join(fuchsia_resources_dir, 'target',
'x64', 'fuchsia.zbi')
qemu_vars['initrd_path'] = os.path.join(fuchsia_resources_dir,
'fuchsia-ssh.zbi')
return qemu_vars
def _fetch_qemu_vars():
"""
Returns a dictionary with variables necessary for configuring and running
Fuchsia via QEMU.
This function assumes the following layout for
fuchsia_resources_dir:
* /qemu-for-fuchsia/*
* /.ssh/*
* target/x64/fvm.blk
* target/x64/fuchsia.zbi
* target/x64/multiboot.bin
* build/out/default/fuzzers.json
* build/out/default/ids.txt
* build/out/default.zircon/tools/*
* build/zircon/prebuilt/downloads/symbolize
* build/buildtools/linux-x64/clang/bin/llvm-symbolizer
If these locations change in the build, they should be changed here as well.
"""
qemu_vars = {}
# First, ensure we have a resources directory.
fuchsia_resources_dir = environment.get_value("FUCHSIA_RESOURCES_DIR")
if not fuchsia_resources_dir:
raise errors.FuchsiaConfigError("Could not find FUCHSIA_RESOURCES_DIR")
# Then, save and chmod the associated paths.
qemu_vars["fuchsia_resources_dir"] = fuchsia_resources_dir
qemu_vars["qemu_path"] = os.path.join(fuchsia_resources_dir,
"qemu-for-fuchsia", "bin",
"qemu-system-x86_64")
os.chmod(qemu_vars["qemu_path"], 0o550)
qemu_vars["kernel_path"] = os.path.join(fuchsia_resources_dir, "target",
"x64", "multiboot.bin")
os.chmod(qemu_vars["kernel_path"], 0o644)
qemu_vars["pkey_path"] = os.path.join(fuchsia_resources_dir, ".ssh",
"pkey")
os.chmod(qemu_vars["pkey_path"], 0o400)
qemu_vars["sharefiles_path"] = os.path.join(fuchsia_resources_dir,
"qemu-for-fuchsia", "share",
"qemu")
qemu_vars["drive_path"] = os.path.join(fuchsia_resources_dir, "target",
"x64", "fvm.blk")
os.chmod(qemu_vars["drive_path"], 0o644)
qemu_vars["fuchsia_zbi"] = os.path.join(fuchsia_resources_dir, "target",
"x64", "fuchsia.zbi")
qemu_vars["initrd_path"] = os.path.join(fuchsia_resources_dir,
"fuchsia-ssh.zbi")
return qemu_vars
def setup_qemu_values(initial_setup=True):
"""Sets up and runs a QEMU VM in the background.
Returns a process.Popen object.
Does not block the calling process, and teardown must be handled by the
caller (use .kill()).
Fuchsia fuzzers assume a QEMU VM is running; call this routine prior to
beginning Fuchsia fuzzing tasks.
This initialization routine assumes the following layout for
fuchsia_resources_dir:
* /qemu-for-fuchsia/*
* /.ssh/*
* target/x64/fvm.blk
* target/x64/fuchsia.zbi
* target/x64/multiboot.bin
* build/out/default/fuzzers.json
* build/out/default/ids.txt
* build/out/default.zircon/tools/*
* build/zircon/prebuilt/downloads/symbolize
* build/buildtools/linux-x64/clang/bin/llvm-symbolizer"""
# First download the Fuchsia resources locally.
fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
if not fuchsia_resources_dir:
raise errors.FuchsiaConfigError('Could not find FUCHSIA_RESOURCES_DIR')
# Then, save paths for necessary commands later.
qemu_path = os.path.join(fuchsia_resources_dir, 'qemu-for-fuchsia', 'bin',
'qemu-system-x86_64')
os.chmod(qemu_path, 0o550)
kernel_path = os.path.join(fuchsia_resources_dir, 'target', 'x64',
'multiboot.bin')
os.chmod(kernel_path, 0o644)
pkey_path = os.path.join(fuchsia_resources_dir, '.ssh', 'pkey')
os.chmod(pkey_path, 0o400)
sharefiles_path = os.path.join(fuchsia_resources_dir, 'qemu-for-fuchsia',
'share', 'qemu')
drive_path = os.path.join(fuchsia_resources_dir, 'target', 'x64', 'fvm.blk')
os.chmod(drive_path, 0o644)
fuchsia_zbi = os.path.join(fuchsia_resources_dir, 'target', 'x64',
'fuchsia.zbi')
initrd_path = os.path.join(fuchsia_resources_dir, 'fuchsia-ssh.zbi')
# Perform some more initiailization steps.
# Only do these the first time you run QEMU after downloading the build.
if initial_setup:
extend_fvm(fuchsia_resources_dir, drive_path)
add_keys_to_zbi(fuchsia_resources_dir, initrd_path, fuchsia_zbi)
# Get a free port for the VM, so we can SSH in later.
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.bind(('localhost', 0))
_, port = tcp.getsockname()
tcp.close()
# Fuzzing jobs that SSH into the QEMU VM need access to this env var.
environment.set_value('FUCHSIA_PORTNUM', port)
environment.set_value('FUCHSIA_RESOURCES_DIR', fuchsia_resources_dir)
# yapf: disable
qemu_args = [
'-m', '2048',
'-nographic',
'-kernel', kernel_path,
'-initrd', initrd_path,
'-smp', '4',
'-drive', 'file=' + drive_path + ',format=raw,if=none,id=blobstore',
'-device', 'virtio-blk-pci,drive=blobstore',
'-monitor', 'none',
'-append', '"kernel.serial=legacy TERM=dumb"',
'-machine', 'q35',
'-display', 'none',
'-netdev',
('user,id=net0,net=192.168.3.0/24,dhcpstart=192.168.3.9,'
'host=192.168.3.2,hostfwd=tcp::') + str(port) + '-:22',
'-device', 'e1000,netdev=net0,mac=52:54:00:63:5e:7b',
'-L', sharefiles_path
]
# yapf: enable
# Detecing KVM is tricky, so use an environment variable to determine whether
# to turn it on or not.
if environment.get_value('FUCHSIA_USE_KVM'):
qemu_args.extend(['-cpu', 'host,migratable=no'])
qemu_args.append('-enable-kvm')
else:
# Can't use host CPU since we don't necessarily have KVM on the machine.
# Emulate a Haswell CPU with a few feature toggles. This mirrors the most
# common configuration for Fuchsia VMs when using in-tree tools.
qemu_args.extend(['-cpu', 'Haswell,+smap,-check,-fsgsbase'])
# Get the list of fuzzers for ClusterFuzz to choose from.
host = Host.from_dir(
os.path.join(fuchsia_resources_dir, 'build', 'out', 'default'))
Device(host, 'localhost', str(port))
Fuzzer.filter(host.fuzzers, '')
# Fuzzing jobs that SSH into the QEMU VM need access to this env var.
environment.set_value('FUCHSIA_PKEY_PATH', pkey_path)
return qemu_path, qemu_args
