def initialize_resources_dir(): """Download Fuchsia QEMU resources from GCS bucket.""" resources_dir = environment.get_value('RESOURCES_DIR') if not resources_dir: raise errors.FuchsiaConfigError('Could not find RESOURCES_DIR') fuchsia_resources_dir = os.path.join(resources_dir, 'fuchsia') shell.create_directory(fuchsia_resources_dir, recreate=True) fuchsia_resources_url = environment.get_value('FUCHSIA_RESOURCES_URL') if not fuchsia_resources_url: raise errors.FuchsiaConfigError( 'Could not find path for remote' 'Fuchsia resources bucket (FUCHSIA_RESOURCES_URL') gsutil_command_arguments = [ '-m', 'cp', '-r', fuchsia_resources_url, fuchsia_resources_dir ] logs.log("Beginning Fuchsia SDK download.") result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments) if result.return_code or result.timed_out: raise errors.FuchsiaSdkError('Failed to download Fuchsia' 'resources: ' + result.output) logs.log("Fuchsia SDK download complete.") return fuchsia_resources_dir
def initialize_resources_dir(): """Download Fuchsia QEMU resources from GCS bucket.""" # This module depends on multiprocessing, which is not available in # appengine, and since appengine *imports* this file (but does not run this # function!), we import it here. from google_cloud_utils import gsutil resources_dir = environment.get_value('RESOURCES_DIR') if not resources_dir: raise errors.FuchsiaConfigError('Could not find RESOURCES_DIR') fuchsia_resources_dir = os.path.join(resources_dir, 'fuchsia') shell.create_directory( fuchsia_resources_dir, create_intermediates=True, recreate=True) # Bucket for QEMU resources. fuchsia_resources_url = environment.get_value('FUCHSIA_RESOURCES_URL') if not fuchsia_resources_url: raise errors.FuchsiaConfigError( 'Could not find path for remote' 'Fuchsia resources bucket (FUCHSIA_RESOURCES_URL') gsutil_command_arguments = [ '-m', 'cp', '-r', fuchsia_resources_url, fuchsia_resources_dir ] logs.log("Beginning Fuchsia SDK download.") result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments) if result.return_code or result.timed_out: raise errors.FuchsiaSdkError('Failed to download Fuchsia ' 'resources: ' + result.output) logs.log("Fuchsia SDK download complete.") # Bucket for build resources. Necessary for fuzzer selection. logs.log("Fetching Fuchsia build.") fuchsia_build_url = environment.get_value('FUCHSIA_BUILD_URL') if not fuchsia_build_url: raise errors.FuchsiaConfigError('Could not find path for remote' 'Fuchsia build bucket (FUCHSIA BUILD URL') gsutil_command_arguments = [ '-m', 'cp', '-r', fuchsia_build_url, fuchsia_resources_dir ] logs.log("Beginning Fuchsia build download.") result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments) if result.return_code or result.timed_out: raise errors.FuchsiaSdkError('Failed to download Fuchsia ' 'resources: ' + result.output) return fuchsia_resources_dir
def initialize_resources_dir(): """Download Fuchsia QEMU resources from GCS bucket.""" # This module depends on multiprocessing, which is not available in # appengine, and since appengine *imports* this file (but does not run this # function!), we import it here. from google_cloud_utils import gsutil resources_dir = environment.get_value('RESOURCES_DIR') if not resources_dir: raise errors.FuchsiaConfigError('Could not find RESOURCES_DIR') fuchsia_resources_dir = os.path.join(resources_dir, 'fuchsia') shell.create_directory(fuchsia_resources_dir, create_intermediates=True, recreate=True) # Bucket for QEMU resources. fuchsia_resources_url = environment.get_value('FUCHSIA_BUILD_URL') if not fuchsia_resources_url: raise errors.FuchsiaConfigError( 'Could not find path for remote' 'Fuchsia resources bucket (FUCHSIA_BUILD_URL') gsutil_command_arguments = [ '-m', 'cp', '-r', fuchsia_resources_url, fuchsia_resources_dir ] logs.log("Fetching Fuchsia build.") result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments) if result.return_code or result.timed_out: raise errors.FuchsiaSdkError('Failed to download Fuchsia ' 'resources: ' + result.output) # Chmod the symbolizers so they can be used easily. symbolizer_path = os.path.join(fuchsia_resources_dir, 'build', 'zircon', 'prebuilt', 'downloads', 'symbolize', 'linux-x64', 'symbolize') llvm_symbolizer_path = os.path.join(fuchsia_resources_dir, 'build', 'buildtools', 'linux-x64', 'clang', 'bin', 'llvm-symbolizer') os.chmod(symbolizer_path, 0o111) os.chmod(llvm_symbolizer_path, 0o111) logs.log("Fuchsia build download complete.") return fuchsia_resources_dir
def _fetch_qemu_vars(): """ Returns a dictionary with variables necessary for configuring and running Fuchsia via QEMU. This function assumes the following layout for fuchsia_resources_dir: * /qemu-for-fuchsia/* * /.ssh/* * target/x64/fvm.blk * target/x64/fuchsia.zbi * target/x64/multiboot.bin * build/out/default/fuzzers.json * build/out/default/ids.txt * build/out/default.zircon/tools/* * build/zircon/prebuilt/downloads/symbolize * build/buildtools/linux-x64/clang/bin/llvm-symbolizer If these locations change in the build, they should be changed here as well. """ qemu_vars = {} # First, ensure we have a resources directory. fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR') if not fuchsia_resources_dir: raise errors.FuchsiaConfigError('Could not find FUCHSIA_RESOURCES_DIR') # Then, save and chmod the associated paths. qemu_vars['fuchsia_resources_dir'] = fuchsia_resources_dir qemu_vars['qemu_path'] = os.path.join(fuchsia_resources_dir, 'qemu-for-fuchsia', 'bin', 'qemu-system-x86_64') os.chmod(qemu_vars['qemu_path'], 0o550) qemu_vars['kernel_path'] = os.path.join(fuchsia_resources_dir, 'target', 'x64', 'multiboot.bin') os.chmod(qemu_vars['kernel_path'], 0o644) qemu_vars['pkey_path'] = os.path.join(fuchsia_resources_dir, '.ssh', 'pkey') os.chmod(qemu_vars['pkey_path'], 0o400) qemu_vars['sharefiles_path'] = os.path.join(fuchsia_resources_dir, 'qemu-for-fuchsia', 'share', 'qemu') qemu_vars['drive_path'] = os.path.join(fuchsia_resources_dir, 'target', 'x64', 'fvm-extended.blk') qemu_vars['orig_drive_path'] = os.path.join(fuchsia_resources_dir, 'target', 'x64', 'fvm.blk') os.chmod(qemu_vars['orig_drive_path'], 0o644) qemu_vars['fuchsia_zbi'] = os.path.join(fuchsia_resources_dir, 'target', 'x64', 'fuchsia.zbi') qemu_vars['initrd_path'] = os.path.join(fuchsia_resources_dir, 'fuchsia-ssh.zbi') return qemu_vars
def _fetch_qemu_vars(): """ Returns a dictionary with variables necessary for configuring and running Fuchsia via QEMU. This function assumes the following layout for fuchsia_resources_dir: * /qemu-for-fuchsia/* * /.ssh/* * target/x64/fvm.blk * target/x64/fuchsia.zbi * target/x64/multiboot.bin * build/out/default/fuzzers.json * build/out/default/ids.txt * build/out/default.zircon/tools/* * build/zircon/prebuilt/downloads/symbolize * build/buildtools/linux-x64/clang/bin/llvm-symbolizer If these locations change in the build, they should be changed here as well. """ qemu_vars = {} # First, ensure we have a resources directory. fuchsia_resources_dir = environment.get_value("FUCHSIA_RESOURCES_DIR") if not fuchsia_resources_dir: raise errors.FuchsiaConfigError("Could not find FUCHSIA_RESOURCES_DIR") # Then, save and chmod the associated paths. qemu_vars["fuchsia_resources_dir"] = fuchsia_resources_dir qemu_vars["qemu_path"] = os.path.join(fuchsia_resources_dir, "qemu-for-fuchsia", "bin", "qemu-system-x86_64") os.chmod(qemu_vars["qemu_path"], 0o550) qemu_vars["kernel_path"] = os.path.join(fuchsia_resources_dir, "target", "x64", "multiboot.bin") os.chmod(qemu_vars["kernel_path"], 0o644) qemu_vars["pkey_path"] = os.path.join(fuchsia_resources_dir, ".ssh", "pkey") os.chmod(qemu_vars["pkey_path"], 0o400) qemu_vars["sharefiles_path"] = os.path.join(fuchsia_resources_dir, "qemu-for-fuchsia", "share", "qemu") qemu_vars["drive_path"] = os.path.join(fuchsia_resources_dir, "target", "x64", "fvm.blk") os.chmod(qemu_vars["drive_path"], 0o644) qemu_vars["fuchsia_zbi"] = os.path.join(fuchsia_resources_dir, "target", "x64", "fuchsia.zbi") qemu_vars["initrd_path"] = os.path.join(fuchsia_resources_dir, "fuchsia-ssh.zbi") return qemu_vars
def setup_qemu_values(initial_setup=True): """Sets up and runs a QEMU VM in the background. Returns a process.Popen object. Does not block the calling process, and teardown must be handled by the caller (use .kill()). Fuchsia fuzzers assume a QEMU VM is running; call this routine prior to beginning Fuchsia fuzzing tasks. This initialization routine assumes the following layout for fuchsia_resources_dir: * /qemu-for-fuchsia/* * /.ssh/* * target/x64/fvm.blk * target/x64/fuchsia.zbi * target/x64/multiboot.bin * build/out/default/fuzzers.json * build/out/default/ids.txt * build/out/default.zircon/tools/* * build/zircon/prebuilt/downloads/symbolize * build/buildtools/linux-x64/clang/bin/llvm-symbolizer""" # First download the Fuchsia resources locally. fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR') if not fuchsia_resources_dir: raise errors.FuchsiaConfigError('Could not find FUCHSIA_RESOURCES_DIR') # Then, save paths for necessary commands later. qemu_path = os.path.join(fuchsia_resources_dir, 'qemu-for-fuchsia', 'bin', 'qemu-system-x86_64') os.chmod(qemu_path, 0o550) kernel_path = os.path.join(fuchsia_resources_dir, 'target', 'x64', 'multiboot.bin') os.chmod(kernel_path, 0o644) pkey_path = os.path.join(fuchsia_resources_dir, '.ssh', 'pkey') os.chmod(pkey_path, 0o400) sharefiles_path = os.path.join(fuchsia_resources_dir, 'qemu-for-fuchsia', 'share', 'qemu') drive_path = os.path.join(fuchsia_resources_dir, 'target', 'x64', 'fvm.blk') os.chmod(drive_path, 0o644) fuchsia_zbi = os.path.join(fuchsia_resources_dir, 'target', 'x64', 'fuchsia.zbi') initrd_path = os.path.join(fuchsia_resources_dir, 'fuchsia-ssh.zbi') # Perform some more initiailization steps. # Only do these the first time you run QEMU after downloading the build. if initial_setup: extend_fvm(fuchsia_resources_dir, drive_path) add_keys_to_zbi(fuchsia_resources_dir, initrd_path, fuchsia_zbi) # Get a free port for the VM, so we can SSH in later. tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tcp.bind(('localhost', 0)) _, port = tcp.getsockname() tcp.close() # Fuzzing jobs that SSH into the QEMU VM need access to this env var. environment.set_value('FUCHSIA_PORTNUM', port) environment.set_value('FUCHSIA_RESOURCES_DIR', fuchsia_resources_dir) # yapf: disable qemu_args = [ '-m', '2048', '-nographic', '-kernel', kernel_path, '-initrd', initrd_path, '-smp', '4', '-drive', 'file=' + drive_path + ',format=raw,if=none,id=blobstore', '-device', 'virtio-blk-pci,drive=blobstore', '-monitor', 'none', '-append', '"kernel.serial=legacy TERM=dumb"', '-machine', 'q35', '-display', 'none', '-netdev', ('user,id=net0,net=192.168.3.0/24,dhcpstart=192.168.3.9,' 'host=192.168.3.2,hostfwd=tcp::') + str(port) + '-:22', '-device', 'e1000,netdev=net0,mac=52:54:00:63:5e:7b', '-L', sharefiles_path ] # yapf: enable # Detecing KVM is tricky, so use an environment variable to determine whether # to turn it on or not. if environment.get_value('FUCHSIA_USE_KVM'): qemu_args.extend(['-cpu', 'host,migratable=no']) qemu_args.append('-enable-kvm') else: # Can't use host CPU since we don't necessarily have KVM on the machine. # Emulate a Haswell CPU with a few feature toggles. This mirrors the most # common configuration for Fuchsia VMs when using in-tree tools. qemu_args.extend(['-cpu', 'Haswell,+smap,-check,-fsgsbase']) # Get the list of fuzzers for ClusterFuzz to choose from. host = Host.from_dir( os.path.join(fuchsia_resources_dir, 'build', 'out', 'default')) Device(host, 'localhost', str(port)) Fuzzer.filter(host.fuzzers, '') # Fuzzing jobs that SSH into the QEMU VM need access to this env var. environment.set_value('FUCHSIA_PKEY_PATH', pkey_path) return qemu_path, qemu_args