示例#1
0
  def _setup_fuzzer_and_device(self):
    """ Build a Fuzzer object based on the QEMU values.
    Call this only after setup_qemu_values()"""
    fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
    fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
    fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
    if (not fuchsia_pkey_path or not fuchsia_portnum or
        not fuchsia_resources_dir):
      raise fuchsia.errors.FuchsiaConfigError(
          ('FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
           'not set'))
    fuchsia_resources_dir_plus_build = os.path.join(fuchsia_resources_dir,
                                                    self.FUCHSIA_BUILD_REL_PATH)
    self.host = Host.from_dir(fuchsia_resources_dir_plus_build)
    self.device = Device(self.host, 'localhost', fuchsia_portnum)
    self.device.set_ssh_option('StrictHostKeyChecking no')
    self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
    self.device.set_ssh_identity(fuchsia_pkey_path)

    # Fuchsia fuzzer names have the format {package_name}/{binary_name}.
    package, target = environment.get_value('FUZZ_TARGET').split('/')
    test_data_dir = os.path.join(fuchsia_resources_dir_plus_build,
                                 self.FUZZER_TEST_DATA_REL_PATH, package,
                                 target)
    self.fuzzer = Fuzzer(
        self.device, package, target, output=test_data_dir, foreground=True)
示例#2
0
    def __init__(self, executable_path, default_args=None):
        fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
        fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
        fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
        if (not fuchsia_pkey_path or not fuchsia_portnum
                or not fuchsia_resources_dir):
            raise fuchsia.errors.FuchsiaConfigError((
                'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
                'not set'))
        fuchsia_resources_dir_plus_build = os.path.join(
            fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH)
        self.host = Host.from_dir(fuchsia_resources_dir_plus_build)
        self.device = Device(self.host, 'localhost', fuchsia_portnum)
        self.device.set_ssh_option('StrictHostKeyChecking no')
        self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
        self.device.set_ssh_identity(fuchsia_pkey_path)
        # Fuchsia fuzzer names have the format {package_name}/{binary_name}.
        package, target = environment.get_value('FUZZ_TARGET').split('/')
        test_data_dir = os.path.join(fuchsia_resources_dir_plus_build,
                                     self.FUZZER_TEST_DATA_REL_PATH, package,
                                     target)
        self.fuzzer = Fuzzer(self.device,
                             package,
                             target,
                             output=test_data_dir,
                             foreground=True)

        super(FuchsiaQemuLibFuzzerRunner,
              self).__init__(executable_path=executable_path,
                             default_args=default_args)
示例#3
0
    def _setup_device_and_fuzzer(self):
        """Build a Device and Fuzzer object based on QEMU's settings."""
        # These environment variables are set when start_qemu is run.
        # We need them in order to ssh / otherwise communicate with the VM.
        fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
        fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
        fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
        if (not fuchsia_pkey_path or not fuchsia_portnum
                or not fuchsia_resources_dir):
            raise fuchsia.errors.FuchsiaConfigError((
                'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
                'not set'))

        # Fuzzer objects communicate with the VM via a Device object,
        # which we set up here.
        fuchsia_resources_dir_plus_build = os.path.join(
            fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH)
        self.host = Host.from_dir(fuchsia_resources_dir_plus_build)
        self.device = Device(self.host, 'localhost', fuchsia_portnum)
        self.device.set_ssh_option('StrictHostKeyChecking no')
        self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
        self.device.set_ssh_identity(fuchsia_pkey_path)

        # Fuchsia fuzzer names have the format {package_name}/{binary_name}.
        package, target = self.executable_path.split('/')
        test_data_dir = os.path.join(fuchsia_resources_dir_plus_build,
                                     self.FUZZER_TEST_DATA_REL_PATH, package,
                                     target)

        # Finally, we set up the Fuzzer object itself, which will run our fuzzer!
        self.fuzzer = Fuzzer(self.device,
                             package,
                             target,
                             output=test_data_dir,
                             foreground=True)
示例#4
0
 def __init__(self, executable_path, default_args=None):
     fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
     fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
     fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
     if (not fuchsia_pkey_path or not fuchsia_portnum
             or not fuchsia_resources_dir):
         raise fuchsia.errors.FuchsiaConfigError((
             'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
             'not set'))
     self.host = Host.from_dir(
         os.path.join(fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH))
     self.device = Device(self.host, 'localhost', fuchsia_portnum)
     # Fuchsia fuzzer names have the format {package_name}/{binary_name}.
     # TODO(ochang): Properly handle fuzzers with '/' in the binary name.
     package, target = environment.get_value('FUZZ_TARGET').split('/')
     self.fuzzer = Fuzzer(self.device, package, target)
     self.device.set_ssh_option('StrictHostKeyChecking no')
     self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
     self.device.set_ssh_identity(fuchsia_pkey_path)
     super(FuchsiaQemuLibFuzzerRunner,
           self).__init__(executable_path=executable_path,
                          default_args=default_args)
示例#5
0
class FuchsiaQemuLibFuzzerRunner(new_process.ProcessRunner, LibFuzzerCommon):
    """libFuzzer runner (when Fuchsia is the target platform)."""

    FUCHSIA_BUILD_REL_PATH = os.path.join('build', 'out', 'default')

    SSH_RETRIES = 3
    SSH_WAIT = 2

    FUZZER_TEST_DATA_REL_PATH = os.path.join('test_data', 'fuzzing')

    def __init__(self, executable_path, default_args=None):
        fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
        fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
        fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
        if (not fuchsia_pkey_path or not fuchsia_portnum
                or not fuchsia_resources_dir):
            raise fuchsia.errors.FuchsiaConfigError((
                'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
                'not set'))
        fuchsia_resources_dir_plus_build = os.path.join(
            fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH)
        self.host = Host.from_dir(fuchsia_resources_dir_plus_build)
        self.device = Device(self.host, 'localhost', fuchsia_portnum)
        self.device.set_ssh_option('StrictHostKeyChecking no')
        self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
        self.device.set_ssh_identity(fuchsia_pkey_path)
        # Fuchsia fuzzer names have the format {package_name}/{binary_name}.
        package, target = environment.get_value('FUZZ_TARGET').split('/')
        test_data_dir = os.path.join(fuchsia_resources_dir_plus_build,
                                     self.FUZZER_TEST_DATA_REL_PATH, package,
                                     target)
        self.fuzzer = Fuzzer(self.device,
                             package,
                             target,
                             output=test_data_dir,
                             foreground=True)

        super(FuchsiaQemuLibFuzzerRunner,
              self).__init__(executable_path=executable_path,
                             default_args=default_args)

    def get_command(self, additional_args=None):
        # TODO(flowerhack): Update this to dynamically pick a result from "fuzz
        # list" and then run that fuzzer.
        return self.ssh_command('ls')

    def fetch_and_process_logs_and_crash(self):
        """Fetch symbolized logs and crashes."""
        # Fetch the symbolized log.
        for logname in os.listdir(self.fuzzer.results_output()):
            if logname == os.path.basename(self.fuzzer.logfile):
                self.device.dlog(self.fuzzer.logfile)

        # Clusterfuzz assumes that the Libfuzzer output points to an absolute path,
        # where it can find the crash file.
        # This doesn't work in our case due to how Fuchsia is run.
        # So, we make a new file, change the appropriate line with a regex to point
        # to the true location. Apologies for the hackery.
        crash_location_regex = r'(.*)(Test unit written to )(data/.*)'
        _, new_file_handle_path = tempfile.mkstemp()
        with open(new_file_handle_path, 'w') as new_file:
            with open(self.fuzzer.logfile) as old_file:
                for line in old_file:
                    line_match = re.match(crash_location_regex, line)
                    if line_match:
                        # We now know the name of our crash file.
                        crash_name = line_match.group(3).replace('data/', '')
                        # Save the crash locally.
                        self.device.fetch(self.fuzzer.data_path(crash_name),
                                          self.fuzzer.results_output())
                        # Then update the crash report to point to that file.
                        crash_testcase_file_path = os.path.join(
                            self.fuzzer.results_output(), crash_name)
                        line = re.sub(crash_location_regex,
                                      r'\1\2' + crash_testcase_file_path, line)
                    new_file.write(line)
        os.remove(self.fuzzer.logfile)
        os.rename(new_file_handle_path, self.fuzzer.logfile)

    def fuzz(self,
             corpus_directories,
             fuzz_timeout,
             artifact_prefix=None,
             additional_args=None,
             extra_env=None):
        """LibFuzzerCommon.fuzz override."""
        self._test_qemu_ssh()
        self.fuzzer.start([])
        self.fetch_and_process_logs_and_crash()

        with open(self.fuzzer.logfile) as logfile:
            symbolized_output = logfile.read()

        # TODO(flowerhack): Would be nice if we could figure out a way to make
        # the "fuzzer start" code return its own ProcessResult. For now, we simply
        # craft one by hand here.
        fuzzer_process_result = new_process.ProcessResult()
        fuzzer_process_result.return_code = 0
        fuzzer_process_result.output = symbolized_output
        fuzzer_process_result.time_executed = 0
        fuzzer_process_result.command = self.fuzzer.last_fuzz_cmd
        return fuzzer_process_result

    def run_single_testcase(self,
                            testcase_path,
                            timeout=None,
                            additional_args=None):
        # TODO(flowerhack): Fill out this command.
        pass

    def ssh_command(self, *args):
        return ['ssh'] + self.ssh_root + list(args)

    @retry.wrap(retries=SSH_RETRIES, delay=SSH_WAIT, function='_test_qemu_ssh')
    def _test_qemu_ssh(self):
        """Tests that a VM is up and can be successfully SSH'd into.
    Raises an exception if no success after MAX_SSH_RETRIES."""
        ssh_test_process = new_process.ProcessRunner(
            'ssh',
            self.device.get_ssh_cmd(
                ['ssh', 'localhost', 'echo running on fuchsia!'])[1:])
        result = ssh_test_process.run_and_wait()
        if result.return_code or result.timed_out:
            raise fuchsia.errors.FuchsiaConnectionError(
                'Failed to establish initial SSH connection: ' +
                str(result.return_code) + " , " + str(result.command) + " , " +
                str(result.output))
        return result
示例#6
0
    def create(self):
        """Configures a QEMU process which can subsequently be `run`.

    Assumes that initial_qemu_setup was already called exactly once.
    """
        qemu_vars = _fetch_qemu_vars()

        # Get a free port for the VM, so we can SSH in later.
        tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        tcp.bind(('localhost', 0))
        _, port = tcp.getsockname()
        tcp.close()
        # Fuzzing jobs that SSH into the QEMU VM need access to this env var.
        environment.set_value('FUCHSIA_PORTNUM', port)
        environment.set_value('FUCHSIA_RESOURCES_DIR',
                              qemu_vars['fuchsia_resources_dir'])

        # yapf: disable
        qemu_args = [
            '-m', '3072',
            '-nographic',
            '-kernel', qemu_vars['kernel_path'],
            '-initrd', qemu_vars['initrd_path'],
            '-smp', '4',
            '-drive',
            ('file=' + qemu_vars['drive_path'] + ',format=raw,if=none,'
             'id=blobstore'),
            '-device', 'virtio-blk-pci,drive=blobstore',
            '-monitor', 'none',
            '-append', 'kernel.serial=legacy TERM=dumb',
            '-machine', 'q35',
            '-display', 'none',
            '-netdev',
            ('user,id=net0,net=192.168.3.0/24,dhcpstart=192.168.3.9,'
             'host=192.168.3.2,hostfwd=tcp::') + str(port) + '-:22',
            '-device', 'e1000,netdev=net0,mac=52:54:00:63:5e:7b',
            '-L', qemu_vars['sharefiles_path']
        ]
        # yapf: enable

        # Detecting KVM is tricky, so use an environment variable to determine
        # whether to turn it on or not.
        if environment.get_value('FUCHSIA_USE_KVM'):
            # In builds before fxrev.dev/375343, a bug prevents booting with newer
            # versions of KVM. On some of these older builds,
            # `kernel.x86.disable-spec-mitigations` also doesn't work as
            # expected, so we work around this by selecting a CPU type where the
            # speculation mitigation will not applied.
            if environment.get_value('APP_REVISION') < 20200414210423:
                qemu_args.extend(['-cpu', 'Opteron_G5,+invtsc'])
            else:
                qemu_args.extend(['-cpu', 'host,migratable=no,+invtsc'])
            qemu_args.append('-enable-kvm')
        else:
            # Can't use host CPU since we don't necessarily have KVM on the machine.
            # Emulate a Haswell CPU with a few feature toggles. This mirrors the most
            # common configuration for Fuchsia VMs when using in-tree tools.
            qemu_args.extend(['-cpu', 'Haswell,+smap,-check,-fsgsbase'])

        # Get the list of fuzzers for ClusterFuzz to choose from.
        host = Host.from_dir(
            os.path.join(qemu_vars['fuchsia_resources_dir'], 'build', 'out',
                         'default'))
        Device(host, 'localhost', str(port))
        Fuzzer.filter(host.fuzzers, '')

        # Fuzzing jobs that SSH into the QEMU VM need access to this env var.
        environment.set_value('FUCHSIA_PKEY_PATH', qemu_vars['pkey_path'])
        logs.log('Ready to run QEMU. Command: ' + qemu_vars['qemu_path'] +
                 ' ' + ' '.join(shlex.quote(arg) for arg in qemu_args))
        self.process_runner = new_process.ProcessRunner(
            qemu_vars['qemu_path'], qemu_args)
示例#7
0
def setup_qemu_values(initial_setup=True):
  """Sets up and runs a QEMU VM in the background.
  Returns a process.Popen object.
  Does not block the calling process, and teardown must be handled by the
  caller (use .kill()).
  Fuchsia fuzzers assume a QEMU VM is running; call this routine prior to
  beginning Fuchsia fuzzing tasks.
  This initialization routine assumes the following layout for
  fuchsia_resources_dir:

  * /qemu-for-fuchsia/*
  * /.ssh/*
  * target/x64/fvm.blk
  * target/x64/fuchsia.zbi
  * target/x64/multiboot.bin

  * build/out/default/fuzzers.json
  * build/out/default/ids.txt
  * build/out/default.zircon/tools/*
  * build/zircon/prebuilt/downloads/symbolize
  * build/buildtools/linux-x64/clang/bin/llvm-symbolizer"""

  # First download the Fuchsia resources locally.
  fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
  if not fuchsia_resources_dir:
    raise errors.FuchsiaConfigError('Could not find FUCHSIA_RESOURCES_DIR')

  # Then, save paths for necessary commands later.
  qemu_path = os.path.join(fuchsia_resources_dir, 'qemu-for-fuchsia', 'bin',
                           'qemu-system-x86_64')
  os.chmod(qemu_path, 0o550)
  kernel_path = os.path.join(fuchsia_resources_dir, 'target', 'x64',
                             'multiboot.bin')
  os.chmod(kernel_path, 0o644)
  pkey_path = os.path.join(fuchsia_resources_dir, '.ssh', 'pkey')
  os.chmod(pkey_path, 0o400)
  sharefiles_path = os.path.join(fuchsia_resources_dir, 'qemu-for-fuchsia',
                                 'share', 'qemu')
  drive_path = os.path.join(fuchsia_resources_dir, 'target', 'x64', 'fvm.blk')
  os.chmod(drive_path, 0o644)
  fuchsia_zbi = os.path.join(fuchsia_resources_dir, 'target', 'x64',
                             'fuchsia.zbi')
  initrd_path = os.path.join(fuchsia_resources_dir, 'fuchsia-ssh.zbi')

  # Perform some more initiailization steps.
  # Only do these the first time you run QEMU after downloading the build.
  if initial_setup:
    extend_fvm(fuchsia_resources_dir, drive_path)
    add_keys_to_zbi(fuchsia_resources_dir, initrd_path, fuchsia_zbi)

  # Get a free port for the VM, so we can SSH in later.
  tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  tcp.bind(('localhost', 0))
  _, port = tcp.getsockname()
  tcp.close()
  # Fuzzing jobs that SSH into the QEMU VM need access to this env var.
  environment.set_value('FUCHSIA_PORTNUM', port)
  environment.set_value('FUCHSIA_RESOURCES_DIR', fuchsia_resources_dir)

  # yapf: disable
  qemu_args = [
      '-m', '2048',
      '-nographic',
      '-kernel', kernel_path,
      '-initrd', initrd_path,
      '-smp', '4',
      '-drive', 'file=' + drive_path + ',format=raw,if=none,id=blobstore',
      '-device', 'virtio-blk-pci,drive=blobstore',
      '-monitor', 'none',
      '-append', '"kernel.serial=legacy TERM=dumb"',
      '-machine', 'q35',
      '-display', 'none',
      '-netdev',
      ('user,id=net0,net=192.168.3.0/24,dhcpstart=192.168.3.9,'
       'host=192.168.3.2,hostfwd=tcp::') + str(port) + '-:22',
      '-device', 'e1000,netdev=net0,mac=52:54:00:63:5e:7b',
      '-L', sharefiles_path
  ]
  # yapf: enable

  # Detecing KVM is tricky, so use an environment variable to determine whether
  # to turn it on or not.
  if environment.get_value('FUCHSIA_USE_KVM'):
    qemu_args.extend(['-cpu', 'host,migratable=no'])
    qemu_args.append('-enable-kvm')
  else:
    # Can't use host CPU since we don't necessarily have KVM on the machine.
    # Emulate a Haswell CPU with a few feature toggles. This mirrors the most
    # common configuration for Fuchsia VMs when using in-tree tools.
    qemu_args.extend(['-cpu', 'Haswell,+smap,-check,-fsgsbase'])

  # Get the list of fuzzers for ClusterFuzz to choose from.
  host = Host.from_dir(
      os.path.join(fuchsia_resources_dir, 'build', 'out', 'default'))
  Device(host, 'localhost', str(port))
  Fuzzer.filter(host.fuzzers, '')

  # Fuzzing jobs that SSH into the QEMU VM need access to this env var.
  environment.set_value('FUCHSIA_PKEY_PATH', pkey_path)
  return qemu_path, qemu_args
示例#8
0
class FuchsiaQemuLibFuzzerRunner(new_process.ProcessRunner, LibFuzzerCommon):
    """libFuzzer runner (when Fuchsia is the target platform)."""

    FUCHSIA_BUILD_REL_PATH = os.path.join('build', 'out', 'default')

    SSH_RETRIES = 3
    SSH_WAIT = 2

    def __init__(self, executable_path, default_args=None):
        fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
        fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
        fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
        if (not fuchsia_pkey_path or not fuchsia_portnum
                or not fuchsia_resources_dir):
            raise fuchsia.errors.FuchsiaConfigError((
                'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
                'not set'))
        self.host = Host.from_dir(
            os.path.join(fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH))
        self.device = Device(self.host, 'localhost', fuchsia_portnum)
        # Fuchsia fuzzer names have the format {package_name}/{binary_name}.
        # TODO(ochang): Properly handle fuzzers with '/' in the binary name.
        package, target = environment.get_value('FUZZ_TARGET').split('/')
        self.fuzzer = Fuzzer(self.device, package, target)
        self.device.set_ssh_option('StrictHostKeyChecking no')
        self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
        self.device.set_ssh_identity(fuchsia_pkey_path)
        super(FuchsiaQemuLibFuzzerRunner,
              self).__init__(executable_path=executable_path,
                             default_args=default_args)

    def get_command(self, additional_args=None):
        # TODO(flowerhack): Update this to dynamically pick a result from "fuzz
        # list" and then run that fuzzer.
        return self.ssh_command('ls')

    def fuzz(self,
             corpus_directories,
             fuzz_timeout,
             artifact_prefix=None,
             additional_args=None,
             extra_env=None):
        """LibFuzzerCommon.fuzz override."""
        self._test_qemu_ssh()
        self.fuzzer.run([])
        # TODO(flowerhack): Modify fuzzer.run() to return a ProcessResult, rather
        # than artisinally handcrafting one here.
        fuzzer_process_result = new_process.ProcessResult()
        fuzzer_process_result.return_code = 0
        fuzzer_process_result.output = ''
        fuzzer_process_result.time_executed = 0
        fuzzer_process_result.command = self.fuzzer.last_fuzz_cmd
        return fuzzer_process_result

    def run_single_testcase(self,
                            testcase_path,
                            timeout=None,
                            additional_args=None):
        # TODO(flowerhack): Fill out this command.
        pass

    def ssh_command(self, *args):
        return ['ssh'] + self.ssh_root + list(args)

    @retry.wrap(retries=SSH_RETRIES, delay=SSH_WAIT, function='_test_qemu_ssh')
    def _test_qemu_ssh(self):
        """Tests that a VM is up and can be successfully SSH'd into.
    Raises an exception if no success after MAX_SSH_RETRIES."""
        ssh_test_process = new_process.ProcessRunner(
            'ssh',
            self.device.get_ssh_cmd(
                ['ssh', 'localhost', 'echo running on fuchsia!'])[1:])
        result = ssh_test_process.run_and_wait()
        if result.return_code or result.timed_out:
            raise fuchsia.errors.FuchsiaConnectionError(
                'Failed to establish initial SSH connection: ' +
                str(result.return_code) + " , " + str(result.command) + " , " +
                str(result.output))
        return result
示例#9
0
    def create(self):
        """Configures a QEMU process which can subsequently be `run`.

        Assumes that initial_qemu_setup was already called exactly once.
        """
        qemu_vars = _fetch_qemu_vars()

        # Get a free port for the VM, so we can SSH in later.
        tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        tcp.bind(("localhost", 0))
        _, port = tcp.getsockname()
        tcp.close()
        # Fuzzing jobs that SSH into the QEMU VM need access to this env var.
        environment.set_value("FUCHSIA_PORTNUM", port)
        environment.set_value("FUCHSIA_RESOURCES_DIR",
                              qemu_vars["fuchsia_resources_dir"])

        # yapf: disable
        qemu_args = [
            '-m', '3072',
            '-nographic',
            '-kernel', qemu_vars['kernel_path'],
            '-initrd', qemu_vars['initrd_path'],
            '-smp', '4',
            '-drive',
            ('file=' + qemu_vars['drive_path'] + ',format=raw,if=none,'
             'id=blobstore'),
            '-device', 'virtio-blk-pci,drive=blobstore',
            '-monitor', 'none',
            '-append', 'kernel.serial=legacy TERM=dumb',
            '-machine', 'q35',
            '-display', 'none',
            '-netdev',
            ('user,id=net0,net=192.168.3.0/24,dhcpstart=192.168.3.9,'
             'host=192.168.3.2,hostfwd=tcp::') + str(port) + '-:22',
            '-device', 'e1000,netdev=net0,mac=52:54:00:63:5e:7b',
            '-L', qemu_vars['sharefiles_path']
        ]
        # yapf: enable

        # Detecing KVM is tricky, so use an environment variable to determine
        # whether to turn it on or not.
        if environment.get_value("FUCHSIA_USE_KVM"):
            qemu_args.extend(["-cpu", "host,migratable=no"])
            qemu_args.append("-enable-kvm")
        else:
            # Can't use host CPU since we don't necessarily have KVM on the machine.
            # Emulate a Haswell CPU with a few feature toggles. This mirrors the most
            # common configuration for Fuchsia VMs when using in-tree tools.
            qemu_args.extend(["-cpu", "Haswell,+smap,-check,-fsgsbase"])

        # Get the list of fuzzers for ClusterFuzz to choose from.
        host = Host.from_dir(
            os.path.join(qemu_vars["fuchsia_resources_dir"], "build", "out",
                         "default"))
        Device(host, "localhost", str(port))
        Fuzzer.filter(host.fuzzers, "")

        # Fuzzing jobs that SSH into the QEMU VM need access to this env var.
        environment.set_value("FUCHSIA_PKEY_PATH", qemu_vars["pkey_path"])
        logs.log("Ready to run QEMU. Command: " + qemu_vars["qemu_path"] +
                 " " + str(qemu_args))
        self.process_runner = new_process.ProcessRunner(
            qemu_vars["qemu_path"], qemu_args)
示例#10
0
class FuchsiaQemuLibFuzzerRunner(new_process.ProcessRunner, LibFuzzerCommon):
    """libFuzzer runner (when Fuchsia is the target platform)."""

    FUCHSIA_BUILD_REL_PATH = os.path.join('build', 'out', 'default')

    SSH_RETRIES = 3
    SSH_WAIT = 3

    FUZZER_TEST_DATA_REL_PATH = os.path.join('test_data', 'fuzzing')

    def _setup_device_and_fuzzer(self):
        """Build a Device and Fuzzer object based on QEMU's settings."""
        # These environment variables are set when start_qemu is run.
        # We need them in order to ssh / otherwise communicate with the VM.
        fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
        fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
        fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
        if (not fuchsia_pkey_path or not fuchsia_portnum
                or not fuchsia_resources_dir):
            raise fuchsia.errors.FuchsiaConfigError((
                'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
                'not set'))

        # Fuzzer objects communicate with the VM via a Device object,
        # which we set up here.
        fuchsia_resources_dir_plus_build = os.path.join(
            fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH)
        self.host = Host.from_dir(fuchsia_resources_dir_plus_build)
        self.device = Device(self.host, 'localhost', fuchsia_portnum)
        self.device.set_ssh_option('StrictHostKeyChecking no')
        self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
        self.device.set_ssh_identity(fuchsia_pkey_path)

        # Fuchsia fuzzer names have the format {package_name}/{binary_name}.
        package, target = self.executable_path.split('/')
        test_data_dir = os.path.join(fuchsia_resources_dir_plus_build,
                                     self.FUZZER_TEST_DATA_REL_PATH, package,
                                     target)

        # Finally, we set up the Fuzzer object itself, which will run our fuzzer!
        sanitizer = environment.get_memory_tool_name(
            environment.get_value('JOB_NAME')).lower()
        self.fuzzer = Fuzzer(self.device,
                             package,
                             target,
                             output=test_data_dir,
                             foreground=True,
                             sanitizer=sanitizer)

    def __init__(self, executable_path, default_args=None):
        # We always assume QEMU is running on __init__, since build_manager sets
        # it up initially. If this isn't the case, _test_ssh will detect and
        # restart QEMU anyway.
        super(FuchsiaQemuLibFuzzerRunner,
              self).__init__(executable_path=executable_path,
                             default_args=default_args)
        self._setup_device_and_fuzzer()

    def process_logs_and_crash(self, artifact_prefix):
        """Fetch symbolized logs and crashes."""
        if not artifact_prefix:
            return

        # Clusterfuzz assumes that the Libfuzzer output points to an absolute path,
        # where it can find the crash file.
        # This doesn't work in our case due to how Fuchsia is run.
        # So, we make a new file, change the appropriate line with a regex to point
        # to the true location. Apologies for the hackery.
        crash_location_regex = r'(.*)(Test unit written to )(data/.*)'
        _, processed_log_path = tempfile.mkstemp()
        with open(processed_log_path, 'w') as new_file:
            with open(self.fuzzer.logfile) as old_file:
                for line in old_file:
                    line_match = re.match(crash_location_regex, line)
                    if line_match:
                        # We now know the name of our crash file.
                        crash_name = line_match.group(3).replace('data/', '')
                        # Save the crash locally.
                        self.device.fetch(self.fuzzer.data_path(crash_name),
                                          artifact_prefix)
                        # Then update the crash report to point to that file.
                        crash_testcase_file_path = os.path.join(
                            artifact_prefix, crash_name)
                        line = re.sub(crash_location_regex,
                                      r'\1\2' + crash_testcase_file_path, line)
                    new_file.write(line)
        os.remove(self.fuzzer.logfile)
        shutil.move(processed_log_path, self.fuzzer.logfile)

    def _test_ssh(self):
        """Test the ssh connection."""
        # Test the connection.  If this works, proceed.
        # - If we fail, restart QEMU and test the connection again.
        # - If that fails, throw the error; we can't seem to recover.
        try:
            self._test_qemu_ssh()
        except fuchsia.errors.FuchsiaConnectionError:
            self._restart_qemu()
            self._test_qemu_ssh()

    @retry.wrap(retries=SSH_RETRIES, delay=SSH_WAIT, function='_test_qemu_ssh')
    def _test_qemu_ssh(self):
        """Tests that a VM is up and can be successfully SSH'd into.
    Raises an exception if no success after MAX_SSH_RETRIES."""
        ssh_test_process = new_process.ProcessRunner(
            'ssh',
            self.device.get_ssh_cmd(
                ['ssh', 'localhost', 'echo running on fuchsia!'])[1:])
        result = ssh_test_process.run_and_wait()
        if result.return_code or result.timed_out:
            raise fuchsia.errors.FuchsiaConnectionError(
                'Failed to establish initial SSH connection: ' +
                str(result.return_code) + " , " + str(result.command) + " , " +
                str(result.output))
        return result

    def _restart_qemu(self):
        """Restart QEMU."""
        logs.log_warn('Connection to fuzzing VM lost. Restarting.')
        stop_qemu()
        start_qemu()
        self._setup_device_and_fuzzer()

    def _corpus_target_subdir(self, relpath):
        """ Returns the absolute path of the corpus subdirectory on the target,
    given "relpath", the name of the specific corpus. """
        return os.path.join(self._corpus_directories_target(), relpath)

    def _corpus_directories_libfuzzer(self, corpus_directories):
        """ Returns the corpus directory paths expected by libfuzzer itself. """
        corpus_directories_libfuzzer = []
        for corpus_dir in corpus_directories:
            corpus_directories_libfuzzer.append(
                os.path.join('data', 'corpus', os.path.basename(corpus_dir)))
        return corpus_directories_libfuzzer

    def _new_corpus_dir_host(self, corpus_directories):
        """ Returns the path of the 'new' corpus directory on the host. """
        return corpus_directories[0]

    def _new_corpus_dir_target(self, corpus_directories):
        """ Returns the path of the 'new' corpus directory on the target. """
        new_corpus_dir_host = self._new_corpus_dir_host(corpus_directories)
        return self.fuzzer.data_path(
            os.path.join('corpus', os.path.basename(new_corpus_dir_host)))

    def _corpus_directories_target(self):
        """ Returns the path of the root corpus directory on the target. """
        return self.fuzzer.data_path('corpus')

    def _push_corpora_from_host_to_target(self, corpus_directories):
        # Push corpus directories to the device.
        self._clear_all_target_corpora()
        logs.log('Push corpora from host to target.')
        for corpus_dir in corpus_directories:
            # Appending '/*' indicates we want all the *files* in the corpus_dir's
            self.fuzzer.device.store(
                corpus_dir + '/*',
                self._corpus_target_subdir(os.path.basename(corpus_dir)))

    def _pull_new_corpus_from_target_to_host(self, corpus_directories):
        # Appending '/*' indicates we want all the *files* in the target's
        # directory, rather than the directory itself.
        logs.log('Fuzzer ran; pull down corpus')
        files_in_new_corpus_dir_target = self._new_corpus_dir_target(
            corpus_directories) + "/*"
        self.fuzzer.device.fetch(files_in_new_corpus_dir_target,
                                 self._new_corpus_dir_host(corpus_directories))

    def _clear_all_target_corpora(self):
        """ Clears out all the corpora on the target. """
        logs.log('Clearing corpora on target')
        self.fuzzer.device.ssh(
            ['rm', '-rf', self._corpus_directories_target()])

    def fuzz(self,
             corpus_directories,
             fuzz_timeout,
             artifact_prefix=None,
             additional_args=None,
             extra_env=None):
        """LibFuzzerCommon.fuzz override."""
        additional_args = copy.copy(additional_args)
        if additional_args is None:
            additional_args = []

        self._test_ssh()
        self._push_corpora_from_host_to_target(corpus_directories)

        max_total_time = self.get_max_total_time(fuzz_timeout)
        if any(arg.startswith(constants.FORK_FLAG) for arg in additional_args):
            max_total_time -= self.LIBFUZZER_FORK_MODE_CLEAN_EXIT_TIME
        assert max_total_time > 0

        additional_args.extend([
            '%s%d' % (constants.MAX_TOTAL_TIME_FLAG, max_total_time),
            constants.PRINT_FINAL_STATS_ARGUMENT,
        ])

        # Run the fuzzer.
        # TODO: actually we want new_corpus_relative_dir_target for *each* corpus
        return_code = self.fuzzer.start(
            self._corpus_directories_libfuzzer(corpus_directories) +
            additional_args)
        self.fuzzer.monitor(return_code)
        self.process_logs_and_crash(artifact_prefix)
        with open(self.fuzzer.logfile) as logfile:
            symbolized_output = logfile.read()

        self._pull_new_corpus_from_target_to_host(corpus_directories)
        self._clear_all_target_corpora()

        # TODO(flowerhack): Would be nice if we could figure out a way to make
        # the "fuzzer start" code return its own ProcessResult. For now, we simply
        # craft one by hand here.
        fuzzer_process_result = new_process.ProcessResult()
        fuzzer_process_result.return_code = 0
        fuzzer_process_result.output = symbolized_output
        fuzzer_process_result.time_executed = 0
        fuzzer_process_result.command = self.fuzzer.last_fuzz_cmd
        return fuzzer_process_result

    def merge(self,
              corpus_directories,
              merge_timeout,
              artifact_prefix=None,
              tmp_dir=None,
              additional_args=None):
        # TODO(flowerhack): Integrate some notion of a merge timeout.
        self._push_corpora_from_host_to_target(corpus_directories)

        # Run merge.
        _, _ = self.fuzzer.merge(
            self._corpus_directories_libfuzzer(corpus_directories) +
            additional_args)

        self._pull_new_corpus_from_target_to_host(corpus_directories)
        self._clear_all_target_corpora()

        merge_result = new_process.ProcessResult()
        merge_result.return_code = 0
        merge_result.timed_out = False
        merge_result.output = ''
        merge_result.time_executed = 0
        merge_result.command = ''
        return merge_result

    def run_single_testcase(self,
                            testcase_path,
                            timeout=None,
                            additional_args=None):
        """Run a single testcase."""
        self._test_ssh()

        # We need to push the testcase to the device and pass in the name.
        testcase_path_name = os.path.basename(os.path.normpath(testcase_path))
        self.device.store(testcase_path, self.fuzzer.data_path())

        return_code = self.fuzzer.start(
            ['repro', 'data/' + testcase_path_name] + additional_args)
        self.fuzzer.monitor(return_code)

        with open(self.fuzzer.logfile) as logfile:
            symbolized_output = logfile.read()

        fuzzer_process_result = new_process.ProcessResult()
        fuzzer_process_result.return_code = 0
        fuzzer_process_result.output = symbolized_output
        fuzzer_process_result.time_executed = 0
        fuzzer_process_result.command = self.fuzzer.last_fuzz_cmd
        return fuzzer_process_result

    def minimize_crash(self,
                       testcase_path,
                       output_path,
                       timeout,
                       artifact_prefix=None,
                       additional_args=None):
        return new_process.ProcessResult()

    def ssh_command(self, *args):
        return ['ssh'] + self.ssh_root + list(args)
示例#11
0
class FuchsiaQemuLibFuzzerRunner(new_process.ProcessRunner, LibFuzzerCommon):
    """libFuzzer runner (when Fuchsia is the target platform)."""

    FUCHSIA_BUILD_REL_PATH = os.path.join('build', 'out', 'default')

    SSH_RETRIES = 3
    SSH_WAIT = 3

    FUZZER_TEST_DATA_REL_PATH = os.path.join('test_data', 'fuzzing')

    def _setup_device_and_fuzzer(self):
        """Build a Device and Fuzzer object based on QEMU's settings."""
        # These environment variables are set when start_qemu is run.
        # We need them in order to ssh / otherwise communicate with the VM.
        fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH')
        fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM')
        fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR')
        if (not fuchsia_pkey_path or not fuchsia_portnum
                or not fuchsia_resources_dir):
            raise fuchsia.errors.FuchsiaConfigError((
                'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was '
                'not set'))

        # Fuzzer objects communicate with the VM via a Device object,
        # which we set up here.
        fuchsia_resources_dir_plus_build = os.path.join(
            fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH)
        self.host = Host.from_dir(fuchsia_resources_dir_plus_build)
        self.device = Device(self.host, 'localhost', fuchsia_portnum)
        self.device.set_ssh_option('StrictHostKeyChecking no')
        self.device.set_ssh_option('UserKnownHostsFile=/dev/null')
        self.device.set_ssh_identity(fuchsia_pkey_path)

        # Fuchsia fuzzer names have the format {package_name}/{binary_name}.
        package, target = self.executable_path.split('/')
        test_data_dir = os.path.join(fuchsia_resources_dir_plus_build,
                                     self.FUZZER_TEST_DATA_REL_PATH, package,
                                     target)

        # Finally, we set up the Fuzzer object itself, which will run our fuzzer!
        self.fuzzer = Fuzzer(self.device,
                             package,
                             target,
                             output=test_data_dir,
                             foreground=True)

    def __init__(self, executable_path, default_args=None):
        # We always assume QEMU is running on __init__, since build_manager sets
        # it up initially. If this isn't the case, _test_ssh will detect and
        # restart QEMU anyway.
        super(FuchsiaQemuLibFuzzerRunner,
              self).__init__(executable_path=executable_path,
                             default_args=default_args)
        self._setup_device_and_fuzzer()

    def get_command(self, additional_args=None):
        # TODO(flowerhack): Update this to dynamically pick a result from "fuzz
        # list" and then run that fuzzer.
        return self.ssh_command('ls')

    def process_logs_and_crash(self, artifact_prefix):
        """Fetch symbolized logs and crashes."""
        if not artifact_prefix:
            return

        # Clusterfuzz assumes that the Libfuzzer output points to an absolute path,
        # where it can find the crash file.
        # This doesn't work in our case due to how Fuchsia is run.
        # So, we make a new file, change the appropriate line with a regex to point
        # to the true location. Apologies for the hackery.
        crash_location_regex = r'(.*)(Test unit written to )(data/.*)'
        _, processed_log_path = tempfile.mkstemp()
        with open(processed_log_path, 'w') as new_file:
            with open(self.fuzzer.logfile) as old_file:
                for line in old_file:
                    line_match = re.match(crash_location_regex, line)
                    if line_match:
                        # We now know the name of our crash file.
                        crash_name = line_match.group(3).replace('data/', '')
                        # Save the crash locally.
                        self.device.fetch(self.fuzzer.data_path(crash_name),
                                          artifact_prefix)
                        # Then update the crash report to point to that file.
                        crash_testcase_file_path = os.path.join(
                            artifact_prefix, crash_name)
                        line = re.sub(crash_location_regex,
                                      r'\1\2' + crash_testcase_file_path, line)
                    new_file.write(line)
        os.remove(self.fuzzer.logfile)
        shutil.move(processed_log_path, self.fuzzer.logfile)

    def _test_ssh(self):
        """Test the ssh connection."""
        # Test the connection.  If this works, proceed.
        # - If we fail, restart QEMU and test the connection again.
        # - If that fails, throw the error; we can't seem to recover.
        try:
            self._test_qemu_ssh()
        except fuchsia.errors.FuchsiaConnectionError:
            self._restart_qemu()
            self._test_qemu_ssh()

    @retry.wrap(retries=SSH_RETRIES, delay=SSH_WAIT, function='_test_qemu_ssh')
    def _test_qemu_ssh(self):
        """Tests that a VM is up and can be successfully SSH'd into.
    Raises an exception if no success after MAX_SSH_RETRIES."""
        ssh_test_process = new_process.ProcessRunner(
            'ssh',
            self.device.get_ssh_cmd(
                ['ssh', 'localhost', 'echo running on fuchsia!'])[1:])
        result = ssh_test_process.run_and_wait()
        if result.return_code or result.timed_out:
            raise fuchsia.errors.FuchsiaConnectionError(
                'Failed to establish initial SSH connection: ' +
                str(result.return_code) + " , " + str(result.command) + " , " +
                str(result.output))
        return result

    def _restart_qemu(self):
        """Restart QEMU."""
        process_handler.terminate_processes_matching_names(
            'qemu_system-x86_64')
        start_qemu()
        self._setup_device_and_fuzzer()

    def fuzz(self,
             corpus_directories,
             fuzz_timeout,
             artifact_prefix=None,
             additional_args=None,
             extra_env=None):
        """LibFuzzerCommon.fuzz override."""
        self._test_ssh()

        #TODO(flowerhack): Pass libfuzzer args (additional_args) here
        return_code = self.fuzzer.start(additional_args)
        self.fuzzer.monitor(return_code)
        self.process_logs_and_crash(artifact_prefix)

        with open(self.fuzzer.logfile) as logfile:
            symbolized_output = logfile.read()

        # TODO(flowerhack): Would be nice if we could figure out a way to make
        # the "fuzzer start" code return its own ProcessResult. For now, we simply
        # craft one by hand here.
        fuzzer_process_result = new_process.ProcessResult()
        fuzzer_process_result.return_code = 0
        fuzzer_process_result.output = symbolized_output
        fuzzer_process_result.time_executed = 0
        fuzzer_process_result.command = self.fuzzer.last_fuzz_cmd
        return fuzzer_process_result

    def run_single_testcase(self,
                            testcase_path,
                            timeout=None,
                            additional_args=None):
        """Run a single testcase."""
        self._test_ssh()

        # We need to push the testcase to the device and pass in the name.
        testcase_path_name = os.path.basename(os.path.normpath(testcase_path))
        self.device.store(testcase_path, self.fuzzer.data_path())

        # TODO(flowerhack): Pass libfuzzer args (additional_args) here
        return_code = self.fuzzer.start(
            ['repro', 'data/' + testcase_path_name] + additional_args)
        self.fuzzer.monitor(return_code)

        with open(self.fuzzer.logfile) as logfile:
            symbolized_output = logfile.read()

        fuzzer_process_result = new_process.ProcessResult()
        fuzzer_process_result.return_code = 0
        fuzzer_process_result.output = symbolized_output
        fuzzer_process_result.time_executed = 0
        fuzzer_process_result.command = self.fuzzer.last_fuzz_cmd
        return fuzzer_process_result

    def minimize_crash(self,
                       testcase_path,
                       output_path,
                       timeout,
                       artifact_prefix=None,
                       additional_args=None):
        return new_process.ProcessResult()

    def ssh_command(self, *args):
        return ['ssh'] + self.ssh_root + list(args)