Example #1
0
    def get(self):
        """Handle a get request."""
        try:
            grouper.group_testcases()
        except:
            logs.log_error('Error occurred while grouping test cases.')
            return

        # Free up memory after group task run.
        utils.python_gc()

        # Get a list of jobs excluded from bug filing.
        excluded_jobs = _get_excluded_jobs()

        # Get a list of all jobs. This is used to filter testcases whose jobs have
        # been removed.
        all_jobs = data_handler.get_all_job_type_names()

        for testcase_id in data_handler.get_open_testcase_id_iterator():
            try:
                testcase = data_handler.get_testcase_by_id(testcase_id)
            except errors.InvalidTestcaseError:
                # Already deleted.
                continue

            # Skip if testcase's job is removed.
            if testcase.job_type not in all_jobs:
                continue

            # Skip if testcase's job is in exclusions list.
            if testcase.job_type in excluded_jobs:
                continue

            # Skip if we are running progression task at this time.
            if testcase.get_metadata('progression_pending'):
                continue

            # If the testcase has a bug filed already, no triage is needed.
            if _is_bug_filed(testcase):
                continue

            # Check if the crash is important, i.e. it is either a reproducible crash
            # or an unreproducible crash happening frequently.
            if not _is_crash_important(testcase):
                continue

            # Require that all tasks like minimizaton, regression testing, etc have
            # finished.
            if not data_handler.critical_tasks_completed(testcase):
                continue

            # For testcases that are not part of a group, wait an additional time till
            # group task completes.
            # FIXME: In future, grouping might be dependent on regression range, so we
            # would have to add an additional wait time.
            if not testcase.group_id and not dates.time_has_expired(
                    testcase.timestamp,
                    hours=data_types.MIN_ELAPSED_TIME_SINCE_REPORT):
                continue

            # If this project does not have an associated issue tracker, we cannot
            # file this crash anywhere.
            issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(
                testcase)
            if not issue_tracker:
                issue_filer.notify_issue_update(testcase, 'new')
                continue

            # If there are similar issues to this test case already filed or recently
            # closed, skip filing a duplicate bug.
            if _check_and_update_similar_bug(testcase, issue_tracker):
                continue

            # Clean up old triage messages that would be not applicable now.
            testcase.delete_metadata(TRIAGE_MESSAGE_KEY, update_testcase=False)

            # File the bug first and then create filed bug metadata.
            try:
                issue_filer.file_issue(testcase, issue_tracker)
            except Exception:
                logs.log_error('Failed to file issue for testcase %d.' %
                               testcase_id)
                continue

            _create_filed_bug_metadata(testcase)
            issue_filer.notify_issue_update(testcase, 'new')

            logs.log('Filed new issue %s for testcase %d.' %
                     (testcase.bug_information, testcase_id))
Example #2
0
  def get(self):
    """Handles a GET request."""
    libfuzzer = data_types.Fuzzer.query(
        data_types.Fuzzer.name == 'libFuzzer').get()
    if not libfuzzer:
      logs.log_error('Failed to get libFuzzer Fuzzer entity.')
      return

    afl = data_types.Fuzzer.query(data_types.Fuzzer.name == 'afl').get()
    if not afl:
      logs.log_error('Failed to get AFL Fuzzer entity.')
      return

    honggfuzz = data_types.Fuzzer.query(
        data_types.Fuzzer.name == 'honggfuzz').get()
    if not honggfuzz:
      logs.log_error('Failed to get honggfuzz Fuzzer entity.')
      return

    project_setup_configs = local_config.ProjectConfig().get('project_setup')
    for setup_config in project_setup_configs:
      bucket_config = setup_config.get('build_buckets')

      if not bucket_config:
        raise ProjectSetupError('Project setup buckets not specified.')

      config = ProjectSetup(
          BUILD_BUCKET_PATH_TEMPLATE,
          REVISION_URL,
          setup_config.get('build_type'),
          config_suffix=setup_config.get('job_suffix', ''),
          segregate_projects=setup_config.get('segregate_projects', False),
          engine_build_buckets={
              'libfuzzer': bucket_config.get('libfuzzer'),
              'libfuzzer-i386': bucket_config.get('libfuzzer_i386'),
              'afl': bucket_config.get('afl'),
              'honggfuzz': bucket_config.get('honggfuzz'),
              'none': bucket_config.get('no_engine'),
              'dataflow': bucket_config.get('dataflow'),
          },
          fuzzer_entities={
              'libfuzzer': libfuzzer,
              'honggfuzz': honggfuzz,
              'afl': afl,
          },
          add_info_labels=setup_config.get('add_info_labels', False),
          add_revision_mappings=setup_config.get('add_revision_mappings',
                                                 False),
          additional_vars=setup_config.get('additional_vars'))

      projects_source = setup_config.get('source')
      if projects_source == 'oss-fuzz':
        projects = get_oss_fuzz_projects()
      elif projects_source.startswith(storage.GS_PREFIX):
        projects = get_projects_from_gcs(projects_source)
      else:
        raise ProjectSetupError('Invalid projects source: ' + projects_source)

      if not projects:
        raise ProjectSetupError('Missing projects list.')

      config.set_up(projects)
Example #3
0
    def _minimize_corpus_two_step(self, target_path, arguments,
                                  existing_corpus_dirs, new_corpus_dir,
                                  output_corpus_dir, reproducers_dir,
                                  max_time):
        """Optional (but recommended): run corpus minimization.

    Args:
      target_path: Path to the target.
      arguments: Additional arguments needed for corpus minimization.
      existing_corpus_dirs: Input corpora that existed before the fuzzing run.
      new_corpus_dir: Input corpus that was generated during the fuzzing run.
          Must have at least one new file.
      output_corpus_dir: Output directory to place minimized corpus.
      reproducers_dir: The directory to put reproducers in when crashes are
          found.
      max_time: Maximum allowed time for the minimization.

    Returns:
      A Result object.
    """
        if not _is_multistep_merge_supported(target_path):
            # Fallback to the old single step merge. It does not support incremental
            # stats and provides only `edge_coverage` and `feature_coverage` stats.
            logs.log(
                'Old version of libFuzzer is used. Using single step merge.')
            return self.minimize_corpus(
                target_path, arguments,
                existing_corpus_dirs + [new_corpus_dir], output_corpus_dir,
                reproducers_dir, max_time)

        # The dir where merge control file is located must persist for both merge
        # steps. The second step re-uses the MCF produced during the first step.
        merge_control_file_dir = self._create_temp_corpus_dir('mcf_tmp_dir')
        self._merge_control_file = os.path.join(merge_control_file_dir, 'MCF')

        # Two step merge process to obtain accurate stats for the new corpus units.
        # See https://reviews.llvm.org/D66107 for a more detailed description.
        merge_stats = {}

        # Step 1. Use only existing corpus and collect "initial" stats.
        result_1 = self.minimize_corpus(target_path, arguments,
                                        existing_corpus_dirs,
                                        output_corpus_dir, reproducers_dir,
                                        max_time)
        merge_stats['initial_edge_coverage'] = result_1.stats['edge_coverage']
        merge_stats['initial_feature_coverage'] = result_1.stats[
            'feature_coverage']

        # Clear the output dir as it does not have any new units at this point.
        engine_common.recreate_directory(output_corpus_dir)

        # Adjust the time limit for the time we spent on the first merge step.
        max_time -= result_1.time_executed
        if max_time <= 0:
            raise TimeoutError('Merging new testcases timed out\n' +
                               result_1.logs)

        # Step 2. Process the new corpus units as well.
        result_2 = self.minimize_corpus(
            target_path, arguments, existing_corpus_dirs + [new_corpus_dir],
            output_corpus_dir, reproducers_dir, max_time)
        merge_stats['edge_coverage'] = result_2.stats['edge_coverage']
        merge_stats['feature_coverage'] = result_2.stats['feature_coverage']

        # Diff the stats to obtain accurate values for the new corpus units.
        merge_stats['new_edges'] = (merge_stats['edge_coverage'] -
                                    merge_stats['initial_edge_coverage'])
        merge_stats['new_features'] = (merge_stats['feature_coverage'] -
                                       merge_stats['initial_feature_coverage'])

        output = result_1.logs + '\n\n' + result_2.logs
        if (merge_stats['new_edges'] < 0 or merge_stats['new_features'] < 0):
            logs.log_error('Two step merge failed.',
                           merge_stats=merge_stats,
                           output=output)
            merge_stats['new_edges'] = 0
            merge_stats['new_features'] = 0

        self._merge_control_file = None

        # TODO(ochang): Get crashes found during merge.
        return engine.FuzzResult(
            output, result_2.command, [], merge_stats,
            result_1.time_executed + result_2.time_executed)
Example #4
0
def update_fuzzer_and_data_bundles(fuzzer_name):
    """Update the fuzzer with a given name if necessary."""
    fuzzer = data_types.Fuzzer.query(
        data_types.Fuzzer.name == fuzzer_name).get()
    if not fuzzer:
        logs.log_error('No fuzzer exists with name %s.' % fuzzer_name)
        raise errors.InvalidFuzzerError

    # Set some helper environment variables.
    fuzzer_directory = get_fuzzer_directory(fuzzer_name)
    environment.set_value('FUZZER_DIR', fuzzer_directory)
    environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content)

    # Adjust the test timeout, if user has provided one.
    if fuzzer.timeout:
        environment.set_value('TEST_TIMEOUT', fuzzer.timeout)

        # Increase fuzz test timeout if the fuzzer timeout is higher than its
        # current value.
        fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT')
        if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout:
            environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout)

    # Adjust the max testcases if this fuzzer has specified a lower limit.
    max_testcases = environment.get_value('MAX_TESTCASES')
    if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases:
        environment.set_value('MAX_TESTCASES', fuzzer.max_testcases)

    # Check for updates to this fuzzer.
    version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name)
    if (not fuzzer.builtin
            and revisions.needs_update(version_file, fuzzer.revision)):
        logs.log('Fuzzer update was found, updating.')

        # Clear the old fuzzer directory if it exists.
        if not shell.remove_directory(fuzzer_directory, recreate=True):
            logs.log_error('Failed to clear fuzzer directory.')
            return False

        # Copy the archive to local disk and unpack it.
        archive_path = os.path.join(fuzzer_directory, fuzzer.filename)
        if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path):
            logs.log_error('Failed to copy fuzzer archive.')
            return False

        try:
            archive.unpack(archive_path, fuzzer_directory)
        except Exception:
            error_message = (
                'Failed to unpack fuzzer archive %s '
                '(bad archive or unsupported format).') % fuzzer.filename
            logs.log_error(error_message)
            fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
                                          fuzzer_name=fuzzer_name)
            return False

        fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path)
        if not os.path.exists(fuzzer_path):
            error_message = (
                'Fuzzer executable %s not found. '
                'Check fuzzer configuration.') % fuzzer.executable_path
            logs.log_error(error_message)
            fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
                                          fuzzer_name=fuzzer_name)
            return False

        # Make fuzzer executable.
        os.chmod(fuzzer_path, 0750)

        # Cleanup unneeded archive.
        shell.remove_file(archive_path)

        # Save the current revision of this fuzzer in a file for later checks.
        revisions.write_revision_to_revision_file(version_file,
                                                  fuzzer.revision)
        logs.log('Updated fuzzer to revision %d.' % fuzzer.revision)

    # Setup data bundles associated with this fuzzer.
    data_bundles = ndb_utils.get_all_from_query(
        data_types.DataBundle.query(
            data_types.DataBundle.name == fuzzer.data_bundle_name))
    for data_bundle in data_bundles:
        if not update_data_bundle(fuzzer, data_bundle):
            return False

    # Setup environment variable for launcher script path.
    if fuzzer.launcher_script:
        fuzzer_launcher_path = utils.get_launch_path_for_script(
            fuzzer_directory, fuzzer.launcher_script)
        environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path)

    return True
Example #5
0
def setup_testcase(testcase):
    """Sets up the testcase and needed dependencies like fuzzer,
  data bundle, etc."""
    fuzzer_name = testcase.fuzzer_name
    job_type = testcase.job_type
    task_name = environment.get_value('TASK_NAME')
    testcase_fail_wait = environment.get_value('FAIL_WAIT')
    testcase_id = testcase.key.id()

    # Clear testcase directories.
    shell.clear_testcase_directories()

    # Setup memory debugging tool environment.
    environment.reset_current_memory_tool_options(
        redzone_size=testcase.redzone)

    # Adjust the test timeout value if this is coming from an user uploaded
    # testcase.
    _set_timeout_value_from_user_upload(testcase_id)

    if task_name == 'minimize':
        # Allow minimizing with a different fuzzer set up.
        minimize_fuzzer_override = environment.get_value(
            'MINIMIZE_FUZZER_OVERRIDE')
        fuzzer_name = minimize_fuzzer_override or fuzzer_name

    # Update the fuzzer if necessary in order to get the updated data bundle.
    if fuzzer_name:
        try:
            update_successful = update_fuzzer_and_data_bundles(fuzzer_name)
        except errors.InvalidFuzzerError:
            # Close testcase and don't recreate tasks if this fuzzer is invalid.
            testcase.open = False
            testcase.fixed = 'NA'
            testcase.set_metadata('fuzzer_was_deleted', True)
            logs.log_error('Closed testcase %d with invalid fuzzer %s.' %
                           (testcase_id, fuzzer_name))

            error_message = 'Fuzzer %s no longer exists.' % fuzzer_name
            data_handler.update_testcase_comment(testcase,
                                                 data_types.TaskState.ERROR,
                                                 error_message)
            return None, None, None

        if not update_successful:
            error_message = 'Unable to setup fuzzer %s.' % fuzzer_name
            data_handler.update_testcase_comment(testcase,
                                                 data_types.TaskState.ERROR,
                                                 error_message)
            tasks.add_task(task_name,
                           testcase_id,
                           job_type,
                           wait_time=testcase_fail_wait)
            return None, None, None

    # Extract the testcase and any of its resources to the input directory.
    file_list, input_directory, testcase_file_path = unpack_testcase(testcase)
    if not file_list:
        error_message = 'Unable to setup testcase %s.' % testcase_file_path
        data_handler.update_testcase_comment(testcase,
                                             data_types.TaskState.ERROR,
                                             error_message)
        tasks.add_task(task_name,
                       testcase_id,
                       job_type,
                       wait_time=testcase_fail_wait)
        return None, None, None

    # For Android/Fuchsia, we need to sync our local testcases directory with the
    # one on the device.
    if environment.platform() == 'ANDROID':
        _copy_testcase_to_device_and_setup_environment(testcase,
                                                       testcase_file_path)

    if environment.platform() == 'FUCHSIA':
        fuchsia.device.copy_testcase_to_device(testcase_file_path)

    # Push testcases to worker.
    if environment.is_trusted_host():
        from bot.untrusted_runner import file_host
        file_host.push_testcases_to_worker()

    # Copy global blacklist into local blacklist.
    is_lsan_enabled = environment.get_value('LSAN')
    if is_lsan_enabled:
        # Get local blacklist without this testcase's entry.
        leak_blacklist.copy_global_to_local_blacklist(
            excluded_testcase=testcase)

    # Setup environment variable for windows size and location properties.
    # Explicitly use empty string to indicate use of default window properties.
    if hasattr(testcase, 'window_argument'):
        environment.set_value('WINDOW_ARG', testcase.window_argument)

    # Adjust timeout based on the stored multiplier (if available).
    if hasattr(testcase, 'timeout_multiplier') and testcase.timeout_multiplier:
        test_timeout = environment.get_value('TEST_TIMEOUT')
        environment.set_value('TEST_TIMEOUT',
                              int(test_timeout * testcase.timeout_multiplier))

    # Override APP_ARGS with minimized arguments (if available).
    if (hasattr(testcase, 'minimized_arguments')
            and testcase.minimized_arguments):
        environment.set_value('APP_ARGS', testcase.minimized_arguments)

    # Add FUZZ_TARGET to environment if this is a fuzz target testcase.
    fuzz_target = testcase.get_metadata('fuzzer_binary_name')
    if fuzz_target:
        environment.set_value('FUZZ_TARGET', fuzz_target)

    return file_list, input_directory, testcase_file_path
Example #6
0
def main():
    """Prepare the configuration options and start requesting tasks."""
    logs.configure('run_bot')

    root_directory = environment.get_value('ROOT_DIR')
    if not root_directory:
        print(
            'Please set ROOT_DIR environment variable to the root of the source '
            'checkout before running. Exiting.')
        print('For an example, check init.bash in the local directory.')
        return

    dates.initialize_timezone_from_environment()
    environment.set_bot_environment()
    monitor.initialize()

    if not profiler.start_if_needed('python_profiler_bot'):
        sys.exit(-1)

    fuzzers_init.run()

    if environment.is_trusted_host(ensure_connected=False):
        from bot.untrusted_runner import host
        host.init()

    if environment.is_untrusted_worker():
        # Track revision since we won't go into the task_loop.
        update_task.track_revision()

        from bot.untrusted_runner import untrusted as untrusted_worker
        untrusted_worker.start_server()
        assert False, 'Unreachable code'

    while True:
        # task_loop should be an infinite loop,
        # unless we run into an exception.
        error_stacktrace, clean_exit, task_payload = task_loop()

        # Print the error trace to the console.
        if not clean_exit:
            print('Exception occurred while running "%s".' % task_payload)
            print('-' * 80)
            print(error_stacktrace)
            print('-' * 80)

        should_terminate = (clean_exit or errors.error_in_list(
            error_stacktrace, errors.BOT_ERROR_TERMINATION_LIST))
        if should_terminate:
            return

        logs.log_error('Task exited with exception (payload="%s").' %
                       task_payload,
                       error_stacktrace=error_stacktrace)

        should_hang = errors.error_in_list(error_stacktrace,
                                           errors.BOT_ERROR_HANG_LIST)
        if should_hang:
            logs.log('Start hanging forever.')
            while True:
                # Sleep to avoid consuming 100% of CPU.
                time.sleep(60)

        # See if our run timed out, if yes bail out.
        if data_handler.bot_run_timed_out():
            return

        # Clear the current exception.
        utils.exc_clear()
Example #7
0
def setup_testcase(testcase, job_type, fuzzer_override=None):
    """Sets up the testcase and needed dependencies like fuzzer,
  data bundle, etc."""
    fuzzer_name = fuzzer_override or testcase.fuzzer_name
    task_name = environment.get_value('TASK_NAME')
    testcase_fail_wait = environment.get_value('FAIL_WAIT')
    testcase_id = testcase.key.id()

    # Clear testcase directories.
    shell.clear_testcase_directories()

    # Adjust the test timeout value if this is coming from an user uploaded
    # testcase.
    _set_timeout_value_from_user_upload(testcase_id)

    # Update the fuzzer if necessary in order to get the updated data bundle.
    if fuzzer_name:
        try:
            update_successful = update_fuzzer_and_data_bundles(fuzzer_name)
        except errors.InvalidFuzzerError:
            # Close testcase and don't recreate tasks if this fuzzer is invalid.
            testcase.open = False
            testcase.fixed = 'NA'
            testcase.set_metadata('fuzzer_was_deleted', True)
            logs.log_error('Closed testcase %d with invalid fuzzer %s.' %
                           (testcase_id, fuzzer_name))

            error_message = 'Fuzzer %s no longer exists' % fuzzer_name
            data_handler.update_testcase_comment(testcase,
                                                 data_types.TaskState.ERROR,
                                                 error_message)
            return None, None, None

        if not update_successful:
            error_message = 'Unable to setup fuzzer %s' % fuzzer_name
            data_handler.update_testcase_comment(testcase,
                                                 data_types.TaskState.ERROR,
                                                 error_message)
            tasks.add_task(task_name,
                           testcase_id,
                           job_type,
                           wait_time=testcase_fail_wait)
            return None, None, None

    # Extract the testcase and any of its resources to the input directory.
    file_list, input_directory, testcase_file_path = unpack_testcase(testcase)
    if not file_list:
        error_message = 'Unable to setup testcase %s' % testcase_file_path
        data_handler.update_testcase_comment(testcase,
                                             data_types.TaskState.ERROR,
                                             error_message)
        tasks.add_task(task_name,
                       testcase_id,
                       job_type,
                       wait_time=testcase_fail_wait)
        return None, None, None

    # For Android/Fuchsia, we need to sync our local testcases directory with the
    # one on the device.
    if environment.platform() == 'ANDROID':
        _copy_testcase_to_device_and_setup_environment(testcase,
                                                       testcase_file_path)

    # Push testcases to worker.
    if environment.is_trusted_host():
        from bot.untrusted_runner import file_host
        file_host.push_testcases_to_worker()

    # Copy global blacklist into local blacklist.
    is_lsan_enabled = environment.get_value('LSAN')
    if is_lsan_enabled:
        # Get local blacklist without this testcase's entry.
        leak_blacklist.copy_global_to_local_blacklist(
            excluded_testcase=testcase)

    prepare_environment_for_testcase(testcase, job_type, task_name)

    return file_list, input_directory, testcase_file_path
Example #8
0
def wait_for_battery_charge_if_needed():
  """Check device battery and make sure it is charged beyond minimum level and
  temperature thresholds."""
  # Battery levels are not applicable on GCE.
  if adb.is_gce():
    return

  # Make sure device is online.
  adb.wait_for_device()

  # Skip battery check if done recently.
  last_battery_check_time = persistent_cache.get_value(
      LAST_BATTERY_CHECK_TIME_KEY,
      constructor=datetime.datetime.utcfromtimestamp)
  if last_battery_check_time and not dates.time_has_expired(
      last_battery_check_time, seconds=BATTERY_CHECK_INTERVAL):
    return

  # Initialize variables.
  battery_level_threshold = environment.get_value('LOW_BATTERY_LEVEL_THRESHOLD',
                                                  LOW_BATTERY_LEVEL_THRESHOLD)
  battery_temperature_threshold = environment.get_value(
      'MAX_BATTERY_TEMPERATURE_THRESHOLD', MAX_BATTERY_TEMPERATURE_THRESHOLD)
  device_restarted = False

  while 1:
    battery_information = get_battery_information()
    if battery_information is None:
      logs.log_error('Failed to get battery information, skipping check.')
      return

    battery_level = battery_information['level']
    battery_temperature = battery_information['temperature']
    logs.log('Battery information: level (%d%%), temperature (%.1f celsius).' %
             (battery_level, battery_temperature))
    if (battery_level >= battery_level_threshold and
        battery_temperature <= battery_temperature_threshold):
      persistent_cache.set_value(LAST_BATTERY_CHECK_TIME_KEY, time.time())
      return

    logs.log('Battery in bad battery state, putting device in sleep mode.')

    if not device_restarted:
      reboot()
      adb.disable_wifi()
      device_restarted = True

    # Change thresholds to expected levels (only if they were below minimum
    # thresholds).
    if battery_level < battery_level_threshold:
      battery_level_threshold = EXPECTED_BATTERY_LEVEL
    if battery_temperature > battery_temperature_threshold:
      battery_temperature_threshold = EXPECTED_BATTERY_TEMPERATURE

    # Stopping shell should help with shutting off a lot of services that would
    # otherwise use up the battery. However, we need to turn it back on to get
    # battery status information. Also, turn off display explicitly (needed for
    # Nexus 9s).
    turn_off_display_if_needed()
    adb.stop_shell()
    time.sleep(BATTERY_CHARGE_INTERVAL)
    adb.start_shell()
Example #9
0
          '*.txt',
          '-x',
          '*-expected.*',
          '-x',
          '*.git*',
          '-x',
          '*.svn*',
      ],
      cwd=tests_directory)
  subprocess.check_call(
      ['gsutil', 'cp', tests_archive_local, tests_archive_remote])

  logs.log('Completed cycle, sleeping for %s seconds.' % sync_interval)
  time.sleep(sync_interval)


if __name__ == '__main__':
  # Make sure environment is correctly configured.
  logs.configure('run_bot')
  environment.set_bot_environment()

  fail_wait = environment.get_value('FAIL_WAIT')

  # Continue this forever.
  while 1:
    try:
      main()
    except Exception:
      logs.log_error('Failed to sync tests.')
      time.sleep(fail_wait)
Example #10
0
def flash_to_latest_build_if_needed():
  """Wipes user data, resetting the device to original factory state."""
  run_timeout = environment.get_value('RUN_TIMEOUT')
  if run_timeout:
    # If we have a run timeout, then we are already scheduled to bail out and
    # will be probably get re-imaged. E.g. using frameworks like Tradefed.
    return

  # Check if a flash is needed based on last recorded flash time.
  last_flash_time = persistent_cache.get_value(
      LAST_FLASH_TIME_KEY, constructor=datetime.datetime.utcfromtimestamp)
  needs_flash = last_flash_time is None or dates.time_has_expired(
      last_flash_time, seconds=adb.FLASH_INTERVAL)
  if not needs_flash:
    return

  build_info = {}
  is_remote_device = environment.get_value('ANDROID_GCE')
  if is_remote_device:
    # To prevent thousands of devices all trying to reimage at the same time,
    # reimages are done at a random time in the future.
    scheduled_reimage_time = persistent_cache.get_value(
        SCHEDULED_GCE_REIMAGE_TIME_KEY,
        constructor=datetime.datetime.utcfromtimestamp)

    if scheduled_reimage_time is None:
      # No reimage scheduled yet, so we need to do so.
      delay = random.randint(0, 3600)
      reimage_time = int(time.time()) + delay
      logs.log('Scheduling a new reimage in %d seconds.' % delay)
      persistent_cache.set_value(SCHEDULED_GCE_REIMAGE_TIME_KEY, reimage_time)
      return

    current_time = datetime.datetime.utcnow()
    if current_time < scheduled_reimage_time:
      time_left = scheduled_reimage_time - current_time
      # Not yet time for the reimage.
      logs.log('Scheduled reimage in %d seconds.' % time_left.seconds)
      return

    # Recreating the virtual device will reimage this to the latest image
    # available (with retry logic).
    logs.log('Reimaging device to latest image.')
    if not adb.recreate_virtual_device():
      logs.log_error('Unable to recreate virtual device. Reimaging failed.')
      adb.bad_state_reached()

  else:
    is_google_device = google_device()
    if is_google_device is None:
      logs.log_error('Unable to query device. Reimaging failed.')
      adb.bad_state_reached()

    if not is_google_device:
      # We can't reimage these, skip.
      logs.log('Non-Google device found, skipping reimage.')
      return

    else:
      # For Google devices.
      # Check if both |BUILD_BRANCH| and |BUILD_TARGET| environment variables
      # are set. If not, we don't have enough data for reimaging and hence
      # we bail out.
      branch = environment.get_value('BUILD_BRANCH')
      target = environment.get_value('BUILD_TARGET')
      if not target:
        # We default to userdebug configuration.
        build_params = get_build_parameters()
        if build_params:
          target = build_params.get('target') + '-userdebug'

          # Cache target in environment. This is also useful for cases when
          # device is bricked and we don't have this information available.
          environment.set_value('BUILD_TARGET', target)

      if not branch or not target:
        return

      # Download the latest build artifact for this branch and target.
      build_info = fetch_artifact.get_latest_artifact_info(branch, target)
      if not build_info:
        logs.log_error(
            'Unable to fetch information on latest build artifact for '
            'branch %s and target %s.' % (branch, target))
        return

      # Check if our local build matches the latest build. If not, we will
      # download it.
      build_id = build_info['bid']
      target = build_info['target']
      image_directory = environment.get_value('IMAGES_DIR')
      last_build_info = persistent_cache.get_value(LAST_FLASH_BUILD_KEY)
      if not last_build_info or last_build_info['bid'] != build_id:
        # Clean up the images directory first.
        shell.remove_directory(image_directory, recreate=True)

        # We have a new build, download the build artifacts for it.
        for image_regex in FLASH_IMAGE_REGEXES:
          image_file_path = fetch_artifact.get(build_id, target, image_regex,
                                               image_directory)
          if not image_file_path:
            logs.log_error(
                'Failed to download image artifact %s for '
                'branch %s and target %s.' % (image_file_path, branch, target))
            return
          if image_file_path.endswith('.zip'):
            archive.unpack(image_file_path, image_directory)

      # We do one device flash at a time on one host, otherwise we run into
      # failures and device being stuck in a bad state.
      flash_lock_key_name = 'flash:%s' % socket.gethostname()
      if not locks.acquire_lock(flash_lock_key_name, by_zone=True):
        logs.log_error('Failed to acquire lock for reimaging, exiting.')
        return

      logs.log('Reimaging started.')
      logs.log('Rebooting into bootloader mode.')
      for _ in xrange(FLASH_RETRIES):
        adb.run_as_root()
        adb.run_adb_command(['reboot-bootloader'])
        time.sleep(FLASH_REBOOT_BOOTLOADER_WAIT)
        adb.run_fastboot_command(['oem', 'off-mode-charge', '0'])
        adb.run_fastboot_command(['-w', 'reboot-bootloader'])

        for partition, partition_image_filename in FLASH_IMAGE_FILES:
          partition_image_file_path = os.path.join(image_directory,
                                                   partition_image_filename)
          adb.run_fastboot_command(
              ['flash', partition, partition_image_file_path])
          if partition in ['bootloader', 'radio']:
            adb.run_fastboot_command(['reboot-bootloader'])
        adb.run_fastboot_command('reboot')
        time.sleep(FLASH_REBOOT_WAIT)

        if adb.get_device_state() == 'device':
          break
        logs.log_error('Reimaging failed, retrying.')

      locks.release_lock(flash_lock_key_name, by_zone=True)

  if adb.get_device_state() != 'device':
    logs.log_error('Unable to find device. Reimaging failed.')
    adb.bad_state_reached()

  logs.log('Reimaging finished.')

  # Reset all of our persistent keys after wipe.
  persistent_cache.delete_value(BUILD_PROP_MD5_KEY)
  persistent_cache.delete_value(LAST_TEST_ACCOUNT_CHECK_KEY)
  persistent_cache.set_value(LAST_FLASH_BUILD_KEY, build_info)
  persistent_cache.set_value(LAST_FLASH_TIME_KEY, time.time())

  if is_remote_device:
    persistent_cache.delete_value(SCHEDULED_GCE_REIMAGE_TIME_KEY)
Example #11
0
def configure_build_properties_if_needed():
  """Edits /system/build.prop for better boot speed and power use."""
  # Check md5 checksum of build.prop to see if already updated,
  # in which case exit. If build.prop does not exist, something
  # is very wrong with the device, so bail.
  old_md5 = persistent_cache.get_value(BUILD_PROP_MD5_KEY)
  current_md5 = adb.get_file_checksum(BUILD_PROP_PATH)
  if current_md5 is None:
    logs.log_error('Unable to find %s on device.' % BUILD_PROP_PATH)
    return
  if old_md5 == current_md5:
    return

  # Pull to tmp file.
  bot_tmp_directory = environment.get_value('BOT_TMPDIR')
  old_build_prop_path = os.path.join(bot_tmp_directory, 'old.prop')
  adb.run_adb_command(['pull', BUILD_PROP_PATH, old_build_prop_path])
  if not os.path.exists(old_build_prop_path):
    logs.log_error('Unable to fetch %s from device.' % BUILD_PROP_PATH)
    return

  # Write new build.prop.
  new_build_prop_path = os.path.join(bot_tmp_directory, 'new.prop')
  old_build_prop_file_content = open(old_build_prop_path, 'r')
  new_build_prop_file_content = open(new_build_prop_path, 'w')
  new_content_notification = '### CHANGED OR ADDED PROPERTIES ###'
  for line in old_build_prop_file_content:
    property_name = line.split('=')[0].strip()
    if property_name in BUILD_PROPERTIES:
      continue
    if new_content_notification in line:
      continue
    new_build_prop_file_content.write(line)

  new_build_prop_file_content.write(new_content_notification + '\n')
  for flag, value in BUILD_PROPERTIES.iteritems():
    new_build_prop_file_content.write('%s=%s\n' % (flag, value))
  old_build_prop_file_content.close()
  new_build_prop_file_content.close()

  # Keep verified boot disabled for M and higher releases. This makes it easy
  # to modify system's app_process to load asan libraries.
  build_version = get_build_version()
  if is_build_at_least(build_version, 'M'):
    adb.run_as_root()
    adb.run_adb_command('disable-verity')
    reboot()

  # Make /system writable.
  adb.run_as_root()
  adb.remount()

  # Remove seccomp policies (on N and higher) as ASan requires extra syscalls.
  if is_build_at_least(build_version, 'N'):
    policy_files = adb.run_adb_shell_command(
        ['find', '/system/etc/seccomp_policy/', '-type', 'f'])
    for policy_file in policy_files.splitlines():
      adb.run_adb_shell_command(['rm', policy_file.strip()])

  # Remove Google Plus app from non-Google devices. Makes it easy to install
  # older Gallery app on these devices. Otherwise, we run into duplicate
  # permission errors.
  if not google_device():
    adb.run_adb_shell_command(['rm', '/system/app/PlusOne.apk'])
    adb.run_adb_shell_command(['rm', '/system/app/PlusOne/PlusOne.apk'])

  # Push new build.prop and backup to device.
  logs.log('Pushing new build properties file on device.')
  adb.run_adb_command(
      ['push', '-p', old_build_prop_path, BUILD_PROP_BACKUP_PATH])
  adb.run_adb_command(['push', '-p', new_build_prop_path, BUILD_PROP_PATH])
  adb.run_adb_shell_command(['chmod', '644', BUILD_PROP_PATH])

  # Set persistent cache key containing and md5sum.
  current_md5 = adb.get_file_checksum(BUILD_PROP_PATH)
  persistent_cache.set_value(BUILD_PROP_MD5_KEY, current_md5)
Example #12
0
def get_component_revisions_dict(revision, job_type):
    """Retrieve revision vars dict."""
    if revision == 0 or revision == '0' or revision is None:
        # Return empty dict for zero start revision.
        return {}

    config = db_config.get()
    component = data_handler.get_component_name(job_type)
    project_name = data_handler.get_project_name(job_type)

    revision_info_url_format = db_config.get_value_for_job(
        config.revision_vars_url, job_type)
    if not revision_info_url_format:
        return None

    revisions_dict = {}

    repository = data_handler.get_repository_for_component(component)

    if repository and not _is_clank(revision_info_url_format):
        revision_hash = _git_commit_position_to_git_hash_for_chromium(
            revision, repository)
        if revision_hash is None:
            return None

        # FIXME: While we check for this explicitly appended component in all
        # applicable cases that we know of within this codebase, if the dict
        # is shared with an external service (e.g. Predator) we may need to clean
        # this up beforehand.
        revisions_dict['/src'] = {
            'name': _get_component_display_name(component, project_name),
            'url': _git_url_for_chromium_repository(repository),
            'rev': revision_hash,
            'commit_pos': revision
        }

        # Use revision hash for info url later.
        revision = revision_hash

    revision_info_url = revision_info_url_format % revision
    url_content = _get_url_content(revision_info_url)
    if not url_content:
        logs.log_error('Failed to get component revisions from %s.' %
                       revision_info_url)
        return None

    # Parse as per DEPS format.
    if _is_deps(revision_info_url):
        deps_revisions_dict = deps_to_revisions_dict(url_content)
        if not deps_revisions_dict:
            return None

        revisions_dict.update(deps_revisions_dict)
        return revisions_dict

    # Parse as per Clank DEPS format.
    if _is_clank(revision_info_url):
        return _clank_revision_file_to_revisions_dict(url_content)

    # Default case: parse content as yaml.
    revisions_dict = _to_yaml_dict(url_content)
    if not revisions_dict:
        logs.log_error('Failed to parse component revisions from %s.' %
                       revision_info_url)
        return None

    # Parse as per source map format.
    if revision_info_url.endswith(SOURCE_MAP_EXTENSION):
        revisions_dict = _src_map_to_revisions_dict(revisions_dict,
                                                    project_name)

    return revisions_dict
Example #13
0
def main(argv):
    """Run libFuzzer as specified by argv."""
    atexit.register(fuzzer_utils.cleanup)

    # Initialize variables.
    arguments = argv[1:]
    testcase_file_path = arguments.pop(0)
    target_name = arguments.pop(0)
    fuzzer_name = data_types.fuzz_target_project_qualified_name(
        utils.current_project(), target_name)

    # Initialize log handler.
    logs.configure(
        'run_fuzzer', {
            'fuzzer': fuzzer_name,
            'engine': 'libFuzzer',
            'job_name': environment.get_value('JOB_NAME')
        })

    profiler.start_if_needed('libfuzzer_launcher')

    # Make sure that the fuzzer binary exists.
    build_directory = environment.get_value('BUILD_DIR')
    fuzzer_path = engine_common.find_fuzzer_path(build_directory, target_name)
    if not fuzzer_path:
        return

    # Install signal handler.
    signal.signal(signal.SIGTERM, engine_common.signal_term_handler)

    # Set up temp dir.
    engine_common.recreate_directory(fuzzer_utils.get_temp_dir())

    # Setup minijail if needed.
    use_minijail = environment.get_value('USE_MINIJAIL')
    runner = libfuzzer.get_runner(fuzzer_path,
                                  temp_dir=fuzzer_utils.get_temp_dir())

    if use_minijail:
        minijail_chroot = runner.chroot
    else:
        minijail_chroot = None

    # Get corpus directory.
    corpus_directory = environment.get_value('FUZZ_CORPUS_DIR')

    # Add common arguments which are necessary to be used for every run.
    arguments = expand_with_common_arguments(arguments)

    # Add sanitizer options to environment that were specified in the .options
    # file and options that this script requires.
    set_sanitizer_options(fuzzer_path)

    # If we don't have a corpus, then that means this is not a fuzzing run.
    # TODO(flowerhack): Implement this to properly load past testcases.
    if not corpus_directory and environment.platform() != 'FUCHSIA':
        load_testcase_if_exists(runner, testcase_file_path, fuzzer_name,
                                use_minijail, arguments)
        return

    # We don't have a crash testcase, fuzz.

    # Check dict argument to make sure that it's valid.
    dict_argument = fuzzer_utils.extract_argument(arguments,
                                                  constants.DICT_FLAG,
                                                  remove=False)
    if dict_argument and not os.path.exists(dict_argument):
        logs.log_error('Invalid dict %s for %s.' %
                       (dict_argument, fuzzer_name))
        fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG)

    # If there's no dict argument, check for %target_binary_name%.dict file.
    if (not fuzzer_utils.extract_argument(
            arguments, constants.DICT_FLAG, remove=False)):
        default_dict_path = dictionary_manager.get_default_dictionary_path(
            fuzzer_path)
        if os.path.exists(default_dict_path):
            arguments.append(constants.DICT_FLAG + default_dict_path)

    # Set up scratch directory for writing new units.
    new_testcases_directory = create_corpus_directory('new')

    # Strategy pool is the list of strategies that we attempt to enable, whereas
    # fuzzing strategies is the list of strategies that are enabled. (e.g. if
    # mutator is selected in the pool, but not available for a given target, it
    # would not be added to fuzzing strategies.)
    strategy_pool = strategy_selection.generate_weighted_strategy_pool(
        strategy_list=strategy.LIBFUZZER_STRATEGY_LIST,
        use_generator=True,
        engine_name='libFuzzer')
    strategy_info = pick_strategies(strategy_pool,
                                    fuzzer_path,
                                    corpus_directory,
                                    arguments,
                                    minijail_chroot=minijail_chroot)
    arguments.extend(strategy_info.arguments)

    # Timeout for fuzzer run.
    fuzz_timeout = get_fuzz_timeout(strategy_info.is_mutations_run)

    # Get list of corpus directories.
    # TODO(flowerhack): Implement this to handle corpus sync'ing.
    if environment.platform() == 'FUCHSIA':
        corpus_directories = []
    else:
        corpus_directories = get_corpus_directories(
            corpus_directory,
            new_testcases_directory,
            fuzzer_path,
            strategy_info.fuzzing_strategies,
            strategy_pool,
            minijail_chroot=minijail_chroot,
            allow_corpus_subset=not strategy_info.use_dataflow_tracing)

    corpus_directories.extend(strategy_info.additional_corpus_dirs)

    # Bind corpus directories in minijail.
    artifact_prefix = os.path.abspath(os.path.dirname(testcase_file_path))
    # Execute the fuzzer binary with original arguments.
    fuzz_result = runner.fuzz(corpus_directories,
                              fuzz_timeout=fuzz_timeout,
                              artifact_prefix=artifact_prefix,
                              additional_args=arguments,
                              extra_env=strategy_info.extra_env)

    if (not use_minijail
            and fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE):
        # Minijail returns 1 if the exit code is nonzero.
        # Otherwise: we can assume that a return code of 1 means that libFuzzer
        # itself ran into an error.
        logs.log_error(ENGINE_ERROR_MESSAGE, engine_output=fuzz_result.output)

    log_lines = fuzz_result.output.splitlines()
    # Output can be large, so save some memory by removing reference to the
    # original output which is no longer needed.
    fuzz_result.output = None

    # Check if we crashed, and get the crash testcase path.
    crash_testcase_file_path = runner.get_testcase_path(log_lines)
    if crash_testcase_file_path:
        # Copy crash testcase contents into the main testcase path.
        shutil.move(crash_testcase_file_path, testcase_file_path)

    # Print the command output.
    bot_name = environment.get_value('BOT_NAME', '')
    command = fuzz_result.command
    if use_minijail:
        # Remove minijail prefix.
        command = engine_common.strip_minijail_command(command, fuzzer_path)
    print(
        engine_common.get_log_header(command, bot_name,
                                     fuzz_result.time_executed))

    # Parse stats information based on libFuzzer output.
    parsed_stats = parse_log_stats(log_lines)

    # Extend parsed stats by additional performance features.
    parsed_stats.update(
        stats.parse_performance_features(log_lines,
                                         strategy_info.fuzzing_strategies,
                                         arguments))

    # Set some initial stat overrides.
    timeout_limit = fuzzer_utils.extract_argument(arguments,
                                                  constants.TIMEOUT_FLAG,
                                                  remove=False)

    expected_duration = runner.get_max_total_time(fuzz_timeout)
    actual_duration = int(fuzz_result.time_executed)
    fuzzing_time_percent = 100 * actual_duration / float(expected_duration)
    stat_overrides = {
        'timeout_limit': int(timeout_limit),
        'expected_duration': expected_duration,
        'actual_duration': actual_duration,
        'fuzzing_time_percent': fuzzing_time_percent,
    }

    # Remove fuzzing arguments before merge and dictionary analysis step.
    remove_fuzzing_arguments(arguments)

    # Make a decision on whether merge step is needed at all. If there are no
    # new units added by libFuzzer run, then no need to do merge at all.
    new_units_added = shell.get_directory_file_count(new_testcases_directory)
    merge_error = None
    if new_units_added:
        # Merge the new units with the initial corpus.
        if corpus_directory not in corpus_directories:
            corpus_directories.append(corpus_directory)

        # If this times out, it's possible that we will miss some units. However, if
        # we're taking >10 minutes to load/merge the corpus something is going very
        # wrong and we probably don't want to make things worse by adding units
        # anyway.

        merge_tmp_dir = None
        if not use_minijail:
            merge_tmp_dir = os.path.join(fuzzer_utils.get_temp_dir(),
                                         'merge_workdir')
            engine_common.recreate_directory(merge_tmp_dir)

        old_corpus_len = shell.get_directory_file_count(corpus_directory)
        merge_directory = create_merge_directory()
        corpus_directories.insert(0, merge_directory)

        if use_minijail:
            bind_corpus_dirs(minijail_chroot, [merge_directory])

        merge_result = runner.merge(
            corpus_directories,
            merge_timeout=engine_common.get_merge_timeout(
                DEFAULT_MERGE_TIMEOUT),
            tmp_dir=merge_tmp_dir,
            additional_args=arguments)

        move_mergeable_units(merge_directory, corpus_directory)
        new_corpus_len = shell.get_directory_file_count(corpus_directory)
        new_units_added = 0

        merge_error = None
        if merge_result.timed_out:
            merge_error = 'Merging new testcases timed out:'
        elif merge_result.return_code != 0:
            merge_error = 'Merging new testcases failed:'
        else:
            new_units_added = new_corpus_len - old_corpus_len

        stat_overrides['new_units_added'] = new_units_added

        if merge_result.output:
            stat_overrides.update(
                stats.parse_stats_from_merge_log(
                    merge_result.output.splitlines()))
    else:
        stat_overrides['new_units_added'] = 0
        logs.log('Skipped corpus merge since no new units added by fuzzing.')

    # Get corpus size after merge. This removes the duplicate units that were
    # created during this fuzzing session.
    # TODO(flowerhack): Remove this workaround once we can handle corpus sync.
    if environment.platform() != 'FUCHSIA':
        stat_overrides['corpus_size'] = shell.get_directory_file_count(
            corpus_directory)

    # Delete all corpus directories except for the main one. These were temporary
    # directories to store new testcase mutations and have already been merged to
    # main corpus directory.
    if corpus_directory in corpus_directories:
        corpus_directories.remove(corpus_directory)
    for directory in corpus_directories:
        shutil.rmtree(directory, ignore_errors=True)

    if use_minijail:
        unbind_corpus_dirs(minijail_chroot, corpus_directories)

    # Apply overridden stats to the parsed stats prior to dumping.
    parsed_stats.update(stat_overrides)

    # Dump stats data for further uploading to BigQuery.
    engine_common.dump_big_query_data(parsed_stats, testcase_file_path,
                                      command)

    # Add custom crash state based on fuzzer name (if needed).
    add_custom_crash_state_if_needed(fuzzer_name, log_lines, parsed_stats)
    for line in log_lines:
        print(line)

    # Add fuzzing strategies used.
    print(
        engine_common.format_fuzzing_strategies(
            strategy_info.fuzzing_strategies))

    # Add merge error (if any).
    if merge_error:
        print(data_types.CRASH_STACKTRACE_END_MARKER)
        print(merge_error)
        print(
            'Command:',
            get_printable_command(merge_result.command, fuzzer_path,
                                  use_minijail))
        print(merge_result.output)

    analyze_and_update_recommended_dictionary(runner, fuzzer_name, log_lines,
                                              corpus_directory, arguments)

    # Close minijail chroot.
    if use_minijail:
        minijail_chroot.close()

    # Record the stats to make them easily searchable in stackdriver.
    if new_units_added:
        logs.log('New units added to corpus: %d.' % new_units_added,
                 stats=parsed_stats)
    else:
        logs.log('No new units found.', stats=parsed_stats)
Example #14
0
def pick_strategies(strategy_pool,
                    fuzzer_path,
                    corpus_directory,
                    existing_arguments,
                    minijail_chroot=None):
    """Pick strategies."""
    build_directory = environment.get_value('BUILD_DIR')
    target_name = os.path.basename(fuzzer_path)
    project_qualified_fuzzer_name = data_types.fuzz_target_project_qualified_name(
        utils.current_project(), target_name)

    fuzzing_strategies = []
    arguments = []
    additional_corpus_dirs = []

    # Select a generator to attempt to use for existing testcase mutations.
    candidate_generator = engine_common.select_generator(
        strategy_pool, fuzzer_path)
    is_mutations_run = candidate_generator != engine_common.Generator.NONE

    # Depends on the presense of DFSan instrumented build.
    dataflow_build_dir = environment.get_value('DATAFLOW_BUILD_DIR')
    use_dataflow_tracing = (dataflow_build_dir and strategy_pool.do_strategy(
        strategy.DATAFLOW_TRACING_STRATEGY))
    if use_dataflow_tracing:
        dataflow_binary_path = os.path.join(
            dataflow_build_dir, os.path.relpath(fuzzer_path, build_directory))
        if os.path.exists(dataflow_binary_path):
            arguments.append(
                '%s%s' %
                (constants.COLLECT_DATA_FLOW_FLAG, dataflow_binary_path))
            fuzzing_strategies.append(strategy.DATAFLOW_TRACING_STRATEGY.name)
        else:
            logs.log_error(
                'Fuzz target is not found in dataflow build, skiping strategy.'
            )
            use_dataflow_tracing = False

    # Generate new testcase mutations using radamsa, etc.
    if is_mutations_run:
        new_testcase_mutations_directory = create_corpus_directory('mutations')
        generator_used = engine_common.generate_new_testcase_mutations(
            corpus_directory, new_testcase_mutations_directory,
            project_qualified_fuzzer_name, candidate_generator)

        # Add the used generator strategy to our fuzzing strategies list.
        if generator_used:
            if candidate_generator == engine_common.Generator.RADAMSA:
                fuzzing_strategies.append(
                    strategy.CORPUS_MUTATION_RADAMSA_STRATEGY.name)
            elif candidate_generator == engine_common.Generator.ML_RNN:
                fuzzing_strategies.append(
                    strategy.CORPUS_MUTATION_ML_RNN_STRATEGY.name)

        additional_corpus_dirs.append(new_testcase_mutations_directory)
        if minijail_chroot:
            bind_corpus_dirs(minijail_chroot,
                             [new_testcase_mutations_directory])

    if strategy_pool.do_strategy(strategy.RANDOM_MAX_LENGTH_STRATEGY):
        max_len_argument = fuzzer_utils.extract_argument(
            existing_arguments, constants.MAX_LEN_FLAG, remove=False)
        if not max_len_argument:
            max_length = random.SystemRandom().randint(
                1, MAX_VALUE_FOR_MAX_LENGTH)
            arguments.append('%s%d' % (constants.MAX_LEN_FLAG, max_length))
            fuzzing_strategies.append(strategy.RANDOM_MAX_LENGTH_STRATEGY.name)

    if (strategy_pool.do_strategy(strategy.RECOMMENDED_DICTIONARY_STRATEGY)
            and add_recommended_dictionary(
                arguments, project_qualified_fuzzer_name, fuzzer_path)):
        fuzzing_strategies.append(
            strategy.RECOMMENDED_DICTIONARY_STRATEGY.name)

    if strategy_pool.do_strategy(strategy.VALUE_PROFILE_STRATEGY):
        arguments.append(constants.VALUE_PROFILE_ARGUMENT)
        fuzzing_strategies.append(strategy.VALUE_PROFILE_STRATEGY.name)

    # DataFlow Tracing requires fork mode, always use it with DFT strategy.
    if use_dataflow_tracing or strategy_pool.do_strategy(
            strategy.FORK_STRATEGY):
        max_fuzz_threads = environment.get_value('MAX_FUZZ_THREADS', 1)
        num_fuzz_processes = max(
            1,
            multiprocessing.cpu_count() // max_fuzz_threads)
        arguments.append('%s%d' % (constants.FORK_FLAG, num_fuzz_processes))
        fuzzing_strategies.append(
            '%s_%d' % (strategy.FORK_STRATEGY.name, num_fuzz_processes))

    extra_env = {}
    if (strategy_pool.do_strategy(strategy.MUTATOR_PLUGIN_STRATEGY)
            and use_mutator_plugin(target_name, extra_env, minijail_chroot)):
        fuzzing_strategies.append(strategy.MUTATOR_PLUGIN_STRATEGY.name)

    return StrategyInfo(fuzzing_strategies, arguments, additional_corpus_dirs,
                        extra_env, use_dataflow_tracing, is_mutations_run)
Example #15
0
def execute_task(metadata_id, job_type):
    """Unpack a bundled testcase archive and create analyze jobs for each item."""
    metadata = ndb.Key(data_types.BundledArchiveMetadata,
                       int(metadata_id)).get()
    if not metadata:
        logs.log_error('Invalid bundle metadata id %s.' % metadata_id)
        return

    bot_name = environment.get_value('BOT_NAME')
    upload_metadata = data_types.TestcaseUploadMetadata.query(
        data_types.TestcaseUploadMetadata.blobstore_key ==
        metadata.blobstore_key).get()
    if not upload_metadata:
        logs.log_error('Invalid upload metadata key %s.' %
                       metadata.blobstore_key)
        return

    # Update the upload metadata with this bot name.
    upload_metadata.bot_name = bot_name
    upload_metadata.put()

    # We can't use FUZZ_INPUTS directory since it is constrained
    # by tmpfs limits.
    testcases_directory = environment.get_value('FUZZ_INPUTS_DISK')

    # Retrieve multi-testcase archive.
    archive_path = os.path.join(testcases_directory, metadata.archive_filename)
    if not blobs.read_blob_to_disk(metadata.blobstore_key, archive_path):
        logs.log_error('Could not retrieve archive for bundle %d.' %
                       metadata_id)
        tasks.add_task('unpack', metadata_id, job_type)
        return

    try:
        archive.unpack(archive_path, testcases_directory)
    except:
        logs.log_error('Could not unpack archive for bundle %d.' % metadata_id)
        tasks.add_task('unpack', metadata_id, job_type)
        return

    # Get additional testcase metadata (if any).
    additional_metadata = None
    if upload_metadata.additional_metadata_string:
        additional_metadata = json.loads(
            upload_metadata.additional_metadata_string)

    archive_state = data_types.ArchiveStatus.NONE
    bundled = True
    file_list = archive.get_file_list(archive_path)

    for file_path in file_list:
        absolute_file_path = os.path.join(testcases_directory, file_path)
        filename = os.path.basename(absolute_file_path)

        # Only files are actual testcases. Skip directories.
        if not os.path.isfile(absolute_file_path):
            continue

        try:
            file_handle = open(absolute_file_path, 'rb')
            blob_key = blobs.write_blob(file_handle)
            file_handle.close()
        except:
            blob_key = None

        if not blob_key:
            logs.log_error('Could not write testcase %s to blobstore.' %
                           absolute_file_path)
            continue

        data_handler.create_user_uploaded_testcase(
            blob_key, metadata.blobstore_key, archive_state,
            metadata.archive_filename, filename, metadata.timeout,
            metadata.job_type, metadata.job_queue, metadata.http_flag,
            metadata.gestures, metadata.additional_arguments,
            metadata.bug_information, metadata.crash_revision,
            metadata.uploader_email, metadata.platform_id,
            metadata.app_launch_command, metadata.fuzzer_name,
            metadata.overridden_fuzzer_name, metadata.fuzzer_binary_name,
            bundled, upload_metadata.retries,
            upload_metadata.bug_summary_update_flag,
            upload_metadata.quiet_flag, additional_metadata)

    # The upload metadata for the archive is not needed anymore since we created
    # one for each testcase.
    upload_metadata.key.delete()

    shell.clear_testcase_directories()
Example #16
0
def unpack_crash_testcases(crash_testcases_directory):
  """Unpacks the old crash testcases in the provided directory."""
  for testcase in ndb_utils.get_all_from_model(data_types.Testcase):
    testcase_id = testcase.key.id()

    # 1. If we have already stored the testcase, then just skip.
    if testcase_id in STORED_TESTCASES_LIST:
      continue

    # 2. Make sure that it is a unique crash testcase. Ignore duplicates,
    # uploaded repros.
    if testcase.status != 'Processed':
      continue

    # 3. Check if the testcase is fixed. If not, skip.
    if testcase.open:
      continue

    # 4. Check if the testcase has a minimized repro. If not, skip.
    if not testcase.minimized_keys or testcase.minimized_keys == 'NA':
      continue

    # 5. Only use testcases that have bugs associated with them.
    if not testcase.bug_information:
      continue

    # 6. Existing IPC testcases are un-interesting and unused in furthur
    # mutations. Due to size bloat, ignoring these for now.
    if testcase.absolute_path.endswith(testcase_manager.IPCDUMP_EXTENSION):
      continue

    # 7. Ignore testcases that are archives (e.g. Langfuzz fuzzer tests).
    if archive.get_archive_type(testcase.absolute_path):
      continue

    # 8. Skip in-process fuzzer testcases, since these are only applicable to
    # fuzz targets and don't run with blackbox binaries.
    if testcase.fuzzer_name and testcase.fuzzer_name in ['afl', 'libFuzzer']:
      continue

    # Un-pack testcase.
    try:
      _, input_directory, _ = setup.unpack_testcase(testcase)
    except Exception:
      logs.log_error('Failed to unpack testcase %d.' % testcase.key.id())
      continue

    # Move this to our crash testcases directory.
    crash_testcase_directory = os.path.join(crash_testcases_directory,
                                            str(testcase_id))
    shell.move(input_directory, crash_testcase_directory)

    # Re-create input directory for unpacking testcase in next iteration.
    shell.create_directory(input_directory)

    STORED_TESTCASES_LIST.append(testcase_id)

  # Remove testcase directories that exceed the max size limit.
  for directory_name in os.listdir(crash_testcases_directory):
    directory_path = os.path.join(crash_testcases_directory, directory_name)
    if not os.path.isdir(directory_path):
      continue

    if shell.get_directory_size(directory_path) <= MAX_TESTCASE_DIRECTORY_SIZE:
      continue

    shell.remove_directory(directory_path)

  # Rename all fuzzed testcase files as regular files.
  for root, _, files in os.walk(crash_testcases_directory):
    for filename in files:
      if not filename.startswith(testcase_manager.FUZZ_PREFIX):
        continue

      file_path = os.path.join(root, filename)
      stripped_file_name = os.path.basename(file_path)[len(
          testcase_manager.FUZZ_PREFIX):]
      stripped_file_path = os.path.join(
          os.path.dirname(file_path), stripped_file_name)
      try:
        os.rename(file_path, stripped_file_path)
      except:
        raise Exception('Failed to rename testcase %s.' % file_path)

  # Remove empty files and dirs to avoid the case where a fuzzer randomly
  # chooses an empty dir/file and generates zero testcases.
  shell.remove_empty_files(crash_testcases_directory)
  shell.remove_empty_directories(crash_testcases_directory)
Example #17
0
def _do_run_testcase_and_return_result_in_queue(crash_queue,
                                                thread_index,
                                                file_path,
                                                gestures,
                                                env_copy,
                                                upload_output=False):
    """Run a single testcase and return crash results in the crash queue."""
    try:
        # Run testcase and check whether a crash occurred or not.
        return_code, crash_time, output = run_testcase(thread_index, file_path,
                                                       gestures, env_copy)

        # Pull testcase directory to host to get any stats files.
        if environment.is_trusted_host():
            from bot.untrusted_runner import file_host
            file_host.pull_testcases_from_worker()

        # Analyze the crash.
        crash_output = _get_crash_output(output)
        crash_result = CrashResult(return_code, crash_time, crash_output)

        # To provide consistency between stats and logs, we use timestamp taken
        # from stats when uploading logs and testcase.
        if upload_output:
            log_time = _get_testcase_time(file_path)

        if crash_result.is_crash():
            # Initialize resource list with the testcase path.
            resource_list = [file_path]
            resource_list += get_resource_paths(crash_output)

            # Store the crash stack file in the crash stacktrace directory
            # with filename as the hash of the testcase path.
            crash_stacks_directory = environment.get_value(
                'CRASH_STACKTRACES_DIR')
            stack_file_path = os.path.join(crash_stacks_directory,
                                           utils.string_hash(file_path))
            utils.write_data_to_file(crash_output, stack_file_path)

            # Put crash/no-crash results in the crash queue.
            crash_queue.put(
                Crash(file_path=file_path,
                      crash_time=crash_time,
                      return_code=return_code,
                      resource_list=resource_list,
                      gestures=gestures,
                      stack_file_path=stack_file_path))

            # Don't upload uninteresting testcases (no crash) or if there is no log to
            # correlate it with (not upload_output).
            if upload_output:
                upload_testcase(file_path, log_time)

        if upload_output:
            # Include full output for uploaded logs (crash output, merge output, etc).
            crash_result_full = CrashResult(return_code, crash_time, output)
            log = prepare_log_for_upload(crash_result_full.get_stacktrace(),
                                         return_code)
            upload_log(log, log_time)
    except Exception:
        logs.log_error('Exception occurred while running '
                       'run_testcase_and_return_result_in_queue.')
def download_system_symbols_if_needed(symbols_directory, is_kernel=False):
    """Download system libraries from |SYMBOLS_URL| and cache locally."""
    # For local testing, we do not have access to the cloud storage bucket with
    # the symbols. In this case, just bail out.
    if environment.get_value('LOCAL_DEVELOPMENT'):
        return

    # When running reproduce tool locally, we do not have access to the cloud
    # storage bucket with the symbols. In this case, just bail out.
    if environment.get_value('REPRODUCE_TOOL'):
        return

    # We have archived symbols for google builds only.
    if not settings.is_google_device():
        return

    # For Android kernel we want to get the repro.prop
    # Note: kasan and non-kasan kernel should have the same repo.prop for a given
    # build_id.
    if is_kernel:
        build_id = settings.get_kernel_build_id()
        target = settings.get_kernel_name()
        if not build_id or not target:
            logs.log_error('Could not get kernel parameters, exiting.')
            return

        artifact_file_name = 'repo.prop'
        symbols_archive_filename = get_symbols_archive_filename(
            build_id, target)
        output_filename_override = symbols_archive_filename
        # We create our own build_params for cache
        build_params = {
            'build_id': build_id,
            'target': target,
            'type': 'kernel'
        }
    else:
        # Get the build fingerprint parameters.
        build_params = settings.get_build_parameters()
        if not build_params:
            logs.log_error('Unable to determine build parameters.')
            return

        build_id = build_params.get('build_id')
        target = build_params.get('target')
        build_type = build_params.get('type')
        if not build_id or not target or not build_type:
            logs.log_error('Null build parameters found, exiting.')
            return

        symbols_archive_filename = '%s-symbols-%s.zip' % (target, build_id)
        artifact_file_name = symbols_archive_filename
        output_filename_override = None

    # Check if we already have the symbols in cache.
    build_params_check_path = os.path.join(symbols_directory,
                                           '.cached_build_params')
    cached_build_params = utils.read_data_from_file(build_params_check_path,
                                                    eval_data=True)
    if cached_build_params and cached_build_params == build_params:
        # No work to do, same system symbols already in cache.
        return

    symbols_archive_path = os.path.join(symbols_directory,
                                        symbols_archive_filename)

    # Delete existing symbols directory first.
    shell.remove_directory(symbols_directory, recreate=True)

    # Fetch symbol file from cloud storage cache (if available).
    found_in_cache = storage.get_file_from_cache_if_exists(
        symbols_archive_path, update_modification_time_on_access=False)
    if not found_in_cache:
        tool_suffix = environment.get_value('SANITIZER_TOOL_NAME')

        if is_kernel:
            # Some kernels are just 'kernel', some are kernel_target
            if tool_suffix:
                targets_with_type_and_san = [
                    'kernel_%s' % tool_suffix,
                    'kernel_%s_%s' % (tool_suffix, target)
                ]
            else:
                targets_with_type_and_san = ['kernel', 'kernel_%s' % target]
        else:
            # Include type and sanitizer information in the target.
            target_with_type_and_san = '%s-%s' % (target, build_type)
            if tool_suffix and not tool_suffix in target_with_type_and_san:
                target_with_type_and_san += '_%s' % tool_suffix

            targets_with_type_and_san = [target_with_type_and_san]

        for target_with_type_and_san in targets_with_type_and_san:
            # Fetch the artifact now.
            fetch_artifact.get(build_id, target_with_type_and_san,
                               artifact_file_name, symbols_directory,
                               output_filename_override)
            if os.path.exists(symbols_archive_path):
                break

    if not os.path.exists(symbols_archive_path):
        logs.log_error('Unable to locate symbols archive %s.' %
                       symbols_archive_path)
        return

    # Store the artifact for later use or for use by other bots.
    storage.store_file_in_cache(symbols_archive_path)

    # repo.prop is not a zip archive.
    if not is_kernel:
        archive.unpack(symbols_archive_path, symbols_directory, trusted=True)
        shell.remove_file(symbols_archive_path)

    utils.write_data_to_file(build_params, build_params_check_path)
Example #19
0
def find_fixed_range(testcase_id, job_type):
    """Attempt to find the revision range where a testcase was fixed."""
    deadline = tasks.get_task_completion_deadline()
    testcase = data_handler.get_testcase_by_id(testcase_id)
    if not testcase:
        return

    if testcase.fixed:
        logs.log_error('Fixed range is already set as %s, skip.' %
                       testcase.fixed)
        return

    # Setup testcase and its dependencies.
    file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type)
    if not file_list:
        return

    # Set a flag to indicate we are running progression task. This shows pending
    # status on testcase report page and avoid conflicting testcase updates by
    # triage cron.
    testcase.set_metadata('progression_pending', True)

    # Custom binaries are handled as special cases.
    if build_manager.is_custom_binary():
        _check_fixed_for_custom_binary(testcase, job_type, testcase_file_path)
        return

    build_bucket_path = build_manager.get_primary_bucket_path()
    revision_list = build_manager.get_revisions_list(build_bucket_path,
                                                     testcase=testcase)
    if not revision_list:
        data_handler.close_testcase_with_error(
            testcase_id, 'Failed to fetch revision list')
        return

    # Use min, max_index to mark the start and end of revision list that is used
    # for bisecting the progression range. Set start to the revision where noticed
    # the crash. Set end to the trunk revision. Also, use min, max from past run
    # if it timed out.
    min_revision = testcase.get_metadata('last_progression_min')
    max_revision = testcase.get_metadata('last_progression_max')

    if min_revision or max_revision:
        # Clear these to avoid using them in next run. If this run fails, then we
        # should try next run without them to see it succeeds. If this run succeeds,
        # we should still clear them to avoid capping max revision in next run.
        testcase = data_handler.get_testcase_by_id(testcase_id)
        testcase.delete_metadata('last_progression_min', update_testcase=False)
        testcase.delete_metadata('last_progression_max', update_testcase=False)
        testcase.put()

    last_tested_revision = testcase.get_metadata('last_tested_crash_revision')
    known_crash_revision = last_tested_revision or testcase.crash_revision
    if not min_revision:
        min_revision = known_crash_revision
    if not max_revision:
        max_revision = revisions.get_last_revision_in_list(revision_list)

    min_index = revisions.find_min_revision_index(revision_list, min_revision)
    if min_index is None:
        raise errors.BuildNotFoundError(min_revision, job_type)
    max_index = revisions.find_max_revision_index(revision_list, max_revision)
    if max_index is None:
        raise errors.BuildNotFoundError(max_revision, job_type)

    testcase = data_handler.get_testcase_by_id(testcase_id)
    data_handler.update_testcase_comment(testcase,
                                         data_types.TaskState.STARTED,
                                         'r%d' % max_revision)

    # Check to see if this testcase is still crashing now. If it is, then just
    # bail out.
    result = _testcase_reproduces_in_revision(testcase,
                                              testcase_file_path,
                                              job_type,
                                              max_revision,
                                              update_metadata=True)
    if result.is_crash():
        logs.log('Found crash with same signature on latest revision r%d.' %
                 max_revision)
        app_path = environment.get_value('APP_PATH')
        command = testcase_manager.get_command_line_for_application(
            testcase_file_path,
            app_path=app_path,
            needs_http=testcase.http_flag)
        symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True)
        unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False)
        stacktrace = utils.get_crash_stacktrace_output(
            command, symbolized_crash_stacktrace,
            unsymbolized_crash_stacktrace)
        testcase = data_handler.get_testcase_by_id(testcase_id)
        testcase.last_tested_crash_stacktrace = data_handler.filter_stacktrace(
            stacktrace)
        data_handler.update_progression_completion_metadata(
            testcase,
            max_revision,
            is_crash=True,
            message='still crashes on latest revision r%s' % max_revision)

        # Since we've verified that the test case is still crashing, clear out any
        # metadata indicating potential flake from previous runs.
        task_creation.mark_unreproducible_if_flaky(testcase, False)

        # For chromium project, save latest crash information for later upload
        # to chromecrash/.
        state = result.get_symbolized_data()
        crash_uploader.save_crash_info_if_needed(testcase_id, max_revision,
                                                 job_type, state.crash_type,
                                                 state.crash_address,
                                                 state.frames)
        return

    # Don't burden NFS server with caching these random builds.
    environment.set_value('CACHE_STORE', False)

    # Verify that we do crash in the min revision. This is assumed to be true
    # while we are doing the bisect.
    result = _testcase_reproduces_in_revision(testcase, testcase_file_path,
                                              job_type, min_revision)
    if result and not result.is_crash():
        testcase = data_handler.get_testcase_by_id(testcase_id)

        # Retry once on another bot to confirm our result.
        if data_handler.is_first_retry_for_task(testcase,
                                                reset_after_retry=True):
            tasks.add_task('progression', testcase_id, job_type)
            error_message = (
                'Known crash revision %d did not crash, will retry on another bot to '
                'confirm result' % known_crash_revision)
            data_handler.update_testcase_comment(testcase,
                                                 data_types.TaskState.ERROR,
                                                 error_message)
            data_handler.update_progression_completion_metadata(
                testcase, max_revision)
            return

        data_handler.clear_progression_pending(testcase)
        error_message = ('Known crash revision %d did not crash' %
                         known_crash_revision)
        data_handler.update_testcase_comment(testcase,
                                             data_types.TaskState.ERROR,
                                             error_message)
        task_creation.mark_unreproducible_if_flaky(testcase, True)
        return

    # Start a binary search to find last non-crashing revision. At this point, we
    # know that we do crash in the min_revision, and do not crash in max_revision.
    while time.time() < deadline:
        min_revision = revision_list[min_index]
        max_revision = revision_list[max_index]

        # If the min and max revisions are one apart this is as much as we can
        # narrow the range.
        if max_index - min_index == 1:
            _save_fixed_range(testcase_id, min_revision, max_revision,
                              testcase_file_path)
            return

        # Occasionally, we get into this bad state. It seems to be related to test
        # cases with flaky stacks, but the exact cause is unknown.
        if max_index - min_index < 1:
            testcase = data_handler.get_testcase_by_id(testcase_id)
            testcase.fixed = 'NA'
            testcase.open = False
            message = ('Fixed testing errored out (min and max revisions '
                       'are both %d)' % min_revision)
            data_handler.update_progression_completion_metadata(
                testcase, max_revision, message=message)

            # Let the bisection service know about the NA status.
            bisection.request_bisection(testcase)
            return

        # Test the middle revision of our range.
        middle_index = (min_index + max_index) // 2
        middle_revision = revision_list[middle_index]

        testcase = data_handler.get_testcase_by_id(testcase_id)
        log_message = 'Testing r%d (current range %d:%d)' % (
            middle_revision, min_revision, max_revision)
        data_handler.update_testcase_comment(testcase,
                                             data_types.TaskState.WIP,
                                             log_message)

        try:
            result = _testcase_reproduces_in_revision(testcase,
                                                      testcase_file_path,
                                                      job_type,
                                                      middle_revision)
        except errors.BadBuildError:
            # Skip this revision.
            del revision_list[middle_index]
            max_index -= 1
            continue

        if result.is_crash():
            min_index = middle_index
        else:
            max_index = middle_index

        _save_current_fixed_range_indices(testcase_id,
                                          revision_list[min_index],
                                          revision_list[max_index])

    # If we've broken out of the loop, we've exceeded the deadline. Recreate the
    # task to pick up where we left off.
    testcase = data_handler.get_testcase_by_id(testcase_id)
    error_message = ('Timed out, current range r%d:r%d' %
                     (revision_list[min_index], revision_list[max_index]))
    data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
                                         error_message)
    tasks.add_task('progression', testcase_id, job_type)
Example #20
0
def execute(input_directory, output_directory, fuzzer_name,
            generation_timeout):
    """Execute ML RNN generator to produce new inputs.

  This method should be called inside launcher, to generate a number of
  new inputs based on ML RNN model.

  It will fetch ML model from GCS bucket specified in environment
  variable `CORPUS_BUCKET`. The script to run the model resides
  in folder `tools/fuzzers/ml/rnn`.

  Args:
    input_directory: Seed corpus path. The directory should not be empty.
    output_directory: The directory to place generated inputs.
    fuzzer_name: Name of the fuzzer, e.g libpng_read_fuzzer. It indicates the
        subdirectory in gcs bucket to store models.
    generation_timeout: Time in seconds for the generator to run. Normally it
        takes <1s to generate an input, assuming the input length is <4KB.
  """
    if environment.platform() != 'LINUX':
        logs.log('Unsupported platform for ML RNN generation, skipping.')
        return

    # Validate corpus folder.
    file_count = shell.get_directory_file_count(input_directory)
    if not file_count:
        logs.log('Corpus is empty. Skip generation.')
        return

    # Number of existing new inputs. They are possibly generated by other
    # generators.
    old_corpus_units = shell.get_directory_file_count(output_directory)
    old_corpus_bytes = shell.get_directory_size(output_directory)

    # Get model path.
    model_path = prepare_model_directory(fuzzer_name)
    if not model_path:
        return

    result = run(input_directory, output_directory, model_path,
                 generation_timeout)

    # Generation process exited abnormally but not caused by timeout, meaning
    # error occurred during execution.
    if result.return_code and not result.timed_out:
        if result.return_code == constants.ExitCode.CORPUS_TOO_SMALL:
            logs.log_warn(
                'ML RNN generation for fuzzer %s aborted due to small corpus.'
                % fuzzer_name)
        else:
            logs.log_error(
                'ML RNN generation for fuzzer %s failed with ExitCode = %d.' %
                (fuzzer_name, result.return_code),
                output=result.output)
        return

    # Timeout is not error, if we have new units generated.
    if result.timed_out:
        logs.log_warn('ML RNN generation for fuzzer %s timed out.' %
                      fuzzer_name)

    new_corpus_units = (shell.get_directory_file_count(output_directory) -
                        old_corpus_units)
    new_corpus_bytes = (shell.get_directory_size(output_directory) -
                        old_corpus_bytes)
    if new_corpus_units:
        logs.log(
            'Added %d new inputs (%d bytes) using ML RNN generator for %s.' %
            (new_corpus_units, new_corpus_bytes, fuzzer_name))
    else:
        logs.log_error('ML RNN generator did not produce any inputs for %s' %
                       fuzzer_name,
                       output=result.output)
Example #21
0
def update_data_bundle(fuzzer, data_bundle):
    """Updates a data bundle to the latest version."""
    # This module can't be in the global imports due to appengine issues
    # with multiprocessing and psutil imports.
    from google_cloud_utils import gsutil

    # If we are using a data bundle on NFS, it is expected that our testcases
    # will usually be large enough that we would fill up our tmpfs directory
    # pretty quickly. So, change it to use an on-disk directory.
    if not data_bundle.is_local:
        testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK')
        environment.set_value('FUZZ_INPUTS', testcase_disk_directory)

    data_bundle_directory = get_data_bundle_directory(fuzzer.name)
    if not data_bundle_directory:
        logs.log_error('Failed to setup data bundle %s.' % data_bundle.name)
        return False

    if not shell.create_directory_if_needed(data_bundle_directory,
                                            create_intermediates=True):
        logs.log_error('Failed to create data bundle %s directory.' %
                       data_bundle.name)
        return False

    # Check if data bundle is up to date. If yes, skip the update.
    if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
        logs.log('Data bundle was recently synced, skip.')
        return True

    # Fetch lock for this data bundle.
    if not _fetch_lock_for_data_bundle_update(data_bundle):
        logs.log_error('Failed to lock data bundle %s.' % data_bundle.name)
        return False

    # Re-check if another bot did the sync already. If yes, skip.
    if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
        logs.log('Another bot finished the sync, skip.')
        _release_lock_for_data_bundle_update(data_bundle)
        return True

    time_before_sync_start = time.time()

    # No need to sync anything if this is a search index data bundle. In that
    # case, the fuzzer will generate testcases from a gcs bucket periodically.
    if not _is_search_index_data_bundle(data_bundle.name):
        bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name)

        if environment.is_trusted_host() and data_bundle.sync_to_worker:
            from bot.untrusted_runner import corpus_manager
            from bot.untrusted_runner import file_host
            worker_data_bundle_directory = file_host.rebase_to_worker_root(
                data_bundle_directory)

            file_host.create_directory(worker_data_bundle_directory,
                                       create_intermediates=True)
            result = corpus_manager.RemoteGSUtilRunner().rsync(
                bucket_url, worker_data_bundle_directory, delete=False)
        else:
            result = gsutil.GSUtilRunner().rsync(bucket_url,
                                                 data_bundle_directory,
                                                 delete=False)

        if result.return_code != 0:
            logs.log_error('Failed to sync data bundle %s: %s.' %
                           (data_bundle.name, result.output))
            _release_lock_for_data_bundle_update(data_bundle)
            return False

    # Update the testcase list file.
    tests.create_testcase_list_file(data_bundle_directory)

    #  Write last synced time in the sync file.
    sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory)
    utils.write_data_to_file(time_before_sync_start, sync_file_path)
    if environment.is_trusted_host() and data_bundle.sync_to_worker:
        from bot.untrusted_runner import file_host
        worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path)
        file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path)

    # Release acquired lock.
    _release_lock_for_data_bundle_update(data_bundle)

    return True
Example #22
0
    def get(self):
        """Handle a get request."""
        try:
            grouper.group_testcases()
        except:
            logs.log_error('Error occurred while grouping test cases.')
            return

        # Free up memory after group task run.
        utils.python_gc()

        # Get list of jobs excluded from bug filing.
        excluded_jobs = get_excluded_jobs()

        for testcase_id in data_handler.get_open_testcase_id_iterator():
            try:
                testcase = data_handler.get_testcase_by_id(testcase_id)
            except errors.InvalidTestcaseError:
                # Already deleted.
                continue

            # Skip if testcase's job type is in exclusions list.
            if testcase.job_type in excluded_jobs:
                continue

            # If the testcase has a bug filed already, no triage is needed.
            if is_bug_filed(testcase):
                continue

            # Check if the crash is important, i.e. it is either a reproducible crash
            # or an unreproducible crash happening frequently.
            if not is_crash_important(testcase):
                continue

            # Require that all tasks like minimizaton, regression testing, etc have
            # finished.
            if not data_handler.critical_tasks_completed(testcase):
                continue

            # For testcases that are not part of a group, wait an additional time till
            # group task completes.
            # FIXME: In future, grouping might be dependent on regression range, so we
            # would have to add an additional wait time.
            if not testcase.group_id and not dates.time_has_expired(
                    testcase.timestamp,
                    hours=data_types.MIN_ELAPSED_TIME_SINCE_REPORT):
                continue

            # If this project does not have an associated issue tracker, we cannot
            # file this crash anywhere.
            issue_tracker_manager = issue_tracker_utils.get_issue_tracker_manager(
                testcase, use_cache=True)
            if not issue_tracker_manager:
                continue

            # If there are similar issues to this test case already filed or recently
            # closed, skip filing a duplicate bug.
            if is_similar_bug_open_or_recently_closed(testcase,
                                                      issue_tracker_manager):
                continue

            # File the bug first and then create filed bug metadata.
            issue_filer.file_issue(testcase, issue_tracker_manager)
            create_filed_bug_metadata(testcase)
            logs.log('Filed new issue %s for testcase %d.' %
                     (testcase.bug_information, testcase_id))
Example #23
0
def archive_testcase_and_dependencies_in_gcs(resource_list, testcase_path):
    """Archive testcase and its dependencies, and store in blobstore."""
    if not os.path.exists(testcase_path):
        logs.log_error('Unable to find testcase %s.' % testcase_path)
        return None, None, None, None

    absolute_filename = testcase_path
    archived = False
    zip_filename = None
    zip_path = None

    if not resource_list:
        resource_list = []

    # Add resource dependencies based on testcase path. These include
    # stuff like extensions directory, dependency files, etc.
    resource_list.extend(tests.get_resource_dependencies(testcase_path))

    # Filter out duplicates, directories, and files that do not exist.
    resource_list = utils.filter_file_list(resource_list)

    logs.log('Testcase and related files :\n%s' % str(resource_list))

    if len(resource_list) <= 1:
        # If this does not have any resources, just save the testcase.
        try:
            file_handle = open(testcase_path, 'rb')
        except IOError:
            logs.log_error('Unable to open testcase %s.' % testcase_path)
            return None, None, None, None
    else:
        # If there are resources, create an archive.

        # Find the common root directory for all of the resources.
        # Assumption: resource_list[0] is the testcase path.
        base_directory_list = resource_list[0].split(os.path.sep)
        for list_index in xrange(1, len(resource_list)):
            current_directory_list = resource_list[list_index].split(
                os.path.sep)
            length = min(len(base_directory_list), len(current_directory_list))
            for directory_index in xrange(length):
                if (current_directory_list[directory_index] !=
                        base_directory_list[directory_index]):
                    base_directory_list = base_directory_list[
                        0:directory_index]
                    break

        base_directory = os.path.sep.join(base_directory_list)
        logs.log('Subresource common base directory: %s' % base_directory)
        if base_directory:
            # Common parent directory, archive sub-paths only.
            base_len = len(base_directory) + len(os.path.sep)
        else:
            # No common parent directory, archive all paths as it-is.
            base_len = 0

        # Prepare the filename for the archive.
        zip_filename, _ = os.path.splitext(os.path.basename(testcase_path))
        zip_filename += _TESTCASE_ARCHIVE_EXTENSION

        # Create the archive.
        zip_path = os.path.join(environment.get_value('INPUT_DIR'),
                                zip_filename)
        zip_file = zipfile.ZipFile(zip_path, 'w')
        for file_name in resource_list:
            if os.path.exists(file_name):
                relative_filename = file_name[base_len:]
                zip_file.write(file_name, relative_filename,
                               zipfile.ZIP_DEFLATED)
        zip_file.close()

        try:
            file_handle = open(zip_path, 'rb')
        except IOError:
            logs.log_error('Unable to open testcase archive %s.' % zip_path)
            return None, None, None, None

        archived = True
        absolute_filename = testcase_path[base_len:]

    fuzzed_key = blobs.write_blob(file_handle)
    file_handle.close()

    # Don't need the archive after writing testcase to blobstore.
    if zip_path:
        shell.remove_file(zip_path)

    return fuzzed_key, archived, absolute_filename, zip_filename
Example #24
0
def update_issue_owner_and_ccs_from_predator_results(policy,
                                                     testcase,
                                                     issue,
                                                     only_allow_ccs=False):
    """Assign the issue to an appropriate owner if possible."""
    if not issue or not issue.is_open:
        return

    # If the issue already has an owner, we don't need to update the bug.
    if issue.assignee:
        return

    # If we've assigned an owner or cc once before, it likely means we were
    # incorrect. Don't try again for this particular issue.
    if (issue_tracker_utils.was_label_added(
            issue, data_types.CHROMIUM_ISSUE_PREDATOR_AUTO_OWNER_LABEL)
            or issue_tracker_utils.was_label_added(
                issue, data_types.CHROMIUM_ISSUE_PREDATOR_AUTO_CC_LABEL)):
        return

    # If there are more than 3 suspected CLs, we can't be confident in the
    # results. Just skip any sort of notification to CL authors in this case.
    suspected_cls = _get_predator_result_item(testcase, 'suspected_cls')
    if not suspected_cls or len(suspected_cls) > 3:
        return

    # Validate that the suspected CLs have all of the information we need before
    # continuing. This allows us to assume that they are well-formed later,
    # avoiding any potential exceptions that would interrupt this task.
    for suspected_cl in suspected_cls:
        url = suspected_cl.get('url')
        description = suspected_cl.get('description')
        author = suspected_cl.get('author')
        if not url or not description or not author:
            logs.log_error(
                'Suspected CL for testcase %d is missing required information.'
                % testcase.key.id())
            return

    if len(suspected_cls) == 1 and not only_allow_ccs:
        suspected_cl = suspected_cls[0]

        # If this owner has already been assigned before but has since been removed,
        # don't assign it to them again.
        for action in issue.actions:
            if action.assignee == suspected_cls[0]['author']:
                return

        # We have high confidence for the single-CL case, so we assign the owner.
        issue.labels.add(data_types.CHROMIUM_ISSUE_PREDATOR_AUTO_OWNER_LABEL)
        issue.assignee = suspected_cl['author']
        issue.status = policy.status('assigned')
        issue_comment = (
            'Automatically assigning owner based on suspected regression '
            'changelist %s (%s).\n\n'
            'If this is incorrect, please let us know why and apply the %s '
            'label. If you aren\'t the correct owner for this issue, please '
            'unassign yourself as soon as possible so it can be re-triaged.' %
            (suspected_cl['url'], suspected_cl['description'],
             data_types.CHROMIUM_ISSUE_PREDATOR_WRONG_CL_LABEL))

    else:
        if testcase.get_metadata('has_issue_ccs_from_predator_results'):
            return

        issue_comment = (
            'Automatically adding ccs based on suspected regression changelists:'
            '\n\n')
        ccs_added = False

        for suspected_cl in suspected_cls:
            # Update the comment with the suspected CL, regardless of whether or not
            # we're ccing the author. This might, for example, catch the attention of
            # someone who has already been cced.
            author = suspected_cl['author']
            issue_comment += '%s by %s - %s\n\n' % (
                suspected_cl['description'], author, suspected_cl['url'])
            if author in issue.ccs:
                continue

            # If an author has previously been manually removed from the cc list,
            # we assume they were incorrectly added. Don't try to add them again.
            author_was_removed = False
            for action in issue.actions:
                if author in action.ccs.removed:
                    author_was_removed = True
                    break

            if author_was_removed:
                continue

            issue.ccs.add(author)
            ccs_added = True

        if not ccs_added:
            # Everyone we'd expect to see has already been cced on the issue. No need
            # to spam it with another comment. Also, set the metadata to avoid doing
            # this again.
            testcase.set_metadata('has_issue_ccs_from_owners_file', True)
            return

        issue.labels.add(data_types.CHROMIUM_ISSUE_PREDATOR_AUTO_CC_LABEL)
        issue_comment += ((
            'If this is incorrect, please let us know why and apply the '
            '{label_text}.').format(label_text=issue.issue_tracker.label_text(
                data_types.CHROMIUM_ISSUE_PREDATOR_WRONG_CL_LABEL)))

    try:
        issue.save(new_comment=issue_comment, notify=True)
    except HttpError:
        # If we see such an error when we aren't setting an owner, it's unexpected.
        if only_allow_ccs or not issue.assignee:
            logs.log_error('Unable to update issue for test case %d.' %
                           testcase.key.id())
            return

        # Retry without setting the owner. They may not be a chromium project
        # member, in which case we can try falling back to cc.
        issue = issue_tracker_utils.get_issue_for_testcase(testcase)
        update_issue_owner_and_ccs_from_predator_results(policy,
                                                         testcase,
                                                         issue,
                                                         only_allow_ccs=True)
Example #25
0
  def _sync_job(self, project, info, corpus_bucket_name, quarantine_bucket_name,
                logs_bucket_name, backup_bucket_name):
    """Sync the config with ClusterFuzz."""
    # Create/update ClusterFuzz jobs.
    for template in get_jobs_for_project(project, info):
      if template.engine == 'none':
        # Engine-less jobs are not automatically managed.
        continue

      fuzzer_entity = self._fuzzer_entities.get(template.engine)
      if not fuzzer_entity:
        raise ProjectSetupError('Invalid fuzzing engine ' + template.engine)

      job_name = template.job_name(project, self._config_suffix)
      job = data_types.Job.query(data_types.Job.name == job_name).get()
      if not job:
        job = data_types.Job()

      if job_name not in fuzzer_entity.jobs and not info.get('disabled', False):
        # Enable new job.
        fuzzer_entity.jobs.append(job_name)

      job.name = job_name
      if self._segregate_projects:
        job.platform = untrusted.platform_name(project, 'linux')
      else:
        # TODO(ochang): Support other platforms?
        job.platform = 'LINUX'

      job.templates = template.cf_job_templates

      job.environment_string = JOB_TEMPLATE.format(
          build_type=self._build_type,
          build_bucket_path=self._get_build_bucket_path(
              project, info, template.engine, template.memory_tool,
              template.architecture),
          engine=template.engine,
          project=project)

      if self._add_revision_mappings:
        revision_vars_url = self._revision_url_template.format(
            project=project,
            bucket=self._get_build_bucket(template.engine,
                                          template.architecture),
            sanitizer=template.memory_tool)

        job.environment_string += (
            'REVISION_VARS_URL = {revision_vars_url}\n'.format(
                revision_vars_url=revision_vars_url))

      if logs_bucket_name:
        job.environment_string += 'FUZZ_LOGS_BUCKET = {logs_bucket}\n'.format(
            logs_bucket=logs_bucket_name)

      if corpus_bucket_name:
        job.environment_string += 'CORPUS_BUCKET = {corpus_bucket}\n'.format(
            corpus_bucket=corpus_bucket_name)

      if quarantine_bucket_name:
        job.environment_string += (
            'QUARANTINE_BUCKET = {quarantine_bucket}\n'.format(
                quarantine_bucket=quarantine_bucket_name))

      if backup_bucket_name:
        job.environment_string += 'BACKUP_BUCKET = {backup_bucket}\n'.format(
            backup_bucket=backup_bucket_name)

      if self._add_info_labels:
        job.environment_string += (
            'AUTOMATIC_LABELS = Proj-{project},Engine-{engine}\n'.format(
                project=project,
                engine=template.engine,
            ))

      help_url = info.get('help_url')
      if help_url:
        job.environment_string += 'HELP_URL = %s\n' % help_url

      if template.experimental:
        job.environment_string += 'EXPERIMENTAL = True\n'

      if template.minimize_job_override:
        minimize_job_override = template.minimize_job_override.job_name(
            project, self._config_suffix)
        job.environment_string += (
            'MINIMIZE_JOB_OVERRIDE = %s\n' % minimize_job_override)

      view_restrictions = info.get('view_restrictions')
      if view_restrictions:
        if view_restrictions in ALLOWED_VIEW_RESTRICTIONS:
          job.environment_string += (
              'ISSUE_VIEW_RESTRICTIONS = %s\n' % view_restrictions)
        else:
          logs.log_error('Invalid view restriction setting %s for project %s.' %
                         (view_restrictions, project))

      selective_unpack = info.get('selective_unpack')
      if selective_unpack:
        job.environment_string += 'UNPACK_ALL_FUZZ_TARGETS_AND_FILES = False\n'

      if (template.engine == 'libfuzzer' and
          template.architecture == 'x86_64' and
          'dataflow' in info.get('fuzzing_engines', DEFAULT_ENGINES)):
        # Dataflow binaries are built with dataflow sanitizer, but can be used
        # as an auxiliary build with libFuzzer builds (e.g. with ASan or UBSan).
        dataflow_build_bucket_path = self._get_build_bucket_path(
            project_name=project,
            info=info,
            engine='dataflow',
            memory_tool='dataflow',
            architecture=template.architecture)
        job.environment_string += (
            'DATAFLOW_BUILD_BUCKET_PATH = %s\n' % dataflow_build_bucket_path)

      if self._additional_vars:
        additional_vars = {}
        additional_vars.update(self._additional_vars.get('all', {}))

        engine_vars = self._additional_vars.get(template.engine, {})
        engine_sanitizer_vars = engine_vars.get(template.memory_tool, {})
        additional_vars.update(engine_sanitizer_vars)

        for key, value in sorted(six.iteritems(additional_vars)):
          job.environment_string += ('{} = {}\n'.format(
              key,
              str(value).encode('unicode-escape').decode('utf-8')))

      job.put()
Example #26
0
    def fuzz(self, target_path, options, reproducers_dir, max_time):
        """Run a fuzz session.

    Args:
      target_path: Path to the target.
      options: The FuzzOptions object returned by prepare().
      reproducers_dir: The directory to put reproducers in when crashes
          are found.
      max_time: Maximum allowed time for the fuzzing to run.

    Returns:
      A FuzzResult object.
    """
        profiler.start_if_needed('libfuzzer_fuzz')
        runner = libfuzzer.get_runner(target_path)
        launcher.set_sanitizer_options(target_path)

        # Directory to place new units.
        new_corpus_dir = self._create_temp_corpus_dir('new')

        corpus_directories = [new_corpus_dir] + options.fuzz_corpus_dirs
        fuzz_timeout = launcher.get_fuzz_timeout(options.is_mutations_run,
                                                 total_timeout=max_time)
        fuzz_result = runner.fuzz(corpus_directories,
                                  fuzz_timeout=fuzz_timeout,
                                  additional_args=options.arguments,
                                  artifact_prefix=reproducers_dir,
                                  extra_env=options.extra_env)

        if (not environment.get_value('USE_MINIJAIL') and
                fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE):
            # Minijail returns 1 if the exit code is nonzero.
            # Otherwise: we can assume that a return code of 1 means that libFuzzer
            # itself ran into an error.
            logs.log_error(ENGINE_ERROR_MESSAGE,
                           engine_output=fuzz_result.output)

        log_lines = utils.decode_to_unicode(fuzz_result.output).splitlines()
        # Output can be large, so save some memory by removing reference to the
        # original output which is no longer needed.
        fuzz_result.output = None

        # Check if we crashed, and get the crash testcase path.
        crash_testcase_file_path = runner.get_testcase_path(log_lines)

        # Parse stats information based on libFuzzer output.
        parsed_stats = launcher.parse_log_stats(log_lines)

        # Extend parsed stats by additional performance features.
        parsed_stats.update(
            stats.parse_performance_features(log_lines,
                                             options.strategies,
                                             options.arguments,
                                             include_strategies=False))

        # Set some initial stat overrides.
        timeout_limit = fuzzer_utils.extract_argument(options.arguments,
                                                      constants.TIMEOUT_FLAG,
                                                      remove=False)

        expected_duration = runner.get_max_total_time(fuzz_timeout)
        actual_duration = int(fuzz_result.time_executed)
        fuzzing_time_percent = 100 * actual_duration / float(expected_duration)
        parsed_stats.update({
            'timeout_limit': int(timeout_limit),
            'expected_duration': expected_duration,
            'actual_duration': actual_duration,
            'fuzzing_time_percent': fuzzing_time_percent,
        })

        # Remove fuzzing arguments before merge and dictionary analysis step.
        arguments = options.arguments[:]
        launcher.remove_fuzzing_arguments(arguments)

        self._merge_new_units(target_path, options.corpus_dir, new_corpus_dir,
                              options.fuzz_corpus_dirs, arguments,
                              parsed_stats)

        fuzz_logs = '\n'.join(log_lines)
        crashes = []
        if crash_testcase_file_path:
            # Write the new testcase.
            # Copy crash testcase contents into the main testcase path.
            crashes.append(
                engine.Crash(crash_testcase_file_path, fuzz_logs, arguments,
                             actual_duration))

        project_qualified_fuzzer_name = (
            data_types.fuzz_target_project_qualified_name(
                utils.current_project(), os.path.basename(target_path)))
        launcher.analyze_and_update_recommended_dictionary(
            runner, project_qualified_fuzzer_name, log_lines,
            options.corpus_dir, arguments)

        return engine.FuzzResult(fuzz_logs, fuzz_result.command, crashes,
                                 parsed_stats, fuzz_result.time_executed)
Example #27
0
    def fuzz(self, target_path, options, reproducers_dir, max_time):
        """Run a fuzz session.

    Args:
      target_path: Path to the target.
      options: The FuzzOptions object returned by prepare().
      reproducers_dir: The directory to put reproducers in when crashes
          are found.
      max_time: Maximum allowed time for the fuzzing to run.

    Returns:
      A FuzzResult object.
    """
        profiler.start_if_needed('libfuzzer_fuzz')
        runner = libfuzzer.get_runner(target_path)
        libfuzzer.set_sanitizer_options(target_path, fuzz_options=options)

        # Directory to place new units.
        new_corpus_dir = self._create_temp_corpus_dir('new')

        corpus_directories = [new_corpus_dir] + options.fuzz_corpus_dirs
        fuzz_timeout = libfuzzer.get_fuzz_timeout(options.is_mutations_run,
                                                  total_timeout=max_time)
        fuzz_result = runner.fuzz(corpus_directories,
                                  fuzz_timeout=fuzz_timeout,
                                  additional_args=options.arguments,
                                  artifact_prefix=reproducers_dir,
                                  extra_env=options.extra_env)

        project_qualified_fuzzer_name = _project_qualified_fuzzer_name(
            target_path)
        dict_error_match = DICT_PARSING_FAILED_REGEX.search(fuzz_result.output)
        if dict_error_match:
            logs.log_error(
                'Dictionary parsing failed (target={target}, line={line}).'.
                format(target=project_qualified_fuzzer_name,
                       line=dict_error_match.group(1)),
                engine_output=fuzz_result.output)
        elif (not environment.get_value('USE_MINIJAIL') and
              fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE):
            # Minijail returns 1 if the exit code is nonzero.
            # Otherwise: we can assume that a return code of 1 means that libFuzzer
            # itself ran into an error.
            logs.log_error(ENGINE_ERROR_MESSAGE + ' (target={target}).'.format(
                target=project_qualified_fuzzer_name),
                           engine_output=fuzz_result.output)

        log_lines = fuzz_result.output.splitlines()
        # Output can be large, so save some memory by removing reference to the
        # original output which is no longer needed.
        fuzz_result.output = None

        # Check if we crashed, and get the crash testcase path.
        crash_testcase_file_path = runner.get_testcase_path(log_lines)

        # If we exited with a non-zero return code with no crash file in output from
        # libFuzzer, this is most likely a startup crash. Use an empty testcase to
        # to store it as a crash.
        if not crash_testcase_file_path and fuzz_result.return_code:
            crash_testcase_file_path = self._create_empty_testcase_file(
                reproducers_dir)

        # Parse stats information based on libFuzzer output.
        parsed_stats = libfuzzer.parse_log_stats(log_lines)

        # Extend parsed stats by additional performance features.
        parsed_stats.update(
            stats.parse_performance_features(log_lines, options.strategies,
                                             options.arguments))

        # Set some initial stat overrides.
        timeout_limit = fuzzer_utils.extract_argument(options.arguments,
                                                      constants.TIMEOUT_FLAG,
                                                      remove=False)

        expected_duration = runner.get_max_total_time(fuzz_timeout)
        actual_duration = int(fuzz_result.time_executed)
        fuzzing_time_percent = 100 * actual_duration / float(expected_duration)
        parsed_stats.update({
            'timeout_limit': int(timeout_limit),
            'expected_duration': expected_duration,
            'actual_duration': actual_duration,
            'fuzzing_time_percent': fuzzing_time_percent,
        })

        # Remove fuzzing arguments before merge and dictionary analysis step.
        merge_arguments = options.arguments[:]
        libfuzzer.remove_fuzzing_arguments(merge_arguments, is_merge=True)
        self._merge_new_units(target_path, options.corpus_dir, new_corpus_dir,
                              options.fuzz_corpus_dirs, merge_arguments,
                              parsed_stats)

        fuzz_logs = '\n'.join(log_lines)
        crashes = []
        if crash_testcase_file_path:
            reproduce_arguments = options.arguments[:]
            libfuzzer.remove_fuzzing_arguments(reproduce_arguments)

            # Use higher timeout for reproduction.
            libfuzzer.fix_timeout_argument_for_reproduction(
                reproduce_arguments)

            # Write the new testcase.
            # Copy crash testcase contents into the main testcase path.
            crashes.append(
                engine.Crash(crash_testcase_file_path, fuzz_logs,
                             reproduce_arguments, actual_duration))

        libfuzzer.analyze_and_update_recommended_dictionary(
            runner, project_qualified_fuzzer_name, log_lines,
            options.corpus_dir, merge_arguments)

        return engine.FuzzResult(fuzz_logs, fuzz_result.command, crashes,
                                 parsed_stats, fuzz_result.time_executed)
Example #28
0
def run_process(cmdline,
                current_working_directory=None,
                timeout=DEFAULT_TEST_TIMEOUT,
                need_shell=False,
                gestures=None,
                env_copy=None,
                testcase_run=True,
                ignore_children=True):
    """Executes a process with a given command line and other parameters."""
    # FIXME(mbarbella): Using LAUNCHER_PATH here is error prone. It forces us to
    # do certain operations before fuzzer setup (e.g. bad build check).
    launcher = environment.get_value('LAUNCHER_PATH')
    if environment.is_trusted_host() and testcase_run and not launcher:
        from bot.untrusted_runner import remote_process_host
        return remote_process_host.run_process(cmdline,
                                               current_working_directory,
                                               timeout, need_shell, gestures,
                                               env_copy, testcase_run,
                                               ignore_children)

    if gestures is None:
        gestures = []

    if env_copy:
        os.environ.update(env_copy)

    # This is used when running scripts on native linux OS and not on the device.
    # E.g. running a fuzzer to generate testcases or launcher script.
    plt = environment.platform()
    if plt in ['ANDROID', 'FUCHSIA'] and (not testcase_run or launcher):
        plt = 'LINUX'
    elif plt == 'IOS' and (not testcase_run or launcher):
        plt = 'MAC'

    # Lower down testcase timeout slightly to account for time for crash analysis.
    timeout -= CRASH_ANALYSIS_TIME

    # LeakSanitizer hack - give time for stdout/stderr processing.
    lsan = environment.get_value('LSAN', False)
    if lsan:
        timeout -= LSAN_ANALYSIS_TIME

    # Initialize variables.
    adb_output = None
    process_output = ''
    process_status = None
    return_code = 0
    process_poll_interval = environment.get_value('PROCESS_POLL_INTERVAL', 0.5)
    start_time = time.time()
    watch_for_process_exit = (environment.get_value('WATCH_FOR_PROCESS_EXIT')
                              if plt == 'ANDROID' else True)
    window_list = []

    # Get gesture start time from last element in gesture list.
    gestures = copy.deepcopy(gestures)
    if gestures and gestures[-1].startswith('Trigger'):
        gesture_start_time = int(gestures[-1].split(':')[1])
        gestures.pop()
    else:
        gesture_start_time = timeout / 2

    logs.log('Process (%s) started.' % str(cmdline), level=logging.DEBUG)

    if plt == 'ANDROID':
        # Clear the log upfront.
        android.logger.clear_log()

        # Run the app.
        adb_output = android.adb.run_adb_command(cmdline, timeout=timeout)
    else:
        cmd, args = shell.get_command_and_arguments(cmdline)

        process_output = mozprocess.processhandler.StoreOutput()
        process_status = ProcessStatus()
        try:
            process_handle = mozprocess.ProcessHandlerMixin(
                cmd,
                args,
                cwd=current_working_directory,
                shell=need_shell,
                processOutputLine=[process_output],
                onFinish=[process_status],
                ignore_children=ignore_children)
            start_process(process_handle)
        except:
            logs.log_error('Exception occurred when running command: %s.' %
                           cmdline)
            return None, None, ''

    while True:
        time.sleep(process_poll_interval)

        # Run the gestures at gesture_start_time or in case we didn't find windows
        # in the last try.
        if (gestures and time.time() - start_time >= gesture_start_time
                and not window_list):
            # In case, we don't find any windows, we increment the gesture start time
            # so that the next check is after 1 second.
            gesture_start_time += 1

            if plt == 'LINUX':
                linux.gestures.run_gestures(gestures, process_handle.pid,
                                            process_status, start_time,
                                            timeout, window_list)
            elif plt == 'WINDOWS':
                windows.gestures.run_gestures(gestures, process_handle.pid,
                                              process_status, start_time,
                                              timeout, window_list)
            elif plt == 'ANDROID':
                android.gestures.run_gestures(gestures, start_time, timeout)

                # TODO(mbarbella): We add a fake window here to prevent gestures on
                # Android from getting executed more than once.
                window_list = ['FAKE']

        if time.time() - start_time >= timeout:
            break

        # Collect the process output.
        output = (android.logger.log_output()
                  if plt == 'ANDROID' else '\n'.join(process_output.output))
        if crash_analyzer.is_memory_tool_crash(output):
            break

        # Check if we need to bail out on process exit.
        if watch_for_process_exit:
            # If |watch_for_process_exit| is set, then we already completed running
            # our app launch command. So, we can bail out.
            if plt == 'ANDROID':
                break

            # On desktop, we bail out as soon as the process finishes.
            if process_status and process_status.finished:
                # Wait for process shutdown and set return code.
                process_handle.wait(timeout=PROCESS_CLEANUP_WAIT_TIME)
                break

    # Process output based on platform.
    if plt == 'ANDROID':
        # Get current log output. If device is in reboot mode, logcat automatically
        # waits for device to be online.
        time.sleep(ANDROID_CRASH_LOGCAT_WAIT_TIME)
        output = android.logger.log_output()

        if android.device.LOW_MEMORY_REGEX.search(output):
            # If the device is low on memory, we should force reboot and bail out to
            # prevent device from getting in a frozen state.
            logs.log('Device is low on memory, rebooting.', output=output)
            android.adb.hard_reset()
            android.adb.wait_for_device()
            android.device.setup_memory_monitor_script_if_needed()

        elif android.adb.time_since_last_reboot() < time.time() - start_time:
            # Check if a reboot has happened, if yes, append log output before reboot
            # and kernel logs content to output.
            log_before_last_reboot = android.logger.log_output_before_last_reboot(
            )
            kernel_log = android.device.get_kernel_log_content()
            output = '%s%s%s%s%s' % (
                log_before_last_reboot,
                utils.get_line_seperator('Device rebooted'), output,
                utils.get_line_seperator('Kernel Log'), kernel_log)
            # Make sure to reset SE Linux Permissive Mode. This can be done cheaply
            # in ~0.15 sec and is needed especially between runs for kernel crashes.
            android.adb.run_as_root()
            android.adb.change_se_linux_to_permissive_mode()
            return_code = 1

        # Add output from adb to the front.
        if adb_output:
            output = '%s\n\n%s' % (adb_output, output)

        # Kill the application if it is still running. We do this at the end to
        # prevent this from adding noise to the logcat output.
        task_name = environment.get_value('TASK_NAME')
        child_process_termination_pattern = environment.get_value(
            'CHILD_PROCESS_TERMINATION_PATTERN')
        if task_name == 'fuzz' and child_process_termination_pattern:
            # In some cases, we do not want to terminate the application after each
            # run to avoid long startup times (e.g. for chrome). Terminate processes
            # matching a particular pattern for light cleanup in this case.
            android.adb.kill_processes_and_children_matching_name(
                child_process_termination_pattern)
        else:
            # There is no special termination behavior. Simply stop the application.
            android.adb.stop_application()

    else:
        # Get the return code in case the process has finished already.
        # If the process hasn't finished, return_code will be None which is what
        # callers expect unless the output indicates a crash.
        return_code = process_handle.poll()

        # If the process is still running, then terminate it.
        if not process_status.finished:
            if launcher and cmdline.startswith(launcher):
                # If this was a launcher script, we KILL all child processes created
                # except for APP_NAME.
                # It is expected that, if the launcher script terminated normally, it
                # cleans up all the child processes it created itself.
                terminate_root_and_child_processes(process_handle.pid)
            else:
                try:
                    # kill() here actually sends SIGTERM on posix.
                    process_handle.kill()
                except:
                    pass

        if lsan:
            time.sleep(LSAN_ANALYSIS_TIME)

        output = '\n'.join(process_output.output)

        # X Server hack when max client reached.
        if ('Maximum number of clients reached' in output
                or 'Unable to get connection to X server' in output):
            logs.log_error('Unable to connect to X server, exiting.')
            os.system('sudo killall -9 Xvfb blackbox >/dev/null 2>&1')
            sys.exit(0)

    if testcase_run and (crash_analyzer.is_memory_tool_crash(output)
                         or crash_analyzer.is_check_failure_crash(output)):
        return_code = 1

    # If a crash is found, then we add the memory state as well.
    if return_code and plt == 'ANDROID':
        ps_output = android.adb.get_ps_output()
        if ps_output:
            output += utils.get_line_seperator('Memory Statistics')
            output += ps_output

    logs.log('Process (%s) ended, exit code (%s), output (%s).' %
             (str(cmdline), str(return_code), str(output)),
             level=logging.DEBUG)

    return return_code, round(time.time() - start_time, 1), output
Example #29
0
    def prepare(self, corpus_dir, target_path, _):
        """Prepare for a fuzzing session, by generating options. Returns a
    FuzzOptions object.

    Args:
      corpus_dir: The main corpus directory.
      target_path: Path to the target.
      build_dir: Path to the build directory.

    Returns:
      A FuzzOptions object.
    """
        arguments = fuzzer.get_arguments(target_path)
        grammar = fuzzer.get_grammar(target_path)
        strategy_pool = strategy_selection.generate_weighted_strategy_pool(
            strategy_list=strategy.LIBFUZZER_STRATEGY_LIST,
            use_generator=True,
            engine_name=self.name)
        strategy_info = libfuzzer.pick_strategies(strategy_pool, target_path,
                                                  corpus_dir, arguments,
                                                  grammar)

        arguments.extend(strategy_info.arguments)

        # Check for seed corpus and add it into corpus directory.
        engine_common.unpack_seed_corpus_if_needed(target_path, corpus_dir)

        # Pick a few testcases from our corpus to use as the initial corpus.
        subset_size = engine_common.random_choice(
            engine_common.CORPUS_SUBSET_NUM_TESTCASES)

        if (not strategy_info.use_dataflow_tracing
                and strategy_pool.do_strategy(strategy.CORPUS_SUBSET_STRATEGY)
                and shell.get_directory_file_count(corpus_dir) > subset_size):
            # Copy |subset_size| testcases into 'subset' directory.
            corpus_subset_dir = self._create_temp_corpus_dir('subset')
            libfuzzer.copy_from_corpus(corpus_subset_dir, corpus_dir,
                                       subset_size)
            strategy_info.fuzzing_strategies.append(
                strategy.CORPUS_SUBSET_STRATEGY.name + '_' + str(subset_size))
            strategy_info.additional_corpus_dirs.append(corpus_subset_dir)
        else:
            strategy_info.additional_corpus_dirs.append(corpus_dir)

        # Check dict argument to make sure that it's valid.
        dict_path = fuzzer_utils.extract_argument(arguments,
                                                  constants.DICT_FLAG,
                                                  remove=False)
        if dict_path and not os.path.exists(dict_path):
            logs.log_error('Invalid dict %s for %s.' %
                           (dict_path, target_path))
            fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG)

        # If there's no dict argument, check for %target_binary_name%.dict file.
        dict_path = fuzzer_utils.extract_argument(arguments,
                                                  constants.DICT_FLAG,
                                                  remove=False)
        if not dict_path:
            dict_path = dictionary_manager.get_default_dictionary_path(
                target_path)
            if os.path.exists(dict_path):
                arguments.append(constants.DICT_FLAG + dict_path)

        # If we have a dictionary, correct any items that are not formatted properly
        # (e.g. quote items that are missing them).
        dictionary_manager.correct_if_needed(dict_path)

        strategies = stats.process_strategies(strategy_info.fuzzing_strategies,
                                              name_modifier=lambda x: x)
        return LibFuzzerOptions(corpus_dir, arguments, strategies,
                                strategy_info.additional_corpus_dirs,
                                strategy_info.extra_env,
                                strategy_info.use_dataflow_tracing,
                                strategy_info.is_mutations_run)
Example #30
0
def main(argv):
    """Run libFuzzer as specified by argv."""
    atexit.register(fuzzer_utils.cleanup)

    # Initialize variables.
    arguments = argv[1:]
    testcase_file_path = arguments.pop(0)
    target_name = arguments.pop(0)
    fuzzer_name = data_types.fuzz_target_project_qualified_name(
        utils.current_project(), target_name)

    # Initialize log handler.
    logs.configure(
        'run_fuzzer', {
            'fuzzer': fuzzer_name,
            'engine': 'libFuzzer',
            'job_name': environment.get_value('JOB_NAME')
        })

    profiler.start_if_needed('libfuzzer_launcher')

    # Make sure that the fuzzer binary exists.
    build_directory = environment.get_value('BUILD_DIR')
    fuzzer_path = engine_common.find_fuzzer_path(build_directory, target_name)
    if not fuzzer_path:
        # This is an expected case when doing regression testing with old builds
        # that do not have that fuzz target. It can also happen when a host sends a
        # message to an untrusted worker that just restarted and lost information on
        # build directory.
        logs.log_warn('Could not find fuzz target %s.' % target_name)
        return

    # Install signal handler.
    signal.signal(signal.SIGTERM, engine_common.signal_term_handler)

    # Set up temp dir.
    engine_common.recreate_directory(fuzzer_utils.get_temp_dir())

    # Setup minijail if needed.
    use_minijail = environment.get_value('USE_MINIJAIL')
    runner = libfuzzer.get_runner(fuzzer_path,
                                  temp_dir=fuzzer_utils.get_temp_dir())

    if use_minijail:
        minijail_chroot = runner.chroot
    else:
        minijail_chroot = None

    # Get corpus directory.
    corpus_directory = environment.get_value('FUZZ_CORPUS_DIR')

    # Add common arguments which are necessary to be used for every run.
    arguments = expand_with_common_arguments(arguments)

    # Add sanitizer options to environment that were specified in the .options
    # file and options that this script requires.
    set_sanitizer_options(fuzzer_path)

    # Minimize test argument.
    minimize_to = fuzzer_utils.extract_argument(arguments,
                                                MINIMIZE_TO_ARGUMENT)
    minimize_timeout = fuzzer_utils.extract_argument(
        arguments, MINIMIZE_TIMEOUT_ARGUMENT)

    if minimize_to and minimize_timeout:
        minimize_testcase(runner, testcase_file_path, minimize_to,
                          int(minimize_timeout), arguments, use_minijail)
        return

    # Cleanse argument.
    cleanse_to = fuzzer_utils.extract_argument(arguments, CLEANSE_TO_ARGUMENT)
    cleanse_timeout = fuzzer_utils.extract_argument(arguments,
                                                    CLEANSE_TIMEOUT_ARGUMENT)

    if cleanse_to and cleanse_timeout:
        cleanse_testcase(runner, testcase_file_path, cleanse_to,
                         int(cleanse_timeout), arguments, use_minijail)
        return

    # If we don't have a corpus, then that means this is not a fuzzing run.
    if not corpus_directory:
        load_testcase_if_exists(runner, testcase_file_path, fuzzer_name,
                                use_minijail, arguments)
        return

    # We don't have a crash testcase, fuzz.

    # Check dict argument to make sure that it's valid.
    dict_argument = fuzzer_utils.extract_argument(arguments,
                                                  constants.DICT_FLAG,
                                                  remove=False)
    if dict_argument and not os.path.exists(dict_argument):
        logs.log_error('Invalid dict %s for %s.' %
                       (dict_argument, fuzzer_name))
        fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG)

    # If there's no dict argument, check for %target_binary_name%.dict file.
    if (not fuzzer_utils.extract_argument(
            arguments, constants.DICT_FLAG, remove=False)):
        default_dict_path = dictionary_manager.get_default_dictionary_path(
            fuzzer_path)
        if os.path.exists(default_dict_path):
            arguments.append(constants.DICT_FLAG + default_dict_path)

    fuzzing_strategies = []

    # Select a generator to use for existing testcase mutations.
    generator = _select_generator()
    is_mutations_run = generator != Generator.NONE

    # Timeout for fuzzer run.
    fuzz_timeout = get_fuzz_timeout(is_mutations_run)

    # Get list of corpus directories.
    corpus_directories = get_corpus_directories(corpus_directory, fuzzer_path,
                                                fuzzing_strategies,
                                                minijail_chroot)

    # Bind corpus directories in minijail.
    if use_minijail:
        artifact_prefix = constants.ARTIFACT_PREFIX_FLAG + '/'
    else:
        artifact_prefix = '%s%s/' % (constants.ARTIFACT_PREFIX_FLAG,
                                     os.path.abspath(
                                         os.path.dirname(testcase_file_path)))

    # Generate new testcase mutations using radamsa, etc.
    if is_mutations_run:
        new_testcase_mutations_directory = generate_new_testcase_mutations(
            corpus_directory, fuzzer_name, generator, fuzzing_strategies)
        corpus_directories.append(new_testcase_mutations_directory)
        if use_minijail:
            bind_corpus_dirs(minijail_chroot,
                             [new_testcase_mutations_directory])

    max_len_argument = fuzzer_utils.extract_argument(arguments,
                                                     constants.MAX_LEN_FLAG,
                                                     remove=False)
    if not max_len_argument and do_random_max_length():
        max_length = random.SystemRandom().randint(1, MAX_VALUE_FOR_MAX_LENGTH)
        arguments.append('%s%d' % (constants.MAX_LEN_FLAG, max_length))
        fuzzing_strategies.append(strategy.RANDOM_MAX_LENGTH_STRATEGY)

    if do_recommended_dictionary():
        if add_recommended_dictionary(arguments, fuzzer_name, fuzzer_path):
            fuzzing_strategies.append(strategy.RECOMMENDED_DICTIONARY_STRATEGY)

    if do_value_profile():
        arguments.append(constants.VALUE_PROFILE_ARGUMENT)
        fuzzing_strategies.append(strategy.VALUE_PROFILE_STRATEGY)

    if do_fork():
        max_fuzz_threads = environment.get_value('MAX_FUZZ_THREADS', 1)
        num_fuzz_processes = max(
            1,
            multiprocessing.cpu_count() / max_fuzz_threads)
        arguments.append('%s%d' % (constants.FORK_FLAG, num_fuzz_processes))
        fuzzing_strategies.append('%s_%d' %
                                  (strategy.FORK_STRATEGY, num_fuzz_processes))

    extra_env = {}
    if do_mutator_plugin():
        if use_mutator_plugin(target_name, extra_env, minijail_chroot):
            fuzzing_strategies.append(strategy.MUTATOR_PLUGIN_STRATEGY)

    # Execute the fuzzer binary with original arguments.
    fuzz_result = runner.fuzz(corpus_directories,
                              fuzz_timeout=fuzz_timeout,
                              additional_args=arguments + [artifact_prefix],
                              extra_env=extra_env)

    if (not use_minijail
            and fuzz_result.return_code == constants.LIBFUZZER_ERROR_EXITCODE):
        # Minijail returns 1 if the exit code is nonzero.
        # Otherwise: we can assume that a return code of 1 means that libFuzzer
        # itself ran into an error.
        logs.log_error(ENGINE_ERROR_MESSAGE, engine_output=fuzz_result.output)

    log_lines = fuzz_result.output.splitlines()
    # Output can be large, so save some memory by removing reference to the
    # original output which is no longer needed.
    fuzz_result.output = None

    # Check if we crashed, and get the crash testcase path.
    crash_testcase_file_path = None
    for line in log_lines:
        match = re.match(CRASH_TESTCASE_REGEX, line)
        if match:
            crash_testcase_file_path = match.group(1)
            break

    if crash_testcase_file_path:
        # Write the new testcase.
        if use_minijail:
            # Convert chroot relative path to host path. Remove the leading '/' before
            # joining.
            crash_testcase_file_path = os.path.join(
                minijail_chroot.directory, crash_testcase_file_path[1:])

        # Copy crash testcase contents into the main testcase path.
        shutil.move(crash_testcase_file_path, testcase_file_path)

    # Print the command output.
    log_header_format = ('Command: %s\n' 'Bot: %s\n' 'Time ran: %f\n')
    bot_name = environment.get_value('BOT_NAME', '')
    command = fuzz_result.command
    if use_minijail:
        # Remove minijail prefix.
        command = engine_common.strip_minijail_command(command, fuzzer_path)
    print(log_header_format % (engine_common.get_command_quoted(command),
                               bot_name, fuzz_result.time_executed))

    # Parse stats information based on libFuzzer output.
    parsed_stats = parse_log_stats(log_lines)

    # Extend parsed stats by additional performance features.
    parsed_stats.update(
        stats.parse_performance_features(log_lines, fuzzing_strategies,
                                         arguments))

    # Set some initial stat overrides.
    timeout_limit = fuzzer_utils.extract_argument(arguments,
                                                  constants.TIMEOUT_FLAG,
                                                  remove=False)

    expected_duration = runner.get_max_total_time(fuzz_timeout)
    actual_duration = int(fuzz_result.time_executed)
    fuzzing_time_percent = 100 * actual_duration / float(expected_duration)
    stat_overrides = {
        'timeout_limit': int(timeout_limit),
        'expected_duration': expected_duration,
        'actual_duration': actual_duration,
        'fuzzing_time_percent': fuzzing_time_percent,
    }

    # Remove fuzzing arguments before merge and dictionary analysis step.
    remove_fuzzing_arguments(arguments)

    # Make a decision on whether merge step is needed at all. If there are no
    # new units added by libFuzzer run, then no need to do merge at all.
    new_units_added = parsed_stats.get('new_units_added', 0)
    merge_error = None
    if new_units_added:
        # Merge the new units with the initial corpus.
        if corpus_directory not in corpus_directories:
            corpus_directories.append(corpus_directory)

        # If this times out, it's possible that we will miss some units. However, if
        # we're taking >10 minutes to load/merge the corpus something is going very
        # wrong and we probably don't want to make things worse by adding units
        # anyway.

        merge_tmp_dir = None
        if not use_minijail:
            merge_tmp_dir = os.path.join(fuzzer_utils.get_temp_dir(),
                                         'merge_workdir')
            engine_common.recreate_directory(merge_tmp_dir)

        old_corpus_len = shell.get_directory_file_count(corpus_directory)
        merge_directory = create_merge_directory()
        corpus_directories.insert(0, merge_directory)

        if use_minijail:
            bind_corpus_dirs(minijail_chroot, [merge_directory])

        merge_result = runner.merge(
            corpus_directories,
            merge_timeout=engine_common.get_merge_timeout(
                DEFAULT_MERGE_TIMEOUT),
            tmp_dir=merge_tmp_dir,
            additional_args=arguments)

        move_mergeable_units(merge_directory, corpus_directory)
        new_corpus_len = shell.get_directory_file_count(corpus_directory)
        new_units_added = 0

        merge_error = None
        if merge_result.timed_out:
            merge_error = 'Merging new testcases timed out:'
        elif merge_result.return_code != 0:
            merge_error = 'Merging new testcases failed:'
        else:
            new_units_added = new_corpus_len - old_corpus_len

        stat_overrides['new_units_added'] = new_units_added

        if merge_result.output:
            stat_overrides.update(
                stats.parse_stats_from_merge_log(
                    merge_result.output.splitlines()))
    else:
        stat_overrides['new_units_added'] = 0
        logs.log('Skipped corpus merge since no new units added by fuzzing.')

    # Get corpus size after merge. This removes the duplicate units that were
    # created during this fuzzing session.
    stat_overrides['corpus_size'] = shell.get_directory_file_count(
        corpus_directory)

    # Delete all corpus directories except for the main one. These were temporary
    # directories to store new testcase mutations and have already been merged to
    # main corpus directory.
    if corpus_directory in corpus_directories:
        corpus_directories.remove(corpus_directory)
    for directory in corpus_directories:
        shutil.rmtree(directory, ignore_errors=True)

    if use_minijail:
        unbind_corpus_dirs(minijail_chroot, corpus_directories)

    # Apply overridden stats to the parsed stats prior to dumping.
    parsed_stats.update(stat_overrides)

    # Dump stats data for further uploading to BigQuery.
    engine_common.dump_big_query_data(parsed_stats, testcase_file_path,
                                      LIBFUZZER_PREFIX, fuzzer_name, command)

    # Add custom crash state based on fuzzer name (if needed).
    add_custom_crash_state_if_needed(fuzzer_name, log_lines, parsed_stats)
    for line in log_lines:
        print(line)

    # Add fuzzing strategies used.
    engine_common.print_fuzzing_strategies(fuzzing_strategies)

    # Add merge error (if any).
    if merge_error:
        print(data_types.CRASH_STACKTRACE_END_MARKER)
        print(merge_error)
        print(
            'Command:',
            get_printable_command(merge_result.command, fuzzer_path,
                                  use_minijail))
        print(merge_result.output)

    analyze_and_update_recommended_dictionary(runner, fuzzer_name, log_lines,
                                              corpus_directory, arguments)

    # Close minijail chroot.
    if use_minijail:
        minijail_chroot.close()

    # Record the stats to make them easily searchable in stackdriver.
    if new_units_added:
        logs.log('New units added to corpus: %d.' % new_units_added,
                 stats=parsed_stats)
    else:
        logs.log('No new units found.', stats=parsed_stats)