def setUp(self):
        if environment.platform() != 'LINUX':
            self.skipTest(
                'Minijail tests are only applicable for linux platform.')

        super(CorpusPruningTestMinijail, self).setUp()
        os.environ['USE_MINIJAIL'] = 'True'
Ejemplo n.º 2
0
def _setup_x():
    """Start Xvfb and blackbox before running the test application."""
    if environment.platform() != 'LINUX':
        return []

    if environment.is_engine_fuzzer_job():
        # For engine fuzzer jobs like AFL, libFuzzer, Xvfb is not needed as the
        # those fuzz targets do not needed a UI.
        return []

    environment.set_value('DISPLAY', DISPLAY)

    print('Creating virtual display...')
    xvfb_runner = new_process.ProcessRunner('/usr/bin/Xvfb')
    xvfb_process = xvfb_runner.run(additional_args=[
        DISPLAY, '-screen', '0', '1280x1024x24', '-ac', '-nolisten', 'tcp'
    ])
    time.sleep(PROCESS_START_WAIT_SECONDS)

    blackbox_runner = new_process.ProcessRunner('/usr/bin/blackbox')
    blackbox_process = blackbox_runner.run()
    time.sleep(PROCESS_START_WAIT_SECONDS)

    # Return all handles we create so they can be terminated properly at exit.
    return [xvfb_process, blackbox_process]
Ejemplo n.º 3
0
def get_build_to_revision_mappings(platform=None):
    """Gets the build information."""
    if not platform:
        platform = environment.platform()

    result = {}
    build_info_json = _fetch_releases_from_chromiumdash(platform)

    for info in build_info_json:
        build_type = info['channel'].lower()
        if build_type == 'extended':
            build_type = 'extended_stable'

        version = info['version']
        revision = str(info['chromium_main_branch_position'])
        result[build_type] = {'revision': revision, 'version': version}

    # Hack: pretend Windows extended stable info to be Linux extended stable info.
    # Because Linux doesn't have extended stable channel.
    if platform.lower() == 'linux':
        es_info = _fetch_releases_from_chromiumdash('WINDOWS',
                                                    channel='Extended')[0]
        result['extended_stable'] = {
            'revision': str(es_info['chromium_main_branch_position']),
            'version': es_info['version']
        }

    return result
Ejemplo n.º 4
0
def _fake_get_libfuzzer_testcase(*_):
    """Fake test case output intended to run "echo -n"."""
    testcase_map = {
        'crash_type': 'type',
        'crash_state': 'state',
        'security_flag': False,
        'gestures': [],
        'flaky_stack': False,
        'job_type': 'test_job',
        'redzone': 32,
        'disable_ubsan': False,
        'additional_metadata': '{}',
        'fuzzer_name': 'libFuzzer',
        'job_definition': 'APP_NAME = launcher.py\n',
        'overridden_fuzzer_name': 'libFuzzer_test_fuzzer',
        'platform': environment.platform().lower(),
        'minimized_arguments': '',
        'window_argument': '',
        'timeout_multiplier': 1.0,
        'serialized_fuzz_target': {
            'binary': 'test_fuzzer',
            'engine': 'libFuzzer',
            'project': 'test_project',
        },
        'one_time_crasher_flag': False,
    }

    return reproduce.SerializedTestcase(testcase_map)
Ejemplo n.º 5
0
def save_crash_info_if_needed(testcase_id, crash_revision, job_type,
                              crash_type, crash_address, crash_frames):
    """Saves crash report for chromium project, skip otherwise."""
    if data_handler.get_project_name(job_type) != 'chromium':
        return None

    serialized_crash_stack_frames = get_symbolized_stack_bytes(
        crash_type, crash_address, crash_frames)
    if not serialized_crash_stack_frames:
        return None

    crash_info = CrashReportInfo(
        serialized_crash_stack_frames=serialized_crash_stack_frames)

    # Get product and version (required).
    platform = environment.platform()
    crash_info.product = PRODUCT_MAP[platform]
    crash_info.version = revisions.get_real_revision(crash_revision,
                                                     job_type,
                                                     display=True)

    # Update crash_info object with bot information and testcase id.
    crash_info.bot_id = environment.get_value('BOT_NAME')
    crash_info.testcase_id = int(testcase_id)

    # Store CrashInfo metadata.
    crash_report_metadata = crash_info.to_report_metadata()
    crash_report_metadata.job_type = job_type
    crash_report_metadata.crash_revision = crash_revision
    crash_report_metadata.put()

    logs.log('Created crash report entry for testcase %s.' % testcase_id)
    return crash_info
Ejemplo n.º 6
0
def default_queue_suffix():
    """Get the queue suffix for the current platform."""
    queue_override = environment.get_value('QUEUE_OVERRIDE')
    if queue_override:
        return queue_suffix_for_platform(queue_override)

    return queue_suffix_for_platform(environment.platform())
Ejemplo n.º 7
0
def _get_testcase_file_and_path(testcase):
    """Figure out the relative path and input directory for this testcase."""
    testcase_absolute_path = testcase.absolute_path

    # This hack is needed so that we can run a testcase generated on windows, on
    # linux. os.path.isabs return false on paths like c:\a\b\c.
    testcase_path_is_absolute = (testcase_absolute_path[1:3] == ':\\'
                                 or os.path.isabs(testcase_absolute_path))

    # Fix os.sep in testcase path if we are running this on non-windows platform.
    # It is unusual to have '\\' on linux paths, so substitution should be safe.
    if environment.platform() != 'WINDOWS' and '\\' in testcase_absolute_path:
        testcase_absolute_path = testcase_absolute_path.replace('\\', os.sep)

    # Default directory for testcases.
    input_directory = environment.get_value('FUZZ_INPUTS')
    if not testcase_path_is_absolute:
        testcase_path = os.path.join(input_directory, testcase_absolute_path)
        return input_directory, testcase_path

    # Check if the testcase is on a nfs data bundle. If yes, then just
    # return it without doing root directory path fix.
    nfs_root = environment.get_value('NFS_ROOT')
    if nfs_root and testcase_absolute_path.startswith(nfs_root):
        return input_directory, testcase_absolute_path

    # Root directory can be different on bots. Fix the path to account for this.
    root_directory = environment.get_value('ROOT_DIR')
    search_string = '%s%s%s' % (os.sep, _BOT_DIR, os.sep)
    search_index = testcase_absolute_path.find(search_string)
    relative_path = testcase_absolute_path[search_index + len(search_string):]
    testcase_path = os.path.join(root_directory, _BOT_DIR, relative_path)

    return input_directory, testcase_path
Ejemplo n.º 8
0
    def _get_fuzzer_binary_name_and_path(self):
        """Returns the fuzzer binary name and its path."""
        # Fuchsia doesn't use file paths to call fuzzers, just the name of the
        # fuzzer, so we set both from FUZZ_TARGET here.
        if environment.platform() == 'FUCHSIA':
            fuzzer_binary_name = fuzzer_path = environment.get_value(
                'FUZZ_TARGET')
            return fuzzer_binary_name, fuzzer_path
        build_directory = environment.get_value('BUILD_DIR')

        if not build_directory:
            raise BuiltinFuzzerException(
                'BUILD_DIR environment variable is not set.')

        fuzzers = fuzzers_utils.get_fuzz_targets(build_directory)

        if not fuzzers:
            raise BuiltinFuzzerException(
                'No fuzzer binaries found in |BUILD_DIR| directory.')

        fuzzer_binary_name = environment.get_value('FUZZ_TARGET')
        if fuzzer_binary_name:
            fuzzer_path = _get_fuzzer_path(fuzzers, fuzzer_binary_name)
        else:
            fuzzer_path = random.SystemRandom().choice(fuzzers)
            fuzzer_binary_name = os.path.basename(fuzzer_path)
        return fuzzer_binary_name, fuzzer_path
Ejemplo n.º 9
0
def initialize_timezone_from_environment():
  """Initializes timezone for date functions based on environment."""
  plt = environment.platform()
  if plt == 'WINDOWS':
    return

  # Only available on Unix platforms.
  time.tzset()
Ejemplo n.º 10
0
def wait_process(process,
                 timeout,
                 input_data=None,
                 terminate_before_kill=False,
                 terminate_wait_time=None):
    """Waits until either the process exits or times out.

  Args:
    process: A subprocess.Popen object.
    timeout: Maximum number of seconds to wait for before sending a signal.
    input_data: Input to be sent to the process.
    terminate_before_kill: A bool indicating that SIGTERM should be sent to
        the process first before SIGKILL (to let the SIGTERM handler run).
    terminate_wait_time: Maximum number of seconds to wait for the SIGTERM
        handler.

  Returns:
    A ProcessResult.
  """
    result = ProcessResult()
    is_windows = environment.platform() == 'WINDOWS'

    # On Windows, terminate() just calls Win32 API function TerminateProcess()
    # which is equivalent to process kill. So, skip terminate_before_kill.
    if terminate_before_kill and not is_windows:
        first_timeout_function = process.terminate

        # Use a second timer to send the process kill.
        second_timer = threading.Timer(timeout + terminate_wait_time,
                                       _end_process, [process.kill, result])
    else:
        first_timeout_function = process.kill
        second_timer = None

    first_timer = threading.Timer(timeout, _end_process,
                                  [first_timeout_function, result])

    output = None
    start_time = time.time()

    try:
        first_timer.start()
        if second_timer:
            second_timer.start()

        output = process.communicate(input_data)[0]
    finally:
        first_timer.cancel()

        if second_timer:
            second_timer.cancel()

    result.return_code = process.poll()
    result.output = output
    result.time_executed = time.time() - start_time
    return result
Ejemplo n.º 11
0
def select_generator(strategy_pool, fuzzer_path):
    """Pick a generator to generate new testcases before fuzzing or return
  Generator.NONE if no generator selected."""
    if environment.is_lib() or environment.platform() == 'FUCHSIA':
        # Unsupported.
        return Generator.NONE

    # We can't use radamsa binary on Windows. Disable ML for now until we know it
    # works on Win.
    # These generators don't produce testcases that LPM fuzzers can use.
    if (environment.platform() == 'WINDOWS'
            or is_lpm_fuzz_target(fuzzer_path)):
        return Generator.NONE
    if strategy_pool.do_strategy(strategy.CORPUS_MUTATION_ML_RNN_STRATEGY):
        return Generator.ML_RNN
    if strategy_pool.do_strategy(strategy.CORPUS_MUTATION_RADAMSA_STRATEGY):
        return Generator.RADAMSA

    return Generator.NONE
Ejemplo n.º 12
0
def android_device_required(func):
    """Skip Android-specific tests if we cannot run them."""
    reason = None
    if not environment.get_value('ANDROID_SERIAL'):
        reason = 'Android device tests require that ANDROID_SERIAL is set.'
    elif not environment.get_value('INTEGRATION'):
        reason = 'Integration tests are not enabled.'
    elif environment.platform() != 'LINUX':
        reason = 'Android device tests can only run on a Linux host.'

    return unittest.skipIf(reason is not None, reason)(func)
Ejemplo n.º 13
0
def get_radamsa_path():
    """Return path to radamsa binary for current platform."""
    bin_directory_path = os.path.join(
        os.path.dirname(os.path.realpath(__file__)), 'bin')
    platform = environment.platform()
    if platform == 'LINUX':
        return os.path.join(bin_directory_path, 'linux', 'radamsa')

    if platform == 'MAC':
        return os.path.join(bin_directory_path, 'mac', 'radamsa')

    return None
Ejemplo n.º 14
0
def get_gestures(gesture_count):
    """Return a list of random gestures."""
    plt = environment.platform()

    if environment.is_android(plt):
        return android.gestures.get_random_gestures(gesture_count)
    if plt == 'LINUX':
        return linux.gestures.get_random_gestures(gesture_count)
    if plt == 'WINDOWS':
        return windows.gestures.get_random_gestures(gesture_count)

    return []
Ejemplo n.º 15
0
def clear_testcase_directories():
    """Clears the testcase directories."""
    remove_directory(environment.get_value('FUZZ_INPUTS'), recreate=True)
    remove_directory(environment.get_value('FUZZ_INPUTS_DISK'), recreate=True)

    if environment.is_android() and environment.get_value('ANDROID_SERIAL'):
        from clusterfuzz._internal.platforms import android
        android.device.clear_testcase_directory()
    if environment.platform() == 'FUCHSIA':
        from clusterfuzz._internal.platforms import fuchsia
        fuchsia.device.clear_testcase_directory()
    if environment.is_trusted_host():
        from clusterfuzz._internal.bot.untrusted_runner import file_host
        file_host.clear_testcase_directories()
Ejemplo n.º 16
0
def cleanup_defunct_processes():
    """Cleans up defunct processes."""
    # Defunct processes happen only on unix platforms.
    if environment.platform() != 'WINDOWS':
        while True:
            try:
                # Matches any defunct child process.
                p, _ = os.waitpid(-1, os.WNOHANG)
                if not p:
                    break

                logs.log('Clearing defunct process %s.' % str(p))
            except:
                break
Ejemplo n.º 17
0
def start_process(process_handle):
    """Start the process using process handle and override list2cmdline for
  Windows."""
    is_win = environment.platform() == 'WINDOWS'
    if is_win:
        # Override list2cmdline on Windows to return first index of list as string.
        # This is to workaround a mozprocess bug since it passes command as list
        # and not as string.
        subprocess.list2cmdline_orig = subprocess.list2cmdline
        subprocess.list2cmdline = lambda s: s[0]
    try:
        process_handle.run()
    finally:
        if is_win:
            subprocess.list2cmdline = subprocess.list2cmdline_orig
Ejemplo n.º 18
0
def symbolize_stacktrace(unsymbolized_crash_stacktrace,
                         enable_inline_frames=True):
  """Symbolize a crash stacktrace."""
  if environment.is_trusted_host():
    from clusterfuzz._internal.bot.untrusted_runner import symbolize_host
    return symbolize_host.symbolize_stacktrace(unsymbolized_crash_stacktrace,
                                               enable_inline_frames)

  platform = environment.platform()
  if platform == 'WINDOWS':
    # Windows Clang ASAN provides symbolized stacktraces anyway.
    return unsymbolized_crash_stacktrace

  if platform == 'FUCHSIA':
    # Fuchsia Clang ASAN provides symbolized stacktraces anyway.
    return unsymbolized_crash_stacktrace

  # FIXME: Support symbolization on ChromeOS device.
  if platform == 'CHROMEOS':
    return unsymbolized_crash_stacktrace

  # Initialize variables.
  global llvm_symbolizer_path
  global pipes
  global stack_inlining
  global symbolizers
  pipes = []
  stack_inlining = str(enable_inline_frames).lower()
  symbolizers = {}

  # Make sure we have a llvm symbolizer for this platform.
  llvm_symbolizer_path = environment.get_llvm_symbolizer_path()
  if not llvm_symbolizer_path:
    return unsymbolized_crash_stacktrace

  # Disable buffering for stdout.
  disable_buffering()

  loop = SymbolizationLoop(
      binary_path_filter=filter_binary_path,
      dsym_hint_producer=chrome_dsym_hints)
  symbolized_crash_stacktrace = loop.process_stacktrace(
      unsymbolized_crash_stacktrace)

  return symbolized_crash_stacktrace
Ejemplo n.º 19
0
def convert_dependency_url_to_local_path(url):
    """Convert a dependency URL to a corresponding local path."""
    # Bot-specific import.
    from clusterfuzz._internal.bot.webserver import http_server

    logs.log('Process dependency: %s.' % url)
    file_match = FILE_URL_REGEX.search(url)
    http_match = HTTP_URL_REGEX.search(url)
    platform = environment.platform()

    local_path = None
    if file_match:
        file_path = file_match.group(1)
        logs.log('Detected file dependency: %s.' % file_path)
        if platform == 'WINDOWS':
            local_path = file_path
        else:
            local_path = '/' + file_path

            # Convert remote to local path for android.
            if environment.is_android():
                remote_testcases_directory = android.constants.DEVICE_TESTCASES_DIR
                local_testcases_directory = environment.get_value(
                    'FUZZ_INPUTS')
                local_path = local_path.replace(remote_testcases_directory,
                                                local_testcases_directory)

    elif http_match:
        relative_http_path = os.path.sep + http_match.group(2)
        logs.log('Detected http dependency: %s.' % relative_http_path)
        local_path = http_server.get_absolute_testcase_file(relative_http_path)
        if not local_path:
            # This needs to be a warning since in many cases, it is actually a
            # non-existent path. For others, we need to add the directory aliases in
            # file http_server.py.
            logs.log_warn('Unable to find server resource %s, skipping.' %
                          relative_http_path)

    if local_path:
        local_path = utils.normalize_path(local_path)

    return local_path
Ejemplo n.º 20
0
    def setUp(self):
        """Setup for minijail test."""
        if environment.platform() != 'LINUX':
            self.skipTest(
                'Minijail tests are only applicable for linux platform.')

        self.setUpPyfakefs()
        for subdir in ['dev', 'lib', 'lib32', 'lib64', 'proc']:
            self.fs.create_dir(os.path.join('/', subdir))
        self.fs.create_dir(os.path.join('/', 'usr', 'lib'))
        self.fs.create_dir(os.path.join('/', 'usr', 'lib32'))

        test_helpers.patch(self, [
            'clusterfuzz._internal.system.minijail._get_minijail_path',
            'clusterfuzz._internal.system.minijail.MinijailChroot._mknod',
        ])

        test_helpers.patch_environ(self)

        self.mock._get_minijail_path.return_value = '/sbin/minijail'  # pylint: disable=protected-access
Ejemplo n.º 21
0
    def get_command(self, additional_args=None):
        """Overridden get_command."""
        if environment.platform() != 'LINUX':
            raise RuntimeError('UnshareProcessRunner only supported on Linux')

        unshare_path = environment.get_default_tool_path('unshare')
        if not unshare_path:
            raise RuntimeError('unshare not found')

        command = [
            unshare_path,
            '-c',  # Map current user to same user in user namespace.
            '-n',  # Enter network namespace.
        ]

        command.append(self._executable_path)
        command.extend(self._default_args)

        if additional_args:
            command.extend(additional_args)

        return command
Ejemplo n.º 22
0
def run():
    """Run custom platform specific init scripts."""
    platform = environment.platform().lower()
    script_path = os.path.join(environment.get_config_directory(), SCRIPT_DIR,
                               platform + _extension(platform))
    if not os.path.exists(script_path):
        return

    os.chmod(script_path, 0o750)
    if script_path.endswith('.ps1'):
        cmd = 'powershell.exe ' + script_path
    else:
        cmd = script_path

    try:
        process_handler.run_process(cmd,
                                    timeout=1800,
                                    need_shell=True,
                                    testcase_run=False,
                                    ignore_children=True)
    except Exception:
        logs.log_error('Failed to execute platform initialization script.')
Ejemplo n.º 23
0
def filter_binary_path(binary_path):
  """Filters binary path to provide a local copy."""
  if environment.is_android() or environment.is_lkl_job():
    return symbols_downloader.filter_binary_path(binary_path)

  if environment.platform() == 'CHROMEOS':
    # FIXME: Add code to pull binaries from ChromeOS device.
    return binary_path

  if environment.is_chromeos_system_job():
    # This conditional is True for ChromeOS system fuzzers that are running on
    # Linux. Ensure that the binary is always looked for in the chroot and not
    # in system directories.
    build_dir = environment.get_value('BUILD_DIR')
    if not binary_path.startswith(build_dir):
      # Fixup path so |binary_path| points to a binary in the chroot (probably
      # a system library).
      return os.path.join(build_dir, binary_path[1:])

  # For Linux and Mac, the binary exists locally. No work to do,
  # just return the same binary path.
  return binary_path
Ejemplo n.º 24
0
def start_if_needed(service):
    """Start Google Cloud Profiler if |USE_PYTHON_PROFILER| environment variable
  is set."""
    if not environment.get_value('USE_PYTHON_PROFILER'):
        return True

    project_id = utils.get_application_id()
    service_with_platform = '{service}_{platform}'.format(
        service=service, platform=environment.platform().lower())

    try:
        # Import the package here since it is only needed when profiler is enabled.
        # Also, this is supported on Linux only.
        import googlecloudprofiler
        googlecloudprofiler.start(project_id=project_id,
                                  service=service_with_platform)
    except Exception:
        logs.log_error('Failed to start the profiler for service %s.' %
                       service_with_platform)
        return False

    return True
Ejemplo n.º 25
0
def find_fuzzer_path(build_directory, fuzzer_name):
    """Find the fuzzer path with the given name."""
    if not build_directory:
        # Grey-box fuzzers might not have the build directory for a particular job
        # configuration when doing variant task testing (e.g. Android on-device
        # fuzz target might not exist on host). In this case, treat it similar to
        # target not found by returning None.
        logs.log_warn('No build directory found for fuzzer: %s' % fuzzer_name)
        return None

    if environment.platform() == 'FUCHSIA':
        # Fuchsia targets are not on disk.
        return fuzzer_name

    if environment.is_android_kernel():
        return os.path.join(build_directory, 'syzkaller', 'bin', 'syz-manager')

    # TODO(ochang): This is necessary for legacy testcases, which include the
    # project prefix in arguments. Remove this in the near future.
    project_name = environment.get_value('PROJECT_NAME')
    legacy_name_prefix = u''
    if project_name:
        legacy_name_prefix = project_name + u'_'

    fuzzer_filename = environment.get_executable_filename(fuzzer_name)
    for root, _, files in shell.walk(build_directory):
        for filename in files:
            if (legacy_name_prefix + filename == fuzzer_name
                    or filename == fuzzer_filename):
                return os.path.join(root, filename)

    # This is an expected case when doing regression testing with old builds
    # that do not have that fuzz target. It can also happen when a host sends a
    # message to an untrusted worker that just restarted and lost information on
    # build directory.
    logs.log_warn('Fuzzer: %s not found in build_directory: %s.' %
                  (fuzzer_name, build_directory))
    return None
Ejemplo n.º 26
0
def close_open_file_handles_if_needed(path):
    """Try to close all open file handle for a specific path."""
    if environment.platform() != 'WINDOWS':
        # Handle closing is only applicable on Windows platform.
        return

    resources_directory = environment.get_platform_resources_directory()
    handle_executable_path = os.path.join(resources_directory, 'handle.exe')
    handle_output = execute_command('%s -accepteula "%s"' %
                                    (handle_executable_path, path))
    for line in handle_output.splitlines():
        match = HANDLE_OUTPUT_FILE_TYPE_REGEX.match(line)
        if not match:
            continue

        process_id = match.group(1).decode('utf-8')
        file_handle_id = match.group(2).decode('utf-8')
        file_path = match.group(3).decode('utf-8')

        logs.log('Closing file handle id %s for path %s.' %
                 (file_handle_id, file_path))
        execute_command('%s -accepteula -c %s -p %s -y' %
                        (handle_executable_path, file_handle_id, process_id))
Ejemplo n.º 27
0
def _get_gsutil_path():
    """Get path to gsutil executable.

  Returns:
    Path to gsutil executable on the system.
  """
    gsutil_executable = 'gsutil'
    if environment.platform() == 'WINDOWS':
        gsutil_executable += '.cmd'

    gsutil_directory = environment.get_value('GSUTIL_PATH')
    if not gsutil_directory:
        # Try searching the binary in path.
        gsutil_absolute_path = shell.which(gsutil_executable)
        if gsutil_absolute_path:
            return gsutil_absolute_path

        logs.log_error(
            'Cannot locate gsutil in PATH, set GSUTIL_PATH to directory '
            'containing gsutil binary.')
        return None

    gsutil_absolute_path = os.path.join(gsutil_directory, gsutil_executable)
    return gsutil_absolute_path
Ejemplo n.º 28
0
def get_fuzz_task_payload(platform=None):
    """Select a fuzzer that can run on this platform."""
    if not platform:
        queue_override = environment.get_value('QUEUE_OVERRIDE')
        platform = queue_override if queue_override else environment.platform()

    if environment.is_local_development():
        query = data_types.FuzzerJob.query()
        query = query.filter(data_types.FuzzerJobs.platform == platform)
        mappings = list(ndb_utils.get_all_from_query(query))
    else:
        query = data_types.FuzzerJobs.query()
        query = query.filter(data_types.FuzzerJobs.platform == platform)

        mappings = []
        for entity in query:
            mappings.extend(entity.fuzzer_jobs)

    if not mappings:
        return None, None

    selection = utils.random_weighted_choice(mappings,
                                             weight_attribute='actual_weight')
    return selection.fuzzer, selection.job
Ejemplo n.º 29
0
def run_platform_init_scripts():
    """Run platform specific initialization scripts."""
    logs.log('Running platform initialization scripts.')

    plt = environment.platform()
    if environment.is_android_emulator():
        # Nothing to do here since emulator is not started yet.
        pass
    elif environment.is_android():
        android_init.run()
    elif plt == 'CHROMEOS':
        chromeos_init.run()
    elif plt == 'FUCHSIA':
        fuchsia_init.run()
    elif plt == 'LINUX':
        linux_init.run()
    elif plt == 'MAC':
        mac_init.run()
    elif plt == 'WINDOWS':
        windows_init.run()
    else:
        raise RuntimeError('Unsupported platform')

    logs.log('Completed running platform initialization scripts.')
Ejemplo n.º 30
0
def execute(input_directory, output_directory, fuzzer_name,
            generation_timeout):
    """Execute ML RNN generator to produce new inputs.

  This method should be called inside launcher, to generate a number of
  new inputs based on ML RNN model.

  It will fetch ML model from GCS bucket specified in environment
  variable `CORPUS_BUCKET`. The script to run the model resides
  in folder `tools/fuzzers/ml/rnn`.

  Args:
    input_directory: Seed corpus path. The directory should not be empty.
    output_directory: The directory to place generated inputs.
    fuzzer_name: Name of the fuzzer, e.g libpng_read_fuzzer. It indicates the
        subdirectory in gcs bucket to store models.
    generation_timeout: Time in seconds for the generator to run. Normally it
        takes <1s to generate an input, assuming the input length is <4KB.
  """
    if environment.platform() != 'LINUX':
        logs.log('Unsupported platform for ML RNN generation, skipping.')
        return

    # Validate corpus folder.
    file_count = shell.get_directory_file_count(input_directory)
    if not file_count:
        logs.log('Corpus is empty. Skip generation.')
        return

    # Number of existing new inputs. They are possibly generated by other
    # generators.
    old_corpus_units = shell.get_directory_file_count(output_directory)
    old_corpus_bytes = shell.get_directory_size(output_directory)

    # Get model path.
    model_path = prepare_model_directory(fuzzer_name)
    if not model_path:
        return

    result = run(input_directory, output_directory, model_path,
                 generation_timeout)

    # Generation process exited abnormally but not caused by timeout, meaning
    # error occurred during execution.
    if result.return_code and not result.timed_out:
        if result.return_code == constants.ExitCode.CORPUS_TOO_SMALL:
            logs.log_warn(
                'ML RNN generation for fuzzer %s aborted due to small corpus.'
                % fuzzer_name)
        else:
            logs.log_error(
                'ML RNN generation for fuzzer %s failed with ExitCode = %d.' %
                (fuzzer_name, result.return_code),
                output=utils.decode_to_unicode(result.output))
        return

    # Timeout is not error, if we have new units generated.
    if result.timed_out:
        logs.log_warn('ML RNN generation for fuzzer %s timed out.' %
                      fuzzer_name)

    new_corpus_units = (shell.get_directory_file_count(output_directory) -
                        old_corpus_units)
    new_corpus_bytes = (shell.get_directory_size(output_directory) -
                        old_corpus_bytes)
    if new_corpus_units:
        logs.log(
            'Added %d new inputs (%d bytes) using ML RNN generator for %s.' %
            (new_corpus_units, new_corpus_bytes, fuzzer_name))
    else:
        logs.log_error('ML RNN generator did not produce any inputs for %s' %
                       fuzzer_name,
                       output=utils.decode_to_unicode(result.output))