Пример #1
0
def update_fuzzer_and_data_bundles(fuzzer_name):
    """Update the fuzzer with a given name if necessary."""
    fuzzer = data_types.Fuzzer.query(
        data_types.Fuzzer.name == fuzzer_name).get()
    if not fuzzer:
        logs.log_error('No fuzzer exists with name %s.' % fuzzer_name)
        raise errors.InvalidFuzzerError

    # Set some helper environment variables.
    fuzzer_directory = get_fuzzer_directory(fuzzer_name)
    environment.set_value('FUZZER_DIR', fuzzer_directory)
    environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content)

    # Adjust the test timeout, if user has provided one.
    if fuzzer.timeout:
        environment.set_value('TEST_TIMEOUT', fuzzer.timeout)

        # Increase fuzz test timeout if the fuzzer timeout is higher than its
        # current value.
        fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT')
        if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout:
            environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout)

    # Adjust the max testcases if this fuzzer has specified a lower limit.
    max_testcases = environment.get_value('MAX_TESTCASES')
    if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases:
        environment.set_value('MAX_TESTCASES', fuzzer.max_testcases)

    # Check for updates to this fuzzer.
    version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name)
    if (not fuzzer.builtin
            and revisions.needs_update(version_file, fuzzer.revision)):
        logs.log('Fuzzer update was found, updating.')

        # Clear the old fuzzer directory if it exists.
        if not shell.remove_directory(fuzzer_directory, recreate=True):
            logs.log_error('Failed to clear fuzzer directory.')
            return False

        # Copy the archive to local disk and unpack it.
        archive_path = os.path.join(fuzzer_directory, fuzzer.filename)
        if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path):
            logs.log_error('Failed to copy fuzzer archive.')
            return False

        try:
            archive.unpack(archive_path, fuzzer_directory)
        except Exception:
            error_message = (
                'Failed to unpack fuzzer archive %s '
                '(bad archive or unsupported format).') % fuzzer.filename
            logs.log_error(error_message)
            fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
                                          fuzzer_name=fuzzer_name)
            return False

        fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path)
        if not os.path.exists(fuzzer_path):
            error_message = (
                'Fuzzer executable %s not found. '
                'Check fuzzer configuration.') % fuzzer.executable_path
            logs.log_error(error_message)
            fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
                                          fuzzer_name=fuzzer_name)
            return False

        # Make fuzzer executable.
        os.chmod(fuzzer_path, 0o750)

        # Cleanup unneeded archive.
        shell.remove_file(archive_path)

        # Save the current revision of this fuzzer in a file for later checks.
        revisions.write_revision_to_revision_file(version_file,
                                                  fuzzer.revision)
        logs.log('Updated fuzzer to revision %d.' % fuzzer.revision)

    # Setup data bundles associated with this fuzzer.
    data_bundles = ndb_utils.get_all_from_query(
        data_types.DataBundle.query(
            data_types.DataBundle.name == fuzzer.data_bundle_name))
    for data_bundle in data_bundles:
        if not update_data_bundle(fuzzer, data_bundle):
            return False

    # Setup environment variable for launcher script path.
    if fuzzer.launcher_script:
        fuzzer_launcher_path = shell.get_execute_command(
            os.path.join(fuzzer_directory, fuzzer.launcher_script))
        environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path)

    return True
Пример #2
0
def get_command_line_for_application(file_to_run='',
                                     user_profile_index=0,
                                     app_path=None,
                                     app_args=None,
                                     needs_http=False,
                                     write_command_line_file=False,
                                     get_arguments_only=False):
    """Returns the complete command line required to execute application."""
    if app_args is None:
        app_args = environment.get_value('APP_ARGS')
    if app_path is None:
        app_path = environment.get_value('APP_PATH')

    if not app_path:
        # No APP_PATH is available for e.g. grey box fuzzers.
        return ''

    additional_command_line_flags = get_additional_command_line_flags(
        file_to_run)
    app_args_append_testcase = environment.get_value(
        'APP_ARGS_APPEND_TESTCASE')
    app_directory = environment.get_value('APP_DIR')
    app_name = environment.get_value('APP_NAME')
    apps_argument = environment.get_value('APPS_ARG')
    crash_stacks_directory = environment.get_value('CRASH_STACKTRACES_DIR')
    debugger = environment.get_value('DEBUGGER_PATH')
    device_testcases_directory = android.constants.DEVICE_TESTCASES_DIR
    fuzzer_directory = environment.get_value('FUZZER_DIR')
    extension_argument = environment.get_value('EXTENSION_ARG')
    input_directory = environment.get_value('INPUT_DIR')
    launcher = environment.get_value('LAUNCHER_PATH')
    is_android = environment.is_android()
    root_directory = environment.get_value('ROOT_DIR')
    temp_directory = environment.get_value('BOT_TMPDIR')
    user_profile_argument = environment.get_value('USER_PROFILE_ARG')
    window_argument = environment.get_value('WINDOW_ARG')
    user_profile_directory = get_user_profile_directory(user_profile_index)

    # Create user profile directory and setup contents if needed.
    setup_user_profile_directory_if_needed(user_profile_directory)

    # Handle spaces in APP_PATH.
    # If application path has spaces, then we need to quote it.
    if ' ' in app_path:
        app_path = '"%s"' % app_path

    interpreter = shell.get_interpreter(app_name)
    if get_arguments_only:
        # If we are only returning the arguments, do not return the application
        # path or anything else required to run it such as an interpreter.
        app_path = ''
    elif interpreter:
        # Prepend command with interpreter if it is a script.
        app_path = '%s %s' % (interpreter, app_path)

    # Start creating the command line.
    command = ''

    # Rebase the file_to_run and launcher paths to the worker's root.
    if environment.is_trusted_host():
        from bot.untrusted_runner import file_host
        file_to_run = file_host.rebase_to_worker_root(file_to_run)
        launcher = file_host.rebase_to_worker_root(launcher)

    # Default case.
    testcase_path = file_to_run
    testcase_filename = os.path.basename(testcase_path)
    testcase_directory = os.path.dirname(testcase_path)
    testcase_file_url = utils.file_path_to_file_url(testcase_path)
    testcase_http_url = ''

    # Determine where |testcase_file_url| should point depending on platform and
    # whether or not a launcher script is used.
    if file_to_run:
        if launcher:
            # In the case of launcher scripts, the testcase file to be run resides on
            # the host running the launcher script. Thus |testcase_file_url|, which
            # may point to a location on the device for Android job types, does not
            # apply. Instead, the launcher script should be passed the original file
            # to run. By setting |testcase_file_url| to |file_to_run|, we avoid
            # duplicating job definitions solely for supporting launcher scripts.
            testcase_file_url = file_to_run
            # Jobs that have a launcher script which needs to be run on the host will
            # have app_name == launcher. In this case don't prepend launcher to
            # command - just use app_name.
            if os.path.basename(launcher) != app_name:
                launcher_with_interpreter = shell.get_execute_command(launcher)
                command += launcher_with_interpreter + ' '
        elif is_android:
            # Android-specific testcase path fixup for fuzzers that don't rely on
            # launcher scripts.
            local_testcases_directory = environment.get_value('FUZZ_INPUTS')

            # Check if the file to run is in fuzzed testcases folder. If yes, then we
            # can substitute with a local device path. Otherwise, it is part of some
            # data bundle with resource dependencies and we just need to use http
            # host forwarder for that.
            if file_to_run.startswith(local_testcases_directory):
                testcase_relative_path = (
                    file_to_run[len(local_testcases_directory) + 1:])
                testcase_path = os.path.join(device_testcases_directory,
                                             testcase_relative_path)
                testcase_file_url = utils.file_path_to_file_url(testcase_path)
            else:
                # Force use of host_forwarder based on comment above.
                needs_http = True

        # Check if the testcase needs to be loaded over http.
        # TODO(ochang): Make this work for trusted/untrusted.
        http_ip = '127.0.0.1'
        http_port_1 = environment.get_value('HTTP_PORT_1', 8000)
        relative_testcase_path = file_to_run[len(input_directory +
                                                 os.path.sep):]
        relative_testcase_path = relative_testcase_path.replace('\\', '/')
        testcase_http_url = 'http://%s:%d/%s' % (http_ip, http_port_1,
                                                 relative_testcase_path)

        if needs_http:
            # TODO(unassigned): Support https.
            testcase_file_url = testcase_http_url
            testcase_path = testcase_http_url

    # Compose app arguments.
    all_app_args = ''

    if user_profile_argument:
        all_app_args += ' %s=%s' % (user_profile_argument,
                                    user_profile_directory)
    if extension_argument and EXTENSIONS_PREFIX in testcase_filename:
        all_app_args += ' %s=%s' % (extension_argument, testcase_directory)
    if apps_argument and APPS_PREFIX in testcase_filename:
        all_app_args += ' %s=%s' % (apps_argument, testcase_directory)
    if window_argument:
        all_app_args += ' %s' % window_argument
    if additional_command_line_flags:
        all_app_args += ' %s' % additional_command_line_flags.strip()
    if app_args:
        all_app_args += ' %s' % app_args.strip()
    # Append %TESTCASE% at end if no testcase pattern is found in app arguments.
    if not utils.sub_string_exists_in(
        ['%TESTCASE%', '%TESTCASE_FILE_URL%', '%TESTCASE_HTTP_URL%'],
            all_app_args) and app_args_append_testcase:
        all_app_args += ' %TESTCASE%'
    all_app_args = all_app_args.strip()

    # Build the actual command to run now.
    if debugger:
        command += '%s ' % debugger
    if app_path:
        command += app_path
    if all_app_args:
        command += ' %s' % all_app_args
    command = command.replace('%APP_DIR%', app_directory)
    command = command.replace('%CRASH_STACKTRACES_DIR%',
                              crash_stacks_directory)
    command = command.replace('%DEVICE_TESTCASES_DIR%',
                              device_testcases_directory)
    command = command.replace('%FUZZER_DIR%', fuzzer_directory)
    command = command.replace('%INPUT_DIR%', input_directory)
    command = command.replace('%ROOT_DIR%', root_directory)
    command = command.replace('%TESTCASE%', testcase_path)
    command = command.replace('%TESTCASE_FILE_URL%', testcase_file_url)
    command = command.replace('%TESTCASE_HTTP_URL%', testcase_http_url)
    command = command.replace('%TMP_DIR%', temp_directory)
    command = command.replace('%USER_PROFILE_DIR%', user_profile_directory)

    if is_android and not launcher:
        # Initial setup phase for command line.
        if write_command_line_file:
            android.adb.write_command_line_file(command, app_path)

        return android.app.get_launch_command(all_app_args, testcase_path,
                                              testcase_file_url)

    # Decide which directory we will run the application from.
    # We are using |app_directory| since it helps to locate pdbs
    # in same directory, other dependencies, etc.
    if os.path.exists(app_directory):
        os.chdir(app_directory)

    return str(command)
Пример #3
0
def run_process(cmdline,
                current_working_directory=None,
                timeout=DEFAULT_TEST_TIMEOUT,
                need_shell=False,
                gestures=None,
                env_copy=None,
                testcase_run=True,
                ignore_children=True):
    """Executes a process with a given command line and other parameters."""
    if environment.is_trusted_host() and testcase_run:
        from bot.untrusted_runner import remote_process_host
        return remote_process_host.run_process(cmdline,
                                               current_working_directory,
                                               timeout, need_shell, gestures,
                                               env_copy, testcase_run,
                                               ignore_children)

    if gestures is None:
        gestures = []

    if env_copy:
        os.environ.update(env_copy)

    # FIXME(mbarbella): Using LAUNCHER_PATH here is error prone. It forces us to
    # do certain operations before fuzzer setup (e.g. bad build check).
    launcher = environment.get_value('LAUNCHER_PATH')
    # This is used when running scripts on native linux OS and not on the device.
    # E.g. running a fuzzer to generate testcases or launcher script.
    plt = environment.platform()
    if plt in ['ANDROID', 'FUCHSIA'] and (not testcase_run or launcher):
        plt = 'LINUX'
    elif plt == 'IOS' and (not testcase_run or launcher):
        plt = 'MAC'

    # Lower down testcase timeout slightly to account for time for crash analysis.
    timeout -= CRASH_ANALYSIS_TIME

    # LeakSanitizer hack - give time for stdout/stderr processing.
    lsan = environment.get_value('LSAN', False)
    if lsan:
        timeout -= LSAN_ANALYSIS_TIME

    # Initialize variables.
    adb_output = None
    process_output = ''
    process_status = None
    return_code = 0
    process_poll_interval = environment.get_value('PROCESS_POLL_INTERVAL', 0.5)
    start_time = time.time()
    watch_for_process_exit = (environment.get_value('WATCH_FOR_PROCESS_EXIT')
                              if plt == 'ANDROID' else True)
    window_list = []

    # Get gesture start time from last element in gesture list.
    gestures = copy.deepcopy(gestures)
    if gestures and gestures[-1].startswith('Trigger'):
        gesture_start_time = int(gestures[-1].split(':')[1])
        gestures.pop()
    else:
        gesture_start_time = timeout // 2

    if plt == 'ANDROID':
        # Clear the log upfront.
        android.logger.clear_log()

        # Run the app.
        adb_output = android.adb.run_command(cmdline, timeout=timeout)
    else:
        cmd, args = shell.get_command_and_arguments(cmdline)

        process_output = mozprocess.processhandler.StoreOutput()
        process_status = ProcessStatus()
        try:
            process_handle = mozprocess.ProcessHandlerMixin(
                cmd,
                args,
                cwd=current_working_directory,
                shell=need_shell,
                processOutputLine=[process_output],
                onFinish=[process_status],
                ignore_children=ignore_children)
            start_process(process_handle)
        except:
            logs.log_error('Exception occurred when running command: %s.' %
                           cmdline)
            return None, None, ''

    while True:
        time.sleep(process_poll_interval)

        # Run the gestures at gesture_start_time or in case we didn't find windows
        # in the last try.
        if (gestures and time.time() - start_time >= gesture_start_time
                and not window_list):
            # In case, we don't find any windows, we increment the gesture start time
            # so that the next check is after 1 second.
            gesture_start_time += 1

            if plt == 'LINUX':
                linux.gestures.run_gestures(gestures, process_handle.pid,
                                            process_status, start_time,
                                            timeout, window_list)
            elif plt == 'WINDOWS':
                windows.gestures.run_gestures(gestures, process_handle.pid,
                                              process_status, start_time,
                                              timeout, window_list)
            elif plt == 'ANDROID':
                android.gestures.run_gestures(gestures, start_time, timeout)

                # TODO(mbarbella): We add a fake window here to prevent gestures on
                # Android from getting executed more than once.
                window_list = ['FAKE']

        if time.time() - start_time >= timeout:
            break

        # Collect the process output.
        output = (android.logger.log_output()
                  if plt == 'ANDROID' else b'\n'.join(process_output.output))
        output = utils.decode_to_unicode(output)
        if crash_analyzer.is_memory_tool_crash(output):
            break

        # Check if we need to bail out on process exit.
        if watch_for_process_exit:
            # If |watch_for_process_exit| is set, then we already completed running
            # our app launch command. So, we can bail out.
            if plt == 'ANDROID':
                break

            # On desktop, we bail out as soon as the process finishes.
            if process_status and process_status.finished:
                # Wait for process shutdown and set return code.
                process_handle.wait(timeout=PROCESS_CLEANUP_WAIT_TIME)
                break

    # Process output based on platform.
    if plt == 'ANDROID':
        # Get current log output. If device is in reboot mode, logcat automatically
        # waits for device to be online.
        time.sleep(ANDROID_CRASH_LOGCAT_WAIT_TIME)
        output = android.logger.log_output()

        if android.constants.LOW_MEMORY_REGEX.search(output):
            # If the device is low on memory, we should force reboot and bail out to
            # prevent device from getting in a frozen state.
            logs.log('Device is low on memory, rebooting.', output=output)
            android.adb.hard_reset()
            android.adb.wait_for_device()

        elif android.adb.time_since_last_reboot() < time.time() - start_time:
            # Check if a reboot has happened, if yes, append log output before reboot
            # and kernel logs content to output.
            log_before_last_reboot = android.logger.log_output_before_last_reboot(
            )
            kernel_log = android.adb.get_kernel_log_content()
            output = '%s%s%s%s%s' % (
                log_before_last_reboot,
                utils.get_line_seperator('Device rebooted'), output,
                utils.get_line_seperator('Kernel Log'), kernel_log)
            # Make sure to reset SE Linux Permissive Mode. This can be done cheaply
            # in ~0.15 sec and is needed especially between runs for kernel crashes.
            android.adb.run_as_root()
            android.settings.change_se_linux_to_permissive_mode()
            return_code = 1

        # Add output from adb to the front.
        if adb_output:
            output = '%s\n\n%s' % (adb_output, output)

        # Kill the application if it is still running. We do this at the end to
        # prevent this from adding noise to the logcat output.
        task_name = environment.get_value('TASK_NAME')
        child_process_termination_pattern = environment.get_value(
            'CHILD_PROCESS_TERMINATION_PATTERN')
        if task_name == 'fuzz' and child_process_termination_pattern:
            # In some cases, we do not want to terminate the application after each
            # run to avoid long startup times (e.g. for chrome). Terminate processes
            # matching a particular pattern for light cleanup in this case.
            android.adb.kill_processes_and_children_matching_name(
                child_process_termination_pattern)
        else:
            # There is no special termination behavior. Simply stop the application.
            android.app.stop()

    else:
        # Get the return code in case the process has finished already.
        # If the process hasn't finished, return_code will be None which is what
        # callers expect unless the output indicates a crash.
        return_code = process_handle.poll()

        # If the process is still running, then terminate it.
        if not process_status.finished:
            launcher_with_interpreter = shell.get_execute_command(
                launcher, is_blackbox_fuzzer=True) if launcher else None
            if (launcher_with_interpreter
                    and cmdline.startswith(launcher_with_interpreter)):
                # If this was a launcher script, we KILL all child processes created
                # except for APP_NAME.
                # It is expected that, if the launcher script terminated normally, it
                # cleans up all the child processes it created itself.
                terminate_root_and_child_processes(process_handle.pid)
            else:
                try:
                    # kill() here actually sends SIGTERM on posix.
                    process_handle.kill()
                except:
                    pass

        if lsan:
            time.sleep(LSAN_ANALYSIS_TIME)

        output = b'\n'.join(process_output.output)
        output = utils.decode_to_unicode(output)

        # X Server hack when max client reached.
        if ('Maximum number of clients reached' in output
                or 'Unable to get connection to X server' in output):
            logs.log_error('Unable to connect to X server, exiting.')
            os.system('sudo killall -9 Xvfb blackbox >/dev/null 2>&1')
            sys.exit(0)

    if testcase_run and (crash_analyzer.is_memory_tool_crash(output)
                         or crash_analyzer.is_check_failure_crash(output)):
        return_code = 1

    # If a crash is found, then we add the memory state as well.
    if return_code and plt == 'ANDROID':
        ps_output = android.adb.get_ps_output()
        if ps_output:
            output += utils.get_line_seperator('Memory Statistics')
            output += ps_output

    if return_code:
        logs.log_warn('Process (%s) ended with exit code (%s).' %
                      (repr(cmdline), str(return_code)),
                      output=output)

    return return_code, round(time.time() - start_time, 1), output
Пример #4
0
 def call_and_assert_helper(self, expected_command, file_to_execute):
     """Call get_execute_command on |file_to_execute| and assert result equal to
 |expected_command|."""
     self.assertEqual(expected_command,
                      shell.get_execute_command(file_to_execute))