def update_data_bundle(fuzzer, data_bundle): """Updates a data bundle to the latest version.""" # This module can't be in the global imports due to appengine issues # with multiprocessing and psutil imports. from google_cloud_utils import gsutil # If we are using a data bundle on NFS, it is expected that our testcases # will usually be large enough that we would fill up our tmpfs directory # pretty quickly. So, change it to use an on-disk directory. if not data_bundle.is_local: testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK') environment.set_value('FUZZ_INPUTS', testcase_disk_directory) data_bundle_directory = get_data_bundle_directory(fuzzer.name) if not data_bundle_directory: logs.log_error('Failed to setup data bundle %s.' % data_bundle.name) return False if not shell.create_directory(data_bundle_directory, create_intermediates=True): logs.log_error('Failed to create data bundle %s directory.' % data_bundle.name) return False # Check if data bundle is up to date. If yes, skip the update. if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): logs.log('Data bundle was recently synced, skip.') return True # Fetch lock for this data bundle. if not _fetch_lock_for_data_bundle_update(data_bundle): logs.log_error('Failed to lock data bundle %s.' % data_bundle.name) return False # Re-check if another bot did the sync already. If yes, skip. if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): logs.log('Another bot finished the sync, skip.') _release_lock_for_data_bundle_update(data_bundle) return True time_before_sync_start = time.time() # No need to sync anything if this is a search index data bundle. In that # case, the fuzzer will generate testcases from a gcs bucket periodically. if not _is_search_index_data_bundle(data_bundle.name): bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name) if environment.is_trusted_host() and data_bundle.sync_to_worker: from bot.untrusted_runner import corpus_manager from bot.untrusted_runner import file_host worker_data_bundle_directory = file_host.rebase_to_worker_root( data_bundle_directory) file_host.create_directory(worker_data_bundle_directory, create_intermediates=True) result = corpus_manager.RemoteGSUtilRunner().rsync( bucket_url, worker_data_bundle_directory, delete=False) else: result = gsutil.GSUtilRunner().rsync(bucket_url, data_bundle_directory, delete=False) if result.return_code != 0: logs.log_error('Failed to sync data bundle %s: %s.' % (data_bundle.name, result.output)) _release_lock_for_data_bundle_update(data_bundle) return False # Update the testcase list file. testcase_manager.create_testcase_list_file(data_bundle_directory) # Write last synced time in the sync file. sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory) utils.write_data_to_file(time_before_sync_start, sync_file_path) if environment.is_trusted_host() and data_bundle.sync_to_worker: from bot.untrusted_runner import file_host worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path) file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path) # Release acquired lock. _release_lock_for_data_bundle_update(data_bundle) return True
def get_command_line_for_application(file_to_run='', user_profile_index=0, app_path=None, app_args=None, needs_http=False, write_command_line_file=False): """Returns the complete command line required to execute application.""" if app_args is None: app_args = environment.get_value('APP_ARGS') if app_path is None: app_path = environment.get_value('APP_PATH') if not app_path: # No APP_PATH is available for e.g. grey box fuzzers. return '' additional_command_line_flags = get_additional_command_line_flags( file_to_run) app_args_append_testcase = environment.get_value( 'APP_ARGS_APPEND_TESTCASE') app_directory = environment.get_value('APP_DIR') app_name = environment.get_value('APP_NAME') apps_argument = environment.get_value('APPS_ARG') crash_stacks_directory = environment.get_value('CRASH_STACKTRACES_DIR') debugger = environment.get_value('DEBUGGER_PATH') device_testcases_directory = android.constants.DEVICE_TESTCASES_DIR fuzzer_directory = environment.get_value('FUZZER_DIR') extension_argument = environment.get_value('EXTENSION_ARG') input_directory = environment.get_value('INPUT_DIR') plt = environment.platform() root_directory = environment.get_value('ROOT_DIR') temp_directory = environment.get_value('BOT_TMPDIR') user_profile_argument = environment.get_value('USER_PROFILE_ARG') window_argument = environment.get_value('WINDOW_ARG') user_profile_directory = get_user_profile_directory(user_profile_index) # Create user profile directory and setup contents if needed. setup_user_profile_directory_if_needed(user_profile_directory) # Handle spaces in APP_PATH. # If application path has spaces, then we need to quote it. if ' ' in app_path: app_path = '"%s"' % app_path # Prepend command with interpreter if it is a script. interpreter = shell.get_interpreter(app_name) if interpreter: app_path = '%s %s' % (interpreter, app_path) # Start creating the command line. command = '' launcher = environment.get_value('LAUNCHER_PATH') if environment.is_trusted_host() and not launcher: # Rebase the file_to_run path to the worker's root (unless we're running # under a launcher, which runs on the host). from bot.untrusted_runner import file_host file_to_run = file_host.rebase_to_worker_root(file_to_run) # Default case. testcase_path = file_to_run testcase_filename = os.path.basename(testcase_path) testcase_directory = os.path.dirname(testcase_path) testcase_file_url = utils.file_path_to_file_url(testcase_path) testcase_http_url = '' # Determine where |testcase_file_url| should point depending on platform and # whether or not a launcher script is used. if file_to_run: if launcher: # In the case of launcher scripts, the testcase file to be run resides on # the host running the launcher script. Thus |testcase_file_url|, which # may point to a location on the device for Android job types, does not # apply. Instead, the launcher script should be passed the original file # to run. By setting |testcase_file_url| to |file_to_run|, we avoid # duplicating job definitions solely for supporting launcher scripts. testcase_file_url = file_to_run # Jobs that have a launcher script which needs to be run on the host will # have app_name == launcher. In this case don't prepend launcher to # command - just use app_name. if os.path.basename(launcher) != app_name: command += launcher + ' ' elif plt in ['ANDROID']: # Android-specific testcase path fixup for fuzzers that don't rely on # launcher scripts. local_testcases_directory = environment.get_value('FUZZ_INPUTS') # Check if the file to run is in fuzzed testcases folder. If yes, then we # can substitute with a local device path. Otherwise, it is part of some # data bundle with resource dependencies and we just need to use http # host forwarder for that. if file_to_run.startswith(local_testcases_directory): testcase_relative_path = ( file_to_run[len(local_testcases_directory) + 1:]) testcase_path = os.path.join(device_testcases_directory, testcase_relative_path) testcase_file_url = utils.file_path_to_file_url(testcase_path) else: # Force use of host_forwarder based on comment above. needs_http = True # Check if the testcase needs to be loaded over http. # TODO(ochang): Make this work for trusted/untrusted. http_ip = '127.0.0.1' http_port_1 = environment.get_value('HTTP_PORT_1', 8000) relative_testcase_path = file_to_run[len(input_directory + os.path.sep):] relative_testcase_path = relative_testcase_path.replace('\\', '/') testcase_http_url = 'http://%s:%d/%s' % (http_ip, http_port_1, relative_testcase_path) if needs_http: # TODO(unassigned): Support https. testcase_file_url = testcase_http_url testcase_path = testcase_http_url # Compose app arguments. all_app_args = '' if user_profile_argument: all_app_args += ' %s=%s' % (user_profile_argument, user_profile_directory) if extension_argument and EXTENSIONS_PREFIX in testcase_filename: all_app_args += ' %s=%s' % (extension_argument, testcase_directory) if apps_argument and APPS_PREFIX in testcase_filename: all_app_args += ' %s=%s' % (apps_argument, testcase_directory) if window_argument: all_app_args += ' %s' % window_argument if additional_command_line_flags: all_app_args += ' %s' % additional_command_line_flags.strip() if app_args: all_app_args += ' %s' % app_args.strip() # Append %TESTCASE% at end if no testcase pattern is found in app arguments. if not utils.sub_string_exists_in( ['%TESTCASE%', '%TESTCASE_FILE_URL%', '%TESTCASE_HTTP_URL%'], all_app_args) and app_args_append_testcase: all_app_args += ' %TESTCASE%' all_app_args = all_app_args.strip() # Build the actual command to run now. if debugger: command += '%s ' % debugger if app_path: command += app_path if all_app_args: command += ' %s' % all_app_args command = command.replace('%APP_DIR%', app_directory) command = command.replace('%CRASH_STACKTRACES_DIR%', crash_stacks_directory) command = command.replace('%DEVICE_TESTCASES_DIR%', device_testcases_directory) command = command.replace('%FUZZER_DIR%', fuzzer_directory) command = command.replace('%INPUT_DIR%', input_directory) command = command.replace('%ROOT_DIR%', root_directory) command = command.replace('%TESTCASE%', testcase_path) command = command.replace('%TESTCASE_FILE_URL%', testcase_file_url) command = command.replace('%TESTCASE_HTTP_URL%', testcase_http_url) command = command.replace('%TMP_DIR%', temp_directory) command = command.replace('%USER_PROFILE_DIR%', user_profile_directory) # Though we attempt to pass all flags that have been used to run html as # a test in our content shell job types for backwards compatibility, a # deprecation warning in recent revisions now causes it to fail. Remove # the --run-layout-test flag to avoid this. content_shell_app_names = [ 'content_shell', 'content_shell.exe', 'Content Shell' ] if (environment.get_value('APP_NAME') in content_shell_app_names and environment.get_value('APP_REVISION', 0) >= 558998): command = command.replace(' --run-layout-test', '') if plt == 'ANDROID' and not launcher: # Initial setup phase for command line. if write_command_line_file: android.adb.write_command_line_file(command, app_path) return android.app.get_launch_command(all_app_args, testcase_path, testcase_file_url) # Decide which directory we will run the application from. # We are using |app_directory| since it helps to locate pdbs # in same directory, other dependencies, etc. if os.path.exists(app_directory): os.chdir(app_directory) return str(command)
def run(self, input_directory, output_directory, no_of_files): """Run the fuzzer to generate testcases.""" build_directory = environment.get_value('BUILD_DIR') if not build_directory: raise BuiltinFuzzerException( 'BUILD_DIR environment variable is not set.') fuzzers = fuzzers_utils.get_fuzz_targets(build_directory) if not fuzzers: raise BuiltinFuzzerException( 'No fuzzer binaries found in |BUILD_DIR| directory.') fuzzer_binary_name = environment.get_value('FUZZ_TARGET') if fuzzer_binary_name: fuzzer_path = _get_fuzzer_path(fuzzers, fuzzer_binary_name) else: fuzzer_path = random.SystemRandom().choice(fuzzers) fuzzer_binary_name = os.path.basename(fuzzer_path) project_qualified_name = data_types.fuzz_target_project_qualified_name( utils.current_project(), fuzzer_binary_name) corpus_directory = os.path.join(input_directory, project_qualified_name) if environment.is_trusted_host(): from bot.untrusted_runner import file_host corpus_directory = file_host.rebase_to_worker_root( corpus_directory) arguments = self.generate_arguments(fuzzer_path) # Create corpus directory if it does not exist already. if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.create_directory(corpus_directory, create_intermediates=True) else: if not os.path.exists(corpus_directory): os.mkdir(corpus_directory) # Create fuzz testcases. for i in range(no_of_files): # Contents of testcase file don't matter at this point. Need to create # something non-null so that it is not ignored. testcase_file_path = os.path.join(output_directory, '%s%d' % (tests.FUZZ_PREFIX, i)) utils.write_data_to_file(' ', testcase_file_path) # Write the flags file containing command line for running launcher # script. flags_file_path = os.path.join(output_directory, '%s%d' % (tests.FLAGS_PREFIX, i)) flags = ['%TESTCASE%', fuzzer_binary_name] if arguments: flags.append(arguments) flags_file_content = ' '.join(flags) utils.write_data_to_file(flags_file_content, flags_file_path) output = 'Generated %d testcase for fuzzer %s.\n' % ( no_of_files, fuzzer_binary_name) output += 'metadata::fuzzer_binary_name: %s\n' % fuzzer_binary_name issue_owners = engine_common.get_issue_owners(fuzzer_path) if issue_owners: output += 'metadata::issue_owners: %s\n' % ','.join(issue_owners) issue_labels = engine_common.get_issue_labels(fuzzer_path) if issue_labels: output += 'metadata::issue_labels: %s\n' % ','.join(issue_labels) # Update *SAN_OPTIONS in current environment from .options file. This # environment is used in fuzz task later for deriving the environment # string in |get_environment_settings_as_string| and embedding this as # part of stacktrace. engine_common.process_sanitizer_options_overrides(fuzzer_path) return BuiltinFuzzerResult(output=output, corpus_directory=corpus_directory)
def update_fuzzer_and_data_bundles(fuzzer_name): """Update the fuzzer with a given name if necessary.""" fuzzer = data_types.Fuzzer.query(data_types.Fuzzer.name == fuzzer_name).get() if not fuzzer: logs.log_error('No fuzzer exists with name %s.' % fuzzer_name) raise errors.InvalidFuzzerError # Set some helper environment variables. fuzzer_directory = get_fuzzer_directory(fuzzer_name) environment.set_value('FUZZER_DIR', fuzzer_directory) environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content) # If the fuzzer generates large testcases or a large number of small ones # that don't fit on tmpfs, then use the larger disk directory. if fuzzer.has_large_testcases: testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK') environment.set_value('FUZZ_INPUTS', testcase_disk_directory) # Adjust the test timeout, if user has provided one. if fuzzer.timeout: environment.set_value('TEST_TIMEOUT', fuzzer.timeout) # Increase fuzz test timeout if the fuzzer timeout is higher than its # current value. fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT') if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout: environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout) # Adjust the max testcases if this fuzzer has specified a lower limit. max_testcases = environment.get_value('MAX_TESTCASES') if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases: environment.set_value('MAX_TESTCASES', fuzzer.max_testcases) # Check for updates to this fuzzer. version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name) if (not fuzzer.builtin and revisions.needs_update(version_file, fuzzer.revision)): logs.log('Fuzzer update was found, updating.') # Clear the old fuzzer directory if it exists. if not shell.remove_directory(fuzzer_directory, recreate=True): logs.log_error('Failed to clear fuzzer directory.') return None # Copy the archive to local disk and unpack it. archive_path = os.path.join(fuzzer_directory, fuzzer.filename) if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path): logs.log_error('Failed to copy fuzzer archive.') return None try: archive.unpack(archive_path, fuzzer_directory) except Exception: error_message = ('Failed to unpack fuzzer archive %s ' '(bad archive or unsupported format).') % fuzzer.filename logs.log_error(error_message) fuzzer_logs.upload_script_log( 'Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return None fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path) if not os.path.exists(fuzzer_path): error_message = ('Fuzzer executable %s not found. ' 'Check fuzzer configuration.') % fuzzer.executable_path logs.log_error(error_message) fuzzer_logs.upload_script_log( 'Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return None # Make fuzzer executable. os.chmod(fuzzer_path, 0o750) # Cleanup unneeded archive. shell.remove_file(archive_path) # Save the current revision of this fuzzer in a file for later checks. revisions.write_revision_to_revision_file(version_file, fuzzer.revision) logs.log('Updated fuzzer to revision %d.' % fuzzer.revision) _clear_old_data_bundles_if_needed() # Setup data bundles associated with this fuzzer. data_bundles = ndb_utils.get_all_from_query( data_types.DataBundle.query( data_types.DataBundle.name == fuzzer.data_bundle_name)) for data_bundle in data_bundles: if not update_data_bundle(fuzzer, data_bundle): return None # Setup environment variable for launcher script path. if fuzzer.launcher_script: fuzzer_launcher_path = os.path.join(fuzzer_directory, fuzzer.launcher_script) environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path) # For launcher script usecase, we need the entire fuzzer directory on the # worker. if environment.is_trusted_host(): from bot.untrusted_runner import file_host worker_fuzzer_directory = file_host.rebase_to_worker_root( fuzzer_directory) file_host.copy_directory_to_worker( fuzzer_directory, worker_fuzzer_directory, replace=True) return fuzzer
def _run_libfuzzer_tool(tool_name, testcase, testcase_file_path, timeout, expected_crash_state, set_dedup_flags=False): """Run libFuzzer tool to either minimize or cleanse.""" memory_tool_options_var = environment.get_current_memory_tool_var() saved_memory_tool_options = environment.get_value(memory_tool_options_var) def _set_dedup_flags(): """Allow libFuzzer to do its own crash comparison during minimization.""" memory_tool_options = environment.get_memory_tool_options( memory_tool_options_var) memory_tool_options['symbolize'] = 1 memory_tool_options['dedup_token_length'] = 3 environment.set_memory_tool_options(memory_tool_options_var, memory_tool_options) def _unset_dedup_flags(): """Reset memory tool options.""" # This is needed so that when we re-run, we can symbolize ourselves # (ignoring inline frames). environment.set_value(memory_tool_options_var, saved_memory_tool_options) output_file_path = get_temporary_file_name(testcase_file_path) rebased_output_file_path = output_file_path if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.copy_file_to_worker( testcase_file_path, file_host.rebase_to_worker_root(testcase_file_path)) rebased_output_file_path = file_host.rebase_to_worker_root(output_file_path) arguments = environment.get_value('APP_ARGS', '') arguments += (' --cf-{tool_name}-timeout={timeout} ' '--cf-{tool_name}-to={output_file_path}').format( tool_name=tool_name, output_file_path=rebased_output_file_path, timeout=timeout) command = tests.get_command_line_for_application( file_to_run=testcase_file_path, app_args=arguments, needs_http=testcase.http_flag) logs.log('Executing command: %s' % command) if set_dedup_flags: _set_dedup_flags() # A small buffer is added to the timeout to allow the current test to # finish, and file to be written. Since we should terminate beforehand, a # long delay only slows fuzzing in cases where it's necessary. _, _, output = process_handler.run_process(command, timeout=timeout + 60) if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.copy_file_from_worker(rebased_output_file_path, output_file_path) if set_dedup_flags: _unset_dedup_flags() if not os.path.exists(output_file_path): logs.log_warn('LibFuzzer %s run failed.' % tool_name, output=output) return None, None # Ensure that the crash parameters match. It's possible that we will # minimize/cleanse to an unrelated bug, such as a timeout. crash_result = _run_libfuzzer_testcase(testcase, output_file_path) state = crash_result.get_symbolized_data() security_flag = crash_result.is_security_issue() if (security_flag != testcase.security_flag or state.crash_state != expected_crash_state): logs.log_warn('Ignoring unrelated crash.\n' 'State: %s (expected %s)\n' 'Security: %s (expected %s)\n' 'Output: %s\n' % (state.crash_state, expected_crash_state, security_flag, testcase.security_flag, state.crash_stacktrace)) return None, None with open(output_file_path, 'rb') as file_handle: minimized_keys = blobs.write_blob(file_handle) testcase.minimized_keys = minimized_keys testcase.put() return output_file_path, crash_result