def update_data_bundle(fuzzer, data_bundle):
"""Updates a data bundle to the latest version."""
# This module can't be in the global imports due to appengine issues
# with multiprocessing and psutil imports.
from google_cloud_utils import gsutil
# If we are using a data bundle on NFS, it is expected that our testcases
# will usually be large enough that we would fill up our tmpfs directory
# pretty quickly. So, change it to use an on-disk directory.
if not data_bundle.is_local:
testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK')
environment.set_value('FUZZ_INPUTS', testcase_disk_directory)
data_bundle_directory = get_data_bundle_directory(fuzzer.name)
if not data_bundle_directory:
logs.log_error('Failed to setup data bundle %s.' % data_bundle.name)
return False
if not shell.create_directory(data_bundle_directory,
create_intermediates=True):
logs.log_error('Failed to create data bundle %s directory.' %
data_bundle.name)
return False
# Check if data bundle is up to date. If yes, skip the update.
if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
logs.log('Data bundle was recently synced, skip.')
return True
# Fetch lock for this data bundle.
if not _fetch_lock_for_data_bundle_update(data_bundle):
logs.log_error('Failed to lock data bundle %s.' % data_bundle.name)
return False
# Re-check if another bot did the sync already. If yes, skip.
if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
logs.log('Another bot finished the sync, skip.')
_release_lock_for_data_bundle_update(data_bundle)
return True
time_before_sync_start = time.time()
# No need to sync anything if this is a search index data bundle. In that
# case, the fuzzer will generate testcases from a gcs bucket periodically.
if not _is_search_index_data_bundle(data_bundle.name):
bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from bot.untrusted_runner import corpus_manager
from bot.untrusted_runner import file_host
worker_data_bundle_directory = file_host.rebase_to_worker_root(
data_bundle_directory)
file_host.create_directory(worker_data_bundle_directory,
create_intermediates=True)
result = corpus_manager.RemoteGSUtilRunner().rsync(
bucket_url, worker_data_bundle_directory, delete=False)
else:
result = gsutil.GSUtilRunner().rsync(bucket_url,
data_bundle_directory,
delete=False)
if result.return_code != 0:
logs.log_error('Failed to sync data bundle %s: %s.' %
(data_bundle.name, result.output))
_release_lock_for_data_bundle_update(data_bundle)
return False
# Update the testcase list file.
testcase_manager.create_testcase_list_file(data_bundle_directory)
# Write last synced time in the sync file.
sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory)
utils.write_data_to_file(time_before_sync_start, sync_file_path)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from bot.untrusted_runner import file_host
worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path)
file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path)
# Release acquired lock.
_release_lock_for_data_bundle_update(data_bundle)
return True
def get_command_line_for_application(file_to_run='',
user_profile_index=0,
app_path=None,
app_args=None,
needs_http=False,
write_command_line_file=False):
"""Returns the complete command line required to execute application."""
if app_args is None:
app_args = environment.get_value('APP_ARGS')
if app_path is None:
app_path = environment.get_value('APP_PATH')
if not app_path:
# No APP_PATH is available for e.g. grey box fuzzers.
return ''
additional_command_line_flags = get_additional_command_line_flags(
file_to_run)
app_args_append_testcase = environment.get_value(
'APP_ARGS_APPEND_TESTCASE')
app_directory = environment.get_value('APP_DIR')
app_name = environment.get_value('APP_NAME')
apps_argument = environment.get_value('APPS_ARG')
crash_stacks_directory = environment.get_value('CRASH_STACKTRACES_DIR')
debugger = environment.get_value('DEBUGGER_PATH')
device_testcases_directory = android.constants.DEVICE_TESTCASES_DIR
fuzzer_directory = environment.get_value('FUZZER_DIR')
extension_argument = environment.get_value('EXTENSION_ARG')
input_directory = environment.get_value('INPUT_DIR')
plt = environment.platform()
root_directory = environment.get_value('ROOT_DIR')
temp_directory = environment.get_value('BOT_TMPDIR')
user_profile_argument = environment.get_value('USER_PROFILE_ARG')
window_argument = environment.get_value('WINDOW_ARG')
user_profile_directory = get_user_profile_directory(user_profile_index)
# Create user profile directory and setup contents if needed.
setup_user_profile_directory_if_needed(user_profile_directory)
# Handle spaces in APP_PATH.
# If application path has spaces, then we need to quote it.
if ' ' in app_path:
app_path = '"%s"' % app_path
# Prepend command with interpreter if it is a script.
interpreter = shell.get_interpreter(app_name)
if interpreter:
app_path = '%s %s' % (interpreter, app_path)
# Start creating the command line.
command = ''
launcher = environment.get_value('LAUNCHER_PATH')
if environment.is_trusted_host() and not launcher:
# Rebase the file_to_run path to the worker's root (unless we're running
# under a launcher, which runs on the host).
from bot.untrusted_runner import file_host
file_to_run = file_host.rebase_to_worker_root(file_to_run)
# Default case.
testcase_path = file_to_run
testcase_filename = os.path.basename(testcase_path)
testcase_directory = os.path.dirname(testcase_path)
testcase_file_url = utils.file_path_to_file_url(testcase_path)
testcase_http_url = ''
# Determine where |testcase_file_url| should point depending on platform and
# whether or not a launcher script is used.
if file_to_run:
if launcher:
# In the case of launcher scripts, the testcase file to be run resides on
# the host running the launcher script. Thus |testcase_file_url|, which
# may point to a location on the device for Android job types, does not
# apply. Instead, the launcher script should be passed the original file
# to run. By setting |testcase_file_url| to |file_to_run|, we avoid
# duplicating job definitions solely for supporting launcher scripts.
testcase_file_url = file_to_run
# Jobs that have a launcher script which needs to be run on the host will
# have app_name == launcher. In this case don't prepend launcher to
# command - just use app_name.
if os.path.basename(launcher) != app_name:
command += launcher + ' '
elif plt in ['ANDROID']:
# Android-specific testcase path fixup for fuzzers that don't rely on
# launcher scripts.
local_testcases_directory = environment.get_value('FUZZ_INPUTS')
# Check if the file to run is in fuzzed testcases folder. If yes, then we
# can substitute with a local device path. Otherwise, it is part of some
# data bundle with resource dependencies and we just need to use http
# host forwarder for that.
if file_to_run.startswith(local_testcases_directory):
testcase_relative_path = (
file_to_run[len(local_testcases_directory) + 1:])
testcase_path = os.path.join(device_testcases_directory,
testcase_relative_path)
testcase_file_url = utils.file_path_to_file_url(testcase_path)
else:
# Force use of host_forwarder based on comment above.
needs_http = True
# Check if the testcase needs to be loaded over http.
# TODO(ochang): Make this work for trusted/untrusted.
http_ip = '127.0.0.1'
http_port_1 = environment.get_value('HTTP_PORT_1', 8000)
relative_testcase_path = file_to_run[len(input_directory +
os.path.sep):]
relative_testcase_path = relative_testcase_path.replace('\\', '/')
testcase_http_url = 'http://%s:%d/%s' % (http_ip, http_port_1,
relative_testcase_path)
if needs_http:
# TODO(unassigned): Support https.
testcase_file_url = testcase_http_url
testcase_path = testcase_http_url
# Compose app arguments.
all_app_args = ''
if user_profile_argument:
all_app_args += ' %s=%s' % (user_profile_argument,
user_profile_directory)
if extension_argument and EXTENSIONS_PREFIX in testcase_filename:
all_app_args += ' %s=%s' % (extension_argument, testcase_directory)
if apps_argument and APPS_PREFIX in testcase_filename:
all_app_args += ' %s=%s' % (apps_argument, testcase_directory)
if window_argument:
all_app_args += ' %s' % window_argument
if additional_command_line_flags:
all_app_args += ' %s' % additional_command_line_flags.strip()
if app_args:
all_app_args += ' %s' % app_args.strip()
# Append %TESTCASE% at end if no testcase pattern is found in app arguments.
if not utils.sub_string_exists_in(
['%TESTCASE%', '%TESTCASE_FILE_URL%', '%TESTCASE_HTTP_URL%'],
all_app_args) and app_args_append_testcase:
all_app_args += ' %TESTCASE%'
all_app_args = all_app_args.strip()
# Build the actual command to run now.
if debugger:
command += '%s ' % debugger
if app_path:
command += app_path
if all_app_args:
command += ' %s' % all_app_args
command = command.replace('%APP_DIR%', app_directory)
command = command.replace('%CRASH_STACKTRACES_DIR%',
crash_stacks_directory)
command = command.replace('%DEVICE_TESTCASES_DIR%',
device_testcases_directory)
command = command.replace('%FUZZER_DIR%', fuzzer_directory)
command = command.replace('%INPUT_DIR%', input_directory)
command = command.replace('%ROOT_DIR%', root_directory)
command = command.replace('%TESTCASE%', testcase_path)
command = command.replace('%TESTCASE_FILE_URL%', testcase_file_url)
command = command.replace('%TESTCASE_HTTP_URL%', testcase_http_url)
command = command.replace('%TMP_DIR%', temp_directory)
command = command.replace('%USER_PROFILE_DIR%', user_profile_directory)
# Though we attempt to pass all flags that have been used to run html as
# a test in our content shell job types for backwards compatibility, a
# deprecation warning in recent revisions now causes it to fail. Remove
# the --run-layout-test flag to avoid this.
content_shell_app_names = [
'content_shell', 'content_shell.exe', 'Content Shell'
]
if (environment.get_value('APP_NAME') in content_shell_app_names
and environment.get_value('APP_REVISION', 0) >= 558998):
command = command.replace(' --run-layout-test', '')
if plt == 'ANDROID' and not launcher:
# Initial setup phase for command line.
if write_command_line_file:
android.adb.write_command_line_file(command, app_path)
return android.app.get_launch_command(all_app_args, testcase_path,
testcase_file_url)
# Decide which directory we will run the application from.
# We are using |app_directory| since it helps to locate pdbs
# in same directory, other dependencies, etc.
if os.path.exists(app_directory):
os.chdir(app_directory)
return str(command)
def run(self, input_directory, output_directory, no_of_files):
"""Run the fuzzer to generate testcases."""
build_directory = environment.get_value('BUILD_DIR')
if not build_directory:
raise BuiltinFuzzerException(
'BUILD_DIR environment variable is not set.')
fuzzers = fuzzers_utils.get_fuzz_targets(build_directory)
if not fuzzers:
raise BuiltinFuzzerException(
'No fuzzer binaries found in |BUILD_DIR| directory.')
fuzzer_binary_name = environment.get_value('FUZZ_TARGET')
if fuzzer_binary_name:
fuzzer_path = _get_fuzzer_path(fuzzers, fuzzer_binary_name)
else:
fuzzer_path = random.SystemRandom().choice(fuzzers)
fuzzer_binary_name = os.path.basename(fuzzer_path)
project_qualified_name = data_types.fuzz_target_project_qualified_name(
utils.current_project(), fuzzer_binary_name)
corpus_directory = os.path.join(input_directory,
project_qualified_name)
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
corpus_directory = file_host.rebase_to_worker_root(
corpus_directory)
arguments = self.generate_arguments(fuzzer_path)
# Create corpus directory if it does not exist already.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.create_directory(corpus_directory,
create_intermediates=True)
else:
if not os.path.exists(corpus_directory):
os.mkdir(corpus_directory)
# Create fuzz testcases.
for i in range(no_of_files):
# Contents of testcase file don't matter at this point. Need to create
# something non-null so that it is not ignored.
testcase_file_path = os.path.join(output_directory,
'%s%d' % (tests.FUZZ_PREFIX, i))
utils.write_data_to_file(' ', testcase_file_path)
# Write the flags file containing command line for running launcher
# script.
flags_file_path = os.path.join(output_directory,
'%s%d' % (tests.FLAGS_PREFIX, i))
flags = ['%TESTCASE%', fuzzer_binary_name]
if arguments:
flags.append(arguments)
flags_file_content = ' '.join(flags)
utils.write_data_to_file(flags_file_content, flags_file_path)
output = 'Generated %d testcase for fuzzer %s.\n' % (
no_of_files, fuzzer_binary_name)
output += 'metadata::fuzzer_binary_name: %s\n' % fuzzer_binary_name
issue_owners = engine_common.get_issue_owners(fuzzer_path)
if issue_owners:
output += 'metadata::issue_owners: %s\n' % ','.join(issue_owners)
issue_labels = engine_common.get_issue_labels(fuzzer_path)
if issue_labels:
output += 'metadata::issue_labels: %s\n' % ','.join(issue_labels)
# Update *SAN_OPTIONS in current environment from .options file. This
# environment is used in fuzz task later for deriving the environment
# string in |get_environment_settings_as_string| and embedding this as
# part of stacktrace.
engine_common.process_sanitizer_options_overrides(fuzzer_path)
return BuiltinFuzzerResult(output=output,
corpus_directory=corpus_directory)
def update_fuzzer_and_data_bundles(fuzzer_name):
"""Update the fuzzer with a given name if necessary."""
fuzzer = data_types.Fuzzer.query(data_types.Fuzzer.name == fuzzer_name).get()
if not fuzzer:
logs.log_error('No fuzzer exists with name %s.' % fuzzer_name)
raise errors.InvalidFuzzerError
# Set some helper environment variables.
fuzzer_directory = get_fuzzer_directory(fuzzer_name)
environment.set_value('FUZZER_DIR', fuzzer_directory)
environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content)
# If the fuzzer generates large testcases or a large number of small ones
# that don't fit on tmpfs, then use the larger disk directory.
if fuzzer.has_large_testcases:
testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK')
environment.set_value('FUZZ_INPUTS', testcase_disk_directory)
# Adjust the test timeout, if user has provided one.
if fuzzer.timeout:
environment.set_value('TEST_TIMEOUT', fuzzer.timeout)
# Increase fuzz test timeout if the fuzzer timeout is higher than its
# current value.
fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT')
if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout:
environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout)
# Adjust the max testcases if this fuzzer has specified a lower limit.
max_testcases = environment.get_value('MAX_TESTCASES')
if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases:
environment.set_value('MAX_TESTCASES', fuzzer.max_testcases)
# Check for updates to this fuzzer.
version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name)
if (not fuzzer.builtin and
revisions.needs_update(version_file, fuzzer.revision)):
logs.log('Fuzzer update was found, updating.')
# Clear the old fuzzer directory if it exists.
if not shell.remove_directory(fuzzer_directory, recreate=True):
logs.log_error('Failed to clear fuzzer directory.')
return None
# Copy the archive to local disk and unpack it.
archive_path = os.path.join(fuzzer_directory, fuzzer.filename)
if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path):
logs.log_error('Failed to copy fuzzer archive.')
return None
try:
archive.unpack(archive_path, fuzzer_directory)
except Exception:
error_message = ('Failed to unpack fuzzer archive %s '
'(bad archive or unsupported format).') % fuzzer.filename
logs.log_error(error_message)
fuzzer_logs.upload_script_log(
'Fatal error: ' + error_message, fuzzer_name=fuzzer_name)
return None
fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path)
if not os.path.exists(fuzzer_path):
error_message = ('Fuzzer executable %s not found. '
'Check fuzzer configuration.') % fuzzer.executable_path
logs.log_error(error_message)
fuzzer_logs.upload_script_log(
'Fatal error: ' + error_message, fuzzer_name=fuzzer_name)
return None
# Make fuzzer executable.
os.chmod(fuzzer_path, 0o750)
# Cleanup unneeded archive.
shell.remove_file(archive_path)
# Save the current revision of this fuzzer in a file for later checks.
revisions.write_revision_to_revision_file(version_file, fuzzer.revision)
logs.log('Updated fuzzer to revision %d.' % fuzzer.revision)
_clear_old_data_bundles_if_needed()
# Setup data bundles associated with this fuzzer.
data_bundles = ndb_utils.get_all_from_query(
data_types.DataBundle.query(
data_types.DataBundle.name == fuzzer.data_bundle_name))
for data_bundle in data_bundles:
if not update_data_bundle(fuzzer, data_bundle):
return None
# Setup environment variable for launcher script path.
if fuzzer.launcher_script:
fuzzer_launcher_path = os.path.join(fuzzer_directory,
fuzzer.launcher_script)
environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path)
# For launcher script usecase, we need the entire fuzzer directory on the
# worker.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
worker_fuzzer_directory = file_host.rebase_to_worker_root(
fuzzer_directory)
file_host.copy_directory_to_worker(
fuzzer_directory, worker_fuzzer_directory, replace=True)
return fuzzer
def _run_libfuzzer_tool(tool_name,
testcase,
testcase_file_path,
timeout,
expected_crash_state,
set_dedup_flags=False):
"""Run libFuzzer tool to either minimize or cleanse."""
memory_tool_options_var = environment.get_current_memory_tool_var()
saved_memory_tool_options = environment.get_value(memory_tool_options_var)
def _set_dedup_flags():
"""Allow libFuzzer to do its own crash comparison during minimization."""
memory_tool_options = environment.get_memory_tool_options(
memory_tool_options_var)
memory_tool_options['symbolize'] = 1
memory_tool_options['dedup_token_length'] = 3
environment.set_memory_tool_options(memory_tool_options_var,
memory_tool_options)
def _unset_dedup_flags():
"""Reset memory tool options."""
# This is needed so that when we re-run, we can symbolize ourselves
# (ignoring inline frames).
environment.set_value(memory_tool_options_var, saved_memory_tool_options)
output_file_path = get_temporary_file_name(testcase_file_path)
rebased_output_file_path = output_file_path
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.copy_file_to_worker(
testcase_file_path, file_host.rebase_to_worker_root(testcase_file_path))
rebased_output_file_path = file_host.rebase_to_worker_root(output_file_path)
arguments = environment.get_value('APP_ARGS', '')
arguments += (' --cf-{tool_name}-timeout={timeout} '
'--cf-{tool_name}-to={output_file_path}').format(
tool_name=tool_name,
output_file_path=rebased_output_file_path,
timeout=timeout)
command = tests.get_command_line_for_application(
file_to_run=testcase_file_path,
app_args=arguments,
needs_http=testcase.http_flag)
logs.log('Executing command: %s' % command)
if set_dedup_flags:
_set_dedup_flags()
# A small buffer is added to the timeout to allow the current test to
# finish, and file to be written. Since we should terminate beforehand, a
# long delay only slows fuzzing in cases where it's necessary.
_, _, output = process_handler.run_process(command, timeout=timeout + 60)
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.copy_file_from_worker(rebased_output_file_path, output_file_path)
if set_dedup_flags:
_unset_dedup_flags()
if not os.path.exists(output_file_path):
logs.log_warn('LibFuzzer %s run failed.' % tool_name, output=output)
return None, None
# Ensure that the crash parameters match. It's possible that we will
# minimize/cleanse to an unrelated bug, such as a timeout.
crash_result = _run_libfuzzer_testcase(testcase, output_file_path)
state = crash_result.get_symbolized_data()
security_flag = crash_result.is_security_issue()
if (security_flag != testcase.security_flag or
state.crash_state != expected_crash_state):
logs.log_warn('Ignoring unrelated crash.\n'
'State: %s (expected %s)\n'
'Security: %s (expected %s)\n'
'Output: %s\n' %
(state.crash_state, expected_crash_state, security_flag,
testcase.security_flag, state.crash_stacktrace))
return None, None
with open(output_file_path, 'rb') as file_handle:
minimized_keys = blobs.write_blob(file_handle)
testcase.minimized_keys = minimized_keys
testcase.put()
return output_file_path, crash_result
