def download_repo_prop_if_needed(symbols_directory, build_id, cache_target,
targets_with_type_and_san, cache_type):
"""Downloads the repo.prop for a branch"""
artifact_file_name = 'repo.prop'
symbols_archive_filename = get_repo_prop_archive_filename(
build_id, cache_target)
output_filename_override = symbols_archive_filename
# We create our own build_params for cache
build_params = {
'build_id': build_id,
'target': cache_target,
'type': cache_type
}
build_params_check_path = os.path.join(symbols_directory,
'.cached_build_params')
symbols_archive_path = os.path.join(symbols_directory,
symbols_archive_filename)
download_artifact_if_needed(build_id, symbols_directory,
symbols_archive_path,
targets_with_type_and_san, artifact_file_name,
output_filename_override, build_params,
build_params_check_path)
if not os.path.exists(symbols_archive_path):
logs.log_error('Unable to locate repo.prop %s.' % symbols_archive_path)
return
# Store the artifact for later use or for use by other bots.
storage.store_file_in_cache(symbols_archive_path)
utils.write_data_to_file(build_params, build_params_check_path)
def _http_request(url,
body=None,
method=_POST_METHOD,
force_reauthorization=False):
"""Make a POST request to the specified URL."""
authorization = _get_authorization(force_reauthorization)
headers = {
'User-Agent': 'clusterfuzz-reproduce',
'Authorization': authorization
}
http = httplib2.Http()
request_body = json_utils.dumps(body) if body else ''
response, content = http.request(url,
method=method,
headers=headers,
body=request_body)
# If the server returns 401 we may need to reauthenticate. Try the request
# a second time if this happens.
if response.status == 401 and not force_reauthorization:
return _http_request(url,
body,
method=method,
force_reauthorization=True)
if 'x-clusterfuzz-authorization' in response:
shell.create_directory(os.path.dirname(AUTHORIZATION_CACHE_FILE),
create_intermediates=True)
utils.write_data_to_file(response['x-clusterfuzz-authorization'],
AUTHORIZATION_CACHE_FILE)
return response, content
def test_recommended_dictionary_merge(self):
"""Test merging with GCS copy of recommended dictionary."""
fake_gcs_dict_path = os.path.join(
DATA_DIRECTORY, 'fake_gcs_recommended_dictionary.txt')
dict_manager = dictionary_manager.DictionaryManager('fuzzer_name')
log_data = utils.read_data_from_file(os.path.join(
DATA_DIRECTORY, 'log_with_recommended_dict.txt'),
eval_data=False).decode('utf-8')
dict_from_log = dict_manager.parse_recommended_dictionary_from_data(
log_data)
utils.write_data_to_file('\n'.join(dict_from_log),
self.local_dict_path)
dictionary_manager.merge_dictionary_files(self.local_dict_path,
fake_gcs_dict_path,
self.local_dict_path)
# Compare resulting dictionary with its expected result.
merged_dictionary = self._parse_dictionary_file(self.local_dict_path)
expected_dictionary_path = os.path.join(
DATA_DIRECTORY, 'expected_merged_recommended_dictionary.txt')
expected_dictionary = self._parse_dictionary_file(
expected_dictionary_path)
self.assertEqual(sorted(merged_dictionary),
sorted(expected_dictionary))
def test(self):
"""Tests copy_local_directory_to_remote."""
utils.write_data_to_file('a', os.path.join(self.local_temp_dir, 'a'))
shell.create_directory(os.path.join(self.local_temp_dir, 'b'))
utils.write_data_to_file('c', os.path.join(self.local_temp_dir, 'b', 'c'))
adb.copy_local_directory_to_remote(self.local_temp_dir,
self.device_temp_dir)
self.assertTrue(adb.file_exists(os.path.join(self.device_temp_dir, 'a')))
self.assertFalse(
adb.directory_exists(os.path.join(self.device_temp_dir, 'a')))
self.assertEqual(
adb.get_file_size(os.path.join(self.device_temp_dir, 'a')), 1)
self.assertTrue(
adb.directory_exists(os.path.join(self.device_temp_dir, 'b')))
self.assertFalse(adb.file_exists(os.path.join(self.device_temp_dir, 'b')))
self.assertTrue(
adb.file_exists(os.path.join(self.device_temp_dir, 'b', 'c')))
self.assertFalse(
adb.directory_exists(os.path.join(self.device_temp_dir, 'b', 'c')))
self.assertEqual(
adb.get_file_size(os.path.join(self.device_temp_dir, 'b', 'c')), 1)
def minimize_resource(test_runner, dependency, input_directory, main_file):
"""Minimize a resource for the test case."""
# TODO(mbarbella): Simplify this with refactoring of setup_testcase.
offset = len(input_directory) + len(os.path.sep)
fixed_testcase_file_path = main_file[offset:]
dependency_absolute_path = os.path.join(input_directory, dependency)
if (dependency == fixed_testcase_file_path or dependency == main_file or
not can_minimize_file(dependency_absolute_path)):
return
get_temp_file = functools.partial(
get_temporary_file, dependency_absolute_path, no_modifications=True)
original_data = utils.get_file_contents_with_fatal_error_on_failure(
dependency_absolute_path)
dependency_data = (
minimize_file(
dependency,
test_runner.test_with_defaults,
get_temp_file,
original_data,
test_runner.deadline,
1,
test_runner.cleanup_interval,
delete_temp_files=False))
utils.write_data_to_file(dependency_data, dependency_absolute_path)
logs.log('Minimized dependency file: %s' % dependency)
def test_sync_with_failed_last_update(self):
"""Test corpus sync when failed to get last update info from gcs."""
corpus = fuzz_task.GcsCorpus('parent', 'child', '/dir', '/dir1')
utils.write_data_to_file(time.time(), '/dir1/.child_sync')
self.mock.last_updated.return_value = None
self.assertTrue(corpus.sync_from_gcs())
self.assertEqual(1, self.mock.rsync_to_disk.call_count)
def test_no_sync(self):
"""Test no corpus sync when bundle is not updated since last sync."""
corpus = fuzz_task.GcsCorpus('parent', 'child', '/dir', '/dir1')
utils.write_data_to_file(time.time(), '/dir1/.child_sync')
self.mock.last_updated.return_value = (
datetime.datetime.utcnow() - datetime.timedelta(days=1))
self.assertTrue(corpus.sync_from_gcs())
self.assertEqual(0, self.mock.rsync_to_disk.call_count)
def run(self, input_directory, output_directory, no_of_files):
"""Run the fuzzer to generate testcases."""
fuzzer_binary_name, fuzzer_path = self._get_fuzzer_binary_name_and_path(
)
project_qualified_name = data_types.fuzz_target_project_qualified_name(
utils.current_project(), fuzzer_binary_name)
arguments = self.generate_arguments(fuzzer_path)
corpus_directory = get_corpus_directory(input_directory,
project_qualified_name)
# Create fuzz testcases.
for i in range(no_of_files):
# Contents of testcase file don't matter at this point. Need to create
# something non-null so that it is not ignored.
testcase_file_path = os.path.join(
output_directory, '%s%d' % (testcase_manager.FUZZ_PREFIX, i))
utils.write_data_to_file(' ', testcase_file_path)
# Write the flags file containing command line for running launcher
# script.
flags_file_path = os.path.join(
output_directory, '%s%d' % (testcase_manager.FLAGS_PREFIX, i))
flags = ['%TESTCASE%', fuzzer_binary_name]
if arguments:
flags.append(arguments)
flags_file_content = ' '.join(flags)
utils.write_data_to_file(flags_file_content, flags_file_path)
output = 'Generated %d testcase for fuzzer %s.\n' % (
no_of_files, fuzzer_binary_name)
output += 'metadata::fuzzer_binary_name: %s\n' % fuzzer_binary_name
issue_owners = engine_common.get_issue_owners(fuzzer_path)
if issue_owners:
output += 'metadata::issue_owners: %s\n' % ','.join(issue_owners)
issue_labels = engine_common.get_issue_labels(fuzzer_path)
if issue_labels:
output += 'metadata::issue_labels: %s\n' % ','.join(issue_labels)
issue_components = engine_common.get_issue_components(fuzzer_path)
if issue_components:
output += 'metadata::issue_components: %s\n' % ','.join(
issue_components)
# Update *SAN_OPTIONS in current environment from .options file. This
# environment is used in fuzz task later for deriving the environment
# string in |get_environment_settings_as_string| and embedding this as
# part of stacktrace.
engine_common.process_sanitizer_options_overrides(fuzzer_path)
return BuiltinFuzzerResult(output=output,
corpus_directory=corpus_directory)
def write_dummy_file(input_dir):
"""Afl will refuse to run if the corpus directory is empty or contains empty
files. So write the bare minimum to get afl to run if there is no corpus
yet."""
# TODO(metzman): Ask lcamtuf to allow AFL to run with an empty input corpus.
dummy_input_path = os.path.join(input_dir, AFL_DUMMY_INPUT)
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.write_data_to_worker(' ', dummy_input_path)
else:
utils.write_data_to_file(' ', dummy_input_path)
def correct_if_needed(dict_path):
"""Corrects obvious errors such as missing quotes in a dictionary."""
if not dict_path or not os.path.exists(dict_path):
return
content = utils.read_data_from_file(dict_path, eval_data=False)
new_content = ''
for current_line in content.splitlines():
new_content += _fix_dictionary_line(current_line, dict_path) + '\n'
# End of file newlines are inconsistent in dictionaries.
if new_content.rstrip('\n') != content.rstrip('\n'):
utils.write_data_to_file(new_content, dict_path)
def download_system_symbols_if_needed(symbols_directory):
"""Download system libraries from |SYMBOLS_URL| and cache locally."""
if not should_download_symbols():
return
# Get the build fingerprint parameters.
build_params = settings.get_build_parameters()
if not build_params:
logs.log_error('Unable to determine build parameters.')
return
build_params_check_path = os.path.join(symbols_directory,
'.cached_build_params')
build_id = build_params.get('build_id')
target = build_params.get('target')
build_type = build_params.get('type')
if not build_id or not target or not build_type:
logs.log_error('Null build parameters found, exiting.')
return
symbols_archive_filename = f'{target}-symbols-{build_id}.zip'
artifact_file_name = symbols_archive_filename
output_filename_override = None
# Include type and sanitizer information in the target.
tool_suffix = environment.get_value('SANITIZER_TOOL_NAME')
target_with_type_and_san = f'{target}-{build_type}'
if tool_suffix and not tool_suffix in target_with_type_and_san:
target_with_type_and_san += f'_{tool_suffix}'
targets_with_type_and_san = [target_with_type_and_san]
symbols_archive_path = os.path.join(symbols_directory,
symbols_archive_filename)
download_artifact_if_needed(build_id, symbols_directory,
symbols_archive_path,
targets_with_type_and_san, artifact_file_name,
output_filename_override, build_params,
build_params_check_path)
if not os.path.exists(symbols_archive_path):
logs.log_error('Unable to locate symbols archive %s.' %
symbols_archive_path)
return
# Store the artifact for later use or for use by other bots.
storage.store_file_in_cache(symbols_archive_path)
archive.unpack(symbols_archive_path, symbols_directory, trusted=True)
shell.remove_file(symbols_archive_path)
utils.write_data_to_file(build_params, build_params_check_path)
def _download_testcase(testcase_id, testcase, configuration):
"""Download the test case and return its path."""
print('Downloading testcase...')
testcase_download_url = '{url}?id={id}'.format(
url=configuration.get('testcase_download_url'), id=testcase_id)
response, content = http_utils.request(
testcase_download_url,
method=http_utils.GET_METHOD,
configuration=configuration)
if response.status != 200:
raise errors.ReproduceToolUnrecoverableError(
'Unable to download test case.')
bot_absolute_filename = response['x-goog-meta-filename']
# Store the test case in the config directory for debuggability.
testcase_directory = os.path.join(CONFIG_DIRECTORY, 'current-testcase')
shell.remove_directory(testcase_directory, recreate=True)
environment.set_value('FUZZ_INPUTS', testcase_directory)
testcase_path = os.path.join(testcase_directory,
os.path.basename(bot_absolute_filename))
utils.write_data_to_file(content, testcase_path)
# Unpack the test case if it's archived.
# TODO(mbarbella): Rewrite setup.unpack_testcase and share this code.
if testcase.minimized_keys and testcase.minimized_keys != 'NA':
mask = data_types.ArchiveStatus.MINIMIZED
else:
mask = data_types.ArchiveStatus.FUZZED
if testcase.archive_state & mask:
archive.unpack(testcase_path, testcase_directory)
file_list = archive.get_file_list(testcase_path)
testcase_path = None
for file_name in file_list:
if os.path.basename(file_name) == os.path.basename(
testcase.absolute_path):
testcase_path = os.path.join(testcase_directory, file_name)
break
if not testcase_path:
raise errors.ReproduceToolUnrecoverableError(
'Test case file was not found in archive.\n'
'Original filename: {absolute_path}.\n'
'Archive contents: {file_list}'.format(
absolute_path=testcase.absolute_path, file_list=file_list))
return testcase_path
def _post_setup_success(self, update_revision=True):
"""Common post-setup."""
if update_revision:
self._write_revision()
# Update timestamp to indicate when this build was last used.
if self.base_build_dir:
timestamp_file_path = os.path.join(self.base_build_dir, TIMESTAMP_FILE)
utils.write_data_to_file(time.time(), timestamp_file_path)
# Update rpaths if necessary (for e.g. instrumented libraries).
instrumented_library_paths = environment.get_instrumented_libraries_paths()
if instrumented_library_paths:
self._patch_rpaths(instrumented_library_paths)
def test_copy_local_directory_to_remote(self):
"""Tests copy_local_directory_to_remote."""
utils.write_data_to_file('a', os.path.join(self.local_temp_dir, 'a'))
shell.create_directory_if_needed(os.path.join(self.local_temp_dir,
'b'))
utils.write_data_to_file('c',
os.path.join(self.local_temp_dir, 'b', 'c'))
adb.copy_local_directory_to_remote(self.local_temp_dir,
self.device_temp_dir)
self.assertTrue(
adb.file_exists(os.path.join(self.device_temp_dir, 'a')))
self.assertTrue(
adb.directory_exists(os.path.join(self.device_temp_dir, 'b')))
self.assertTrue(
adb.file_exists(os.path.join(self.device_temp_dir, 'b', 'c')))
def setup_user_profile_directory_if_needed(user_profile_directory):
"""Set user profile directory if it does not exist."""
if os.path.exists(user_profile_directory):
# User profile directory already exists. Bail out.
return
shell.create_directory(user_profile_directory)
# Create a file in user profile directory based on format:
# filename;base64 encoded zlib compressed file contents.
user_profile_file = environment.get_value('USER_PROFILE_FILE')
if user_profile_file and ';' in user_profile_file:
user_profile_filename, encoded_file_contents = (
user_profile_file.split(';', 1))
user_profile_file_contents = zlib.decompress(
base64.b64decode(encoded_file_contents))
user_profile_file_path = os.path.join(user_profile_directory,
user_profile_filename)
utils.write_data_to_file(user_profile_file_contents,
user_profile_file_path)
# For Firefox, we need to install a special fuzzPriv extension that exposes
# special functions to javascript, e.g. gc(), etc.
app_name = environment.get_value('APP_NAME')
if app_name.startswith('firefox'):
# Create extensions directory.
extensions_directory = os.path.join(user_profile_directory,
'extensions')
shell.create_directory(extensions_directory)
# Unpack the fuzzPriv extension.
extension_archive = os.path.join(environment.get_resources_directory(),
'firefox', 'fuzzPriv-extension.zip')
archive.unpack(extension_archive, extensions_directory)
# Add this extension in the extensions configuration file.
extension_config_file_path = os.path.join(user_profile_directory,
'extensions.ini')
fuzz_extension_directory = os.path.join(extensions_directory,
'*****@*****.**')
extension_config_file_contents = ('[ExtensionDirs]\r\n'
'Extension0=%s\r\n'
'\r\n'
'[ThemeDirs]\r\n' %
fuzz_extension_directory)
utils.write_data_to_file(extension_config_file_contents,
extension_config_file_path)
def test_file_exists(self):
"""Tests file_exists."""
utils.write_data_to_file('a', os.path.join(self.local_temp_dir, 'a'))
shell.create_directory(os.path.join(self.local_temp_dir, 'b'))
utils.write_data_to_file('c', os.path.join(self.local_temp_dir, 'b', 'c'))
adb.copy_local_directory_to_remote(self.local_temp_dir,
self.device_temp_dir)
existent_file_path_remote = os.path.join(self.device_temp_dir, 'a')
existent_directory_path_remote = os.path.join(self.device_temp_dir, 'b')
non_existent_file_path_remote = os.path.join(self.device_temp_dir, 'd')
non_existent_directory_path_remote = os.path.join(self.device_temp_dir, 'e')
self.assertTrue(adb.file_exists(existent_file_path_remote))
self.assertFalse(adb.file_exists(existent_directory_path_remote))
self.assertFalse(adb.file_exists(non_existent_file_path_remote))
self.assertFalse(adb.file_exists(non_existent_directory_path_remote))
def _download_testcase(testcase_id, testcase):
"""Download the test case and return its path."""
response, content = _http_request(
TESTCASE_DOWNLOAD_URL.format(testcase_id=testcase_id),
method=_GET_METHOD)
if response.status != 200:
raise ReproduceToolException('Unable to download test case.')
# Create a temporary directory where we can store the test case.
bot_absolute_filename = response['x-goog-meta-filename']
testcase_directory = os.path.join(environment.get_value('ROOT_DIR'),
'current-testcase')
shell.create_directory(testcase_directory)
testcase_path = os.path.join(testcase_directory,
os.path.basename(bot_absolute_filename))
utils.write_data_to_file(content, testcase_path)
# Unpack the test case if it's archived.
# TODO(mbarbella): Rewrite setup.unpack_testcase and share this code.
if testcase.minimized_keys and testcase.minimized_keys != 'NA':
mask = data_types.ArchiveStatus.MINIMIZED
else:
mask = data_types.ArchiveStatus.FUZZED
if testcase.archive_state & mask:
archive.unpack(testcase_path, testcase_directory)
file_list = archive.get_file_list(testcase_path)
testcase_path = None
for file_name in file_list:
if testcase.absolute_path.endswith(file_name):
testcase_path = os.path.join(testcase_directory, file_name)
break
if not testcase_path:
raise ReproduceToolException(
'Test case file was not found in archive.\n'
'Original filename: {absolute_path}.\n'
'Archive contents: {file_list}'.format(
absolute_path=testcase.absolute_path, file_list=file_list))
return testcase_path
def create_testcase_list_file(output_directory):
"""Create a testcase list file for tests in a directory."""
files_list = []
files_list_file_path = os.path.join(output_directory, TESTCASE_LIST_FILENAME)
for root, _, files in os.walk(output_directory):
for filename in files:
if filename.endswith(INFO_FILE_EXTENSION):
# Skip an info file.
continue
file_path = os.path.join(root, filename)
if not utils.is_valid_testcase_file(file_path, check_if_exists=False):
continue
normalized_relative_file_path = utils.get_normalized_relative_path(
file_path, output_directory)
files_list.append(normalized_relative_file_path)
utils.write_data_to_file('\n'.join(sorted(files_list)), files_list_file_path)
def request(url,
body=None,
method=POST_METHOD,
force_reauthorization=False,
configuration=None):
"""Make an HTTP request to the specified URL."""
if configuration:
authorization = _get_authorization(force_reauthorization,
configuration)
headers = {
"User-Agent": "clusterfuzz-reproduce",
"Authorization": authorization,
}
else:
headers = {}
http = httplib2.Http()
request_body = json_utils.dumps(body) if body is not None else ""
response, content = http.request(url,
method=method,
headers=headers,
body=request_body)
# If the server returns 401 we may need to reauthenticate. Try the request
# a second time if this happens.
if response.status == 401 and not force_reauthorization:
return request(
url,
body,
method=method,
force_reauthorization=True,
configuration=configuration,
)
if AUTHORIZATION_HEADER in response:
shell.create_directory(os.path.dirname(AUTHORIZATION_CACHE_FILE),
create_intermediates=True)
utils.write_data_to_file(response[AUTHORIZATION_HEADER],
AUTHORIZATION_CACHE_FILE)
return response, content
def merge_dictionary_files(original_dictionary_path,
recommended_dictionary_path, merged_dictionary_path):
"""Merge a list of dictionaries with given paths into a singe dictionary."""
if original_dictionary_path and os.path.exists(original_dictionary_path):
merged_dictionary_data = utils.read_data_from_file(
original_dictionary_path, eval_data=False)
else:
merged_dictionary_data = ''
recommended_dictionary_lines = utils.read_data_from_file(
recommended_dictionary_path, eval_data=False).splitlines()
dictionary_lines_to_add = set()
for line in recommended_dictionary_lines:
if line not in merged_dictionary_data:
dictionary_lines_to_add.add(line)
merged_dictionary_data += '\n%s\n' % RECOMMENDED_DICTIONARY_HEADER
merged_dictionary_data += '\n'.join(dictionary_lines_to_add)
utils.write_data_to_file(merged_dictionary_data, merged_dictionary_path)
def remount_if_needed():
"""Remount nfs volume if it is not working."""
nfs_root = environment.get_value('NFS_ROOT')
if not nfs_root:
return
nfs_host = environment.get_value('NFS_HOST')
nfs_volume = environment.get_value('NFS_VOLUME')
check_file_path = os.path.join(nfs_root, 'check')
if os.path.exists(check_file_path):
# Volume is mounted correctly and readable, bail out.
return
# Un-mount the nfs drive first. Ignore the return code as we might have
# not mounted the drive at all.
subprocess.call(['umount', '-f', nfs_root])
# Mount the nfs drive.
logs.log_warn('Trying to remount the NFS volume.')
nfs_volume_path = '%s:/%s' % (nfs_host, nfs_volume)
subprocess.check_call([
'mount', '-o', 'anon', '-o', 'nolock', '-o', 'retry=10',
nfs_volume_path, nfs_root
])
if os.path.exists(check_file_path):
# Volume is mounted correctly and readable, bail out.
return
# Update check file if needed.
utils.write_data_to_file('ok', check_file_path)
# Make sure that check file exists.
if not os.path.exists(check_file_path):
raise Exception('Failed to write check file on nfs volume.')
def download_system_symbols_if_needed(symbols_directory):
"""Download system libraries from |SYMBOLS_URL| and cache locally."""
# For local testing, we do not have access to the cloud storage bucket with
# the symbols. In this case, just bail out.
if environment.get_value('LOCAL_DEVELOPMENT'):
return
# When running reproduce tool locally, we do not have access to the cloud
# storage bucket with the symbols. In this case, just bail out.
if environment.get_value('REPRODUCE_TOOL'):
return
# We have archived symbols for google builds only.
if not settings.is_google_device():
return
# Get the build fingerprint parameters.
build_params = settings.get_build_parameters()
if not build_params:
logs.log_error('Unable to determine build parameters.')
return
build_id = build_params.get('build_id')
target = build_params.get('target')
type = build_params.get('type')
if not build_id or not target or not type:
logs.log_error('Null build parameters found, exiting.')
return
# Check if we already have the symbols in cache.
build_params_check_path = os.path.join(symbols_directory,
'.cached_build_params')
cached_build_params = utils.read_data_from_file(
build_params_check_path, eval_data=True)
if cached_build_params and cmp(cached_build_params, build_params) == 0:
# No work to do, same system symbols already in cache.
return
symbols_archive_filename = '%s-symbols-%s.zip' % (target, build_id)
symbols_archive_path = os.path.join(symbols_directory,
symbols_archive_filename)
# Delete existing symbols directory first.
shell.remove_directory(symbols_directory, recreate=True)
# Fetch symbol file from cloud storage cache (if available).
found_in_cache = storage.get_file_from_cache_if_exists(
symbols_archive_path, update_modification_time_on_access=False)
if not found_in_cache:
# Include type and sanitizer information in the target.
target_with_type_and_san = '%s-%s' % (target, type)
tool_suffix = environment.get_value('SANITIZER_TOOL_NAME')
if tool_suffix and not tool_suffix in target_with_type_and_san:
target_with_type_and_san += '_%s' % tool_suffix
# Fetch the artifact now.
fetch_artifact.get(build_id, target_with_type_and_san,
symbols_archive_filename, symbols_directory)
if not os.path.exists(symbols_archive_path):
logs.log_error(
'Unable to locate symbols archive %s.' % symbols_archive_path)
return
# Store the artifact for later use or for use by other bots.
storage.store_file_in_cache(symbols_archive_path)
archive.unpack(symbols_archive_path, symbols_directory, trusted=True)
shell.remove_file(symbols_archive_path)
utils.write_data_to_file(build_params, build_params_check_path)
def run(self, input_directory, output_directory, no_of_files):
"""Run the fuzzer to generate testcases."""
build_directory = environment.get_value('BUILD_DIR')
if not build_directory:
raise BuiltinFuzzerException(
'BUILD_DIR environment variable is not set.')
fuzzers = fuzzers_utils.get_fuzz_targets(build_directory)
if not fuzzers:
raise BuiltinFuzzerException(
'No fuzzer binaries found in |BUILD_DIR| directory.')
fuzzer_binary_name = environment.get_value('FUZZ_TARGET')
if fuzzer_binary_name:
fuzzer_path = _get_fuzzer_path(fuzzers, fuzzer_binary_name)
else:
fuzzer_path = random.SystemRandom().choice(fuzzers)
fuzzer_binary_name = os.path.basename(fuzzer_path)
project_qualified_name = data_types.fuzz_target_project_qualified_name(
utils.current_project(), fuzzer_binary_name)
corpus_directory = os.path.join(input_directory,
project_qualified_name)
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
corpus_directory = file_host.rebase_to_worker_root(
corpus_directory)
arguments = self.generate_arguments(fuzzer_path)
# Create corpus directory if it does not exist already.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.create_directory(corpus_directory,
create_intermediates=True)
else:
if not os.path.exists(corpus_directory):
os.mkdir(corpus_directory)
# Create fuzz testcases.
for i in range(no_of_files):
# Contents of testcase file don't matter at this point. Need to create
# something non-null so that it is not ignored.
testcase_file_path = os.path.join(output_directory,
'%s%d' % (tests.FUZZ_PREFIX, i))
utils.write_data_to_file(' ', testcase_file_path)
# Write the flags file containing command line for running launcher
# script.
flags_file_path = os.path.join(output_directory,
'%s%d' % (tests.FLAGS_PREFIX, i))
flags = ['%TESTCASE%', fuzzer_binary_name]
if arguments:
flags.append(arguments)
flags_file_content = ' '.join(flags)
utils.write_data_to_file(flags_file_content, flags_file_path)
output = 'Generated %d testcase for fuzzer %s.\n' % (
no_of_files, fuzzer_binary_name)
output += 'metadata::fuzzer_binary_name: %s\n' % fuzzer_binary_name
issue_owners = engine_common.get_issue_owners(fuzzer_path)
if issue_owners:
output += 'metadata::issue_owners: %s\n' % ','.join(issue_owners)
issue_labels = engine_common.get_issue_labels(fuzzer_path)
if issue_labels:
output += 'metadata::issue_labels: %s\n' % ','.join(issue_labels)
# Update *SAN_OPTIONS in current environment from .options file. This
# environment is used in fuzz task later for deriving the environment
# string in |get_environment_settings_as_string| and embedding this as
# part of stacktrace.
engine_common.process_sanitizer_options_overrides(fuzzer_path)
return BuiltinFuzzerResult(output=output,
corpus_directory=corpus_directory)
def update_data_bundle(fuzzer, data_bundle):
"""Updates a data bundle to the latest version."""
# This module can't be in the global imports due to appengine issues
# with multiprocessing and psutil imports.
from google_cloud_utils import gsutil
# If we are using a data bundle on NFS, it is expected that our testcases
# will usually be large enough that we would fill up our tmpfs directory
# pretty quickly. So, change it to use an on-disk directory.
if not data_bundle.is_local:
testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK')
environment.set_value('FUZZ_INPUTS', testcase_disk_directory)
data_bundle_directory = get_data_bundle_directory(fuzzer.name)
if not data_bundle_directory:
logs.log_error('Failed to setup data bundle %s.' % data_bundle.name)
return False
if not shell.create_directory(data_bundle_directory,
create_intermediates=True):
logs.log_error('Failed to create data bundle %s directory.' %
data_bundle.name)
return False
# Check if data bundle is up to date. If yes, skip the update.
if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
logs.log('Data bundle was recently synced, skip.')
return True
# Fetch lock for this data bundle.
if not _fetch_lock_for_data_bundle_update(data_bundle):
logs.log_error('Failed to lock data bundle %s.' % data_bundle.name)
return False
# Re-check if another bot did the sync already. If yes, skip.
if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
logs.log('Another bot finished the sync, skip.')
_release_lock_for_data_bundle_update(data_bundle)
return True
time_before_sync_start = time.time()
# No need to sync anything if this is a search index data bundle. In that
# case, the fuzzer will generate testcases from a gcs bucket periodically.
if not _is_search_index_data_bundle(data_bundle.name):
bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from bot.untrusted_runner import corpus_manager
from bot.untrusted_runner import file_host
worker_data_bundle_directory = file_host.rebase_to_worker_root(
data_bundle_directory)
file_host.create_directory(worker_data_bundle_directory,
create_intermediates=True)
result = corpus_manager.RemoteGSUtilRunner().rsync(
bucket_url, worker_data_bundle_directory, delete=False)
else:
result = gsutil.GSUtilRunner().rsync(bucket_url,
data_bundle_directory,
delete=False)
if result.return_code != 0:
logs.log_error('Failed to sync data bundle %s: %s.' %
(data_bundle.name, result.output))
_release_lock_for_data_bundle_update(data_bundle)
return False
# Update the testcase list file.
testcase_manager.create_testcase_list_file(data_bundle_directory)
# Write last synced time in the sync file.
sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory)
utils.write_data_to_file(time_before_sync_start, sync_file_path)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from bot.untrusted_runner import file_host
worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path)
file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path)
# Release acquired lock.
_release_lock_for_data_bundle_update(data_bundle)
return True
def store_minimized_testcase(testcase, base_directory, file_list,
file_to_run_data, file_to_run):
"""Store all files that make up this testcase."""
# Write the main file data.
utils.write_data_to_file(file_to_run_data, file_to_run)
# Prepare the file.
zip_path = None
if testcase.archive_state:
if len(file_list) > 1:
testcase.archive_state |= data_types.ArchiveStatus.MINIMIZED
zip_path = os.path.join(
environment.get_value('INPUT_DIR'), '%d.zip' % testcase.key.id())
zip_file = zipfile.ZipFile(zip_path, 'w')
count = 0
filtered_file_list = []
for file_name in file_list:
absolute_filename = os.path.join(base_directory, file_name)
is_file = os.path.isfile(absolute_filename)
if file_to_run_data and is_file and os.path.getsize(
absolute_filename) == 0 and (
os.path.basename(absolute_filename) not in file_to_run_data):
continue
if not os.path.exists(absolute_filename):
continue
zip_file.write(absolute_filename, file_name, zipfile.ZIP_DEFLATED)
if is_file:
count += 1
filtered_file_list.append(absolute_filename)
zip_file.close()
try:
if count > 1:
file_handle = open(zip_path, 'rb')
else:
if not filtered_file_list:
# We minimized everything. The only thing needed to reproduce is the
# interaction gesture.
file_path = file_list[0]
file_handle = open(file_path, 'wb')
file_handle.close()
else:
file_path = filtered_file_list[0]
file_handle = open(file_path, 'rb')
testcase.absolute_path = os.path.join(base_directory,
os.path.basename(file_path))
testcase.archive_state &= ~data_types.ArchiveStatus.MINIMIZED
except IOError:
testcase.put() # Preserve what we can.
logs.log_error('Unable to open archive for blobstore write.')
return
else:
absolute_filename = os.path.join(base_directory, file_list[0])
file_handle = open(absolute_filename, 'rb')
testcase.archive_state &= ~data_types.ArchiveStatus.MINIMIZED
else:
file_handle = open(file_list[0], 'rb')
testcase.archive_state &= ~data_types.ArchiveStatus.MINIMIZED
# Store the testcase.
minimized_keys = blobs.write_blob(file_handle)
file_handle.close()
testcase.minimized_keys = minimized_keys
testcase.put()
if zip_path:
shell.remove_file(zip_path)
def write_cache_file_metadata(cache_file_path, file_path):
"""Write cache file metadata."""
cache_file_metadata_path = get_cache_file_metadata_path(cache_file_path)
utils.write_data_to_file({
'size': os.path.getsize(file_path)
}, cache_file_metadata_path)
def download_system_symbols_if_needed(symbols_directory, is_kernel=False):
"""Download system libraries from |SYMBOLS_URL| and cache locally."""
# For local testing, we do not have access to the cloud storage bucket with
# the symbols. In this case, just bail out.
if environment.get_value('LOCAL_DEVELOPMENT'):
return
# When running reproduce tool locally, we do not have access to the cloud
# storage bucket with the symbols. In this case, just bail out.
if environment.get_value('REPRODUCE_TOOL'):
return
# We have archived symbols for google builds only.
if not settings.is_google_device():
return
# For Android kernel we want to get the repro.prop
# Note: kasan and non-kasan kernel should have the same repo.prop for a given
# build_id.
if is_kernel:
_, build_id = kernel_utils.get_kernel_hash_and_build_id()
target = kernel_utils.get_kernel_name()
if not build_id or not target:
logs.log_error('Could not get kernel parameters, exiting.')
return
artifact_file_name = 'repo.prop'
symbols_archive_filename = get_symbols_archive_filename(build_id, target)
output_filename_override = symbols_archive_filename
# We create our own build_params for cache
build_params = {'build_id': build_id, 'target': target, 'type': 'kernel'}
else:
# Get the build fingerprint parameters.
build_params = settings.get_build_parameters()
if not build_params:
logs.log_error('Unable to determine build parameters.')
return
build_id = build_params.get('build_id')
target = build_params.get('target')
build_type = build_params.get('type')
if not build_id or not target or not build_type:
logs.log_error('Null build parameters found, exiting.')
return
symbols_archive_filename = '%s-symbols-%s.zip' % (target, build_id)
artifact_file_name = symbols_archive_filename
output_filename_override = None
# Check if we already have the symbols in cache.
build_params_check_path = os.path.join(symbols_directory,
'.cached_build_params')
cached_build_params = utils.read_data_from_file(
build_params_check_path, eval_data=True)
if cached_build_params and cached_build_params == build_params:
# No work to do, same system symbols already in cache.
return
symbols_archive_path = os.path.join(symbols_directory,
symbols_archive_filename)
# Delete existing symbols directory first.
shell.remove_directory(symbols_directory, recreate=True)
# Fetch symbol file from cloud storage cache (if available).
found_in_cache = storage.get_file_from_cache_if_exists(
symbols_archive_path, update_modification_time_on_access=False)
if not found_in_cache:
tool_suffix = environment.get_value('SANITIZER_TOOL_NAME')
if is_kernel:
# Some kernels are just 'kernel', some are kernel_target
if tool_suffix:
targets_with_type_and_san = [
'kernel_%s' % tool_suffix,
'kernel_%s_%s' % (tool_suffix, target)
]
else:
targets_with_type_and_san = ['kernel', 'kernel_%s' % target]
else:
# Include type and sanitizer information in the target.
target_with_type_and_san = '%s-%s' % (target, build_type)
if tool_suffix and not tool_suffix in target_with_type_and_san:
target_with_type_and_san += '_%s' % tool_suffix
targets_with_type_and_san = [target_with_type_and_san]
for target_with_type_and_san in targets_with_type_and_san:
# Fetch the artifact now.
fetch_artifact.get(build_id, target_with_type_and_san, artifact_file_name,
symbols_directory, output_filename_override)
if os.path.exists(symbols_archive_path):
break
if not os.path.exists(symbols_archive_path):
logs.log_error(
'Unable to locate symbols archive %s.' % symbols_archive_path)
return
# Store the artifact for later use or for use by other bots.
storage.store_file_in_cache(symbols_archive_path)
# repo.prop is not a zip archive.
if not is_kernel:
archive.unpack(symbols_archive_path, symbols_directory, trusted=True)
shell.remove_file(symbols_archive_path)
utils.write_data_to_file(build_params, build_params_check_path)
def _unpack_build(base_build_dir, build_dir, build_url, target_weights=None):
"""Unpacks a build from a build url into the build directory."""
# Track time taken to unpack builds so that it doesn't silently regress.
start_time = time.time()
# Free up memory.
utils.python_gc()
# Remove the current build.
logs.log('Removing build directory %s.' % build_dir)
if not shell.remove_directory(build_dir, recreate=True):
logs.log_error('Unable to clear build directory %s.' % build_dir)
_handle_unrecoverable_error_on_windows()
return False
# Decide whether to use cache build archives or not.
use_cache = environment.get_value('CACHE_STORE', False)
# Download build archive locally.
build_local_archive = os.path.join(build_dir, os.path.basename(build_url))
# Make the disk space necessary for the archive available.
archive_size = storage.get_download_file_size(
build_url, build_local_archive, use_cache=True)
if archive_size is not None and not _make_space(archive_size, base_build_dir):
shell.clear_data_directories()
logs.log_fatal_and_exit(
'Failed to make space for download. '
'Cleared all data directories to free up space, exiting.')
logs.log('Downloading build from url %s.' % build_url)
try:
storage.copy_file_from(build_url, build_local_archive, use_cache=use_cache)
except:
logs.log_error('Unable to download build url %s.' % build_url)
return False
unpack_everything = environment.get_value('UNPACK_ALL_FUZZ_TARGETS_AND_FILES')
if not unpack_everything:
# For fuzzing, pick a random fuzz target so that we only un-archive that
# particular fuzz target and its dependencies and save disk space.
# If we are going to unpack everythng in archive based on
# |UNPACK_ALL_FUZZ_TARGETS_AND_FILES| in the job defition, then don't set a
# random fuzz target before we've unpacked the build. It won't actually save
# us anything in this case and can be really expensive for large builds
# (such as Chrome OS). Defer setting it until after the build has been
# unpacked.
_set_random_fuzz_target_for_fuzzing_if_needed(
_get_fuzz_targets_from_archive(build_local_archive), target_weights)
# Actual list of files to unpack can be smaller if we are only unarchiving
# a particular fuzz target.
file_match_callback = _get_file_match_callback()
assert not (unpack_everything and file_match_callback is not None)
if not _make_space_for_build(build_local_archive, base_build_dir,
file_match_callback):
shell.clear_data_directories()
logs.log_fatal_and_exit(
'Failed to make space for build. '
'Cleared all data directories to free up space, exiting.')
# Unpack the local build archive.
logs.log('Unpacking build archive %s.' % build_local_archive)
trusted = not utils.is_oss_fuzz()
try:
archive.unpack(
build_local_archive,
build_dir,
trusted=trusted,
file_match_callback=file_match_callback)
except:
logs.log_error('Unable to unpack build archive %s.' % build_local_archive)
return False
if unpack_everything:
# Set a random fuzz target now that the build has been unpacked, if we
# didn't set one earlier.
_set_random_fuzz_target_for_fuzzing_if_needed(
_get_fuzz_targets_from_dir(build_dir), target_weights)
# If this is partial build due to selected build files, then mark it as such
# so that it is not re-used.
if file_match_callback:
partial_build_file_path = os.path.join(build_dir, PARTIAL_BUILD_FILE)
utils.write_data_to_file('', partial_build_file_path)
# No point in keeping the archive around.
shell.remove_file(build_local_archive)
end_time = time.time()
elapsed_time = end_time - start_time
log_func = logs.log_warn if elapsed_time > UNPACK_TIME_LIMIT else logs.log
log_func('Build took %0.02f minutes to unpack.' % (elapsed_time / 60.))
return True
def parse_mime_to_crash_report_info(local_minidump_mime_path):
"""Read the (local) minidump MIME file into a CrashReportInfo object."""
# Get the minidump name and path.
minidump_path_match = re.match(r'(.*)\.mime', local_minidump_mime_path)
if minidump_path_match is None:
logs.log_error('Minidump filename in unexpected format: \'%s\'.' %
local_minidump_mime_path)
return None
minidump_path = '%s.dmp' % minidump_path_match.group(1).strip()
# Reformat the minidump MIME to include the boundary.
with open(local_minidump_mime_path, 'rb') as minidump_mime_file_content:
# The boundary is the first line after the first two dashes.
boundary = minidump_mime_file_content.readline().strip()[2:]
minidump_mime_string = (
'Content-Type: multipart/form-data; boundary=\"%s\"\r\n--%s\r\n' %
(boundary, boundary))
minidump_mime_string += minidump_mime_file_content.read()
minidump_mime_contents = email.message_from_string(minidump_mime_string)
# Parse the MIME contents, extracting the parameters needed for upload.
mime_key_values = {}
for mime_part in minidump_mime_contents.get_payload():
if isinstance(mime_part, str):
mime_part = utils.decode_to_unicode(mime_part)
logs.log_error('Unexpected str mime_part from mime path %s: %s' %
(local_minidump_mime_path, mime_part))
continue
part_descriptor = list(mime_part.values())
key_tokens = part_descriptor[0].split('; ')
key_match = re.match(r'name="(.*)".*', key_tokens[1])
# Extract from the MIME part the key-value pairs used by report uploading.
if key_match is not None:
report_key = key_match.group(1)
report_value = mime_part.get_payload()
if report_key == MINIDUMP_FILE_KEY:
utils.write_data_to_file(report_value, minidump_path)
else:
# Take care of aliases.
if report_key == 'prod' or report_key == 'buildTargetId':
report_key = PRODUCT_KEY
elif report_key == 'ver':
report_key = VERSION_KEY
# Save the key-value pair.
mime_key_values[report_key] = report_value
# Pull out product and version explicitly since these are required
# for upload.
product, version = None, None
if PRODUCT_KEY in mime_key_values:
product = mime_key_values.pop(PRODUCT_KEY)
else:
logs.log_error(
'Could not find \'%s\' or alias in mime_key_values key.' %
PRODUCT_KEY)
if VERSION_KEY in mime_key_values:
version = mime_key_values.pop(VERSION_KEY)
else:
logs.log_error(
'Could not find \'%s\' or alias in mime_key_values key.' %
VERSION_KEY)
# If missing, return None and log keys that do exist; otherwise, construct
# CrashReportInfo and return.
if product is None or version is None:
logs.log_error('mime_key_values dict keys:\n%s' %
str(list(mime_key_values.keys())))
return None
return CrashReportInfo(minidump_path=minidump_path,
product=product,
version=version,
optional_params=mime_key_values)
def run_testcase_and_return_result_in_queue(crash_queue,
thread_index,
file_path,
gestures,
env_copy,
upload_output=False):
"""Run a single testcase and return crash results in the crash queue."""
# Since this is running in its own process, initialize the log handler again.
# This is needed for Windows where instances are not shared across child
# processes. See:
# https://stackoverflow.com/questions/34724643/python-logging-with-multiprocessing-root-logger-different-in-windows
logs.configure('run_testcase', {
'testcase_path': file_path,
})
try:
# Run testcase and check whether a crash occurred or not.
return_code, crash_time, output = run_testcase(thread_index, file_path,
gestures, env_copy)
# Pull testcase directory to host to get any stats files.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.pull_testcases_from_worker()
# Analyze the crash.
crash_output = _get_crash_output(output)
crash_result = CrashResult(return_code, crash_time, crash_output)
# To provide consistency between stats and logs, we use timestamp taken
# from stats when uploading logs and testcase.
if upload_output:
log_time = _get_testcase_time(file_path)
if crash_result.is_crash():
# Initialize resource list with the testcase path.
resource_list = [file_path]
resource_list += get_resource_paths(crash_output)
# Store the crash stack file in the crash stacktrace directory
# with filename as the hash of the testcase path.
crash_stacks_directory = environment.get_value(
'CRASH_STACKTRACES_DIR')
stack_file_path = os.path.join(crash_stacks_directory,
utils.string_hash(file_path))
utils.write_data_to_file(crash_output, stack_file_path)
# Put crash/no-crash results in the crash queue.
crash_queue.put(
Crash(file_path=file_path,
crash_time=crash_time,
return_code=return_code,
resource_list=resource_list,
gestures=gestures,
stack_file_path=stack_file_path))
# Don't upload uninteresting testcases (no crash) or if there is no log to
# correlate it with (not upload_output).
if upload_output:
upload_testcase(file_path, log_time)
if upload_output:
# Include full output for uploaded logs (crash output, merge output, etc).
crash_result_full = CrashResult(return_code, crash_time, output)
log = prepare_log_for_upload(crash_result_full.get_stacktrace(),
return_code)
upload_log(log, log_time)
except Exception:
logs.log_error('Exception occurred while running '
'run_testcase_and_return_result_in_queue.')
