def initialize_resources_dir():
"""Download Fuchsia QEMU resources from GCS bucket."""
resources_dir = environment.get_value('RESOURCES_DIR')
if not resources_dir:
raise errors.FuchsiaConfigError('Could not find RESOURCES_DIR')
fuchsia_resources_dir = os.path.join(resources_dir, 'fuchsia')
shell.create_directory(fuchsia_resources_dir, recreate=True)
fuchsia_resources_url = environment.get_value('FUCHSIA_RESOURCES_URL')
if not fuchsia_resources_url:
raise errors.FuchsiaConfigError(
'Could not find path for remote'
'Fuchsia resources bucket (FUCHSIA_RESOURCES_URL')
gsutil_command_arguments = [
'-m', 'cp', '-r', fuchsia_resources_url, fuchsia_resources_dir
]
logs.log("Beginning Fuchsia SDK download.")
result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments)
if result.return_code or result.timed_out:
raise errors.FuchsiaSdkError('Failed to download Fuchsia'
'resources: ' + result.output)
logs.log("Fuchsia SDK download complete.")
return fuchsia_resources_dir
def test(self):
"""Tests copy_local_directory_to_remote."""
utils.write_data_to_file('a', os.path.join(self.local_temp_dir, 'a'))
shell.create_directory(os.path.join(self.local_temp_dir, 'b'))
utils.write_data_to_file('c', os.path.join(self.local_temp_dir, 'b', 'c'))
adb.copy_local_directory_to_remote(self.local_temp_dir,
self.device_temp_dir)
self.assertTrue(adb.file_exists(os.path.join(self.device_temp_dir, 'a')))
self.assertFalse(
adb.directory_exists(os.path.join(self.device_temp_dir, 'a')))
self.assertEqual(
adb.get_file_size(os.path.join(self.device_temp_dir, 'a')), 1)
self.assertTrue(
adb.directory_exists(os.path.join(self.device_temp_dir, 'b')))
self.assertFalse(adb.file_exists(os.path.join(self.device_temp_dir, 'b')))
self.assertTrue(
adb.file_exists(os.path.join(self.device_temp_dir, 'b', 'c')))
self.assertFalse(
adb.directory_exists(os.path.join(self.device_temp_dir, 'b', 'c')))
self.assertEqual(
adb.get_file_size(os.path.join(self.device_temp_dir, 'b', 'c')), 1)
def get_temp_dir():
"""Return the temp dir."""
temp_dirname = 'temp-' + str(os.getpid())
temp_directory = os.path.join(
environment.get_value('FUZZ_INPUTS_DISK'), temp_dirname)
shell.create_directory(temp_directory)
return temp_directory
def _http_request(url,
body=None,
method=_POST_METHOD,
force_reauthorization=False):
"""Make a POST request to the specified URL."""
authorization = _get_authorization(force_reauthorization)
headers = {
'User-Agent': 'clusterfuzz-reproduce',
'Authorization': authorization
}
http = httplib2.Http()
request_body = json_utils.dumps(body) if body else ''
response, content = http.request(url,
method=method,
headers=headers,
body=request_body)
# If the server returns 401 we may need to reauthenticate. Try the request
# a second time if this happens.
if response.status == 401 and not force_reauthorization:
return _http_request(url,
body,
method=method,
force_reauthorization=True)
if 'x-clusterfuzz-authorization' in response:
shell.create_directory(os.path.dirname(AUTHORIZATION_CACHE_FILE),
create_intermediates=True)
utils.write_data_to_file(response['x-clusterfuzz-authorization'],
AUTHORIZATION_CACHE_FILE)
return response, content
def create_merge_directory():
"""Create the merge directory and return its path."""
merge_directory_path = get_merge_directory()
shell.create_directory(merge_directory_path,
create_intermediates=True,
recreate=True)
return merge_directory_path
def initialize():
"""Initialize the persistent cache, creating the directory used to store the
values."""
cache_directory_path = environment.get_value('CACHE_DIR')
if os.path.exists(cache_directory_path):
clear_values()
else:
shell.create_directory(cache_directory_path, create_intermediates=True)
def _create_temp_corpus_directory(self, name):
"""Create temporary corpus directory. Returns path to the created
directory."""
testcases_directory = environment.get_value("FUZZ_INPUTS_DISK")
directory_path = os.path.join(testcases_directory, name)
shell.create_directory(directory_path)
self._created_directories.append(directory_path)
return directory_path
def get_runner(fuzzer_path, temp_dir=None):
"""Get a libfuzzer runner."""
use_minijail = environment.get_value('USE_MINIJAIL')
build_dir = environment.get_value('BUILD_DIR')
dataflow_build_dir = environment.get_value('DATAFLOW_BUILD_DIR')
if use_minijail:
# Set up chroot and runner.
if environment.is_chromeos_system_job():
minijail_chroot = minijail.ChromeOSChroot(build_dir)
else:
minijail_chroot = minijail.MinijailChroot(base_dir=temp_dir)
# While it's possible for dynamic binaries to run without this, they need
# to be accessible for symbolization etc. For simplicity we bind BUILD_DIR
# to the same location within the chroot, which leaks the directory
# structure of CF but this shouldn't be a big deal.
minijail_chroot.add_binding(
minijail.ChrootBinding(build_dir, build_dir, False))
if dataflow_build_dir:
minijail_chroot.add_binding(
minijail.ChrootBinding(dataflow_build_dir, dataflow_build_dir,
False))
# Also bind the build dir to /out to make it easier to hardcode references
# to data files.
minijail_chroot.add_binding(
minijail.ChrootBinding(build_dir, '/out', False))
minijail_bin = os.path.join(minijail_chroot.directory, 'bin')
shell.create_directory(minijail_bin)
# Set up /bin with llvm-symbolizer to allow symbolized stacktraces.
# Don't copy if it already exists (e.g. ChromeOS chroot jail).
llvm_symbolizer_source_path = environment.get_llvm_symbolizer_path()
llvm_symbolizer_destination_path = os.path.join(
minijail_bin, 'llvm-symbolizer')
if not os.path.exists(llvm_symbolizer_destination_path):
shutil.copy(llvm_symbolizer_source_path,
llvm_symbolizer_destination_path)
# copy /bin/sh, necessary for system().
if not environment.is_chromeos_system_job():
# The chroot has its own shell we don't need to copy (and probably
# shouldn't because of library differences).
shutil.copy(os.path.realpath('/bin/sh'),
os.path.join(minijail_bin, 'sh'))
runner = MinijailLibFuzzerRunner(fuzzer_path, minijail_chroot)
elif environment.platform() == 'FUCHSIA':
runner = FuchsiaQemuLibFuzzerRunner(fuzzer_path)
else:
runner = LibFuzzerRunner(fuzzer_path)
return runner
def _makedirs(self, directory):
"""Create directories for binding in chroot.
Args:
directory: The absolute path to the directory target in the chroot.
"""
if directory[0] == '/':
directory = directory[1:]
shell.create_directory(os.path.join(self._chroot_dir, directory),
create_intermediates=True)
def init_corpus(self, source_dir, destination_dir):
"""Uses corpus from the cloud to initialize syzkaller corpus.
Args:
source_dir: Folder where the corpus is downloaded from the cloud.
destination_dir: Folder where syzkaller will be looking for corpus.
"""
source_file = os.path.join(source_dir,
self._get_device_corpus_db_filename())
shell.create_directory(destination_dir)
destination_file = os.path.join(destination_dir, CORPUS_DB_FILENAME)
if os.path.isfile(source_file):
shutil.copy(source_file, destination_file)
def convert_path_for_write(self, remote_path, directory=OBJECTS_DIR):
"""Get the local FS path for writing to the remote path. Creates any
intermediate directories if necessary (except for the parent bucket
directory)."""
bucket, path = get_bucket_name_and_path(remote_path)
if not os.path.exists(self._fs_bucket_path(bucket)):
raise RuntimeError(
'Bucket {bucket} does not exist.'.format(bucket=bucket))
fs_path = self._fs_path(bucket, path, directory)
shell.create_directory(os.path.dirname(fs_path), create_intermediates=True)
return fs_path
def _cross_pollinate_other_fuzzer_corpuses(self):
"""Add other fuzzer corpuses to shared corpus path for cross-pollination."""
corpus_backup_date = utils.utcnow().date() - datetime.timedelta(
days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS)
for cross_pollinate_fuzzer in self.cross_pollinate_fuzzers:
project_qualified_name = (
cross_pollinate_fuzzer.fuzz_target.project_qualified_name())
backup_bucket_name = cross_pollinate_fuzzer.backup_bucket_name
corpus_engine_name = cross_pollinate_fuzzer.corpus_engine_name
corpus_backup_url = corpus_manager.gcs_url_for_backup_file(
backup_bucket_name,
corpus_engine_name,
project_qualified_name,
corpus_backup_date,
)
corpus_backup_local_filename = "%s-%s" % (
project_qualified_name,
os.path.basename(corpus_backup_url),
)
corpus_backup_local_path = os.path.join(
self.shared_corpus_path, corpus_backup_local_filename)
if not storage.exists(corpus_backup_url, ignore_errors=True):
# This can happen in cases when a new fuzz target is checked in or if
# missed to capture a backup for a particular day (for OSS-Fuzz, this
# will result in a 403 instead of 404 since that GCS path belongs to
# other project). So, just log a warning for debugging purposes only.
logs.log_warn("Corpus backup does not exist, ignoring: %s." %
corpus_backup_url)
continue
if not storage.copy_file_from(corpus_backup_url,
corpus_backup_local_path):
continue
corpus_backup_output_directory = os.path.join(
self.shared_corpus_path, project_qualified_name)
shell.create_directory(corpus_backup_output_directory)
result = archive.unpack(corpus_backup_local_path,
corpus_backup_output_directory)
shell.remove_file(corpus_backup_local_path)
if result:
logs.log(
"Corpus backup url %s successfully unpacked into shared corpus."
% corpus_backup_url)
else:
logs.log_error("Failed to unpack corpus backup from url %s." %
corpus_backup_url)
def get_corpus_directory(input_directory, project_qualified_name):
"""Get the corpus directory given a project qualified fuzz target name."""
corpus_directory = os.path.join(input_directory, project_qualified_name)
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
corpus_directory = file_host.rebase_to_worker_root(corpus_directory)
# Create corpus directory if it does not exist already.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.create_directory(corpus_directory, create_intermediates=True)
else:
shell.create_directory(corpus_directory)
return corpus_directory
def _download_testcase(testcase_id, testcase, configuration):
"""Download the test case and return its path."""
testcase_download_url = '{url}?id={id}'.format(
url=configuration.get('testcase_download_url'), id=testcase_id)
response, content = http_utils.request(testcase_download_url,
method=http_utils.GET_METHOD,
configuration=configuration)
if response.status != 200:
raise errors.ReproduceToolUnrecoverableError(
'Unable to download test case.')
# Create a temporary directory where we can store the test case.
bot_absolute_filename = response['x-goog-meta-filename']
testcase_directory = os.path.join(environment.get_value('ROOT_DIR'),
'current-testcase')
shell.create_directory(testcase_directory)
environment.set_value('FUZZ_INPUTS', testcase_directory)
testcase_path = os.path.join(testcase_directory,
os.path.basename(bot_absolute_filename))
utils.write_data_to_file(content, testcase_path)
# Unpack the test case if it's archived.
# TODO(mbarbella): Rewrite setup.unpack_testcase and share this code.
if testcase.minimized_keys and testcase.minimized_keys != 'NA':
mask = data_types.ArchiveStatus.MINIMIZED
else:
mask = data_types.ArchiveStatus.FUZZED
if testcase.archive_state & mask:
archive.unpack(testcase_path, testcase_directory)
file_list = archive.get_file_list(testcase_path)
testcase_path = None
for file_name in file_list:
if testcase.absolute_path.endswith(file_name):
testcase_path = os.path.join(testcase_directory, file_name)
break
if not testcase_path:
raise errors.ReproduceToolUnrecoverableError(
'Test case file was not found in archive.\n'
'Original filename: {absolute_path}.\n'
'Archive contents: {file_list}'.format(
absolute_path=testcase.absolute_path, file_list=file_list))
return testcase_path
def initialize_resources_dir():
"""Download Fuchsia QEMU resources from GCS bucket."""
# This module depends on multiprocessing, which is not available in
# appengine, and since appengine *imports* this file (but does not run this
# function!), we import it here.
from google_cloud_utils import gsutil
resources_dir = environment.get_value('RESOURCES_DIR')
if not resources_dir:
raise errors.FuchsiaConfigError('Could not find RESOURCES_DIR')
fuchsia_resources_dir = os.path.join(resources_dir, 'fuchsia')
shell.create_directory(
fuchsia_resources_dir, create_intermediates=True, recreate=True)
# Bucket for QEMU resources.
fuchsia_resources_url = environment.get_value('FUCHSIA_RESOURCES_URL')
if not fuchsia_resources_url:
raise errors.FuchsiaConfigError(
'Could not find path for remote'
'Fuchsia resources bucket (FUCHSIA_RESOURCES_URL')
gsutil_command_arguments = [
'-m', 'cp', '-r', fuchsia_resources_url, fuchsia_resources_dir
]
logs.log("Beginning Fuchsia SDK download.")
result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments)
if result.return_code or result.timed_out:
raise errors.FuchsiaSdkError('Failed to download Fuchsia '
'resources: ' + result.output)
logs.log("Fuchsia SDK download complete.")
# Bucket for build resources. Necessary for fuzzer selection.
logs.log("Fetching Fuchsia build.")
fuchsia_build_url = environment.get_value('FUCHSIA_BUILD_URL')
if not fuchsia_build_url:
raise errors.FuchsiaConfigError('Could not find path for remote'
'Fuchsia build bucket (FUCHSIA BUILD URL')
gsutil_command_arguments = [
'-m', 'cp', '-r', fuchsia_build_url, fuchsia_resources_dir
]
logs.log("Beginning Fuchsia build download.")
result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments)
if result.return_code or result.timed_out:
raise errors.FuchsiaSdkError('Failed to download Fuchsia '
'resources: ' + result.output)
return fuchsia_resources_dir
def save_corpus(self, source_dir, destination_dir):
"""Saves syzkaller to folder so it is backed up to the cloud.
Args:
source_dir: Folder where syzkaller corpus is.
destination_dir: Folder where the corpus is synced with the cloud.
"""
source_file = os.path.join(source_dir, CORPUS_DB_FILENAME)
shell.create_directory(destination_dir)
destination_file = os.path.join(destination_dir,
self._get_device_corpus_db_filename())
if os.path.isfile(source_file) and (
not os.path.exists(destination_file) or
(os.path.getsize(source_file) >
os.path.getsize(destination_file))):
shutil.copy(source_file, destination_file)
def _setup_build_directories(base_build_dir):
"""Set up build directories for a job."""
# Create the root build directory for this job.
shell.create_directory(base_build_dir, create_intermediates=True)
custom_binary_directory = os.path.join(base_build_dir, 'custom')
revision_build_directory = os.path.join(base_build_dir, 'revisions')
sym_build_directory = os.path.join(base_build_dir, 'symbolized')
sym_debug_build_directory = os.path.join(sym_build_directory, 'debug')
sym_release_build_directory = os.path.join(sym_build_directory, 'release')
build_directories = [
custom_binary_directory, revision_build_directory, sym_build_directory,
sym_debug_build_directory, sym_release_build_directory
]
for build_directory in build_directories:
shell.create_directory(build_directory)
def setup_user_profile_directory_if_needed(user_profile_directory):
"""Set user profile directory if it does not exist."""
if os.path.exists(user_profile_directory):
# User profile directory already exists. Bail out.
return
shell.create_directory(user_profile_directory)
# Create a file in user profile directory based on format:
# filename;base64 encoded zlib compressed file contents.
user_profile_file = environment.get_value('USER_PROFILE_FILE')
if user_profile_file and ';' in user_profile_file:
user_profile_filename, encoded_file_contents = (
user_profile_file.split(';', 1))
user_profile_file_contents = zlib.decompress(
base64.b64decode(encoded_file_contents))
user_profile_file_path = os.path.join(user_profile_directory,
user_profile_filename)
utils.write_data_to_file(user_profile_file_contents,
user_profile_file_path)
# For Firefox, we need to install a special fuzzPriv extension that exposes
# special functions to javascript, e.g. gc(), etc.
app_name = environment.get_value('APP_NAME')
if app_name.startswith('firefox'):
# Create extensions directory.
extensions_directory = os.path.join(user_profile_directory,
'extensions')
shell.create_directory(extensions_directory)
# Unpack the fuzzPriv extension.
extension_archive = os.path.join(environment.get_resources_directory(),
'firefox', 'fuzzPriv-extension.zip')
archive.unpack(extension_archive, extensions_directory)
# Add this extension in the extensions configuration file.
extension_config_file_path = os.path.join(user_profile_directory,
'extensions.ini')
fuzz_extension_directory = os.path.join(extensions_directory,
'*****@*****.**')
extension_config_file_contents = ('[ExtensionDirs]\r\n'
'Extension0=%s\r\n'
'\r\n'
'[ThemeDirs]\r\n' %
fuzz_extension_directory)
utils.write_data_to_file(extension_config_file_contents,
extension_config_file_path)
def mocked_create_merge_directory(_):
"""A mocked version of create_merge_directory that adds some interesting
files to the merge corpus and initial corpus."""
merge_directory_path = libfuzzer.create_corpus_directory('merge-corpus')
shell.create_directory(
merge_directory_path, create_intermediates=True, recreate=True)
# Write the minimal unit to the merge directory.
minimal_unit_path = os.path.join(merge_directory_path, minimal_unit_hash)
with open(minimal_unit_path, 'w+') as file_handle:
file_handle.write(minimal_unit_contents)
# Write the nonminimal unit to the corpus directory.
nonminimal_unit_path = os.path.join(corpus_path, nonminimal_unit_hash)
with open(nonminimal_unit_path, 'w+') as file_handle:
file_handle.write(nonminimal_unit_contents)
return merge_directory_path
def test_file_exists(self):
"""Tests file_exists."""
utils.write_data_to_file('a', os.path.join(self.local_temp_dir, 'a'))
shell.create_directory(os.path.join(self.local_temp_dir, 'b'))
utils.write_data_to_file('c', os.path.join(self.local_temp_dir, 'b', 'c'))
adb.copy_local_directory_to_remote(self.local_temp_dir,
self.device_temp_dir)
existent_file_path_remote = os.path.join(self.device_temp_dir, 'a')
existent_directory_path_remote = os.path.join(self.device_temp_dir, 'b')
non_existent_file_path_remote = os.path.join(self.device_temp_dir, 'd')
non_existent_directory_path_remote = os.path.join(self.device_temp_dir, 'e')
self.assertTrue(adb.file_exists(existent_file_path_remote))
self.assertFalse(adb.file_exists(existent_directory_path_remote))
self.assertFalse(adb.file_exists(non_existent_file_path_remote))
self.assertFalse(adb.file_exists(non_existent_directory_path_remote))
def initialize_resources_dir():
"""Download Fuchsia QEMU resources from GCS bucket."""
# This module depends on multiprocessing, which is not available in
# appengine, and since appengine *imports* this file (but does not run this
# function!), we import it here.
from google_cloud_utils import gsutil
resources_dir = environment.get_value('RESOURCES_DIR')
if not resources_dir:
raise errors.FuchsiaConfigError('Could not find RESOURCES_DIR')
fuchsia_resources_dir = os.path.join(resources_dir, 'fuchsia')
shell.create_directory(fuchsia_resources_dir,
create_intermediates=True,
recreate=True)
# Bucket for QEMU resources.
fuchsia_resources_url = environment.get_value('FUCHSIA_BUILD_URL')
if not fuchsia_resources_url:
raise errors.FuchsiaConfigError(
'Could not find path for remote'
'Fuchsia resources bucket (FUCHSIA_BUILD_URL')
gsutil_command_arguments = [
'-m', 'cp', '-r', fuchsia_resources_url, fuchsia_resources_dir
]
logs.log("Fetching Fuchsia build.")
result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments)
if result.return_code or result.timed_out:
raise errors.FuchsiaSdkError('Failed to download Fuchsia '
'resources: ' + result.output)
# Chmod the symbolizers so they can be used easily.
symbolizer_path = os.path.join(fuchsia_resources_dir, 'build', 'zircon',
'prebuilt', 'downloads', 'symbolize',
'linux-x64', 'symbolize')
llvm_symbolizer_path = os.path.join(fuchsia_resources_dir, 'build',
'buildtools', 'linux-x64', 'clang',
'bin', 'llvm-symbolizer')
os.chmod(symbolizer_path, 0o111)
os.chmod(llvm_symbolizer_path, 0o111)
logs.log("Fuchsia build download complete.")
return fuchsia_resources_dir
def mocked_create_merge_directory():
"""A mocked version of create_merge_directory that adds some interesting
files to the merge corpus and initial corpus."""
merge_directory_path = launcher.get_merge_directory()
shell.create_directory(
merge_directory_path, create_intermediates=True, recreate=True)
# Write the minimal unit to the merge directory.
minimal_unit_path = os.path.join(merge_directory_path, minimal_unit_hash)
with open(minimal_unit_path, 'w+') as file_handle:
file_handle.write(minimal_unit_contents)
# Write the nonminimal unit to the corpus directory.
corpus_directory = os.getenv('FUZZ_CORPUS_DIR')
nonminimal_unit_path = os.path.join(corpus_directory,
nonminimal_unit_hash)
with open(nonminimal_unit_path, 'w+') as file_handle:
file_handle.write(nonminimal_unit_contents)
return merge_directory_path
def request(url,
body=None,
method=POST_METHOD,
force_reauthorization=False,
configuration=None):
"""Make an HTTP request to the specified URL."""
if configuration:
authorization = _get_authorization(force_reauthorization,
configuration)
headers = {
"User-Agent": "clusterfuzz-reproduce",
"Authorization": authorization,
}
else:
headers = {}
http = httplib2.Http()
request_body = json_utils.dumps(body) if body is not None else ""
response, content = http.request(url,
method=method,
headers=headers,
body=request_body)
# If the server returns 401 we may need to reauthenticate. Try the request
# a second time if this happens.
if response.status == 401 and not force_reauthorization:
return request(
url,
body,
method=method,
force_reauthorization=True,
configuration=configuration,
)
if AUTHORIZATION_HEADER in response:
shell.create_directory(os.path.dirname(AUTHORIZATION_CACHE_FILE),
create_intermediates=True)
utils.write_data_to_file(response[AUTHORIZATION_HEADER],
AUTHORIZATION_CACHE_FILE)
return response, content
def rsync_to_disk(self,
directory,
timeout=CORPUS_FILES_SYNC_TIMEOUT,
delete=True):
"""Run gsutil to download corpus files from GCS.
Args:
directory: Path to directory to sync to.
timeout: Timeout for gsutil.
delete: Whether or not to delete files on disk that don't exist locally.
Returns:
A bool indicating whether or not the command succeeded.
"""
shell.create_directory(directory, create_intermediates=True)
corpus_gcs_url = self.get_gcs_url()
result = self._gsutil_runner.rsync(corpus_gcs_url, directory, timeout,
delete)
# Allow a small number of files to fail to be synced.
return _handle_rsync_result(result, max_errors=MAX_SYNC_ERRORS)
def create_directories(self):
"""Creates directories needed to use mutator plugins."""
# TODO(320): Change mutator plugin downloads so that they don't need to be
# deleted and redownloaded on each run of launcher.py.
shell.create_directory(environment.get_value('MUTATOR_PLUGINS_DIR'),
create_intermediates=True,
recreate=True)
shell.create_directory(_get_mutator_plugins_archives_dir(),
create_intermediates=True,
recreate=True)
shell.create_directory(_get_mutator_plugins_unpacked_dir(),
create_intermediates=True,
recreate=True)
def unpack_crash_testcases(crash_testcases_directory):
"""Unpacks the old crash testcases in the provided directory."""
for testcase in ndb_utils.get_all_from_model(data_types.Testcase):
testcase_id = testcase.key.id()
# 1. If we have already stored the testcase, then just skip.
if testcase_id in STORED_TESTCASES_LIST:
continue
# 2. Make sure that it is a unique crash testcase. Ignore duplicates,
# uploaded repros.
if testcase.status != 'Processed':
continue
# 3. Check if the testcase is fixed. If not, skip.
if testcase.open:
continue
# 4. Check if the testcase has a minimized repro. If not, skip.
if not testcase.minimized_keys or testcase.minimized_keys == 'NA':
continue
# 5. Only use testcases that have bugs associated with them.
if not testcase.bug_information:
continue
# 6. Existing IPC testcases are un-interesting and unused in furthur
# mutations. Due to size bloat, ignoring these for now.
if testcase.absolute_path.endswith(testcase_manager.IPCDUMP_EXTENSION):
continue
# 7. Ignore testcases that are archives (e.g. Langfuzz fuzzer tests).
if archive.get_archive_type(testcase.absolute_path):
continue
# 8. Skip in-process fuzzer testcases, since these are only applicable to
# fuzz targets and don't run with blackbox binaries.
if testcase.fuzzer_name and testcase.fuzzer_name in [
'afl', 'libFuzzer'
]:
continue
# Un-pack testcase.
try:
_, input_directory, _ = setup.unpack_testcase(testcase)
except Exception:
logs.log_error('Failed to unpack testcase %d.' % testcase.key.id())
continue
# Move this to our crash testcases directory.
crash_testcase_directory = os.path.join(crash_testcases_directory,
str(testcase_id))
shell.move(input_directory, crash_testcase_directory)
# Re-create input directory for unpacking testcase in next iteration.
shell.create_directory(input_directory)
STORED_TESTCASES_LIST.append(testcase_id)
# Remove testcase directories that exceed the max size limit.
for directory_name in os.listdir(crash_testcases_directory):
directory_path = os.path.join(crash_testcases_directory,
directory_name)
if not os.path.isdir(directory_path):
continue
if shell.get_directory_size(
directory_path) <= MAX_TESTCASE_DIRECTORY_SIZE:
continue
shell.remove_directory(directory_path)
# Rename all fuzzed testcase files as regular files.
for root, _, files in os.walk(crash_testcases_directory):
for filename in files:
if not filename.startswith(testcase_manager.FUZZ_PREFIX):
continue
file_path = os.path.join(root, filename)
stripped_file_name = os.path.basename(
file_path)[len(testcase_manager.FUZZ_PREFIX):]
stripped_file_path = os.path.join(os.path.dirname(file_path),
stripped_file_name)
try:
os.rename(file_path, stripped_file_path)
except:
raise Exception('Failed to rename testcase %s.' % file_path)
# Remove empty files and dirs to avoid the case where a fuzzer randomly
# chooses an empty dir/file and generates zero testcases.
shell.remove_empty_files(crash_testcases_directory)
shell.remove_empty_directories(crash_testcases_directory)
def main():
"""Main sync routine."""
tests_archive_bucket = environment.get_value('TESTS_ARCHIVE_BUCKET')
tests_archive_name = environment.get_value('TESTS_ARCHIVE_NAME')
tests_directory = environment.get_value('TESTS_DIR')
sync_interval = environment.get_value('SYNC_INTERVAL') # in seconds.
shell.create_directory(tests_directory)
# Sync old crash tests.
logs.log('Syncing old crash tests.')
crash_testcases_directory = os.path.join(tests_directory, 'CrashTests')
shell.create_directory(crash_testcases_directory)
unpack_crash_testcases(crash_testcases_directory)
# Sync web tests.
logs.log('Syncing web tests.')
src_directory = os.path.join(tests_directory, 'src')
gclient_file_path = os.path.join(tests_directory, '.gclient')
if not os.path.exists(gclient_file_path):
subprocess.check_call(
['fetch', '--no-history', 'chromium', '--nosvn=True'],
cwd=tests_directory)
if os.path.exists(src_directory):
subprocess.check_call(['gclient', 'revert'], cwd=src_directory)
subprocess.check_call(['git', 'pull'], cwd=src_directory)
subprocess.check_call(['gclient', 'sync'], cwd=src_directory)
else:
raise Exception('Unable to checkout web tests.')
clone_git_repository(tests_directory, 'v8',
'https://chromium.googlesource.com/v8/v8')
clone_git_repository(tests_directory, 'ChakraCore',
'https://github.com/Microsoft/ChakraCore.git')
clone_git_repository(tests_directory, 'gecko-dev',
'https://github.com/mozilla/gecko-dev.git')
clone_git_repository(tests_directory, 'webgl-conformance-tests',
'https://github.com/KhronosGroup/WebGL.git')
checkout_svn_repository(
tests_directory, 'WebKit/LayoutTests',
'http://svn.webkit.org/repository/webkit/trunk/LayoutTests')
checkout_svn_repository(
tests_directory, 'WebKit/JSTests/stress',
'http://svn.webkit.org/repository/webkit/trunk/JSTests/stress')
checkout_svn_repository(
tests_directory, 'WebKit/JSTests/es6',
'http://svn.webkit.org/repository/webkit/trunk/JSTests/es6')
create_gecko_tests_directory(tests_directory, 'gecko-dev', 'gecko-tests')
# Upload tests archive to google cloud storage.
logs.log('Uploading tests archive to cloud.')
tests_archive_local = os.path.join(tests_directory, tests_archive_name)
tests_archive_remote = 'gs://{bucket_name}/{archive_name}'.format(
bucket_name=tests_archive_bucket, archive_name=tests_archive_name)
shell.remove_file(tests_archive_local)
create_symbolic_link(tests_directory, 'gecko-dev/js/src/tests',
'spidermonkey')
create_symbolic_link(tests_directory, 'ChakraCore/test', 'chakra')
# FIXME: Find a way to rename LayoutTests to web_tests without breaking
# compatability with older testcases.
create_symbolic_link(tests_directory, 'src/third_party/blink/web_tests',
'LayoutTests')
subprocess.check_call([
'zip',
'-r',
tests_archive_local,
'CrashTests',
'LayoutTests',
'WebKit',
'gecko-tests',
'v8/test/mjsunit',
'spidermonkey',
'chakra',
'webgl-conformance-tests',
'-x',
'*.cc',
'-x',
'*.cpp',
'-x',
'*.py',
'-x',
'*.txt',
'-x',
'*-expected.*',
'-x',
'*.git*',
'-x',
'*.svn*',
],
cwd=tests_directory)
subprocess.check_call(
['gsutil', 'cp', tests_archive_local, tests_archive_remote])
logs.log('Completed cycle, sleeping for %s seconds.' % sync_interval)
time.sleep(sync_interval)
def update_data_bundle(fuzzer, data_bundle):
"""Updates a data bundle to the latest version."""
# This module can't be in the global imports due to appengine issues
# with multiprocessing and psutil imports.
from google_cloud_utils import gsutil
# If we are using a data bundle on NFS, it is expected that our testcases
# will usually be large enough that we would fill up our tmpfs directory
# pretty quickly. So, change it to use an on-disk directory.
if not data_bundle.is_local:
testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK')
environment.set_value('FUZZ_INPUTS', testcase_disk_directory)
data_bundle_directory = get_data_bundle_directory(fuzzer.name)
if not data_bundle_directory:
logs.log_error('Failed to setup data bundle %s.' % data_bundle.name)
return False
if not shell.create_directory(data_bundle_directory,
create_intermediates=True):
logs.log_error('Failed to create data bundle %s directory.' %
data_bundle.name)
return False
# Check if data bundle is up to date. If yes, skip the update.
if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
logs.log('Data bundle was recently synced, skip.')
return True
# Fetch lock for this data bundle.
if not _fetch_lock_for_data_bundle_update(data_bundle):
logs.log_error('Failed to lock data bundle %s.' % data_bundle.name)
return False
# Re-check if another bot did the sync already. If yes, skip.
if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
logs.log('Another bot finished the sync, skip.')
_release_lock_for_data_bundle_update(data_bundle)
return True
time_before_sync_start = time.time()
# No need to sync anything if this is a search index data bundle. In that
# case, the fuzzer will generate testcases from a gcs bucket periodically.
if not _is_search_index_data_bundle(data_bundle.name):
bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from bot.untrusted_runner import corpus_manager
from bot.untrusted_runner import file_host
worker_data_bundle_directory = file_host.rebase_to_worker_root(
data_bundle_directory)
file_host.create_directory(worker_data_bundle_directory,
create_intermediates=True)
result = corpus_manager.RemoteGSUtilRunner().rsync(
bucket_url, worker_data_bundle_directory, delete=False)
else:
result = gsutil.GSUtilRunner().rsync(bucket_url,
data_bundle_directory,
delete=False)
if result.return_code != 0:
logs.log_error('Failed to sync data bundle %s: %s.' %
(data_bundle.name, result.output))
_release_lock_for_data_bundle_update(data_bundle)
return False
# Update the testcase list file.
testcase_manager.create_testcase_list_file(data_bundle_directory)
# Write last synced time in the sync file.
sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory)
utils.write_data_to_file(time_before_sync_start, sync_file_path)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from bot.untrusted_runner import file_host
worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path)
file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path)
# Release acquired lock.
_release_lock_for_data_bundle_update(data_bundle)
return True
def copy_remote_file_to_local(remote_file_path, local_file_path):
"""Copies device file to a local file."""
shell.create_directory(
os.path.dirname(local_file_path), create_intermediates=True)
run_command(['pull', remote_file_path, local_file_path])
def store_file_in_cache(file_path,
cached_files_per_directory_limit=True,
force_update=False):
"""Get file from nfs cache if available."""
if not os.path.exists(file_path):
logs.log_error(
'Local file %s does not exist, nothing to store in cache.' % file_path)
return
if os.path.getsize(file_path) > CACHE_SIZE_LIMIT:
logs.log('File %s is too large to store in cache, skipping.' % file_path)
return
nfs_root = environment.get_value('NFS_ROOT')
if not nfs_root:
# No NFS, nothing to store in cache.
return
# If NFS server is not available due to heavy load, skip storage operation
# altogether as we would fail to store file.
if not os.path.exists(os.path.join(nfs_root, '.')): # Use . to iterate mount.
logs.log_warn('Cache %s not available.' % nfs_root)
return
cache_file_path = get_cache_file_path(file_path)
cache_directory = os.path.dirname(cache_file_path)
filename = os.path.basename(file_path)
if not os.path.exists(cache_directory):
if not shell.create_directory(cache_directory, create_intermediates=True):
logs.log_error('Failed to create cache directory %s.' % cache_directory)
return
# Check if the file already exists in cache.
if file_exists_in_cache(cache_file_path):
if not force_update:
return
# If we are forcing update, we need to remove current cached file and its
# metadata.
remove_cache_file_and_metadata(cache_file_path)
# Delete old cached files beyond our maximum storage limit.
if cached_files_per_directory_limit:
# Get a list of cached files.
cached_files_list = []
for cached_filename in os.listdir(cache_directory):
if cached_filename.endswith(CACHE_METADATA_FILE_EXTENSION):
continue
cached_file_path = os.path.join(cache_directory, cached_filename)
cached_files_list.append(cached_file_path)
mtime = lambda f: os.stat(f).st_mtime
last_used_cached_files_list = list(
sorted(cached_files_list, key=mtime, reverse=True))
for cached_file_path in (
last_used_cached_files_list[MAX_CACHED_FILES_PER_DIRECTORY - 1:]):
remove_cache_file_and_metadata(cached_file_path)
# Start storing the actual file in cache now.
logs.log('Started storing file %s into cache.' % filename)
# Fetch lock to store this file. Try only once since if any other bot has
# started to store it, we don't need to do it ourselves. Just bail out.
lock_name = 'store:cache_file:%s' % utils.string_hash(cache_file_path)
if not locks.acquire_lock(
lock_name, max_hold_seconds=CACHE_LOCK_TIMEOUT, retries=1, by_zone=True):
logs.log_warn(
'Unable to fetch lock to update cache file %s, skipping.' % filename)
return
# Check if another bot already updated it.
if file_exists_in_cache(cache_file_path):
locks.release_lock(lock_name, by_zone=True)
return
shell.copy_file(file_path, cache_file_path)
write_cache_file_metadata(cache_file_path, file_path)
time.sleep(CACHE_COPY_WAIT_TIME)
error_occurred = not file_exists_in_cache(cache_file_path)
locks.release_lock(lock_name, by_zone=True)
if error_occurred:
logs.log_error('Failed to store file %s into cache.' % filename)
else:
logs.log('Completed storing file %s into cache.' % filename)
