def get_stacktrace(testcase, stack_attribute='crash_stacktrace'):
"""Returns the stacktrace for a test case.
This may require a blobstore read.
"""
result = getattr(testcase, stack_attribute)
if not result or not result.startswith(data_types.BLOBSTORE_STACK_PREFIX):
return result
# For App Engine, we can't write to local file, so use blobs.read_key instead.
if environment.is_running_on_app_engine():
key = result[len(data_types.BLOBSTORE_STACK_PREFIX):]
return unicode(blobs.read_key(key), errors='replace')
key = result[len(data_types.BLOBSTORE_STACK_PREFIX):]
tmpdir = environment.get_value('BOT_TMPDIR')
tmp_stacktrace_file = os.path.join(tmpdir, 'stacktrace.tmp')
blobs.read_blob_to_disk(key, tmp_stacktrace_file)
try:
handle = open(tmp_stacktrace_file)
result = handle.read()
handle.close()
except:
logs.log_error(
'Unable to read stacktrace for testcase %d.' % testcase.key.id())
result = ''
shell.remove_file(tmp_stacktrace_file)
return result
def _unpack_build(self):
"""Unpack the custom build."""
if not shell.remove_directory(self.build_dir, recreate=True):
logs.log_error('Unable to clear custom binary directory.')
_handle_unrecoverable_error_on_windows()
return False
build_local_archive = os.path.join(self.build_dir,
self.custom_binary_filename)
if not blobs.read_blob_to_disk(self.custom_binary_key, build_local_archive):
return False
# If custom binary is an archive, then unpack it.
if archive.is_archive(self.custom_binary_filename):
if not _make_space_for_build(build_local_archive, self.base_build_dir):
# Remove downloaded archive to free up space and otherwise, it won't get
# deleted until next job run.
shell.remove_file(build_local_archive)
logs.log_fatal_and_exit('Could not make space for build.')
try:
archive.unpack(build_local_archive, self.build_dir, trusted=True)
except:
logs.log_error(
'Unable to unpack build archive %s.' % build_local_archive)
return False
# Remove the archive.
shell.remove_file(build_local_archive)
return True
def combine_tokens(tokens):
"""Use the ipc_message_util utility to create a file for these tokens."""
partial_ipcdumps = []
for start_index in range(0, len(tokens), TOKENS_PER_IPCDUMP):
end_index = min(start_index + TOKENS_PER_IPCDUMP, len(tokens))
current_tokens = tokens[start_index:end_index]
partial_ipcdumps.append(
create_partial_ipc_dump(current_tokens, file_path))
combined_file_path = None
if len(partial_ipcdumps) > 1:
combined_file_path = combine_ipc_dumps(partial_ipcdumps, file_path)
elif len(partial_ipcdumps) == 1:
combined_file_path = partial_ipcdumps[0]
if not combined_file_path:
# This can happen in the case of a timeout or other error. The actual
# error should already be logged, so no need to do it again here.
return ''
# TODO(mbarbella): Allow token combining functions to write files directly.
handle = open(combined_file_path, 'rb')
result = handle.read()
handle.close()
shell.remove_file(combined_file_path)
return result
def get_corpus(corpus_directory, fuzzer_name):
"""Get corpus directory.
This function will download latest corpus backup file from GCS, unzip
the file and put them in corpus directory.
Args:
directory: The directory to place corpus.
fuzzer_name: Fuzzer name, e.g. libpng_read_fuzzer, xml_parser_fuzzer, etc.
Returns:
True if the corpus can be acquired and False otherwise.
"""
backup_bucket_name = environment.get_value('BACKUP_BUCKET')
corpus_fuzzer_name = environment.get_value('CORPUS_FUZZER_NAME_OVERRIDE')
# Get GCS backup path.
gcs_backup_path = corpus_manager.gcs_url_for_backup_file(
backup_bucket_name, corpus_fuzzer_name, fuzzer_name,
corpus_manager.LATEST_BACKUP_TIMESTAMP)
# Get local backup path.
local_backup_name = os.path.basename(gcs_backup_path)
local_backup_path = os.path.join(corpus_directory, local_backup_name)
# Download latest backup.
if not storage.copy_file_from(gcs_backup_path, local_backup_path):
logs.log_error('Failed to download corpus from GCS bucket {}.'.format(
gcs_backup_path))
return False
# Extract corpus from zip file.
archive.unpack(local_backup_path, corpus_directory)
shell.remove_file(local_backup_path)
return True
def create(self, work_dir):
"""Configures a emulator process which can subsequently be `run`."""
# Download emulator image.
if not environment.get_value('ANDROID_EMULATOR_BUCKET_PATH'):
logs.log_error('ANDROID_EMULATOR_BUCKET_PATH is not set.')
return
archive_src_path = environment.get_value(
'ANDROID_EMULATOR_BUCKET_PATH')
archive_dst_path = os.path.join(work_dir, 'emulator_bundle.zip')
storage.copy_file_from(archive_src_path, archive_dst_path)
# Extract emulator image.
self.emulator_path = os.path.join(work_dir, 'emulator')
shell.remove_directory(self.emulator_path)
archive.unpack(archive_dst_path, self.emulator_path)
shell.remove_file(archive_dst_path)
# Stop any stale emulator instances.
stop_script_path = os.path.join(self.emulator_path, 'stop')
stop_proc = new_process.ProcessRunner(stop_script_path)
stop_proc.run_and_wait()
# Run emulator.
run_script_path = os.path.join(self.emulator_path, 'run')
self.process_runner = new_process.ProcessRunner(run_script_path)
def delete(self, remote_path):
"""Delete a remote file."""
fs_path = self.convert_path(remote_path)
shell.remove_file(fs_path)
fs_metadata_path = self.convert_path(remote_path, self.METADATA_DIR)
shell.remove_file(fs_metadata_path)
return True
def clear_pyc_files(directory):
"""Recursively remove all .pyc files from the given directory"""
for root_directory, _, filenames in os.walk(directory):
for filename in filenames:
if not filename.endswith('.pyc'):
continue
file_path = os.path.join(root_directory, filename)
shell.remove_file(file_path)
def clear_old_files(directory, extracted_file_set):
"""Remove files from the directory that isn't in the given file list."""
for root_directory, _, filenames in os.walk(directory):
for filename in filenames:
file_path = os.path.join(root_directory, filename)
if file_path not in extracted_file_set:
shell.remove_file(file_path)
shell.remove_empty_directories(directory)
def backup_corpus(backup_bucket_name, corpus, directory):
"""Archive and store corpus as a backup.
Args:
backup_bucket_name: Backup bucket.
corpus: The FuzzTargetCorpus.
directory: Path to directory to be archived and backuped.
Returns:
The backup GCS url, or None on failure.
"""
if not backup_bucket_name:
logs.log('No backup bucket provided, skipping corpus backup.')
return None
dated_backup_url = None
timestamp = str(utils.utcnow().date())
# The archive path for shutil.make_archive should be without an extension.
backup_archive_path = os.path.join(
os.path.dirname(os.path.normpath(directory)), timestamp)
try:
backup_archive_path = shutil.make_archive(backup_archive_path,
BACKUP_ARCHIVE_FORMAT,
directory)
logs.log('Created corpus backup file.',
backup_archive_path=backup_archive_path,
directory=directory,
size=os.path.getsize(backup_archive_path))
dated_backup_url = gcs_url_for_backup_file(
backup_bucket_name, corpus.engine,
corpus.project_qualified_target_name, timestamp)
if not storage.copy_file_to(backup_archive_path, dated_backup_url):
return None
latest_backup_url = gcs_url_for_backup_file(
backup_bucket_name, corpus.engine,
corpus.project_qualified_target_name, LATEST_BACKUP_TIMESTAMP)
if not storage.copy_blob(dated_backup_url, latest_backup_url):
logs.log_error('Failed to update latest corpus backup at "%s"' %
latest_backup_url)
except Exception as ex:
logs.log_error('backup_corpus failed: %s\n' % str(ex),
backup_bucket_name=backup_bucket_name,
directory=directory,
backup_archive_path=backup_archive_path)
finally:
# Remove backup archive.
shell.remove_file(backup_archive_path)
return dated_backup_url
def update_tests_if_needed():
"""Updates layout tests every day."""
data_directory = environment.get_value('FUZZ_DATA')
error_occured = False
expected_task_duration = 60 * 60 # 1 hour.
retry_limit = environment.get_value('FAIL_RETRIES')
temp_archive = os.path.join(data_directory, 'temp.zip')
tests_url = environment.get_value('WEB_TESTS_URL')
# Check if we have a valid tests url.
if not tests_url:
return
# Layout test updates are usually disabled to speedup local testing.
if environment.get_value('LOCAL_DEVELOPMENT'):
return
# |UPDATE_WEB_TESTS| env variable can be used to control our update behavior.
if not environment.get_value('UPDATE_WEB_TESTS'):
return
last_modified_time = persistent_cache.get_value(
LAYOUT_TEST_LAST_UPDATE_KEY,
constructor=datetime.datetime.utcfromtimestamp)
if (last_modified_time is not None and not dates.time_has_expired(
last_modified_time, days=LAYOUT_TEST_UPDATE_INTERVAL_DAYS)):
return
logs.log('Updating layout tests.')
tasks.track_task_start(tasks.Task('update_tests', '', ''),
expected_task_duration)
# Download and unpack the tests archive.
for _ in xrange(retry_limit):
try:
shell.remove_directory(data_directory, recreate=True)
storage.copy_file_from(tests_url, temp_archive)
archive.unpack(temp_archive, data_directory, trusted=True)
shell.remove_file(temp_archive)
error_occured = False
break
except:
logs.log_error(
'Could not retrieve and unpack layout tests archive. Retrying.'
)
error_occured = True
if not error_occured:
persistent_cache.set_value(LAYOUT_TEST_LAST_UPDATE_KEY,
time.time(),
persist_across_reboots=True)
tasks.track_task_end()
def unpack_testcase(testcase):
"""Unpack a testcase and return all files it is composed of."""
# Figure out where the testcase file should be stored.
input_directory, testcase_file_path = _get_testcase_file_and_path(testcase)
minimized = testcase.minimized_keys and testcase.minimized_keys != 'NA'
if minimized:
key = testcase.minimized_keys
archived = bool(testcase.archive_state
& data_types.ArchiveStatus.MINIMIZED)
else:
key = testcase.fuzzed_keys
archived = bool(testcase.archive_state
& data_types.ArchiveStatus.FUZZED)
if archived:
if minimized:
temp_filename = (os.path.join(
input_directory,
str(testcase.key.id()) + _TESTCASE_ARCHIVE_EXTENSION))
else:
temp_filename = os.path.join(input_directory,
testcase.archive_filename)
else:
temp_filename = testcase_file_path
if not blobs.read_blob_to_disk(key, temp_filename):
return None, input_directory, testcase_file_path
file_list = []
if archived:
archive.unpack(temp_filename, input_directory)
file_list = archive.get_file_list(temp_filename)
shell.remove_file(temp_filename)
file_exists = False
for file_name in file_list:
if os.path.basename(file_name) == os.path.basename(
testcase_file_path):
file_exists = True
break
if not file_exists:
logs.log_error(
'Expected file to run %s is not in archive. Base directory is %s and '
'files in archive are [%s].' %
(testcase_file_path, input_directory, ','.join(file_list)))
return None, input_directory, testcase_file_path
else:
file_list.append(testcase_file_path)
return file_list, input_directory, testcase_file_path
def _limit_corpus_size(corpus_directory, size_limit):
"""Limit number of files in a corpus directory."""
files_list = os.listdir(corpus_directory)
corpus_size = len(files_list)
if corpus_size <= size_limit:
# Corpus directory size is within limit, no more work to do.
return
files_to_delete = random.sample(files_list, corpus_size - size_limit)
for file_to_delete in files_to_delete:
file_to_delete_full_path = os.path.join(corpus_directory, file_to_delete)
shell.remove_file(file_to_delete_full_path)
def download_system_symbols_if_needed(symbols_directory):
"""Download system libraries from |SYMBOLS_URL| and cache locally."""
if not should_download_symbols():
return
# Get the build fingerprint parameters.
build_params = settings.get_build_parameters()
if not build_params:
logs.log_error('Unable to determine build parameters.')
return
build_params_check_path = os.path.join(symbols_directory,
'.cached_build_params')
build_id = build_params.get('build_id')
target = build_params.get('target')
build_type = build_params.get('type')
if not build_id or not target or not build_type:
logs.log_error('Null build parameters found, exiting.')
return
symbols_archive_filename = f'{target}-symbols-{build_id}.zip'
artifact_file_name = symbols_archive_filename
output_filename_override = None
# Include type and sanitizer information in the target.
tool_suffix = environment.get_value('SANITIZER_TOOL_NAME')
target_with_type_and_san = f'{target}-{build_type}'
if tool_suffix and not tool_suffix in target_with_type_and_san:
target_with_type_and_san += f'_{tool_suffix}'
targets_with_type_and_san = [target_with_type_and_san]
symbols_archive_path = os.path.join(symbols_directory,
symbols_archive_filename)
download_artifact_if_needed(build_id, symbols_directory,
symbols_archive_path,
targets_with_type_and_san, artifact_file_name,
output_filename_override, build_params,
build_params_check_path)
if not os.path.exists(symbols_archive_path):
logs.log_error('Unable to locate symbols archive %s.' %
symbols_archive_path)
return
# Store the artifact for later use or for use by other bots.
storage.store_file_in_cache(symbols_archive_path)
archive.unpack(symbols_archive_path, symbols_directory, trusted=True)
shell.remove_file(symbols_archive_path)
utils.write_data_to_file(build_params, build_params_check_path)
def _cross_pollinate_other_fuzzer_corpuses(self):
"""Add other fuzzer corpuses to shared corpus path for cross-pollination."""
corpus_backup_date = utils.utcnow().date() - datetime.timedelta(
days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS)
for cross_pollinate_fuzzer in self.cross_pollinate_fuzzers:
project_qualified_name = (
cross_pollinate_fuzzer.fuzz_target.project_qualified_name())
backup_bucket_name = cross_pollinate_fuzzer.backup_bucket_name
corpus_engine_name = cross_pollinate_fuzzer.corpus_engine_name
corpus_backup_url = corpus_manager.gcs_url_for_backup_file(
backup_bucket_name,
corpus_engine_name,
project_qualified_name,
corpus_backup_date,
)
corpus_backup_local_filename = "%s-%s" % (
project_qualified_name,
os.path.basename(corpus_backup_url),
)
corpus_backup_local_path = os.path.join(
self.shared_corpus_path, corpus_backup_local_filename)
if not storage.exists(corpus_backup_url, ignore_errors=True):
# This can happen in cases when a new fuzz target is checked in or if
# missed to capture a backup for a particular day (for OSS-Fuzz, this
# will result in a 403 instead of 404 since that GCS path belongs to
# other project). So, just log a warning for debugging purposes only.
logs.log_warn("Corpus backup does not exist, ignoring: %s." %
corpus_backup_url)
continue
if not storage.copy_file_from(corpus_backup_url,
corpus_backup_local_path):
continue
corpus_backup_output_directory = os.path.join(
self.shared_corpus_path, project_qualified_name)
shell.create_directory(corpus_backup_output_directory)
result = archive.unpack(corpus_backup_local_path,
corpus_backup_output_directory)
shell.remove_file(corpus_backup_local_path)
if result:
logs.log(
"Corpus backup url %s successfully unpacked into shared corpus."
% corpus_backup_url)
else:
logs.log_error("Failed to unpack corpus backup from url %s." %
corpus_backup_url)
def read_from_disk(testcase_file_path, delete=False):
"""Read the TestcaseRun for the given testcase."""
stats_file_path = TestcaseRun.get_stats_filename(testcase_file_path)
if not os.path.exists(stats_file_path):
return None
fuzzer_run = None
with open(stats_file_path) as f:
fuzzer_run = BaseRun.from_json(f.read())
if delete:
shell.remove_file(stats_file_path)
return fuzzer_run
def generate_new_testcase_mutations_using_radamsa(
corpus_directory, new_testcase_mutations_directory,
generation_timeout):
"""Generate new testcase mutations based on Radamsa."""
radamsa_path = get_radamsa_path()
if not radamsa_path:
# Mutations using radamsa are not supported on current platform, bail out.
return
radamsa_runner = new_process.ProcessRunner(radamsa_path)
files_list = shell.get_files_list(corpus_directory)
filtered_files_list = [
f for f in files_list if os.path.getsize(f) <= CORPUS_INPUT_SIZE_LIMIT
]
if not filtered_files_list:
# No mutations to do on an empty corpus or one with very large files.
return
old_corpus_size = shell.get_directory_file_count(
new_testcase_mutations_directory)
expected_completion_time = time.time() + generation_timeout
for i in range(RADAMSA_MUTATIONS):
original_file_path = random_choice(filtered_files_list)
original_filename = os.path.basename(original_file_path)
output_path = os.path.join(
new_testcase_mutations_directory,
get_radamsa_output_filename(original_filename, i))
result = radamsa_runner.run_and_wait(
['-o', output_path, original_file_path], timeout=RADAMSA_TIMEOUT)
if (os.path.exists(output_path)
and os.path.getsize(output_path) > CORPUS_INPUT_SIZE_LIMIT):
# Skip large files to avoid furthur mutations and impact fuzzing
# efficiency.
shell.remove_file(output_path)
elif result.return_code or result.timed_out:
logs.log_warn('Radamsa failed to mutate or timed out.',
output=result.output)
# Check if we exceeded our timeout. If yes, do no more mutations and break.
if time.time() > expected_completion_time:
break
new_corpus_size = shell.get_directory_file_count(
new_testcase_mutations_directory)
logs.log('Added %d tests using Radamsa mutations.' %
(new_corpus_size - old_corpus_size))
def combine_ipc_dumps(ipcdumps, original_file_path):
"""Combines a list of ipcdump files into a single dump."""
input_file_string = ','.join(ipcdumps)
executable = get_ipc_message_util_executable()
output_file_path = get_temporary_file_name(original_file_path)
command_line = shell.get_command_line_from_argument_list(
[executable, input_file_string, output_file_path])
return_code, _, output = process_handler.run_process(
command_line, testcase_run=False, timeout=COMBINED_IPCDUMP_TIMEOUT)
for ipcdump in ipcdumps:
shell.remove_file(ipcdump)
if return_code or not os.path.exists(output_file_path):
logs.log_error('Failed to create ipc dump file %s.' % output)
return None
return output_file_path
def create(self):
"""Configures a emulator process which can subsequently be `run`."""
# Download emulator image.
if not environment.get_value('ANDROID_EMULATOR_BUCKET_PATH'):
logs.log_error('ANDROID_EMULATOR_BUCKET_PATH is not set.')
return
temp_directory = environment.get_value('BOT_TMPDIR')
archive_src_path = environment.get_value(
'ANDROID_EMULATOR_BUCKET_PATH')
archive_dst_path = os.path.join(temp_directory, 'emulator_bundle.zip')
storage.copy_file_from(archive_src_path, archive_dst_path)
# Extract emulator image.
self.emulator_path = os.path.join(temp_directory, 'emulator')
archive.unpack(archive_dst_path, self.emulator_path)
shell.remove_file(archive_dst_path)
# Run emulator.
script_path = os.path.join(self.emulator_path, 'run')
self.process_runner = new_process.ProcessRunner(script_path)
def recreate_gce_device():
"""Recreate gce device, restoring from backup images."""
logs.log('Reimaging gce device.')
cvd_dir = environment.get_value('CVD_DIR')
stop_gce_device()
# Delete all existing images.
image_dir = cvd_dir
for image_file_path in glob.glob(os.path.join(image_dir, '*.img')):
shell.remove_file(image_file_path)
# Restore images from backup.
backup_image_dir = os.path.join(cvd_dir, 'backup')
for image_filename in os.listdir(backup_image_dir):
image_src = os.path.join(backup_image_dir, image_filename)
image_dest = os.path.join(image_dir, image_filename)
shell.copy_file(image_src, image_dest)
start_gce_device()
def remove_testcases_from_directories(directories):
"""Removes all testcases and their dependencies from testcase directories."""
generators = []
for directory in directories:
if not directory.strip():
continue
# If there is a bot-specific files list, delete it now.
bot_testcases_file_path = utils.get_bot_testcases_file_path(directory)
shell.remove_file(bot_testcases_file_path)
generators.append(os.walk(directory))
for generator in generators:
for structure in generator:
base_directory = structure[0]
for filename in structure[2]:
if not is_testcase_resource(filename):
continue
if filename.startswith(RESOURCES_PREFIX):
# In addition to removing this file, remove all resources.
resources_file_path = os.path.join(base_directory,
filename)
resources = read_resource_list(resources_file_path)
for resource in resources:
shell.remove_file(resource)
file_path = os.path.join(base_directory, filename)
shell.remove_file(file_path)
def filter_stacktrace(stacktrace):
"""Filters stacktrace and returns content appropriate for storage as an
appengine entity."""
unicode_stacktrace = utils.decode_to_unicode(stacktrace)
if len(unicode_stacktrace) <= data_types.STACKTRACE_LENGTH_LIMIT:
return unicode_stacktrace
tmpdir = environment.get_value('BOT_TMPDIR')
tmp_stacktrace_file = os.path.join(tmpdir, 'stacktrace.tmp')
try:
with open(tmp_stacktrace_file, 'w') as handle:
handle.write(stacktrace)
with open(tmp_stacktrace_file, 'r') as handle:
key = blobs.write_blob(handle)
except Exception:
logs.log_error('Unable to write crash stacktrace to temporary file.')
shell.remove_file(tmp_stacktrace_file)
return unicode_stacktrace[(-1 * data_types.STACKTRACE_LENGTH_LIMIT):]
shell.remove_file(tmp_stacktrace_file)
return '%s%s' % (data_types.BLOBSTORE_STACK_PREFIX, key)
def _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
"""Return true if the data bundle is up to date, false otherwise."""
sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from bot.untrusted_runner import file_host
worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path)
shell.remove_file(sync_file_path)
file_host.copy_file_from_worker(worker_sync_file_path, sync_file_path)
if not os.path.exists(sync_file_path):
return False
last_sync_time = datetime.datetime.utcfromtimestamp(
utils.read_data_from_file(sync_file_path))
# Check if we recently synced.
if not dates.time_has_expired(
last_sync_time, seconds=_DATA_BUNDLE_SYNC_INTERVAL_IN_SECONDS):
return True
# For search index data bundle, we don't sync them from bucket. Instead, we
# rely on the fuzzer to generate testcases periodically.
if _is_search_index_data_bundle(data_bundle.name):
return False
# Check when the bucket url had last updates. If no new updates, no need to
# update directory.
bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name)
last_updated_time = storage.last_updated(bucket_url)
if last_updated_time and last_sync_time > last_updated_time:
logs.log("Data bundle %s has no new content from last sync." %
data_bundle.name)
return True
return False
def archive_testcase_and_dependencies_in_gcs(resource_list, testcase_path):
"""Archive testcase and its dependencies, and store in blobstore."""
if not os.path.exists(testcase_path):
logs.log_error('Unable to find testcase %s.' % testcase_path)
return None, None, None, None
absolute_filename = testcase_path
archived = False
zip_filename = None
zip_path = None
if not resource_list:
resource_list = []
# Add resource dependencies based on testcase path. These include
# stuff like extensions directory, dependency files, etc.
resource_list.extend(
testcase_manager.get_resource_dependencies(testcase_path))
# Filter out duplicates, directories, and files that do not exist.
resource_list = utils.filter_file_list(resource_list)
logs.log('Testcase and related files :\n%s' % str(resource_list))
if len(resource_list) <= 1:
# If this does not have any resources, just save the testcase.
# TODO(flowerhack): Update this when we teach CF how to download testcases.
try:
file_handle = open(testcase_path, 'rb')
except IOError:
logs.log_error('Unable to open testcase %s.' % testcase_path)
return None, None, None, None
else:
# If there are resources, create an archive.
# Find the common root directory for all of the resources.
# Assumption: resource_list[0] is the testcase path.
base_directory_list = resource_list[0].split(os.path.sep)
for list_index in range(1, len(resource_list)):
current_directory_list = resource_list[list_index].split(
os.path.sep)
length = min(len(base_directory_list), len(current_directory_list))
for directory_index in range(length):
if (current_directory_list[directory_index] !=
base_directory_list[directory_index]):
base_directory_list = base_directory_list[
0:directory_index]
break
base_directory = os.path.sep.join(base_directory_list)
logs.log('Subresource common base directory: %s' % base_directory)
if base_directory:
# Common parent directory, archive sub-paths only.
base_len = len(base_directory) + len(os.path.sep)
else:
# No common parent directory, archive all paths as it-is.
base_len = 0
# Prepare the filename for the archive.
zip_filename, _ = os.path.splitext(os.path.basename(testcase_path))
zip_filename += _TESTCASE_ARCHIVE_EXTENSION
# Create the archive.
zip_path = os.path.join(environment.get_value('INPUT_DIR'),
zip_filename)
zip_file = zipfile.ZipFile(zip_path, 'w')
for file_name in resource_list:
if os.path.exists(file_name):
relative_filename = file_name[base_len:]
zip_file.write(file_name, relative_filename,
zipfile.ZIP_DEFLATED)
zip_file.close()
try:
file_handle = open(zip_path, 'rb')
except IOError:
logs.log_error('Unable to open testcase archive %s.' % zip_path)
return None, None, None, None
archived = True
absolute_filename = testcase_path[base_len:]
fuzzed_key = blobs.write_blob(file_handle)
file_handle.close()
# Don't need the archive after writing testcase to blobstore.
if zip_path:
shell.remove_file(zip_path)
return fuzzed_key, archived, absolute_filename, zip_filename
def update_fuzzer_and_data_bundles(fuzzer_name):
"""Update the fuzzer with a given name if necessary."""
fuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == fuzzer_name).get()
if not fuzzer:
logs.log_error('No fuzzer exists with name %s.' % fuzzer_name)
raise errors.InvalidFuzzerError
# Set some helper environment variables.
fuzzer_directory = get_fuzzer_directory(fuzzer_name)
environment.set_value('FUZZER_DIR', fuzzer_directory)
environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content)
# Adjust the test timeout, if user has provided one.
if fuzzer.timeout:
environment.set_value('TEST_TIMEOUT', fuzzer.timeout)
# Increase fuzz test timeout if the fuzzer timeout is higher than its
# current value.
fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT')
if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout:
environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout)
# Adjust the max testcases if this fuzzer has specified a lower limit.
max_testcases = environment.get_value('MAX_TESTCASES')
if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases:
environment.set_value('MAX_TESTCASES', fuzzer.max_testcases)
# Check for updates to this fuzzer.
version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name)
if (not fuzzer.builtin
and revisions.needs_update(version_file, fuzzer.revision)):
logs.log('Fuzzer update was found, updating.')
# Clear the old fuzzer directory if it exists.
if not shell.remove_directory(fuzzer_directory, recreate=True):
logs.log_error('Failed to clear fuzzer directory.')
return False
# Copy the archive to local disk and unpack it.
archive_path = os.path.join(fuzzer_directory, fuzzer.filename)
if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path):
logs.log_error('Failed to copy fuzzer archive.')
return False
try:
archive.unpack(archive_path, fuzzer_directory)
except Exception:
error_message = (
'Failed to unpack fuzzer archive %s '
'(bad archive or unsupported format).') % fuzzer.filename
logs.log_error(error_message)
fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
fuzzer_name=fuzzer_name)
return False
fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path)
if not os.path.exists(fuzzer_path):
error_message = (
'Fuzzer executable %s not found. '
'Check fuzzer configuration.') % fuzzer.executable_path
logs.log_error(error_message)
fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
fuzzer_name=fuzzer_name)
return False
# Make fuzzer executable.
os.chmod(fuzzer_path, 0o750)
# Cleanup unneeded archive.
shell.remove_file(archive_path)
# Save the current revision of this fuzzer in a file for later checks.
revisions.write_revision_to_revision_file(version_file,
fuzzer.revision)
logs.log('Updated fuzzer to revision %d.' % fuzzer.revision)
# Setup data bundles associated with this fuzzer.
data_bundles = ndb_utils.get_all_from_query(
data_types.DataBundle.query(
data_types.DataBundle.name == fuzzer.data_bundle_name))
for data_bundle in data_bundles:
if not update_data_bundle(fuzzer, data_bundle):
return False
# Setup environment variable for launcher script path.
if fuzzer.launcher_script:
fuzzer_launcher_path = shell.get_execute_command(
os.path.join(fuzzer_directory, fuzzer.launcher_script))
environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path)
return True
def delete_value(key): """Removes the value for a key.""" value_path = get_value_file_path(key) shell.remove_file(value_path)
def main():
"""Main sync routine."""
tests_archive_bucket = environment.get_value('TESTS_ARCHIVE_BUCKET')
tests_archive_name = environment.get_value('TESTS_ARCHIVE_NAME')
tests_directory = environment.get_value('TESTS_DIR')
sync_interval = environment.get_value('SYNC_INTERVAL') # in seconds.
shell.create_directory(tests_directory)
# Sync old crash tests.
logs.log('Syncing old crash tests.')
crash_testcases_directory = os.path.join(tests_directory, 'CrashTests')
shell.create_directory(crash_testcases_directory)
unpack_crash_testcases(crash_testcases_directory)
# Sync web tests.
logs.log('Syncing web tests.')
src_directory = os.path.join(tests_directory, 'src')
gclient_file_path = os.path.join(tests_directory, '.gclient')
if not os.path.exists(gclient_file_path):
subprocess.check_call(
['fetch', '--no-history', 'chromium', '--nosvn=True'],
cwd=tests_directory)
if os.path.exists(src_directory):
subprocess.check_call(['gclient', 'revert'], cwd=src_directory)
subprocess.check_call(['git', 'pull'], cwd=src_directory)
subprocess.check_call(['gclient', 'sync'], cwd=src_directory)
else:
raise Exception('Unable to checkout web tests.')
clone_git_repository(tests_directory, 'v8',
'https://chromium.googlesource.com/v8/v8')
clone_git_repository(tests_directory, 'ChakraCore',
'https://github.com/Microsoft/ChakraCore.git')
clone_git_repository(tests_directory, 'gecko-dev',
'https://github.com/mozilla/gecko-dev.git')
clone_git_repository(tests_directory, 'webgl-conformance-tests',
'https://github.com/KhronosGroup/WebGL.git')
checkout_svn_repository(
tests_directory, 'WebKit/LayoutTests',
'http://svn.webkit.org/repository/webkit/trunk/LayoutTests')
checkout_svn_repository(
tests_directory, 'WebKit/JSTests/stress',
'http://svn.webkit.org/repository/webkit/trunk/JSTests/stress')
checkout_svn_repository(
tests_directory, 'WebKit/JSTests/es6',
'http://svn.webkit.org/repository/webkit/trunk/JSTests/es6')
create_gecko_tests_directory(tests_directory, 'gecko-dev', 'gecko-tests')
# Upload tests archive to google cloud storage.
logs.log('Uploading tests archive to cloud.')
tests_archive_local = os.path.join(tests_directory, tests_archive_name)
tests_archive_remote = 'gs://{bucket_name}/{archive_name}'.format(
bucket_name=tests_archive_bucket, archive_name=tests_archive_name)
shell.remove_file(tests_archive_local)
create_symbolic_link(tests_directory, 'gecko-dev/js/src/tests',
'spidermonkey')
create_symbolic_link(tests_directory, 'ChakraCore/test', 'chakra')
# FIXME: Find a way to rename LayoutTests to web_tests without breaking
# compatability with older testcases.
create_symbolic_link(tests_directory, 'src/third_party/blink/web_tests',
'LayoutTests')
subprocess.check_call([
'zip',
'-r',
tests_archive_local,
'CrashTests',
'LayoutTests',
'WebKit',
'gecko-tests',
'v8/test/mjsunit',
'spidermonkey',
'chakra',
'webgl-conformance-tests',
'-x',
'*.cc',
'-x',
'*.cpp',
'-x',
'*.py',
'-x',
'*.txt',
'-x',
'*-expected.*',
'-x',
'*.git*',
'-x',
'*.svn*',
],
cwd=tests_directory)
subprocess.check_call(
['gsutil', 'cp', tests_archive_local, tests_archive_remote])
logs.log('Completed cycle, sleeping for %s seconds.' % sync_interval)
time.sleep(sync_interval)
def remove_cache_file_and_metadata(cache_file_path):
"""Removes cache file and its metadata."""
logs.log('Removing cache file %s and its metadata.' % cache_file_path)
shell.remove_file(get_cache_file_metadata_path(cache_file_path))
shell.remove_file(cache_file_path)
def main():
"""Main sync routine."""
tests_archive_bucket = environment.get_value("TESTS_ARCHIVE_BUCKET")
tests_archive_name = environment.get_value("TESTS_ARCHIVE_NAME")
tests_directory = environment.get_value("TESTS_DIR")
sync_interval = environment.get_value("SYNC_INTERVAL") # in seconds.
shell.create_directory(tests_directory)
# Sync old crash tests.
logs.log("Syncing old crash tests.")
crash_testcases_directory = os.path.join(tests_directory, "CrashTests")
shell.create_directory(crash_testcases_directory)
unpack_crash_testcases(crash_testcases_directory)
# Sync web tests.
logs.log("Syncing web tests.")
src_directory = os.path.join(tests_directory, "src")
gclient_file_path = os.path.join(tests_directory, ".gclient")
if not os.path.exists(gclient_file_path):
subprocess.check_call(
["fetch", "--no-history", "chromium", "--nosvn=True"],
cwd=tests_directory)
if os.path.exists(src_directory):
subprocess.check_call(["gclient", "revert"], cwd=src_directory)
subprocess.check_call(["git", "pull"], cwd=src_directory)
subprocess.check_call(["gclient", "sync"], cwd=src_directory)
else:
raise Exception("Unable to checkout web tests.")
clone_git_repository(tests_directory, "v8",
"https://chromium.googlesource.com/v8/v8")
clone_git_repository(tests_directory, "ChakraCore",
"https://github.com/Microsoft/ChakraCore.git")
clone_git_repository(tests_directory, "gecko-dev",
"https://github.com/mozilla/gecko-dev.git")
clone_git_repository(
tests_directory,
"webgl-conformance-tests",
"https://github.com/KhronosGroup/WebGL.git",
)
checkout_svn_repository(
tests_directory,
"WebKit/LayoutTests",
"http://svn.webkit.org/repository/webkit/trunk/LayoutTests",
)
checkout_svn_repository(
tests_directory,
"WebKit/JSTests/stress",
"http://svn.webkit.org/repository/webkit/trunk/JSTests/stress",
)
checkout_svn_repository(
tests_directory,
"WebKit/JSTests/es6",
"http://svn.webkit.org/repository/webkit/trunk/JSTests/es6",
)
create_gecko_tests_directory(tests_directory, "gecko-dev", "gecko-tests")
# Upload tests archive to google cloud storage.
logs.log("Uploading tests archive to cloud.")
tests_archive_local = os.path.join(tests_directory, tests_archive_name)
tests_archive_remote = "gs://{bucket_name}/{archive_name}".format(
bucket_name=tests_archive_bucket, archive_name=tests_archive_name)
shell.remove_file(tests_archive_local)
create_symbolic_link(tests_directory, "gecko-dev/js/src/tests",
"spidermonkey")
create_symbolic_link(tests_directory, "ChakraCore/test", "chakra")
# FIXME: Find a way to rename LayoutTests to web_tests without breaking
# compatability with older testcases.
create_symbolic_link(tests_directory, "src/third_party/blink/web_tests",
"LayoutTests")
subprocess.check_call(
[
"zip",
"-r",
tests_archive_local,
"CrashTests",
"LayoutTests",
"WebKit",
"gecko-tests",
"v8/test/mjsunit",
"spidermonkey",
"chakra",
"webgl-conformance-tests",
"-x",
"*.cc",
"-x",
"*.cpp",
"-x",
"*.py",
"-x",
"*.txt",
"-x",
"*-expected.*",
"-x",
"*.git*",
"-x",
"*.svn*",
],
cwd=tests_directory,
)
subprocess.check_call(
["gsutil", "cp", tests_archive_local, tests_archive_remote])
logs.log("Completed cycle, sleeping for %s seconds." % sync_interval)
time.sleep(sync_interval)
def download_system_symbols_if_needed(symbols_directory):
"""Download system libraries from |SYMBOLS_URL| and cache locally."""
# For local testing, we do not have access to the cloud storage bucket with
# the symbols. In this case, just bail out.
if environment.get_value('LOCAL_DEVELOPMENT'):
return
# When running reproduce tool locally, we do not have access to the cloud
# storage bucket with the symbols. In this case, just bail out.
if environment.get_value('REPRODUCE_TOOL'):
return
# We have archived symbols for google builds only.
if not settings.is_google_device():
return
# Get the build fingerprint parameters.
build_params = settings.get_build_parameters()
if not build_params:
logs.log_error('Unable to determine build parameters.')
return
build_id = build_params.get('build_id')
target = build_params.get('target')
type = build_params.get('type')
if not build_id or not target or not type:
logs.log_error('Null build parameters found, exiting.')
return
# Check if we already have the symbols in cache.
build_params_check_path = os.path.join(symbols_directory,
'.cached_build_params')
cached_build_params = utils.read_data_from_file(
build_params_check_path, eval_data=True)
if cached_build_params and cmp(cached_build_params, build_params) == 0:
# No work to do, same system symbols already in cache.
return
symbols_archive_filename = '%s-symbols-%s.zip' % (target, build_id)
symbols_archive_path = os.path.join(symbols_directory,
symbols_archive_filename)
# Delete existing symbols directory first.
shell.remove_directory(symbols_directory, recreate=True)
# Fetch symbol file from cloud storage cache (if available).
found_in_cache = storage.get_file_from_cache_if_exists(
symbols_archive_path, update_modification_time_on_access=False)
if not found_in_cache:
# Include type and sanitizer information in the target.
target_with_type_and_san = '%s-%s' % (target, type)
tool_suffix = environment.get_value('SANITIZER_TOOL_NAME')
if tool_suffix and not tool_suffix in target_with_type_and_san:
target_with_type_and_san += '_%s' % tool_suffix
# Fetch the artifact now.
fetch_artifact.get(build_id, target_with_type_and_san,
symbols_archive_filename, symbols_directory)
if not os.path.exists(symbols_archive_path):
logs.log_error(
'Unable to locate symbols archive %s.' % symbols_archive_path)
return
# Store the artifact for later use or for use by other bots.
storage.store_file_in_cache(symbols_archive_path)
archive.unpack(symbols_archive_path, symbols_directory, trusted=True)
shell.remove_file(symbols_archive_path)
utils.write_data_to_file(build_params, build_params_check_path)
def _unpack_build(base_build_dir, build_dir, build_url, target_weights=None):
"""Unpacks a build from a build url into the build directory."""
# Track time taken to unpack builds so that it doesn't silently regress.
start_time = time.time()
# Free up memory.
utils.python_gc()
# Remove the current build.
logs.log('Removing build directory %s.' % build_dir)
if not shell.remove_directory(build_dir, recreate=True):
logs.log_error('Unable to clear build directory %s.' % build_dir)
_handle_unrecoverable_error_on_windows()
return False
# Decide whether to use cache build archives or not.
use_cache = environment.get_value('CACHE_STORE', False)
# Download build archive locally.
build_local_archive = os.path.join(build_dir, os.path.basename(build_url))
# Make the disk space necessary for the archive available.
archive_size = storage.get_download_file_size(
build_url, build_local_archive, use_cache=True)
if archive_size is not None and not _make_space(archive_size, base_build_dir):
shell.clear_data_directories()
logs.log_fatal_and_exit(
'Failed to make space for download. '
'Cleared all data directories to free up space, exiting.')
logs.log('Downloading build from url %s.' % build_url)
try:
storage.copy_file_from(build_url, build_local_archive, use_cache=use_cache)
except:
logs.log_error('Unable to download build url %s.' % build_url)
return False
unpack_everything = environment.get_value('UNPACK_ALL_FUZZ_TARGETS_AND_FILES')
if not unpack_everything:
# For fuzzing, pick a random fuzz target so that we only un-archive that
# particular fuzz target and its dependencies and save disk space.
# If we are going to unpack everythng in archive based on
# |UNPACK_ALL_FUZZ_TARGETS_AND_FILES| in the job defition, then don't set a
# random fuzz target before we've unpacked the build. It won't actually save
# us anything in this case and can be really expensive for large builds
# (such as Chrome OS). Defer setting it until after the build has been
# unpacked.
_set_random_fuzz_target_for_fuzzing_if_needed(
_get_fuzz_targets_from_archive(build_local_archive), target_weights)
# Actual list of files to unpack can be smaller if we are only unarchiving
# a particular fuzz target.
file_match_callback = _get_file_match_callback()
assert not (unpack_everything and file_match_callback is not None)
if not _make_space_for_build(build_local_archive, base_build_dir,
file_match_callback):
shell.clear_data_directories()
logs.log_fatal_and_exit(
'Failed to make space for build. '
'Cleared all data directories to free up space, exiting.')
# Unpack the local build archive.
logs.log('Unpacking build archive %s.' % build_local_archive)
trusted = not utils.is_oss_fuzz()
try:
archive.unpack(
build_local_archive,
build_dir,
trusted=trusted,
file_match_callback=file_match_callback)
except:
logs.log_error('Unable to unpack build archive %s.' % build_local_archive)
return False
if unpack_everything:
# Set a random fuzz target now that the build has been unpacked, if we
# didn't set one earlier.
_set_random_fuzz_target_for_fuzzing_if_needed(
_get_fuzz_targets_from_dir(build_dir), target_weights)
# If this is partial build due to selected build files, then mark it as such
# so that it is not re-used.
if file_match_callback:
partial_build_file_path = os.path.join(build_dir, PARTIAL_BUILD_FILE)
utils.write_data_to_file('', partial_build_file_path)
# No point in keeping the archive around.
shell.remove_file(build_local_archive)
end_time = time.time()
elapsed_time = end_time - start_time
log_func = logs.log_warn if elapsed_time > UNPACK_TIME_LIMIT else logs.log
log_func('Build took %0.02f minutes to unpack.' % (elapsed_time / 60.))
return True
