def unpack_testcase(testcase):
"""Unpack a testcase and return all files it is composed of."""
# Figure out where the testcase file should be stored.
input_directory, testcase_file_path = _get_testcase_file_and_path(testcase)
minimized = testcase.minimized_keys and testcase.minimized_keys != 'NA'
if minimized:
key = testcase.minimized_keys
archived = bool(testcase.archive_state
& data_types.ArchiveStatus.MINIMIZED)
else:
key = testcase.fuzzed_keys
archived = bool(testcase.archive_state
& data_types.ArchiveStatus.FUZZED)
if archived:
if minimized:
temp_filename = (os.path.join(
input_directory,
str(testcase.key.id()) + _TESTCASE_ARCHIVE_EXTENSION))
else:
temp_filename = os.path.join(input_directory,
testcase.archive_filename)
else:
temp_filename = testcase_file_path
if not blobs.read_blob_to_disk(key, temp_filename):
return None, input_directory, testcase_file_path
file_list = []
if archived:
archive.unpack(temp_filename, input_directory)
file_list = archive.get_file_list(temp_filename)
shell.remove_file(temp_filename)
file_exists = False
for file_name in file_list:
if os.path.basename(file_name) == os.path.basename(
testcase_file_path):
file_exists = True
break
if not file_exists:
logs.log_error(
'Expected file to run %s is not in archive. Base directory is %s and '
'files in archive are [%s].' %
(testcase_file_path, input_directory, ','.join(file_list)))
return None, input_directory, testcase_file_path
else:
file_list.append(testcase_file_path)
return file_list, input_directory, testcase_file_path
def _download_testcase(testcase_id, testcase, configuration):
"""Download the test case and return its path."""
print('Downloading testcase...')
testcase_download_url = '{url}?id={id}'.format(
url=configuration.get('testcase_download_url'), id=testcase_id)
response, content = http_utils.request(
testcase_download_url,
method=http_utils.GET_METHOD,
configuration=configuration)
if response.status != 200:
raise errors.ReproduceToolUnrecoverableError(
'Unable to download test case.')
bot_absolute_filename = response['x-goog-meta-filename']
# Store the test case in the config directory for debuggability.
testcase_directory = os.path.join(CONFIG_DIRECTORY, 'current-testcase')
shell.remove_directory(testcase_directory, recreate=True)
environment.set_value('FUZZ_INPUTS', testcase_directory)
testcase_path = os.path.join(testcase_directory,
os.path.basename(bot_absolute_filename))
utils.write_data_to_file(content, testcase_path)
# Unpack the test case if it's archived.
# TODO(mbarbella): Rewrite setup.unpack_testcase and share this code.
if testcase.minimized_keys and testcase.minimized_keys != 'NA':
mask = data_types.ArchiveStatus.MINIMIZED
else:
mask = data_types.ArchiveStatus.FUZZED
if testcase.archive_state & mask:
archive.unpack(testcase_path, testcase_directory)
file_list = archive.get_file_list(testcase_path)
testcase_path = None
for file_name in file_list:
if os.path.basename(file_name) == os.path.basename(
testcase.absolute_path):
testcase_path = os.path.join(testcase_directory, file_name)
break
if not testcase_path:
raise errors.ReproduceToolUnrecoverableError(
'Test case file was not found in archive.\n'
'Original filename: {absolute_path}.\n'
'Archive contents: {file_list}'.format(
absolute_path=testcase.absolute_path, file_list=file_list))
return testcase_path
def _download_testcase(testcase_id, testcase):
"""Download the test case and return its path."""
response, content = _http_request(
TESTCASE_DOWNLOAD_URL.format(testcase_id=testcase_id),
method=_GET_METHOD)
if response.status != 200:
raise ReproduceToolException('Unable to download test case.')
# Create a temporary directory where we can store the test case.
bot_absolute_filename = response['x-goog-meta-filename']
testcase_directory = os.path.join(environment.get_value('ROOT_DIR'),
'current-testcase')
shell.create_directory(testcase_directory)
testcase_path = os.path.join(testcase_directory,
os.path.basename(bot_absolute_filename))
utils.write_data_to_file(content, testcase_path)
# Unpack the test case if it's archived.
# TODO(mbarbella): Rewrite setup.unpack_testcase and share this code.
if testcase.minimized_keys and testcase.minimized_keys != 'NA':
mask = data_types.ArchiveStatus.MINIMIZED
else:
mask = data_types.ArchiveStatus.FUZZED
if testcase.archive_state & mask:
archive.unpack(testcase_path, testcase_directory)
file_list = archive.get_file_list(testcase_path)
testcase_path = None
for file_name in file_list:
if testcase.absolute_path.endswith(file_name):
testcase_path = os.path.join(testcase_directory, file_name)
break
if not testcase_path:
raise ReproduceToolException(
'Test case file was not found in archive.\n'
'Original filename: {absolute_path}.\n'
'Archive contents: {file_list}'.format(
absolute_path=testcase.absolute_path, file_list=file_list))
return testcase_path
def update_source_code():
"""Updates source code files with latest version from appengine."""
process_handler.cleanup_stale_processes()
shell.clear_temp_directory()
root_directory = environment.get_value('ROOT_DIR')
temp_directory = environment.get_value('BOT_TMPDIR')
temp_archive = os.path.join(temp_directory, 'clusterfuzz-source.zip')
try:
storage.copy_file_from(get_source_url(), temp_archive)
except Exception:
logs.log_error('Could not retrieve source code archive from url.')
return
try:
file_list = archive.get_file_list(temp_archive)
zip_archive = zipfile.ZipFile(temp_archive, 'r')
except Exception:
logs.log_error('Bad zip file.')
return
src_directory = os.path.join(root_directory, 'src')
output_directory = os.path.dirname(root_directory)
error_occurred = False
normalized_file_set = set()
for filepath in file_list:
filename = os.path.basename(filepath)
# This file cannot be updated on the fly since it is running as server.
if filename == 'adb':
continue
absolute_filepath = os.path.join(output_directory, filepath)
if os.path.altsep:
absolute_filepath = absolute_filepath.replace(os.path.altsep, os.path.sep)
if os.path.realpath(absolute_filepath) != absolute_filepath:
continue
normalized_file_set.add(absolute_filepath)
try:
file_extension = os.path.splitext(filename)[1]
# Remove any .so files first before overwriting, as they can be loaded
# in the memory of existing processes. Overwriting them directly causes
# segfaults in existing processes (e.g. run.py).
if file_extension == '.so' and os.path.exists(absolute_filepath):
os.remove(absolute_filepath)
# On Windows, to update DLLs (and native .pyd extensions), we rename it
# first so that we can install the new version.
if (environment.platform() == 'WINDOWS' and
file_extension in ['.dll', '.pyd'] and
os.path.exists(absolute_filepath)):
_rename_dll_for_update(absolute_filepath)
except Exception:
logs.log_error('Failed to remove or move %s before extracting new '
'version.' % absolute_filepath)
try:
extracted_path = zip_archive.extract(filepath, output_directory)
external_attr = zip_archive.getinfo(filepath).external_attr
mode = (external_attr >> 16) & 0o777
mode |= 0o440
os.chmod(extracted_path, mode)
except:
error_occurred = True
logs.log_error(
'Failed to extract file %s from source archive.' % filepath)
zip_archive.close()
if error_occurred:
return
clear_pyc_files(src_directory)
clear_old_files(src_directory, normalized_file_set)
local_manifest_path = os.path.join(root_directory,
utils.LOCAL_SOURCE_MANIFEST)
source_version = utils.read_data_from_file(
local_manifest_path, eval_data=False)
logs.log('Source code updated to %s.' % source_version)
def execute_task(metadata_id, job_type):
"""Unpack a bundled testcase archive and create analyze jobs for each item."""
metadata = ndb.Key(data_types.BundledArchiveMetadata, int(metadata_id)).get()
if not metadata:
logs.log_error('Invalid bundle metadata id %s.' % metadata_id)
return
bot_name = environment.get_value('BOT_NAME')
upload_metadata = data_types.TestcaseUploadMetadata.query(
data_types.TestcaseUploadMetadata.blobstore_key ==
metadata.blobstore_key).get()
if not upload_metadata:
logs.log_error('Invalid upload metadata key %s.' % metadata.blobstore_key)
return
# Update the upload metadata with this bot name.
upload_metadata.bot_name = bot_name
upload_metadata.put()
# We can't use FUZZ_INPUTS directory since it is constrained
# by tmpfs limits.
testcases_directory = environment.get_value('FUZZ_INPUTS_DISK')
# Retrieve multi-testcase archive.
archive_path = os.path.join(testcases_directory, metadata.archive_filename)
if not blobs.read_blob_to_disk(metadata.blobstore_key, archive_path):
logs.log_error('Could not retrieve archive for bundle %d.' % metadata_id)
tasks.add_task('unpack', metadata_id, job_type)
return
try:
archive.unpack(archive_path, testcases_directory)
except:
logs.log_error('Could not unpack archive for bundle %d.' % metadata_id)
tasks.add_task('unpack', metadata_id, job_type)
return
archive_state = data_types.ArchiveStatus.NONE
bundled = True
file_list = archive.get_file_list(archive_path)
for file_path in file_list:
absolute_file_path = os.path.join(testcases_directory, file_path)
filename = os.path.basename(absolute_file_path)
# Only files are actual testcases. Skip directories.
if not os.path.isfile(absolute_file_path):
continue
try:
file_handle = open(absolute_file_path, 'rb')
blob_key = blobs.write_blob(file_handle)
file_handle.close()
except:
blob_key = None
if not blob_key:
logs.log_error(
'Could not write testcase %s to blobstore.' % absolute_file_path)
continue
data_handler.create_user_uploaded_testcase(
blob_key, metadata.blobstore_key, archive_state,
metadata.archive_filename, filename, metadata.timeout,
metadata.job_type, metadata.job_queue, metadata.http_flag,
metadata.gestures, metadata.additional_arguments,
metadata.bug_information, metadata.crash_revision,
metadata.uploader_email, metadata.platform_id,
metadata.app_launch_command, metadata.fuzzer_name,
metadata.overridden_fuzzer_name, metadata.fuzzer_binary_name, bundled,
upload_metadata.retries, upload_metadata.bug_summary_update_flag)
# The upload metadata for the archive is not needed anymore since we created
# one for each testcase.
upload_metadata.key.delete()
shell.clear_testcase_directories()
