def test_setup_testcase(self):
"""Test setup_testcase."""
job_type = 'job'
self._setup_env(job_type=job_type)
fuzz_inputs = os.environ['FUZZ_INPUTS']
testcase = data_types.Testcase()
testcase.job_type = job_type
testcase.absolute_path = os.path.join(fuzz_inputs, 'testcase.ext')
with tempfile.NamedTemporaryFile() as f:
f.write(b'contents')
f.seek(0)
testcase.fuzzed_keys = blobs.write_blob(f)
testcase.put()
file_list, input_directory, testcase_file_path = (setup.setup_testcase(
testcase, job_type))
six.assertCountEqual(self, file_list, [
testcase.absolute_path,
])
self.assertEqual(input_directory, fuzz_inputs)
self.assertEqual(testcase_file_path, testcase.absolute_path)
worker_fuzz_inputs = file_host.rebase_to_worker_root(fuzz_inputs)
self.assert_dirs_equal(fuzz_inputs, worker_fuzz_inputs)
def get_upload(self):
"""Get the upload."""
uploaded_file = request.files.get('file')
if not uploaded_file:
raise helpers.EarlyExitException('File upload not found.', 400)
bytes_io = NamedBytesIO(uploaded_file.filename,
uploaded_file.stream.read())
key = blobs.write_blob(bytes_io)
return blobs.get_blob_info(key)
def test_minimize(self):
"""Test minimize."""
helpers.patch(self, ['clusterfuzz._internal.base.utils.is_oss_fuzz'])
self.mock.is_oss_fuzz.return_value = True
testcase_file_path = os.path.join(self.temp_dir, 'testcase')
with open(testcase_file_path, 'wb') as f:
f.write(b'EEE')
with open(testcase_file_path) as f:
fuzzed_keys = blobs.write_blob(f)
testcase_path = os.path.join(self.temp_dir, 'testcase')
testcase = data_types.Testcase(
crash_type='Null-dereference WRITE',
crash_address='',
crash_state='Foo\n',
crash_stacktrace='',
crash_revision=1337,
fuzzed_keys=fuzzed_keys,
fuzzer_name='libFuzzer',
overridden_fuzzer_name='libFuzzer_test_fuzzer',
job_type='libfuzzer_asan_job',
absolute_path=testcase_path,
minimized_arguments='%TESTCASE% test_fuzzer')
testcase.put()
data_types.FuzzTarget(engine='libFuzzer', binary='test_fuzzer').put()
fuzzers_init.run()
self._setup_env(job_type='libfuzzer_asan_job')
environment.set_value('APP_ARGS', testcase.minimized_arguments)
environment.set_value('LIBFUZZER_MINIMIZATION_ROUNDS', 3)
environment.set_value('UBSAN_OPTIONS',
'unneeded_option=1:silence_unsigned_overflow=1')
minimize_task.execute_task(testcase.key.id(), 'libfuzzer_asan_job')
testcase = data_handler.get_testcase_by_id(testcase.key.id())
self.assertNotEqual('', testcase.minimized_keys)
self.assertNotEqual('NA', testcase.minimized_keys)
self.assertNotEqual(testcase.fuzzed_keys, testcase.minimized_keys)
self.assertEqual(
{
'ASAN_OPTIONS': {},
'UBSAN_OPTIONS': {
'silence_unsigned_overflow': 1
}
}, testcase.get_metadata('env'))
blobs.read_blob_to_disk(testcase.minimized_keys, testcase_path)
with open(testcase_path, 'rb') as f:
self.assertEqual(1, len(f.read()))
def store_minidump(self):
"""Store the crash minidump in appengine and return key."""
if not self.minidump_info.path:
return ''
minidump_key = ''
logs.log('Storing minidump (%s) in blobstore.' %
self.minidump_info.path)
try:
minidump_key = ''
with open(self.minidump_info.path, 'rb') as file_handle:
minidump_key = blobs.write_blob(file_handle)
except:
logs.log_error('Failed to store minidump.')
if minidump_key:
self.minidump_info = FileMetadataInfo(path=self.minidump_info.path,
key=minidump_key)
return minidump_key
def archive_testcase_and_dependencies_in_gcs(resource_list, testcase_path):
"""Archive testcase and its dependencies, and store in blobstore."""
if not os.path.exists(testcase_path):
logs.log_error('Unable to find testcase %s.' % testcase_path)
return None, None, None, None
absolute_filename = testcase_path
archived = False
zip_filename = None
zip_path = None
if not resource_list:
resource_list = []
# Add resource dependencies based on testcase path. These include
# stuff like extensions directory, dependency files, etc.
resource_list.extend(
testcase_manager.get_resource_dependencies(testcase_path))
# Filter out duplicates, directories, and files that do not exist.
resource_list = utils.filter_file_list(resource_list)
logs.log('Testcase and related files :\n%s' % str(resource_list))
if len(resource_list) <= 1:
# If this does not have any resources, just save the testcase.
# TODO(flowerhack): Update this when we teach CF how to download testcases.
try:
file_handle = open(testcase_path, 'rb')
except IOError:
logs.log_error('Unable to open testcase %s.' % testcase_path)
return None, None, None, None
else:
# If there are resources, create an archive.
# Find the common root directory for all of the resources.
# Assumption: resource_list[0] is the testcase path.
base_directory_list = resource_list[0].split(os.path.sep)
for list_index in range(1, len(resource_list)):
current_directory_list = resource_list[list_index].split(
os.path.sep)
length = min(len(base_directory_list), len(current_directory_list))
for directory_index in range(length):
if (current_directory_list[directory_index] !=
base_directory_list[directory_index]):
base_directory_list = base_directory_list[
0:directory_index]
break
base_directory = os.path.sep.join(base_directory_list)
logs.log('Subresource common base directory: %s' % base_directory)
if base_directory:
# Common parent directory, archive sub-paths only.
base_len = len(base_directory) + len(os.path.sep)
else:
# No common parent directory, archive all paths as it-is.
base_len = 0
# Prepare the filename for the archive.
zip_filename, _ = os.path.splitext(os.path.basename(testcase_path))
zip_filename += _TESTCASE_ARCHIVE_EXTENSION
# Create the archive.
zip_path = os.path.join(environment.get_value('INPUT_DIR'),
zip_filename)
zip_file = zipfile.ZipFile(zip_path, 'w')
for file_name in resource_list:
if os.path.exists(file_name):
relative_filename = file_name[base_len:]
zip_file.write(file_name, relative_filename,
zipfile.ZIP_DEFLATED)
zip_file.close()
try:
file_handle = open(zip_path, 'rb')
except IOError:
logs.log_error('Unable to open testcase archive %s.' % zip_path)
return None, None, None, None
archived = True
absolute_filename = testcase_path[base_len:]
fuzzed_key = blobs.write_blob(file_handle)
file_handle.close()
# Don't need the archive after writing testcase to blobstore.
if zip_path:
shell.remove_file(zip_path)
return fuzzed_key, archived, absolute_filename, zip_filename
def execute_task(metadata_id, job_type):
"""Unpack a bundled testcase archive and create analyze jobs for each item."""
metadata = ndb.Key(data_types.BundledArchiveMetadata,
int(metadata_id)).get()
if not metadata:
logs.log_error('Invalid bundle metadata id %s.' % metadata_id)
return
bot_name = environment.get_value('BOT_NAME')
upload_metadata = data_types.TestcaseUploadMetadata.query(
data_types.TestcaseUploadMetadata.blobstore_key ==
metadata.blobstore_key).get()
if not upload_metadata:
logs.log_error('Invalid upload metadata key %s.' %
metadata.blobstore_key)
return
job = data_types.Job.query(data_types.Job.name == metadata.job_type).get()
if not job:
logs.log_error('Invalid job_type %s.' % metadata.job_type)
return
# Update the upload metadata with this bot name.
upload_metadata.bot_name = bot_name
upload_metadata.put()
# We can't use FUZZ_INPUTS directory since it is constrained
# by tmpfs limits.
testcases_directory = environment.get_value('FUZZ_INPUTS_DISK')
# Retrieve multi-testcase archive.
archive_path = os.path.join(testcases_directory, metadata.archive_filename)
if not blobs.read_blob_to_disk(metadata.blobstore_key, archive_path):
logs.log_error('Could not retrieve archive for bundle %d.' %
metadata_id)
tasks.add_task('unpack', metadata_id, job_type)
return
try:
archive.unpack(archive_path, testcases_directory)
except:
logs.log_error('Could not unpack archive for bundle %d.' % metadata_id)
tasks.add_task('unpack', metadata_id, job_type)
return
# Get additional testcase metadata (if any).
additional_metadata = None
if upload_metadata.additional_metadata_string:
additional_metadata = json.loads(
upload_metadata.additional_metadata_string)
archive_state = data_types.ArchiveStatus.NONE
bundled = True
file_list = archive.get_file_list(archive_path)
for file_path in file_list:
absolute_file_path = os.path.join(testcases_directory, file_path)
filename = os.path.basename(absolute_file_path)
# Only files are actual testcases. Skip directories.
if not os.path.isfile(absolute_file_path):
continue
try:
file_handle = open(absolute_file_path, 'rb')
blob_key = blobs.write_blob(file_handle)
file_handle.close()
except:
blob_key = None
if not blob_key:
logs.log_error('Could not write testcase %s to blobstore.' %
absolute_file_path)
continue
data_handler.create_user_uploaded_testcase(
blob_key, metadata.blobstore_key, archive_state,
metadata.archive_filename, filename, metadata.timeout, job,
metadata.job_queue, metadata.http_flag, metadata.gestures,
metadata.additional_arguments, metadata.bug_information,
metadata.crash_revision, metadata.uploader_email,
metadata.platform_id, metadata.app_launch_command,
metadata.fuzzer_name, metadata.overridden_fuzzer_name,
metadata.fuzzer_binary_name, bundled, upload_metadata.retries,
upload_metadata.bug_summary_update_flag,
upload_metadata.quiet_flag, additional_metadata)
# The upload metadata for the archive is not needed anymore since we created
# one for each testcase.
upload_metadata.key.delete()
shell.clear_testcase_directories()
def _process_corpus_crashes(context, result):
"""Process crashes found in the corpus."""
# Default Testcase entity values.
crash_revision = result.revision
job_type = environment.get_value('JOB_NAME')
minimized_arguments = '%TESTCASE% ' + context.fuzz_target.binary
project_name = data_handler.get_project_name(job_type)
comment = 'Fuzzer %s generated corpus testcase crashed (r%s)' % (
context.fuzz_target.project_qualified_name(), crash_revision)
# Generate crash reports.
for crash in result.crashes:
existing_testcase = data_handler.find_testcase(project_name,
crash.crash_type,
crash.crash_state,
crash.security_flag)
if existing_testcase:
continue
# Upload/store testcase.
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
unit_path = os.path.join(context.bad_units_path,
os.path.basename(crash.unit_path))
# Prevent the worker from escaping out of |context.bad_units_path|.
if not file_host.is_directory_parent(unit_path,
context.bad_units_path):
raise CorpusPruningException('Invalid units path from worker.')
file_host.copy_file_from_worker(crash.unit_path, unit_path)
else:
unit_path = crash.unit_path
with open(unit_path, 'rb') as f:
key = blobs.write_blob(f)
# Set the absolute_path property of the Testcase to a file in FUZZ_INPUTS
# instead of the local quarantine directory.
absolute_testcase_path = os.path.join(
environment.get_value('FUZZ_INPUTS'), 'testcase')
testcase_id = data_handler.store_testcase(
crash=crash,
fuzzed_keys=key,
minimized_keys='',
regression='',
fixed='',
one_time_crasher_flag=False,
crash_revision=crash_revision,
comment=comment,
absolute_path=absolute_testcase_path,
fuzzer_name=context.fuzz_target.engine,
fully_qualified_fuzzer_name=context.fuzz_target.
fully_qualified_name(),
job_type=job_type,
archived=False,
archive_filename='',
binary_flag=True,
http_flag=False,
gestures=None,
redzone=DEFAULT_REDZONE,
disable_ubsan=False,
minidump_keys=None,
window_argument=None,
timeout_multiplier=1.0,
minimized_arguments=minimized_arguments)
# Set fuzzer_binary_name in testcase metadata.
testcase = data_handler.get_testcase_by_id(testcase_id)
testcase.set_metadata('fuzzer_binary_name', result.fuzzer_binary_name)
issue_metadata = engine_common.get_all_issue_metadata_for_testcase(
testcase)
if issue_metadata:
for key, value in issue_metadata.items():
testcase.set_metadata(key, value, update_testcase=False)
testcase.put()
# Create additional tasks for testcase (starting with minimization).
testcase = data_handler.get_testcase_by_id(testcase_id)
task_creation.create_tasks(testcase)