def __init__(self,
fuzz_target,
cross_pollinate_fuzzers,
cross_pollination_method=Pollination.RANDOM,
tag=None):
self.fuzz_target = fuzz_target
self.cross_pollinate_fuzzers = cross_pollinate_fuzzers
self.cross_pollination_method = cross_pollination_method
self.tag = tag
self.merge_tmp_dir = None
self.engine = engine.get(self.fuzz_target.engine)
if not self.engine:
raise CorpusPruningException('Engine {} not found'.format(engine))
self._created_directories = []
# Set up temporary directories where corpora will be synced to.
# Initial synced corpus.
self.initial_corpus_path = self._create_temp_corpus_directory(
'%s_initial_corpus' % self.fuzz_target.project_qualified_name())
# Minimized corpus.
self.minimized_corpus_path = self._create_temp_corpus_directory(
'%s_minimized_corpus' % self.fuzz_target.project_qualified_name())
# Synced quarantine corpus.
self.quarantine_corpus_path = self._create_temp_corpus_directory(
'%s_quarantine' % self.fuzz_target.project_qualified_name())
# Synced shared corpus.
self.shared_corpus_path = self._create_temp_corpus_directory(
'%s_shared' % self.fuzz_target.project_qualified_name())
# Bad units.
self.bad_units_path = self._create_temp_corpus_directory(
'%s_bad_units' % self.fuzz_target.project_qualified_name())
self.merge_tmp_dir = self._create_temp_corpus_directory(
'merge_workdir')
self.corpus = corpus_manager.FuzzTargetCorpus(
self.fuzz_target.engine,
self.fuzz_target.project_qualified_name(),
include_regressions=True)
self.quarantine_corpus = corpus_manager.FuzzTargetCorpus(
self.fuzz_target.engine,
self.fuzz_target.project_qualified_name(),
quarantine=True)
shared_corpus_bucket = environment.get_value('SHARED_CORPUS_BUCKET')
self.shared_corpus = corpus_manager.GcsCorpus(shared_corpus_bucket)
def test_rsync_to_disk(self):
"""Test rsync_to_disk."""
corpus = corpus_manager.FuzzTargetCorpus('libFuzzer', 'fuzzer')
self.assertTrue(corpus.rsync_to_disk('/dir', timeout=60))
commands = [
call_arg[0][0] for call_arg in self.mock.Popen.call_args_list
]
self.assertEqual(commands, [
[
'/gsutil_path/gsutil',
'-m',
'-q',
'rsync',
'-r',
'-d',
'gs://bucket/libFuzzer/fuzzer/',
'/dir',
],
[
'/gsutil_path/gsutil',
'-m',
'-q',
'rsync',
'-r',
'gs://bucket/libFuzzer/fuzzer_static/',
'/dir',
],
])
def _store_testcase_for_regression_testing(testcase, testcase_file_path):
"""Stores reproduction testcase for future regression testing in corpus
pruning task."""
if testcase.open:
# Store testcase only after the crash is fixed.
return
if not testcase.bug_information:
# Only store crashes with bugs associated with them.
return
fuzz_target = data_handler.get_fuzz_target(testcase.overridden_fuzzer_name)
if not fuzz_target:
# No work to do, only applicable for engine fuzzers.
return
corpus = corpus_manager.FuzzTargetCorpus(
fuzz_target.engine, fuzz_target.project_qualified_name())
regression_testcase_url = os.path.join(
corpus.get_regressions_corpus_gcs_url(),
utils.file_hash(testcase_file_path))
if storage.copy_file_to(testcase_file_path, regression_testcase_url):
logs.log('Successfully stored testcase for regression testing: ' +
regression_testcase_url)
else:
logs.log_error('Failed to store testcase for regression testing: ' +
regression_testcase_url)
def test_rsync_to_disk_with_regressions(self):
"""Test rsync_to_disk, with regressions set."""
corpus = corpus_manager.FuzzTargetCorpus('libFuzzer',
'fuzzer',
include_regressions=True)
self.assertTrue(corpus.rsync_to_disk('/dir', timeout=60))
commands = [
call_arg[0][0] for call_arg in self.mock.Popen.call_args_list
]
self.assertEqual(commands, [
[
'/gsutil_path/gsutil',
'-m',
'-q',
'rsync',
'-r',
'-d',
'gs://bucket/libFuzzer/fuzzer/',
'/dir',
],
[
'/gsutil_path/gsutil',
'-m',
'-q',
'rsync',
'-r',
'gs://bucket/libFuzzer/fuzzer_regressions/',
'/dir/regressions',
],
])
def test_rsync_from_disk(self):
"""Test rsync_from_disk."""
corpus = corpus_manager.FuzzTargetCorpus('libFuzzer', 'fuzzer')
self.assertTrue(corpus.rsync_from_disk('/dir'))
self.assertEqual(self.mock.Popen.call_args[0][0], [
'/gsutil_path/gsutil', '-m', '-q', 'rsync', '-r', '-d', '/dir',
'gs://bucket/libFuzzer/fuzzer/'
])
def test_upload_files(self):
"""Test upload_files."""
mock_popen = self.mock.Popen.return_value
corpus = corpus_manager.FuzzTargetCorpus('libFuzzer', 'fuzzer')
self.assertTrue(corpus.upload_files(['/dir/a', '/dir/b']))
mock_popen.communicate.assert_called_with('/dir/a\n/dir/b')
self.assertEqual(self.mock.Popen.call_args[0][0], [
'/gsutil_path/gsutil', '-m', 'cp', '-I', 'gs://bucket/libFuzzer/fuzzer/'
])
def __init__(self, fuzz_target, cross_pollinate_fuzzers):
self.fuzz_target = fuzz_target
self.cross_pollinate_fuzzers = cross_pollinate_fuzzers
self.merge_tmp_dir = None
self.engine = engine.get(self.fuzz_target.engine)
if not self.engine:
raise CorpusPruningException("Engine {} not found".format(engine))
self._created_directories = []
# Set up temporary directories where corpora will be synced to.
# Initial synced corpus.
self.initial_corpus_path = self._create_temp_corpus_directory(
"%s_initial_corpus" % self.fuzz_target.project_qualified_name())
# Minimized corpus.
self.minimized_corpus_path = self._create_temp_corpus_directory(
"%s_minimized_corpus" % self.fuzz_target.project_qualified_name())
# Synced quarantine corpus.
self.quarantine_corpus_path = self._create_temp_corpus_directory(
"%s_quarantine" % self.fuzz_target.project_qualified_name())
# Synced shared corpus.
self.shared_corpus_path = self._create_temp_corpus_directory(
"%s_shared" % self.fuzz_target.project_qualified_name())
# Bad units.
self.bad_units_path = self._create_temp_corpus_directory(
"%s_bad_units" % self.fuzz_target.project_qualified_name())
self.merge_tmp_dir = self._create_temp_corpus_directory(
"merge_workdir")
self.corpus = corpus_manager.FuzzTargetCorpus(
self.fuzz_target.engine, self.fuzz_target.project_qualified_name())
self.quarantine_corpus = corpus_manager.FuzzTargetCorpus(
self.fuzz_target.engine,
self.fuzz_target.project_qualified_name(),
quarantine=True,
)
shared_corpus_bucket = environment.get_value("SHARED_CORPUS_BUCKET")
self.shared_corpus = corpus_manager.GcsCorpus(shared_corpus_bucket)
def __init__(self, fuzz_target, cross_pollinate_fuzzers, use_minijail):
self.fuzz_target = fuzz_target
self.cross_pollinate_fuzzers = cross_pollinate_fuzzers
self.use_minijail = use_minijail
self.merge_tmp_dir = None
self._created_directories = []
# Set up temporary directories where corpora will be synced to.
# Initial synced corpus.
self.initial_corpus_path = self._create_temp_corpus_directory(
'%s_initial_corpus' % self.fuzz_target.project_qualified_name())
# Minimized corpus.
self.minimized_corpus_path = self._create_temp_corpus_directory(
'%s_minimized_corpus' % self.fuzz_target.project_qualified_name())
# Synced quarantine corpus.
self.quarantine_corpus_path = self._create_temp_corpus_directory(
'%s_quarantine' % self.fuzz_target.project_qualified_name())
# Synced shared corpus.
self.shared_corpus_path = self._create_temp_corpus_directory(
'%s_shared' % self.fuzz_target.project_qualified_name())
# Bad units.
self.bad_units_path = self._create_temp_corpus_directory(
'%s_bad_units' % self.fuzz_target.project_qualified_name())
if not self.use_minijail:
self.merge_tmp_dir = self._create_temp_corpus_directory(
'merge_workdir')
self.corpus = corpus_manager.FuzzTargetCorpus(
self.fuzz_target.engine, self.fuzz_target.project_qualified_name())
self.quarantine_corpus = corpus_manager.FuzzTargetCorpus(
self.fuzz_target.engine,
self.fuzz_target.project_qualified_name(),
quarantine=True)
shared_corpus_bucket = environment.get_value('SHARED_CORPUS_BUCKET')
self.shared_corpus = corpus_manager.GcsCorpus(shared_corpus_bucket)
def test_backup_corpus(self):
"""Test backup_corpus."""
libfuzzer_corpus = corpus_manager.FuzzTargetCorpus('libFuzzer', 'fuzzer')
corpus_manager.backup_corpus('backup_bucket', libfuzzer_corpus, '/dir')
self.mock.copy_file_to.assert_has_calls([
mock.call('/2017-01-01.zip',
'gs://backup_bucket/corpus/libFuzzer/fuzzer/2017-01-01.zip')
])
self.mock.copy_blob.assert_has_calls([
mock.call('gs://backup_bucket/corpus/libFuzzer/fuzzer/2017-01-01.zip',
'gs://backup_bucket/corpus/libFuzzer/fuzzer/latest.zip'),
])
def setUp(self):
"""Set up."""
super(CorpusPruningTestUntrusted, self).setUp()
environment.set_value('JOB_NAME', 'libfuzzer_asan_job')
helpers.patch(self, [
'bot.fuzzers.engine.get', 'bot.tasks.setup.get_fuzzer_directory',
'base.tasks.add_task',
'bot.tasks.corpus_pruning_task._record_cross_pollination_stats'
])
self.mock.get.return_value = libFuzzer_engine.LibFuzzerEngine()
self.mock.get_fuzzer_directory.return_value = os.path.join(
environment.get_value('ROOT_DIR'), 'src', 'python', 'bot',
'fuzzers', 'libFuzzer')
self.corpus_bucket = os.environ['CORPUS_BUCKET']
self.quarantine_bucket = os.environ['QUARANTINE_BUCKET']
self.backup_bucket = os.environ['BACKUP_BUCKET']
job = data_types.Job(
name='libfuzzer_asan_job',
environment_string=(
'APP_NAME = test_fuzzer\n'
'CORPUS_BUCKET = {corpus_bucket}\n'
'QUARANTINE_BUCKET = {quarantine_bucket}\n'
'BACKUP_BUCKET={backup_bucket}\n'
'RELEASE_BUILD_BUCKET_PATH = '
'gs://clusterfuzz-test-data/test_libfuzzer_builds/'
'test-libfuzzer-build-([0-9]+).zip\n'
'REVISION_VARS_URL = gs://clusterfuzz-test-data/'
'test_libfuzzer_builds/'
'test-libfuzzer-build-%s.srcmap.json\n'.format(
corpus_bucket=self.corpus_bucket,
quarantine_bucket=self.quarantine_bucket,
backup_bucket=self.backup_bucket)))
job.put()
job = data_types.Job(
name='libfuzzer_asan_job2',
environment_string=(
'APP_NAME = test2_fuzzer\n'
'BACKUP_BUCKET = {backup_bucket}\n'
'CORPUS_FUZZER_NAME_OVERRIDE = libfuzzer\n'.format(
backup_bucket=self.backup_bucket)))
job.put()
os.environ['PROJECT_NAME'] = 'oss-fuzz'
data_types.FuzzTarget(engine='libFuzzer',
project='test',
binary='test_fuzzer').put()
data_types.FuzzTargetJob(fuzz_target_name='libFuzzer_test_fuzzer',
engine='libFuzzer',
job='libfuzzer_asan_job',
last_run=datetime.datetime.now()).put()
data_types.FuzzTarget(engine='libFuzzer',
project='test2',
binary='fuzzer').put()
data_types.FuzzTargetJob(fuzz_target_name='libFuzzer_test2_fuzzer',
engine='libFuzzer',
job='libfuzzer_asan_job2',
last_run=datetime.datetime.now()).put()
environment.set_value('USE_MINIJAIL', True)
environment.set_value('SHARED_CORPUS_BUCKET', TEST_SHARED_BUCKET)
# Set up remote corpora.
self.corpus = corpus_manager.FuzzTargetCorpus('libFuzzer',
'test_fuzzer')
self.corpus.rsync_from_disk(os.path.join(TEST_DIR, 'corpus'),
delete=True)
self.quarantine_corpus = corpus_manager.FuzzTargetCorpus(
'libFuzzer', 'test_fuzzer', quarantine=True)
self.quarantine_corpus.rsync_from_disk(os.path.join(
TEST_DIR, 'quarantine'),
delete=True)
self.mock.get_data_bundle_bucket_name.return_value = TEST_GLOBAL_BUCKET
data_types.DataBundle(name='bundle',
is_local=True,
sync_to_worker=True).put()
data_types.Fuzzer(revision=1,
file_size='builtin',
source='builtin',
name='libFuzzer',
max_testcases=4,
builtin=True,
data_bundle_name='bundle').put()
self.temp_dir = tempfile.mkdtemp()
# Copy corpus backup in the older date format.
corpus_backup_date = (
datetime.datetime.utcnow().date() - datetime.timedelta(
days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS))
corpus_backup_dir = ('gs://{bucket}/corpus/libfuzzer/test2_fuzzer/')
gsutil.GSUtilRunner().run_gsutil([
'cp', (corpus_backup_dir +
'backup.zip').format(bucket=TEST2_BACKUP_BUCKET),
(corpus_backup_dir +
'%s.zip' % corpus_backup_date).format(bucket=self.backup_bucket)
])
def setUp(self):
"""Set up."""
super(CorpusPruningTestUntrusted, self).setUp()
environment.set_value("JOB_NAME", "libfuzzer_asan_job")
helpers.patch(
self,
[
"bot.fuzzers.engine.get",
"bot.fuzzers.libFuzzer.fuzzer.LibFuzzer.fuzzer_directory",
"base.tasks.add_task",
"datastore.data_handler.get_data_bundle_bucket_name",
],
)
self.mock.get.return_value = libFuzzer_engine.LibFuzzerEngine()
self.mock.fuzzer_directory.return_value = os.path.join(
environment.get_value("ROOT_DIR"),
"src",
"python",
"bot",
"fuzzers",
"libFuzzer",
)
self.corpus_bucket = os.environ["CORPUS_BUCKET"]
self.quarantine_bucket = os.environ["QUARANTINE_BUCKET"]
self.backup_bucket = os.environ["BACKUP_BUCKET"]
job = data_types.Job(
name="libfuzzer_asan_job",
environment_string=("APP_NAME = test_fuzzer\n"
"CORPUS_BUCKET = {corpus_bucket}\n"
"QUARANTINE_BUCKET = {quarantine_bucket}\n"
"BACKUP_BUCKET={backup_bucket}\n"
"RELEASE_BUILD_BUCKET_PATH = "
"gs://clusterfuzz-test-data/test_libfuzzer_builds/"
"test-libfuzzer-build-([0-9]+).zip\n"
"REVISION_VARS_URL = gs://clusterfuzz-test-data/"
"test_libfuzzer_builds/"
"test-libfuzzer-build-%s.srcmap.json\n".format(
corpus_bucket=self.corpus_bucket,
quarantine_bucket=self.quarantine_bucket,
backup_bucket=self.backup_bucket,
)),
)
job.put()
job = data_types.Job(
name="libfuzzer_asan_job2",
environment_string=("APP_NAME = test2_fuzzer\n"
"BACKUP_BUCKET = {backup_bucket}\n"
"CORPUS_FUZZER_NAME_OVERRIDE = libfuzzer\n".format(
backup_bucket=self.backup_bucket)),
)
job.put()
os.environ["PROJECT_NAME"] = "oss-fuzz"
data_types.FuzzTarget(
engine="libFuzzer", project="test", binary="test_fuzzer").put()
data_types.FuzzTargetJob(
fuzz_target_name="libFuzzer_test_fuzzer",
engine="libFuzzer",
job="libfuzzer_asan_job",
last_run=datetime.datetime.now(),
).put()
data_types.FuzzTarget(
engine="libFuzzer", project="test2", binary="fuzzer").put()
data_types.FuzzTargetJob(
fuzz_target_name="libFuzzer_test2_fuzzer",
engine="libFuzzer",
job="libfuzzer_asan_job2",
last_run=datetime.datetime.now(),
).put()
environment.set_value("USE_MINIJAIL", True)
environment.set_value("SHARED_CORPUS_BUCKET", TEST_SHARED_BUCKET)
# Set up remote corpora.
self.corpus = corpus_manager.FuzzTargetCorpus("libFuzzer", "test_fuzzer")
self.corpus.rsync_from_disk(os.path.join(TEST_DIR, "corpus"), delete=True)
self.quarantine_corpus = corpus_manager.FuzzTargetCorpus(
"libFuzzer", "test_fuzzer", quarantine=True)
self.quarantine_corpus.rsync_from_disk(
os.path.join(TEST_DIR, "quarantine"), delete=True)
self.mock.get_data_bundle_bucket_name.return_value = TEST_GLOBAL_BUCKET
data_types.DataBundle(
name="bundle", is_local=True, sync_to_worker=True).put()
data_types.Fuzzer(
revision=1,
file_size="builtin",
source="builtin",
name="libFuzzer",
max_testcases=4,
builtin=True,
data_bundle_name="bundle",
).put()
self.temp_dir = tempfile.mkdtemp()
# Copy corpus backup in the older date format.
corpus_backup_date = datetime.datetime.utcnow().date() - datetime.timedelta(
days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS)
corpus_backup_dir = "gs://{bucket}/corpus/libfuzzer/test2_fuzzer/"
gsutil.GSUtilRunner().run_gsutil([
"cp",
(corpus_backup_dir + "backup.zip").format(bucket=TEST2_BACKUP_BUCKET),
(corpus_backup_dir +
"%s.zip" % corpus_backup_date).format(bucket=self.backup_bucket),
])
