def __init__(self, fuzz_target, cross_pollinate_fuzzers, cross_pollination_method=Pollination.RANDOM, tag=None): self.fuzz_target = fuzz_target self.cross_pollinate_fuzzers = cross_pollinate_fuzzers self.cross_pollination_method = cross_pollination_method self.tag = tag self.merge_tmp_dir = None self.engine = engine.get(self.fuzz_target.engine) if not self.engine: raise CorpusPruningException('Engine {} not found'.format(engine)) self._created_directories = [] # Set up temporary directories where corpora will be synced to. # Initial synced corpus. self.initial_corpus_path = self._create_temp_corpus_directory( '%s_initial_corpus' % self.fuzz_target.project_qualified_name()) # Minimized corpus. self.minimized_corpus_path = self._create_temp_corpus_directory( '%s_minimized_corpus' % self.fuzz_target.project_qualified_name()) # Synced quarantine corpus. self.quarantine_corpus_path = self._create_temp_corpus_directory( '%s_quarantine' % self.fuzz_target.project_qualified_name()) # Synced shared corpus. self.shared_corpus_path = self._create_temp_corpus_directory( '%s_shared' % self.fuzz_target.project_qualified_name()) # Bad units. self.bad_units_path = self._create_temp_corpus_directory( '%s_bad_units' % self.fuzz_target.project_qualified_name()) self.merge_tmp_dir = self._create_temp_corpus_directory( 'merge_workdir') self.corpus = corpus_manager.FuzzTargetCorpus( self.fuzz_target.engine, self.fuzz_target.project_qualified_name(), include_regressions=True) self.quarantine_corpus = corpus_manager.FuzzTargetCorpus( self.fuzz_target.engine, self.fuzz_target.project_qualified_name(), quarantine=True) shared_corpus_bucket = environment.get_value('SHARED_CORPUS_BUCKET') self.shared_corpus = corpus_manager.GcsCorpus(shared_corpus_bucket)
def test_rsync_to_disk(self): """Test rsync_to_disk.""" corpus = corpus_manager.FuzzTargetCorpus('libFuzzer', 'fuzzer') self.assertTrue(corpus.rsync_to_disk('/dir', timeout=60)) commands = [ call_arg[0][0] for call_arg in self.mock.Popen.call_args_list ] self.assertEqual(commands, [ [ '/gsutil_path/gsutil', '-m', '-q', 'rsync', '-r', '-d', 'gs://bucket/libFuzzer/fuzzer/', '/dir', ], [ '/gsutil_path/gsutil', '-m', '-q', 'rsync', '-r', 'gs://bucket/libFuzzer/fuzzer_static/', '/dir', ], ])
def _store_testcase_for_regression_testing(testcase, testcase_file_path): """Stores reproduction testcase for future regression testing in corpus pruning task.""" if testcase.open: # Store testcase only after the crash is fixed. return if not testcase.bug_information: # Only store crashes with bugs associated with them. return fuzz_target = data_handler.get_fuzz_target(testcase.overridden_fuzzer_name) if not fuzz_target: # No work to do, only applicable for engine fuzzers. return corpus = corpus_manager.FuzzTargetCorpus( fuzz_target.engine, fuzz_target.project_qualified_name()) regression_testcase_url = os.path.join( corpus.get_regressions_corpus_gcs_url(), utils.file_hash(testcase_file_path)) if storage.copy_file_to(testcase_file_path, regression_testcase_url): logs.log('Successfully stored testcase for regression testing: ' + regression_testcase_url) else: logs.log_error('Failed to store testcase for regression testing: ' + regression_testcase_url)
def test_rsync_to_disk_with_regressions(self): """Test rsync_to_disk, with regressions set.""" corpus = corpus_manager.FuzzTargetCorpus('libFuzzer', 'fuzzer', include_regressions=True) self.assertTrue(corpus.rsync_to_disk('/dir', timeout=60)) commands = [ call_arg[0][0] for call_arg in self.mock.Popen.call_args_list ] self.assertEqual(commands, [ [ '/gsutil_path/gsutil', '-m', '-q', 'rsync', '-r', '-d', 'gs://bucket/libFuzzer/fuzzer/', '/dir', ], [ '/gsutil_path/gsutil', '-m', '-q', 'rsync', '-r', 'gs://bucket/libFuzzer/fuzzer_regressions/', '/dir/regressions', ], ])
def test_rsync_from_disk(self): """Test rsync_from_disk.""" corpus = corpus_manager.FuzzTargetCorpus('libFuzzer', 'fuzzer') self.assertTrue(corpus.rsync_from_disk('/dir')) self.assertEqual(self.mock.Popen.call_args[0][0], [ '/gsutil_path/gsutil', '-m', '-q', 'rsync', '-r', '-d', '/dir', 'gs://bucket/libFuzzer/fuzzer/' ])
def test_upload_files(self): """Test upload_files.""" mock_popen = self.mock.Popen.return_value corpus = corpus_manager.FuzzTargetCorpus('libFuzzer', 'fuzzer') self.assertTrue(corpus.upload_files(['/dir/a', '/dir/b'])) mock_popen.communicate.assert_called_with('/dir/a\n/dir/b') self.assertEqual(self.mock.Popen.call_args[0][0], [ '/gsutil_path/gsutil', '-m', 'cp', '-I', 'gs://bucket/libFuzzer/fuzzer/' ])
def __init__(self, fuzz_target, cross_pollinate_fuzzers): self.fuzz_target = fuzz_target self.cross_pollinate_fuzzers = cross_pollinate_fuzzers self.merge_tmp_dir = None self.engine = engine.get(self.fuzz_target.engine) if not self.engine: raise CorpusPruningException("Engine {} not found".format(engine)) self._created_directories = [] # Set up temporary directories where corpora will be synced to. # Initial synced corpus. self.initial_corpus_path = self._create_temp_corpus_directory( "%s_initial_corpus" % self.fuzz_target.project_qualified_name()) # Minimized corpus. self.minimized_corpus_path = self._create_temp_corpus_directory( "%s_minimized_corpus" % self.fuzz_target.project_qualified_name()) # Synced quarantine corpus. self.quarantine_corpus_path = self._create_temp_corpus_directory( "%s_quarantine" % self.fuzz_target.project_qualified_name()) # Synced shared corpus. self.shared_corpus_path = self._create_temp_corpus_directory( "%s_shared" % self.fuzz_target.project_qualified_name()) # Bad units. self.bad_units_path = self._create_temp_corpus_directory( "%s_bad_units" % self.fuzz_target.project_qualified_name()) self.merge_tmp_dir = self._create_temp_corpus_directory( "merge_workdir") self.corpus = corpus_manager.FuzzTargetCorpus( self.fuzz_target.engine, self.fuzz_target.project_qualified_name()) self.quarantine_corpus = corpus_manager.FuzzTargetCorpus( self.fuzz_target.engine, self.fuzz_target.project_qualified_name(), quarantine=True, ) shared_corpus_bucket = environment.get_value("SHARED_CORPUS_BUCKET") self.shared_corpus = corpus_manager.GcsCorpus(shared_corpus_bucket)
def __init__(self, fuzz_target, cross_pollinate_fuzzers, use_minijail): self.fuzz_target = fuzz_target self.cross_pollinate_fuzzers = cross_pollinate_fuzzers self.use_minijail = use_minijail self.merge_tmp_dir = None self._created_directories = [] # Set up temporary directories where corpora will be synced to. # Initial synced corpus. self.initial_corpus_path = self._create_temp_corpus_directory( '%s_initial_corpus' % self.fuzz_target.project_qualified_name()) # Minimized corpus. self.minimized_corpus_path = self._create_temp_corpus_directory( '%s_minimized_corpus' % self.fuzz_target.project_qualified_name()) # Synced quarantine corpus. self.quarantine_corpus_path = self._create_temp_corpus_directory( '%s_quarantine' % self.fuzz_target.project_qualified_name()) # Synced shared corpus. self.shared_corpus_path = self._create_temp_corpus_directory( '%s_shared' % self.fuzz_target.project_qualified_name()) # Bad units. self.bad_units_path = self._create_temp_corpus_directory( '%s_bad_units' % self.fuzz_target.project_qualified_name()) if not self.use_minijail: self.merge_tmp_dir = self._create_temp_corpus_directory( 'merge_workdir') self.corpus = corpus_manager.FuzzTargetCorpus( self.fuzz_target.engine, self.fuzz_target.project_qualified_name()) self.quarantine_corpus = corpus_manager.FuzzTargetCorpus( self.fuzz_target.engine, self.fuzz_target.project_qualified_name(), quarantine=True) shared_corpus_bucket = environment.get_value('SHARED_CORPUS_BUCKET') self.shared_corpus = corpus_manager.GcsCorpus(shared_corpus_bucket)
def test_backup_corpus(self): """Test backup_corpus.""" libfuzzer_corpus = corpus_manager.FuzzTargetCorpus('libFuzzer', 'fuzzer') corpus_manager.backup_corpus('backup_bucket', libfuzzer_corpus, '/dir') self.mock.copy_file_to.assert_has_calls([ mock.call('/2017-01-01.zip', 'gs://backup_bucket/corpus/libFuzzer/fuzzer/2017-01-01.zip') ]) self.mock.copy_blob.assert_has_calls([ mock.call('gs://backup_bucket/corpus/libFuzzer/fuzzer/2017-01-01.zip', 'gs://backup_bucket/corpus/libFuzzer/fuzzer/latest.zip'), ])
def setUp(self): """Set up.""" super(CorpusPruningTestUntrusted, self).setUp() environment.set_value('JOB_NAME', 'libfuzzer_asan_job') helpers.patch(self, [ 'bot.fuzzers.engine.get', 'bot.tasks.setup.get_fuzzer_directory', 'base.tasks.add_task', 'bot.tasks.corpus_pruning_task._record_cross_pollination_stats' ]) self.mock.get.return_value = libFuzzer_engine.LibFuzzerEngine() self.mock.get_fuzzer_directory.return_value = os.path.join( environment.get_value('ROOT_DIR'), 'src', 'python', 'bot', 'fuzzers', 'libFuzzer') self.corpus_bucket = os.environ['CORPUS_BUCKET'] self.quarantine_bucket = os.environ['QUARANTINE_BUCKET'] self.backup_bucket = os.environ['BACKUP_BUCKET'] job = data_types.Job( name='libfuzzer_asan_job', environment_string=( 'APP_NAME = test_fuzzer\n' 'CORPUS_BUCKET = {corpus_bucket}\n' 'QUARANTINE_BUCKET = {quarantine_bucket}\n' 'BACKUP_BUCKET={backup_bucket}\n' 'RELEASE_BUILD_BUCKET_PATH = ' 'gs://clusterfuzz-test-data/test_libfuzzer_builds/' 'test-libfuzzer-build-([0-9]+).zip\n' 'REVISION_VARS_URL = gs://clusterfuzz-test-data/' 'test_libfuzzer_builds/' 'test-libfuzzer-build-%s.srcmap.json\n'.format( corpus_bucket=self.corpus_bucket, quarantine_bucket=self.quarantine_bucket, backup_bucket=self.backup_bucket))) job.put() job = data_types.Job( name='libfuzzer_asan_job2', environment_string=( 'APP_NAME = test2_fuzzer\n' 'BACKUP_BUCKET = {backup_bucket}\n' 'CORPUS_FUZZER_NAME_OVERRIDE = libfuzzer\n'.format( backup_bucket=self.backup_bucket))) job.put() os.environ['PROJECT_NAME'] = 'oss-fuzz' data_types.FuzzTarget(engine='libFuzzer', project='test', binary='test_fuzzer').put() data_types.FuzzTargetJob(fuzz_target_name='libFuzzer_test_fuzzer', engine='libFuzzer', job='libfuzzer_asan_job', last_run=datetime.datetime.now()).put() data_types.FuzzTarget(engine='libFuzzer', project='test2', binary='fuzzer').put() data_types.FuzzTargetJob(fuzz_target_name='libFuzzer_test2_fuzzer', engine='libFuzzer', job='libfuzzer_asan_job2', last_run=datetime.datetime.now()).put() environment.set_value('USE_MINIJAIL', True) environment.set_value('SHARED_CORPUS_BUCKET', TEST_SHARED_BUCKET) # Set up remote corpora. self.corpus = corpus_manager.FuzzTargetCorpus('libFuzzer', 'test_fuzzer') self.corpus.rsync_from_disk(os.path.join(TEST_DIR, 'corpus'), delete=True) self.quarantine_corpus = corpus_manager.FuzzTargetCorpus( 'libFuzzer', 'test_fuzzer', quarantine=True) self.quarantine_corpus.rsync_from_disk(os.path.join( TEST_DIR, 'quarantine'), delete=True) self.mock.get_data_bundle_bucket_name.return_value = TEST_GLOBAL_BUCKET data_types.DataBundle(name='bundle', is_local=True, sync_to_worker=True).put() data_types.Fuzzer(revision=1, file_size='builtin', source='builtin', name='libFuzzer', max_testcases=4, builtin=True, data_bundle_name='bundle').put() self.temp_dir = tempfile.mkdtemp() # Copy corpus backup in the older date format. corpus_backup_date = ( datetime.datetime.utcnow().date() - datetime.timedelta( days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS)) corpus_backup_dir = ('gs://{bucket}/corpus/libfuzzer/test2_fuzzer/') gsutil.GSUtilRunner().run_gsutil([ 'cp', (corpus_backup_dir + 'backup.zip').format(bucket=TEST2_BACKUP_BUCKET), (corpus_backup_dir + '%s.zip' % corpus_backup_date).format(bucket=self.backup_bucket) ])
def setUp(self): """Set up.""" super(CorpusPruningTestUntrusted, self).setUp() environment.set_value("JOB_NAME", "libfuzzer_asan_job") helpers.patch( self, [ "bot.fuzzers.engine.get", "bot.fuzzers.libFuzzer.fuzzer.LibFuzzer.fuzzer_directory", "base.tasks.add_task", "datastore.data_handler.get_data_bundle_bucket_name", ], ) self.mock.get.return_value = libFuzzer_engine.LibFuzzerEngine() self.mock.fuzzer_directory.return_value = os.path.join( environment.get_value("ROOT_DIR"), "src", "python", "bot", "fuzzers", "libFuzzer", ) self.corpus_bucket = os.environ["CORPUS_BUCKET"] self.quarantine_bucket = os.environ["QUARANTINE_BUCKET"] self.backup_bucket = os.environ["BACKUP_BUCKET"] job = data_types.Job( name="libfuzzer_asan_job", environment_string=("APP_NAME = test_fuzzer\n" "CORPUS_BUCKET = {corpus_bucket}\n" "QUARANTINE_BUCKET = {quarantine_bucket}\n" "BACKUP_BUCKET={backup_bucket}\n" "RELEASE_BUILD_BUCKET_PATH = " "gs://clusterfuzz-test-data/test_libfuzzer_builds/" "test-libfuzzer-build-([0-9]+).zip\n" "REVISION_VARS_URL = gs://clusterfuzz-test-data/" "test_libfuzzer_builds/" "test-libfuzzer-build-%s.srcmap.json\n".format( corpus_bucket=self.corpus_bucket, quarantine_bucket=self.quarantine_bucket, backup_bucket=self.backup_bucket, )), ) job.put() job = data_types.Job( name="libfuzzer_asan_job2", environment_string=("APP_NAME = test2_fuzzer\n" "BACKUP_BUCKET = {backup_bucket}\n" "CORPUS_FUZZER_NAME_OVERRIDE = libfuzzer\n".format( backup_bucket=self.backup_bucket)), ) job.put() os.environ["PROJECT_NAME"] = "oss-fuzz" data_types.FuzzTarget( engine="libFuzzer", project="test", binary="test_fuzzer").put() data_types.FuzzTargetJob( fuzz_target_name="libFuzzer_test_fuzzer", engine="libFuzzer", job="libfuzzer_asan_job", last_run=datetime.datetime.now(), ).put() data_types.FuzzTarget( engine="libFuzzer", project="test2", binary="fuzzer").put() data_types.FuzzTargetJob( fuzz_target_name="libFuzzer_test2_fuzzer", engine="libFuzzer", job="libfuzzer_asan_job2", last_run=datetime.datetime.now(), ).put() environment.set_value("USE_MINIJAIL", True) environment.set_value("SHARED_CORPUS_BUCKET", TEST_SHARED_BUCKET) # Set up remote corpora. self.corpus = corpus_manager.FuzzTargetCorpus("libFuzzer", "test_fuzzer") self.corpus.rsync_from_disk(os.path.join(TEST_DIR, "corpus"), delete=True) self.quarantine_corpus = corpus_manager.FuzzTargetCorpus( "libFuzzer", "test_fuzzer", quarantine=True) self.quarantine_corpus.rsync_from_disk( os.path.join(TEST_DIR, "quarantine"), delete=True) self.mock.get_data_bundle_bucket_name.return_value = TEST_GLOBAL_BUCKET data_types.DataBundle( name="bundle", is_local=True, sync_to_worker=True).put() data_types.Fuzzer( revision=1, file_size="builtin", source="builtin", name="libFuzzer", max_testcases=4, builtin=True, data_bundle_name="bundle", ).put() self.temp_dir = tempfile.mkdtemp() # Copy corpus backup in the older date format. corpus_backup_date = datetime.datetime.utcnow().date() - datetime.timedelta( days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS) corpus_backup_dir = "gs://{bucket}/corpus/libfuzzer/test2_fuzzer/" gsutil.GSUtilRunner().run_gsutil([ "cp", (corpus_backup_dir + "backup.zip").format(bucket=TEST2_BACKUP_BUCKET), (corpus_backup_dir + "%s.zip" % corpus_backup_date).format(bucket=self.backup_bucket), ])