def setUp(self): """Set up.""" super(UntrustedRunnerIntegrationTest, self).setUp() test_helpers.patch(self, [ 'datastore.data_handler.get_data_bundle_bucket_name', ]) data_types.Config().put() environment_string = ('APP_NAME = app\n' 'RELEASE_BUILD_BUCKET_PATH = ' 'gs://clusterfuzz-test-data/test_builds/' 'test-build-([0-9]+).zip\n') data_types.Job(name='job', environment_string=environment_string).put() environment_string = ('APP_NAME = launcher.py\n' 'RELEASE_BUILD_BUCKET_PATH = ' 'gs://clusterfuzz-test-data/test_libfuzzer_builds/' 'test-libfuzzer-build-([0-9]+).zip\n' 'UNPACK_ALL_FUZZ_TARGETS_AND_FILES = True') data_types.Job( name='libfuzzer_asan_job', environment_string=environment_string).put() data_types.Fuzzer(name='fuzzer', data_bundle_name='bundle').put() data_types.DataBundle( name='bundle', is_local=True, sync_to_worker=True).put()
def post(self): """Handle a post request.""" name = self.request.get('name') if not name: raise helpers.EarlyExitException('Please give this corpus a name!', 400) if not data_types.DataBundle.VALID_NAME_REGEX.match(name): raise helpers.EarlyExitException( 'Name can only contain letters, numbers, dashes and underscores.', 400) user_email = helpers.get_user_email() bucket_name = data_handler.get_data_bundle_bucket_name(name) bucket_url = data_handler.get_data_bundle_bucket_url(name) is_local = not self.request.get('nfs', False) if not data_handler.create_data_bundle_bucket_and_iams( name, [user_email]): raise helpers.EarlyExitException( 'Failed to create bucket %s.' % bucket_name, 400) data_bundle = data_types.DataBundle.query( data_types.DataBundle.name == name).get() if not data_bundle: data_bundle = data_types.DataBundle() data_bundle.name = name data_bundle.bucket_name = bucket_name data_bundle.is_local = is_local data_bundle.put() template_values = { 'title': 'Success', 'message': ('Upload data to the corpus using: ' 'gsutil -d -m rsync -r <local_corpus_directory> %s' % bucket_url), } self.render('message.html', template_values)
def setUp(self): """Set up.""" super(CorpusPruningTestUntrusted, self).setUp() environment.set_value('JOB_NAME', 'libfuzzer_asan_job') helpers.patch(self, [ 'bot.fuzzers.engine.get', 'bot.tasks.setup.get_fuzzer_directory', 'base.tasks.add_task', 'bot.tasks.corpus_pruning_task._record_cross_pollination_stats' ]) self.mock.get.return_value = libFuzzer_engine.LibFuzzerEngine() self.mock.get_fuzzer_directory.return_value = os.path.join( environment.get_value('ROOT_DIR'), 'src', 'python', 'bot', 'fuzzers', 'libFuzzer') self.corpus_bucket = os.environ['CORPUS_BUCKET'] self.quarantine_bucket = os.environ['QUARANTINE_BUCKET'] self.backup_bucket = os.environ['BACKUP_BUCKET'] job = data_types.Job( name='libfuzzer_asan_job', environment_string=( 'APP_NAME = test_fuzzer\n' 'CORPUS_BUCKET = {corpus_bucket}\n' 'QUARANTINE_BUCKET = {quarantine_bucket}\n' 'BACKUP_BUCKET={backup_bucket}\n' 'RELEASE_BUILD_BUCKET_PATH = ' 'gs://clusterfuzz-test-data/test_libfuzzer_builds/' 'test-libfuzzer-build-([0-9]+).zip\n' 'REVISION_VARS_URL = gs://clusterfuzz-test-data/' 'test_libfuzzer_builds/' 'test-libfuzzer-build-%s.srcmap.json\n'.format( corpus_bucket=self.corpus_bucket, quarantine_bucket=self.quarantine_bucket, backup_bucket=self.backup_bucket))) job.put() job = data_types.Job( name='libfuzzer_asan_job2', environment_string=( 'APP_NAME = test2_fuzzer\n' 'BACKUP_BUCKET = {backup_bucket}\n' 'CORPUS_FUZZER_NAME_OVERRIDE = libfuzzer\n'.format( backup_bucket=self.backup_bucket))) job.put() os.environ['PROJECT_NAME'] = 'oss-fuzz' data_types.FuzzTarget(engine='libFuzzer', project='test', binary='test_fuzzer').put() data_types.FuzzTargetJob(fuzz_target_name='libFuzzer_test_fuzzer', engine='libFuzzer', job='libfuzzer_asan_job', last_run=datetime.datetime.now()).put() data_types.FuzzTarget(engine='libFuzzer', project='test2', binary='fuzzer').put() data_types.FuzzTargetJob(fuzz_target_name='libFuzzer_test2_fuzzer', engine='libFuzzer', job='libfuzzer_asan_job2', last_run=datetime.datetime.now()).put() environment.set_value('USE_MINIJAIL', True) environment.set_value('SHARED_CORPUS_BUCKET', TEST_SHARED_BUCKET) # Set up remote corpora. self.corpus = corpus_manager.FuzzTargetCorpus('libFuzzer', 'test_fuzzer') self.corpus.rsync_from_disk(os.path.join(TEST_DIR, 'corpus'), delete=True) self.quarantine_corpus = corpus_manager.FuzzTargetCorpus( 'libFuzzer', 'test_fuzzer', quarantine=True) self.quarantine_corpus.rsync_from_disk(os.path.join( TEST_DIR, 'quarantine'), delete=True) self.mock.get_data_bundle_bucket_name.return_value = TEST_GLOBAL_BUCKET data_types.DataBundle(name='bundle', is_local=True, sync_to_worker=True).put() data_types.Fuzzer(revision=1, file_size='builtin', source='builtin', name='libFuzzer', max_testcases=4, builtin=True, data_bundle_name='bundle').put() self.temp_dir = tempfile.mkdtemp() # Copy corpus backup in the older date format. corpus_backup_date = ( datetime.datetime.utcnow().date() - datetime.timedelta( days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS)) corpus_backup_dir = ('gs://{bucket}/corpus/libfuzzer/test2_fuzzer/') gsutil.GSUtilRunner().run_gsutil([ 'cp', (corpus_backup_dir + 'backup.zip').format(bucket=TEST2_BACKUP_BUCKET), (corpus_backup_dir + '%s.zip' % corpus_backup_date).format(bucket=self.backup_bucket) ])
def setUp(self): """Set up.""" super(CorpusPruningTestUntrusted, self).setUp() environment.set_value("JOB_NAME", "libfuzzer_asan_job") helpers.patch( self, [ "bot.fuzzers.engine.get", "bot.fuzzers.libFuzzer.fuzzer.LibFuzzer.fuzzer_directory", "base.tasks.add_task", "datastore.data_handler.get_data_bundle_bucket_name", ], ) self.mock.get.return_value = libFuzzer_engine.LibFuzzerEngine() self.mock.fuzzer_directory.return_value = os.path.join( environment.get_value("ROOT_DIR"), "src", "python", "bot", "fuzzers", "libFuzzer", ) self.corpus_bucket = os.environ["CORPUS_BUCKET"] self.quarantine_bucket = os.environ["QUARANTINE_BUCKET"] self.backup_bucket = os.environ["BACKUP_BUCKET"] job = data_types.Job( name="libfuzzer_asan_job", environment_string=("APP_NAME = test_fuzzer\n" "CORPUS_BUCKET = {corpus_bucket}\n" "QUARANTINE_BUCKET = {quarantine_bucket}\n" "BACKUP_BUCKET={backup_bucket}\n" "RELEASE_BUILD_BUCKET_PATH = " "gs://clusterfuzz-test-data/test_libfuzzer_builds/" "test-libfuzzer-build-([0-9]+).zip\n" "REVISION_VARS_URL = gs://clusterfuzz-test-data/" "test_libfuzzer_builds/" "test-libfuzzer-build-%s.srcmap.json\n".format( corpus_bucket=self.corpus_bucket, quarantine_bucket=self.quarantine_bucket, backup_bucket=self.backup_bucket, )), ) job.put() job = data_types.Job( name="libfuzzer_asan_job2", environment_string=("APP_NAME = test2_fuzzer\n" "BACKUP_BUCKET = {backup_bucket}\n" "CORPUS_FUZZER_NAME_OVERRIDE = libfuzzer\n".format( backup_bucket=self.backup_bucket)), ) job.put() os.environ["PROJECT_NAME"] = "oss-fuzz" data_types.FuzzTarget( engine="libFuzzer", project="test", binary="test_fuzzer").put() data_types.FuzzTargetJob( fuzz_target_name="libFuzzer_test_fuzzer", engine="libFuzzer", job="libfuzzer_asan_job", last_run=datetime.datetime.now(), ).put() data_types.FuzzTarget( engine="libFuzzer", project="test2", binary="fuzzer").put() data_types.FuzzTargetJob( fuzz_target_name="libFuzzer_test2_fuzzer", engine="libFuzzer", job="libfuzzer_asan_job2", last_run=datetime.datetime.now(), ).put() environment.set_value("USE_MINIJAIL", True) environment.set_value("SHARED_CORPUS_BUCKET", TEST_SHARED_BUCKET) # Set up remote corpora. self.corpus = corpus_manager.FuzzTargetCorpus("libFuzzer", "test_fuzzer") self.corpus.rsync_from_disk(os.path.join(TEST_DIR, "corpus"), delete=True) self.quarantine_corpus = corpus_manager.FuzzTargetCorpus( "libFuzzer", "test_fuzzer", quarantine=True) self.quarantine_corpus.rsync_from_disk( os.path.join(TEST_DIR, "quarantine"), delete=True) self.mock.get_data_bundle_bucket_name.return_value = TEST_GLOBAL_BUCKET data_types.DataBundle( name="bundle", is_local=True, sync_to_worker=True).put() data_types.Fuzzer( revision=1, file_size="builtin", source="builtin", name="libFuzzer", max_testcases=4, builtin=True, data_bundle_name="bundle", ).put() self.temp_dir = tempfile.mkdtemp() # Copy corpus backup in the older date format. corpus_backup_date = datetime.datetime.utcnow().date() - datetime.timedelta( days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS) corpus_backup_dir = "gs://{bucket}/corpus/libfuzzer/test2_fuzzer/" gsutil.GSUtilRunner().run_gsutil([ "cp", (corpus_backup_dir + "backup.zip").format(bucket=TEST2_BACKUP_BUCKET), (corpus_backup_dir + "%s.zip" % corpus_backup_date).format(bucket=self.backup_bucket), ])
def setUp(self): helpers.patch_environ(self) project_config_get = local_config.ProjectConfig.get helpers.patch(self, [ 'base.utils.default_project_name', 'config.db_config.get', ('project_config_get', 'config.local_config.ProjectConfig.get'), ]) self.job = data_types.Job( name='linux_asan_chrome', environment_string=('SUMMARY_PREFIX = project\n' 'PROJECT_NAME = project\n' 'HELP_URL = help_url\n')) self.job2 = data_types.Job( name='windows_asan_chrome', environment_string=('SUMMARY_PREFIX = project\n' 'PROJECT_NAME = project\n' 'HELP_URL = help_url\n')) self.testcase = data_types.Testcase( job_type='linux_asan_chrome', fuzzer_name='libfuzzer_binary_name', crash_type='Crash-type', crash_address='0x1337', crash_state='A\nB\nC\n') self.testcase.set_metadata('fuzzer_binary_name', 'binary_name', update_testcase=False) self.testcase_assert = data_types.Testcase( job_type='linux_asan_chrome', fuzzer_name='libfuzzer_binary_name', crash_type='ASSERT', crash_address='0x1337', crash_state='foo != bar\nB\nC\n') self.testcase_assert.set_metadata('fuzzer_binary_name', 'binary_name', update_testcase=False) self.testcase_null = data_types.Testcase( job_type='linux_asan_chrome', fuzzer_name='libfuzzer_binary_name', crash_type='UNKNOWN', crash_address='0x1337', crash_state='NULL') self.testcase_bad_cast = data_types.Testcase( job_type='linux_asan_chrome', fuzzer_name='libfuzzer_binary_name', crash_type='Bad-cast', crash_address='0x1337', crash_state= ('Bad-cast to blink::LayoutBlock from blink::LayoutTableSection\n' 'blink::LayoutObject::ContainerForFixedPosition\n' 'blink::LayoutObject::Container\n')) self.testcase_bad_cast_without_crash_function = data_types.Testcase( job_type='linux_asan_chrome', fuzzer_name='libfuzzer_binary_name', crash_type='Bad-cast', crash_address='0x1337', crash_state= ('Bad-cast to blink::LayoutBlock from blink::LayoutTableSection\n' )) self.local_data_bundle = data_types.DataBundle( name='local_data_bundle') self.cloud_data_bundle = data_types.DataBundle( name='cloud_data_bundle') self.fuzzer1 = data_types.Fuzzer(name='fuzzer1', data_bundle_name=None, jobs=['linux_asan_chrome']) self.fuzzer2 = data_types.Fuzzer(name='fuzzer2', data_bundle_name='local_data_bundle', jobs=['linux_asan_chrome']) self.fuzzer3 = data_types.Fuzzer(name='fuzzer3', data_bundle_name='cloud_data_bundle', jobs=['linux_asan_chrome']) entities_to_put = [ self.testcase, self.testcase_assert, self.testcase_null, self.testcase_bad_cast, self.testcase_bad_cast_without_crash_function, self.job, self.job2, self.local_data_bundle, self.cloud_data_bundle, self.fuzzer1, self.fuzzer2, self.fuzzer3 ] for entity in entities_to_put: entity.put() environment.set_value('FUZZ_DATA', '/tmp/inputs/fuzzer-common-data-bundles') environment.set_value('FUZZERS_DIR', '/tmp/inputs/fuzzers') self.mock.default_project_name.return_value = 'project' self.mock.project_config_get.side_effect = project_config_get