def get(self): """Handles a GET request.""" libfuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == 'libFuzzer').get() if not libfuzzer: logs.log_error('Failed to get libFuzzer Fuzzer entity.') return afl = data_types.Fuzzer.query(data_types.Fuzzer.name == 'afl').get() if not afl: logs.log_error('Failed to get AFL Fuzzer entity.') return # Create storage client. client = storage.create_discovery_storage_client() # Clear old job associations. libfuzzer.jobs = [] afl.jobs = [] data_bundles = set([ libfuzzer.data_bundle_name, afl.data_bundle_name, ]) projects = get_projects() for project, info in projects: logs.log('Syncing configs for %s.' % project) if not VALID_PROJECT_NAME_REGEX.match(project): logs.log_error('Invalid project name: ' + project) continue service_account = service_accounts.get_or_create_service_account( project) service_accounts.set_service_account_roles(service_account) # Create GCS buckets. backup_bucket_name = get_backup_bucket_name(project) corpus_bucket_name = get_corpus_bucket_name(project) logs_bucket_name = get_logs_bucket_name(project) quarantine_bucket_name = get_quarantine_bucket_name(project) storage.create_bucket_if_needed(backup_bucket_name, BACKUPS_LIFECYCLE) storage.create_bucket_if_needed(corpus_bucket_name) storage.create_bucket_if_needed(quarantine_bucket_name, QUARANTINE_LIFECYCLE) storage.create_bucket_if_needed(logs_bucket_name, LOGS_LIFECYCLE) try: add_bucket_iams(info, client, backup_bucket_name, service_account) add_bucket_iams(info, client, corpus_bucket_name, service_account) add_bucket_iams(info, client, logs_bucket_name, service_account) add_bucket_iams(info, client, quarantine_bucket_name, service_account) except Exception as e: logs.log_error('Failed to add bucket IAMs for %s: %s' % (project, e)) # Grant the service account read access to deployment, shared corpus and # mutator plugin buckets. add_service_account_to_bucket(client, _deployment_bucket_name(), service_account, OBJECT_VIEWER_IAM_ROLE) add_service_account_to_bucket(client, _shared_corpus_bucket_name(), service_account, OBJECT_VIEWER_IAM_ROLE) add_service_account_to_bucket(client, _mutator_plugins_bucket_name(), service_account, OBJECT_VIEWER_IAM_ROLE) for data_bundle in data_bundles: # Workers also need to be able to set up these global bundles. data_bundle_bucket_name = data_handler.get_data_bundle_bucket_name( data_bundle) add_service_account_to_bucket(client, data_bundle_bucket_name, service_account, OBJECT_VIEWER_IAM_ROLE) # Create CF jobs for project. sync_cf_job(project, info, corpus_bucket_name, quarantine_bucket_name, logs_bucket_name, backup_bucket_name, libfuzzer, afl) # Create revision mappings for CF. sync_cf_revision_mappings(project, info) sync_user_permissions(project, info) # Create Pub/Sub topics for tasks. create_pubsub_topics(project) # Set up projects settings (such as CPU distribution settings). if not info.get('disabled', False): create_project_settings(project, info, service_account) # Update CF Fuzzer entities for new jobs added. libfuzzer.put() afl.put() # Update job task queues. refresh_fuzzer_job_mappings([libfuzzer, afl]) # Delete old jobs. project_names = [project[0] for project in projects] cleanup_old_jobs(project_names) # Delete old pubsub topics. cleanup_pubsub_topics(project_names) # Delete old/disabled project settings. enabled_projects = [ project for project, info in projects if not info.get('disabled', False) ] cleanup_old_projects_settings(enabled_projects)
def test_get_data_bundle_name_default(self): """Test getting the default data bundle bucket name.""" self.assertEqual('test-corpus.test-clusterfuzz.appspot.com', data_handler.get_data_bundle_bucket_name('test'))
def test_get_data_bundle_name_custom_suffix(self): """Test getting the data bundle bucket name with custom suffix.""" self.mock.project_config_get.side_effect = None self.mock.project_config_get.return_value = 'custom.suffix.com' self.assertEqual('test-corpus.custom.suffix.com', data_handler.get_data_bundle_bucket_name('test'))