def get(self):
"""Handles a GET request."""
libfuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == 'libFuzzer').get()
if not libfuzzer:
logs.log_error('Failed to get libFuzzer Fuzzer entity.')
return
afl = data_types.Fuzzer.query(data_types.Fuzzer.name == 'afl').get()
if not afl:
logs.log_error('Failed to get AFL Fuzzer entity.')
return
# Create storage client.
client = storage.create_discovery_storage_client()
# Clear old job associations.
libfuzzer.jobs = []
afl.jobs = []
data_bundles = set([
libfuzzer.data_bundle_name,
afl.data_bundle_name,
])
projects = get_projects()
for project, info in projects:
logs.log('Syncing configs for %s.' % project)
if not VALID_PROJECT_NAME_REGEX.match(project):
logs.log_error('Invalid project name: ' + project)
continue
service_account = service_accounts.get_or_create_service_account(
project)
service_accounts.set_service_account_roles(service_account)
# Create GCS buckets.
backup_bucket_name = get_backup_bucket_name(project)
corpus_bucket_name = get_corpus_bucket_name(project)
logs_bucket_name = get_logs_bucket_name(project)
quarantine_bucket_name = get_quarantine_bucket_name(project)
storage.create_bucket_if_needed(backup_bucket_name,
BACKUPS_LIFECYCLE)
storage.create_bucket_if_needed(corpus_bucket_name)
storage.create_bucket_if_needed(quarantine_bucket_name,
QUARANTINE_LIFECYCLE)
storage.create_bucket_if_needed(logs_bucket_name, LOGS_LIFECYCLE)
try:
add_bucket_iams(info, client, backup_bucket_name,
service_account)
add_bucket_iams(info, client, corpus_bucket_name,
service_account)
add_bucket_iams(info, client, logs_bucket_name,
service_account)
add_bucket_iams(info, client, quarantine_bucket_name,
service_account)
except Exception as e:
logs.log_error('Failed to add bucket IAMs for %s: %s' %
(project, e))
# Grant the service account read access to deployment, shared corpus and
# mutator plugin buckets.
add_service_account_to_bucket(client, _deployment_bucket_name(),
service_account,
OBJECT_VIEWER_IAM_ROLE)
add_service_account_to_bucket(client, _shared_corpus_bucket_name(),
service_account,
OBJECT_VIEWER_IAM_ROLE)
add_service_account_to_bucket(client,
_mutator_plugins_bucket_name(),
service_account,
OBJECT_VIEWER_IAM_ROLE)
for data_bundle in data_bundles:
# Workers also need to be able to set up these global bundles.
data_bundle_bucket_name = data_handler.get_data_bundle_bucket_name(
data_bundle)
add_service_account_to_bucket(client, data_bundle_bucket_name,
service_account,
OBJECT_VIEWER_IAM_ROLE)
# Create CF jobs for project.
sync_cf_job(project, info, corpus_bucket_name,
quarantine_bucket_name, logs_bucket_name,
backup_bucket_name, libfuzzer, afl)
# Create revision mappings for CF.
sync_cf_revision_mappings(project, info)
sync_user_permissions(project, info)
# Create Pub/Sub topics for tasks.
create_pubsub_topics(project)
# Set up projects settings (such as CPU distribution settings).
if not info.get('disabled', False):
create_project_settings(project, info, service_account)
# Update CF Fuzzer entities for new jobs added.
libfuzzer.put()
afl.put()
# Update job task queues.
refresh_fuzzer_job_mappings([libfuzzer, afl])
# Delete old jobs.
project_names = [project[0] for project in projects]
cleanup_old_jobs(project_names)
# Delete old pubsub topics.
cleanup_pubsub_topics(project_names)
# Delete old/disabled project settings.
enabled_projects = [
project for project, info in projects
if not info.get('disabled', False)
]
cleanup_old_projects_settings(enabled_projects)
def test_get_data_bundle_name_default(self):
"""Test getting the default data bundle bucket name."""
self.assertEqual('test-corpus.test-clusterfuzz.appspot.com',
data_handler.get_data_bundle_bucket_name('test'))
def test_get_data_bundle_name_custom_suffix(self):
"""Test getting the data bundle bucket name with custom suffix."""
self.mock.project_config_get.side_effect = None
self.mock.project_config_get.return_value = 'custom.suffix.com'
self.assertEqual('test-corpus.custom.suffix.com',
data_handler.get_data_bundle_bucket_name('test'))