def _create_service_accounts_and_buckets(self, project, info):
"""Create per-project service account and buckets."""
service_account = service_accounts.get_or_create_service_account(
project)
service_accounts.set_service_account_roles(service_account)
# Create GCS buckets.
backup_bucket_name = self._backup_bucket_name(project)
corpus_bucket_name = self._corpus_bucket_name(project)
logs_bucket_name = self._logs_bucket_name(project)
quarantine_bucket_name = self._quarantine_bucket_name(project)
storage.create_bucket_if_needed(backup_bucket_name, BACKUPS_LIFECYCLE)
storage.create_bucket_if_needed(corpus_bucket_name)
storage.create_bucket_if_needed(quarantine_bucket_name,
QUARANTINE_LIFECYCLE)
storage.create_bucket_if_needed(logs_bucket_name, LOGS_LIFECYCLE)
client = storage.create_discovery_storage_client()
try:
add_bucket_iams(info, client, backup_bucket_name, service_account)
add_bucket_iams(info, client, corpus_bucket_name, service_account)
add_bucket_iams(info, client, logs_bucket_name, service_account)
add_bucket_iams(info, client, quarantine_bucket_name,
service_account)
except Exception as e:
logs.log_error('Failed to add bucket IAMs for %s: %s' %
(project, e))
# Grant the service account read access to deployment, shared corpus and
# mutator plugin buckets.
add_service_account_to_bucket(client, self._deployment_bucket_name(),
service_account, OBJECT_VIEWER_IAM_ROLE)
add_service_account_to_bucket(client,
self._shared_corpus_bucket_name(),
service_account, OBJECT_VIEWER_IAM_ROLE)
add_service_account_to_bucket(client,
self._mutator_plugins_bucket_name(),
service_account, OBJECT_VIEWER_IAM_ROLE)
data_bundles = {
fuzzer_entity.data_bundle_name
for fuzzer_entity in six.itervalues(self._fuzzer_entities)
if fuzzer_entity.data_bundle_name
}
for data_bundle in data_bundles:
# Workers also need to be able to set up these global bundles.
data_bundle_bucket_name = data_handler.get_data_bundle_bucket_name(
data_bundle)
add_service_account_to_bucket(client, data_bundle_bucket_name,
service_account,
OBJECT_VIEWER_IAM_ROLE)
return (service_account, backup_bucket_name, corpus_bucket_name,
logs_bucket_name, quarantine_bucket_name)
def create_data_bundle_bucket_and_iams(data_bundle_name, emails):
"""Creates a data bundle bucket and adds iams for access."""
bucket_name = get_data_bundle_bucket_name(data_bundle_name)
if not storage.create_bucket_if_needed(bucket_name):
return False
client = storage.create_discovery_storage_client()
iam_policy = storage.get_bucket_iam_policy(client, bucket_name)
if not iam_policy:
return False
members = []
# Add access for the domains allowed in project.
domains = local_config.AuthConfig().get('whitelisted_domains', default=[])
for domain in domains:
members.append('domain:%s' % domain)
# Add access for the emails provided in function arguments.
for email in emails:
members.append('user:%s' % email)
if not members:
# No members to add, bail out.
return True
binding = storage.get_bucket_iam_binding(iam_policy,
DATA_BUNDLE_DEFAULT_BUCKET_IAM_ROLE)
if binding:
binding['members'] = members
else:
binding = {
'role': DATA_BUNDLE_DEFAULT_BUCKET_IAM_ROLE,
'members': members,
}
iam_policy['bindings'].append(binding)
return bool(storage.set_bucket_iam_policy(client, bucket_name, iam_policy))
def get(self):
"""Handles a GET request."""
libfuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == 'libFuzzer').get()
if not libfuzzer:
logs.log_error('Failed to get libFuzzer Fuzzer entity.')
return
afl = data_types.Fuzzer.query(data_types.Fuzzer.name == 'afl').get()
if not afl:
logs.log_error('Failed to get AFL Fuzzer entity.')
return
# Create storage client.
client = storage.create_discovery_storage_client()
# Clear old job associations.
libfuzzer.jobs = []
afl.jobs = []
data_bundles = set([
libfuzzer.data_bundle_name,
afl.data_bundle_name,
])
projects = get_projects()
for project, info in projects:
logs.log('Syncing configs for %s.' % project)
if not VALID_PROJECT_NAME_REGEX.match(project):
logs.log_error('Invalid project name: ' + project)
continue
service_account = service_accounts.get_or_create_service_account(project)
service_accounts.set_service_account_roles(service_account)
# Create GCS buckets.
backup_bucket_name = get_backup_bucket_name(project)
corpus_bucket_name = get_corpus_bucket_name(project)
logs_bucket_name = get_logs_bucket_name(project)
quarantine_bucket_name = get_quarantine_bucket_name(project)
storage.create_bucket_if_needed(backup_bucket_name, BACKUPS_LIFECYCLE)
storage.create_bucket_if_needed(corpus_bucket_name)
storage.create_bucket_if_needed(quarantine_bucket_name,
QUARANTINE_LIFECYCLE)
storage.create_bucket_if_needed(logs_bucket_name, LOGS_LIFECYCLE)
try:
add_bucket_iams(info, client, backup_bucket_name, service_account)
add_bucket_iams(info, client, corpus_bucket_name, service_account)
add_bucket_iams(info, client, logs_bucket_name, service_account)
add_bucket_iams(info, client, quarantine_bucket_name, service_account)
except Exception as e:
logs.log_error('Failed to add bucket IAMs for %s: %s' % (project, e))
# Grant the service account read access to deployment, shared corpus and
# mutator plugin buckets.
add_service_account_to_bucket(client, _deployment_bucket_name(),
service_account, OBJECT_VIEWER_IAM_ROLE)
add_service_account_to_bucket(client, _shared_corpus_bucket_name(),
service_account, OBJECT_VIEWER_IAM_ROLE)
add_service_account_to_bucket(client, _mutator_plugins_bucket_name(),
service_account, OBJECT_VIEWER_IAM_ROLE)
for data_bundle in data_bundles:
# Workers also need to be able to set up these global bundles.
data_bundle_bucket_name = data_handler.get_data_bundle_bucket_name(
data_bundle)
add_service_account_to_bucket(client, data_bundle_bucket_name,
service_account, OBJECT_VIEWER_IAM_ROLE)
# Create CF jobs for project.
sync_cf_job(project, info, corpus_bucket_name, quarantine_bucket_name,
logs_bucket_name, backup_bucket_name, libfuzzer, afl)
# Create revision mappings for CF.
sync_cf_revision_mappings(project, info)
sync_user_permissions(project, info)
# Create Pub/Sub topics for tasks.
create_pubsub_topics(project)
# Set up projects settings (such as CPU distribution settings).
if not info.get('disabled', False):
create_project_settings(project, info, service_account)
# Update CF Fuzzer entities for new jobs added.
libfuzzer.put()
afl.put()
# Update job task queues.
refresh_fuzzer_job_mappings([libfuzzer, afl])
# Delete old jobs.
project_names = [project[0] for project in projects]
cleanup_old_jobs(project_names)
# Delete old pubsub topics.
cleanup_pubsub_topics(project_names)
# Delete old/disabled project settings.
enabled_projects = [
project for project, info in projects
if not info.get('disabled', False)
]
cleanup_old_projects_settings(enabled_projects)
