def _create_service_accounts_and_buckets(self, project, info): """Create per-project service account and buckets.""" service_account = service_accounts.get_or_create_service_account( project) service_accounts.set_service_account_roles(service_account) # Create GCS buckets. backup_bucket_name = self._backup_bucket_name(project) corpus_bucket_name = self._corpus_bucket_name(project) logs_bucket_name = self._logs_bucket_name(project) quarantine_bucket_name = self._quarantine_bucket_name(project) storage.create_bucket_if_needed(backup_bucket_name, BACKUPS_LIFECYCLE) storage.create_bucket_if_needed(corpus_bucket_name) storage.create_bucket_if_needed(quarantine_bucket_name, QUARANTINE_LIFECYCLE) storage.create_bucket_if_needed(logs_bucket_name, LOGS_LIFECYCLE) client = storage.create_discovery_storage_client() try: add_bucket_iams(info, client, backup_bucket_name, service_account) add_bucket_iams(info, client, corpus_bucket_name, service_account) add_bucket_iams(info, client, logs_bucket_name, service_account) add_bucket_iams(info, client, quarantine_bucket_name, service_account) except Exception as e: logs.log_error('Failed to add bucket IAMs for %s: %s' % (project, e)) # Grant the service account read access to deployment, shared corpus and # mutator plugin buckets. add_service_account_to_bucket(client, self._deployment_bucket_name(), service_account, OBJECT_VIEWER_IAM_ROLE) add_service_account_to_bucket(client, self._shared_corpus_bucket_name(), service_account, OBJECT_VIEWER_IAM_ROLE) add_service_account_to_bucket(client, self._mutator_plugins_bucket_name(), service_account, OBJECT_VIEWER_IAM_ROLE) data_bundles = { fuzzer_entity.data_bundle_name for fuzzer_entity in six.itervalues(self._fuzzer_entities) if fuzzer_entity.data_bundle_name } for data_bundle in data_bundles: # Workers also need to be able to set up these global bundles. data_bundle_bucket_name = data_handler.get_data_bundle_bucket_name( data_bundle) add_service_account_to_bucket(client, data_bundle_bucket_name, service_account, OBJECT_VIEWER_IAM_ROLE) return (service_account, backup_bucket_name, corpus_bucket_name, logs_bucket_name, quarantine_bucket_name)
def create_data_bundle_bucket_and_iams(data_bundle_name, emails): """Creates a data bundle bucket and adds iams for access.""" bucket_name = get_data_bundle_bucket_name(data_bundle_name) if not storage.create_bucket_if_needed(bucket_name): return False client = storage.create_discovery_storage_client() iam_policy = storage.get_bucket_iam_policy(client, bucket_name) if not iam_policy: return False members = [] # Add access for the domains allowed in project. domains = local_config.AuthConfig().get('whitelisted_domains', default=[]) for domain in domains: members.append('domain:%s' % domain) # Add access for the emails provided in function arguments. for email in emails: members.append('user:%s' % email) if not members: # No members to add, bail out. return True binding = storage.get_bucket_iam_binding(iam_policy, DATA_BUNDLE_DEFAULT_BUCKET_IAM_ROLE) if binding: binding['members'] = members else: binding = { 'role': DATA_BUNDLE_DEFAULT_BUCKET_IAM_ROLE, 'members': members, } iam_policy['bindings'].append(binding) return bool(storage.set_bucket_iam_policy(client, bucket_name, iam_policy))
def get(self): """Handles a GET request.""" libfuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == 'libFuzzer').get() if not libfuzzer: logs.log_error('Failed to get libFuzzer Fuzzer entity.') return afl = data_types.Fuzzer.query(data_types.Fuzzer.name == 'afl').get() if not afl: logs.log_error('Failed to get AFL Fuzzer entity.') return # Create storage client. client = storage.create_discovery_storage_client() # Clear old job associations. libfuzzer.jobs = [] afl.jobs = [] data_bundles = set([ libfuzzer.data_bundle_name, afl.data_bundle_name, ]) projects = get_projects() for project, info in projects: logs.log('Syncing configs for %s.' % project) if not VALID_PROJECT_NAME_REGEX.match(project): logs.log_error('Invalid project name: ' + project) continue service_account = service_accounts.get_or_create_service_account(project) service_accounts.set_service_account_roles(service_account) # Create GCS buckets. backup_bucket_name = get_backup_bucket_name(project) corpus_bucket_name = get_corpus_bucket_name(project) logs_bucket_name = get_logs_bucket_name(project) quarantine_bucket_name = get_quarantine_bucket_name(project) storage.create_bucket_if_needed(backup_bucket_name, BACKUPS_LIFECYCLE) storage.create_bucket_if_needed(corpus_bucket_name) storage.create_bucket_if_needed(quarantine_bucket_name, QUARANTINE_LIFECYCLE) storage.create_bucket_if_needed(logs_bucket_name, LOGS_LIFECYCLE) try: add_bucket_iams(info, client, backup_bucket_name, service_account) add_bucket_iams(info, client, corpus_bucket_name, service_account) add_bucket_iams(info, client, logs_bucket_name, service_account) add_bucket_iams(info, client, quarantine_bucket_name, service_account) except Exception as e: logs.log_error('Failed to add bucket IAMs for %s: %s' % (project, e)) # Grant the service account read access to deployment, shared corpus and # mutator plugin buckets. add_service_account_to_bucket(client, _deployment_bucket_name(), service_account, OBJECT_VIEWER_IAM_ROLE) add_service_account_to_bucket(client, _shared_corpus_bucket_name(), service_account, OBJECT_VIEWER_IAM_ROLE) add_service_account_to_bucket(client, _mutator_plugins_bucket_name(), service_account, OBJECT_VIEWER_IAM_ROLE) for data_bundle in data_bundles: # Workers also need to be able to set up these global bundles. data_bundle_bucket_name = data_handler.get_data_bundle_bucket_name( data_bundle) add_service_account_to_bucket(client, data_bundle_bucket_name, service_account, OBJECT_VIEWER_IAM_ROLE) # Create CF jobs for project. sync_cf_job(project, info, corpus_bucket_name, quarantine_bucket_name, logs_bucket_name, backup_bucket_name, libfuzzer, afl) # Create revision mappings for CF. sync_cf_revision_mappings(project, info) sync_user_permissions(project, info) # Create Pub/Sub topics for tasks. create_pubsub_topics(project) # Set up projects settings (such as CPU distribution settings). if not info.get('disabled', False): create_project_settings(project, info, service_account) # Update CF Fuzzer entities for new jobs added. libfuzzer.put() afl.put() # Update job task queues. refresh_fuzzer_job_mappings([libfuzzer, afl]) # Delete old jobs. project_names = [project[0] for project in projects] cleanup_old_jobs(project_names) # Delete old pubsub topics. cleanup_pubsub_topics(project_names) # Delete old/disabled project settings. enabled_projects = [ project for project, info in projects if not info.get('disabled', False) ] cleanup_old_projects_settings(enabled_projects)