def auth_domain(): """Get the auth domain.""" domain = local_config.ProjectConfig().get('firebase.auth_domain') if domain: return domain return utils.get_application_id() + '.firebaseapp.com'
def set_service_account_roles(service_account): """Set roles for service account.""" project_id = utils.get_application_id() resource_manager = _create_client('cloudresourcemanager') request = resource_manager.projects().getIamPolicy(resource=project_id, body={}) policy = request.execute() # Set logging and metrics permissions. policy_changed = False policy_changed |= _add_service_account_role(policy, 'roles/logging.logWriter', service_account['email']) policy_changed |= _add_service_account_role( policy, 'roles/monitoring.metricWriter', service_account['email']) if not policy_changed: return request = resource_manager.projects().setIamPolicy(resource=project_id, body={ 'policy': policy, }) request.execute()
def cleanup_pubsub_topics(project_names): """Delete old pubsub topics and subscriptions.""" client = pubsub.PubSubClient() application_id = utils.get_application_id() expected_topics = set() for platform in PUBSUB_PLATFORMS: expected_topics.update([ untrusted.queue_name(project, platform) for project in project_names ]) pubsub_config = local_config.Config('pubsub.queues') unmanaged_queues = [ queue['name'] for queue in pubsub_config.get('resources') ] for topic in client.list_topics(pubsub.project_name(application_id)): _, name = pubsub.parse_name(topic) if (not name.startswith(tasks.JOBS_PREFIX) and not name.startswith(tasks.HIGH_END_JOBS_PREFIX)): # Some topic created by another service, ignore. continue if name in unmanaged_queues: continue if name in expected_topics: continue for subscription in client.list_topic_subscriptions(topic): client.delete_subscription(subscription) client.delete_topic(topic)
def run(self): """Run the flusher thread.""" create_time_series = _retry_wrap(_monitoring_v3_client.create_time_series) project_path = _monitoring_v3_client.project_path( utils.get_application_id()) while True: try: if self.stop_event.wait(FLUSH_INTERVAL_SECONDS): return time_series = [] end_time = time.time() for metric, labels, start_time, value in _metrics_store.iter_values(): if (metric.metric_kind == monitoring_v3.enums.MetricDescriptor.MetricKind.GAUGE): start_time = end_time series = monitoring_v3.types.TimeSeries() metric.monitoring_v3_time_series(series, labels, start_time, end_time, value) time_series.append(series) if len(time_series) == MAX_TIME_SERIES_PER_CALL: create_time_series(project_path, time_series) time_series = [] if time_series: create_time_series(project_path, time_series) except Exception: logs.log_error('Failed to flush metrics.')
def _table_name(self): """Return the table name for the query.""" app_id = utils.get_application_id() dataset = dataset_name(self.fuzzer_or_engine_name) return '`%s`.%s.%s' % (app_id, dataset, self.base_table)
def _create_storage_client_new(): """Create a storage client.""" creds, project = credentials.get_default() if not project: project = utils.get_application_id() return gcs.Client(project=project, credentials=creds)
def add_task(command, argument, job_type, queue=None, wait_time=None): """Add a new task to the job queue.""" # Old testcases may pass in queue=None explicitly, # so we must check this here. if not queue: queue = default_queue() if wait_time is None: wait_time = random.randint(1, TASK_CREATION_WAIT_INTERVAL) if job_type != 'none': job = data_types.Job.query(data_types.Job.name == job_type).get() if not job: raise Error(f'Job {job_type} not found.') if job.is_external(): external_tasks.add_external_task(command, argument, job) return # Add the task. eta = utils.utcnow() + datetime.timedelta(seconds=wait_time) task = Task(command, argument, job_type, eta=eta) pubsub_client = pubsub.PubSubClient() pubsub_client.publish(pubsub.topic_name(utils.get_application_id(), queue), [task.to_pubsub_message()])
def get_regular_task(queue=None): """Get a regular task.""" if not queue: queue = regular_queue() pubsub_client = pubsub.PubSubClient() application_id = utils.get_application_id() while True: messages = pubsub_client.pull_from_subscription( pubsub.subscription_name(application_id, queue), max_messages=1) if not messages: return None try: task = PubSubTask(messages[0]) except KeyError: logs.log_error('Received an invalid task, discarding...') messages[0].ack() continue # Check that this task should be run now (past the ETA). Otherwise we defer # its execution. if not task.defer(): return task
def _load_data(self, bigquery, fuzzer): """Load yesterday's stats into BigQuery.""" project_id = utils.get_application_id() yesterday = (self._utc_now().date() - datetime.timedelta(days=1)) date_string = yesterday.strftime('%Y%m%d') timestamp = utils.utc_date_to_timestamp(yesterday) dataset_id = fuzzer_stats.dataset_name(fuzzer) if not self._create_dataset_if_needed(bigquery, dataset_id): return for kind in STATS_KINDS: kind_name = kind.__name__ table_id = kind_name if kind == fuzzer_stats.TestcaseRun: schema = fuzzer_stats_schema.get(fuzzer) else: schema = kind.SCHEMA if not self._create_table_if_needed(bigquery, dataset_id, table_id, schema): continue gcs_path = fuzzer_stats.get_gcs_stats_path(kind_name, fuzzer, timestamp) load = { 'destinationTable': { 'projectId': project_id, 'tableId': table_id + '$' + date_string, 'datasetId': dataset_id, }, 'schemaUpdateOptions': [ 'ALLOW_FIELD_ADDITION', ], 'sourceFormat': 'NEWLINE_DELIMITED_JSON', 'sourceUris': ['gs:/' + gcs_path + '*.json'], 'writeDisposition': 'WRITE_TRUNCATE', } if schema is not None: load['schema'] = schema job_body = { 'configuration': { 'load': load, }, } logs.log("Uploading job to BigQuery.", job_body=job_body) request = bigquery.jobs().insert(projectId=project_id, body=job_body) response = request.execute() # We cannot really check the response here, as the query might be still # running, but having a BigQuery jobId in the log would make our life # simpler if we ever have to manually check the status of the query. # See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query. logs.log('Response from BigQuery: %s' % response)
def _client(): """Get or initialize the NDB client.""" global _ndb_client global _initial_pid if not _ndb_client: with _ndb_client_lock: if not _ndb_client: _ndb_client = ndb.Client(project=utils.get_application_id()) _initial_pid = os.getpid() return _ndb_client
def _create_dataset_if_needed(self, bigquery, dataset_id): """Create a new dataset if necessary.""" project_id = utils.get_application_id() dataset_body = { 'datasetReference': { 'datasetId': dataset_id, 'projectId': project_id, }, } dataset_insert = bigquery.datasets().insert(projectId=project_id, body=dataset_body) return self._execute_insert_request(dataset_insert)
def create_pubsub_topics(project): """Create pubsub topics for tasks.""" for platform in PUBSUB_PLATFORMS: name = untrusted.queue_name(project, platform) client = pubsub.PubSubClient() application_id = utils.get_application_id() topic_name = pubsub.topic_name(application_id, name) if client.get_topic(topic_name) is None: client.create_topic(topic_name) subscription_name = pubsub.subscription_name(application_id, name) if client.get_subscription(subscription_name) is None: client.create_subscription(subscription_name, topic_name)
def _create_table_if_needed(self, bigquery, dataset_id, table_id): """Create a new table if needed.""" project_id = utils.get_application_id() table_body = { 'tableReference': { 'datasetId': dataset_id, 'projectId': project_id, 'tableId': table_id, }, 'timePartitioning': { 'type': 'DAY', }, } table_insert = bigquery.tables().insert(projectId=project_id, datasetId=dataset_id, body=table_body) return self._execute_insert_request(table_insert)
def create_bucket(self, name, object_lifecycle, cors): """Create a new bucket.""" project_id = utils.get_application_id() request_body = {'name': name} if object_lifecycle: request_body['lifecycle'] = object_lifecycle if cors: request_body['cors'] = cors client = create_discovery_storage_client() try: client.buckets().insert(project=project_id, body=request_body).execute() except HttpError as e: logs.log_warn('Failed to create bucket %s: %s' % (name, e)) raise return True
def setup_metrics(non_dry_run): """Set up metrics.""" client = monitoring_v3.MetricServiceClient() project_name = utils.get_application_id() project_path = client.project_path(project_name) for name in dir(monitoring_metrics): metric = getattr(monitoring_metrics, name) if not isinstance(metric, monitor.Metric): continue descriptor = monitoring_v3.types.MetricDescriptor() metric.monitoring_v3_metric_descriptor(descriptor) if non_dry_run: print('Creating metric', descriptor) client.create_metric_descriptor(project_path, descriptor) else: print('Skip creating metric', descriptor, '(dry-run mode)')
def setUp(self): helpers.patch_environ(self) helpers.patch(self, [ 'clusterfuzz._internal.bot.tasks.minimize_task._run_libfuzzer_testcase', 'clusterfuzz._internal.bot.tasks.minimize_task._run_libfuzzer_tool', ]) test_utils.setup_pubsub(utils.get_application_id()) environment.set_value('APP_ARGS', '%TESTCASE% fuzz_target') environment.set_value('APP_DIR', '/libfuzzer') environment.set_value('APP_NAME', '') environment.set_value('APP_PATH', '') environment.set_value('BOT_TMPDIR', '/bot_tmpdir') environment.set_value('CRASH_STACKTRACES_DIR', '/crash_stacks') environment.set_value('FUZZER_DIR', '/fuzzer_dir') environment.set_value('INPUT_DIR', '/input_dir') environment.set_value('JOB_NAME', 'libfuzzer_asan_test') environment.set_value('USER_PROFILE_IN_MEMORY', True)
def _validate_iap_jwt(iap_jwt): """Validate JWT assertion.""" project_id = utils.get_application_id() expected_audience = '/projects/{}/apps/{}'.format( _project_number_from_id(project_id), project_id) try: key_id = jwt.get_unverified_header(iap_jwt).get('kid') if not key_id: raise AuthError('No key ID.') key = _get_iap_key(key_id) decoded_jwt = jwt.decode(iap_jwt, key, algorithms=['ES256'], issuer='https://cloud.google.com/iap', audience=expected_audience) return decoded_jwt['email'] except (jwt.exceptions.InvalidTokenError, requests.exceptions.RequestException) as e: raise AuthError('JWT assertion decode error: ' + str(e))
def get_or_create_service_account(project): """Get or create service account for the project.""" iam = _create_client('iam') project_id = utils.get_application_id() service_account_id = _service_account_id(project) service_account = get_service_account(iam, project_id, service_account_id) if service_account: logging.info('Using existing new service account for %s.', project) return service_account logging.info('Creating new service account for %s.', project) request = iam.projects().serviceAccounts().create( name='projects/' + project_id, body={ 'accountId': service_account_id, 'serviceAccount': { 'displayName': project, } }) return request.execute()
def start_if_needed(service): """Start Google Cloud Profiler if |USE_PYTHON_PROFILER| environment variable is set.""" if not environment.get_value('USE_PYTHON_PROFILER'): return True project_id = utils.get_application_id() service_with_platform = '{service}_{platform}'.format( service=service, platform=environment.platform().lower()) try: # Import the package here since it is only needed when profiler is enabled. # Also, this is supported on Linux only. import googlecloudprofiler googlecloudprofiler.start(project_id=project_id, service=service_with_platform) except Exception: logs.log_error('Failed to start the profiler for service %s.' % service_with_platform) return False return True
def get(self): """Handle a cron job.""" backup_bucket = local_config.Config( local_config.PROJECT_PATH).get('backup.bucket') if not backup_bucket: logs.log('No backup bucket is set, skipping.') return 'OK' kinds = [ kind for kind in ndb.Model._kind_map # pylint: disable=protected-access if (not kind.startswith('_') and kind not in EXCLUDED_MODELS) ] app_id = utils.get_application_id() timestamp = datetime.datetime.utcnow().strftime('%Y-%m-%d-%H:%M:%S') output_url_prefix = ( 'gs://{backup_bucket}/datastore-backups/{timestamp}'.format( backup_bucket=backup_bucket, timestamp=timestamp)) body = { 'output_url_prefix': output_url_prefix, 'entity_filter': { 'kinds': kinds } } try: request = _datastore_client().projects().export( projectId=app_id, body=body) response = request.execute() message = 'Datastore export succeeded.' status_code = 200 logs.log(message, response=response) except googleapiclient.errors.HttpError as e: message = 'Datastore export failed.' status_code = e.resp.status logs.log_error(message, error=str(e)) return (message, status_code, {'Content-Type': 'text/plain'})
def _initialize_monitored_resource(): """Monitored resources.""" global _monitored_resource _monitored_resource = monitoring_v3.types.MonitoredResource() # TODO(ochang): Use generic_node when that is available. _monitored_resource.type = 'gce_instance' # The project ID must be the same as the one we write metrics to, not the ID # where the instance lives. _monitored_resource.labels['project_id'] = utils.get_application_id() # Use bot name here instance as that's more useful to us. _monitored_resource.labels['instance_id'] = environment.get_value('BOT_NAME') if compute_metadata.is_gce(): # Returned in the form projects/{id}/zones/{zone} zone = compute_metadata.get('instance/zone').split('/')[-1] _monitored_resource.labels['zone'] = zone else: # Default zone for instances not on GCE. _monitored_resource.labels['zone'] = 'us-central1-f'
def _deployment_bucket_name(self): """Deployment bucket name.""" return '{project}-deployment'.format( project=utils.get_application_id())
def internal_network_domain(): """Return the internal network domain.""" return '.c.%s.internal' % utils.get_application_id()
def test_app_id_with_domain_and_partition(self): """Test app id with domain and partition separator.""" os.environ['APPLICATION_ID'] = 's~company:app_id' self.assertEqual('company:app_id', utils.get_application_id())
def test_app_id_with_partition(self): """Test app id with partition separator, but no domain separator.""" os.environ['APPLICATION_ID'] = 'dev~app_id' self.assertEqual('app_id', utils.get_application_id())
def test_simple_app_id(self): """Test simple app id without domain or partition separator.""" os.environ['APPLICATION_ID'] = 'app_id' self.assertEqual('app_id', utils.get_application_id())
def __init__(self, dataset_id=None, table_id=None): self.project_id = utils.get_application_id() self.dataset_id = dataset_id self.table_id = table_id self.client = get_api_client()
def test_no_app_id(self): """Test with no app id set in environment.""" del os.environ['APPLICATION_ID'] self.assertEqual(None, utils.get_application_id())