def auth_domain():
"""Get the auth domain."""
domain = local_config.ProjectConfig().get('firebase.auth_domain')
if domain:
return domain
return utils.get_application_id() + '.firebaseapp.com'
def set_service_account_roles(service_account):
"""Set roles for service account."""
project_id = utils.get_application_id()
resource_manager = _create_client('cloudresourcemanager')
request = resource_manager.projects().getIamPolicy(resource=project_id,
body={})
policy = request.execute()
# Set logging and metrics permissions.
policy_changed = False
policy_changed |= _add_service_account_role(policy,
'roles/logging.logWriter',
service_account['email'])
policy_changed |= _add_service_account_role(
policy, 'roles/monitoring.metricWriter', service_account['email'])
if not policy_changed:
return
request = resource_manager.projects().setIamPolicy(resource=project_id,
body={
'policy': policy,
})
request.execute()
def cleanup_pubsub_topics(project_names):
"""Delete old pubsub topics and subscriptions."""
client = pubsub.PubSubClient()
application_id = utils.get_application_id()
expected_topics = set()
for platform in PUBSUB_PLATFORMS:
expected_topics.update([
untrusted.queue_name(project, platform)
for project in project_names
])
pubsub_config = local_config.Config('pubsub.queues')
unmanaged_queues = [
queue['name'] for queue in pubsub_config.get('resources')
]
for topic in client.list_topics(pubsub.project_name(application_id)):
_, name = pubsub.parse_name(topic)
if (not name.startswith(tasks.JOBS_PREFIX)
and not name.startswith(tasks.HIGH_END_JOBS_PREFIX)):
# Some topic created by another service, ignore.
continue
if name in unmanaged_queues:
continue
if name in expected_topics:
continue
for subscription in client.list_topic_subscriptions(topic):
client.delete_subscription(subscription)
client.delete_topic(topic)
def run(self):
"""Run the flusher thread."""
create_time_series = _retry_wrap(_monitoring_v3_client.create_time_series)
project_path = _monitoring_v3_client.project_path(
utils.get_application_id())
while True:
try:
if self.stop_event.wait(FLUSH_INTERVAL_SECONDS):
return
time_series = []
end_time = time.time()
for metric, labels, start_time, value in _metrics_store.iter_values():
if (metric.metric_kind ==
monitoring_v3.enums.MetricDescriptor.MetricKind.GAUGE):
start_time = end_time
series = monitoring_v3.types.TimeSeries()
metric.monitoring_v3_time_series(series, labels, start_time, end_time,
value)
time_series.append(series)
if len(time_series) == MAX_TIME_SERIES_PER_CALL:
create_time_series(project_path, time_series)
time_series = []
if time_series:
create_time_series(project_path, time_series)
except Exception:
logs.log_error('Failed to flush metrics.')
def _table_name(self):
"""Return the table name for the query."""
app_id = utils.get_application_id()
dataset = dataset_name(self.fuzzer_or_engine_name)
return '`%s`.%s.%s' % (app_id, dataset, self.base_table)
def _create_storage_client_new():
"""Create a storage client."""
creds, project = credentials.get_default()
if not project:
project = utils.get_application_id()
return gcs.Client(project=project, credentials=creds)
def add_task(command, argument, job_type, queue=None, wait_time=None):
"""Add a new task to the job queue."""
# Old testcases may pass in queue=None explicitly,
# so we must check this here.
if not queue:
queue = default_queue()
if wait_time is None:
wait_time = random.randint(1, TASK_CREATION_WAIT_INTERVAL)
if job_type != 'none':
job = data_types.Job.query(data_types.Job.name == job_type).get()
if not job:
raise Error(f'Job {job_type} not found.')
if job.is_external():
external_tasks.add_external_task(command, argument, job)
return
# Add the task.
eta = utils.utcnow() + datetime.timedelta(seconds=wait_time)
task = Task(command, argument, job_type, eta=eta)
pubsub_client = pubsub.PubSubClient()
pubsub_client.publish(pubsub.topic_name(utils.get_application_id(), queue),
[task.to_pubsub_message()])
def get_regular_task(queue=None):
"""Get a regular task."""
if not queue:
queue = regular_queue()
pubsub_client = pubsub.PubSubClient()
application_id = utils.get_application_id()
while True:
messages = pubsub_client.pull_from_subscription(
pubsub.subscription_name(application_id, queue), max_messages=1)
if not messages:
return None
try:
task = PubSubTask(messages[0])
except KeyError:
logs.log_error('Received an invalid task, discarding...')
messages[0].ack()
continue
# Check that this task should be run now (past the ETA). Otherwise we defer
# its execution.
if not task.defer():
return task
def _load_data(self, bigquery, fuzzer):
"""Load yesterday's stats into BigQuery."""
project_id = utils.get_application_id()
yesterday = (self._utc_now().date() - datetime.timedelta(days=1))
date_string = yesterday.strftime('%Y%m%d')
timestamp = utils.utc_date_to_timestamp(yesterday)
dataset_id = fuzzer_stats.dataset_name(fuzzer)
if not self._create_dataset_if_needed(bigquery, dataset_id):
return
for kind in STATS_KINDS:
kind_name = kind.__name__
table_id = kind_name
if kind == fuzzer_stats.TestcaseRun:
schema = fuzzer_stats_schema.get(fuzzer)
else:
schema = kind.SCHEMA
if not self._create_table_if_needed(bigquery, dataset_id, table_id,
schema):
continue
gcs_path = fuzzer_stats.get_gcs_stats_path(kind_name, fuzzer,
timestamp)
load = {
'destinationTable': {
'projectId': project_id,
'tableId': table_id + '$' + date_string,
'datasetId': dataset_id,
},
'schemaUpdateOptions': [
'ALLOW_FIELD_ADDITION',
],
'sourceFormat': 'NEWLINE_DELIMITED_JSON',
'sourceUris': ['gs:/' + gcs_path + '*.json'],
'writeDisposition': 'WRITE_TRUNCATE',
}
if schema is not None:
load['schema'] = schema
job_body = {
'configuration': {
'load': load,
},
}
logs.log("Uploading job to BigQuery.", job_body=job_body)
request = bigquery.jobs().insert(projectId=project_id,
body=job_body)
response = request.execute()
# We cannot really check the response here, as the query might be still
# running, but having a BigQuery jobId in the log would make our life
# simpler if we ever have to manually check the status of the query.
# See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query.
logs.log('Response from BigQuery: %s' % response)
def _client():
"""Get or initialize the NDB client."""
global _ndb_client
global _initial_pid
if not _ndb_client:
with _ndb_client_lock:
if not _ndb_client:
_ndb_client = ndb.Client(project=utils.get_application_id())
_initial_pid = os.getpid()
return _ndb_client
def _create_dataset_if_needed(self, bigquery, dataset_id):
"""Create a new dataset if necessary."""
project_id = utils.get_application_id()
dataset_body = {
'datasetReference': {
'datasetId': dataset_id,
'projectId': project_id,
},
}
dataset_insert = bigquery.datasets().insert(projectId=project_id,
body=dataset_body)
return self._execute_insert_request(dataset_insert)
def create_pubsub_topics(project):
"""Create pubsub topics for tasks."""
for platform in PUBSUB_PLATFORMS:
name = untrusted.queue_name(project, platform)
client = pubsub.PubSubClient()
application_id = utils.get_application_id()
topic_name = pubsub.topic_name(application_id, name)
if client.get_topic(topic_name) is None:
client.create_topic(topic_name)
subscription_name = pubsub.subscription_name(application_id, name)
if client.get_subscription(subscription_name) is None:
client.create_subscription(subscription_name, topic_name)
def _create_table_if_needed(self, bigquery, dataset_id, table_id):
"""Create a new table if needed."""
project_id = utils.get_application_id()
table_body = {
'tableReference': {
'datasetId': dataset_id,
'projectId': project_id,
'tableId': table_id,
},
'timePartitioning': {
'type': 'DAY',
},
}
table_insert = bigquery.tables().insert(projectId=project_id,
datasetId=dataset_id,
body=table_body)
return self._execute_insert_request(table_insert)
def create_bucket(self, name, object_lifecycle, cors):
"""Create a new bucket."""
project_id = utils.get_application_id()
request_body = {'name': name}
if object_lifecycle:
request_body['lifecycle'] = object_lifecycle
if cors:
request_body['cors'] = cors
client = create_discovery_storage_client()
try:
client.buckets().insert(project=project_id, body=request_body).execute()
except HttpError as e:
logs.log_warn('Failed to create bucket %s: %s' % (name, e))
raise
return True
def setup_metrics(non_dry_run):
"""Set up metrics."""
client = monitoring_v3.MetricServiceClient()
project_name = utils.get_application_id()
project_path = client.project_path(project_name)
for name in dir(monitoring_metrics):
metric = getattr(monitoring_metrics, name)
if not isinstance(metric, monitor.Metric):
continue
descriptor = monitoring_v3.types.MetricDescriptor()
metric.monitoring_v3_metric_descriptor(descriptor)
if non_dry_run:
print('Creating metric', descriptor)
client.create_metric_descriptor(project_path, descriptor)
else:
print('Skip creating metric', descriptor, '(dry-run mode)')
def setUp(self):
helpers.patch_environ(self)
helpers.patch(self, [
'clusterfuzz._internal.bot.tasks.minimize_task._run_libfuzzer_testcase',
'clusterfuzz._internal.bot.tasks.minimize_task._run_libfuzzer_tool',
])
test_utils.setup_pubsub(utils.get_application_id())
environment.set_value('APP_ARGS', '%TESTCASE% fuzz_target')
environment.set_value('APP_DIR', '/libfuzzer')
environment.set_value('APP_NAME', '')
environment.set_value('APP_PATH', '')
environment.set_value('BOT_TMPDIR', '/bot_tmpdir')
environment.set_value('CRASH_STACKTRACES_DIR', '/crash_stacks')
environment.set_value('FUZZER_DIR', '/fuzzer_dir')
environment.set_value('INPUT_DIR', '/input_dir')
environment.set_value('JOB_NAME', 'libfuzzer_asan_test')
environment.set_value('USER_PROFILE_IN_MEMORY', True)
def _validate_iap_jwt(iap_jwt):
"""Validate JWT assertion."""
project_id = utils.get_application_id()
expected_audience = '/projects/{}/apps/{}'.format(
_project_number_from_id(project_id), project_id)
try:
key_id = jwt.get_unverified_header(iap_jwt).get('kid')
if not key_id:
raise AuthError('No key ID.')
key = _get_iap_key(key_id)
decoded_jwt = jwt.decode(iap_jwt,
key,
algorithms=['ES256'],
issuer='https://cloud.google.com/iap',
audience=expected_audience)
return decoded_jwt['email']
except (jwt.exceptions.InvalidTokenError,
requests.exceptions.RequestException) as e:
raise AuthError('JWT assertion decode error: ' + str(e))
def get_or_create_service_account(project):
"""Get or create service account for the project."""
iam = _create_client('iam')
project_id = utils.get_application_id()
service_account_id = _service_account_id(project)
service_account = get_service_account(iam, project_id, service_account_id)
if service_account:
logging.info('Using existing new service account for %s.', project)
return service_account
logging.info('Creating new service account for %s.', project)
request = iam.projects().serviceAccounts().create(
name='projects/' + project_id,
body={
'accountId': service_account_id,
'serviceAccount': {
'displayName': project,
}
})
return request.execute()
def start_if_needed(service):
"""Start Google Cloud Profiler if |USE_PYTHON_PROFILER| environment variable
is set."""
if not environment.get_value('USE_PYTHON_PROFILER'):
return True
project_id = utils.get_application_id()
service_with_platform = '{service}_{platform}'.format(
service=service, platform=environment.platform().lower())
try:
# Import the package here since it is only needed when profiler is enabled.
# Also, this is supported on Linux only.
import googlecloudprofiler
googlecloudprofiler.start(project_id=project_id,
service=service_with_platform)
except Exception:
logs.log_error('Failed to start the profiler for service %s.' %
service_with_platform)
return False
return True
def get(self):
"""Handle a cron job."""
backup_bucket = local_config.Config(
local_config.PROJECT_PATH).get('backup.bucket')
if not backup_bucket:
logs.log('No backup bucket is set, skipping.')
return 'OK'
kinds = [
kind for kind in ndb.Model._kind_map # pylint: disable=protected-access
if (not kind.startswith('_') and kind not in EXCLUDED_MODELS)
]
app_id = utils.get_application_id()
timestamp = datetime.datetime.utcnow().strftime('%Y-%m-%d-%H:%M:%S')
output_url_prefix = (
'gs://{backup_bucket}/datastore-backups/{timestamp}'.format(
backup_bucket=backup_bucket, timestamp=timestamp))
body = {
'output_url_prefix': output_url_prefix,
'entity_filter': {
'kinds': kinds
}
}
try:
request = _datastore_client().projects().export(
projectId=app_id, body=body)
response = request.execute()
message = 'Datastore export succeeded.'
status_code = 200
logs.log(message, response=response)
except googleapiclient.errors.HttpError as e:
message = 'Datastore export failed.'
status_code = e.resp.status
logs.log_error(message, error=str(e))
return (message, status_code, {'Content-Type': 'text/plain'})
def _initialize_monitored_resource():
"""Monitored resources."""
global _monitored_resource
_monitored_resource = monitoring_v3.types.MonitoredResource()
# TODO(ochang): Use generic_node when that is available.
_monitored_resource.type = 'gce_instance'
# The project ID must be the same as the one we write metrics to, not the ID
# where the instance lives.
_monitored_resource.labels['project_id'] = utils.get_application_id()
# Use bot name here instance as that's more useful to us.
_monitored_resource.labels['instance_id'] = environment.get_value('BOT_NAME')
if compute_metadata.is_gce():
# Returned in the form projects/{id}/zones/{zone}
zone = compute_metadata.get('instance/zone').split('/')[-1]
_monitored_resource.labels['zone'] = zone
else:
# Default zone for instances not on GCE.
_monitored_resource.labels['zone'] = 'us-central1-f'
def _deployment_bucket_name(self):
"""Deployment bucket name."""
return '{project}-deployment'.format(
project=utils.get_application_id())
def internal_network_domain():
"""Return the internal network domain."""
return '.c.%s.internal' % utils.get_application_id()
def test_app_id_with_domain_and_partition(self):
"""Test app id with domain and partition separator."""
os.environ['APPLICATION_ID'] = 's~company:app_id'
self.assertEqual('company:app_id', utils.get_application_id())
def test_app_id_with_partition(self):
"""Test app id with partition separator, but no domain separator."""
os.environ['APPLICATION_ID'] = 'dev~app_id'
self.assertEqual('app_id', utils.get_application_id())
def test_simple_app_id(self):
"""Test simple app id without domain or partition separator."""
os.environ['APPLICATION_ID'] = 'app_id'
self.assertEqual('app_id', utils.get_application_id())
def __init__(self, dataset_id=None, table_id=None):
self.project_id = utils.get_application_id()
self.dataset_id = dataset_id
self.table_id = table_id
self.client = get_api_client()
def test_no_app_id(self):
"""Test with no app id set in environment."""
del os.environ['APPLICATION_ID']
self.assertEqual(None, utils.get_application_id())
