def get_datasets_for_projectid(self, project_id):
"""Return BigQuery datasets stored in the requested project_id.
Args:
project_id (str): String representing the project id.
Returns:
list: A list of datasetReference objects for a given project_id.
[{'datasetId': 'dataset-id',
'projectId': 'project-id'},
{...}]
"""
try:
results = self.repository.datasets.list(
resource=project_id,
fields='datasets/datasetReference,nextPageToken',
all=True)
flattened = api_helpers.flatten_list_results(results, 'datasets')
except (errors.HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError(project_id, e)
datasets = [
result.get('datasetReference') for result in flattened
if 'datasetReference' in result
]
return datasets
def _flatten_list_results(project_id, paged_results, item_key):
"""Flatten results and handle exceptions.
Args:
project_id (str): The project id the results are for.
paged_results (list): A list of paged API response objects.
[{page 1 results}, {page 2 results}, {page 3 results}, ...]
item_key (str): The name of the key within the inner "items" lists
containing the objects of interest.
Returns:
list: A list of items.
Raises:
ApiNotEnabledError: Raised if the API is not enabled for the project.
ApiExecutionError: Raised if there is another error while calling the
API method.
"""
try:
return api_helpers.flatten_list_results(paged_results, item_key)
except (errors.HttpError, HttpLib2Error) as e:
api_not_enabled, details = _api_not_enabled(e)
if api_not_enabled:
raise api_errors.ApiNotEnabledError(details, e)
raise api_errors.ApiExecutionError(project_id, e)
def get_folders(self, resource_name, parent=None, show_deleted=False):
"""Find all folders that the authenticated account has access to.
If no parent is passed in, then all folders the caller has visibility
to are returned. This is significantly less efficient then listing by
parent.
Args:
resource_name (str): The resource type.
parent (str): Optional parent resource, either
'organizations/{org_id}' or 'folders/{folder_id}'.
show_deleted (bool): Determines if deleted folders should be
returned in the results.
Returns:
list: A list of Folder dicts as returned by the API.
Raises:
ApiExecutionError: An error has occurred when executing the API.
"""
if parent:
paged_results = self.repository.folders.list(
parent, showDeleted=show_deleted)
else:
query = ''
if not show_deleted:
query = 'lifecycleState=ACTIVE'
paged_results = self.repository.folders.search(query=query)
try:
return api_helpers.flatten_list_results(paged_results, 'folders')
except (errors.HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError(resource_name, e)
def get_instances(self, project_id):
"""Gets all CloudSQL instances for a project.
Args:
project_id (int): The project id for a GCP project.
Returns:
list: A list of database Instance resource dicts for a project_id.
https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances
[{"kind": "sql#instance", "name": "sql_instance1", ...}
{"kind": "sql#instance", "name": "sql_instance2", ...},
{...}]
Raises:
ApiExecutionError: ApiExecutionError is raised if the call to the
GCP ClodSQL API fails
"""
try:
paged_results = self.repository.instances.list(project_id)
return api_helpers.flatten_list_results(paged_results, 'items')
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(project_id, e))
raise api_errors.ApiExecutionError('instances', e)
def get_service_account_keys(self, name, key_type=None):
"""Get keys associated with the given Service Account.
Args:
name (str): The service account name to query, must be in the format
projects/{PROJECT_ID}/serviceAccounts/{SERVICE_ACCOUNT_EMAIL}
key_type (str): Optional, the key type to include in the results.
Can be None, USER_MANAGED or SYSTEM_MANAGED. Defaults to
returning all key types.
Returns:
list: List with a dict for each key associated with the account.
Raises:
ValueError: Raised if an invalid key_type is specified.
"""
try:
kwargs = {}
if key_type:
if key_type not in self.KEY_TYPES:
raise ValueError('Key type %s is not a valid key type.' %
key_type)
kwargs['keyTypes'] = key_type
results = self.repository.projects_serviceaccounts_keys.list(
name, **kwargs)
return api_helpers.flatten_list_results(results, 'keys')
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(name, e))
raise api_errors.ApiExecutionError('serviceAccountKeys', e)
def get_clusters(self, project_id, zone='-'):
"""Gets the clusters and their node pools for a project and zone.
If zone is not specified, it lists clusters for all zones in the
project.
Args:
project_id (int): The project id for a GCP project.
zone (str): Name of the zone to get the configuration for. Use
'-' to return clusters from all zones.
Returns:
list: A list of Cluster dicts.
https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.zones.clusters#Cluster
[{"name": "cluster-1", ...}
{"name": "cluster-2", ...},
{...}]
Raises:
ApiExecutionError: ApiExecutionError is raised if the call to the
GCP API fails
"""
try:
repository = self.repository.projects_zones_clusters
results = repository.list(project_id, zone=zone)
return api_helpers.flatten_list_results(results, 'clusters')
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(project_id, e))
raise api_errors.ApiExecutionError(project_id, e)
def get_organizations(self, resource_name):
"""Get organizations that the authenticated account has access to.
Args:
resource_name (str): The resource type.
Returns:
list: A list of Folder dicts as returned by the API.
"""
try:
paged_results = self.repository.organizations.search()
return api_helpers.flatten_list_results(paged_results,
'organizations')
except (errors.HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError(resource_name, e)
def list_services(self, project_id):
"""Lists services of a project.
Args:
project_id (str): The id of the project.
Returns:
list: A list of Service resource dicts for a project_id.
"""
try:
paged_results = self.repository.app_services.list(project_id)
return api_helpers.flatten_list_results(paged_results, 'services')
except (errors.HttpError, HttpLib2Error) as e:
if isinstance(e, errors.HttpError) and e.resp.status == 404:
return []
raise api_errors.ApiExecutionError(project_id, e)
def get_service_account_keys(self, name):
"""Get keys associated with the given Service Account.
Args:
name (str): The service account name to query, must be in the format
projects/{PROJECT_ID}/serviceAccounts/{SERVICE_ACCOUNT_EMAIL}
Returns:
list: List with a dict for each key associated with the account.
"""
try:
results = self.repository.projects_serviceaccounts_keys.list(name)
return api_helpers.flatten_list_results(results, 'keys')
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(name, e))
raise api_errors.ApiExecutionError('serviceAccountKeys', e)
def get_service_accounts(self, project_id):
"""Get Service Accounts associated with a project.
Args:
project_id (str): The project ID to get Service Accounts for.
Returns:
list: List of service accounts associated with the project.
"""
name = self.repository.projects_serviceaccounts.get_name(project_id)
try:
paged_results = self.repository.projects_serviceaccounts.list(name)
return api_helpers.flatten_list_results(paged_results, 'accounts')
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(name, e))
raise api_errors.ApiExecutionError('serviceAccounts', e)
def get_group_members(self, group_key):
"""Get all the members for specified groups.
Args:
group_key (str): The group's unique id assigned by the Admin API.
Returns:
list: A list of member objects from the API.
Raises:
api_errors.ApiExecutionError: If group member retrieval fails.
"""
try:
paged_results = self.repository.members.list(group_key)
return api_helpers.flatten_list_results(paged_results, 'members')
except (errors.HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError(group_key, e)
def list_versions(self, project_id, service_id):
"""Lists versions of a given service.
Args:
project_id (str): The id of the project.
service_id (str): The id of the service to query.
Returns:
list: A list of Version resource dicts for a given Service.
"""
try:
paged_results = self.repository.service_versions.list(
project_id, services_id=service_id)
return api_helpers.flatten_list_results(paged_results, 'versions')
except (errors.HttpError, HttpLib2Error) as e:
if isinstance(e, errors.HttpError) and e.resp.status == 404:
return []
raise api_errors.ApiExecutionError(project_id, e)
def get_objects(self, bucket):
"""Gets all GCS buckets for a project.
Args:
bucket (str): The bucket to list to objects in.
Returns:
list: a list of object resource dicts.
https://cloud.google.com/storage/docs/json_api/v1/objects
Raises:
ApiExecutionError: ApiExecutionError is raised if the call to the
GCP ClodSQL API fails
"""
try:
paged_results = self.repository.objects.list(bucket,
projection='full')
return api_helpers.flatten_list_results(paged_results, 'items')
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(bucket, e))
raise api_errors.ApiExecutionError('objects', e)
def get_buckets(self, project_id):
"""Gets all GCS buckets for a project.
Args:
project_id (int): The project id for a GCP project.
Returns:
list: a list of bucket resource dicts.
https://cloud.google.com/storage/docs/json_api/v1/buckets
Raises:
ApiExecutionError: ApiExecutionError is raised if the call to the
GCP ClodSQL API fails
"""
try:
paged_results = self.repository.buckets.list(project_id,
projection='full')
return api_helpers.flatten_list_results(paged_results, 'items')
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(project_id, e))
raise api_errors.ApiExecutionError('buckets', e)
def get_groups(self, customer_id='my_customer'):
"""Get all the groups for a given customer_id.
A note on customer_id='my_customer'. This is a magic string instead
of using the real customer id. See:
https://developers.google.com/admin-sdk/directory/v1/guides/manage-groups#get_all_domain_groups
Args:
customer_id (str): The customer id to scope the request to.
Returns:
list: A list of group objects returned from the API.
Raises:
api_errors.ApiExecutionError: If groups retrieval fails.
"""
try:
paged_results = self.repository.groups.list(customer=customer_id)
return api_helpers.flatten_list_results(paged_results, 'groups')
except (errors.HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError('groups', e)
def get_bigquery_projectids(self):
"""Request and page through bigquery projectids.
Returns:
list: A list of project_ids enabled for bigquery.
['project-id',
'project-id',
'...']
If there are no project_ids enabled for bigquery an empty list will
be returned.
"""
try:
results = self.repository.projects.list(
fields='nextPageToken,projects/id')
flattened = api_helpers.flatten_list_results(results, 'projects')
except (errors.HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError('bigquery', e)
project_ids = [
result.get('id') for result in flattened if 'id' in result
]
return project_ids
