def get_buckets(self, project_id):
"""Gets all GCS buckets for a project.
Args:
project_id (int): The project id for a GCP project.
Returns:
dict: If successful, this function returns a dictionary for the
instances in the project.
{
"kind": "storage#buckets",
"nextPageToken": string,
"items": [
buckets Resource
]
}
Raises:
ApiExecutionError: ApiExecutionError is raised if the call to the
GCP ClodSQL API fails
"""
buckets_api = self.service.buckets()
try:
buckets_request = buckets_api.list(project=project_id,
projection='full')
buckets = buckets_request.execute()
return buckets
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.error(api_errors.ApiExecutionError(project_id, e))
# TODO: pass in "buckets" as resource_name variable
raise api_errors.ApiExecutionError('buckets', e)
def get_clusters(self, project_id, zone='-'):
"""Gets the clusters and their node pools for a project and zone.
If zone is not specified, it lists clusters for all zones in the
project.
Args:
project_id (int): The project id for a GCP project.
zone (str): Name of the zone to get the configuration for. Use
'-' to return clusters from all zones.
Returns:
list: A list of Cluster dicts.
https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.zones.clusters#Cluster
[{"name": "cluster-1", ...}
{"name": "cluster-2", ...},
{...}]
Raises:
ApiExecutionError: ApiExecutionError is raised if the call to the
GCP API fails
"""
try:
repository = self.repository.projects_zones_clusters
results = repository.list(project_id, zone=zone)
return api_helpers.flatten_list_results(results, 'clusters')
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(project_id, e))
raise api_errors.ApiExecutionError(project_id, e)
def get_instances(self, project_id):
"""Gets all CloudSQL instances for a project.
Args:
project_id (int): The project id for a GCP project.
Returns:
dict: If successful, this function returns a dictionary for the
instances in the project.
{
"kind": "sql#instancesList",
"nextPageToken": string,
"items": [
instances Resource
]
}
Raises:
ApiExecutionError: ApiExecutionError is raised if the call to the
GCP ClodSQL API fails
"""
instances_api = self.service.instances()
try:
instances_request = instances_api.list(project=project_id)
instances = self._execute(instances_request, self.rate_limiter)
return instances
except (HttpError, HttpLib2Error) as e:
LOGGER.error(api_errors.ApiExecutionError(project_id, e))
raise api_errors.ApiExecutionError('instances', e)
def get_billing_info(self, project_id):
"""Gets the biling information for a project.
Args:
project_id (int): The project id for a GCP project.
Returns:
dict: A ProjectBillingInfo resource.
https://cloud.google.com/billing/reference/rest/v1/ProjectBillingInfo
{
"name": string,
"projectId": string,
"billingAccountName": string,
"billingEnabled": boolean,
}
Raises:
ApiExecutionError: ApiExecutionError is raised if the call to the
GCP ClodSQL API fails
"""
try:
name = self.repository.projects.get_name(project_id)
return self.repository.projects.get_billing_info(name)
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(project_id, e))
raise api_errors.ApiExecutionError('billing_info', e)
def get_instances(self, project_id):
"""Gets all CloudSQL instances for a project.
Args:
project_id (int): The project id for a GCP project.
Returns:
list: A list of database Instance resource dicts for a project_id.
https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances
[{"kind": "sql#instance", "name": "sql_instance1", ...}
{"kind": "sql#instance", "name": "sql_instance2", ...},
{...}]
Raises:
ApiExecutionError: ApiExecutionError is raised if the call to the
GCP ClodSQL API fails
"""
try:
paged_results = self.repository.instances.list(project_id)
return api_helpers.flatten_list_results(paged_results, 'items')
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(project_id, e))
raise api_errors.ApiExecutionError('instances', e)
def get_service_accounts(self, project_id):
"""Get Service Accounts associated with a project.
Args:
project_id (str): The project ID to get Service Accounts for.
Yields:
dict: Service account associated with the project.
"""
endpoint = self.service.projects().serviceAccounts().list
project_name = 'projects/{}'.format(project_id)
next_token = ''
while True:
api_call = endpoint(name=project_name, pageToken=next_token)
try:
result = api_call.execute()
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.error(api_errors.ApiExecutionError(project_name, e))
raise api_errors.ApiExecutionError('serviceAccounts', e)
# Does the result have any objects listed?
if 'accounts' not in result:
break
# Yield objects
for item in result['accounts']:
yield item
# Are we finished?
if 'nextPageToken' not in result:
break
next_token = result['nextPageToken']
def get_service_account_keys(self, name, key_type=None):
"""Get keys associated with the given Service Account.
Args:
name (str): The service account name to query, must be in the format
projects/{PROJECT_ID}/serviceAccounts/{SERVICE_ACCOUNT_EMAIL}
key_type (str): Optional, the key type to include in the results.
Can be None, USER_MANAGED or SYSTEM_MANAGED. Defaults to
returning all key types.
Returns:
list: List with a dict for each key associated with the account.
Raises:
ValueError: Raised if an invalid key_type is specified.
"""
try:
kwargs = {}
if key_type:
if key_type not in self.KEY_TYPES:
raise ValueError('Key type %s is not a valid key type.' %
key_type)
kwargs['keyTypes'] = key_type
results = self.repository.projects_serviceaccounts_keys.list(
name, **kwargs)
return api_helpers.flatten_list_results(results, 'keys')
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(name, e))
raise api_errors.ApiExecutionError('serviceAccountKeys', e)
def _build_paged_result(self, request, api_stub, rate_limiter):
"""Execute results and page through the results.
Use of this method requires the API having a .list_next() method.
Args:
request: GCP API client request object.
api_stub: The API stub used to build the request.
rate_limiter: An instance of RateLimiter to use.
Returns:
A list of API response objects (dict).
Raises:
api_errors.ApiExecutionError when there is no list_next() method
on the api_stub.
"""
if not hasattr(api_stub, 'list_next'):
raise api_errors.ApiExecutionError(
api_stub, 'No list_next() method.')
results = []
while request is not None:
try:
with rate_limiter:
response = self._execute(request)
results.append(response)
request = api_stub.list_next(request, response)
except (HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError(api_stub, e)
return results
def get_bucket_iam_policy(self, bucket):
"""Gets the IAM policy for a bucket.
Args:
bucket (str): The bucket to fetch the policy for.
Returns:
dict: The IAM policies for the bucket.
"""
try:
return self.repository.buckets.get_iam_policy(bucket)
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(bucket, e))
raise api_errors.ApiExecutionError('bucketIamPolicy', e)
def get_bucket_acls(self, bucket_name):
"""Gets acls for GCS bucket.
Args:
bucket_name: The name of the bucket.
Returns: ACL json for bucket
"""
bucket_access_controls_api = self.service.bucketAccessControls()
bucket_acl_request = bucket_access_controls_api.list(bucket=bucket_name)
try:
return bucket_acl_request.execute()
except (HttpError, HttpLib2Error) as e:
LOGGER.error(api_errors.ApiExecutionError(bucket_name, e))
# TODO: pass in "buckets" as resource_name variable
raise api_errors.ApiExecutionError('buckets', e)
def test_retrieve_dataset_project_map_raises(self):
self.pipeline.api_client.get_datasets_for_projectid.side_effect = (
api_errors.ApiExecutionError('', mock.MagicMock()))
self.assertEqual([],
self.pipeline._retrieve_dataset_project_map(
['1', '2']))
def get_organizations(self, resource_name):
"""Get organizations that this application has access to.
Args:
resource_name: String of the resource's type.
Yields:
An iterator of the response from the organizations API, which
contains is paginated and contains a list of organizations.
"""
organizations_api = self.service.organizations()
next_page_token = None
try:
while True:
req_body = {}
if next_page_token:
req_body['pageToken'] = next_page_token
request = organizations_api.search(body=req_body)
response = self._execute(request, self.rate_limiter)
yield response
next_page_token = response.get('nextPageToken')
if not next_page_token:
break
except (HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError(resource_name, e)
def get_org_iam_policies(self, resource_name, org_id):
"""Get all the iam policies of an org.
Args:
resource_name: String of the resource's name.
org_id: Integer of the org id.
Returns:
Organization IAM policy for given org_id.
https://cloud.google.com/resource-manager/reference/rest/Shared.Types/Policy
Raises:
ApiExecutionError: An error has occurred when executing the API.
"""
organizations_api = self.service.organizations()
resource_id = 'organizations/%s' % org_id
try:
request = organizations_api.getIamPolicy(resource=resource_id,
body={})
return {
'org_id': org_id,
'iam_policy': self._execute(request, self.rate_limiter)
}
except (HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError(resource_name, e)
def get_service_account_keys(self, name):
"""Get keys associated with the given Service Account.
Args:
name (str): The service account name to query, must be in the format
projects/{PROJECT_ID}/serviceAccounts/{SERVICE_ACCOUNT_EMAIL}
Returns:
list: List with a dict for each key associated with the account.
"""
try:
results = self.repository.projects_serviceaccounts_keys.list(name)
return api_helpers.flatten_list_results(results, 'keys')
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(name, e))
raise api_errors.ApiExecutionError('serviceAccountKeys', e)
def get_forwarding_rules(self, project_id, region=None):
"""Get the forwarding rules for a project.
If no region name is specified, use aggregatedList() to query for
forwarding rules in all regions.
Args:
project_id: The project id.
region: The region name.
Yield:
An iterator of forwarding rules for this project.
Raise:
api_errors.ApiExecutionError if API raises an error.
"""
forwarding_rules_api = self.service.forwardingRules()
if region:
list_request = forwarding_rules_api.list(project=project_id,
region=region)
list_next_request = forwarding_rules_api.list_next
else:
list_request = forwarding_rules_api.aggregatedList(
project=project_id)
list_next_request = forwarding_rules_api.aggregatedList_next
try:
while list_request is not None:
response = self._execute(list_request)
yield response
list_request = list_next_request(previous_request=list_request,
previous_response=response)
except (HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError('forwarding_rules', e)
def get_projects(self, resource_name, **filterargs):
"""Get all the projects this application has access to.
Args:
resource_name: String of the resource's type.
filterargs: Extra project filter args.
Yields:
An iterable of the projects.list() response.
https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#response-body
Raises:
ApiExecutionError: An error has occurred when executing the API.
"""
projects_api = self.service.projects()
project_filter = []
for filter_key in filterargs:
project_filter.append('%s:%s' %
(filter_key, filterargs[filter_key]))
request = projects_api.list(filter=' '.join(project_filter))
# TODO: Convert this over to _base_client._build_paged_result().
try:
while request is not None:
response = self._execute(request, self.rate_limiter)
yield response
request = projects_api.list_next(previous_request=request,
previous_response=response)
except (HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError(resource_name, e)
def get_service_account_iam_policy(self, name):
"""Get IAM policy associated with a service account.
Args:
name (str): The service account name to query, must be in the format
projects/{PROJECT_ID}/serviceAccounts/{SERVICE_ACCOUNT_EMAIL}
Returns:
dict: The IAM policies for the service account.
"""
try:
return self.repository.projects_serviceaccounts.get_iam_policy(
name)
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(name, e))
raise api_errors.ApiExecutionError('serviceAccountIamPolicy', e)
def test_retrieve_dataset_project_map_raises(self):
self.pipeline.api_client.get_datasets_for_projectid.side_effect = (
api_errors.ApiExecutionError('', mock.MagicMock())
)
with self.assertRaises(inventory_errors.LoadDataPipelineError):
self.pipeline._retrieve_dataset_project_map([''])
def test_retrieve_dataset_access_raises(self):
self.pipeline.api_client.get_dataset_access.side_effect = (
api_errors.ApiExecutionError('', mock.MagicMock())
)
with self.assertRaises(inventory_errors.LoadDataPipelineError):
self.pipeline._retrieve_dataset_access('1', '2')
def get_datasets_for_projectid(self, project_id):
"""Return BigQuery datasets stored in the requested project_id.
Args:
project_id (str): String representing the project id.
Returns:
list: A list of datasetReference objects for a given project_id.
[{'datasetId': 'dataset-id',
'projectId': 'project-id'},
{...}]
"""
try:
results = self.repository.datasets.list(
resource=project_id,
fields='datasets/datasetReference,nextPageToken',
all=True)
flattened = api_helpers.flatten_list_results(results, 'datasets')
except (errors.HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError(project_id, e)
datasets = [
result.get('datasetReference') for result in flattened
if 'datasetReference' in result
]
return datasets
def _flatten_aggregated_list_results(project_id, paged_results, item_key,
sort_key='name'):
"""Flatten results and handle exceptions.
Args:
project_id (str): The project id the results are for.
paged_results (list): A list of paged API response objects.
[{page 1 results}, {page 2 results}, {page 3 results}, ...]
item_key (str): The name of the key within the inner "items" lists
containing the objects of interest.
sort_key (str): The name of the key to sort the results by before
returning.
Returns:
list: A sorted list of items.
Raises:
ApiNotEnabledError: Raised if the API is not enabled for the project.
ApiExecutionError: Raised if there is another error while calling the
API method.
"""
try:
return sorted(
api_helpers.flatten_aggregated_list_results(paged_results,
item_key),
key=lambda d: d.get(sort_key, ''))
except (errors.HttpError, HttpLib2Error) as e:
api_not_enabled, details = _api_not_enabled(e)
if api_not_enabled:
raise api_errors.ApiNotEnabledError(details, e)
raise api_errors.ApiExecutionError(project_id, e)
def get_folders(self, resource_name, parent=None, show_deleted=False):
"""Find all folders that the authenticated account has access to.
If no parent is passed in, then all folders the caller has visibility
to are returned. This is significantly less efficient then listing by
parent.
Args:
resource_name (str): The resource type.
parent (str): Optional parent resource, either
'organizations/{org_id}' or 'folders/{folder_id}'.
show_deleted (bool): Determines if deleted folders should be
returned in the results.
Returns:
list: A list of Folder dicts as returned by the API.
Raises:
ApiExecutionError: An error has occurred when executing the API.
"""
if parent:
paged_results = self.repository.folders.list(
parent, showDeleted=show_deleted)
else:
query = ''
if not show_deleted:
query = 'lifecycleState=ACTIVE'
paged_results = self.repository.folders.search(query=query)
try:
return api_helpers.flatten_list_results(paged_results, 'folders')
except (errors.HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError(resource_name, e)
def get_folders(self, resource_name, **kwargs):
"""Find all folders Forseti can access.
Args:
kwargs: Extra args.
resource_name: The resource name. TODO: why include this?
Returns:
The folders API response.
Raises:
ApiExecutionError: An error has occurred when executing the API.
"""
folders_api = self.service.folders()
next_page_token = None
lifecycle_state_filter = kwargs.get('lifecycle_state')
try:
while True:
req_body = {}
if next_page_token:
req_body['pageToken'] = next_page_token
if lifecycle_state_filter:
req_body['lifecycleState'] = lifecycle_state_filter
request = folders_api.search(body=req_body)
response = self._execute(request, self.rate_limiter)
yield response
next_page_token = response.get('nextPageToken')
if not next_page_token:
break
except (HttpError, HttpLib2Error) as e:
raise api_errors.ApiExecutionError(resource_name, e)
def _build_paged_result(self,
request,
api_stub,
rate_limiter,
next_stub=None):
"""Execute results and page through the results.
Use of this method requires the API having a .list_next() method.
Args:
request (HttpRequest): GCP API client request object.
api_stub (object): The API stub used to build the request.
rate_limiter (RateLimiter): An instance of RateLimiter to use.
Will be None for APIs without any rate limits.
next_stub (object): The API stub used to get the next page
of results.
Returns:
list: A list of paged API response objects.
[{page 1 results}, {page 2 results}, {page 3 results}, ...]
Raises:
api_errors.ApiExecutionError: When there is no list_next() method
on the api_stub.
"""
if next_stub is None:
if not hasattr(api_stub, 'list_next'):
raise api_errors.ApiExecutionError(api_stub,
'No list_next() method.')
next_stub = api_stub.list_next
results = []
while request is not None:
try:
response = self._execute(request, rate_limiter)
results.append(response)
request = next_stub(request, response)
except api_errors.ApiNotEnabledError:
# If the API isn't enabled on the resource, there must
# not be any resources. So, just swallow the error:
# we're done!
break
except (errors.HttpError, httplib2.HttpLib2Error) as e:
raise api_errors.ApiExecutionError(api_stub, e)
return results
def test_retrieve_error_logged_when_api_error(self, mock_logger):
"""Test that LOGGER.error() is called when there is an API error."""
self.mock_dao.get_folders.return_value = [folder.Folder(self.fake_id)]
self.pipeline.api_client.get_folder_iam_policies.side_effect = (
api_errors.ApiExecutionError('11111', mock.MagicMock()))
results = self.pipeline._retrieve()
self.assertEqual([], results)
self.assertEqual(1, mock_logger.error.call_count)
def get_service_accounts(self, project_id):
"""Get Service Accounts associated with a project.
Args:
project_id (str): The project ID to get Service Accounts for.
Returns:
list: List of service accounts associated with the project.
"""
name = self.repository.projects_serviceaccounts.get_name(project_id)
try:
paged_results = self.repository.projects_serviceaccounts.list(name)
return api_helpers.flatten_list_results(paged_results, 'accounts')
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(name, e))
raise api_errors.ApiExecutionError('serviceAccounts', e)
def test_retrieve_errors_are_handled(self):
"""Test that errors are handled when retrieving."""
self.pipeline.api_client.get_groups.side_effect = (
api_errors.ApiExecutionError('11111', mock.MagicMock()))
with self.assertRaises(inventory_errors.LoadDataPipelineError):
self.pipeline._retrieve()
def test_retrieve_errors_are_handled(self):
"""Test that errors are handled when retrieving."""
self.pipeline.api_client.get_projects.side_effect = (
api_errors.ApiExecutionError('11111', mock.MagicMock()))
projects = self.pipeline._retrieve()
self.assertEqual(None, projects)
def get_serverconfig(self, project_id, zone=None, location=None):
"""Gets the serverconfig for a project and zone or location.
Requires either a zone or a location, if both are passed in, the
location is used instead of the zone.
Args:
project_id (int): The project id for a GCP project.
zone (str): Name of the zone to get the configuration for.
location (str): Name of the location to get the configuration for.
Returns:
dict: A serverconfig for a given Compute Engine zone.
https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/ServerConfig
{
"defaultClusterVersion": string,
"validNodeVersions": [
string
],
"defaultImageType": string,
"validImageTypes": [
string
],
"validMasterVersions": [
string
],
}
Raises:
ApiExecutionError: ApiExecutionError is raised if the call to the
GCP API fails
ValueError: Raised if neither zone nor location are passed in.
"""
try:
if location:
return self.repository.projects_locations.get_serverconfig(
project_id, location=location)
elif zone:
return self.repository.projects_zones.get_serverconfig(
project_id, zone=zone)
raise ValueError('get_serverconfig takes either zone or location, '
'got zone: %s, location: %s' % (zone, location))
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.warn(api_errors.ApiExecutionError(project_id, e))
raise api_errors.ApiExecutionError(project_id, e)
def get_service_account_keys(self, service_account_name):
"""Get keys associated with the given Service Account.
Args:
service_account_name (str): Name of the Service Account for which
to get keys.
Returns:
list: List with a dict for each key associated with the account.
"""
endpoint = self.service.projects().serviceAccounts().keys().list
api_call = endpoint(name=service_account_name)
try:
result = api_call.execute()
except (errors.HttpError, HttpLib2Error) as e:
LOGGER.error(api_errors.ApiExecutionError(service_account_name, e))
raise api_errors.ApiExecutionError('serviceAccountKeys', e)
return result
