def list_instances(self, zone_id=None, max_results=None, filter_str=None):
"""
Abstraction of instances().list() method with inbuilt iteration functionality. [https://cloud.google.com/compute/docs/reference/latest/instances/list]
:param zone_id: Zone name. If None, all Zones are iterated over and returned.
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_str: Check documentation link for more details.
:return: Generator for dictionary objects representing resources.
"""
if zone_id is None:
return_list = [
iterate_list(self._service.instances(),
'items',
max_results,
self._max_retries,
project=self.project_id,
zone=zone['name'],
filter=filter_str) for zone in self.list_zones()
]
return chain(*return_list)
else:
return iterate_list(self._service.instances(),
'items',
max_results,
self._max_retries,
project=self.project_id,
zone=zone_id,
filter=filter_str)
def list_objects(self,
bucket_name,
max_results=None,
prefix=None,
projection=None,
filter_exp=None):
"""
Abstraction of objects().list() method with inbuilt iteration functionality. [https://cloud.google.com/storage/docs/json_api/v1/objects/list]
:param bucket_name: Bucket identifier.
:type bucket_name: string
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param prefix: Pre-filter (on API call) results to objects whose names begin with this prefix.
:type prefix: string
:param projection: Set of properties to return.
:param filter_exp: Function that filters entries if filter_exp evaluates to True.
:type filter_exp: function
:return: List of dictionary objects representing object resources.
"""
return iterate_list(self._service.objects(),
'items',
max_results,
self._max_retries,
filter_exp,
bucket=bucket_name,
prefix=prefix,
projection=projection)
def list_objects(self, bucket_name, max_results=None, prefix=None, projection=None, filter_exp=None):
"""
Abstraction of objects().list() method with inbuilt iteration functionality. [https://cloud.google.com/storage/docs/json_api/v1/objects/list]
:param bucket_name: Bucket identifier.
:type bucket_name: string
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param prefix: Pre-filter (on API call) results to objects whose names begin with this prefix.
:type prefix: string
:param projection: Set of properties to return.
:param filter_exp: Function that filters entries if filter_exp evaluates to True.
:type filter_exp: function
:return: List of dictionary objects representing object resources.
"""
return iterate_list(
self._service.objects(),
'items',
max_results,
self._max_retries,
filter_exp,
bucket=bucket_name,
prefix=prefix,
projection=projection
)
def list_datasets(self,
project_id,
show_all=False,
max_results=None,
filter_exp=None):
"""
Abstraction of datasets().list() method with inbuilt iteration functionality. [https://cloud.google.com/bigquery/docs/reference/v2/datasets/list]
:param project_id: Unique project identifier.
:type project_id: string
:param show_all: Include hidden datasets generated when running queries on the UI.
:type show_all: boolean
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_exp: Function that filters entries if filter_exp evaluates to True.
:type filter_exp: function
:return: List of dictionary objects representing dataset resources.
"""
return iterate_list(self._service.datasets(),
'datasets',
max_results,
self._max_retries,
filter_exp,
projectId=project_id,
all=show_all)
def list_files(self, max_results=None, **kwargs):
"""
Abstraction of files().list() method with inbuilt iteration functionality. [https://developers.google.com/drive/v3/reference/files/list]
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:keyword fields: Available properties can be found here: https://developers.google.com/drive/v3/reference/about
:keyword spaces: A comma-separated list of spaces to query within the corpus. Supported values are 'drive', 'appDataFolder' and 'photos'.
:keyword q: A query for filtering the file results. Reference here: https://developers.google.com/drive/v3/web/search-parameters
:return: List of dictionary objects representing file resources.
"""
fields = kwargs.get('fields', None)
if isinstance(fields, list):
if 'nextPageToken' not in fields:
fields.append('nextPageToken')
fields = ', '.join(fields)
return iterate_list(self._service.files(),
'files',
max_results,
self._max_retries,
filter_exp=None,
fields=fields,
q=kwargs.get('q', None),
spaces=kwargs.get('spaces', None))
def list_tables(self,
project_id,
dataset_id,
max_results=None,
filter_exp=None):
"""
Abstraction of tables().list() method with inbuilt iteration functionality. [https://cloud.google.com/bigquery/docs/reference/v2/tables/list]
:param project_id: Unique project identifier.
:type project_id: string
:param dataset_id: Unique dataset identifier.
:type dataset_id: string
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_exp: Function that filters entries if filter_exp evaluates to True.
:type filter_exp: function
:return: List of dictionary objects representing table resources.
"""
return iterate_list(self._service.tables(),
'tables',
max_results,
self._max_retries,
filter_exp,
projectId=project_id,
datasetId=dataset_id)
def list_files(self, max_results=None, **kwargs):
"""
Abstraction of files().list() method with inbuilt iteration functionality. [https://developers.google.com/drive/v3/reference/files/list]
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:keyword orderBy: List of keys to sort by. Refer to documentation.
:keyword spaces: A comma-separated list of spaces to query within the corpus. Supported values are 'drive', 'appDataFolder' and 'photos'.
:keyword q: A query for filtering the file results. Reference here: https://developers.google.com/drive/v3/web/search-parameters
:return: List of dictionary objects representing file resources.
"""
order_by = kwargs.get('orderBy', None)
if isinstance(order_by, list):
order_by = ','.join(order_by)
return iterate_list(
self._service.files(),
'files',
max_results,
self._max_retries,
filter_exp=kwargs.get('filter_exp', None),
orderBy=order_by,
q=kwargs.get('q', None),
spaces=kwargs.get('spaces', None)
)
def list_projects(self, max_results=None, filter_exp=None):
"""
Abstraction of projects().list() method with inbuilt iteration functionality. [https://cloud.google.com/bigquery/docs/reference/v2/projects/list]
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_exp: Function that filters entries if filter_exp evaluates to True.
:type filter_exp: function
:return: List of dictionary objects representing project resources.
"""
return iterate_list(self._service.projects(), 'projects', max_results,
self._max_retries, filter_exp)
def list_jobs(self,
project_id,
state_filter=None,
show_all=False,
projection='full',
max_results=None,
earliest_date=None,
filter_exp=None):
"""
Abstraction of jobs().list() method with inbuilt iteration functionality. [https://cloud.google.com/bigquery/docs/reference/v2/jobs/list]
**Note** - All jobs are stored in BigQuery. Do set *max_results* or *earliest_date* to limit data returned.
:param project_id: Unique project identifier.
:type project_id: string
:param state_filter: Pre-filter API request for job state. Acceptable values are "done", "pending" and "running". [Equivalent API param: stateFilter]
:type state_filter: string
:param show_all: Whether to display jobs owned by all users in the project. [Equivalent API param: allUsers]
:type show_all: boolean
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param projection: Acceptable values are *'full'*, *'minimal'*. *'full'* includes job configuration.
:type projection: string
:param earliest_date: Only returns data after this date.
:type earliest_date: datetime object or string representation of datetime in %Y-%m-%d format.
:param filter_exp: Function that filters entries if filter_exp evaluates to True.
:type filter_exp: function
:return: List of dictionary objects representing job resources.
"""
if earliest_date is not None:
# bigquery timestamp is in milliseconds
earliest_date_timestamp = datetime_to_timestamp(
earliest_date) * 1000
break_condition = lambda x: int(x['statistics']['creationTime']
) < earliest_date_timestamp
else:
break_condition = None
return iterate_list(self._service.jobs(),
'jobs',
max_results,
self._max_retries,
filter_exp,
break_condition=break_condition,
projectId=project_id,
allUsers=show_all,
projection=projection,
stateFilter=state_filter)
def list_instances(self, zone_id=None, max_results=None, filter_str=None):
"""
Abstraction of instances().list() method with inbuilt iteration functionality. [https://cloud.google.com/compute/docs/reference/latest/instances/list]
:param zone_id: Zone name. If None, all Zones are iterated over and returned.
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_str: Check documentation link for more details.
:return: Generator for dictionary objects representing resources.
"""
if zone_id is None:
return_list = [
iterate_list(
self._service.instances(),
'items',
max_results,
self._max_retries,
project=self.project_id,
zone=zone['name'],
filter=filter_str
)
for zone in self.list_zones()
]
return chain(*return_list)
else:
return iterate_list(
self._service.instances(),
'items',
max_results,
self._max_retries,
project=self.project_id,
zone=zone_id,
filter=filter_str
)
def list_zones(self, max_results=None, filter_str=None):
"""
Abstraction of zones().list() method with inbuilt iteration functionality. [https://cloud.google.com/compute/docs/reference/latest/zones/list]
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_str: Check documentation link for more details.
:return: Generator for dictionary objects representing resources.
"""
return iterate_list(self._service.zones(),
'items',
max_results,
self._max_retries,
project=self.project_id,
filter=filter_str)
def list_projects(self, max_results=None, filter_exp=None):
"""
Abstraction of projects().list() method with inbuilt iteration functionality. [https://cloud.google.com/bigquery/docs/reference/v2/projects/list]
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_exp: Function that filters entries if filter_exp evaluates to True.
:type filter_exp: function
:return: List of dictionary objects representing project resources.
"""
return iterate_list(
self._service.projects(),
'projects',
max_results,
self._max_retries,
filter_exp
)
def list_zones(self, max_results=None, filter_str=None):
"""
Abstraction of zones().list() method with inbuilt iteration functionality. [https://cloud.google.com/compute/docs/reference/latest/zones/list]
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_str: Check documentation link for more details.
:return: Generator for dictionary objects representing resources.
"""
return iterate_list(
self._service.zones(),
'items',
max_results,
self._max_retries,
project=self.project_id,
filter=filter_str
)
def list_jobs(self, project_id, state_filter=None, show_all=False, projection='full', max_results=None, earliest_date=None, filter_exp=None):
"""
Abstraction of jobs().list() method with inbuilt iteration functionality. [https://cloud.google.com/bigquery/docs/reference/v2/jobs/list]
**Note** - All jobs are stored in BigQuery. Do set *max_results* or *earliest_date* to limit data returned.
:param project_id: Unique project identifier.
:type project_id: string
:param state_filter: Pre-filter API request for job state. Acceptable values are "done", "pending" and "running". [Equivalent API param: stateFilter]
:type state_filter: string
:param show_all: Whether to display jobs owned by all users in the project. [Equivalent API param: allUsers]
:type show_all: boolean
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param projection: Acceptable values are *'full'*, *'minimal'*. *'full'* includes job configuration.
:type projection: string
:param earliest_date: Only returns data after this date.
:type earliest_date: datetime object or string representation of datetime in %Y-%m-%d format.
:param filter_exp: Function that filters entries if filter_exp evaluates to True.
:type filter_exp: function
:return: List of dictionary objects representing job resources.
"""
if earliest_date is not None:
# bigquery timestamp is in milliseconds
earliest_date_timestamp = datetime_to_timestamp(earliest_date) * 1000
break_condition = lambda x: int(x['statistics']['creationTime']) < earliest_date_timestamp
else:
break_condition = None
return iterate_list(
self._service.jobs(),
'jobs',
max_results,
self._max_retries,
filter_exp,
break_condition=break_condition,
projectId=project_id,
allUsers=show_all,
projection=projection,
stateFilter=state_filter
)
def list_buckets(self, project_id, max_results=None, filter_exp=None):
"""
Abstraction of buckets().list() method with inbuilt iteration functionality. [https://cloud.google.com/storage/docs/json_api/v1/buckets/list]
:param project_id: Unique project identifier.
:type project_id: string
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_exp: Function that filters entries if filter_exp evaluates to True.
:type filter_exp: function
:return: List of dictionary objects representing bucket resources.
"""
return iterate_list(self._service.buckets(),
'items',
max_results,
self._max_retries,
filter_exp,
project=project_id)
def list_operations(self, max_results=None, filter=None):
"""
Abstraction of projects().regions().operations().list() method with inbuilt iteration functionality. [https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.operations/list]
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter: Query param [https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.operations/list#query-parameters]
:type filter: String
:return: List of dictionary objects representing operation resources.
"""
return iterate_list(
self._service.projects().regions().operations(),
'operations',
max_results,
self._max_retries,
name='projects/{project_id}/regions/{region}/operations'.format(project_id=self.project_id, region='global'),
filter=filter
)
def list_buckets(self, project_id, max_results=None, filter_exp=None):
"""
Abstraction of buckets().list() method with inbuilt iteration functionality. [https://cloud.google.com/storage/docs/json_api/v1/buckets/list]
:param project_id: Unique project identifier.
:type project_id: string
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_exp: Function that filters entries if filter_exp evaluates to True.
:type filter_exp: function
:return: List of dictionary objects representing bucket resources.
"""
return iterate_list(
self._service.buckets(),
'items',
max_results,
self._max_retries,
filter_exp,
project=project_id
)
def list_drafts(self, max_results=None, full_messages=True, **kwargs):
"""
Abstraction of users().drafts().list() method with inbuilt iteration functionality. [https://developers.google.com/gmail/api/v1/reference/users/drafts/list]
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param full_messages: Convenience toggle to call self.get_draft() for each message returned.
:type full_messages: boolean
:keyword q: A query for filtering the file results. Can be generated from gwrappy.gmail.utils.generate_q
:return: List of dictionary objects representing draft resources.
"""
kwargs['userId'] = 'me'
results = iterate_list(self._service.users().drafts(), 'drafts',
max_results, self._max_retries, **kwargs)
if full_messages:
results = [self.get_draft(x['id']) for x in results]
return results
def list_datasets(self, project_id, show_all=False, max_results=None, filter_exp=None):
"""
Abstraction of datasets().list() method with inbuilt iteration functionality. [https://cloud.google.com/bigquery/docs/reference/v2/datasets/list]
:param project_id: Unique project identifier.
:type project_id: string
:param show_all: Include hidden datasets generated when running queries on the UI.
:type show_all: boolean
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_exp: Function that filters entries if filter_exp evaluates to True.
:type filter_exp: function
:return: List of dictionary objects representing dataset resources.
"""
return iterate_list(
self._service.datasets(),
'datasets',
max_results,
self._max_retries,
filter_exp,
projectId=project_id,
all=show_all
)
def list_tables(self, project_id, dataset_id, max_results=None, filter_exp=None):
"""
Abstraction of tables().list() method with inbuilt iteration functionality. [https://cloud.google.com/bigquery/docs/reference/v2/tables/list]
:param project_id: Unique project identifier.
:type project_id: string
:param dataset_id: Unique dataset identifier.
:type dataset_id: string
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_exp: Function that filters entries if filter_exp evaluates to True.
:type filter_exp: function
:return: List of dictionary objects representing table resources.
"""
return iterate_list(
self._service.tables(),
'tables',
max_results,
self._max_retries,
filter_exp,
projectId=project_id,
datasetId=dataset_id
)
def list_jobs(self, cluster_name=None, job_state='ACTIVE', max_results=None, filter=None):
"""
Abstraction of projects().regions().jobs().list() method with inbuilt iteration functionality. [https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs/list]
:param cluster_name: Cluster name, if unset, will return jobs from all clusters.
:param job_state: Category of jobs to return. [https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs/list#JobStateMatcher]
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter: Query param [https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs/list#query-parameters]
:type filter: String
:return: List of dictionary objects representing cluster resources.
"""
return iterate_list(
self._service.projects().regions().jobs(),
'jobs',
max_results,
self._max_retries,
projectId=self.project_id,
region='global',
clusterName=cluster_name,
jobStateMatcher=job_state,
filter=filter
)
def list_operations(self, operation_type, location_id=None, max_results=None, filter_str=None):
"""
Choose between region or zone operations with operation_type.
Abstraction of zoneOperations()/regionOperations().list() method with inbuilt iteration functionality.
https://cloud.google.com/compute/docs/reference/latest/zoneOperations/list
https://cloud.google.com/compute/docs/reference/latest/regionOperations/list
:param operation_type: 'zone' or 'region' type operations.
:param location_id: Zone/Region name. If None, all Zones/Regions are iterated over and returned.
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_str: Check documentation link for more details.
:return: Generator for dictionary objects representing resources.
"""
assert operation_type in ('region', 'zone')
if location_id is None:
if operation_type == 'region':
return_list = [
iterate_list(
self._service.regionOperations(),
'items',
max_results,
self._max_retries,
project=self.project_id,
region=region['name'],
filter=filter_str
)
for region in self.list_regions()
]
else:
return_list = [
iterate_list(
self._service.zoneOperations(),
'items',
max_results,
self._max_retries,
project=self.project_id,
zone=zone['name'],
filter=filter_str
)
for zone in self.list_zones()
]
return chain(*return_list)
else:
if operation_type == 'region':
return iterate_list(
self._service.regionOperations(),
'items',
max_results,
self._max_retries,
project=self.project_id,
region=location_id,
filter=filter_str
)
else:
return iterate_list(
self._service.zoneOperations(),
'items',
max_results,
self._max_retries,
project=self.project_id,
zone=location_id,
filter=filter_str
)
def list_operations(self,
operation_type,
location_id=None,
max_results=None,
filter_str=None):
"""
Choose between region or zone operations with operation_type.
Abstraction of zoneOperations()/regionOperations().list() method with inbuilt iteration functionality.
https://cloud.google.com/compute/docs/reference/latest/zoneOperations/list
https://cloud.google.com/compute/docs/reference/latest/regionOperations/list
:param operation_type: 'zone' or 'region' type operations.
:param location_id: Zone/Region name. If None, all Zones/Regions are iterated over and returned.
:param max_results: If None, all results are iterated over and returned.
:type max_results: integer
:param filter_str: Check documentation link for more details.
:return: Generator for dictionary objects representing resources.
"""
assert operation_type in ('region', 'zone')
if location_id is None:
if operation_type == 'region':
return_list = [
iterate_list(self._service.regionOperations(),
'items',
max_results,
self._max_retries,
project=self.project_id,
region=region['name'],
filter=filter_str)
for region in self.list_regions()
]
else:
return_list = [
iterate_list(self._service.zoneOperations(),
'items',
max_results,
self._max_retries,
project=self.project_id,
zone=zone['name'],
filter=filter_str)
for zone in self.list_zones()
]
return chain(*return_list)
else:
if operation_type == 'region':
return iterate_list(self._service.regionOperations(),
'items',
max_results,
self._max_retries,
project=self.project_id,
region=location_id,
filter=filter_str)
else:
return iterate_list(self._service.zoneOperations(),
'items',
max_results,
self._max_retries,
project=self.project_id,
zone=location_id,
filter=filter_str)
