def list_zones(self, max_results=None, page_token=None):
"""List zones for the project associated with this client.
See
https://cloud.google.com/dns/api/v1/managedZones/list
:type max_results: int
:param max_results: maximum number of zones to return, If not
passed, defaults to a value set by the API.
:type page_token: str
:param page_token: opaque marker for the next "page" of zones. If
not passed, the API will return the first page of
zones.
:rtype: :class:`~google.api_core.page_iterator.Iterator`
:returns: Iterator of :class:`~google.cloud.dns.zone.ManagedZone`
belonging to this project.
"""
path = "/projects/%s/managedZones" % (self.project,)
return page_iterator.HTTPIterator(
client=self,
api_request=self._connection.api_request,
path=path,
item_to_value=_item_to_zone,
items_key="managedZones",
page_token=page_token,
max_results=max_results,
)
def list_environments(self,
location,
max_results=None,
page_token=None,
prefix=None,
projection="noAcl",
fields=None,
project=None):
"""Get all environments in the project associated to the client.
"""
if project is None:
project = self.project
if project is None:
raise ValueError(
"Client project not set: pass an explicit project.")
return page_iterator.HTTPIterator(
client=self,
api_request=self._connection.api_request,
path=f'/projects/{project}/locations/{location}/environments',
items_key='environments',
item_to_value=_item_to_environment,
page_token=page_token,
max_results=max_results)
def list_sub_prefixes(self, bucket: str, prefix: str) -> None:
"""
List sub folders in folder specified by prefix
SO link: https://stackoverflow.com/a/59008580
:param bucket: GCS bucket name as string
:param prefix: path to directory to look into e.g. a/b/c/
:return: list of strings of sub-directories e.g. [a/b/c/v1/, a/b/c/v2/]
"""
if not prefix.endswith('/'):
prefix += '/'
extra_params: dict = {
'projection': 'noAcl',
'prefix': prefix,
'delimiter': '/'
}
path: str = f'/b/{bucket}/o'
pg_iterator = page_iterator.HTTPIterator(
client=self,
api_request=self._connection.api_request,
path=path,
items_key='prefixes',
item_to_value=lambda _, x: x,
extra_params=extra_params,
)
return list(pg_iterator)
def list_zones(self, max_results=None, page_token=None):
"""List zones for the project associated with this client.
See
https://cloud.google.com/dns/api/v1/managedZones/list
:type max_results: int
:param max_results: maximum number of zones to return, If not
passed, defaults to a value set by the API.
:type page_token: str
:param page_token: Optional. If present, return the next batch of
zones, using the value, which must correspond to the
``nextPageToken`` value returned in the previous response.
Deprecated: use the ``pages`` property of the returned iterator
instead of manually passing the token.
:rtype: :class:`~google.api_core.page_iterator.Iterator`
:returns: Iterator of :class:`~google.cloud.dns.zone.ManagedZone`
belonging to this project.
"""
path = "/projects/%s/managedZones" % (self.project,)
return page_iterator.HTTPIterator(
client=self,
api_request=self._connection.api_request,
path=path,
item_to_value=_item_to_zone,
items_key="managedZones",
page_token=page_token,
max_results=max_results,
)
def test_iterate(self):
path = "/foo"
item1 = {"name": "1"}
item2 = {"name": "2"}
api_request = mock.Mock(return_value={"items": [item1, item2]})
iterator = page_iterator.HTTPIterator(
mock.sentinel.client,
api_request,
path=path,
item_to_value=page_iterator._item_to_value_identity,
)
assert iterator.num_results == 0
items_iter = iter(iterator)
val1 = six.next(items_iter)
assert val1 == item1
assert iterator.num_results == 1
val2 = six.next(items_iter)
assert val2 == item2
assert iterator.num_results == 2
with pytest.raises(StopIteration):
six.next(items_iter)
api_request.assert_called_once_with(method="GET",
path=path,
query_params={})
def list_tables(self, max_results=None, page_token=None):
"""List tables for the project associated with this client.
See
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/list
:type max_results: int
:param max_results: (Optional) Maximum number of tables to return.
If not passed, defaults to a value set by the API.
:type page_token: str
:param page_token: (Optional) Opaque marker for the next "page" of
datasets. If not passed, the API will return the
first page of datasets.
:rtype: :class:`~google.api_core.page_iterator.Iterator`
:returns: Iterator of :class:`~google.cloud.bigquery.table.Table`
contained within the current dataset.
"""
path = '/projects/%s/datasets/%s/tables' % (self.project, self.name)
result = page_iterator.HTTPIterator(
client=self._client,
api_request=self._client._connection.api_request,
path=path,
item_to_value=_item_to_table,
items_key='tables',
page_token=page_token,
max_results=max_results)
result.dataset = self
return result
def list_notifications(self, client=None):
"""List Pub / Sub notifications for this bucket.
See:
https://cloud.google.com/storage/docs/json_api/v1/notifications/list
If :attr:`user_project` is set, bills the API request to that project.
:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the current bucket.
:rtype: list of :class:`.BucketNotification`
:returns: notification instances
"""
client = self._require_client(client)
path = self.path + '/notificationConfigs'
iterator = page_iterator.HTTPIterator(
client=client,
api_request=client._connection.api_request,
path=path,
item_to_value=_item_to_notification)
iterator.bucket = self
return iterator
def list_sub_prefixes(client, bucket, prefix):
"""
List sub folders in folder specified by prefix
SO link: https://stackoverflow.com/a/59008580
:param client: GCS client
:param bucket: GCS bucket name as string
:param prefix: path to directory to look into e.g. a/b/c/
:return: list of strings of sub-directories e.g. [a/b/c/v1/, a/b/c/v2/]
"""
if not prefix.endswith('/'):
prefix += '/'
extra_params = {"projection": "noAcl", "prefix": prefix, "delimiter": '/'}
path = f"/b/{bucket}/o"
pg_iterator = page_iterator.HTTPIterator(
client=client,
api_request=client._connection.api_request,
path=path,
items_key='prefixes',
item_to_value=lambda _, x: x,
extra_params=extra_params,
)
return list(pg_iterator)
def list_metrics(self, project, *, page_size=None, page_token=None):
"""List metrics for the project associated with this client.
See
https://cloud.google.com/logging/docs/reference/v2/rest/v2/projects.metrics/list
Args:
page_size (Optional[int]): The maximum number of sinks in each
page of results from this request. Non-positive values are ignored. Defaults to a
sensible value set by the API.
page_token (Optional[str]): If present, return the next batch of sinks, using the
value, which must correspond to the ``nextPageToken`` value
returned in the previous response. Deprecated: use the ``pages``
property ofthe returned iterator instead of manually passing the
token.
Returns:
Iterator[google.cloud.logging_v2.metric.Metric]
"""
extra_params = {}
if page_size is not None:
extra_params["pageSize"] = page_size
path = f"/projects/{project}/metrics"
return page_iterator.HTTPIterator(
client=self._client,
api_request=self._client._connection.api_request,
path=path,
item_to_value=_item_to_metric,
items_key="metrics",
page_token=page_token,
extra_params=extra_params,
)
def list_projects(self, max_results=None, page_token=None):
"""List projects for the project associated with this client.
See
https://cloud.google.com/bigquery/docs/reference/rest/v2/projects/list
:type max_results: int
:param max_results: maximum number of projects to return, If not
passed, defaults to a value set by the API.
:type page_token: str
:param page_token: opaque marker for the next "page" of projects. If
not passed, the API will return the first page of
projects.
:rtype: :class:`~google.api_core.page_iterator.Iterator`
:returns: Iterator of :class:`~google.cloud.bigquery.client.Project`
accessible to the current client.
"""
return page_iterator.HTTPIterator(
client=self,
api_request=self._connection.api_request,
path='/projects',
item_to_value=_item_to_project,
items_key='projects',
page_token=page_token,
max_results=max_results)
def test__get_query_params_no_token(self):
iterator = page_iterator.HTTPIterator(mock.sentinel.client,
mock.sentinel.api_request,
mock.sentinel.path,
mock.sentinel.item_to_value)
assert iterator._get_query_params() == {}
def test__has_next_page_new(self):
iterator = page_iterator.HTTPIterator(mock.sentinel.client,
mock.sentinel.api_request,
mock.sentinel.path,
mock.sentinel.item_to_value)
# The iterator should *always* indicate that it has a next page
# when created so that it can fetch the initial page.
assert iterator._has_next_page()
def test_constructor_w_extra_param_collision(self):
extra_params = {'pageToken': 'val'}
with pytest.raises(ValueError):
page_iterator.HTTPIterator(mock.sentinel.client,
mock.sentinel.api_request,
mock.sentinel.path,
mock.sentinel.item_to_value,
extra_params=extra_params)
def list_hmac_keys(
self,
max_results=None,
service_account_email=None,
show_deleted_keys=None,
project_id=None,
user_project=None,
):
"""List HMAC keys for a project.
:type max_results: int
:param max_results:
(Optional) max number of keys to return in a given page.
:type service_account_email: str
:param service_account_email:
(Optional) limit keys to those created by the given service account.
:type show_deleted_keys: bool
:param show_deleted_keys:
(Optional) included deleted keys in the list. Default is to
exclude them.
:type project_id: str
:param project_id: (Optional) explicit project ID for the key.
Defaults to the client's project.
:type user_project: str
:param user_project: (Optional) This parameter is currently ignored.
:rtype:
Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
:returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
"""
if project_id is None:
project_id = self.project
path = "/projects/{}/hmacKeys".format(project_id)
extra_params = {}
if service_account_email is not None:
extra_params["serviceAccountEmail"] = service_account_email
if show_deleted_keys is not None:
extra_params["showDeletedKeys"] = show_deleted_keys
if user_project is not None:
extra_params["userProject"] = user_project
return page_iterator.HTTPIterator(
client=self,
api_request=self._connection.api_request,
path=path,
item_to_value=_item_to_hmac_key_metadata,
max_results=max_results,
extra_params=extra_params,
)
def test__get_query_params_extra_params(self):
extra_params = {'key': 'val'}
iterator = page_iterator.HTTPIterator(mock.sentinel.client,
mock.sentinel.api_request,
mock.sentinel.path,
mock.sentinel.item_to_value,
extra_params=extra_params)
assert iterator._get_query_params() == extra_params
def test__get_next_page_bad_http_method(self):
iterator = page_iterator.HTTPIterator(mock.sentinel.client,
mock.sentinel.api_request,
mock.sentinel.path,
mock.sentinel.item_to_value)
iterator._HTTP_METHOD = 'NOT-A-VERB'
with pytest.raises(ValueError):
iterator._get_next_page_response()
def test_page_size_items(self, page_size, max_results, pages):
path = "/foo"
NITEMS = 10
n = [0] # blast you python 2!
def api_request(*args, **kw):
assert not args
query_params = dict(maxResults=(page_size if max_results is None
else min(page_size, max_results -
n[0])))
if n[0]:
query_params.update(pageToken='test')
assert kw == {
'method': 'GET',
'path': '/foo',
'query_params': query_params
}
n_items = min(kw['query_params']['maxResults'], NITEMS - n[0])
items = [dict(name=str(i + n[0])) for i in range(n_items)]
n[0] += n_items
result = dict(items=items)
if n[0] < NITEMS:
result.update(nextPageToken='test')
return result
iterator = page_iterator.HTTPIterator(
mock.sentinel.client,
api_request,
path=path,
item_to_value=page_iterator._item_to_value_identity,
page_size=page_size,
max_results=max_results,
)
assert iterator.num_results == 0
n_results = max_results if max_results is not None else NITEMS
if pages:
items_iter = iter(iterator.pages)
npages = int(math.ceil(float(n_results) / page_size))
for ipage in range(npages):
assert (list(six.next(items_iter)) == [
dict(name=str(i)) for i in range(
ipage * page_size,
min((ipage + 1) * page_size, n_results),
)
])
else:
items_iter = iter(iterator)
for i in range(n_results):
assert six.next(items_iter) == dict(name=str(i))
assert iterator.num_results == i + 1
with pytest.raises(StopIteration):
six.next(items_iter)
def models(self, max_results=None, page_token=None):
model_client = ModelClient(self)
return page_iterator.HTTPIterator(
client=self,
api_request=self._connection.api_request,
path=model_client.path,
item_to_value=ModelClient._item_to_model,
items_key="models",
page_token=page_token,
max_results=max_results)
def test__get_query_params_w_token(self):
iterator = page_iterator.HTTPIterator(mock.sentinel.client,
mock.sentinel.api_request,
mock.sentinel.path,
mock.sentinel.item_to_value)
iterator.next_page_token = 'token'
assert iterator._get_query_params() == {
'pageToken': iterator.next_page_token
}
def fetch_data(self, max_results=None, page_token=None, client=None):
"""API call: fetch the table data via a GET request
See
https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/list
.. note::
This method assumes that its instance's ``schema`` attribute is
up-to-date with the schema as defined on the back-end: if the
two schemas are not identical, the values returned may be
incomplete. To ensure that the local copy of the schema is
up-to-date, call :meth:`reload`.
:type max_results: int
:param max_results: (Optional) Maximum number of rows to return.
:type page_token: str
:param page_token: (Optional) Token representing a cursor into the
table's rows.
:type client: :class:`~google.cloud.bigquery.client.Client`
:param client: (Optional) The client to use. If not passed, falls
back to the ``client`` stored on the current dataset.
:rtype: :class:`~google.api_core.page_iterator.Iterator`
:returns: Iterator of row data :class:`tuple`s. During each page, the
iterator will have the ``total_rows`` attribute set,
which counts the total number of rows **in the table**
(this is distinct from the total number of rows in the
current page: ``iterator.page.num_items``).
"""
if len(self._schema) == 0:
raise ValueError(_TABLE_HAS_NO_SCHEMA)
params = {}
if max_results is not None:
params['maxResults'] = max_results
client = self._require_client(client)
path = '%s/data' % (self.path,)
iterator = page_iterator.HTTPIterator(
client=client,
api_request=client._connection.api_request,
path=path,
item_to_value=_item_to_row,
items_key='rows',
page_token=page_token,
page_start=_rows_page_start,
next_token='pageToken',
extra_params=params)
iterator.schema = self._schema
return iterator
def test__has_next_page_without_token(self):
iterator = page_iterator.HTTPIterator(mock.sentinel.client,
mock.sentinel.api_request,
mock.sentinel.path,
mock.sentinel.item_to_value)
iterator.page_number = 1
# The iterator should not indicate that it has a new page if the
# initial page has been requested and there's no page token.
assert not iterator._has_next_page()
def test__get_query_params_w_max_results(self):
max_results = 3
iterator = page_iterator.HTTPIterator(mock.sentinel.client,
mock.sentinel.api_request,
mock.sentinel.path,
mock.sentinel.item_to_value,
max_results=max_results)
iterator.num_results = 1
local_max = max_results - iterator.num_results
assert iterator._get_query_params() == {'maxResults': local_max}
def get_iam_folder(self, resource):
extra_params = [""]
return page_iterator.HTTPIterator(
client=self,
api_request=self._connection_v2.api_request,
path="/folders",
item_to_value=_item_to_folder,
items_key="folders",
extra_params=extra_params,
)
def list_jobs(self,
max_results=None,
page_token=None,
all_users=None,
state_filter=None):
"""List jobs for the project associated with this client.
See
https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/list
:type max_results: int
:param max_results: maximum number of jobs to return, If not
passed, defaults to a value set by the API.
:type page_token: str
:param page_token: opaque marker for the next "page" of jobs. If
not passed, the API will return the first page of
jobs.
:type all_users: bool
:param all_users: if true, include jobs owned by all users in the
project.
:type state_filter: str
:param state_filter: if passed, include only jobs matching the given
state. One of
* ``"done"``
* ``"pending"``
* ``"running"``
:rtype: :class:`~google.api_core.page_iterator.Iterator`
:returns: Iterable of job instances.
"""
extra_params = {'projection': 'full'}
if all_users is not None:
extra_params['allUsers'] = all_users
if state_filter is not None:
extra_params['stateFilter'] = state_filter
path = '/projects/%s/jobs' % (self.project, )
return page_iterator.HTTPIterator(
client=self,
api_request=self._connection.api_request,
path=path,
item_to_value=_item_to_job,
items_key='jobs',
page_token=page_token,
max_results=max_results,
extra_params=extra_params)
def jobs(self, filter="", max_results=None, page_token=None):
job_client = JobClient(self)
extra_params = {}
extra_params["filter"] = filter
return page_iterator.HTTPIterator(
client=self,
api_request=self._connection.api_request,
path=job_client.path,
item_to_value=JobClient._item_to_job,
items_key="jobs",
page_token=page_token,
max_results=max_results,
extra_params=extra_params)
def test__has_next_page_w_max_results_not_done(self):
iterator = page_iterator.HTTPIterator(mock.sentinel.client,
mock.sentinel.api_request,
mock.sentinel.path,
mock.sentinel.item_to_value,
max_results=3,
page_token=mock.sentinel.token)
iterator.page_number = 1
# The iterator should indicate that it has a new page if there
# is a page token and it has not consumed more than max_results.
assert iterator.num_results < iterator.max_results
assert iterator._has_next_page()
def test__has_next_page_w_number_w_token(self):
iterator = page_iterator.HTTPIterator(
mock.sentinel.client,
mock.sentinel.api_request,
mock.sentinel.path,
mock.sentinel.item_to_value,
)
iterator.page_number = 1
iterator.next_page_token = mock.sentinel.token
# The iterator should indicate that it has a new page if the
# initial page has been requested and there's is a page token.
assert iterator._has_next_page()
def list_sinks(self,
parent,
*,
max_results=None,
page_size=None,
page_token=None):
"""List sinks for the parent resource.
See
https://cloud.google.com/logging/docs/reference/v2/rest/v2/projects.sinks/list
Args:
parent (str): The parent resource whose sinks are to be listed:
::
"projects/[PROJECT_ID]"
"organizations/[ORGANIZATION_ID]"
"billingAccounts/[BILLING_ACCOUNT_ID]"
"folders/[FOLDER_ID]".
max_results (Optional[int]):
Optional. The maximum number of entries to return.
Non-positive values are treated as 0. If None, uses API defaults.
page_size (int): number of entries to fetch in each API call. Although
requests are paged internally, logs are returned by the generator
one at a time. If not passed, defaults to a value set by the API.
page_token (str): opaque marker for the starting "page" of entries. If not
passed, the API will return the first page of entries.
Returns:
Generator[~logging_v2.Sink]
"""
extra_params = {}
if page_size is not None:
extra_params["pageSize"] = page_size
path = f"/{parent}/sinks"
iterator = page_iterator.HTTPIterator(
client=self._client,
api_request=self._client._connection.api_request,
path=path,
item_to_value=_item_to_sink,
items_key="sinks",
page_token=page_token,
extra_params=extra_params,
)
return _entries_pager(iterator, max_results)
def list_folders(self, parent, page_size=None):
extra_params = {"parent": parent}
if page_size is not None:
extra_params["pageSize"] = page_size
return page_iterator.HTTPIterator(
client=self,
api_request=self._connection_v2.api_request,
path="/folders",
item_to_value=_item_to_folder,
items_key="folders",
extra_params=extra_params,
)
def test__get_next_page_response_with_post(self):
path = '/foo'
page_response = {'items': ['one', 'two']}
api_request = mock.Mock(return_value=page_response)
iterator = page_iterator.HTTPIterator(
mock.sentinel.client,
api_request,
path=path,
item_to_value=page_iterator._item_to_value_identity)
iterator._HTTP_METHOD = 'POST'
response = iterator._get_next_page_response()
assert response == page_response
api_request.assert_called_once_with(method='POST', path=path, data={})
