def list_zones(self, max_results=None, page_token=None): """List zones for the project associated with this client. See https://cloud.google.com/dns/api/v1/managedZones/list :type max_results: int :param max_results: maximum number of zones to return, If not passed, defaults to a value set by the API. :type page_token: str :param page_token: opaque marker for the next "page" of zones. If not passed, the API will return the first page of zones. :rtype: :class:`~google.api_core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.dns.zone.ManagedZone` belonging to this project. """ path = "/projects/%s/managedZones" % (self.project,) return page_iterator.HTTPIterator( client=self, api_request=self._connection.api_request, path=path, item_to_value=_item_to_zone, items_key="managedZones", page_token=page_token, max_results=max_results, )
def list_environments(self, location, max_results=None, page_token=None, prefix=None, projection="noAcl", fields=None, project=None): """Get all environments in the project associated to the client. """ if project is None: project = self.project if project is None: raise ValueError( "Client project not set: pass an explicit project.") return page_iterator.HTTPIterator( client=self, api_request=self._connection.api_request, path=f'/projects/{project}/locations/{location}/environments', items_key='environments', item_to_value=_item_to_environment, page_token=page_token, max_results=max_results)
def list_sub_prefixes(self, bucket: str, prefix: str) -> None: """ List sub folders in folder specified by prefix SO link: https://stackoverflow.com/a/59008580 :param bucket: GCS bucket name as string :param prefix: path to directory to look into e.g. a/b/c/ :return: list of strings of sub-directories e.g. [a/b/c/v1/, a/b/c/v2/] """ if not prefix.endswith('/'): prefix += '/' extra_params: dict = { 'projection': 'noAcl', 'prefix': prefix, 'delimiter': '/' } path: str = f'/b/{bucket}/o' pg_iterator = page_iterator.HTTPIterator( client=self, api_request=self._connection.api_request, path=path, items_key='prefixes', item_to_value=lambda _, x: x, extra_params=extra_params, ) return list(pg_iterator)
def list_zones(self, max_results=None, page_token=None): """List zones for the project associated with this client. See https://cloud.google.com/dns/api/v1/managedZones/list :type max_results: int :param max_results: maximum number of zones to return, If not passed, defaults to a value set by the API. :type page_token: str :param page_token: Optional. If present, return the next batch of zones, using the value, which must correspond to the ``nextPageToken`` value returned in the previous response. Deprecated: use the ``pages`` property of the returned iterator instead of manually passing the token. :rtype: :class:`~google.api_core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.dns.zone.ManagedZone` belonging to this project. """ path = "/projects/%s/managedZones" % (self.project,) return page_iterator.HTTPIterator( client=self, api_request=self._connection.api_request, path=path, item_to_value=_item_to_zone, items_key="managedZones", page_token=page_token, max_results=max_results, )
def test_iterate(self): path = "/foo" item1 = {"name": "1"} item2 = {"name": "2"} api_request = mock.Mock(return_value={"items": [item1, item2]}) iterator = page_iterator.HTTPIterator( mock.sentinel.client, api_request, path=path, item_to_value=page_iterator._item_to_value_identity, ) assert iterator.num_results == 0 items_iter = iter(iterator) val1 = six.next(items_iter) assert val1 == item1 assert iterator.num_results == 1 val2 = six.next(items_iter) assert val2 == item2 assert iterator.num_results == 2 with pytest.raises(StopIteration): six.next(items_iter) api_request.assert_called_once_with(method="GET", path=path, query_params={})
def list_tables(self, max_results=None, page_token=None): """List tables for the project associated with this client. See https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/list :type max_results: int :param max_results: (Optional) Maximum number of tables to return. If not passed, defaults to a value set by the API. :type page_token: str :param page_token: (Optional) Opaque marker for the next "page" of datasets. If not passed, the API will return the first page of datasets. :rtype: :class:`~google.api_core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.bigquery.table.Table` contained within the current dataset. """ path = '/projects/%s/datasets/%s/tables' % (self.project, self.name) result = page_iterator.HTTPIterator( client=self._client, api_request=self._client._connection.api_request, path=path, item_to_value=_item_to_table, items_key='tables', page_token=page_token, max_results=max_results) result.dataset = self return result
def list_notifications(self, client=None): """List Pub / Sub notifications for this bucket. See: https://cloud.google.com/storage/docs/json_api/v1/notifications/list If :attr:`user_project` is set, bills the API request to that project. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :rtype: list of :class:`.BucketNotification` :returns: notification instances """ client = self._require_client(client) path = self.path + '/notificationConfigs' iterator = page_iterator.HTTPIterator( client=client, api_request=client._connection.api_request, path=path, item_to_value=_item_to_notification) iterator.bucket = self return iterator
def list_sub_prefixes(client, bucket, prefix): """ List sub folders in folder specified by prefix SO link: https://stackoverflow.com/a/59008580 :param client: GCS client :param bucket: GCS bucket name as string :param prefix: path to directory to look into e.g. a/b/c/ :return: list of strings of sub-directories e.g. [a/b/c/v1/, a/b/c/v2/] """ if not prefix.endswith('/'): prefix += '/' extra_params = {"projection": "noAcl", "prefix": prefix, "delimiter": '/'} path = f"/b/{bucket}/o" pg_iterator = page_iterator.HTTPIterator( client=client, api_request=client._connection.api_request, path=path, items_key='prefixes', item_to_value=lambda _, x: x, extra_params=extra_params, ) return list(pg_iterator)
def list_metrics(self, project, *, page_size=None, page_token=None): """List metrics for the project associated with this client. See https://cloud.google.com/logging/docs/reference/v2/rest/v2/projects.metrics/list Args: page_size (Optional[int]): The maximum number of sinks in each page of results from this request. Non-positive values are ignored. Defaults to a sensible value set by the API. page_token (Optional[str]): If present, return the next batch of sinks, using the value, which must correspond to the ``nextPageToken`` value returned in the previous response. Deprecated: use the ``pages`` property ofthe returned iterator instead of manually passing the token. Returns: Iterator[google.cloud.logging_v2.metric.Metric] """ extra_params = {} if page_size is not None: extra_params["pageSize"] = page_size path = f"/projects/{project}/metrics" return page_iterator.HTTPIterator( client=self._client, api_request=self._client._connection.api_request, path=path, item_to_value=_item_to_metric, items_key="metrics", page_token=page_token, extra_params=extra_params, )
def list_projects(self, max_results=None, page_token=None): """List projects for the project associated with this client. See https://cloud.google.com/bigquery/docs/reference/rest/v2/projects/list :type max_results: int :param max_results: maximum number of projects to return, If not passed, defaults to a value set by the API. :type page_token: str :param page_token: opaque marker for the next "page" of projects. If not passed, the API will return the first page of projects. :rtype: :class:`~google.api_core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.bigquery.client.Project` accessible to the current client. """ return page_iterator.HTTPIterator( client=self, api_request=self._connection.api_request, path='/projects', item_to_value=_item_to_project, items_key='projects', page_token=page_token, max_results=max_results)
def test__get_query_params_no_token(self): iterator = page_iterator.HTTPIterator(mock.sentinel.client, mock.sentinel.api_request, mock.sentinel.path, mock.sentinel.item_to_value) assert iterator._get_query_params() == {}
def test__has_next_page_new(self): iterator = page_iterator.HTTPIterator(mock.sentinel.client, mock.sentinel.api_request, mock.sentinel.path, mock.sentinel.item_to_value) # The iterator should *always* indicate that it has a next page # when created so that it can fetch the initial page. assert iterator._has_next_page()
def test_constructor_w_extra_param_collision(self): extra_params = {'pageToken': 'val'} with pytest.raises(ValueError): page_iterator.HTTPIterator(mock.sentinel.client, mock.sentinel.api_request, mock.sentinel.path, mock.sentinel.item_to_value, extra_params=extra_params)
def list_hmac_keys( self, max_results=None, service_account_email=None, show_deleted_keys=None, project_id=None, user_project=None, ): """List HMAC keys for a project. :type max_results: int :param max_results: (Optional) max number of keys to return in a given page. :type service_account_email: str :param service_account_email: (Optional) limit keys to those created by the given service account. :type show_deleted_keys: bool :param show_deleted_keys: (Optional) included deleted keys in the list. Default is to exclude them. :type project_id: str :param project_id: (Optional) explicit project ID for the key. Defaults to the client's project. :type user_project: str :param user_project: (Optional) This parameter is currently ignored. :rtype: Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str] :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string. """ if project_id is None: project_id = self.project path = "/projects/{}/hmacKeys".format(project_id) extra_params = {} if service_account_email is not None: extra_params["serviceAccountEmail"] = service_account_email if show_deleted_keys is not None: extra_params["showDeletedKeys"] = show_deleted_keys if user_project is not None: extra_params["userProject"] = user_project return page_iterator.HTTPIterator( client=self, api_request=self._connection.api_request, path=path, item_to_value=_item_to_hmac_key_metadata, max_results=max_results, extra_params=extra_params, )
def test__get_query_params_extra_params(self): extra_params = {'key': 'val'} iterator = page_iterator.HTTPIterator(mock.sentinel.client, mock.sentinel.api_request, mock.sentinel.path, mock.sentinel.item_to_value, extra_params=extra_params) assert iterator._get_query_params() == extra_params
def test__get_next_page_bad_http_method(self): iterator = page_iterator.HTTPIterator(mock.sentinel.client, mock.sentinel.api_request, mock.sentinel.path, mock.sentinel.item_to_value) iterator._HTTP_METHOD = 'NOT-A-VERB' with pytest.raises(ValueError): iterator._get_next_page_response()
def test_page_size_items(self, page_size, max_results, pages): path = "/foo" NITEMS = 10 n = [0] # blast you python 2! def api_request(*args, **kw): assert not args query_params = dict(maxResults=(page_size if max_results is None else min(page_size, max_results - n[0]))) if n[0]: query_params.update(pageToken='test') assert kw == { 'method': 'GET', 'path': '/foo', 'query_params': query_params } n_items = min(kw['query_params']['maxResults'], NITEMS - n[0]) items = [dict(name=str(i + n[0])) for i in range(n_items)] n[0] += n_items result = dict(items=items) if n[0] < NITEMS: result.update(nextPageToken='test') return result iterator = page_iterator.HTTPIterator( mock.sentinel.client, api_request, path=path, item_to_value=page_iterator._item_to_value_identity, page_size=page_size, max_results=max_results, ) assert iterator.num_results == 0 n_results = max_results if max_results is not None else NITEMS if pages: items_iter = iter(iterator.pages) npages = int(math.ceil(float(n_results) / page_size)) for ipage in range(npages): assert (list(six.next(items_iter)) == [ dict(name=str(i)) for i in range( ipage * page_size, min((ipage + 1) * page_size, n_results), ) ]) else: items_iter = iter(iterator) for i in range(n_results): assert six.next(items_iter) == dict(name=str(i)) assert iterator.num_results == i + 1 with pytest.raises(StopIteration): six.next(items_iter)
def models(self, max_results=None, page_token=None): model_client = ModelClient(self) return page_iterator.HTTPIterator( client=self, api_request=self._connection.api_request, path=model_client.path, item_to_value=ModelClient._item_to_model, items_key="models", page_token=page_token, max_results=max_results)
def test__get_query_params_w_token(self): iterator = page_iterator.HTTPIterator(mock.sentinel.client, mock.sentinel.api_request, mock.sentinel.path, mock.sentinel.item_to_value) iterator.next_page_token = 'token' assert iterator._get_query_params() == { 'pageToken': iterator.next_page_token }
def fetch_data(self, max_results=None, page_token=None, client=None): """API call: fetch the table data via a GET request See https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/list .. note:: This method assumes that its instance's ``schema`` attribute is up-to-date with the schema as defined on the back-end: if the two schemas are not identical, the values returned may be incomplete. To ensure that the local copy of the schema is up-to-date, call :meth:`reload`. :type max_results: int :param max_results: (Optional) Maximum number of rows to return. :type page_token: str :param page_token: (Optional) Token representing a cursor into the table's rows. :type client: :class:`~google.cloud.bigquery.client.Client` :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current dataset. :rtype: :class:`~google.api_core.page_iterator.Iterator` :returns: Iterator of row data :class:`tuple`s. During each page, the iterator will have the ``total_rows`` attribute set, which counts the total number of rows **in the table** (this is distinct from the total number of rows in the current page: ``iterator.page.num_items``). """ if len(self._schema) == 0: raise ValueError(_TABLE_HAS_NO_SCHEMA) params = {} if max_results is not None: params['maxResults'] = max_results client = self._require_client(client) path = '%s/data' % (self.path,) iterator = page_iterator.HTTPIterator( client=client, api_request=client._connection.api_request, path=path, item_to_value=_item_to_row, items_key='rows', page_token=page_token, page_start=_rows_page_start, next_token='pageToken', extra_params=params) iterator.schema = self._schema return iterator
def test__has_next_page_without_token(self): iterator = page_iterator.HTTPIterator(mock.sentinel.client, mock.sentinel.api_request, mock.sentinel.path, mock.sentinel.item_to_value) iterator.page_number = 1 # The iterator should not indicate that it has a new page if the # initial page has been requested and there's no page token. assert not iterator._has_next_page()
def test__get_query_params_w_max_results(self): max_results = 3 iterator = page_iterator.HTTPIterator(mock.sentinel.client, mock.sentinel.api_request, mock.sentinel.path, mock.sentinel.item_to_value, max_results=max_results) iterator.num_results = 1 local_max = max_results - iterator.num_results assert iterator._get_query_params() == {'maxResults': local_max}
def get_iam_folder(self, resource): extra_params = [""] return page_iterator.HTTPIterator( client=self, api_request=self._connection_v2.api_request, path="/folders", item_to_value=_item_to_folder, items_key="folders", extra_params=extra_params, )
def list_jobs(self, max_results=None, page_token=None, all_users=None, state_filter=None): """List jobs for the project associated with this client. See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/list :type max_results: int :param max_results: maximum number of jobs to return, If not passed, defaults to a value set by the API. :type page_token: str :param page_token: opaque marker for the next "page" of jobs. If not passed, the API will return the first page of jobs. :type all_users: bool :param all_users: if true, include jobs owned by all users in the project. :type state_filter: str :param state_filter: if passed, include only jobs matching the given state. One of * ``"done"`` * ``"pending"`` * ``"running"`` :rtype: :class:`~google.api_core.page_iterator.Iterator` :returns: Iterable of job instances. """ extra_params = {'projection': 'full'} if all_users is not None: extra_params['allUsers'] = all_users if state_filter is not None: extra_params['stateFilter'] = state_filter path = '/projects/%s/jobs' % (self.project, ) return page_iterator.HTTPIterator( client=self, api_request=self._connection.api_request, path=path, item_to_value=_item_to_job, items_key='jobs', page_token=page_token, max_results=max_results, extra_params=extra_params)
def jobs(self, filter="", max_results=None, page_token=None): job_client = JobClient(self) extra_params = {} extra_params["filter"] = filter return page_iterator.HTTPIterator( client=self, api_request=self._connection.api_request, path=job_client.path, item_to_value=JobClient._item_to_job, items_key="jobs", page_token=page_token, max_results=max_results, extra_params=extra_params)
def test__has_next_page_w_max_results_not_done(self): iterator = page_iterator.HTTPIterator(mock.sentinel.client, mock.sentinel.api_request, mock.sentinel.path, mock.sentinel.item_to_value, max_results=3, page_token=mock.sentinel.token) iterator.page_number = 1 # The iterator should indicate that it has a new page if there # is a page token and it has not consumed more than max_results. assert iterator.num_results < iterator.max_results assert iterator._has_next_page()
def test__has_next_page_w_number_w_token(self): iterator = page_iterator.HTTPIterator( mock.sentinel.client, mock.sentinel.api_request, mock.sentinel.path, mock.sentinel.item_to_value, ) iterator.page_number = 1 iterator.next_page_token = mock.sentinel.token # The iterator should indicate that it has a new page if the # initial page has been requested and there's is a page token. assert iterator._has_next_page()
def list_sinks(self, parent, *, max_results=None, page_size=None, page_token=None): """List sinks for the parent resource. See https://cloud.google.com/logging/docs/reference/v2/rest/v2/projects.sinks/list Args: parent (str): The parent resource whose sinks are to be listed: :: "projects/[PROJECT_ID]" "organizations/[ORGANIZATION_ID]" "billingAccounts/[BILLING_ACCOUNT_ID]" "folders/[FOLDER_ID]". max_results (Optional[int]): Optional. The maximum number of entries to return. Non-positive values are treated as 0. If None, uses API defaults. page_size (int): number of entries to fetch in each API call. Although requests are paged internally, logs are returned by the generator one at a time. If not passed, defaults to a value set by the API. page_token (str): opaque marker for the starting "page" of entries. If not passed, the API will return the first page of entries. Returns: Generator[~logging_v2.Sink] """ extra_params = {} if page_size is not None: extra_params["pageSize"] = page_size path = f"/{parent}/sinks" iterator = page_iterator.HTTPIterator( client=self._client, api_request=self._client._connection.api_request, path=path, item_to_value=_item_to_sink, items_key="sinks", page_token=page_token, extra_params=extra_params, ) return _entries_pager(iterator, max_results)
def list_folders(self, parent, page_size=None): extra_params = {"parent": parent} if page_size is not None: extra_params["pageSize"] = page_size return page_iterator.HTTPIterator( client=self, api_request=self._connection_v2.api_request, path="/folders", item_to_value=_item_to_folder, items_key="folders", extra_params=extra_params, )
def test__get_next_page_response_with_post(self): path = '/foo' page_response = {'items': ['one', 'two']} api_request = mock.Mock(return_value=page_response) iterator = page_iterator.HTTPIterator( mock.sentinel.client, api_request, path=path, item_to_value=page_iterator._item_to_value_identity) iterator._HTTP_METHOD = 'POST' response = iterator._get_next_page_response() assert response == page_response api_request.assert_called_once_with(method='POST', path=path, data={})