def _next_page(self): """Get the next page in the iterator. :rtype: :class:`~google.cloud.iterator.Page` :returns: The next page in the iterator (or :data:`None` if there are no pages left). """ if not self._more_results: return None query_pb = self._build_protobuf() transaction = self.client.current_transaction if transaction is None: transaction_id = None else: transaction_id = transaction.id read_options = helpers.get_read_options(self._eventual, transaction_id) partition_id = entity_pb2.PartitionId( project_id=self._query.project, namespace_id=self._query.namespace) response_pb = self.client._datastore_api.run_query( self._query.project, partition_id, read_options, query=query_pb, ) entity_pbs = self._process_query_results(response_pb) return page_iterator.Page(self, entity_pbs, self.item_to_value)
def _next_page(self): """Get the next page in the iterator. :rtype: :class:`~google.cloud.iterator.Page` :returns: The next page in the iterator (or :data:`None` if there are no pages left). """ if not self._more_results: return None query_pb = self._build_protobuf() transaction = self.client.current_transaction if transaction is None: transaction_id = None else: transaction_id = transaction.id read_options = helpers.get_read_options(self._eventual, transaction_id) partition_id = entity_pb2.PartitionId( project_id=self._query.project, namespace_id=self._query.namespace) response_pb = self.client._datastore_api.run_query( self._query.project, partition_id, read_options, query=query_pb, ) entity_pbs = self._process_query_results(response_pb) return page_iterator.Page(self, entity_pbs, self.item_to_value)
def _next_page(self): """Get the next page in the iterator. :rtype: :class:`~google.cloud.iterator.Page` :returns: The next page in the iterator (or :data:`None` if there are no pages left). """ if not self._more_results: return None query_pb = self._build_protobuf() transaction = self.client.current_transaction if transaction is None: transaction_id = None else: transaction_id = transaction.id read_options = helpers.get_read_options(self._eventual, transaction_id) partition_id = entity_pb2.PartitionId( project_id=self._query.project, namespace_id=self._query.namespace) kwargs = {} if self._retry is not None: kwargs["retry"] = self._retry if self._timeout is not None: kwargs["timeout"] = self._timeout response_pb = self.client._datastore_api.run_query( request={ "project_id": self._query.project, "partition_id": partition_id, "read_options": read_options, "query": query_pb, }, **kwargs, ) while (response_pb.batch.more_results == _NOT_FINISHED and response_pb.batch.skipped_results < query_pb.offset): # We haven't finished processing. A likely reason is we haven't # skipped all of the results yet. Don't return any results. # Instead, rerun query, adjusting offsets. Datastore doesn't process # more than 1000 skipped results in a query. query_pb.start_cursor = response_pb.batch.skipped_cursor query_pb.offset -= response_pb.batch.skipped_results response_pb = self.client._datastore_api.run_query( request={ "project_id": self._query.project, "partition_id": partition_id, "read_options": read_options, "query": query_pb, }, **kwargs, ) entity_pbs = self._process_query_results(response_pb) return page_iterator.Page(self, entity_pbs, self.item_to_value)
def _extended_lookup( datastore_api, project, key_pbs, missing=None, deferred=None, eventual=False, transaction_id=None, ): """Repeat lookup until all keys found (unless stop requested). Helper function for :meth:`Client.get_multi`. :type datastore_api: :class:`google.cloud.datastore._http.HTTPDatastoreAPI` or :class:`google.cloud.datastore_v1.gapic.DatastoreClient` :param datastore_api: The datastore API object used to connect to datastore. :type project: str :param project: The project to make the request for. :type key_pbs: list of :class:`.entity_pb2.Key` :param key_pbs: The keys to retrieve from the datastore. :type missing: list :param missing: (Optional) If a list is passed, the key-only entity protobufs returned by the backend as "missing" will be copied into it. :type deferred: list :param deferred: (Optional) If a list is passed, the key protobufs returned by the backend as "deferred" will be copied into it. :type eventual: bool :param eventual: If False (the default), request ``STRONG`` read consistency. If True, request ``EVENTUAL`` read consistency. :type transaction_id: str :param transaction_id: If passed, make the request in the scope of the given transaction. Incompatible with ``eventual==True``. :rtype: list of :class:`.entity_pb2.Entity` :returns: The requested entities. :raises: :class:`ValueError` if missing / deferred are not null or empty list. """ if missing is not None and missing != []: raise ValueError("missing must be None or an empty list") if deferred is not None and deferred != []: raise ValueError("deferred must be None or an empty list") results = [] loop_num = 0 read_options = helpers.get_read_options(eventual, transaction_id) while loop_num < _MAX_LOOPS: # loop against possible deferred. loop_num += 1 lookup_response = datastore_api.lookup( project, key_pbs, read_options=read_options ) # Accumulate the new results. results.extend(result.entity for result in lookup_response.found) if missing is not None: missing.extend(result.entity for result in lookup_response.missing) if deferred is not None: deferred.extend(lookup_response.deferred) break if len(lookup_response.deferred) == 0: break # We have deferred keys, and the user didn't ask to know about # them, so retry (but only with the deferred ones). key_pbs = lookup_response.deferred return results
def _call_fut(self, eventual, transaction_id): from google.cloud.datastore.helpers import get_read_options return get_read_options(eventual, transaction_id)
def _call_fut(self, eventual, transaction_id): from google.cloud.datastore.helpers import get_read_options return get_read_options(eventual, transaction_id)
def _extended_lookup( datastore_api, project, key_pbs, missing=None, deferred=None, eventual=False, transaction_id=None, retry=None, timeout=None, ): """Repeat lookup until all keys found (unless stop requested). Helper function for :meth:`Client.get_multi`. :type datastore_api: :class:`google.cloud.datastore._http.HTTPDatastoreAPI` or :class:`google.cloud.datastore_v1.gapic.DatastoreClient` :param datastore_api: The datastore API object used to connect to datastore. :type project: str :param project: The project to make the request for. :type key_pbs: list of :class:`.entity_pb2.Key` :param key_pbs: The keys to retrieve from the datastore. :type missing: list :param missing: (Optional) If a list is passed, the key-only entity protobufs returned by the backend as "missing" will be copied into it. :type deferred: list :param deferred: (Optional) If a list is passed, the key protobufs returned by the backend as "deferred" will be copied into it. :type eventual: bool :param eventual: If False (the default), request ``STRONG`` read consistency. If True, request ``EVENTUAL`` read consistency. :type transaction_id: str :param transaction_id: If passed, make the request in the scope of the given transaction. Incompatible with ``eventual==True``. :type retry: :class:`google.api_core.retry.Retry` :param retry: A retry object used to retry requests. If ``None`` is specified, requests will be retried using a default configuration. :type timeout: float :param timeout: Time, in seconds, to wait for the request to complete. Note that if ``retry`` is specified, the timeout applies to each individual attempt. :rtype: list of :class:`.entity_pb2.Entity` :returns: The requested entities. :raises: :class:`ValueError` if missing / deferred are not null or empty list. """ if missing is not None and missing != []: raise ValueError("missing must be None or an empty list") if deferred is not None and deferred != []: raise ValueError("deferred must be None or an empty list") kwargs = _make_retry_timeout_kwargs(retry, timeout) results = [] loop_num = 0 read_options = helpers.get_read_options(eventual, transaction_id) while loop_num < _MAX_LOOPS: # loop against possible deferred. loop_num += 1 lookup_response = datastore_api.lookup( request={ "project_id": project, "keys": key_pbs, "read_options": read_options, }, **kwargs, ) # Accumulate the new results. results.extend(result.entity for result in lookup_response.found) if missing is not None: missing.extend(result.entity for result in lookup_response.missing) if deferred is not None: deferred.extend(lookup_response.deferred) break if len(lookup_response.deferred) == 0: break # We have deferred keys, and the user didn't ask to know about # them, so retry (but only with the deferred ones). key_pbs = lookup_response.deferred return results
def _extended_lookup(datastore_api, project, key_pbs, missing=None, deferred=None, eventual=False, transaction_id=None): """Repeat lookup until all keys found (unless stop requested). Helper function for :meth:`Client.get_multi`. :type datastore_api: :class:`google.cloud.datastore._http.HTTPDatastoreAPI` or :class:`google.cloud.datastore._gax.GAPICDatastoreAPI` :param datastore_api: The datastore API object used to connect to datastore. :type project: str :param project: The project to make the request for. :type key_pbs: list of :class:`.entity_pb2.Key` :param key_pbs: The keys to retrieve from the datastore. :type missing: list :param missing: (Optional) If a list is passed, the key-only entity protobufs returned by the backend as "missing" will be copied into it. :type deferred: list :param deferred: (Optional) If a list is passed, the key protobufs returned by the backend as "deferred" will be copied into it. :type eventual: bool :param eventual: If False (the default), request ``STRONG`` read consistency. If True, request ``EVENTUAL`` read consistency. :type transaction_id: str :param transaction_id: If passed, make the request in the scope of the given transaction. Incompatible with ``eventual==True``. :rtype: list of :class:`.entity_pb2.Entity` :returns: The requested entities. :raises: :class:`ValueError` if missing / deferred are not null or empty list. """ if missing is not None and missing != []: raise ValueError('missing must be None or an empty list') if deferred is not None and deferred != []: raise ValueError('deferred must be None or an empty list') results = [] loop_num = 0 read_options = helpers.get_read_options(eventual, transaction_id) while loop_num < _MAX_LOOPS: # loop against possible deferred. loop_num += 1 lookup_response = datastore_api.lookup( project_id=project, read_options=read_options, keys=key_pbs, ) # Accumulate the new results. results.extend(result.entity for result in lookup_response.found) if missing is not None: missing.extend(result.entity for result in lookup_response.missing) if deferred is not None: deferred.extend(lookup_response.deferred) break if len(lookup_response.deferred) == 0: break # We have deferred keys, and the user didn't ask to know about # them, so retry (but only with the deferred ones). key_pbs = lookup_response.deferred return results