def get_multi(self, keys, missing=None, deferred=None, transaction=None):
"""Retrieve entities, along with their attributes.
:type keys: list of :class:`google.cloud.datastore.key.Key`
:param keys: The keys to be retrieved from the datastore.
:type missing: list
:param missing: (Optional) If a list is passed, the key-only entities
returned by the backend as "missing" will be copied
into it. If the list is not empty, an error will occur.
:type deferred: list
:param deferred: (Optional) If a list is passed, the keys returned
by the backend as "deferred" will be copied into it.
If the list is not empty, an error will occur.
:type transaction: :class:`~.transaction.Transaction`
:param transaction: (Optional) Transaction to use for read consistency.
If not passed, uses current transaction, if set.
:rtype: list of :class:`google.cloud.datastore.entity.Entity`
:returns: The requested entities.
:raises: :class:`ValueError` if one or more of ``keys`` has a project
which does not match our project.
"""
if not keys:
return []
ids = set(key.project for key in keys)
for current_id in ids:
if current_id != self.project:
raise ValueError('Keys do not match project')
if transaction is None:
transaction = self.current_transaction
entity_pbs = _extended_lookup(
connection=self.connection,
project=self.project,
key_pbs=[k.to_protobuf() for k in keys],
missing=missing,
deferred=deferred,
transaction_id=transaction and transaction.id,
)
if missing is not None:
missing[:] = [
helpers.entity_from_protobuf(missed_pb)
for missed_pb in missing
]
if deferred is not None:
deferred[:] = [
helpers.key_from_protobuf(deferred_pb)
for deferred_pb in deferred
]
return [
helpers.entity_from_protobuf(entity_pb) for entity_pb in entity_pbs
]
def get_multi(self, keys, missing=None, deferred=None, transaction=None):
"""Retrieve entities, along with their attributes.
:type keys: list of :class:`google.cloud.datastore.key.Key`
:param keys: The keys to be retrieved from the datastore.
:type missing: list
:param missing: (Optional) If a list is passed, the key-only entities
returned by the backend as "missing" will be copied
into it. If the list is not empty, an error will occur.
:type deferred: list
:param deferred: (Optional) If a list is passed, the keys returned
by the backend as "deferred" will be copied into it.
If the list is not empty, an error will occur.
:type transaction: :class:`~.transaction.Transaction`
:param transaction: (Optional) Transaction to use for read consistency.
If not passed, uses current transaction, if set.
:rtype: list of :class:`google.cloud.datastore.entity.Entity`
:returns: The requested entities.
:raises: :class:`ValueError` if one or more of ``keys`` has a project
which does not match our project.
"""
if not keys:
return []
ids = set(key.project for key in keys)
for current_id in ids:
if current_id != self.project:
raise ValueError('Keys do not match project')
if transaction is None:
transaction = self.current_transaction
entity_pbs = _extended_lookup(
connection=self.connection,
project=self.project,
key_pbs=[k.to_protobuf() for k in keys],
missing=missing,
deferred=deferred,
transaction_id=transaction and transaction.id,
)
if missing is not None:
missing[:] = [
helpers.entity_from_protobuf(missed_pb)
for missed_pb in missing]
if deferred is not None:
deferred[:] = [
helpers.key_from_protobuf(deferred_pb)
for deferred_pb in deferred]
return [helpers.entity_from_protobuf(entity_pb)
for entity_pb in entity_pbs]
def _deserialize_entity(pb_bytes):
"""Converts a serialized proto string to an Entity object."""
if pb_bytes is None:
return None
entity = entity_pb2.Entity()
entity._pb.ParseFromString(pb_bytes)
return helpers.entity_from_protobuf(entity._pb)
def _get_value_from_value_pb(value_pb):
"""Given a protobuf for a Value, get the correct value.
The Cloud Datastore Protobuf API returns a Property Protobuf which
has one value set and the rest blank. This function retrieves the
the one value provided.
Some work is done to coerce the return value into a more useful type
(particularly in the case of a timestamp value, or a key value).
:type value_pb: :class:`.entity_pb2.Value`
:param value_pb: The Value Protobuf.
:rtype: object
:returns: The value provided by the Protobuf.
:raises: :class:`ValueError <exceptions.ValueError>` if no value type
has been set.
"""
value_type = value_pb.WhichOneof('value_type')
if value_type == 'timestamp_value':
result = pb_timestamp_to_datetime(value_pb.timestamp_value)
elif value_type == 'key_value':
result = CustomIterator.key_from_protobuf(value_pb.key_value)
result._type = SubclassMap.get()[result.kind]
elif value_type == 'boolean_value':
result = value_pb.boolean_value
elif value_type == 'double_value':
result = value_pb.double_value
elif value_type == 'integer_value':
result = value_pb.integer_value
elif value_type == 'string_value':
result = value_pb.string_value
elif value_type == 'blob_value':
result = value_pb.blob_value
elif value_type == 'entity_value':
result = helpers.entity_from_protobuf(value_pb.entity_value)
elif value_type == 'array_value':
result = [CustomIterator._get_value_from_value_pb(value)
for value in value_pb.array_value.values]
elif value_type == 'geo_point_value':
result = helpers.GeoPoint(value_pb.geo_point_value.latitude,
value_pb.geo_point_value.longitude)
elif value_type == 'null_value':
result = None
else:
raise ValueError('Value protobuf did not have any value set')
return result
def next_page(self):
"""Fetch a single "page" of query results.
Low-level API for fine control: the more convenient API is
to iterate on the current Iterator.
:rtype: tuple, (entities, more_results, cursor)
:returns: The next page of results.
"""
pb = _pb_from_query(self._query)
start_cursor = self._start_cursor
if start_cursor is not None:
pb.start_cursor = base64.urlsafe_b64decode(start_cursor)
end_cursor = self._end_cursor
if end_cursor is not None:
pb.end_cursor = base64.urlsafe_b64decode(end_cursor)
if self._limit is not None:
pb.limit.value = self._limit
if self._offset is not None:
pb.offset = self._offset
transaction = self._client.current_transaction
query_results = self._client.connection.run_query(
query_pb=pb,
project=self._query.project,
namespace=self._query.namespace,
transaction_id=transaction and transaction.id,
)
(entity_pbs, cursor_as_bytes,
more_results_enum, self._skipped_results) = query_results
if cursor_as_bytes == b'':
self._start_cursor = None
else:
self._start_cursor = base64.urlsafe_b64encode(cursor_as_bytes)
self._end_cursor = None
if more_results_enum == self._NOT_FINISHED:
self._more_results = True
elif more_results_enum in self._FINISHED:
self._more_results = False
else:
raise ValueError('Unexpected value returned for `more_results`.')
self._page = [
helpers.entity_from_protobuf(entity)
for entity in entity_pbs]
return self._page, self._more_results, self._start_cursor
def test_inverts_to_protobuf(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.helpers import _new_value_pb
from google.cloud.datastore.helpers import entity_from_protobuf
original_pb = entity_pb2.Entity()
# Add a key.
original_pb.key.partition_id.project_id = project = 'PROJECT'
elem1 = original_pb.key.path.add()
elem1.kind = 'Family'
elem1.id = 1234
elem2 = original_pb.key.path.add()
elem2.kind = 'King'
elem2.name = 'Spades'
# Add an integer property.
val_pb1 = _new_value_pb(original_pb, 'foo')
val_pb1.integer_value = 1337
val_pb1.exclude_from_indexes = True
# Add a string property.
val_pb2 = _new_value_pb(original_pb, 'bar')
val_pb2.string_value = u'hello'
# Add a nested (entity) property.
val_pb3 = _new_value_pb(original_pb, 'entity-baz')
sub_pb = entity_pb2.Entity()
sub_val_pb1 = _new_value_pb(sub_pb, 'x')
sub_val_pb1.double_value = 3.14
sub_val_pb2 = _new_value_pb(sub_pb, 'y')
sub_val_pb2.double_value = 2.718281828
val_pb3.meaning = 9
val_pb3.entity_value.CopyFrom(sub_pb)
# Add a list property.
val_pb4 = _new_value_pb(original_pb, 'list-quux')
array_val1 = val_pb4.array_value.values.add()
array_val1.exclude_from_indexes = False
array_val1.meaning = meaning = 22
array_val1.blob_value = b'\xe2\x98\x83'
array_val2 = val_pb4.array_value.values.add()
array_val2.exclude_from_indexes = False
array_val2.meaning = meaning
array_val2.blob_value = b'\xe2\x98\x85'
# Convert to the user-space Entity.
entity = entity_from_protobuf(original_pb)
# Convert the user-space Entity back to a protobuf.
new_pb = self._call_fut(entity)
# NOTE: entity_to_protobuf() strips the project so we "cheat".
new_pb.key.partition_id.project_id = project
self._compareEntityProto(original_pb, new_pb)
def test_inverts_to_protobuf(self):
from google.cloud.datastore_v1.proto import entity_pb2
from google.cloud.datastore.helpers import _new_value_pb
from google.cloud.datastore.helpers import entity_from_protobuf
original_pb = entity_pb2.Entity()
# Add a key.
original_pb.key.partition_id.project_id = project = "PROJECT"
elem1 = original_pb.key.path.add()
elem1.kind = "Family"
elem1.id = 1234
elem2 = original_pb.key.path.add()
elem2.kind = "King"
elem2.name = "Spades"
# Add an integer property.
val_pb1 = _new_value_pb(original_pb, "foo")
val_pb1.integer_value = 1337
val_pb1.exclude_from_indexes = True
# Add a string property.
val_pb2 = _new_value_pb(original_pb, "bar")
val_pb2.string_value = u"hello"
# Add a nested (entity) property.
val_pb3 = _new_value_pb(original_pb, "entity-baz")
sub_pb = entity_pb2.Entity()
sub_val_pb1 = _new_value_pb(sub_pb, "x")
sub_val_pb1.double_value = 3.14
sub_val_pb2 = _new_value_pb(sub_pb, "y")
sub_val_pb2.double_value = 2.718281828
val_pb3.meaning = 9
val_pb3.entity_value.CopyFrom(sub_pb)
# Add a list property.
val_pb4 = _new_value_pb(original_pb, "list-quux")
array_val1 = val_pb4.array_value.values.add()
array_val1.exclude_from_indexes = False
array_val1.meaning = meaning = 22
array_val1.blob_value = b"\xe2\x98\x83"
array_val2 = val_pb4.array_value.values.add()
array_val2.exclude_from_indexes = False
array_val2.meaning = meaning
array_val2.blob_value = b"\xe2\x98\x85"
# Convert to the user-space Entity.
entity = entity_from_protobuf(original_pb)
# Convert the user-space Entity back to a protobuf.
new_pb = self._call_fut(entity)
# NOTE: entity_to_protobuf() strips the project so we "cheat".
new_pb.key.partition_id.project_id = project
self._compare_entity_proto(original_pb, new_pb)
def test_inverts_to_protobuf(self):
from google.cloud.datastore_v1.types import entity as entity_pb2
from google.cloud.datastore.helpers import _new_value_pb
from google.cloud.datastore.helpers import entity_from_protobuf
original_pb = entity_pb2.Entity()
# Add a key.
original_pb.key.partition_id.project_id = project = "PROJECT"
elem1 = original_pb._pb.key.path.add()
elem1.kind = "Family"
elem1.id = 1234
elem2 = original_pb._pb.key.path.add()
elem2.kind = "King"
elem2.name = "Spades"
# Add an integer property.
val_pb1 = _new_value_pb(original_pb, "foo")
val_pb1.integer_value = 1337
val_pb1.exclude_from_indexes = True
# Add a string property.
val_pb2 = _new_value_pb(original_pb, "bar")
val_pb2.string_value = u"hello"
# Add a nested (entity) property.
val_pb3 = _new_value_pb(original_pb, "entity-baz")
sub_pb = entity_pb2.Entity()
sub_val_pb1 = _new_value_pb(sub_pb, "x")
sub_val_pb1.double_value = 3.14
sub_val_pb2 = _new_value_pb(sub_pb, "y")
sub_val_pb2.double_value = 2.718281828
val_pb3.meaning = 9
val_pb3.entity_value.CopyFrom(sub_pb._pb)
# Add a list property.
val_pb4 = _new_value_pb(original_pb, "list-quux")
array_val1 = val_pb4.array_value.values.add()
array_val1.exclude_from_indexes = False
array_val1.meaning = meaning = 22
array_val1.blob_value = b"\xe2\x98\x83"
array_val2 = val_pb4.array_value.values.add()
array_val2.exclude_from_indexes = False
array_val2.meaning = meaning
array_val2.blob_value = b"\xe2\x98\x85"
# Convert to the user-space Entity.
entity = entity_from_protobuf(original_pb)
# Convert the user-space Entity back to a protobuf.
new_pb = self._call_fut(entity)
# NOTE: entity_to_protobuf() strips the project so we "cheat".
new_pb.key.partition_id.project_id = project
self._compare_entity_proto(original_pb, new_pb)
def test_inverts_to_protobuf(self):
from google.cloud.datastore._generated import entity_pb2
from google.cloud.datastore.helpers import _new_value_pb
from google.cloud.datastore.helpers import entity_from_protobuf
original_pb = entity_pb2.Entity()
# Add a key.
original_pb.key.partition_id.project_id = project = 'PROJECT'
elem1 = original_pb.key.path.add()
elem1.kind = 'Family'
elem1.id = 1234
elem2 = original_pb.key.path.add()
elem2.kind = 'King'
elem2.name = 'Spades'
# Add an integer property.
val_pb1 = _new_value_pb(original_pb, 'foo')
val_pb1.integer_value = 1337
val_pb1.exclude_from_indexes = True
# Add a string property.
val_pb2 = _new_value_pb(original_pb, 'bar')
val_pb2.string_value = u'hello'
# Add a nested (entity) property.
val_pb3 = _new_value_pb(original_pb, 'entity-baz')
sub_pb = entity_pb2.Entity()
sub_val_pb1 = _new_value_pb(sub_pb, 'x')
sub_val_pb1.double_value = 3.14
sub_val_pb2 = _new_value_pb(sub_pb, 'y')
sub_val_pb2.double_value = 2.718281828
val_pb3.meaning = 9
val_pb3.entity_value.CopyFrom(sub_pb)
# Add a list property.
val_pb4 = _new_value_pb(original_pb, 'list-quux')
array_val1 = val_pb4.array_value.values.add()
array_val1.exclude_from_indexes = False
array_val1.meaning = meaning = 22
array_val1.blob_value = b'\xe2\x98\x83'
array_val2 = val_pb4.array_value.values.add()
array_val2.exclude_from_indexes = False
array_val2.meaning = meaning
array_val2.blob_value = b'\xe2\x98\x85'
# Convert to the user-space Entity.
entity = entity_from_protobuf(original_pb)
# Convert the user-space Entity back to a protobuf.
new_pb = self._call_fut(entity)
# NOTE: entity_to_protobuf() strips the project so we "cheat".
new_pb.key.partition_id.project_id = project
self._compareEntityProto(original_pb, new_pb)
def _item_to_entity(iterator, entity_pb):
"""Convert a raw protobuf entity to the native object.
:type iterator: :class:`~google.api_core.page_iterator.Iterator`
:param iterator: The iterator that is currently in use.
:type entity_pb:
:class:`.entity_pb2.Entity`
:param entity_pb: An entity protobuf to convert to a native entity.
:rtype: :class:`~google.cloud.datastore.entity.Entity`
:returns: The next entity in the page.
"""
return helpers.entity_from_protobuf(entity_pb)
def _item_to_entity(iterator, entity_pb):
"""Convert a raw protobuf entity to the native object.
:type iterator: :class:`~google.cloud.iterator.Iterator`
:param iterator: The iterator that is currently in use.
:type entity_pb:
:class:`google.cloud.datastore._generated.entity_pb2.Entity`
:param entity_pb: An entity protobuf to convert to a native entity.
:rtype: :class:`~google.cloud.datastore.entity.Entity`
:returns: The next entity in the page.
"""
return helpers.entity_from_protobuf(entity_pb)
def _call_fut(self, val):
from google.cloud.datastore.helpers import entity_from_protobuf
return entity_from_protobuf(val)
def _call_fut(self, val):
from google.cloud.datastore.helpers import entity_from_protobuf
return entity_from_protobuf(val)
def ConvertToEntity(element):
return helpers.entity_from_protobuf(element)
def get_multi(
self,
keys,
missing=None,
deferred=None,
transaction=None,
eventual=False,
retry=None,
timeout=None,
):
"""Retrieve entities, along with their attributes.
:type keys: list of :class:`google.cloud.datastore.key.Key`
:param keys: The keys to be retrieved from the datastore.
:type missing: list
:param missing: (Optional) If a list is passed, the key-only entities
returned by the backend as "missing" will be copied
into it. If the list is not empty, an error will occur.
:type deferred: list
:param deferred: (Optional) If a list is passed, the keys returned
by the backend as "deferred" will be copied into it.
If the list is not empty, an error will occur.
:type transaction:
:class:`~google.cloud.datastore.transaction.Transaction`
:param transaction: (Optional) Transaction to use for read consistency.
If not passed, uses current transaction, if set.
:type eventual: bool
:param eventual: (Optional) Defaults to strongly consistent (False).
Setting True will use eventual consistency, but cannot
be used inside a transaction or will raise ValueError.
:type retry: :class:`google.api_core.retry.Retry`
:param retry:
A retry object used to retry requests. If ``None`` is specified,
requests will be retried using a default configuration.
:type timeout: float
:param timeout:
Time, in seconds, to wait for the request to complete.
Note that if ``retry`` is specified, the timeout applies
to each individual attempt.
:rtype: list of :class:`google.cloud.datastore.entity.Entity`
:returns: The requested entities.
:raises: :class:`ValueError` if one or more of ``keys`` has a project
which does not match our project.
:raises: :class:`ValueError` if eventual is True and in a transaction.
"""
if not keys:
return []
ids = set(key.project for key in keys)
for current_id in ids:
if current_id != self.project:
raise ValueError("Keys do not match project")
if transaction is None:
transaction = self.current_transaction
entity_pbs = _extended_lookup(
datastore_api=self._datastore_api,
project=self.project,
key_pbs=[key.to_protobuf() for key in keys],
eventual=eventual,
missing=missing,
deferred=deferred,
transaction_id=transaction and transaction.id,
retry=retry,
timeout=timeout,
)
if missing is not None:
missing[:] = [
helpers.entity_from_protobuf(missed_pb)
for missed_pb in missing
]
if deferred is not None:
deferred[:] = [
helpers.key_from_protobuf(deferred_pb)
for deferred_pb in deferred
]
return [
helpers.entity_from_protobuf(entity_pb._pb)
for entity_pb in entity_pbs
]
def ConvertToEntity(element):
return helpers.entity_from_protobuf(element)