def get_multi(self, keys, missing=None, deferred=None, transaction=None): """Retrieve entities, along with their attributes. :type keys: list of :class:`google.cloud.datastore.key.Key` :param keys: The keys to be retrieved from the datastore. :type missing: list :param missing: (Optional) If a list is passed, the key-only entities returned by the backend as "missing" will be copied into it. If the list is not empty, an error will occur. :type deferred: list :param deferred: (Optional) If a list is passed, the keys returned by the backend as "deferred" will be copied into it. If the list is not empty, an error will occur. :type transaction: :class:`~.transaction.Transaction` :param transaction: (Optional) Transaction to use for read consistency. If not passed, uses current transaction, if set. :rtype: list of :class:`google.cloud.datastore.entity.Entity` :returns: The requested entities. :raises: :class:`ValueError` if one or more of ``keys`` has a project which does not match our project. """ if not keys: return [] ids = set(key.project for key in keys) for current_id in ids: if current_id != self.project: raise ValueError('Keys do not match project') if transaction is None: transaction = self.current_transaction entity_pbs = _extended_lookup( connection=self.connection, project=self.project, key_pbs=[k.to_protobuf() for k in keys], missing=missing, deferred=deferred, transaction_id=transaction and transaction.id, ) if missing is not None: missing[:] = [ helpers.entity_from_protobuf(missed_pb) for missed_pb in missing ] if deferred is not None: deferred[:] = [ helpers.key_from_protobuf(deferred_pb) for deferred_pb in deferred ] return [ helpers.entity_from_protobuf(entity_pb) for entity_pb in entity_pbs ]
def get_multi(self, keys, missing=None, deferred=None, transaction=None): """Retrieve entities, along with their attributes. :type keys: list of :class:`google.cloud.datastore.key.Key` :param keys: The keys to be retrieved from the datastore. :type missing: list :param missing: (Optional) If a list is passed, the key-only entities returned by the backend as "missing" will be copied into it. If the list is not empty, an error will occur. :type deferred: list :param deferred: (Optional) If a list is passed, the keys returned by the backend as "deferred" will be copied into it. If the list is not empty, an error will occur. :type transaction: :class:`~.transaction.Transaction` :param transaction: (Optional) Transaction to use for read consistency. If not passed, uses current transaction, if set. :rtype: list of :class:`google.cloud.datastore.entity.Entity` :returns: The requested entities. :raises: :class:`ValueError` if one or more of ``keys`` has a project which does not match our project. """ if not keys: return [] ids = set(key.project for key in keys) for current_id in ids: if current_id != self.project: raise ValueError('Keys do not match project') if transaction is None: transaction = self.current_transaction entity_pbs = _extended_lookup( connection=self.connection, project=self.project, key_pbs=[k.to_protobuf() for k in keys], missing=missing, deferred=deferred, transaction_id=transaction and transaction.id, ) if missing is not None: missing[:] = [ helpers.entity_from_protobuf(missed_pb) for missed_pb in missing] if deferred is not None: deferred[:] = [ helpers.key_from_protobuf(deferred_pb) for deferred_pb in deferred] return [helpers.entity_from_protobuf(entity_pb) for entity_pb in entity_pbs]
def _deserialize_entity(pb_bytes): """Converts a serialized proto string to an Entity object.""" if pb_bytes is None: return None entity = entity_pb2.Entity() entity._pb.ParseFromString(pb_bytes) return helpers.entity_from_protobuf(entity._pb)
def _get_value_from_value_pb(value_pb): """Given a protobuf for a Value, get the correct value. The Cloud Datastore Protobuf API returns a Property Protobuf which has one value set and the rest blank. This function retrieves the the one value provided. Some work is done to coerce the return value into a more useful type (particularly in the case of a timestamp value, or a key value). :type value_pb: :class:`.entity_pb2.Value` :param value_pb: The Value Protobuf. :rtype: object :returns: The value provided by the Protobuf. :raises: :class:`ValueError <exceptions.ValueError>` if no value type has been set. """ value_type = value_pb.WhichOneof('value_type') if value_type == 'timestamp_value': result = pb_timestamp_to_datetime(value_pb.timestamp_value) elif value_type == 'key_value': result = CustomIterator.key_from_protobuf(value_pb.key_value) result._type = SubclassMap.get()[result.kind] elif value_type == 'boolean_value': result = value_pb.boolean_value elif value_type == 'double_value': result = value_pb.double_value elif value_type == 'integer_value': result = value_pb.integer_value elif value_type == 'string_value': result = value_pb.string_value elif value_type == 'blob_value': result = value_pb.blob_value elif value_type == 'entity_value': result = helpers.entity_from_protobuf(value_pb.entity_value) elif value_type == 'array_value': result = [CustomIterator._get_value_from_value_pb(value) for value in value_pb.array_value.values] elif value_type == 'geo_point_value': result = helpers.GeoPoint(value_pb.geo_point_value.latitude, value_pb.geo_point_value.longitude) elif value_type == 'null_value': result = None else: raise ValueError('Value protobuf did not have any value set') return result
def next_page(self): """Fetch a single "page" of query results. Low-level API for fine control: the more convenient API is to iterate on the current Iterator. :rtype: tuple, (entities, more_results, cursor) :returns: The next page of results. """ pb = _pb_from_query(self._query) start_cursor = self._start_cursor if start_cursor is not None: pb.start_cursor = base64.urlsafe_b64decode(start_cursor) end_cursor = self._end_cursor if end_cursor is not None: pb.end_cursor = base64.urlsafe_b64decode(end_cursor) if self._limit is not None: pb.limit.value = self._limit if self._offset is not None: pb.offset = self._offset transaction = self._client.current_transaction query_results = self._client.connection.run_query( query_pb=pb, project=self._query.project, namespace=self._query.namespace, transaction_id=transaction and transaction.id, ) (entity_pbs, cursor_as_bytes, more_results_enum, self._skipped_results) = query_results if cursor_as_bytes == b'': self._start_cursor = None else: self._start_cursor = base64.urlsafe_b64encode(cursor_as_bytes) self._end_cursor = None if more_results_enum == self._NOT_FINISHED: self._more_results = True elif more_results_enum in self._FINISHED: self._more_results = False else: raise ValueError('Unexpected value returned for `more_results`.') self._page = [ helpers.entity_from_protobuf(entity) for entity in entity_pbs] return self._page, self._more_results, self._start_cursor
def test_inverts_to_protobuf(self): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.helpers import _new_value_pb from google.cloud.datastore.helpers import entity_from_protobuf original_pb = entity_pb2.Entity() # Add a key. original_pb.key.partition_id.project_id = project = 'PROJECT' elem1 = original_pb.key.path.add() elem1.kind = 'Family' elem1.id = 1234 elem2 = original_pb.key.path.add() elem2.kind = 'King' elem2.name = 'Spades' # Add an integer property. val_pb1 = _new_value_pb(original_pb, 'foo') val_pb1.integer_value = 1337 val_pb1.exclude_from_indexes = True # Add a string property. val_pb2 = _new_value_pb(original_pb, 'bar') val_pb2.string_value = u'hello' # Add a nested (entity) property. val_pb3 = _new_value_pb(original_pb, 'entity-baz') sub_pb = entity_pb2.Entity() sub_val_pb1 = _new_value_pb(sub_pb, 'x') sub_val_pb1.double_value = 3.14 sub_val_pb2 = _new_value_pb(sub_pb, 'y') sub_val_pb2.double_value = 2.718281828 val_pb3.meaning = 9 val_pb3.entity_value.CopyFrom(sub_pb) # Add a list property. val_pb4 = _new_value_pb(original_pb, 'list-quux') array_val1 = val_pb4.array_value.values.add() array_val1.exclude_from_indexes = False array_val1.meaning = meaning = 22 array_val1.blob_value = b'\xe2\x98\x83' array_val2 = val_pb4.array_value.values.add() array_val2.exclude_from_indexes = False array_val2.meaning = meaning array_val2.blob_value = b'\xe2\x98\x85' # Convert to the user-space Entity. entity = entity_from_protobuf(original_pb) # Convert the user-space Entity back to a protobuf. new_pb = self._call_fut(entity) # NOTE: entity_to_protobuf() strips the project so we "cheat". new_pb.key.partition_id.project_id = project self._compareEntityProto(original_pb, new_pb)
def test_inverts_to_protobuf(self): from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore.helpers import _new_value_pb from google.cloud.datastore.helpers import entity_from_protobuf original_pb = entity_pb2.Entity() # Add a key. original_pb.key.partition_id.project_id = project = "PROJECT" elem1 = original_pb.key.path.add() elem1.kind = "Family" elem1.id = 1234 elem2 = original_pb.key.path.add() elem2.kind = "King" elem2.name = "Spades" # Add an integer property. val_pb1 = _new_value_pb(original_pb, "foo") val_pb1.integer_value = 1337 val_pb1.exclude_from_indexes = True # Add a string property. val_pb2 = _new_value_pb(original_pb, "bar") val_pb2.string_value = u"hello" # Add a nested (entity) property. val_pb3 = _new_value_pb(original_pb, "entity-baz") sub_pb = entity_pb2.Entity() sub_val_pb1 = _new_value_pb(sub_pb, "x") sub_val_pb1.double_value = 3.14 sub_val_pb2 = _new_value_pb(sub_pb, "y") sub_val_pb2.double_value = 2.718281828 val_pb3.meaning = 9 val_pb3.entity_value.CopyFrom(sub_pb) # Add a list property. val_pb4 = _new_value_pb(original_pb, "list-quux") array_val1 = val_pb4.array_value.values.add() array_val1.exclude_from_indexes = False array_val1.meaning = meaning = 22 array_val1.blob_value = b"\xe2\x98\x83" array_val2 = val_pb4.array_value.values.add() array_val2.exclude_from_indexes = False array_val2.meaning = meaning array_val2.blob_value = b"\xe2\x98\x85" # Convert to the user-space Entity. entity = entity_from_protobuf(original_pb) # Convert the user-space Entity back to a protobuf. new_pb = self._call_fut(entity) # NOTE: entity_to_protobuf() strips the project so we "cheat". new_pb.key.partition_id.project_id = project self._compare_entity_proto(original_pb, new_pb)
def test_inverts_to_protobuf(self): from google.cloud.datastore_v1.types import entity as entity_pb2 from google.cloud.datastore.helpers import _new_value_pb from google.cloud.datastore.helpers import entity_from_protobuf original_pb = entity_pb2.Entity() # Add a key. original_pb.key.partition_id.project_id = project = "PROJECT" elem1 = original_pb._pb.key.path.add() elem1.kind = "Family" elem1.id = 1234 elem2 = original_pb._pb.key.path.add() elem2.kind = "King" elem2.name = "Spades" # Add an integer property. val_pb1 = _new_value_pb(original_pb, "foo") val_pb1.integer_value = 1337 val_pb1.exclude_from_indexes = True # Add a string property. val_pb2 = _new_value_pb(original_pb, "bar") val_pb2.string_value = u"hello" # Add a nested (entity) property. val_pb3 = _new_value_pb(original_pb, "entity-baz") sub_pb = entity_pb2.Entity() sub_val_pb1 = _new_value_pb(sub_pb, "x") sub_val_pb1.double_value = 3.14 sub_val_pb2 = _new_value_pb(sub_pb, "y") sub_val_pb2.double_value = 2.718281828 val_pb3.meaning = 9 val_pb3.entity_value.CopyFrom(sub_pb._pb) # Add a list property. val_pb4 = _new_value_pb(original_pb, "list-quux") array_val1 = val_pb4.array_value.values.add() array_val1.exclude_from_indexes = False array_val1.meaning = meaning = 22 array_val1.blob_value = b"\xe2\x98\x83" array_val2 = val_pb4.array_value.values.add() array_val2.exclude_from_indexes = False array_val2.meaning = meaning array_val2.blob_value = b"\xe2\x98\x85" # Convert to the user-space Entity. entity = entity_from_protobuf(original_pb) # Convert the user-space Entity back to a protobuf. new_pb = self._call_fut(entity) # NOTE: entity_to_protobuf() strips the project so we "cheat". new_pb.key.partition_id.project_id = project self._compare_entity_proto(original_pb, new_pb)
def test_inverts_to_protobuf(self): from google.cloud.datastore._generated import entity_pb2 from google.cloud.datastore.helpers import _new_value_pb from google.cloud.datastore.helpers import entity_from_protobuf original_pb = entity_pb2.Entity() # Add a key. original_pb.key.partition_id.project_id = project = 'PROJECT' elem1 = original_pb.key.path.add() elem1.kind = 'Family' elem1.id = 1234 elem2 = original_pb.key.path.add() elem2.kind = 'King' elem2.name = 'Spades' # Add an integer property. val_pb1 = _new_value_pb(original_pb, 'foo') val_pb1.integer_value = 1337 val_pb1.exclude_from_indexes = True # Add a string property. val_pb2 = _new_value_pb(original_pb, 'bar') val_pb2.string_value = u'hello' # Add a nested (entity) property. val_pb3 = _new_value_pb(original_pb, 'entity-baz') sub_pb = entity_pb2.Entity() sub_val_pb1 = _new_value_pb(sub_pb, 'x') sub_val_pb1.double_value = 3.14 sub_val_pb2 = _new_value_pb(sub_pb, 'y') sub_val_pb2.double_value = 2.718281828 val_pb3.meaning = 9 val_pb3.entity_value.CopyFrom(sub_pb) # Add a list property. val_pb4 = _new_value_pb(original_pb, 'list-quux') array_val1 = val_pb4.array_value.values.add() array_val1.exclude_from_indexes = False array_val1.meaning = meaning = 22 array_val1.blob_value = b'\xe2\x98\x83' array_val2 = val_pb4.array_value.values.add() array_val2.exclude_from_indexes = False array_val2.meaning = meaning array_val2.blob_value = b'\xe2\x98\x85' # Convert to the user-space Entity. entity = entity_from_protobuf(original_pb) # Convert the user-space Entity back to a protobuf. new_pb = self._call_fut(entity) # NOTE: entity_to_protobuf() strips the project so we "cheat". new_pb.key.partition_id.project_id = project self._compareEntityProto(original_pb, new_pb)
def _item_to_entity(iterator, entity_pb): """Convert a raw protobuf entity to the native object. :type iterator: :class:`~google.api_core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type entity_pb: :class:`.entity_pb2.Entity` :param entity_pb: An entity protobuf to convert to a native entity. :rtype: :class:`~google.cloud.datastore.entity.Entity` :returns: The next entity in the page. """ return helpers.entity_from_protobuf(entity_pb)
def _item_to_entity(iterator, entity_pb): """Convert a raw protobuf entity to the native object. :type iterator: :class:`~google.cloud.iterator.Iterator` :param iterator: The iterator that is currently in use. :type entity_pb: :class:`google.cloud.datastore._generated.entity_pb2.Entity` :param entity_pb: An entity protobuf to convert to a native entity. :rtype: :class:`~google.cloud.datastore.entity.Entity` :returns: The next entity in the page. """ return helpers.entity_from_protobuf(entity_pb)
def _call_fut(self, val): from google.cloud.datastore.helpers import entity_from_protobuf return entity_from_protobuf(val)
def _call_fut(self, val): from google.cloud.datastore.helpers import entity_from_protobuf return entity_from_protobuf(val)
def ConvertToEntity(element): return helpers.entity_from_protobuf(element)
def get_multi( self, keys, missing=None, deferred=None, transaction=None, eventual=False, retry=None, timeout=None, ): """Retrieve entities, along with their attributes. :type keys: list of :class:`google.cloud.datastore.key.Key` :param keys: The keys to be retrieved from the datastore. :type missing: list :param missing: (Optional) If a list is passed, the key-only entities returned by the backend as "missing" will be copied into it. If the list is not empty, an error will occur. :type deferred: list :param deferred: (Optional) If a list is passed, the keys returned by the backend as "deferred" will be copied into it. If the list is not empty, an error will occur. :type transaction: :class:`~google.cloud.datastore.transaction.Transaction` :param transaction: (Optional) Transaction to use for read consistency. If not passed, uses current transaction, if set. :type eventual: bool :param eventual: (Optional) Defaults to strongly consistent (False). Setting True will use eventual consistency, but cannot be used inside a transaction or will raise ValueError. :type retry: :class:`google.api_core.retry.Retry` :param retry: A retry object used to retry requests. If ``None`` is specified, requests will be retried using a default configuration. :type timeout: float :param timeout: Time, in seconds, to wait for the request to complete. Note that if ``retry`` is specified, the timeout applies to each individual attempt. :rtype: list of :class:`google.cloud.datastore.entity.Entity` :returns: The requested entities. :raises: :class:`ValueError` if one or more of ``keys`` has a project which does not match our project. :raises: :class:`ValueError` if eventual is True and in a transaction. """ if not keys: return [] ids = set(key.project for key in keys) for current_id in ids: if current_id != self.project: raise ValueError("Keys do not match project") if transaction is None: transaction = self.current_transaction entity_pbs = _extended_lookup( datastore_api=self._datastore_api, project=self.project, key_pbs=[key.to_protobuf() for key in keys], eventual=eventual, missing=missing, deferred=deferred, transaction_id=transaction and transaction.id, retry=retry, timeout=timeout, ) if missing is not None: missing[:] = [ helpers.entity_from_protobuf(missed_pb) for missed_pb in missing ] if deferred is not None: deferred[:] = [ helpers.key_from_protobuf(deferred_pb) for deferred_pb in deferred ] return [ helpers.entity_from_protobuf(entity_pb._pb) for entity_pb in entity_pbs ]
def ConvertToEntity(element): return helpers.entity_from_protobuf(element)