Пример #1
0
    def get_multi(self, keys, missing=None, deferred=None, transaction=None):
        """Retrieve entities, along with their attributes.

        :type keys: list of :class:`google.cloud.datastore.key.Key`
        :param keys: The keys to be retrieved from the datastore.

        :type missing: list
        :param missing: (Optional) If a list is passed, the key-only entities
                        returned by the backend as "missing" will be copied
                        into it. If the list is not empty, an error will occur.

        :type deferred: list
        :param deferred: (Optional) If a list is passed, the keys returned
                         by the backend as "deferred" will be copied into it.
                         If the list is not empty, an error will occur.

        :type transaction: :class:`~.transaction.Transaction`
        :param transaction: (Optional) Transaction to use for read consistency.
                            If not passed, uses current transaction, if set.

        :rtype: list of :class:`google.cloud.datastore.entity.Entity`
        :returns: The requested entities.
        :raises: :class:`ValueError` if one or more of ``keys`` has a project
                 which does not match our project.
        """
        if not keys:
            return []

        ids = set(key.project for key in keys)
        for current_id in ids:
            if current_id != self.project:
                raise ValueError('Keys do not match project')

        if transaction is None:
            transaction = self.current_transaction

        entity_pbs = _extended_lookup(
            connection=self.connection,
            project=self.project,
            key_pbs=[k.to_protobuf() for k in keys],
            missing=missing,
            deferred=deferred,
            transaction_id=transaction and transaction.id,
        )

        if missing is not None:
            missing[:] = [
                helpers.entity_from_protobuf(missed_pb)
                for missed_pb in missing
            ]

        if deferred is not None:
            deferred[:] = [
                helpers.key_from_protobuf(deferred_pb)
                for deferred_pb in deferred
            ]

        return [
            helpers.entity_from_protobuf(entity_pb) for entity_pb in entity_pbs
        ]
Пример #2
0
    def get_multi(self, keys, missing=None, deferred=None, transaction=None):
        """Retrieve entities, along with their attributes.

        :type keys: list of :class:`google.cloud.datastore.key.Key`
        :param keys: The keys to be retrieved from the datastore.

        :type missing: list
        :param missing: (Optional) If a list is passed, the key-only entities
                        returned by the backend as "missing" will be copied
                        into it. If the list is not empty, an error will occur.

        :type deferred: list
        :param deferred: (Optional) If a list is passed, the keys returned
                         by the backend as "deferred" will be copied into it.
                         If the list is not empty, an error will occur.

        :type transaction: :class:`~.transaction.Transaction`
        :param transaction: (Optional) Transaction to use for read consistency.
                            If not passed, uses current transaction, if set.

        :rtype: list of :class:`google.cloud.datastore.entity.Entity`
        :returns: The requested entities.
        :raises: :class:`ValueError` if one or more of ``keys`` has a project
                 which does not match our project.
        """
        if not keys:
            return []

        ids = set(key.project for key in keys)
        for current_id in ids:
            if current_id != self.project:
                raise ValueError('Keys do not match project')

        if transaction is None:
            transaction = self.current_transaction

        entity_pbs = _extended_lookup(
            connection=self.connection,
            project=self.project,
            key_pbs=[k.to_protobuf() for k in keys],
            missing=missing,
            deferred=deferred,
            transaction_id=transaction and transaction.id,
        )

        if missing is not None:
            missing[:] = [
                helpers.entity_from_protobuf(missed_pb)
                for missed_pb in missing]

        if deferred is not None:
            deferred[:] = [
                helpers.key_from_protobuf(deferred_pb)
                for deferred_pb in deferred]

        return [helpers.entity_from_protobuf(entity_pb)
                for entity_pb in entity_pbs]
Пример #3
0
def _deserialize_entity(pb_bytes):
    """Converts a serialized proto string to an Entity object."""
    if pb_bytes is None:
        return None
    entity = entity_pb2.Entity()
    entity._pb.ParseFromString(pb_bytes)
    return helpers.entity_from_protobuf(entity._pb)
Пример #4
0
    def _get_value_from_value_pb(value_pb):
        """Given a protobuf for a Value, get the correct value.

        The Cloud Datastore Protobuf API returns a Property Protobuf which
        has one value set and the rest blank.  This function retrieves the
        the one value provided.

        Some work is done to coerce the return value into a more useful type
        (particularly in the case of a timestamp value, or a key value).

        :type value_pb: :class:`.entity_pb2.Value`
        :param value_pb: The Value Protobuf.

        :rtype: object
        :returns: The value provided by the Protobuf.
        :raises: :class:`ValueError <exceptions.ValueError>` if no value type
                 has been set.
        """
        value_type = value_pb.WhichOneof('value_type')

        if value_type == 'timestamp_value':
            result = pb_timestamp_to_datetime(value_pb.timestamp_value)

        elif value_type == 'key_value':
            result = CustomIterator.key_from_protobuf(value_pb.key_value)
            result._type = SubclassMap.get()[result.kind]

        elif value_type == 'boolean_value':
            result = value_pb.boolean_value

        elif value_type == 'double_value':
            result = value_pb.double_value

        elif value_type == 'integer_value':
            result = value_pb.integer_value

        elif value_type == 'string_value':
            result = value_pb.string_value

        elif value_type == 'blob_value':
            result = value_pb.blob_value

        elif value_type == 'entity_value':
            result = helpers.entity_from_protobuf(value_pb.entity_value)

        elif value_type == 'array_value':
            result = [CustomIterator._get_value_from_value_pb(value)
                      for value in value_pb.array_value.values]

        elif value_type == 'geo_point_value':
            result = helpers.GeoPoint(value_pb.geo_point_value.latitude,
                                      value_pb.geo_point_value.longitude)

        elif value_type == 'null_value':
            result = None
        else:
            raise ValueError('Value protobuf did not have any value set')

        return result
Пример #5
0
    def next_page(self):
        """Fetch a single "page" of query results.

        Low-level API for fine control:  the more convenient API is
        to iterate on the current Iterator.

        :rtype: tuple, (entities, more_results, cursor)
        :returns: The next page of results.
        """
        pb = _pb_from_query(self._query)

        start_cursor = self._start_cursor
        if start_cursor is not None:
            pb.start_cursor = base64.urlsafe_b64decode(start_cursor)

        end_cursor = self._end_cursor
        if end_cursor is not None:
            pb.end_cursor = base64.urlsafe_b64decode(end_cursor)

        if self._limit is not None:
            pb.limit.value = self._limit

        if self._offset is not None:
            pb.offset = self._offset

        transaction = self._client.current_transaction

        query_results = self._client.connection.run_query(
            query_pb=pb,
            project=self._query.project,
            namespace=self._query.namespace,
            transaction_id=transaction and transaction.id,
            )
        (entity_pbs, cursor_as_bytes,
         more_results_enum, self._skipped_results) = query_results

        if cursor_as_bytes == b'':
            self._start_cursor = None
        else:
            self._start_cursor = base64.urlsafe_b64encode(cursor_as_bytes)
        self._end_cursor = None

        if more_results_enum == self._NOT_FINISHED:
            self._more_results = True
        elif more_results_enum in self._FINISHED:
            self._more_results = False
        else:
            raise ValueError('Unexpected value returned for `more_results`.')

        self._page = [
            helpers.entity_from_protobuf(entity)
            for entity in entity_pbs]
        return self._page, self._more_results, self._start_cursor
    def test_inverts_to_protobuf(self):
        from google.cloud.proto.datastore.v1 import entity_pb2
        from google.cloud.datastore.helpers import _new_value_pb
        from google.cloud.datastore.helpers import entity_from_protobuf

        original_pb = entity_pb2.Entity()
        # Add a key.
        original_pb.key.partition_id.project_id = project = 'PROJECT'
        elem1 = original_pb.key.path.add()
        elem1.kind = 'Family'
        elem1.id = 1234
        elem2 = original_pb.key.path.add()
        elem2.kind = 'King'
        elem2.name = 'Spades'

        # Add an integer property.
        val_pb1 = _new_value_pb(original_pb, 'foo')
        val_pb1.integer_value = 1337
        val_pb1.exclude_from_indexes = True
        # Add a string property.
        val_pb2 = _new_value_pb(original_pb, 'bar')
        val_pb2.string_value = u'hello'

        # Add a nested (entity) property.
        val_pb3 = _new_value_pb(original_pb, 'entity-baz')
        sub_pb = entity_pb2.Entity()
        sub_val_pb1 = _new_value_pb(sub_pb, 'x')
        sub_val_pb1.double_value = 3.14
        sub_val_pb2 = _new_value_pb(sub_pb, 'y')
        sub_val_pb2.double_value = 2.718281828
        val_pb3.meaning = 9
        val_pb3.entity_value.CopyFrom(sub_pb)

        # Add a list property.
        val_pb4 = _new_value_pb(original_pb, 'list-quux')
        array_val1 = val_pb4.array_value.values.add()
        array_val1.exclude_from_indexes = False
        array_val1.meaning = meaning = 22
        array_val1.blob_value = b'\xe2\x98\x83'
        array_val2 = val_pb4.array_value.values.add()
        array_val2.exclude_from_indexes = False
        array_val2.meaning = meaning
        array_val2.blob_value = b'\xe2\x98\x85'

        # Convert to the user-space Entity.
        entity = entity_from_protobuf(original_pb)
        # Convert the user-space Entity back to a protobuf.
        new_pb = self._call_fut(entity)

        # NOTE: entity_to_protobuf() strips the project so we "cheat".
        new_pb.key.partition_id.project_id = project
        self._compareEntityProto(original_pb, new_pb)
    def test_inverts_to_protobuf(self):
        from google.cloud.datastore_v1.proto import entity_pb2
        from google.cloud.datastore.helpers import _new_value_pb
        from google.cloud.datastore.helpers import entity_from_protobuf

        original_pb = entity_pb2.Entity()
        # Add a key.
        original_pb.key.partition_id.project_id = project = "PROJECT"
        elem1 = original_pb.key.path.add()
        elem1.kind = "Family"
        elem1.id = 1234
        elem2 = original_pb.key.path.add()
        elem2.kind = "King"
        elem2.name = "Spades"

        # Add an integer property.
        val_pb1 = _new_value_pb(original_pb, "foo")
        val_pb1.integer_value = 1337
        val_pb1.exclude_from_indexes = True
        # Add a string property.
        val_pb2 = _new_value_pb(original_pb, "bar")
        val_pb2.string_value = u"hello"

        # Add a nested (entity) property.
        val_pb3 = _new_value_pb(original_pb, "entity-baz")
        sub_pb = entity_pb2.Entity()
        sub_val_pb1 = _new_value_pb(sub_pb, "x")
        sub_val_pb1.double_value = 3.14
        sub_val_pb2 = _new_value_pb(sub_pb, "y")
        sub_val_pb2.double_value = 2.718281828
        val_pb3.meaning = 9
        val_pb3.entity_value.CopyFrom(sub_pb)

        # Add a list property.
        val_pb4 = _new_value_pb(original_pb, "list-quux")
        array_val1 = val_pb4.array_value.values.add()
        array_val1.exclude_from_indexes = False
        array_val1.meaning = meaning = 22
        array_val1.blob_value = b"\xe2\x98\x83"
        array_val2 = val_pb4.array_value.values.add()
        array_val2.exclude_from_indexes = False
        array_val2.meaning = meaning
        array_val2.blob_value = b"\xe2\x98\x85"

        # Convert to the user-space Entity.
        entity = entity_from_protobuf(original_pb)
        # Convert the user-space Entity back to a protobuf.
        new_pb = self._call_fut(entity)

        # NOTE: entity_to_protobuf() strips the project so we "cheat".
        new_pb.key.partition_id.project_id = project
        self._compare_entity_proto(original_pb, new_pb)
Пример #8
0
    def test_inverts_to_protobuf(self):
        from google.cloud.datastore_v1.types import entity as entity_pb2
        from google.cloud.datastore.helpers import _new_value_pb
        from google.cloud.datastore.helpers import entity_from_protobuf

        original_pb = entity_pb2.Entity()
        # Add a key.
        original_pb.key.partition_id.project_id = project = "PROJECT"
        elem1 = original_pb._pb.key.path.add()
        elem1.kind = "Family"
        elem1.id = 1234
        elem2 = original_pb._pb.key.path.add()
        elem2.kind = "King"
        elem2.name = "Spades"

        # Add an integer property.
        val_pb1 = _new_value_pb(original_pb, "foo")
        val_pb1.integer_value = 1337
        val_pb1.exclude_from_indexes = True
        # Add a string property.
        val_pb2 = _new_value_pb(original_pb, "bar")
        val_pb2.string_value = u"hello"

        # Add a nested (entity) property.
        val_pb3 = _new_value_pb(original_pb, "entity-baz")
        sub_pb = entity_pb2.Entity()
        sub_val_pb1 = _new_value_pb(sub_pb, "x")
        sub_val_pb1.double_value = 3.14
        sub_val_pb2 = _new_value_pb(sub_pb, "y")
        sub_val_pb2.double_value = 2.718281828
        val_pb3.meaning = 9
        val_pb3.entity_value.CopyFrom(sub_pb._pb)

        # Add a list property.
        val_pb4 = _new_value_pb(original_pb, "list-quux")
        array_val1 = val_pb4.array_value.values.add()
        array_val1.exclude_from_indexes = False
        array_val1.meaning = meaning = 22
        array_val1.blob_value = b"\xe2\x98\x83"
        array_val2 = val_pb4.array_value.values.add()
        array_val2.exclude_from_indexes = False
        array_val2.meaning = meaning
        array_val2.blob_value = b"\xe2\x98\x85"

        # Convert to the user-space Entity.
        entity = entity_from_protobuf(original_pb)
        # Convert the user-space Entity back to a protobuf.
        new_pb = self._call_fut(entity)

        # NOTE: entity_to_protobuf() strips the project so we "cheat".
        new_pb.key.partition_id.project_id = project
        self._compare_entity_proto(original_pb, new_pb)
Пример #9
0
    def test_inverts_to_protobuf(self):
        from google.cloud.datastore._generated import entity_pb2
        from google.cloud.datastore.helpers import _new_value_pb
        from google.cloud.datastore.helpers import entity_from_protobuf

        original_pb = entity_pb2.Entity()
        # Add a key.
        original_pb.key.partition_id.project_id = project = 'PROJECT'
        elem1 = original_pb.key.path.add()
        elem1.kind = 'Family'
        elem1.id = 1234
        elem2 = original_pb.key.path.add()
        elem2.kind = 'King'
        elem2.name = 'Spades'

        # Add an integer property.
        val_pb1 = _new_value_pb(original_pb, 'foo')
        val_pb1.integer_value = 1337
        val_pb1.exclude_from_indexes = True
        # Add a string property.
        val_pb2 = _new_value_pb(original_pb, 'bar')
        val_pb2.string_value = u'hello'

        # Add a nested (entity) property.
        val_pb3 = _new_value_pb(original_pb, 'entity-baz')
        sub_pb = entity_pb2.Entity()
        sub_val_pb1 = _new_value_pb(sub_pb, 'x')
        sub_val_pb1.double_value = 3.14
        sub_val_pb2 = _new_value_pb(sub_pb, 'y')
        sub_val_pb2.double_value = 2.718281828
        val_pb3.meaning = 9
        val_pb3.entity_value.CopyFrom(sub_pb)

        # Add a list property.
        val_pb4 = _new_value_pb(original_pb, 'list-quux')
        array_val1 = val_pb4.array_value.values.add()
        array_val1.exclude_from_indexes = False
        array_val1.meaning = meaning = 22
        array_val1.blob_value = b'\xe2\x98\x83'
        array_val2 = val_pb4.array_value.values.add()
        array_val2.exclude_from_indexes = False
        array_val2.meaning = meaning
        array_val2.blob_value = b'\xe2\x98\x85'

        # Convert to the user-space Entity.
        entity = entity_from_protobuf(original_pb)
        # Convert the user-space Entity back to a protobuf.
        new_pb = self._call_fut(entity)

        # NOTE: entity_to_protobuf() strips the project so we "cheat".
        new_pb.key.partition_id.project_id = project
        self._compareEntityProto(original_pb, new_pb)
Пример #10
0
def _item_to_entity(iterator, entity_pb):
    """Convert a raw protobuf entity to the native object.

    :type iterator: :class:`~google.api_core.page_iterator.Iterator`
    :param iterator: The iterator that is currently in use.

    :type entity_pb:
        :class:`.entity_pb2.Entity`
    :param entity_pb: An entity protobuf to convert to a native entity.

    :rtype: :class:`~google.cloud.datastore.entity.Entity`
    :returns: The next entity in the page.
    """
    return helpers.entity_from_protobuf(entity_pb)
Пример #11
0
def _item_to_entity(iterator, entity_pb):
    """Convert a raw protobuf entity to the native object.

    :type iterator: :class:`~google.cloud.iterator.Iterator`
    :param iterator: The iterator that is currently in use.

    :type entity_pb:
        :class:`google.cloud.datastore._generated.entity_pb2.Entity`
    :param entity_pb: An entity protobuf to convert to a native entity.

    :rtype: :class:`~google.cloud.datastore.entity.Entity`
    :returns: The next entity in the page.
    """
    return helpers.entity_from_protobuf(entity_pb)
    def _call_fut(self, val):
        from google.cloud.datastore.helpers import entity_from_protobuf

        return entity_from_protobuf(val)
    def _call_fut(self, val):
        from google.cloud.datastore.helpers import entity_from_protobuf

        return entity_from_protobuf(val)
Пример #14
0
def ConvertToEntity(element):
    return helpers.entity_from_protobuf(element)
Пример #15
0
    def get_multi(
        self,
        keys,
        missing=None,
        deferred=None,
        transaction=None,
        eventual=False,
        retry=None,
        timeout=None,
    ):
        """Retrieve entities, along with their attributes.

        :type keys: list of :class:`google.cloud.datastore.key.Key`
        :param keys: The keys to be retrieved from the datastore.

        :type missing: list
        :param missing: (Optional) If a list is passed, the key-only entities
                        returned by the backend as "missing" will be copied
                        into it. If the list is not empty, an error will occur.

        :type deferred: list
        :param deferred: (Optional) If a list is passed, the keys returned
                         by the backend as "deferred" will be copied into it.
                         If the list is not empty, an error will occur.

        :type transaction:
            :class:`~google.cloud.datastore.transaction.Transaction`
        :param transaction: (Optional) Transaction to use for read consistency.
                            If not passed, uses current transaction, if set.

        :type eventual: bool
        :param eventual: (Optional) Defaults to strongly consistent (False).
                         Setting True will use eventual consistency, but cannot
                         be used inside a transaction or will raise ValueError.

        :type retry: :class:`google.api_core.retry.Retry`
        :param retry:
            A retry object used to retry requests. If ``None`` is specified,
            requests will be retried using a default configuration.

        :type timeout: float
        :param timeout:
            Time, in seconds, to wait for the request to complete.
            Note that if ``retry`` is specified, the timeout applies
            to each individual attempt.

        :rtype: list of :class:`google.cloud.datastore.entity.Entity`
        :returns: The requested entities.
        :raises: :class:`ValueError` if one or more of ``keys`` has a project
                 which does not match our project.
        :raises: :class:`ValueError` if eventual is True and in a transaction.
        """
        if not keys:
            return []

        ids = set(key.project for key in keys)
        for current_id in ids:
            if current_id != self.project:
                raise ValueError("Keys do not match project")

        if transaction is None:
            transaction = self.current_transaction

        entity_pbs = _extended_lookup(
            datastore_api=self._datastore_api,
            project=self.project,
            key_pbs=[key.to_protobuf() for key in keys],
            eventual=eventual,
            missing=missing,
            deferred=deferred,
            transaction_id=transaction and transaction.id,
            retry=retry,
            timeout=timeout,
        )

        if missing is not None:
            missing[:] = [
                helpers.entity_from_protobuf(missed_pb)
                for missed_pb in missing
            ]

        if deferred is not None:
            deferred[:] = [
                helpers.key_from_protobuf(deferred_pb)
                for deferred_pb in deferred
            ]

        return [
            helpers.entity_from_protobuf(entity_pb._pb)
            for entity_pb in entity_pbs
        ]
Пример #16
0
def ConvertToEntity(element):
    return helpers.entity_from_protobuf(element)