def key_from_protobuf(pb):
"""Factory method for creating a key based on a protobuf.
The protobuf should be one returned from the Cloud Datastore
Protobuf API.
:type pb: :class:`gcloud.datastore._generated.entity_pb2.Key`
:param pb: The Protobuf representing the key.
:rtype: :class:`gcloud.datastore.key.Key`
:returns: a new `Key` instance
"""
path_args = []
for element in pb.path_element:
path_args.append(element.kind)
if _has_field(element, 'id'):
path_args.append(element.id)
# This is safe: we expect proto objects returned will only have
# one of `name` or `id` set.
if _has_field(element, 'name'):
path_args.append(element.name)
project = None
if _has_field(pb.partition_id, 'dataset_id'):
project = pb.partition_id.dataset_id
namespace = None
if _has_field(pb.partition_id, 'namespace'):
namespace = pb.partition_id.namespace
return Key(*path_args, namespace=namespace, project=project)
def key_from_protobuf(pb):
"""Factory method for creating a key based on a protobuf.
The protobuf should be one returned from the Cloud Datastore
Protobuf API.
:type pb: :class:`gcloud.datastore._generated.entity_pb2.Key`
:param pb: The Protobuf representing the key.
:rtype: :class:`gcloud.datastore.key.Key`
:returns: a new `Key` instance
"""
path_args = []
for element in pb.path_element:
path_args.append(element.kind)
if _has_field(element, 'id'):
path_args.append(element.id)
# This is safe: we expect proto objects returned will only have
# one of `name` or `id` set.
if _has_field(element, 'name'):
path_args.append(element.name)
project = None
if _has_field(pb.partition_id, 'dataset_id'):
project = pb.partition_id.dataset_id
namespace = None
if _has_field(pb.partition_id, 'namespace'):
namespace = pb.partition_id.namespace
return Key(*path_args, namespace=namespace, project=project)
def _prepare_key_for_request(key_pb):
"""Add protobuf keys to a request object.
:type key_pb: :class:`gcloud.datastore._generated.entity_pb2.Key`
:param key_pb: A key to be added to a request.
:rtype: :class:`gcloud.datastore._generated.entity_pb2.Key`
:returns: A key which will be added to a request. It will be the
original if nothing needs to be changed.
"""
if _has_field(key_pb.partition_id, 'dataset_id'):
# We remove the dataset_id from the protobuf. This is because
# the backend fails a request if the key contains un-prefixed
# project. The backend fails because requests to
# /datastore/.../datasets/foo/...
# and
# /datastore/.../datasets/s~foo/...
# both go to the datastore given by 's~foo'. So if the key
# protobuf in the request body has dataset_id='foo', the
# backend will reject since 'foo' != 's~foo'.
new_key_pb = _entity_pb2.Key()
new_key_pb.CopyFrom(key_pb)
new_key_pb.partition_id.ClearField('dataset_id')
key_pb = new_key_pb
return key_pb
def _prepare_key_for_request(key_pb):
"""Add protobuf keys to a request object.
:type key_pb: :class:`gcloud.datastore._generated.entity_pb2.Key`
:param key_pb: A key to be added to a request.
:rtype: :class:`gcloud.datastore._generated.entity_pb2.Key`
:returns: A key which will be added to a request. It will be the
original if nothing needs to be changed.
"""
if _has_field(key_pb.partition_id, 'dataset_id'):
# We remove the dataset_id from the protobuf. This is because
# the backend fails a request if the key contains un-prefixed
# project. The backend fails because requests to
# /datastore/.../datasets/foo/...
# and
# /datastore/.../datasets/s~foo/...
# both go to the datastore given by 's~foo'. So if the key
# protobuf in the request body has dataset_id='foo', the
# backend will reject since 'foo' != 's~foo'.
new_key_pb = _entity_pb2.Key()
new_key_pb.CopyFrom(key_pb)
new_key_pb.partition_id.ClearField('dataset_id')
key_pb = new_key_pb
return key_pb
def _compare_key_pb_after_request(test, key_before, key_after):
from gcloud._helpers import _has_field
test.assertFalse(_has_field(key_after.partition_id, 'dataset_id'))
test.assertEqual(key_before.partition_id.namespace,
key_after.partition_id.namespace)
test.assertEqual(len(key_before.path_element), len(key_after.path_element))
for elt1, elt2 in zip(key_before.path_element, key_after.path_element):
test.assertEqual(elt1, elt2)
def test_to_protobuf_w_no_kind(self):
from gcloud._helpers import _has_field
key = self._makeOne('KIND', project=self._DEFAULT_PROJECT)
# Force the 'kind' to be unset. Maybe `to_protobuf` should fail
# on this? The backend certainly will.
key._path[-1].pop('kind')
pb = key.to_protobuf()
self.assertFalse(_has_field(pb.path_element[0], 'kind'))
def _compare_key_pb_after_request(test, key_before, key_after):
from gcloud._helpers import _has_field
test.assertFalse(_has_field(key_after.partition_id, 'dataset_id'))
test.assertEqual(key_before.partition_id.namespace,
key_after.partition_id.namespace)
test.assertEqual(len(key_before.path_element),
len(key_after.path_element))
for elt1, elt2 in zip(key_before.path_element, key_after.path_element):
test.assertEqual(elt1, elt2)
def test_to_protobuf_defaults(self):
from gcloud._helpers import _has_field
from gcloud.datastore._generated import entity_pb2
_KIND = 'KIND'
key = self._makeOne(_KIND, project=self._DEFAULT_PROJECT)
pb = key.to_protobuf()
self.assertTrue(isinstance(pb, entity_pb2.Key))
# Check partition ID.
self.assertEqual(pb.partition_id.dataset_id, self._DEFAULT_PROJECT)
self.assertEqual(pb.partition_id.namespace, '')
self.assertFalse(_has_field(pb.partition_id, 'namespace'))
# Check the element PB matches the partial key and kind.
elem, = list(pb.path_element)
self.assertEqual(elem.kind, _KIND)
self.assertEqual(elem.name, '')
self.assertFalse(_has_field(elem, 'name'))
self.assertEqual(elem.id, 0)
self.assertFalse(_has_field(elem, 'id'))
def _get_value_from_value_pb(value_pb):
"""Given a protobuf for a Value, get the correct value.
The Cloud Datastore Protobuf API returns a Property Protobuf which
has one value set and the rest blank. This function retrieves the
the one value provided.
Some work is done to coerce the return value into a more useful type
(particularly in the case of a timestamp value, or a key value).
:type value_pb: :class:`gcloud.datastore._generated.entity_pb2.Value`
:param value_pb: The Value Protobuf.
:returns: The value provided by the Protobuf.
"""
result = None
if _has_field(value_pb, 'timestamp_microseconds_value'):
microseconds = value_pb.timestamp_microseconds_value
result = _datetime_from_microseconds(microseconds)
elif _has_field(value_pb, 'key_value'):
result = key_from_protobuf(value_pb.key_value)
elif _has_field(value_pb, 'boolean_value'):
result = value_pb.boolean_value
elif _has_field(value_pb, 'double_value'):
result = value_pb.double_value
elif _has_field(value_pb, 'integer_value'):
result = value_pb.integer_value
elif _has_field(value_pb, 'string_value'):
result = value_pb.string_value
elif _has_field(value_pb, 'blob_value'):
result = value_pb.blob_value
elif _has_field(value_pb, 'entity_value'):
result = entity_from_protobuf(value_pb.entity_value)
elif value_pb.list_value:
result = [
_get_value_from_value_pb(value) for value in value_pb.list_value
]
return result
def entity_from_protobuf(pb):
"""Factory method for creating an entity based on a protobuf.
The protobuf should be one returned from the Cloud Datastore
Protobuf API.
:type pb: :class:`gcloud.datastore._generated.entity_pb2.Entity`
:param pb: The Protobuf representing the entity.
:rtype: :class:`gcloud.datastore.entity.Entity`
:returns: The entity derived from the protobuf.
"""
key = None
if _has_field(pb, 'key'):
key = key_from_protobuf(pb.key)
entity_props = {}
entity_meanings = {}
exclude_from_indexes = []
for prop_name, value_pb in _property_tuples(pb):
value = _get_value_from_value_pb(value_pb)
entity_props[prop_name] = value
# Check if the property has an associated meaning.
is_list = isinstance(value, list)
meaning = _get_meaning(value_pb, is_list=is_list)
if meaning is not None:
entity_meanings[prop_name] = (meaning, value)
# Check if ``value_pb`` was indexed. Lists need to be special-cased
# and we require all ``indexed`` values in a list agree.
if is_list:
indexed_values = set(value_pb.indexed
for value_pb in value_pb.list_value)
if len(indexed_values) != 1:
raise ValueError('For a list_value, subvalues must either all '
'be indexed or all excluded from indexes.')
if not indexed_values.pop():
exclude_from_indexes.append(prop_name)
else:
if not value_pb.indexed:
exclude_from_indexes.append(prop_name)
entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes)
entity.update(entity_props)
entity._meanings.update(entity_meanings)
return entity
def entity_from_protobuf(pb):
"""Factory method for creating an entity based on a protobuf.
The protobuf should be one returned from the Cloud Datastore
Protobuf API.
:type pb: :class:`gcloud.datastore._generated.entity_pb2.Entity`
:param pb: The Protobuf representing the entity.
:rtype: :class:`gcloud.datastore.entity.Entity`
:returns: The entity derived from the protobuf.
"""
key = None
if _has_field(pb, 'key'):
key = key_from_protobuf(pb.key)
entity_props = {}
entity_meanings = {}
exclude_from_indexes = []
for prop_name, value_pb in _property_tuples(pb):
value = _get_value_from_value_pb(value_pb)
entity_props[prop_name] = value
# Check if the property has an associated meaning.
is_list = isinstance(value, list)
meaning = _get_meaning(value_pb, is_list=is_list)
if meaning is not None:
entity_meanings[prop_name] = (meaning, value)
# Check if ``value_pb`` was indexed. Lists need to be special-cased
# and we require all ``indexed`` values in a list agree.
if is_list:
indexed_values = set(value_pb.indexed
for value_pb in value_pb.list_value)
if len(indexed_values) != 1:
raise ValueError('For a list_value, subvalues must either all '
'be indexed or all excluded from indexes.')
if not indexed_values.pop():
exclude_from_indexes.append(prop_name)
else:
if not value_pb.indexed:
exclude_from_indexes.append(prop_name)
entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes)
entity.update(entity_props)
entity._meanings.update(entity_meanings)
return entity
def _get_value_from_value_pb(value_pb):
"""Given a protobuf for a Value, get the correct value.
The Cloud Datastore Protobuf API returns a Property Protobuf which
has one value set and the rest blank. This function retrieves the
the one value provided.
Some work is done to coerce the return value into a more useful type
(particularly in the case of a timestamp value, or a key value).
:type value_pb: :class:`gcloud.datastore._generated.entity_pb2.Value`
:param value_pb: The Value Protobuf.
:returns: The value provided by the Protobuf.
"""
result = None
if _has_field(value_pb, 'timestamp_microseconds_value'):
microseconds = value_pb.timestamp_microseconds_value
result = _datetime_from_microseconds(microseconds)
elif _has_field(value_pb, 'key_value'):
result = key_from_protobuf(value_pb.key_value)
elif _has_field(value_pb, 'boolean_value'):
result = value_pb.boolean_value
elif _has_field(value_pb, 'double_value'):
result = value_pb.double_value
elif _has_field(value_pb, 'integer_value'):
result = value_pb.integer_value
elif _has_field(value_pb, 'string_value'):
result = value_pb.string_value
elif _has_field(value_pb, 'blob_value'):
result = value_pb.blob_value
elif _has_field(value_pb, 'entity_value'):
result = entity_from_protobuf(value_pb.entity_value)
elif value_pb.list_value:
result = [_get_value_from_value_pb(value)
for value in value_pb.list_value]
return result
def _compareEntityProto(self, entity_pb1, entity_pb2):
from gcloud._helpers import _has_field
from gcloud.datastore.helpers import _property_tuples
self.assertEqual(entity_pb1.key, entity_pb2.key)
value_list1 = sorted(_property_tuples(entity_pb1))
value_list2 = sorted(_property_tuples(entity_pb2))
self.assertEqual(len(value_list1), len(value_list2))
for pair1, pair2 in zip(value_list1, value_list2):
name1, val1 = pair1
name2, val2 = pair2
self.assertEqual(name1, name2)
if _has_field(val1, 'entity_value'):
self.assertEqual(val1.meaning, val2.meaning)
self._compareEntityProto(val1.entity_value, val2.entity_value)
else:
self.assertEqual(val1, val2)
def _compareEntityProto(self, entity_pb1, entity_pb2):
from gcloud._helpers import _has_field
from gcloud.datastore.helpers import _property_tuples
self.assertEqual(entity_pb1.key, entity_pb2.key)
value_list1 = sorted(_property_tuples(entity_pb1))
value_list2 = sorted(_property_tuples(entity_pb2))
self.assertEqual(len(value_list1), len(value_list2))
for pair1, pair2 in zip(value_list1, value_list2):
name1, val1 = pair1
name2, val2 = pair2
self.assertEqual(name1, name2)
if _has_field(val1, 'entity_value'):
self.assertEqual(val1.meaning, val2.meaning)
self._compareEntityProto(val1.entity_value,
val2.entity_value)
else:
self.assertEqual(val1, val2)
def _prepare_key_for_request(key_pb): # pragma: NO COVER copied from helpers
"""Add protobuf keys to a request object.
.. note::
This is copied from `helpers` to avoid a cycle:
_implicit_environ -> connection -> helpers -> key -> _implicit_environ
:type key_pb: :class:`gcloud.datastore._generated.entity_pb2.Key`
:param key_pb: A key to be added to a request.
:rtype: :class:`gcloud.datastore._generated.entity_pb2.Key`
:returns: A key which will be added to a request. It will be the
original if nothing needs to be changed.
"""
if _has_field(key_pb.partition_id, 'dataset_id'):
new_key_pb = _entity_pb2.Key()
new_key_pb.CopyFrom(key_pb)
new_key_pb.partition_id.ClearField('dataset_id')
key_pb = new_key_pb
return key_pb
def test_unprefixed_bogus_key_hit(self):
from gcloud._helpers import _has_field
UNPREFIXED = 'PROJECT'
PREFIX = 'e~'
CONNECTION = _Connection(PREFIX, from_missing=True)
result = self._callFUT(UNPREFIXED, CONNECTION)
self.assertEqual(CONNECTION._called_project, UNPREFIXED)
self.assertEqual(CONNECTION._lookup_result, [])
# Make sure just one.
called_key_pb, = CONNECTION._called_key_pbs
path_element = called_key_pb.path_element
self.assertEqual(len(path_element), 1)
self.assertEqual(path_element[0].kind, '__MissingLookupKind')
self.assertEqual(path_element[0].id, 1)
self.assertFalse(_has_field(path_element[0], 'name'))
PREFIXED = PREFIX + UNPREFIXED
self.assertEqual(result, PREFIXED)
def test_unprefixed_bogus_key_hit(self):
from gcloud._helpers import _has_field
UNPREFIXED = 'PROJECT'
PREFIX = 'e~'
CONNECTION = _Connection(PREFIX, from_missing=True)
result = self._callFUT(UNPREFIXED, CONNECTION)
self.assertEqual(CONNECTION._called_project, UNPREFIXED)
self.assertEqual(CONNECTION._lookup_result, [])
# Make sure just one.
called_key_pb, = CONNECTION._called_key_pbs
path_element = called_key_pb.path_element
self.assertEqual(len(path_element), 1)
self.assertEqual(path_element[0].kind, '__MissingLookupKind')
self.assertEqual(path_element[0].id, 1)
self.assertFalse(_has_field(path_element[0], 'name'))
PREFIXED = PREFIX + UNPREFIXED
self.assertEqual(result, PREFIXED)
def _get_meaning(value_pb, is_list=False):
"""Get the meaning from a protobuf value.
:type value_pb: :class:`gcloud.datastore._generated.entity_pb2.Value`
:param value_pb: The protobuf value to be checked for an
associated meaning.
:type is_list: bool
:param is_list: Boolean indicating if the ``value_pb`` contains
a list value.
:rtype: int
:returns: The meaning for the ``value_pb`` if one is set, else
:data:`None`.
:raises: :class:`ValueError <exceptions.ValueError>` if a list value
has disagreeing meanings (in sub-elements) or has some
elements with meanings and some without.
"""
meaning = None
if is_list:
# An empty list will have no values, hence no shared meaning
# set among them.
if len(value_pb.list_value) == 0:
return None
# We check among all the meanings, some of which may be None,
# the rest which may be enum/int values.
all_meanings = set(
_get_meaning(sub_value_pb) for sub_value_pb in value_pb.list_value)
meaning = all_meanings.pop()
# The value we popped off should have been unique. If not
# then we can't handle a list with values that have more
# than one meaning.
if all_meanings:
raise ValueError('Different meanings set on values '
'within a list_value')
elif _has_field(value_pb, 'meaning'):
meaning = value_pb.meaning
return meaning
def _get_meaning(value_pb, is_list=False):
"""Get the meaning from a protobuf value.
:type value_pb: :class:`gcloud.datastore._generated.entity_pb2.Value`
:param value_pb: The protobuf value to be checked for an
associated meaning.
:type is_list: bool
:param is_list: Boolean indicating if the ``value_pb`` contains
a list value.
:rtype: int
:returns: The meaning for the ``value_pb`` if one is set, else
:data:`None`.
:raises: :class:`ValueError <exceptions.ValueError>` if a list value
has disagreeing meanings (in sub-elements) or has some
elements with meanings and some without.
"""
meaning = None
if is_list:
# An empty list will have no values, hence no shared meaning
# set among them.
if len(value_pb.list_value) == 0:
return None
# We check among all the meanings, some of which may be None,
# the rest which may be enum/int values.
all_meanings = set(_get_meaning(sub_value_pb)
for sub_value_pb in value_pb.list_value)
meaning = all_meanings.pop()
# The value we popped off should have been unique. If not
# then we can't handle a list with values that have more
# than one meaning.
if all_meanings:
raise ValueError('Different meanings set on values '
'within a list_value')
elif _has_field(value_pb, 'meaning'):
meaning = value_pb.meaning
return meaning
def _callFUT(self, message_pb, property_name):
from gcloud._helpers import _has_field
return _has_field(message_pb, property_name)
def _callFUT(self, message_pb, property_name):
from gcloud._helpers import _has_field
return _has_field(message_pb, property_name)
