def test_ancestor_with_composite_filter():
key = key_module.Key("Foo", 123)
foo = model.StringProperty("foo")
food = model.StringProperty("food")
query = query_module.QueryOptions(
ancestor=key,
filters=query_module.AND(foo == "bar", food == "barn"),
)
query_pb = _datastore_query._query_to_protobuf(query)
filter_pb1 = query_pb2.PropertyFilter(
property=query_pb2.PropertyReference(name="foo"),
op=query_pb2.PropertyFilter.EQUAL,
value=entity_pb2.Value(string_value="bar"),
)
filter_pb2 = query_pb2.PropertyFilter(
property=query_pb2.PropertyReference(name="food"),
op=query_pb2.PropertyFilter.EQUAL,
value=entity_pb2.Value(string_value="barn"),
)
ancestor_pb = query_pb2.PropertyFilter(
property=query_pb2.PropertyReference(name="__key__"),
op=query_pb2.PropertyFilter.HAS_ANCESTOR,
)
ancestor_pb.value.key_value.CopyFrom(key._key.to_protobuf())
expected_pb = query_pb2.Query(filter=query_pb2.Filter(
composite_filter=query_pb2.CompositeFilter(
op=query_pb2.CompositeFilter.AND,
filters=[
query_pb2.Filter(property_filter=filter_pb1),
query_pb2.Filter(property_filter=filter_pb2),
query_pb2.Filter(property_filter=ancestor_pb),
],
)))
assert query_pb == expected_pb
def result(foo, bar=0, baz=""):
return _datastore_query._Result(
result_type=None,
result_pb=query_pb2.EntityResult(entity=entity_pb2.Entity(
properties={
"foo": entity_pb2.Value(string_value=foo),
"bar": entity_pb2.Value(integer_value=bar),
"baz": entity_pb2.Value(string_value=baz),
})),
order_by=[
query_module.PropertyOrder("foo"),
query_module.PropertyOrder("bar", reverse=True),
],
)
def test_dict_to_entity(self):
from google.cloud.datastore_v1.proto import entity_pb2
from google.cloud.datastore.entity import Entity
entity = Entity()
entity["a"] = {"b": u"c"}
entity_pb = self._call_fut(entity)
expected_pb = entity_pb2.Entity(
properties={
"a":
entity_pb2.Value(entity_value=entity_pb2.Entity(
properties={"b": entity_pb2.Value(string_value="c")}))
})
self.assertEqual(entity_pb, expected_pb)
def test_dict_to_entity(self):
from google.cloud.datastore_v1.proto import entity_pb2
from google.cloud.datastore.entity import Entity
entity = Entity()
entity['a'] = {'b': u'c'}
entity_pb = self._call_fut(entity)
expected_pb = entity_pb2.Entity(properties={
'a':
entity_pb2.Value(entity_value=entity_pb2.Entity(properties={
'b':
entity_pb2.Value(string_value='c', ),
}, ), ),
}, )
self.assertEqual(entity_pb, expected_pb)
def test_make_filter():
expected = query_pb2.PropertyFilter(
property=query_pb2.PropertyReference(name="harry"),
op=query_pb2.PropertyFilter.EQUAL,
value=entity_pb2.Value(string_value="Harold"),
)
assert _datastore_query.make_filter("harry", "=", u"Harold") == expected
def test_null(self):
from google.protobuf import struct_pb2
from google.cloud.datastore_v1.proto import entity_pb2
pb = entity_pb2.Value(null_value=struct_pb2.NULL_VALUE)
result = self._call_fut(pb)
self.assertIsNone(result)
def set_value_pb_item_value(value_pb, value, is_struct=False):
# type: (entity_pb2.Value, Any, bool) -> entity_pb2.Value
"""
Set a value attribute on the Value object based on the type of the provided value.
NOTE: For complex nested types (e.g. dicts and structs this function uses recursion).
:param is_struct: True if the provided value is part of a struct. This is important because
numbers inside struct field types are handled differently (only double number
types are supported).
"""
if isinstance(value, struct_pb2.ListValue):
# Cast special ListValue type to a list
value = cast(Any, value)
value = list(value)
if isinstance(value, float) and value.is_integer() and not is_struct:
# Special case because of how Protobuf handles ints in some scenarios (e.g. Struct)
# Regular Entity value supports integeres and double number types, but Struct mimics
# JSON so it only supports "number" type which is always a double
value = cast(Any, value)
value = int(value)
if isinstance(value, six.text_type):
value_pb.string_value = value
elif isinstance(value, bool):
value_pb.boolean_value = value
elif isinstance(value, int):
value_pb.integer_value = value
elif isinstance(value, float):
value_pb.double_value = value
elif isinstance(value, six.binary_type):
value_pb.blob_value = value
elif isinstance(value, list):
if len(value) == 0:
array_value = entity_pb2.ArrayValue(values=[])
value_pb.array_value.CopyFrom(array_value)
else:
for value in value:
value_pb_item = entity_pb2.Value()
value_pb_item = set_value_pb_item_value(value_pb=value_pb_item,
value=value,
is_struct=is_struct)
value_pb.array_value.values.append(value_pb_item)
elif isinstance(value, struct_pb2.Value):
item_value = _GetStructValue(value)
set_value_pb_item_value(value_pb, item_value, is_struct=is_struct)
elif hasattr(value, 'DESCRIPTOR'):
# Custom user-defined type
entity_pb_item = model_pb_to_entity_pb(value,
exclude_falsy_values=True)
value_pb.entity_value.CopyFrom(entity_pb_item)
elif value is None:
value_pb.null_value = struct_pb2.NULL_VALUE
else:
raise ValueError('Unsupported type for value: %s' % (value))
return value_pb
def test_single(self):
from google.cloud.datastore_v1.proto import entity_pb2
value_pb = entity_pb2.Value()
value_pb.meaning = meaning = 22
value_pb.string_value = u'hi'
result = self._call_fut(value_pb)
self.assertEqual(meaning, result)
def test_key(self):
from google.cloud.datastore_v1.proto import entity_pb2
from google.cloud.datastore.key import Key
pb = entity_pb2.Value()
expected = Key('KIND', 1234, project='PROJECT').to_protobuf()
pb.key_value.CopyFrom(expected)
found = self._call_fut(pb)
self.assertEqual(found.to_protobuf(), expected)
def test_empty_array_value(self):
from google.cloud.datastore_v1.proto import entity_pb2
value_pb = entity_pb2.Value()
value_pb.array_value.values.add()
value_pb.array_value.values.pop()
result = self._call_fut(value_pb, is_list=True)
self.assertEqual(None, result)
def test_array(self):
from google.cloud.datastore_v1.proto import entity_pb2
pb = entity_pb2.Value()
array_pb = pb.array_value.values
item_pb = array_pb.add()
item_pb.string_value = 'Foo'
item_pb = array_pb.add()
item_pb.string_value = 'Bar'
items = self._call_fut(pb)
self.assertEqual(items, ['Foo', 'Bar'])
def test_datetime(self):
import calendar
import datetime
from google.cloud._helpers import UTC
from google.cloud.datastore_v1.proto import entity_pb2
micros = 4375
utc = datetime.datetime(2014, 9, 16, 10, 19, 32, micros, UTC)
pb = entity_pb2.Value()
pb.timestamp_value.seconds = calendar.timegm(utc.timetuple())
pb.timestamp_value.nanos = 1000 * micros
self.assertEqual(self._call_fut(pb), utc)
def test_make_composite_and_filter():
filters = [
query_pb2.PropertyFilter(
property=query_pb2.PropertyReference(name="harry"),
op=query_pb2.PropertyFilter.EQUAL,
value=entity_pb2.Value(string_value="Harold"),
),
query_pb2.PropertyFilter(
property=query_pb2.PropertyReference(name="josie"),
op=query_pb2.PropertyFilter.EQUAL,
value=entity_pb2.Value(string_value="Josephine"),
),
]
expected = query_pb2.CompositeFilter(
op=query_pb2.CompositeFilter.AND,
filters=[
query_pb2.Filter(property_filter=sub_filter)
for sub_filter in filters
],
)
assert _datastore_query.make_composite_and_filter(filters) == expected
def test_dict_to_entity_recursive(self):
from google.cloud.datastore_v1.proto import entity_pb2
from google.cloud.datastore.entity import Entity
entity = Entity()
entity["a"] = {"b": {"c": {"d": 1.25}, "e": True}, "f": 10}
entity_pb = self._call_fut(entity)
b_entity_pb = entity_pb2.Entity(
properties={
"c":
entity_pb2.Value(entity_value=entity_pb2.Entity(
properties={"d": entity_pb2.Value(double_value=1.25)})),
"e":
entity_pb2.Value(boolean_value=True),
})
expected_pb = entity_pb2.Entity(
properties={
"a":
entity_pb2.Value(entity_value=entity_pb2.Entity(
properties={
"b": entity_pb2.Value(entity_value=b_entity_pb),
"f": entity_pb2.Value(integer_value=10),
}))
})
self.assertEqual(entity_pb, expected_pb)
def test_geo_point(self):
from google.type import latlng_pb2
from google.cloud.datastore_v1.proto import entity_pb2
from google.cloud.datastore.helpers import GeoPoint
lat = -3.14
lng = 13.37
geo_pt_pb = latlng_pb2.LatLng(latitude=lat, longitude=lng)
pb = entity_pb2.Value(geo_point_value=geo_pt_pb)
result = self._call_fut(pb)
self.assertIsInstance(result, GeoPoint)
self.assertEqual(result.latitude, lat)
self.assertEqual(result.longitude, lng)
def test_array_value(self):
from google.cloud.datastore_v1.proto import entity_pb2
value_pb = entity_pb2.Value()
meaning = 9
sub_value_pb1 = value_pb.array_value.values.add()
sub_value_pb2 = value_pb.array_value.values.add()
sub_value_pb1.meaning = sub_value_pb2.meaning = meaning
sub_value_pb1.string_value = u'hi'
sub_value_pb2.string_value = u'bye'
result = self._call_fut(value_pb, is_list=True)
self.assertEqual(meaning, result)
def test_array_value_meaning_partially_unset(self):
from google.cloud.datastore_v1.proto import entity_pb2
value_pb = entity_pb2.Value()
meaning1 = 9
sub_value_pb1 = value_pb.array_value.values.add()
sub_value_pb2 = value_pb.array_value.values.add()
sub_value_pb1.meaning = meaning1
sub_value_pb1.string_value = u'hi'
sub_value_pb2.string_value = u'bye'
result = self._call_fut(value_pb, is_list=True)
self.assertEqual(result, [meaning1, None])
def test_filter_pb():
foo = model.StringProperty("foo")
query = query_module.QueryOptions(kind="Foo", filters=(foo == "bar"))
query_pb = _datastore_query._query_to_protobuf(query)
filter_pb = query_pb2.PropertyFilter(
property=query_pb2.PropertyReference(name="foo"),
op=query_pb2.PropertyFilter.EQUAL,
value=entity_pb2.Value(string_value="bar"),
)
expected_pb = query_pb2.Query(
kind=[query_pb2.KindExpression(name="Foo")],
filter=query_pb2.Filter(property_filter=filter_pb),
)
assert query_pb == expected_pb
def test_entity(self):
from google.cloud.datastore_v1.proto import entity_pb2
from google.cloud.datastore.entity import Entity
from google.cloud.datastore.helpers import _new_value_pb
pb = entity_pb2.Value()
entity_pb = pb.entity_value
entity_pb.key.path.add(kind='KIND')
entity_pb.key.partition_id.project_id = 'PROJECT'
value_pb = _new_value_pb(entity_pb, 'foo')
value_pb.string_value = 'Foo'
entity = self._call_fut(pb)
self.assertIsInstance(entity, Entity)
self.assertEqual(entity['foo'], 'Foo')
def test_array_value_multiple_meanings(self):
from google.cloud.datastore_v1.proto import entity_pb2
value_pb = entity_pb2.Value()
meaning1 = 9
meaning2 = 10
sub_value_pb1 = value_pb.array_value.values.add()
sub_value_pb2 = value_pb.array_value.values.add()
sub_value_pb1.meaning = meaning1
sub_value_pb2.meaning = meaning2
sub_value_pb1.string_value = u"hi"
sub_value_pb2.string_value = u"bye"
result = self._call_fut(value_pb, is_list=True)
self.assertEqual(result, [meaning1, meaning2])
def test_index_mismatch_ignores_empty_list(self):
from google.cloud.datastore_v1.proto import entity_pb2
_PROJECT = "PROJECT"
_KIND = "KIND"
_ID = 1234
array_val_pb = entity_pb2.Value(array_value=entity_pb2.ArrayValue(
values=[]))
entity_pb = entity_pb2.Entity(properties={"baz": array_val_pb})
entity_pb.key.partition_id.project_id = _PROJECT
entity_pb.key.path.add(kind=_KIND, id=_ID)
entity = self._call_fut(entity_pb)
entity_dict = dict(entity)
self.assertEqual(entity_dict["baz"], [])
def test_dict_to_entity_recursive(self):
from google.cloud.datastore_v1.proto import entity_pb2
from google.cloud.datastore.entity import Entity
entity = Entity()
entity['a'] = {
'b': {
'c': {
'd': 1.25,
},
'e': True,
},
'f': 10,
}
entity_pb = self._call_fut(entity)
b_entity_pb = entity_pb2.Entity(
properties={
'c':
entity_pb2.Value(entity_value=entity_pb2.Entity(properties={
'd':
entity_pb2.Value(double_value=1.25, ),
}, ), ),
'e':
entity_pb2.Value(boolean_value=True),
})
expected_pb = entity_pb2.Entity(properties={
'a':
entity_pb2.Value(entity_value=entity_pb2.Entity(properties={
'b':
entity_pb2.Value(entity_value=b_entity_pb, ),
'f':
entity_pb2.Value(integer_value=10, ),
}, ), ),
}, )
self.assertEqual(entity_pb, expected_pb)
def model_pb_to_entity_pb(model_pb,
exclude_falsy_values=False,
exclude_from_index=None):
# type: (message.Message, bool, Optional[List[str]]) -> entity_pb2.Entity
"""
Translate Protobuf based database model object to Entity object which can be used with Google
Datastore client library.
:param model_pb: Instance of a custom Protobuf object to translate.
:param exclude_falsy_values: True to exclude field values which are falsy (e.g. None, False,
'', 0, etc.) and match the default values.
NOTE: Due to the design of protobuf v3, there is no way to
distinguish between a user explicitly providing a value which is
the same as a default value (e.g. 0 for an integer field) and
user not providing a value and default value being used instead.
:param exclude_from_index: Optional list of field names which should not be indexed. By
default, all the simple fields are indexed.
NOTE: If provided, this value has high precedence over
"exclude_from_index" message option defined on the model.
"""
exclude_from_index = exclude_from_index or []
if not isinstance(model_pb, message.Message):
raise ValueError(
'model_pb argument is not a valid Protobuf class instance')
fields = list(iter(model_pb.DESCRIPTOR.fields))
fields = [field for field in fields if field not in ['key']]
entity_pb = entity_pb2.Entity()
exclude_from_index = cast(list, exclude_from_index)
for field_descriptor in fields:
field_type = field_descriptor.type
field_name = field_descriptor.name
field_value = getattr(model_pb, field_name, None)
if field_value is None:
# Value not set or it uses a default value, skip it
# NOTE: proto3 syntax doesn't support HasField() anymore so there is now way for us to
# determine if a value is set / provided so we just use and return default values.
continue
if exclude_falsy_values and not field_value:
continue
attr_type = get_pb_attr_type(field_value)
value_pb = None
if attr_type == 'array_value':
if len(field_value) == 0:
value_pb = datastore.helpers._new_value_pb(
entity_pb, field_name)
array_value = entity_pb2.ArrayValue(values=[])
value_pb.array_value.CopyFrom(array_value)
else:
value_pb = datastore.helpers._new_value_pb(
entity_pb, field_name)
for value in field_value:
if field_type == descriptor.FieldDescriptor.TYPE_MESSAGE:
# Nested message type
entity_pb_item = model_pb_to_entity_pb(value)
value_pb_item = entity_pb2.Value()
# pylint: disable=no-member
value_pb_item.entity_value.CopyFrom(entity_pb_item)
# pylint: enable=no-member
else:
# Simple type
value_pb_item = entity_pb2.Value()
value_pb_item = set_value_pb_item_value(
value_pb=value_pb_item, value=value)
value_pb.array_value.values.append(value_pb_item)
elif field_type == descriptor.FieldDescriptor.TYPE_STRING:
value_pb = datastore.helpers._new_value_pb(entity_pb, field_name)
value_pb.string_value = field_value
elif field_type in [
descriptor.FieldDescriptor.TYPE_DOUBLE,
descriptor.FieldDescriptor.TYPE_FLOAT
]:
# NOTE: Datastore only supports double type so we map float to double
value_pb = datastore.helpers._new_value_pb(entity_pb, field_name)
value_pb.double_value = field_value
elif field_type in [
descriptor.FieldDescriptor.TYPE_INT32,
descriptor.FieldDescriptor.TYPE_INT64
]:
value_pb = datastore.helpers._new_value_pb(entity_pb, field_name)
value_pb.integer_value = field_value
elif field_type == descriptor.FieldDescriptor.TYPE_ENUM:
value_pb = datastore.helpers._new_value_pb(entity_pb, field_name)
if field_descriptor.enum_type.name == 'NullValue':
# NULL value
value_pb.null_value = struct_pb2.NULL_VALUE
else:
# Regular ENUM
value_pb.integer_value = field_value
elif field_type == descriptor.FieldDescriptor.TYPE_BOOL:
value_pb = datastore.helpers._new_value_pb(entity_pb, field_name)
value_pb.boolean_value = field_value
elif field_type == descriptor.FieldDescriptor.TYPE_BYTES:
value_pb = datastore.helpers._new_value_pb(entity_pb, field_name)
if isinstance(field_value, six.string_types):
field_value = field_value.encode('utf-8')
value_pb.blob_value = field_value
elif field_type == descriptor.FieldDescriptor.TYPE_MESSAGE:
# Complex type, convert to entity
field_type = model_pb.DESCRIPTOR.fields_by_name[field_name]
if field_type.message_type.full_name == 'google.protobuf.Timestamp':
if str(field_value) == '':
# Value not set
# TODO: Include default empty value?
# value_pb = datastore.helpers._new_value_pb(entity_pb, field_name)
# value_pb.timestamp_value.CopyFrom(field_value)
continue
value_pb = datastore.helpers._new_value_pb(
entity_pb, field_name)
value_pb.timestamp_value.CopyFrom(field_value)
elif field_type.message_type.full_name == 'google.type.LatLng':
if str(field_value) == '':
# Value not set
continue
value_pb = datastore.helpers._new_value_pb(
entity_pb, field_name)
value_pb.geo_point_value.CopyFrom(field_value)
elif isinstance(field_value, MessageMapContainer):
# Nested dictionary on a struct, set a value directory on a passed in pb object
# which is a parent Struct entity
entity_pb_item = get_entity_pb_for_value(value=field_value)
entity_pb.CopyFrom(entity_pb_item)
elif isinstance(field_value, ScalarMapContainer):
# Custom user defined type, recurse into it
value_pb = datastore.helpers._new_value_pb(
entity_pb, field_name)
entity_pb_item = get_entity_pb_for_value(value=field_value)
value_pb.entity_value.CopyFrom(entity_pb_item)
elif field_type.message_type.full_name == 'google.protobuf.Struct':
if not dict(field_value):
# Value not set, skip it
continue
value_pb = datastore.helpers._new_value_pb(
entity_pb, field_name)
entity_pb_item = get_entity_pb_for_value(value=field_value)
value_pb.entity_value.CopyFrom(entity_pb_item)
else:
# Nested type, potentially referenced from another Protobuf definition file
value_pb = datastore.helpers._new_value_pb(
entity_pb, field_name)
entity_pb_item = model_pb_to_entity_pb(field_value)
value_pb.entity_value.CopyFrom(entity_pb_item)
else:
raise ValueError('Unsupported field type for field "%s"' %
(field_name))
if not value_pb:
continue
value_pb = cast(Value, value_pb)
# Determine if field should be excluded from index
exclude_field_from_indexes = exclude_field_from_index(
model=model_pb,
field_descriptor=field_descriptor,
exclude_from_index=exclude_from_index)
if exclude_field_from_indexes:
# Field should be excluded from the index, mark that on the Entity Value
value_pb.exclude_from_indexes = True
return entity_pb
def _makePB(self, attr_name, value):
from google.cloud.datastore_v1.proto import entity_pb2
pb = entity_pb2.Value()
setattr(pb, attr_name, value)
return pb
def test_no_meaning(self):
from google.cloud.datastore_v1.proto import entity_pb2
value_pb = entity_pb2.Value()
result = self._call_fut(value_pb)
self.assertIsNone(result)
def _makePB(self):
from google.cloud.datastore_v1.proto import entity_pb2
return entity_pb2.Value()
def test_unknown(self):
from google.cloud.datastore_v1.proto import entity_pb2
pb = entity_pb2.Value()
with self.assertRaises(ValueError):
self._call_fut(pb)