def test_ancestor_with_composite_filter(): key = key_module.Key("Foo", 123) foo = model.StringProperty("foo") food = model.StringProperty("food") query = query_module.QueryOptions( ancestor=key, filters=query_module.AND(foo == "bar", food == "barn"), ) query_pb = _datastore_query._query_to_protobuf(query) filter_pb1 = query_pb2.PropertyFilter( property=query_pb2.PropertyReference(name="foo"), op=query_pb2.PropertyFilter.EQUAL, value=entity_pb2.Value(string_value="bar"), ) filter_pb2 = query_pb2.PropertyFilter( property=query_pb2.PropertyReference(name="food"), op=query_pb2.PropertyFilter.EQUAL, value=entity_pb2.Value(string_value="barn"), ) ancestor_pb = query_pb2.PropertyFilter( property=query_pb2.PropertyReference(name="__key__"), op=query_pb2.PropertyFilter.HAS_ANCESTOR, ) ancestor_pb.value.key_value.CopyFrom(key._key.to_protobuf()) expected_pb = query_pb2.Query(filter=query_pb2.Filter( composite_filter=query_pb2.CompositeFilter( op=query_pb2.CompositeFilter.AND, filters=[ query_pb2.Filter(property_filter=filter_pb1), query_pb2.Filter(property_filter=filter_pb2), query_pb2.Filter(property_filter=ancestor_pb), ], ))) assert query_pb == expected_pb
def result(foo, bar=0, baz=""): return _datastore_query._Result( result_type=None, result_pb=query_pb2.EntityResult(entity=entity_pb2.Entity( properties={ "foo": entity_pb2.Value(string_value=foo), "bar": entity_pb2.Value(integer_value=bar), "baz": entity_pb2.Value(string_value=baz), })), order_by=[ query_module.PropertyOrder("foo"), query_module.PropertyOrder("bar", reverse=True), ], )
def test_dict_to_entity(self): from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore.entity import Entity entity = Entity() entity["a"] = {"b": u"c"} entity_pb = self._call_fut(entity) expected_pb = entity_pb2.Entity( properties={ "a": entity_pb2.Value(entity_value=entity_pb2.Entity( properties={"b": entity_pb2.Value(string_value="c")})) }) self.assertEqual(entity_pb, expected_pb)
def test_dict_to_entity(self): from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore.entity import Entity entity = Entity() entity['a'] = {'b': u'c'} entity_pb = self._call_fut(entity) expected_pb = entity_pb2.Entity(properties={ 'a': entity_pb2.Value(entity_value=entity_pb2.Entity(properties={ 'b': entity_pb2.Value(string_value='c', ), }, ), ), }, ) self.assertEqual(entity_pb, expected_pb)
def test_make_filter(): expected = query_pb2.PropertyFilter( property=query_pb2.PropertyReference(name="harry"), op=query_pb2.PropertyFilter.EQUAL, value=entity_pb2.Value(string_value="Harold"), ) assert _datastore_query.make_filter("harry", "=", u"Harold") == expected
def test_null(self): from google.protobuf import struct_pb2 from google.cloud.datastore_v1.proto import entity_pb2 pb = entity_pb2.Value(null_value=struct_pb2.NULL_VALUE) result = self._call_fut(pb) self.assertIsNone(result)
def set_value_pb_item_value(value_pb, value, is_struct=False): # type: (entity_pb2.Value, Any, bool) -> entity_pb2.Value """ Set a value attribute on the Value object based on the type of the provided value. NOTE: For complex nested types (e.g. dicts and structs this function uses recursion). :param is_struct: True if the provided value is part of a struct. This is important because numbers inside struct field types are handled differently (only double number types are supported). """ if isinstance(value, struct_pb2.ListValue): # Cast special ListValue type to a list value = cast(Any, value) value = list(value) if isinstance(value, float) and value.is_integer() and not is_struct: # Special case because of how Protobuf handles ints in some scenarios (e.g. Struct) # Regular Entity value supports integeres and double number types, but Struct mimics # JSON so it only supports "number" type which is always a double value = cast(Any, value) value = int(value) if isinstance(value, six.text_type): value_pb.string_value = value elif isinstance(value, bool): value_pb.boolean_value = value elif isinstance(value, int): value_pb.integer_value = value elif isinstance(value, float): value_pb.double_value = value elif isinstance(value, six.binary_type): value_pb.blob_value = value elif isinstance(value, list): if len(value) == 0: array_value = entity_pb2.ArrayValue(values=[]) value_pb.array_value.CopyFrom(array_value) else: for value in value: value_pb_item = entity_pb2.Value() value_pb_item = set_value_pb_item_value(value_pb=value_pb_item, value=value, is_struct=is_struct) value_pb.array_value.values.append(value_pb_item) elif isinstance(value, struct_pb2.Value): item_value = _GetStructValue(value) set_value_pb_item_value(value_pb, item_value, is_struct=is_struct) elif hasattr(value, 'DESCRIPTOR'): # Custom user-defined type entity_pb_item = model_pb_to_entity_pb(value, exclude_falsy_values=True) value_pb.entity_value.CopyFrom(entity_pb_item) elif value is None: value_pb.null_value = struct_pb2.NULL_VALUE else: raise ValueError('Unsupported type for value: %s' % (value)) return value_pb
def test_single(self): from google.cloud.datastore_v1.proto import entity_pb2 value_pb = entity_pb2.Value() value_pb.meaning = meaning = 22 value_pb.string_value = u'hi' result = self._call_fut(value_pb) self.assertEqual(meaning, result)
def test_key(self): from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore.key import Key pb = entity_pb2.Value() expected = Key('KIND', 1234, project='PROJECT').to_protobuf() pb.key_value.CopyFrom(expected) found = self._call_fut(pb) self.assertEqual(found.to_protobuf(), expected)
def test_empty_array_value(self): from google.cloud.datastore_v1.proto import entity_pb2 value_pb = entity_pb2.Value() value_pb.array_value.values.add() value_pb.array_value.values.pop() result = self._call_fut(value_pb, is_list=True) self.assertEqual(None, result)
def test_array(self): from google.cloud.datastore_v1.proto import entity_pb2 pb = entity_pb2.Value() array_pb = pb.array_value.values item_pb = array_pb.add() item_pb.string_value = 'Foo' item_pb = array_pb.add() item_pb.string_value = 'Bar' items = self._call_fut(pb) self.assertEqual(items, ['Foo', 'Bar'])
def test_datetime(self): import calendar import datetime from google.cloud._helpers import UTC from google.cloud.datastore_v1.proto import entity_pb2 micros = 4375 utc = datetime.datetime(2014, 9, 16, 10, 19, 32, micros, UTC) pb = entity_pb2.Value() pb.timestamp_value.seconds = calendar.timegm(utc.timetuple()) pb.timestamp_value.nanos = 1000 * micros self.assertEqual(self._call_fut(pb), utc)
def test_make_composite_and_filter(): filters = [ query_pb2.PropertyFilter( property=query_pb2.PropertyReference(name="harry"), op=query_pb2.PropertyFilter.EQUAL, value=entity_pb2.Value(string_value="Harold"), ), query_pb2.PropertyFilter( property=query_pb2.PropertyReference(name="josie"), op=query_pb2.PropertyFilter.EQUAL, value=entity_pb2.Value(string_value="Josephine"), ), ] expected = query_pb2.CompositeFilter( op=query_pb2.CompositeFilter.AND, filters=[ query_pb2.Filter(property_filter=sub_filter) for sub_filter in filters ], ) assert _datastore_query.make_composite_and_filter(filters) == expected
def test_dict_to_entity_recursive(self): from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore.entity import Entity entity = Entity() entity["a"] = {"b": {"c": {"d": 1.25}, "e": True}, "f": 10} entity_pb = self._call_fut(entity) b_entity_pb = entity_pb2.Entity( properties={ "c": entity_pb2.Value(entity_value=entity_pb2.Entity( properties={"d": entity_pb2.Value(double_value=1.25)})), "e": entity_pb2.Value(boolean_value=True), }) expected_pb = entity_pb2.Entity( properties={ "a": entity_pb2.Value(entity_value=entity_pb2.Entity( properties={ "b": entity_pb2.Value(entity_value=b_entity_pb), "f": entity_pb2.Value(integer_value=10), })) }) self.assertEqual(entity_pb, expected_pb)
def test_geo_point(self): from google.type import latlng_pb2 from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore.helpers import GeoPoint lat = -3.14 lng = 13.37 geo_pt_pb = latlng_pb2.LatLng(latitude=lat, longitude=lng) pb = entity_pb2.Value(geo_point_value=geo_pt_pb) result = self._call_fut(pb) self.assertIsInstance(result, GeoPoint) self.assertEqual(result.latitude, lat) self.assertEqual(result.longitude, lng)
def test_array_value(self): from google.cloud.datastore_v1.proto import entity_pb2 value_pb = entity_pb2.Value() meaning = 9 sub_value_pb1 = value_pb.array_value.values.add() sub_value_pb2 = value_pb.array_value.values.add() sub_value_pb1.meaning = sub_value_pb2.meaning = meaning sub_value_pb1.string_value = u'hi' sub_value_pb2.string_value = u'bye' result = self._call_fut(value_pb, is_list=True) self.assertEqual(meaning, result)
def test_array_value_meaning_partially_unset(self): from google.cloud.datastore_v1.proto import entity_pb2 value_pb = entity_pb2.Value() meaning1 = 9 sub_value_pb1 = value_pb.array_value.values.add() sub_value_pb2 = value_pb.array_value.values.add() sub_value_pb1.meaning = meaning1 sub_value_pb1.string_value = u'hi' sub_value_pb2.string_value = u'bye' result = self._call_fut(value_pb, is_list=True) self.assertEqual(result, [meaning1, None])
def test_filter_pb(): foo = model.StringProperty("foo") query = query_module.QueryOptions(kind="Foo", filters=(foo == "bar")) query_pb = _datastore_query._query_to_protobuf(query) filter_pb = query_pb2.PropertyFilter( property=query_pb2.PropertyReference(name="foo"), op=query_pb2.PropertyFilter.EQUAL, value=entity_pb2.Value(string_value="bar"), ) expected_pb = query_pb2.Query( kind=[query_pb2.KindExpression(name="Foo")], filter=query_pb2.Filter(property_filter=filter_pb), ) assert query_pb == expected_pb
def test_entity(self): from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore.entity import Entity from google.cloud.datastore.helpers import _new_value_pb pb = entity_pb2.Value() entity_pb = pb.entity_value entity_pb.key.path.add(kind='KIND') entity_pb.key.partition_id.project_id = 'PROJECT' value_pb = _new_value_pb(entity_pb, 'foo') value_pb.string_value = 'Foo' entity = self._call_fut(pb) self.assertIsInstance(entity, Entity) self.assertEqual(entity['foo'], 'Foo')
def test_array_value_multiple_meanings(self): from google.cloud.datastore_v1.proto import entity_pb2 value_pb = entity_pb2.Value() meaning1 = 9 meaning2 = 10 sub_value_pb1 = value_pb.array_value.values.add() sub_value_pb2 = value_pb.array_value.values.add() sub_value_pb1.meaning = meaning1 sub_value_pb2.meaning = meaning2 sub_value_pb1.string_value = u"hi" sub_value_pb2.string_value = u"bye" result = self._call_fut(value_pb, is_list=True) self.assertEqual(result, [meaning1, meaning2])
def test_index_mismatch_ignores_empty_list(self): from google.cloud.datastore_v1.proto import entity_pb2 _PROJECT = "PROJECT" _KIND = "KIND" _ID = 1234 array_val_pb = entity_pb2.Value(array_value=entity_pb2.ArrayValue( values=[])) entity_pb = entity_pb2.Entity(properties={"baz": array_val_pb}) entity_pb.key.partition_id.project_id = _PROJECT entity_pb.key.path.add(kind=_KIND, id=_ID) entity = self._call_fut(entity_pb) entity_dict = dict(entity) self.assertEqual(entity_dict["baz"], [])
def test_dict_to_entity_recursive(self): from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore.entity import Entity entity = Entity() entity['a'] = { 'b': { 'c': { 'd': 1.25, }, 'e': True, }, 'f': 10, } entity_pb = self._call_fut(entity) b_entity_pb = entity_pb2.Entity( properties={ 'c': entity_pb2.Value(entity_value=entity_pb2.Entity(properties={ 'd': entity_pb2.Value(double_value=1.25, ), }, ), ), 'e': entity_pb2.Value(boolean_value=True), }) expected_pb = entity_pb2.Entity(properties={ 'a': entity_pb2.Value(entity_value=entity_pb2.Entity(properties={ 'b': entity_pb2.Value(entity_value=b_entity_pb, ), 'f': entity_pb2.Value(integer_value=10, ), }, ), ), }, ) self.assertEqual(entity_pb, expected_pb)
def model_pb_to_entity_pb(model_pb, exclude_falsy_values=False, exclude_from_index=None): # type: (message.Message, bool, Optional[List[str]]) -> entity_pb2.Entity """ Translate Protobuf based database model object to Entity object which can be used with Google Datastore client library. :param model_pb: Instance of a custom Protobuf object to translate. :param exclude_falsy_values: True to exclude field values which are falsy (e.g. None, False, '', 0, etc.) and match the default values. NOTE: Due to the design of protobuf v3, there is no way to distinguish between a user explicitly providing a value which is the same as a default value (e.g. 0 for an integer field) and user not providing a value and default value being used instead. :param exclude_from_index: Optional list of field names which should not be indexed. By default, all the simple fields are indexed. NOTE: If provided, this value has high precedence over "exclude_from_index" message option defined on the model. """ exclude_from_index = exclude_from_index or [] if not isinstance(model_pb, message.Message): raise ValueError( 'model_pb argument is not a valid Protobuf class instance') fields = list(iter(model_pb.DESCRIPTOR.fields)) fields = [field for field in fields if field not in ['key']] entity_pb = entity_pb2.Entity() exclude_from_index = cast(list, exclude_from_index) for field_descriptor in fields: field_type = field_descriptor.type field_name = field_descriptor.name field_value = getattr(model_pb, field_name, None) if field_value is None: # Value not set or it uses a default value, skip it # NOTE: proto3 syntax doesn't support HasField() anymore so there is now way for us to # determine if a value is set / provided so we just use and return default values. continue if exclude_falsy_values and not field_value: continue attr_type = get_pb_attr_type(field_value) value_pb = None if attr_type == 'array_value': if len(field_value) == 0: value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) array_value = entity_pb2.ArrayValue(values=[]) value_pb.array_value.CopyFrom(array_value) else: value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) for value in field_value: if field_type == descriptor.FieldDescriptor.TYPE_MESSAGE: # Nested message type entity_pb_item = model_pb_to_entity_pb(value) value_pb_item = entity_pb2.Value() # pylint: disable=no-member value_pb_item.entity_value.CopyFrom(entity_pb_item) # pylint: enable=no-member else: # Simple type value_pb_item = entity_pb2.Value() value_pb_item = set_value_pb_item_value( value_pb=value_pb_item, value=value) value_pb.array_value.values.append(value_pb_item) elif field_type == descriptor.FieldDescriptor.TYPE_STRING: value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) value_pb.string_value = field_value elif field_type in [ descriptor.FieldDescriptor.TYPE_DOUBLE, descriptor.FieldDescriptor.TYPE_FLOAT ]: # NOTE: Datastore only supports double type so we map float to double value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) value_pb.double_value = field_value elif field_type in [ descriptor.FieldDescriptor.TYPE_INT32, descriptor.FieldDescriptor.TYPE_INT64 ]: value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) value_pb.integer_value = field_value elif field_type == descriptor.FieldDescriptor.TYPE_ENUM: value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) if field_descriptor.enum_type.name == 'NullValue': # NULL value value_pb.null_value = struct_pb2.NULL_VALUE else: # Regular ENUM value_pb.integer_value = field_value elif field_type == descriptor.FieldDescriptor.TYPE_BOOL: value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) value_pb.boolean_value = field_value elif field_type == descriptor.FieldDescriptor.TYPE_BYTES: value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) if isinstance(field_value, six.string_types): field_value = field_value.encode('utf-8') value_pb.blob_value = field_value elif field_type == descriptor.FieldDescriptor.TYPE_MESSAGE: # Complex type, convert to entity field_type = model_pb.DESCRIPTOR.fields_by_name[field_name] if field_type.message_type.full_name == 'google.protobuf.Timestamp': if str(field_value) == '': # Value not set # TODO: Include default empty value? # value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) # value_pb.timestamp_value.CopyFrom(field_value) continue value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) value_pb.timestamp_value.CopyFrom(field_value) elif field_type.message_type.full_name == 'google.type.LatLng': if str(field_value) == '': # Value not set continue value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) value_pb.geo_point_value.CopyFrom(field_value) elif isinstance(field_value, MessageMapContainer): # Nested dictionary on a struct, set a value directory on a passed in pb object # which is a parent Struct entity entity_pb_item = get_entity_pb_for_value(value=field_value) entity_pb.CopyFrom(entity_pb_item) elif isinstance(field_value, ScalarMapContainer): # Custom user defined type, recurse into it value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) entity_pb_item = get_entity_pb_for_value(value=field_value) value_pb.entity_value.CopyFrom(entity_pb_item) elif field_type.message_type.full_name == 'google.protobuf.Struct': if not dict(field_value): # Value not set, skip it continue value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) entity_pb_item = get_entity_pb_for_value(value=field_value) value_pb.entity_value.CopyFrom(entity_pb_item) else: # Nested type, potentially referenced from another Protobuf definition file value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) entity_pb_item = model_pb_to_entity_pb(field_value) value_pb.entity_value.CopyFrom(entity_pb_item) else: raise ValueError('Unsupported field type for field "%s"' % (field_name)) if not value_pb: continue value_pb = cast(Value, value_pb) # Determine if field should be excluded from index exclude_field_from_indexes = exclude_field_from_index( model=model_pb, field_descriptor=field_descriptor, exclude_from_index=exclude_from_index) if exclude_field_from_indexes: # Field should be excluded from the index, mark that on the Entity Value value_pb.exclude_from_indexes = True return entity_pb
def _makePB(self, attr_name, value): from google.cloud.datastore_v1.proto import entity_pb2 pb = entity_pb2.Value() setattr(pb, attr_name, value) return pb
def test_no_meaning(self): from google.cloud.datastore_v1.proto import entity_pb2 value_pb = entity_pb2.Value() result = self._call_fut(value_pb) self.assertIsNone(result)
def _makePB(self): from google.cloud.datastore_v1.proto import entity_pb2 return entity_pb2.Value()
def test_unknown(self): from google.cloud.datastore_v1.proto import entity_pb2 pb = entity_pb2.Value() with self.assertRaises(ValueError): self._call_fut(pb)