def result(foo, bar=0, baz=""): return _datastore_query._Result( result_type=None, result_pb=query_pb2.EntityResult( entity=entity_pb2.Entity( properties={ "foo": entity_pb2.Value(string_value=foo), "bar": entity_pb2.Value(integer_value=bar), "baz": entity_pb2.Value(string_value=baz), } ) ), order_by=[ query_module.PropertyOrder("foo"), query_module.PropertyOrder("bar", reverse=True), ], )
def test_meaning_with_change(self): from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore.entity import Entity from google.cloud.datastore.helpers import _new_value_pb entity = Entity() name = 'foo' entity[name] = value = 42 entity._meanings[name] = (9, 1337) entity_pb = self._call_fut(entity) expected_pb = entity_pb2.Entity() value_pb = _new_value_pb(expected_pb, name) value_pb.integer_value = value # NOTE: No meaning is used since the value differs from the # value stored. self._compare_entity_proto(entity_pb, expected_pb)
def test_index_mismatch_ignores_empty_list(self): from google.cloud.datastore_v1.proto import entity_pb2 _PROJECT = "PROJECT" _KIND = "KIND" _ID = 1234 array_val_pb = entity_pb2.Value(array_value=entity_pb2.ArrayValue( values=[])) entity_pb = entity_pb2.Entity(properties={"baz": array_val_pb}) entity_pb.key.partition_id.project_id = _PROJECT entity_pb.key.path.add(kind=_KIND, id=_ID) entity = self._call_fut(entity_pb) entity_dict = dict(entity) self.assertEqual(entity_dict["baz"], [])
def test_key_only(self): from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore.entity import Entity from google.cloud.datastore.key import Key kind, name = 'PATH', 'NAME' project = 'PROJECT' key = Key(kind, name, project=project) entity = Entity(key=key) entity_pb = self._call_fut(entity) expected_pb = entity_pb2.Entity() expected_pb.key.partition_id.project_id = project path_elt = expected_pb.key.path.add() path_elt.kind = kind path_elt.name = name self._compare_entity_proto(entity_pb, expected_pb)
def test_run_query_w_namespace_nonempty_result(self): from google.cloud.datastore_v1.proto import datastore_pb2 from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore_v1.proto import query_pb2 project = "PROJECT" kind = "Kind" namespace = "NS" query_pb = self._make_query_pb(kind) partition_id = entity_pb2.PartitionId( project_id=project, namespace_id=namespace ) read_options = datastore_pb2.ReadOptions() rsp_pb = datastore_pb2.RunQueryResponse( batch=query_pb2.QueryResultBatch( entity_result_type=query_pb2.EntityResult.FULL, entity_results=[query_pb2.EntityResult(entity=entity_pb2.Entity())], more_results=query_pb2.QueryResultBatch.NO_MORE_RESULTS, ) ) # Create mock HTTP and client with response. http = _make_requests_session( [_make_response(content=rsp_pb.SerializeToString())] ) client_info = _make_client_info() client = mock.Mock( _http=http, _base_url="test.invalid", _client_info=client_info, spec=["_http", "_base_url", "_client_info"], ) # Make request. ds_api = self._make_one(client) response = ds_api.run_query(project, partition_id, read_options, query=query_pb) # Check the result and verify the callers. self.assertEqual(response, rsp_pb) uri = _build_expected_url(client._base_url, project, "runQuery") request = _verify_protobuf_call(http, uri, datastore_pb2.RunQueryRequest()) self.assertEqual(request.partition_id, partition_id) self.assertEqual(request.query, query_pb)
def test_simple_fields(self): from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore.entity import Entity from google.cloud.datastore.helpers import _new_value_pb entity = Entity() name1 = 'foo' entity[name1] = value1 = 42 name2 = 'bar' entity[name2] = value2 = u'some-string' entity_pb = self._call_fut(entity) expected_pb = entity_pb2.Entity() val_pb1 = _new_value_pb(expected_pb, name1) val_pb1.integer_value = value1 val_pb2 = _new_value_pb(expected_pb, name2) val_pb2.string_value = value2 self._compare_entity_proto(entity_pb, expected_pb)
def test_lookup_single_key_nonempty_response(self): from google.cloud.datastore_v1.proto import datastore_pb2 from google.cloud.datastore_v1.proto import entity_pb2 project = "PROJECT" key_pb = _make_key_pb(project) rsp_pb = datastore_pb2.LookupResponse() entity = entity_pb2.Entity() entity.key.CopyFrom(key_pb) rsp_pb.found.add(entity=entity) read_options = datastore_pb2.ReadOptions() # Create mock HTTP and client with response. http = _make_requests_session( [_make_response(content=rsp_pb.SerializeToString())] ) client_info = _make_client_info() client = mock.Mock( _http=http, _base_url="test.invalid", _client_info=client_info, spec=["_http", "_base_url", "_client_info"], ) # Make request. ds_api = self._make_one(client) response = ds_api.lookup(project, [key_pb], read_options=read_options) # Check the result and verify the callers. self.assertEqual(response, rsp_pb) uri = _build_expected_url(client._base_url, project, "lookup") self.assertEqual(len(response.found), 1) self.assertEqual(len(response.missing), 0) self.assertEqual(len(response.deferred), 0) found = response.found[0].entity self.assertEqual(found.key.path[0].kind, "Kind") self.assertEqual(found.key.path[0].id, 1234) request = _verify_protobuf_call(http, uri, datastore_pb2.LookupRequest()) self.assertEqual(list(request.keys), [key_pb]) self.assertEqual(request.read_options, read_options)
def entity_to_protobuf(entity): """Converts an entity into a protobuf. :type entity: :class:`google.cloud.datastore.entity.Entity` :param entity: The entity to be turned into a protobuf. :rtype: :class:`.entity_pb2.Entity` :returns: The protobuf representing the entity. """ entity_pb = entity_pb2.Entity() if entity.key is not None: key_pb = entity.key.to_protobuf() entity_pb.key.CopyFrom(key_pb) for name, value in entity.items(): value_is_list = isinstance(value, list) if value_is_list and len(value) == 0: continue value_pb = _new_value_pb(entity_pb, name) # Set the appropriate value. _set_protobuf_value(value_pb, value) # Add index information to protobuf. if name in entity.exclude_from_indexes: if not value_is_list: value_pb.exclude_from_indexes = True for sub_value in value_pb.array_value.values: sub_value.exclude_from_indexes = True # Add meaning information to protobuf. _set_pb_meaning_from_entity(entity, name, value, value_pb, is_list=value_is_list) return entity_pb
def get_entity_pb_for_value(value): # type: (Any) -> entity_pb2.Entity """ Return Entity protobuf object for the provided Python value. """ entity_pb = entity_pb2.Entity() attr_type = get_pb_attr_type(value) if attr_type == 'dict_value': if six.PY2: value = dict(value) for key, value in six.iteritems(value): value_pb = datastore.helpers._new_value_pb(entity_pb, key) value_pb = set_value_pb_item_value(value_pb=value_pb, value=value, is_struct=True) else: raise ValueError('Unsupported attribute type: %s' % (attr_type)) return entity_pb
def test_mismatched_value_indexed(self): from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore.helpers import _new_value_pb _PROJECT = 'PROJECT' _KIND = 'KIND' _ID = 1234 entity_pb = entity_pb2.Entity() entity_pb.key.partition_id.project_id = _PROJECT entity_pb.key.path.add(kind=_KIND, id=_ID) array_val_pb = _new_value_pb(entity_pb, 'baz') array_pb = array_val_pb.array_value.values unindexed_value_pb1 = array_pb.add() unindexed_value_pb1.integer_value = 10 unindexed_value_pb1.exclude_from_indexes = True unindexed_value_pb2 = array_pb.add() unindexed_value_pb2.integer_value = 11 with self.assertRaises(ValueError): self._call_fut(entity_pb)
def test_get_multi_miss_w_missing(self): from google.cloud.datastore_v1.proto import entity_pb2 from google.cloud.datastore_v1.proto import datastore_pb2 from google.cloud.datastore.key import Key KIND = 'Kind' ID = 1234 # Make a missing entity pb to be returned from mock backend. missed = entity_pb2.Entity() missed.key.partition_id.project_id = self.PROJECT path_element = missed.key.path.add() path_element.kind = KIND path_element.id = ID creds = _make_credentials() client = self._make_one(credentials=creds) # Set missing entity on mock connection. lookup_response = _make_lookup_response(missing=[missed]) ds_api = _make_datastore_api(lookup_response=lookup_response) client._datastore_api_internal = ds_api key = Key(KIND, ID, project=self.PROJECT) missing = [] entities = client.get_multi([key], missing=missing) self.assertEqual(entities, []) key_pb = key.to_protobuf() self.assertEqual( [missed.key.to_protobuf() for missed in missing], [key_pb]) read_options = datastore_pb2.ReadOptions() ds_api.lookup.assert_called_once_with( self.PROJECT, [key_pb], read_options=read_options, )
def Mutation(): path = [entity_pb2.Key.PathElement(kind="SomeKind")] return datastore_pb2.Mutation( upsert=entity_pb2.Entity(key=entity_pb2.Key(path=path)) )
def test_complex_model2(self): import binascii import datetime from protorpc import messages from ndb_orm import msgprop class Items(ndb.Model): has_hat = ndb.BooleanProperty("hh") number_of_socks = ndb.IntegerProperty("ns") class Gender(messages.Enum): male = 1 female = 2 neutral = 3 class Human(ndb.Model): name = ndb.StringProperty("na", indexed=True) gender = msgprop.EnumProperty(Gender, "g", required=True, indexed=True) age = ndb.IntegerProperty("ag", indexed=False) items = ndb.StructuredProperty(Items, "i", required=True) numbers = ndb.JsonProperty('json', indexed=False) description = ndb.TextProperty("t", indexed=False) description2 = ndb.TextProperty("t2", compressed=True, indexed=False) meters_tall = ndb.FloatProperty("mtrs", indexed=False) datetime_of_birth = ndb.DateTimeProperty("dtb", indexed=False) date_of_birth = ndb.DateProperty("db", indexed=False) time_of_birth = ndb.TimeProperty("tb", indexed=False) hobbies = ndb.StringProperty('hob', repeated=True, indexed=False) pickle = ndb.PickleProperty('pi', indexed=False) binary = ndb.BlobProperty("bi", indexed=False) home = ndb.GeoPtProperty("ho", indexed=False) generic = ndb.GenericProperty("gen", indexed=False) model = ndb.LocalStructuredProperty(Items, "mo", indexed=False) number_of_hobbies = ndb.ComputedProperty( name="num_hob", func=lambda self: len(self.hobbies), indexed=False) default_info = ndb.StringProperty("di", indexed=False, default='unknown') update = ndb.DateTimeProperty("up", indexed=False, auto_now=True) # the entity was created from within app engine standard with the code below # and the the protocolbuffer was fetch with google-cloud-datastore and hexlified pb_binary_string = binascii.unhexlify( '0a240a10120e6b756e7374616b726f626174656e12100a0548756d616e108080808080e4910a1a4b0a027069124592013f80025d71017d7102550b666f6f7462616c6c5f61747103636461746574696d650a6461746574696d650a7104550a07e1081a0f0a2a01e2408552710573612e9801011a1f0a026d6f121932141a080a026e73120210031a080a026868120208019801011a100a026269120a920104616263009801011a140a027462120e520908f2aa03108094ef3a9801011a140a026e61120e8a010b4172746875722044656e741a1a0a046a736f6e121292010c5b31322c2031332c2031345d9801011a100a076e756d5f686f62120510029801011a0a0a04692e6e73120210031a0c0a0367656e120510079801011a0a0a04692e6868120208011a150a026469120f700f8a0107756e6b6e6f776e9801011a110a026462120b52060880f482cd059801011a270a03686f6212204a1e0a10700f8a0108666f6f7462616c6c9801010a0a700f8a010274769801011a0b0a0261671205102a9801011a1d0a02686f12174212098fc2f5285c2f4a401185eb51b81e8513409801011a070a0167120210011a290a0274321223701692011b789c4b54284a4dcc51c84dccd351c8cf50a84c4dcc00003f14066c9801011a140a046d747273120c191f85eb51b81efd3f9801011a170a036474621210520b08f29e86cd05108094ef3a9801011a170a0275701211520c08abc791cd051080abfd8e039801011a170a01741212700f8a010a61207265616c206d616e980101' ) # pylint:disable=line-too-long # human = Human( # name='Arthur Dent', # gender=Gender.male, # age=42, # items=Items(has_hat=True, number_of_socks=3), #namespace=namespace), # numbers=[12, 13, 14], # description="a real man", # description2="a real man, oh yeah", # meters_tall=1.82, # datetime_of_birth=datetime.datetime(2017, 8, 26, 15, 10, 42, 123456), # date_of_birth=datetime.datetime(2017, 8, 26, 15, 10, 42, 123456).date(), # time_of_birth=datetime.datetime(2017, 8, 26, 15, 10, 42, 123456).time(), # hobbies=[u"football", u"tv"], # pickle=[{"football_at": datetime.datetime(2017, 8, 26, 15, 10, 42, 123456)}], # binary=binascii.unhexlify("61626300"), # home=ndb.GeoPt("52.37, 4.88"), # generic=7, # model=Items(has_hat=True, number_of_socks=3), #namespace=namespace), # ) # human.put() pb = entity_pb2.Entity() pb.ParseFromString(pb_binary_string) human_recovered = ndb.helpers.model_from_protobuf(pb) # now do the tests self.assertEqual(human_recovered.name, 'Arthur Dent') self.assertEqual(human_recovered.gender, Gender.male) self.assertEqual(human_recovered.age, 42) self.assertEqual(human_recovered.items.has_hat, True) self.assertEqual(human_recovered.items.number_of_socks, 3) self.assertEqual(human_recovered.numbers, [12, 13, 14]) self.assertEqual(human_recovered.description, "a real man") self.assertEqual(human_recovered.description2, "a real man, oh yeah") self.assertAlmostEqual(human_recovered.meters_tall, 1.82) # DateTime always have a timezone attached self.assertEqual( human_recovered.datetime_of_birth.replace(tzinfo=None), datetime.datetime(2017, 8, 26, 15, 10, 42, 123456)) self.assertEqual( human_recovered.date_of_birth, datetime.datetime(2017, 8, 26, 15, 10, 42, 123456).date()) self.assertEqual(human_recovered.hobbies, [u"football", u"tv"]) # following is a PickleProperty and thus not recoverable from python3 #self.assertEqual(human_recovered.pickle[0]["football_at"], datetime.datetime(2017, 8, 26, 15, 10, 42, 123456)) self.assertEqual(human_recovered.binary, binascii.unhexlify("61626300")) self.assertAlmostEqual(human_recovered.home.lat, 52.37) self.assertAlmostEqual(human_recovered.home.lon, 4.88) self.assertEqual(human_recovered.generic, 7) self.assertEqual(human_recovered.model.has_hat, True) self.assertEqual(human_recovered.model.number_of_socks, 3) # these were set automatically self.assertEqual(human_recovered.number_of_hobbies, 2) self.assertEqual(human_recovered.default_info, "unknown") self.assertEqual(isinstance(human_recovered.update, datetime.date), True)
def model_pb_to_entity_pb(model_pb, exclude_falsy_values=False, exclude_from_index=None): # type: (message.Message, bool, Optional[List[str]]) -> entity_pb2.Entity """ Translate Protobuf based database model object to Entity object which can be used with Google Datastore client library. :param model_pb: Instance of a custom Protobuf object to translate. :param exclude_falsy_values: True to exclude field values which are falsy (e.g. None, False, '', 0, etc.) and match the default values. NOTE: Due to the design of protobuf v3, there is no way to distinguish between a user explicitly providing a value which is the same as a default value (e.g. 0 for an integer field) and user not providing a value and default value being used instead. :param exclude_from_index: Optional list of field names which should not be indexed. By default, all the simple fields are indexed. NOTE: If provided, this value has high precedence over "exclude_from_index" message option defined on the model. """ exclude_from_index = exclude_from_index or [] if not isinstance(model_pb, message.Message): raise ValueError( 'model_pb argument is not a valid Protobuf class instance') fields = list(iter(model_pb.DESCRIPTOR.fields)) fields = [field for field in fields if field not in ['key']] entity_pb = entity_pb2.Entity() exclude_from_index = cast(list, exclude_from_index) for field_descriptor in fields: field_type = field_descriptor.type field_name = field_descriptor.name field_value = getattr(model_pb, field_name, None) if field_value is None: # Value not set or it uses a default value, skip it # NOTE: proto3 syntax doesn't support HasField() anymore so there is now way for us to # determine if a value is set / provided so we just use and return default values. continue if exclude_falsy_values and not field_value: continue attr_type = get_pb_attr_type(field_value) value_pb = None if attr_type == 'array_value': if len(field_value) == 0: value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) array_value = entity_pb2.ArrayValue(values=[]) value_pb.array_value.CopyFrom(array_value) else: value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) for value in field_value: if field_type == descriptor.FieldDescriptor.TYPE_MESSAGE: # Nested message type entity_pb_item = model_pb_to_entity_pb(value) value_pb_item = entity_pb2.Value() # pylint: disable=no-member value_pb_item.entity_value.CopyFrom(entity_pb_item) # pylint: enable=no-member else: # Simple type value_pb_item = entity_pb2.Value() value_pb_item = set_value_pb_item_value( value_pb=value_pb_item, value=value) value_pb.array_value.values.append(value_pb_item) elif field_type == descriptor.FieldDescriptor.TYPE_STRING: value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) value_pb.string_value = field_value elif field_type in [ descriptor.FieldDescriptor.TYPE_DOUBLE, descriptor.FieldDescriptor.TYPE_FLOAT ]: # NOTE: Datastore only supports double type so we map float to double value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) value_pb.double_value = field_value elif field_type in [ descriptor.FieldDescriptor.TYPE_INT32, descriptor.FieldDescriptor.TYPE_INT64 ]: value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) value_pb.integer_value = field_value elif field_type == descriptor.FieldDescriptor.TYPE_ENUM: value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) if field_descriptor.enum_type.name == 'NullValue': # NULL value value_pb.null_value = struct_pb2.NULL_VALUE else: # Regular ENUM value_pb.integer_value = field_value elif field_type == descriptor.FieldDescriptor.TYPE_BOOL: value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) value_pb.boolean_value = field_value elif field_type == descriptor.FieldDescriptor.TYPE_BYTES: value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) if isinstance(field_value, six.string_types): field_value = field_value.encode('utf-8') value_pb.blob_value = field_value elif field_type == descriptor.FieldDescriptor.TYPE_MESSAGE: # Complex type, convert to entity field_type = model_pb.DESCRIPTOR.fields_by_name[field_name] if field_type.message_type.full_name == 'google.protobuf.Timestamp': if str(field_value) == '': # Value not set # TODO: Include default empty value? # value_pb = datastore.helpers._new_value_pb(entity_pb, field_name) # value_pb.timestamp_value.CopyFrom(field_value) continue value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) value_pb.timestamp_value.CopyFrom(field_value) elif field_type.message_type.full_name == 'google.type.LatLng': if str(field_value) == '': # Value not set continue value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) value_pb.geo_point_value.CopyFrom(field_value) elif isinstance(field_value, MessageMapContainer): # Nested dictionary on a struct, set a value directory on a passed in pb object # which is a parent Struct entity entity_pb_item = get_entity_pb_for_value(value=field_value) entity_pb.CopyFrom(entity_pb_item) elif isinstance(field_value, ScalarMapContainer): # Custom user defined type, recurse into it value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) entity_pb_item = get_entity_pb_for_value(value=field_value) value_pb.entity_value.CopyFrom(entity_pb_item) elif field_type.message_type.full_name == 'google.protobuf.Struct': if not dict(field_value): # Value not set, skip it continue value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) entity_pb_item = get_entity_pb_for_value(value=field_value) value_pb.entity_value.CopyFrom(entity_pb_item) else: # Nested type, potentially referenced from another Protobuf definition file value_pb = datastore.helpers._new_value_pb( entity_pb, field_name) entity_pb_item = model_pb_to_entity_pb(field_value) value_pb.entity_value.CopyFrom(entity_pb_item) else: raise ValueError('Unsupported field type for field "%s"' % (field_name)) if not value_pb: continue value_pb = cast(Value, value_pb) # Determine if field should be excluded from index exclude_field_from_indexes = exclude_field_from_index( model=model_pb, field_descriptor=field_descriptor, exclude_from_index=exclude_from_index) if exclude_field_from_indexes: # Field should be excluded from the index, mark that on the Entity Value value_pb.exclude_from_indexes = True return entity_pb
def lookup(key, options): """Look up a Datastore entity. Gets an entity from Datastore, asynchronously. Checks the global cache, first, if appropriate. Uses batching. Args: key (~datastore.Key): The key for the entity to retrieve. options (_options.ReadOptions): The options for the request. For example, ``{"read_consistency": EVENTUAL}``. Returns: :class:`~tasklets.Future`: If not an exception, future's result will be either an entity protocol buffer or _NOT_FOUND. """ context = context_module.get_context() use_datastore = context._use_datastore(key, options) if use_datastore and options.transaction: use_global_cache = False else: use_global_cache = context._use_global_cache(key, options) if not (use_global_cache or use_datastore): raise TypeError("use_global_cache and use_datastore can't both be False") entity_pb = _NOT_FOUND key_locked = False if use_global_cache: cache_key = _cache.global_cache_key(key) result = yield _cache.global_get(cache_key) key_locked = _cache.is_locked_value(result) if not key_locked: if result: entity_pb = entity_pb2.Entity() entity_pb.MergeFromString(result) elif use_datastore: lock = yield _cache.global_lock_for_read(cache_key, result) if lock: yield _cache.global_watch(cache_key, lock) else: # Another thread locked or wrote to this key after the call to # _cache.global_get above. Behave as though the key was locked by # another thread and don't attempt to write our value below key_locked = True if entity_pb is _NOT_FOUND and use_datastore: batch = _batch.get_batch(_LookupBatch, options) entity_pb = yield batch.add(key) # Do not cache misses if use_global_cache and not key_locked: if entity_pb is not _NOT_FOUND: expires = context._global_cache_timeout(key, options) serialized = entity_pb.SerializeToString() yield _cache.global_compare_and_swap( cache_key, serialized, expires=expires ) else: yield _cache.global_unwatch(cache_key) raise tasklets.Return(entity_pb)