def key_from_protobuf(pb): """Factory method for creating a key based on a protobuf. The protobuf should be one returned from the Cloud Datastore Protobuf API. :type pb: :class:`gcloud.datastore._generated.entity_pb2.Key` :param pb: The Protobuf representing the key. :rtype: :class:`gcloud.datastore.key.Key` :returns: a new `Key` instance """ path_args = [] for element in pb.path_element: path_args.append(element.kind) if _has_field(element, 'id'): path_args.append(element.id) # This is safe: we expect proto objects returned will only have # one of `name` or `id` set. if _has_field(element, 'name'): path_args.append(element.name) project = None if _has_field(pb.partition_id, 'dataset_id'): project = pb.partition_id.dataset_id namespace = None if _has_field(pb.partition_id, 'namespace'): namespace = pb.partition_id.namespace return Key(*path_args, namespace=namespace, project=project)
def _prepare_key_for_request(key_pb): """Add protobuf keys to a request object. :type key_pb: :class:`gcloud.datastore._generated.entity_pb2.Key` :param key_pb: A key to be added to a request. :rtype: :class:`gcloud.datastore._generated.entity_pb2.Key` :returns: A key which will be added to a request. It will be the original if nothing needs to be changed. """ if _has_field(key_pb.partition_id, 'dataset_id'): # We remove the dataset_id from the protobuf. This is because # the backend fails a request if the key contains un-prefixed # project. The backend fails because requests to # /datastore/.../datasets/foo/... # and # /datastore/.../datasets/s~foo/... # both go to the datastore given by 's~foo'. So if the key # protobuf in the request body has dataset_id='foo', the # backend will reject since 'foo' != 's~foo'. new_key_pb = _entity_pb2.Key() new_key_pb.CopyFrom(key_pb) new_key_pb.partition_id.ClearField('dataset_id') key_pb = new_key_pb return key_pb
def _compare_key_pb_after_request(test, key_before, key_after): from gcloud._helpers import _has_field test.assertFalse(_has_field(key_after.partition_id, 'dataset_id')) test.assertEqual(key_before.partition_id.namespace, key_after.partition_id.namespace) test.assertEqual(len(key_before.path_element), len(key_after.path_element)) for elt1, elt2 in zip(key_before.path_element, key_after.path_element): test.assertEqual(elt1, elt2)
def test_to_protobuf_w_no_kind(self): from gcloud._helpers import _has_field key = self._makeOne('KIND', project=self._DEFAULT_PROJECT) # Force the 'kind' to be unset. Maybe `to_protobuf` should fail # on this? The backend certainly will. key._path[-1].pop('kind') pb = key.to_protobuf() self.assertFalse(_has_field(pb.path_element[0], 'kind'))
def test_to_protobuf_defaults(self): from gcloud._helpers import _has_field from gcloud.datastore._generated import entity_pb2 _KIND = 'KIND' key = self._makeOne(_KIND, project=self._DEFAULT_PROJECT) pb = key.to_protobuf() self.assertTrue(isinstance(pb, entity_pb2.Key)) # Check partition ID. self.assertEqual(pb.partition_id.dataset_id, self._DEFAULT_PROJECT) self.assertEqual(pb.partition_id.namespace, '') self.assertFalse(_has_field(pb.partition_id, 'namespace')) # Check the element PB matches the partial key and kind. elem, = list(pb.path_element) self.assertEqual(elem.kind, _KIND) self.assertEqual(elem.name, '') self.assertFalse(_has_field(elem, 'name')) self.assertEqual(elem.id, 0) self.assertFalse(_has_field(elem, 'id'))
def _get_value_from_value_pb(value_pb): """Given a protobuf for a Value, get the correct value. The Cloud Datastore Protobuf API returns a Property Protobuf which has one value set and the rest blank. This function retrieves the the one value provided. Some work is done to coerce the return value into a more useful type (particularly in the case of a timestamp value, or a key value). :type value_pb: :class:`gcloud.datastore._generated.entity_pb2.Value` :param value_pb: The Value Protobuf. :returns: The value provided by the Protobuf. """ result = None if _has_field(value_pb, 'timestamp_microseconds_value'): microseconds = value_pb.timestamp_microseconds_value result = _datetime_from_microseconds(microseconds) elif _has_field(value_pb, 'key_value'): result = key_from_protobuf(value_pb.key_value) elif _has_field(value_pb, 'boolean_value'): result = value_pb.boolean_value elif _has_field(value_pb, 'double_value'): result = value_pb.double_value elif _has_field(value_pb, 'integer_value'): result = value_pb.integer_value elif _has_field(value_pb, 'string_value'): result = value_pb.string_value elif _has_field(value_pb, 'blob_value'): result = value_pb.blob_value elif _has_field(value_pb, 'entity_value'): result = entity_from_protobuf(value_pb.entity_value) elif value_pb.list_value: result = [ _get_value_from_value_pb(value) for value in value_pb.list_value ] return result
def entity_from_protobuf(pb): """Factory method for creating an entity based on a protobuf. The protobuf should be one returned from the Cloud Datastore Protobuf API. :type pb: :class:`gcloud.datastore._generated.entity_pb2.Entity` :param pb: The Protobuf representing the entity. :rtype: :class:`gcloud.datastore.entity.Entity` :returns: The entity derived from the protobuf. """ key = None if _has_field(pb, 'key'): key = key_from_protobuf(pb.key) entity_props = {} entity_meanings = {} exclude_from_indexes = [] for prop_name, value_pb in _property_tuples(pb): value = _get_value_from_value_pb(value_pb) entity_props[prop_name] = value # Check if the property has an associated meaning. is_list = isinstance(value, list) meaning = _get_meaning(value_pb, is_list=is_list) if meaning is not None: entity_meanings[prop_name] = (meaning, value) # Check if ``value_pb`` was indexed. Lists need to be special-cased # and we require all ``indexed`` values in a list agree. if is_list: indexed_values = set(value_pb.indexed for value_pb in value_pb.list_value) if len(indexed_values) != 1: raise ValueError('For a list_value, subvalues must either all ' 'be indexed or all excluded from indexes.') if not indexed_values.pop(): exclude_from_indexes.append(prop_name) else: if not value_pb.indexed: exclude_from_indexes.append(prop_name) entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes) entity.update(entity_props) entity._meanings.update(entity_meanings) return entity
def _get_value_from_value_pb(value_pb): """Given a protobuf for a Value, get the correct value. The Cloud Datastore Protobuf API returns a Property Protobuf which has one value set and the rest blank. This function retrieves the the one value provided. Some work is done to coerce the return value into a more useful type (particularly in the case of a timestamp value, or a key value). :type value_pb: :class:`gcloud.datastore._generated.entity_pb2.Value` :param value_pb: The Value Protobuf. :returns: The value provided by the Protobuf. """ result = None if _has_field(value_pb, 'timestamp_microseconds_value'): microseconds = value_pb.timestamp_microseconds_value result = _datetime_from_microseconds(microseconds) elif _has_field(value_pb, 'key_value'): result = key_from_protobuf(value_pb.key_value) elif _has_field(value_pb, 'boolean_value'): result = value_pb.boolean_value elif _has_field(value_pb, 'double_value'): result = value_pb.double_value elif _has_field(value_pb, 'integer_value'): result = value_pb.integer_value elif _has_field(value_pb, 'string_value'): result = value_pb.string_value elif _has_field(value_pb, 'blob_value'): result = value_pb.blob_value elif _has_field(value_pb, 'entity_value'): result = entity_from_protobuf(value_pb.entity_value) elif value_pb.list_value: result = [_get_value_from_value_pb(value) for value in value_pb.list_value] return result
def _compareEntityProto(self, entity_pb1, entity_pb2): from gcloud._helpers import _has_field from gcloud.datastore.helpers import _property_tuples self.assertEqual(entity_pb1.key, entity_pb2.key) value_list1 = sorted(_property_tuples(entity_pb1)) value_list2 = sorted(_property_tuples(entity_pb2)) self.assertEqual(len(value_list1), len(value_list2)) for pair1, pair2 in zip(value_list1, value_list2): name1, val1 = pair1 name2, val2 = pair2 self.assertEqual(name1, name2) if _has_field(val1, 'entity_value'): self.assertEqual(val1.meaning, val2.meaning) self._compareEntityProto(val1.entity_value, val2.entity_value) else: self.assertEqual(val1, val2)
def _prepare_key_for_request(key_pb): # pragma: NO COVER copied from helpers """Add protobuf keys to a request object. .. note:: This is copied from `helpers` to avoid a cycle: _implicit_environ -> connection -> helpers -> key -> _implicit_environ :type key_pb: :class:`gcloud.datastore._generated.entity_pb2.Key` :param key_pb: A key to be added to a request. :rtype: :class:`gcloud.datastore._generated.entity_pb2.Key` :returns: A key which will be added to a request. It will be the original if nothing needs to be changed. """ if _has_field(key_pb.partition_id, 'dataset_id'): new_key_pb = _entity_pb2.Key() new_key_pb.CopyFrom(key_pb) new_key_pb.partition_id.ClearField('dataset_id') key_pb = new_key_pb return key_pb
def test_unprefixed_bogus_key_hit(self): from gcloud._helpers import _has_field UNPREFIXED = 'PROJECT' PREFIX = 'e~' CONNECTION = _Connection(PREFIX, from_missing=True) result = self._callFUT(UNPREFIXED, CONNECTION) self.assertEqual(CONNECTION._called_project, UNPREFIXED) self.assertEqual(CONNECTION._lookup_result, []) # Make sure just one. called_key_pb, = CONNECTION._called_key_pbs path_element = called_key_pb.path_element self.assertEqual(len(path_element), 1) self.assertEqual(path_element[0].kind, '__MissingLookupKind') self.assertEqual(path_element[0].id, 1) self.assertFalse(_has_field(path_element[0], 'name')) PREFIXED = PREFIX + UNPREFIXED self.assertEqual(result, PREFIXED)
def _get_meaning(value_pb, is_list=False): """Get the meaning from a protobuf value. :type value_pb: :class:`gcloud.datastore._generated.entity_pb2.Value` :param value_pb: The protobuf value to be checked for an associated meaning. :type is_list: bool :param is_list: Boolean indicating if the ``value_pb`` contains a list value. :rtype: int :returns: The meaning for the ``value_pb`` if one is set, else :data:`None`. :raises: :class:`ValueError <exceptions.ValueError>` if a list value has disagreeing meanings (in sub-elements) or has some elements with meanings and some without. """ meaning = None if is_list: # An empty list will have no values, hence no shared meaning # set among them. if len(value_pb.list_value) == 0: return None # We check among all the meanings, some of which may be None, # the rest which may be enum/int values. all_meanings = set( _get_meaning(sub_value_pb) for sub_value_pb in value_pb.list_value) meaning = all_meanings.pop() # The value we popped off should have been unique. If not # then we can't handle a list with values that have more # than one meaning. if all_meanings: raise ValueError('Different meanings set on values ' 'within a list_value') elif _has_field(value_pb, 'meaning'): meaning = value_pb.meaning return meaning
def _get_meaning(value_pb, is_list=False): """Get the meaning from a protobuf value. :type value_pb: :class:`gcloud.datastore._generated.entity_pb2.Value` :param value_pb: The protobuf value to be checked for an associated meaning. :type is_list: bool :param is_list: Boolean indicating if the ``value_pb`` contains a list value. :rtype: int :returns: The meaning for the ``value_pb`` if one is set, else :data:`None`. :raises: :class:`ValueError <exceptions.ValueError>` if a list value has disagreeing meanings (in sub-elements) or has some elements with meanings and some without. """ meaning = None if is_list: # An empty list will have no values, hence no shared meaning # set among them. if len(value_pb.list_value) == 0: return None # We check among all the meanings, some of which may be None, # the rest which may be enum/int values. all_meanings = set(_get_meaning(sub_value_pb) for sub_value_pb in value_pb.list_value) meaning = all_meanings.pop() # The value we popped off should have been unique. If not # then we can't handle a list with values that have more # than one meaning. if all_meanings: raise ValueError('Different meanings set on values ' 'within a list_value') elif _has_field(value_pb, 'meaning'): meaning = value_pb.meaning return meaning
def _callFUT(self, message_pb, property_name): from gcloud._helpers import _has_field return _has_field(message_pb, property_name)