def save_entity(self, dataset_id, key_pb, properties, exclude_from_indexes=()): """Save an entity to the Cloud Datastore with the provided properties. .. note:: Any existing properties for the entity identified by 'key_pb' will be replaced by those passed in 'properties'; properties not passed in 'properties' no longer be set for the entity. :type dataset_id: string :param dataset_id: The dataset in which to save the entity. :type key_pb: :class:`gcloud.datastore.datastore_v1_pb2.Key` :param key_pb: The complete or partial key for the entity. :type properties: dict :param properties: The properties to store on the entity. :type exclude_from_indexes: sequence of str :param exclude_from_indexes: Names of properties *not* to be indexed. """ mutation = self.mutation() # If the Key is complete, we should upsert # instead of using insert_auto_id. path = key_pb.path_element[-1] auto_id = not (path.HasField('id') or path.HasField('name')) if auto_id: insert = mutation.insert_auto_id.add() else: insert = mutation.upsert.add() insert.key.CopyFrom(key_pb) for name, value in properties.items(): prop = insert.property.add() # Set the name of the property. prop.name = name # Set the appropriate value. helpers._set_protobuf_value(prop.value, value) if name in exclude_from_indexes: prop.value.indexed = False # If this is in a transaction, we should just return True. The # transaction will handle assigning any keys as necessary. if self.transaction(): return True result = self.commit(dataset_id, mutation) # If this was an auto-assigned ID, return the new Key. if auto_id: return result.insert_auto_id_key[0] return True
def filter(self, property_name, operator, value): """Filter the query based on a property name, operator and a value. This will return a clone of the current :class:`Query` filtered by the expression and value provided. Expressions take the form of:: .filter('<property>', '<operator>', <value>) where property is a property stored on the entity in the datastore and operator is one of ``OPERATORS`` (ie, ``=``, ``<``, ``<=``, ``>``, ``>=``):: >>> query = Query('Person') >>> filtered_query = query.filter('name', '=', 'James') >>> filtered_query = query.filter('age', '>', 50) Because each call to ``.filter()`` returns a cloned ``Query`` object we are able to string these together:: >>> query = Query('Person').filter( ... 'name', '=', 'James').filter('age', '>', 50) :type property_name: string :param property_name: A property name. :type operator: string :param operator: One of ``=``, ``<``, ``<=``, ``>``, ``>=``. :type value: integer, string, boolean, float, None, datetime :param value: The value to filter on. :rtype: :class:`Query` :returns: A Query filtered by the expression and value provided. :raises: `ValueError` if `operation` is not one of the specified values. """ clone = self._clone() pb_op_enum = self.OPERATORS.get(operator) if pb_op_enum is None: error_message = 'Invalid expression: "%s"' % (operator,) choices_message = 'Please use one of: =, <, <=, >, >=.' raise ValueError(error_message, choices_message) # Build a composite filter AND'd together. composite_filter = clone._pb.filter.composite_filter composite_filter.operator = datastore_pb.CompositeFilter.AND # Add the specific filter property_filter = composite_filter.filter.add().property_filter property_filter.property.name = property_name property_filter.operator = pb_op_enum # Set the value to filter on based on the type. helpers._set_protobuf_value(property_filter.value, value) return clone
def _assign_entity_to_mutation(mutation_pb, entity, auto_id_entities): """Copy ``entity`` into appropriate slot of ``mutation_pb``. If ``entity.key`` is incomplete, append ``entity`` to ``auto_id_entities`` for later fixup during ``commit``. Helper method for ``Batch.put``. :type mutation_pb: :class:`gcloud.datastore._datastore_v1_pb2.Mutation` :param mutation_pb; the Mutation protobuf for the batch / transaction. :type entity: :class:`gcloud.datastore.entity.Entity` :param entity; the entity being updated within the batch / transaction. :type auto_id_entities: list of :class:`gcloud.datastore.entity.Entity` :param auto_id_entities: entiites with partial keys, to be fixed up during commit. """ auto_id = entity.key.is_partial key_pb = entity.key.to_protobuf() key_pb = helpers._prepare_key_for_request(key_pb) if auto_id: insert = mutation_pb.insert_auto_id.add() auto_id_entities.append(entity) else: # We use ``upsert`` for entities with completed keys, rather than # ``insert`` or ``update``, in order not to create race conditions # based on prior existence / removal of the entity. insert = mutation_pb.upsert.add() insert.key.CopyFrom(key_pb) for name, value in entity.items(): value_is_list = isinstance(value, list) if value_is_list and len(value) == 0: continue prop = insert.property.add() # Set the name of the property. prop.name = name # Set the appropriate value. helpers._set_protobuf_value(prop.value, value) if name in entity.exclude_from_indexes: if not value_is_list: prop.value.indexed = False for sub_value in prop.value.list_value: sub_value.indexed = False
def _assign_entity_to_mutation(mutation_pb, entity, auto_id_entities): """Copy ``entity`` into appropriate slot of ``mutation_pb``. If ``entity.key`` is incomplete, append ``entity`` to ``auto_id_entities`` for later fixup during ``commit``. Helper method for ``Batch.put``. :type mutation_pb: :class:`gcloud.datastore._datastore_v1_pb2.Mutation` :param mutation_pb: The Mutation protobuf for the batch / transaction. :type entity: :class:`gcloud.datastore.entity.Entity` :param entity: The entity being updated within the batch / transaction. :type auto_id_entities: list of :class:`gcloud.datastore.entity.Entity` :param auto_id_entities: Entities with partial keys, to be fixed up during commit. """ auto_id = entity.key.is_partial key_pb = entity.key.to_protobuf() key_pb = helpers._prepare_key_for_request(key_pb) if auto_id: insert = mutation_pb.insert_auto_id.add() auto_id_entities.append(entity) else: # We use ``upsert`` for entities with completed keys, rather than # ``insert`` or ``update``, in order not to create race conditions # based on prior existence / removal of the entity. insert = mutation_pb.upsert.add() insert.key.CopyFrom(key_pb) for name, value in entity.items(): value_is_list = isinstance(value, list) if value_is_list and len(value) == 0: continue prop = insert.property.add() # Set the name of the property. prop.name = name # Set the appropriate value. helpers._set_protobuf_value(prop.value, value) if name in entity.exclude_from_indexes: if not value_is_list: prop.value.indexed = False for sub_value in prop.value.list_value: sub_value.indexed = False
def _pb_from_query(query): """Convert a Query instance to the corresponding protobuf. :type query: :class:`Query` :param query: The source query. :rtype: :class:`gcloud.datastore._generated.query_pb2.Query` :returns: A protobuf that can be sent to the protobuf API. N.b. that it does not contain "in-flight" fields for ongoing query executions (cursors, offset, limit). """ pb = _query_pb2.Query() for projection_name in query.projection: pb.projection.add().property.name = projection_name if query.kind: pb.kind.add().name = query.kind composite_filter = pb.filter.composite_filter composite_filter.operator = _query_pb2.CompositeFilter.AND if query.ancestor: ancestor_pb = helpers._prepare_key_for_request( query.ancestor.to_protobuf()) # Filter on __key__ HAS_ANCESTOR == ancestor. ancestor_filter = composite_filter.filter.add().property_filter ancestor_filter.property.name = '__key__' ancestor_filter.operator = _query_pb2.PropertyFilter.HAS_ANCESTOR ancestor_filter.value.key_value.CopyFrom(ancestor_pb) for property_name, operator, value in query.filters: pb_op_enum = query.OPERATORS.get(operator) # Add the specific filter property_filter = composite_filter.filter.add().property_filter property_filter.property.name = property_name property_filter.operator = pb_op_enum # Set the value to filter on based on the type. if property_name == '__key__': key_pb = value.to_protobuf() property_filter.value.key_value.CopyFrom( helpers._prepare_key_for_request(key_pb)) else: helpers._set_protobuf_value(property_filter.value, value) if not composite_filter.filter: pb.ClearField('filter') for prop in query.order: property_order = pb.order.add() if prop.startswith('-'): property_order.property.name = prop[1:] property_order.direction = property_order.DESCENDING else: property_order.property.name = prop property_order.direction = property_order.ASCENDING for group_by_name in query.group_by: pb.group_by.add().name = group_by_name return pb
def _pb_from_query(query): """Convert a Query instance to the corresponding protobuf. :type query: :class:`Query` :param query: The source query. :rtype: :class:`gcloud.datastore._generated.query_pb2.Query` :returns: A protobuf that can be sent to the protobuf API. N.b. that it does not contain "in-flight" fields for ongoing query executions (cursors, offset, limit). """ pb = _query_pb2.Query() for projection_name in query.projection: pb.projection.add().property.name = projection_name if query.kind: pb.kind.add().name = query.kind composite_filter = pb.filter.composite_filter composite_filter.op = _query_pb2.CompositeFilter.AND if query.ancestor: ancestor_pb = query.ancestor.to_protobuf() # Filter on __key__ HAS_ANCESTOR == ancestor. ancestor_filter = composite_filter.filters.add().property_filter ancestor_filter.property.name = '__key__' ancestor_filter.op = _query_pb2.PropertyFilter.HAS_ANCESTOR ancestor_filter.value.key_value.CopyFrom(ancestor_pb) for property_name, operator, value in query.filters: pb_op_enum = query.OPERATORS.get(operator) # Add the specific filter property_filter = composite_filter.filters.add().property_filter property_filter.property.name = property_name property_filter.op = pb_op_enum # Set the value to filter on based on the type. if property_name == '__key__': key_pb = value.to_protobuf() property_filter.value.key_value.CopyFrom(key_pb) else: helpers._set_protobuf_value(property_filter.value, value) if not composite_filter.filters: pb.ClearField('filter') for prop in query.order: property_order = pb.order.add() if prop.startswith('-'): property_order.property.name = prop[1:] property_order.direction = property_order.DESCENDING else: property_order.property.name = prop property_order.direction = property_order.ASCENDING for distinct_on_name in query.distinct_on: pb.distinct_on.add().name = distinct_on_name return pb
def _callFUT(self, value_pb, val): from gcloud.datastore.helpers import _set_protobuf_value return _set_protobuf_value(value_pb, val)
def filter(self, expression, value): """Filter the query based on an expression and a value. This will return a clone of the current :class:`Query` filtered by the expression and value provided. Expressions take the form of:: .filter('<property> <operator>', <value>) where property is a property stored on the entity in the datastore and operator is one of ``OPERATORS`` (ie, ``=``, ``<``, ``<=``, ``>``, ``>=``):: >>> query = Query('Person') >>> filtered_query = query.filter('name =', 'James') >>> filtered_query = query.filter('age >', 50) Because each call to ``.filter()`` returns a cloned ``Query`` object we are able to string these together:: >>> query = Query('Person').filter( ... 'name =', 'James').filter('age >', 50) :type expression: string :param expression: An expression of a property and an operator (ie, ``=``). :type value: integer, string, boolean, float, None, datetime :param value: The value to filter on. :rtype: :class:`Query` :returns: A Query filtered by the expression and value provided. """ clone = self._clone() # Take an expression like 'property >=', and parse it into # useful pieces. property_name, operator = None, None expression = expression.strip() # Use None to split on *any* whitespace. expr_pieces = expression.rsplit(None, 1) if len(expr_pieces) == 2: property_name, operator = expr_pieces property_name = property_name.strip() # If no whitespace in `expression`, `operator` will be `None` and # self.OPERATORS[None] will be `None` as well. pb_op_enum = self.OPERATORS.get(operator) if pb_op_enum is None: raise ValueError('Invalid expression: "%s"' % expression) # Build a composite filter AND'd together. composite_filter = clone._pb.filter.composite_filter composite_filter.operator = datastore_pb.CompositeFilter.AND # Add the specific filter property_filter = composite_filter.filter.add().property_filter property_filter.property.name = property_name property_filter.operator = pb_op_enum # Set the value to filter on based on the type. helpers._set_protobuf_value(property_filter.value, value) return clone
duration = time.time() - start check_key_pb(key_pb) return duration MUTATION = CONNECTION.mutation() INSERT_AUTO = MUTATION.insert_auto_id.add() INSERT_AUTO.key.CopyFrom(PARTIAL_KEY_PB) for name, value in BASE_DATA.items(): prop = INSERT_AUTO.property.add() # Set the name of the property. prop.name = name # Set the appropriate value. helpers._set_protobuf_value(prop.value, value) def save_entity_fresh(): start = time.time() result = CONNECTION.commit(DATASET_ID, MUTATION) duration = time.time() - start key_pb = result.insert_auto_id_key[0] check_key_pb(key_pb) return duration COMMIT_REQUEST = datastore_pb.CommitRequest() COMMIT_REQUEST.mode = datastore_pb.CommitRequest.NON_TRANSACTIONAL COMMIT_REQUEST.mutation.CopyFrom(MUTATION)