def save_entity(self,
dataset_id,
key_pb,
properties,
exclude_from_indexes=()):
"""Save an entity to the Cloud Datastore with the provided properties.
.. note::
Any existing properties for the entity identified by 'key_pb'
will be replaced by those passed in 'properties'; properties
not passed in 'properties' no longer be set for the entity.
:type dataset_id: string
:param dataset_id: The dataset in which to save the entity.
:type key_pb: :class:`gcloud.datastore.datastore_v1_pb2.Key`
:param key_pb: The complete or partial key for the entity.
:type properties: dict
:param properties: The properties to store on the entity.
:type exclude_from_indexes: sequence of str
:param exclude_from_indexes: Names of properties *not* to be indexed.
"""
mutation = self.mutation()
# If the Key is complete, we should upsert
# instead of using insert_auto_id.
path = key_pb.path_element[-1]
auto_id = not (path.HasField('id') or path.HasField('name'))
if auto_id:
insert = mutation.insert_auto_id.add()
else:
insert = mutation.upsert.add()
insert.key.CopyFrom(key_pb)
for name, value in properties.items():
prop = insert.property.add()
# Set the name of the property.
prop.name = name
# Set the appropriate value.
helpers._set_protobuf_value(prop.value, value)
if name in exclude_from_indexes:
prop.value.indexed = False
# If this is in a transaction, we should just return True. The
# transaction will handle assigning any keys as necessary.
if self.transaction():
return True
result = self.commit(dataset_id, mutation)
# If this was an auto-assigned ID, return the new Key.
if auto_id:
return result.insert_auto_id_key[0]
return True
def filter(self, property_name, operator, value):
"""Filter the query based on a property name, operator and a value.
This will return a clone of the current :class:`Query`
filtered by the expression and value provided.
Expressions take the form of::
.filter('<property>', '<operator>', <value>)
where property is a property stored on the entity in the datastore
and operator is one of ``OPERATORS``
(ie, ``=``, ``<``, ``<=``, ``>``, ``>=``)::
>>> query = Query('Person')
>>> filtered_query = query.filter('name', '=', 'James')
>>> filtered_query = query.filter('age', '>', 50)
Because each call to ``.filter()`` returns a cloned ``Query`` object
we are able to string these together::
>>> query = Query('Person').filter(
... 'name', '=', 'James').filter('age', '>', 50)
:type property_name: string
:param property_name: A property name.
:type operator: string
:param operator: One of ``=``, ``<``, ``<=``, ``>``, ``>=``.
:type value: integer, string, boolean, float, None, datetime
:param value: The value to filter on.
:rtype: :class:`Query`
:returns: A Query filtered by the expression and value provided.
:raises: `ValueError` if `operation` is not one of the specified
values.
"""
clone = self._clone()
pb_op_enum = self.OPERATORS.get(operator)
if pb_op_enum is None:
error_message = 'Invalid expression: "%s"' % (operator,)
choices_message = 'Please use one of: =, <, <=, >, >=.'
raise ValueError(error_message, choices_message)
# Build a composite filter AND'd together.
composite_filter = clone._pb.filter.composite_filter
composite_filter.operator = datastore_pb.CompositeFilter.AND
# Add the specific filter
property_filter = composite_filter.filter.add().property_filter
property_filter.property.name = property_name
property_filter.operator = pb_op_enum
# Set the value to filter on based on the type.
helpers._set_protobuf_value(property_filter.value, value)
return clone
def save_entity(self, dataset_id, key_pb, properties,
exclude_from_indexes=()):
"""Save an entity to the Cloud Datastore with the provided properties.
.. note::
Any existing properties for the entity identified by 'key_pb'
will be replaced by those passed in 'properties'; properties
not passed in 'properties' no longer be set for the entity.
:type dataset_id: string
:param dataset_id: The dataset in which to save the entity.
:type key_pb: :class:`gcloud.datastore.datastore_v1_pb2.Key`
:param key_pb: The complete or partial key for the entity.
:type properties: dict
:param properties: The properties to store on the entity.
:type exclude_from_indexes: sequence of str
:param exclude_from_indexes: Names of properties *not* to be indexed.
"""
mutation = self.mutation()
# If the Key is complete, we should upsert
# instead of using insert_auto_id.
path = key_pb.path_element[-1]
auto_id = not (path.HasField('id') or path.HasField('name'))
if auto_id:
insert = mutation.insert_auto_id.add()
else:
insert = mutation.upsert.add()
insert.key.CopyFrom(key_pb)
for name, value in properties.items():
prop = insert.property.add()
# Set the name of the property.
prop.name = name
# Set the appropriate value.
helpers._set_protobuf_value(prop.value, value)
if name in exclude_from_indexes:
prop.value.indexed = False
# If this is in a transaction, we should just return True. The
# transaction will handle assigning any keys as necessary.
if self.transaction():
return True
result = self.commit(dataset_id, mutation)
# If this was an auto-assigned ID, return the new Key.
if auto_id:
return result.insert_auto_id_key[0]
return True
def _assign_entity_to_mutation(mutation_pb, entity, auto_id_entities):
"""Copy ``entity`` into appropriate slot of ``mutation_pb``.
If ``entity.key`` is incomplete, append ``entity`` to ``auto_id_entities``
for later fixup during ``commit``.
Helper method for ``Batch.put``.
:type mutation_pb: :class:`gcloud.datastore._datastore_v1_pb2.Mutation`
:param mutation_pb; the Mutation protobuf for the batch / transaction.
:type entity: :class:`gcloud.datastore.entity.Entity`
:param entity; the entity being updated within the batch / transaction.
:type auto_id_entities: list of :class:`gcloud.datastore.entity.Entity`
:param auto_id_entities: entiites with partial keys, to be fixed up
during commit.
"""
auto_id = entity.key.is_partial
key_pb = entity.key.to_protobuf()
key_pb = helpers._prepare_key_for_request(key_pb)
if auto_id:
insert = mutation_pb.insert_auto_id.add()
auto_id_entities.append(entity)
else:
# We use ``upsert`` for entities with completed keys, rather than
# ``insert`` or ``update``, in order not to create race conditions
# based on prior existence / removal of the entity.
insert = mutation_pb.upsert.add()
insert.key.CopyFrom(key_pb)
for name, value in entity.items():
value_is_list = isinstance(value, list)
if value_is_list and len(value) == 0:
continue
prop = insert.property.add()
# Set the name of the property.
prop.name = name
# Set the appropriate value.
helpers._set_protobuf_value(prop.value, value)
if name in entity.exclude_from_indexes:
if not value_is_list:
prop.value.indexed = False
for sub_value in prop.value.list_value:
sub_value.indexed = False
def _assign_entity_to_mutation(mutation_pb, entity, auto_id_entities):
"""Copy ``entity`` into appropriate slot of ``mutation_pb``.
If ``entity.key`` is incomplete, append ``entity`` to ``auto_id_entities``
for later fixup during ``commit``.
Helper method for ``Batch.put``.
:type mutation_pb: :class:`gcloud.datastore._datastore_v1_pb2.Mutation`
:param mutation_pb: The Mutation protobuf for the batch / transaction.
:type entity: :class:`gcloud.datastore.entity.Entity`
:param entity: The entity being updated within the batch / transaction.
:type auto_id_entities: list of :class:`gcloud.datastore.entity.Entity`
:param auto_id_entities: Entities with partial keys, to be fixed up
during commit.
"""
auto_id = entity.key.is_partial
key_pb = entity.key.to_protobuf()
key_pb = helpers._prepare_key_for_request(key_pb)
if auto_id:
insert = mutation_pb.insert_auto_id.add()
auto_id_entities.append(entity)
else:
# We use ``upsert`` for entities with completed keys, rather than
# ``insert`` or ``update``, in order not to create race conditions
# based on prior existence / removal of the entity.
insert = mutation_pb.upsert.add()
insert.key.CopyFrom(key_pb)
for name, value in entity.items():
value_is_list = isinstance(value, list)
if value_is_list and len(value) == 0:
continue
prop = insert.property.add()
# Set the name of the property.
prop.name = name
# Set the appropriate value.
helpers._set_protobuf_value(prop.value, value)
if name in entity.exclude_from_indexes:
if not value_is_list:
prop.value.indexed = False
for sub_value in prop.value.list_value:
sub_value.indexed = False
def _pb_from_query(query):
"""Convert a Query instance to the corresponding protobuf.
:type query: :class:`Query`
:param query: The source query.
:rtype: :class:`gcloud.datastore._generated.query_pb2.Query`
:returns: A protobuf that can be sent to the protobuf API. N.b. that
it does not contain "in-flight" fields for ongoing query
executions (cursors, offset, limit).
"""
pb = _query_pb2.Query()
for projection_name in query.projection:
pb.projection.add().property.name = projection_name
if query.kind:
pb.kind.add().name = query.kind
composite_filter = pb.filter.composite_filter
composite_filter.operator = _query_pb2.CompositeFilter.AND
if query.ancestor:
ancestor_pb = helpers._prepare_key_for_request(
query.ancestor.to_protobuf())
# Filter on __key__ HAS_ANCESTOR == ancestor.
ancestor_filter = composite_filter.filter.add().property_filter
ancestor_filter.property.name = '__key__'
ancestor_filter.operator = _query_pb2.PropertyFilter.HAS_ANCESTOR
ancestor_filter.value.key_value.CopyFrom(ancestor_pb)
for property_name, operator, value in query.filters:
pb_op_enum = query.OPERATORS.get(operator)
# Add the specific filter
property_filter = composite_filter.filter.add().property_filter
property_filter.property.name = property_name
property_filter.operator = pb_op_enum
# Set the value to filter on based on the type.
if property_name == '__key__':
key_pb = value.to_protobuf()
property_filter.value.key_value.CopyFrom(
helpers._prepare_key_for_request(key_pb))
else:
helpers._set_protobuf_value(property_filter.value, value)
if not composite_filter.filter:
pb.ClearField('filter')
for prop in query.order:
property_order = pb.order.add()
if prop.startswith('-'):
property_order.property.name = prop[1:]
property_order.direction = property_order.DESCENDING
else:
property_order.property.name = prop
property_order.direction = property_order.ASCENDING
for group_by_name in query.group_by:
pb.group_by.add().name = group_by_name
return pb
def _pb_from_query(query):
"""Convert a Query instance to the corresponding protobuf.
:type query: :class:`Query`
:param query: The source query.
:rtype: :class:`gcloud.datastore._generated.query_pb2.Query`
:returns: A protobuf that can be sent to the protobuf API. N.b. that
it does not contain "in-flight" fields for ongoing query
executions (cursors, offset, limit).
"""
pb = _query_pb2.Query()
for projection_name in query.projection:
pb.projection.add().property.name = projection_name
if query.kind:
pb.kind.add().name = query.kind
composite_filter = pb.filter.composite_filter
composite_filter.op = _query_pb2.CompositeFilter.AND
if query.ancestor:
ancestor_pb = query.ancestor.to_protobuf()
# Filter on __key__ HAS_ANCESTOR == ancestor.
ancestor_filter = composite_filter.filters.add().property_filter
ancestor_filter.property.name = '__key__'
ancestor_filter.op = _query_pb2.PropertyFilter.HAS_ANCESTOR
ancestor_filter.value.key_value.CopyFrom(ancestor_pb)
for property_name, operator, value in query.filters:
pb_op_enum = query.OPERATORS.get(operator)
# Add the specific filter
property_filter = composite_filter.filters.add().property_filter
property_filter.property.name = property_name
property_filter.op = pb_op_enum
# Set the value to filter on based on the type.
if property_name == '__key__':
key_pb = value.to_protobuf()
property_filter.value.key_value.CopyFrom(key_pb)
else:
helpers._set_protobuf_value(property_filter.value, value)
if not composite_filter.filters:
pb.ClearField('filter')
for prop in query.order:
property_order = pb.order.add()
if prop.startswith('-'):
property_order.property.name = prop[1:]
property_order.direction = property_order.DESCENDING
else:
property_order.property.name = prop
property_order.direction = property_order.ASCENDING
for distinct_on_name in query.distinct_on:
pb.distinct_on.add().name = distinct_on_name
return pb
def _callFUT(self, value_pb, val):
from gcloud.datastore.helpers import _set_protobuf_value
return _set_protobuf_value(value_pb, val)
def _callFUT(self, value_pb, val):
from gcloud.datastore.helpers import _set_protobuf_value
return _set_protobuf_value(value_pb, val)
def filter(self, expression, value):
"""Filter the query based on an expression and a value.
This will return a clone of the current :class:`Query`
filtered by the expression and value provided.
Expressions take the form of::
.filter('<property> <operator>', <value>)
where property is a property stored on the entity in the datastore
and operator is one of ``OPERATORS``
(ie, ``=``, ``<``, ``<=``, ``>``, ``>=``)::
>>> query = Query('Person')
>>> filtered_query = query.filter('name =', 'James')
>>> filtered_query = query.filter('age >', 50)
Because each call to ``.filter()`` returns a cloned ``Query`` object
we are able to string these together::
>>> query = Query('Person').filter(
... 'name =', 'James').filter('age >', 50)
:type expression: string
:param expression: An expression of a property and an
operator (ie, ``=``).
:type value: integer, string, boolean, float, None, datetime
:param value: The value to filter on.
:rtype: :class:`Query`
:returns: A Query filtered by the expression and value provided.
"""
clone = self._clone()
# Take an expression like 'property >=', and parse it into
# useful pieces.
property_name, operator = None, None
expression = expression.strip()
# Use None to split on *any* whitespace.
expr_pieces = expression.rsplit(None, 1)
if len(expr_pieces) == 2:
property_name, operator = expr_pieces
property_name = property_name.strip()
# If no whitespace in `expression`, `operator` will be `None` and
# self.OPERATORS[None] will be `None` as well.
pb_op_enum = self.OPERATORS.get(operator)
if pb_op_enum is None:
raise ValueError('Invalid expression: "%s"' % expression)
# Build a composite filter AND'd together.
composite_filter = clone._pb.filter.composite_filter
composite_filter.operator = datastore_pb.CompositeFilter.AND
# Add the specific filter
property_filter = composite_filter.filter.add().property_filter
property_filter.property.name = property_name
property_filter.operator = pb_op_enum
# Set the value to filter on based on the type.
helpers._set_protobuf_value(property_filter.value, value)
return clone
duration = time.time() - start
check_key_pb(key_pb)
return duration
MUTATION = CONNECTION.mutation()
INSERT_AUTO = MUTATION.insert_auto_id.add()
INSERT_AUTO.key.CopyFrom(PARTIAL_KEY_PB)
for name, value in BASE_DATA.items():
prop = INSERT_AUTO.property.add()
# Set the name of the property.
prop.name = name
# Set the appropriate value.
helpers._set_protobuf_value(prop.value, value)
def save_entity_fresh():
start = time.time()
result = CONNECTION.commit(DATASET_ID, MUTATION)
duration = time.time() - start
key_pb = result.insert_auto_id_key[0]
check_key_pb(key_pb)
return duration
COMMIT_REQUEST = datastore_pb.CommitRequest()
COMMIT_REQUEST.mode = datastore_pb.CommitRequest.NON_TRANSACTIONAL
COMMIT_REQUEST.mutation.CopyFrom(MUTATION)
def filter(self, expression, value):
"""Filter the query based on an expression and a value.
This will return a clone of the current :class:`Query`
filtered by the expression and value provided.
Expressions take the form of::
.filter('<property> <operator>', <value>)
where property is a property stored on the entity in the datastore
and operator is one of ``OPERATORS``
(ie, ``=``, ``<``, ``<=``, ``>``, ``>=``)::
>>> query = Query('Person')
>>> filtered_query = query.filter('name =', 'James')
>>> filtered_query = query.filter('age >', 50)
Because each call to ``.filter()`` returns a cloned ``Query`` object
we are able to string these together::
>>> query = Query('Person').filter(
... 'name =', 'James').filter('age >', 50)
:type expression: string
:param expression: An expression of a property and an
operator (ie, ``=``).
:type value: integer, string, boolean, float, None, datetime
:param value: The value to filter on.
:rtype: :class:`Query`
:returns: A Query filtered by the expression and value provided.
"""
clone = self._clone()
# Take an expression like 'property >=', and parse it into
# useful pieces.
property_name, operator = None, None
expression = expression.strip()
# Use None to split on *any* whitespace.
expr_pieces = expression.rsplit(None, 1)
if len(expr_pieces) == 2:
property_name, operator = expr_pieces
property_name = property_name.strip()
# If no whitespace in `expression`, `operator` will be `None` and
# self.OPERATORS[None] will be `None` as well.
pb_op_enum = self.OPERATORS.get(operator)
if pb_op_enum is None:
raise ValueError('Invalid expression: "%s"' % expression)
# Build a composite filter AND'd together.
composite_filter = clone._pb.filter.composite_filter
composite_filter.operator = datastore_pb.CompositeFilter.AND
# Add the specific filter
property_filter = composite_filter.filter.add().property_filter
property_filter.property.name = property_name
property_filter.operator = pb_op_enum
# Set the value to filter on based on the type.
helpers._set_protobuf_value(property_filter.value, value)
return clone
