def Run(self, limit, offset):
opts = self._gae_query._Query__query_options
if opts.keys_only or opts.projection:
return self._gae_query.Run(limit=limit, offset=offset)
ret = caching.get_from_cache(self._identifier, self._namespace)
if ret is not None and not utils.entity_matches_query(
ret, self._gae_query):
ret = None
if ret is None:
# We do a fast keys_only query to get the result
keys_query = rpc.Query(self._gae_query._Query__kind,
keys_only=True,
namespace=self._namespace)
keys_query.update(self._gae_query)
keys = keys_query.Run(limit=limit, offset=offset)
# Do a consistent get so we don't cache stale data, and recheck the result matches the query
ret = [
x for x in rpc.Get(keys)
if x and utils.entity_matches_query(x, self._gae_query)
]
if len(ret) == 1:
caching.add_entities_to_cache(
self._model,
[ret[0]],
caching.CachingSituation.DATASTORE_GET,
self._namespace,
)
return iter(ret)
return iter([ret])
def iter_results(results):
returned = 0
# This is safe, because Django is fetching all results any way :(
sorted_results = sorted(results, cmp=partial(utils.django_ordering_comparison, self.ordering))
sorted_results = [result for result in sorted_results if result is not None]
if cache and sorted_results:
caching.add_entities_to_cache(self.model, sorted_results, caching.CachingSituation.DATASTORE_GET)
for result in sorted_results:
if not any([ utils.entity_matches_query(result, qry) for qry in self.queries_by_key[result.key()]]):
continue
if offset and returned < offset:
# Skip entities based on offset
returned += 1
continue
else:
yield _convert_entity_based_on_query_options(result, opts)
returned += 1
# If there is a limit, we might be done!
if limit is not None and returned == (offset or 0) + limit:
break
def txn():
for key in keys:
if check_existence and key is not None:
if utils.key_exists(key):
raise IntegrityError("Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name, basestring) and id_or_name.startswith("__"):
raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name)
# Notify App Engine of any keys we're specifying intentionally
reserve_id(key.kind(), key.id_or_name(), self.namespace)
results = datastore.Put(entities)
for entity in entities:
markers.extend(constraints.acquire(self.model, entity))
caching.add_entities_to_cache(
self.model,
entities,
caching.CachingSituation.DATASTORE_GET_PUT,
self.namespace,
skip_memcache=True
)
return results
def txn():
for key in keys:
if check_existence and key is not None:
if utils.key_exists(key):
raise IntegrityError("Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name, six.string_types) and id_or_name.startswith("__"):
raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name)
# Notify App Engine of any keys we're specifying intentionally
reserve_id(key.kind(), key.id_or_name(), self.namespace)
results = perform_insert(entities)
for entity, _ in entities:
markers.extend(constraints.acquire(self.model, entity))
caching.add_entities_to_cache(
self.model,
[x[0] for x in entities],
caching.CachingSituation.DATASTORE_GET_PUT,
self.namespace,
skip_memcache=True
)
return results
def txn():
if key is not None:
if utils.key_exists(key):
raise IntegrityError(
"Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name,
basestring) and id_or_name.startswith("__"):
raise NotSupportedError(
"Datastore ids cannot start with __. Id was %s" %
id_or_name)
if not constraints.constraint_checks_enabled(self.model):
# Fast path, just insert
results.append(datastore.Put(ent))
else:
markers = constraints.acquire(self.model, ent)
try:
results.append(datastore.Put(ent))
if not was_in_transaction:
# We can cache if we weren't in a transaction before this little nested one
caching.add_entities_to_cache(
self.model, [ent],
caching.CachingSituation.DATASTORE_GET_PUT)
except:
# Make sure we delete any created markers before we re-raise
constraints.release_markers(markers)
raise
def txn():
if key is not None:
if utils.key_exists(key):
raise IntegrityError("Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name, basestring) and id_or_name.startswith("__"):
raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name)
if not constraints.constraint_checks_enabled(self.model):
# Fast path, just insert
results.append(datastore.Put(ent))
else:
markers = constraints.acquire(self.model, ent)
try:
results.append(datastore.Put(ent))
if not was_in_transaction:
# We can cache if we weren't in a transaction before this little nested one
caching.add_entities_to_cache(
self.model,
[ent],
caching.CachingSituation.DATASTORE_GET_PUT,
self.namespace,
)
except:
# Make sure we delete any created markers before we re-raise
constraints.release_markers(markers)
raise
def Run(self, limit, offset):
opts = self._gae_query._Query__query_options
if opts.keys_only or opts.projection:
return self._gae_query.Run(limit=limit, offset=offset)
ret = caching.get_from_cache(self._identifier, self._namespace)
if ret is not None and not utils.entity_matches_query(ret, self._gae_query):
ret = None
if ret is None:
# We do a fast keys_only query to get the result
keys_query = Query(self._gae_query._Query__kind, keys_only=True, namespace=self._namespace)
keys_query.update(self._gae_query)
keys = keys_query.Run(limit=limit, offset=offset)
# Do a consistent get so we don't cache stale data, and recheck the result matches the query
ret = [x for x in datastore.Get(keys) if x and utils.entity_matches_query(x, self._gae_query)]
if len(ret) == 1:
caching.add_entities_to_cache(
self._model,
[ret[0]],
caching.CachingSituation.DATASTORE_GET,
self._namespace,
)
return iter(ret)
return iter([ret])
def iter_results(results):
returned = 0
# This is safe, because Django is fetching all results any way :(
sorted_results = sorted(results,
cmp=partial(
utils.django_ordering_comparison,
self.ordering))
sorted_results = [
result for result in sorted_results if result is not None
]
if cache_results and sorted_results:
caching.add_entities_to_cache(
self.model,
sorted_results[:max_cache_count],
caching.CachingSituation.DATASTORE_GET,
self.namespace,
)
for result in sorted_results:
if is_projection:
entity_matches_query = True
else:
entity_matches_query = any(
utils.entity_matches_query(result, qry)
for qry in self.queries_by_key[result.key()])
if not entity_matches_query:
continue
if offset and returned < offset:
# Skip entities based on offset
returned += 1
continue
else:
yield _convert_entity_based_on_query_options(result, opts)
returned += 1
# If there is a limit, we might be done!
if limit is not None and returned == (offset or 0) + limit:
break
def execute(self):
if self.has_pk and not has_concrete_parents(self.model):
results = []
# We are inserting, but we specified an ID, we need to check for existence before we Put()
# We do it in a loop so each check/put is transactional - because it's an ancestor query it shouldn't
# cost any entity groups
was_in_transaction = datastore.IsInTransaction()
for key, ent in zip(self.included_keys, self.entities):
@db.transactional
def txn():
if key is not None:
if utils.key_exists(key):
raise IntegrityError("Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name, basestring) and id_or_name.startswith("__"):
raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name)
if not constraints.constraint_checks_enabled(self.model):
# Fast path, just insert
results.append(datastore.Put(ent))
else:
markers = constraints.acquire(self.model, ent)
try:
results.append(datastore.Put(ent))
if not was_in_transaction:
# We can cache if we weren't in a transaction before this little nested one
caching.add_entities_to_cache(
self.model,
[ent],
caching.CachingSituation.DATASTORE_GET_PUT,
self.namespace,
)
except:
# Make sure we delete any created markers before we re-raise
constraints.release_markers(markers)
raise
# Make sure we notify app engine that we are using this ID
# FIXME: Copy ancestor across to the template key
reserve_id(key.kind(), key.id_or_name(), self.namespace)
txn()
return results
else:
if not constraints.constraint_checks_enabled(self.model):
# Fast path, just bulk insert
results = datastore.Put(self.entities)
caching.add_entities_to_cache(
self.model,
self.entities,
caching.CachingSituation.DATASTORE_PUT,
self.namespace
)
return results
else:
markers = []
try:
#FIXME: We should rearrange this so that each entity is handled individually like above. We'll
# lose insert performance, but gain consistency on errors which is more important
markers = constraints.acquire_bulk(self.model, self.entities)
results = datastore.Put(self.entities)
caching.add_entities_to_cache(
self.model,
self.entities,
caching.CachingSituation.DATASTORE_PUT,
self.namespace,
)
except:
to_delete = chain(*markers)
constraints.release_markers(to_delete)
raise
for ent, k, m in zip(self.entities, results, markers):
ent.__key = k
constraints.update_instance_on_markers(ent, m)
return results
def _update_entity(self, key):
caching.remove_entities_from_cache_by_key([key], self.namespace)
try:
result = datastore.Get(key)
except datastore_errors.EntityNotFoundError:
# Return false to indicate update failure
return False
if (
isinstance(self.select.gae_query, (Query, UniqueQuery)) # ignore QueryByKeys and NoOpQuery
and not utils.entity_matches_query(result, self.select.gae_query)
):
# Due to eventual consistency they query may have returned an entity which no longer
# matches the query
return False
original = copy.deepcopy(result)
instance_kwargs = {field.attname:value for field, param, value in self.values}
# Note: If you replace MockInstance with self.model, you'll find that some delete
# tests fail in the test app. This is because any unspecified fields would then call
# get_default (even though we aren't going to use them) which may run a query which
# fails inside this transaction. Given as we are just using MockInstance so that we can
# call django_instance_to_entity it on it with the subset of fields we pass in,
# what we have is fine.
meta = self.model._meta
instance = MockInstance(
_original=MockInstance(_meta=meta, **result),
_meta=meta,
**instance_kwargs
)
# We need to add to the class attribute, rather than replace it!
original_class = result.get(POLYMODEL_CLASS_ATTRIBUTE, [])
# Update the entity we read above with the new values
result.update(django_instance_to_entity(
self.connection, self.model,
[ x[0] for x in self.values], # Pass in the fields that were updated
True, instance)
)
# Make sure we keep all classes in the inheritence tree!
if original_class:
if result[POLYMODEL_CLASS_ATTRIBUTE] is not None:
result[POLYMODEL_CLASS_ATTRIBUTE].extend(original_class)
# Make sure we don't add duplicates
else:
result[POLYMODEL_CLASS_ATTRIBUTE] = original_class
if POLYMODEL_CLASS_ATTRIBUTE in result:
result[POLYMODEL_CLASS_ATTRIBUTE] = list(set(result[POLYMODEL_CLASS_ATTRIBUTE]))
if not constraints.constraint_checks_enabled(self.model):
# The fast path, no constraint checking
datastore.Put(result)
caching.add_entities_to_cache(
self.model,
[result],
caching.CachingSituation.DATASTORE_PUT,
self.namespace,
)
else:
to_acquire, to_release = constraints.get_markers_for_update(self.model, original, result)
# Acquire first, because if that fails then we don't want to alter what's already there
constraints.acquire_identifiers(to_acquire, result.key())
try:
datastore.Put(result)
caching.add_entities_to_cache(
self.model,
[result],
caching.CachingSituation.DATASTORE_PUT,
self.namespace,
)
except:
constraints.release_identifiers(to_acquire, namespace=self.namespace)
raise
else:
# Now we release the ones we don't want anymore
constraints.release_identifiers(to_release, self.namespace)
# Return true to indicate update success
return True
def execute(self):
if self.has_pk and not has_concrete_parents(self.model):
results = []
# We are inserting, but we specified an ID, we need to check for existence before we Put()
# We do it in a loop so each check/put is transactional - because it's an ancestor query it shouldn't
# cost any entity groups
was_in_transaction = datastore.IsInTransaction()
for key, ent in zip(self.included_keys, self.entities):
@db.transactional
def txn():
if key is not None:
if utils.key_exists(key):
raise IntegrityError(
"Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name,
basestring) and id_or_name.startswith("__"):
raise NotSupportedError(
"Datastore ids cannot start with __. Id was %s" %
id_or_name)
if not constraints.constraint_checks_enabled(self.model):
# Fast path, just insert
results.append(datastore.Put(ent))
else:
markers = constraints.acquire(self.model, ent)
try:
results.append(datastore.Put(ent))
if not was_in_transaction:
# We can cache if we weren't in a transaction before this little nested one
caching.add_entities_to_cache(
self.model, [ent],
caching.CachingSituation.DATASTORE_GET_PUT)
except:
# Make sure we delete any created markers before we re-raise
constraints.release_markers(markers)
raise
# Make sure we notify app engine that we are using this ID
# FIXME: Copy ancestor across to the template key
reserve_id(key.kind(), key.id_or_name())
txn()
return results
else:
if not constraints.constraint_checks_enabled(self.model):
# Fast path, just bulk insert
results = datastore.Put(self.entities)
caching.add_entities_to_cache(
self.model, self.entities,
caching.CachingSituation.DATASTORE_PUT)
return results
else:
markers = []
try:
#FIXME: We should rearrange this so that each entity is handled individually like above. We'll
# lose insert performance, but gain consistency on errors which is more important
markers = constraints.acquire_bulk(self.model,
self.entities)
results = datastore.Put(self.entities)
caching.add_entities_to_cache(
self.model, self.entities,
caching.CachingSituation.DATASTORE_PUT)
except:
to_delete = chain(*markers)
constraints.release_markers(to_delete)
raise
for ent, k, m in zip(self.entities, results, markers):
ent.__key = k
constraints.update_instance_on_markers(ent, m)
return results
def txn():
caching.remove_entities_from_cache_by_key([key], self.namespace)
try:
result = datastore.Get(key)
except datastore_errors.EntityNotFoundError:
# Return false to indicate update failure
return False
if (
isinstance(self.select.gae_query, (Query, UniqueQuery)) # ignore QueryByKeys and NoOpQuery
and not utils.entity_matches_query(result, self.select.gae_query)
):
# Due to eventual consistency they query may have returned an entity which no longer
# matches the query
return False
original = copy.deepcopy(result)
instance_kwargs = {field.attname:value for field, param, value in self.values}
# Note: If you replace MockInstance with self.model, you'll find that some delete
# tests fail in the test app. This is because any unspecified fields would then call
# get_default (even though we aren't going to use them) which may run a query which
# fails inside this transaction. Given as we are just using MockInstance so that we can
# call django_instance_to_entity it on it with the subset of fields we pass in,
# what we have is fine.
meta = self.model._meta
instance = MockInstance(
_original=MockInstance(_meta=meta, **result),
_meta=meta,
**instance_kwargs
)
# We need to add to the class attribute, rather than replace it!
original_class = result.get(POLYMODEL_CLASS_ATTRIBUTE, [])
# Update the entity we read above with the new values
result.update(django_instance_to_entity(
self.connection, self.model,
[ x[0] for x in self.values], # Pass in the fields that were updated
True, instance)
)
# Make sure we keep all classes in the inheritence tree!
if original_class:
if result[POLYMODEL_CLASS_ATTRIBUTE] is not None:
result[POLYMODEL_CLASS_ATTRIBUTE].extend(original_class)
# Make sure we don't add duplicates
else:
result[POLYMODEL_CLASS_ATTRIBUTE] = original_class
if POLYMODEL_CLASS_ATTRIBUTE in result:
result[POLYMODEL_CLASS_ATTRIBUTE] = list(set(result[POLYMODEL_CLASS_ATTRIBUTE]))
if not constraints.has_active_unique_constraints(self.model):
# The fast path, no constraint checking
datastore.Put(result)
caching.add_entities_to_cache(
self.model,
[result],
caching.CachingSituation.DATASTORE_PUT,
self.namespace,
skip_memcache=True,
)
else:
markers_to_acquire[:], markers_to_release[:] = constraints.get_markers_for_update(
self.model, original, result
)
datastore.Put(result)
constraints.update_identifiers(markers_to_acquire, markers_to_release, result.key())
# If the datastore.Put() fails then the exception will only be raised when the
# transaction applies, which means that we will still get to here and will still have
# applied the marker changes (because they're in a nested, independent transaction).
# Hence we set this flag to tell us that we got this far and that we should roll them back.
rollback_markers[0] = True
# If something dies between here and the `return` statement then we'll have stale unique markers
try:
# Update the cache before dealing with unique markers, as CachingSituation.DATASTORE_PUT
# will only update the context cache
caching.add_entities_to_cache(
self.model,
[result],
caching.CachingSituation.DATASTORE_PUT,
self.namespace,
skip_memcache=True,
)
except:
# We ignore the exception because raising will rollback the transaction causing
# an inconsistent state
logger.exception("Unable to update the context cache")
pass
# Return true to indicate update success
return True
def txn():
caching.remove_entities_from_cache_by_key([key], self.namespace)
try:
result = rpc.Get(key)
except datastore_errors.EntityNotFoundError:
# Return false to indicate update failure
return False
if (
isinstance(self.select.gae_query, (Query, meta_queries.UniqueQuery)) # ignore QueryByKeys and NoOpQuery
and not utils.entity_matches_query(result, self.select.gae_query)
):
# Due to eventual consistency they query may have returned an entity which no longer
# matches the query
return False
original = copy.deepcopy(result)
instance_kwargs = {field.attname: value for field, param, value in self.values}
# Note: If you replace MockInstance with self.model, you'll find that some delete
# tests fail in the test app. This is because any unspecified fields would then call
# get_default (even though we aren't going to use them) which may run a query which
# fails inside this transaction. Given as we are just using MockInstance so that we can
# call django_instance_to_entities it on it with the subset of fields we pass in,
# what we have is fine.
meta = self.model._meta
instance = MockInstance(
_original=MockInstance(_meta=meta, **result),
_meta=meta,
**instance_kwargs
)
# Convert the instance to an entity
primary, descendents = django_instance_to_entities(
self.connection,
[x[0] for x in self.values], # Pass in the fields that were updated
True, instance,
model=self.model
)
# Update the entity we read above with the new values
result.update(primary)
# Remove fields which have been marked to be unindexed
for col in getattr(primary, "_properties_to_remove", []):
if col in result:
del result[col]
# Make sure that any polymodel classes which were in the original entity are kept,
# as django_instance_to_entities may have wiped them as well as added them.
polymodel_classes = list(set(
original.get(POLYMODEL_CLASS_ATTRIBUTE, []) + result.get(POLYMODEL_CLASS_ATTRIBUTE, [])
))
if polymodel_classes:
result[POLYMODEL_CLASS_ATTRIBUTE] = polymodel_classes
def perform_insert():
"""
Inserts result, and any descendents with their ancestor
value set
"""
inserted_key = rpc.Put(result)
if descendents:
for i, descendent in enumerate(descendents):
descendents[i] = Entity(
descendent.kind(),
parent=inserted_key,
namespace=inserted_key.namespace(),
id=descendent.key().id() or None,
name=descendent.key().name() or None
)
descendents[i].update(descendent)
rpc.Put(descendents)
if not constraints.has_active_unique_constraints(self.model):
# The fast path, no constraint checking
perform_insert()
caching.add_entities_to_cache(
self.model,
[result],
caching.CachingSituation.DATASTORE_PUT,
self.namespace,
skip_memcache=True,
)
else:
markers_to_acquire[:], markers_to_release[:] = constraints.get_markers_for_update(
self.model, original, result
)
perform_insert()
constraints.update_identifiers(markers_to_acquire, markers_to_release, result.key())
# If the rpc.Put() fails then the exception will only be raised when the
# transaction applies, which means that we will still get to here and will still have
# applied the marker changes (because they're in a nested, independent transaction).
# Hence we set this flag to tell us that we got this far and that we should roll them back.
rollback_markers[0] = True
# If something dies between here and the `return` statement then we'll have stale unique markers
try:
# Update the cache before dealing with unique markers, as CachingSituation.DATASTORE_PUT
# will only update the context cache
caching.add_entities_to_cache(
self.model,
[result],
caching.CachingSituation.DATASTORE_PUT,
self.namespace,
skip_memcache=True,
)
except:
# We ignore the exception because raising will rollback the transaction causing
# an inconsistent state
logger.exception("Unable to update the context cache")
pass
# Return true to indicate update success
return True
def execute(self):
check_existence = self.has_pk and not has_concrete_parents(self.model)
def perform_insert(entities):
results = []
for primary, descendents in entities:
new_key = rpc.Put(primary)
if descendents:
for i, descendent in enumerate(descendents):
descendents[i] = Entity(
descendent.kind(),
parent=new_key,
namespace=new_key.namespace(),
id=descendent.key().id() or None,
name=descendent.key().name() or None
)
descendents[i].update(descendent)
rpc.Put(descendents)
results.append(new_key)
return results
if not constraints.has_active_unique_constraints(self.model) and not check_existence:
# Fast path, no constraint checks and no keys mean we can just do a normal rpc.Put
# which isn't limited to 25
results = perform_insert(self.entities) # This modifies self.entities and sets their keys
caching.add_entities_to_cache(
self.model,
[x[0] for x in self.entities],
caching.CachingSituation.DATASTORE_GET_PUT,
self.namespace,
skip_memcache=True
)
return results
entity_group_count = len(self.entities)
def insert_chunk(keys, entities):
# Note that this is limited to a maximum of 25 entities.
markers = []
@db.transactional(xg=entity_group_count > 1)
def txn():
for key in keys:
if check_existence and key is not None:
if utils.key_exists(key):
raise IntegrityError("Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name, six.string_types) and id_or_name.startswith("__"):
raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name)
# Notify App Engine of any keys we're specifying intentionally
reserve_id(key.kind(), key.id_or_name(), self.namespace)
results = perform_insert(entities)
for entity, _ in entities:
markers.extend(constraints.acquire(self.model, entity))
caching.add_entities_to_cache(
self.model,
[x[0] for x in entities],
caching.CachingSituation.DATASTORE_GET_PUT,
self.namespace,
skip_memcache=True
)
return results
try:
return txn()
except:
# There are 3 possible reasons why we've ended up here:
# 1. The rpc.Put() failed, but note that because it's a transaction, the
# exception isn't raised until the END of the transaction block.
# 2. Some of the markers were acquired, but then we hit a unique constraint
# conflict and so the outer transaction was rolled back.
# 3. Something else went wrong after we'd acquired markers, e.g. the
# caching.add_entities_to_cache call got hit by a metaphorical bus.
# In any of these cases, we (may) have acquired markers via (a) nested, independent
# transaction(s), and so we need to release them again.
constraints.release_markers(markers)
raise
# We can't really support this and maintain expected behaviour. If we chunked the insert and one of the
# chunks fails it will mean some of the data would be saved and rather than trying to communicate that back
# to the user it's better that they chunk the data themselves as they can deal with the failure better
if entity_group_count > datastore_stub_util._MAX_EG_PER_TXN:
raise BulkInsertError("Bulk inserts with unique constraints, or pre-defined keys are limited to {} instances on the datastore".format(
datastore_stub_util._MAX_EG_PER_TXN
))
return insert_chunk(self.included_keys, self.entities)
def txn():
caching.remove_entities_from_cache_by_key([key], self.namespace)
try:
result = datastore.Get(key)
except datastore_errors.EntityNotFoundError:
# Return false to indicate update failure
return False
if (isinstance(
self.select.gae_query,
(Query, UniqueQuery)) # ignore QueryByKeys and NoOpQuery
and not utils.entity_matches_query(result,
self.select.gae_query)):
# Due to eventual consistency they query may have returned an entity which no longer
# matches the query
return False
original = copy.deepcopy(result)
instance_kwargs = {
field.attname: value
for field, param, value in self.values
}
# Note: If you replace MockInstance with self.model, you'll find that some delete
# tests fail in the test app. This is because any unspecified fields would then call
# get_default (even though we aren't going to use them) which may run a query which
# fails inside this transaction. Given as we are just using MockInstance so that we can
# call django_instance_to_entity it on it with the subset of fields we pass in,
# what we have is fine.
meta = self.model._meta
instance = MockInstance(_original=MockInstance(_meta=meta,
**result),
_meta=meta,
**instance_kwargs)
# We need to add to the class attribute, rather than replace it!
original_class = result.get(POLYMODEL_CLASS_ATTRIBUTE, [])
# Update the entity we read above with the new values
result.update(
django_instance_to_entity(
self.connection,
self.model,
[x[0] for x in self.values
], # Pass in the fields that were updated
True,
instance))
# Make sure we keep all classes in the inheritence tree!
if original_class:
if result[POLYMODEL_CLASS_ATTRIBUTE] is not None:
result[POLYMODEL_CLASS_ATTRIBUTE].extend(original_class)
# Make sure we don't add duplicates
else:
result[POLYMODEL_CLASS_ATTRIBUTE] = original_class
if POLYMODEL_CLASS_ATTRIBUTE in result:
result[POLYMODEL_CLASS_ATTRIBUTE] = list(
set(result[POLYMODEL_CLASS_ATTRIBUTE]))
if not constraints.has_active_unique_constraints(self.model):
# The fast path, no constraint checking
datastore.Put(result)
caching.add_entities_to_cache(
self.model,
[result],
caching.CachingSituation.DATASTORE_PUT,
self.namespace,
skip_memcache=True,
)
else:
markers_to_acquire[:], markers_to_release[:] = constraints.get_markers_for_update(
self.model, original, result)
datastore.Put(result)
constraints.update_identifiers(markers_to_acquire,
markers_to_release,
result.key())
# If the datastore.Put() fails then the exception will only be raised when the
# transaction applies, which means that we will still get to here and will still have
# applied the marker changes (because they're in a nested, independent transaction).
# Hence we set this flag to tell us that we got this far and that we should roll them back.
rollback_markers[0] = True
# If something dies between here and the `return` statement then we'll have stale unique markers
try:
# Update the cache before dealing with unique markers, as CachingSituation.DATASTORE_PUT
# will only update the context cache
caching.add_entities_to_cache(
self.model,
[result],
caching.CachingSituation.DATASTORE_PUT,
self.namespace,
skip_memcache=True,
)
except:
# We ignore the exception because raising will rollback the transaction causing
# an inconsistent state
logging.exception("Unable to update the context cache")
pass
# Return true to indicate update success
return True
def execute(self):
check_existence = self.has_pk and not has_concrete_parents(self.model)
if not constraints.has_active_unique_constraints(self.model) and not check_existence:
# Fast path, no constraint checks and no keys mean we can just do a normal datastore.Put
# which isn't limited to 25
results = datastore.Put(self.entities) # This modifies self.entities and sets their keys
caching.add_entities_to_cache(
self.model,
self.entities,
caching.CachingSituation.DATASTORE_GET_PUT,
self.namespace,
skip_memcache=True
)
return results
def insert_chunk(keys, entities):
# Note that this is limited to a maximum of 25 entities.
markers = []
@db.transactional(xg=len(entities) > 1)
def txn():
for key in keys:
if check_existence and key is not None:
if utils.key_exists(key):
raise IntegrityError("Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name, basestring) and id_or_name.startswith("__"):
raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name)
# Notify App Engine of any keys we're specifying intentionally
reserve_id(key.kind(), key.id_or_name(), self.namespace)
results = datastore.Put(entities)
for entity in entities:
markers.extend(constraints.acquire(self.model, entity))
caching.add_entities_to_cache(
self.model,
entities,
caching.CachingSituation.DATASTORE_GET_PUT,
self.namespace,
skip_memcache=True
)
return results
try:
return txn()
except:
# There are 3 possible reasons why we've ended up here:
# 1. The datastore.Put() failed, but note that because it's a transaction, the
# exception isn't raised until the END of the transaction block.
# 2. Some of the markers were acquired, but then we hit a unique constraint
# conflict and so the outer transaction was rolled back.
# 3. Something else went wrong after we'd acquired markers, e.g. the
# caching.add_entities_to_cache call got hit by a metaphorical bus.
# In any of these cases, we (may) have acquired markers via (a) nested, independent
# transaction(s), and so we need to release them again.
constraints.release_markers(markers)
raise
# We can't really support this and maintain expected behaviour. If we chunked the insert and one of the
# chunks fails it will mean some of the data would be saved and rather than trying to communicate that back
# to the user it's better that they chunk the data themselves as they can deal with the failure better
if len(self.entities) > datastore_stub_util._MAX_EG_PER_TXN:
raise BulkInsertError("Bulk inserts with unique constraints, or pre-defined keys are limited to {} instances on the datastore".format(
datastore_stub_util._MAX_EG_PER_TXN
))
return insert_chunk(self.included_keys, self.entities)
def _update_entity(self, key):
caching.remove_entities_from_cache_by_key([key])
try:
result = datastore.Get(key)
except datastore_errors.EntityNotFoundError:
# Return false to indicate update failure
return False
if (isinstance(
self.select.gae_query,
(Query, UniqueQuery)) # ignore QueryByKeys and NoOpQuery
and
not utils.entity_matches_query(result, self.select.gae_query)):
# Due to eventual consistency they query may have returned an entity which no longer
# matches the query
return False
original = copy.deepcopy(result)
# Note: If you replace MockInstance with self.model, you'll find that some delete
# tests fail in the test app. This is because any unspecified fields would then call
# get_default (even though we aren't going to use them) which may run a query which
# fails inside this transaction. Given as we are just using MockInstance so that we can
# call django_instance_to_entity it on it with the subset of fields we pass in,
# what we have is fine.
meta = self.model._meta
instance_kwargs = {
field.attname: value
for field, param, value in self.values
}
instance = MockInstance(_original=MockInstance(_meta=meta, **result),
_meta=meta,
**instance_kwargs)
# We need to add to the class attribute, rather than replace it!
original_class = result.get(POLYMODEL_CLASS_ATTRIBUTE, [])
# Update the entity we read above with the new values
result.update(
django_instance_to_entity(
self.connection,
self.model,
[x[0]
for x in self.values], # Pass in the fields that were updated
True,
instance))
# Make sure we keep all classes in the inheritence tree!
if original_class:
if result[POLYMODEL_CLASS_ATTRIBUTE] is not None:
result[POLYMODEL_CLASS_ATTRIBUTE].extend(original_class)
# Make sure we don't add duplicates
else:
result[POLYMODEL_CLASS_ATTRIBUTE] = original_class
if POLYMODEL_CLASS_ATTRIBUTE in result:
result[POLYMODEL_CLASS_ATTRIBUTE] = list(
set(result[POLYMODEL_CLASS_ATTRIBUTE]))
if not constraints.constraint_checks_enabled(self.model):
# The fast path, no constraint checking
datastore.Put(result)
caching.add_entities_to_cache(
self.model, [result], caching.CachingSituation.DATASTORE_PUT)
else:
to_acquire, to_release = constraints.get_markers_for_update(
self.model, original, result)
# Acquire first, because if that fails then we don't want to alter what's already there
constraints.acquire_identifiers(to_acquire, result.key())
try:
datastore.Put(result)
caching.add_entities_to_cache(
self.model, [result],
caching.CachingSituation.DATASTORE_PUT)
except:
constraints.release_identifiers(to_acquire)
raise
else:
# Now we release the ones we don't want anymore
constraints.release_identifiers(to_release)
# Return true to indicate update success
return True
