def txn():
if key is not None:
if utils.key_exists(key):
raise IntegrityError(
"Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name,
basestring) and id_or_name.startswith("__"):
raise NotSupportedError(
"Datastore ids cannot start with __. Id was %s" %
id_or_name)
if not constraints.constraint_checks_enabled(self.model):
# Fast path, just insert
results.append(datastore.Put(ent))
else:
markers = constraints.acquire(self.model, ent)
try:
results.append(datastore.Put(ent))
if not was_in_transaction:
# We can cache if we weren't in a transaction before this little nested one
caching.add_entity_to_cache(
self.model, ent,
caching.CachingSituation.DATASTORE_GET_PUT)
except:
# Make sure we delete any created markers before we re-raise
constraints.release_markers(markers)
raise
def execute(self):
self.select.execute()
# This is a little bit more inefficient than just doing a keys_only query and
# sending it to delete, but I think this is the sacrifice to make for the unique caching layer
keys = []
def spawn_query(kind, key):
qry = Query(kind)
qry["__key__ ="] = key
return qry
queries = [spawn_query(self.select.db_table, x.key()) for x in self.select.results]
if not queries:
return
for entity in QueryByKeys(self.select.model, queries, []).Run():
keys.append(entity.key())
# Delete constraints if that's enabled
if constraints.constraint_checks_enabled(self.select.model):
constraints.release(self.select.model, entity)
caching.remove_entity_from_cache_by_key(entity.key())
datastore.Delete(keys)
def txn():
if key is not None:
if utils.key_exists(key):
raise IntegrityError("Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name, basestring) and id_or_name.startswith("__"):
raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name)
if not constraints.constraint_checks_enabled(self.model):
# Fast path, just insert
results.append(datastore.Put(ent))
else:
markers = constraints.acquire(self.model, ent)
try:
results.append(datastore.Put(ent))
if not was_in_transaction:
# We can cache if we weren't in a transaction before this little nested one
caching.add_entities_to_cache(
self.model,
[ent],
caching.CachingSituation.DATASTORE_GET_PUT,
self.namespace,
)
except:
# Make sure we delete any created markers before we re-raise
constraints.release_markers(markers)
raise
def execute(self):
self.select.execute()
# This is a little bit more inefficient than just doing a keys_only query and
# sending it to delete, but I think this is the sacrifice to make for the unique caching layer
keys = []
def spawn_query(kind, key):
qry = Query(
kind, namespace=key.namespace() or None
) # TODO: is the namespace necessary if we're passing the key?
qry["__key__ ="] = key
return qry
queries = [
spawn_query(x.key().kind(), x.key()) for x in self.select.results
]
if not queries:
return
for entity in QueryByKeys(self.model, queries, [],
self.namespace).Run():
keys.append(entity.key())
# Delete constraints if that's enabled
if constraints.constraint_checks_enabled(self.model):
constraints.release(self.model, entity)
caching.remove_entities_from_cache_by_key(keys, self.namespace)
datastore.Delete(keys)
def execute(self):
self.select.execute()
# This is a little bit more inefficient than just doing a keys_only query and
# sending it to delete, but I think this is the sacrifice to make for the unique caching layer
keys = []
def spawn_query(kind, key):
qry = Query(kind, namespace=key.namespace() or None) # TODO: is the namespace necessary if we're passing the key?
qry["__key__ ="] = key
return qry
queries = [spawn_query(x.key().kind(), x.key()) for x in self.select.results]
if not queries:
return
for entity in QueryByKeys(self.model, queries, [], self.namespace).Run():
keys.append(entity.key())
# Delete constraints if that's enabled
if constraints.constraint_checks_enabled(self.model):
constraints.release(self.model, entity)
caching.remove_entities_from_cache_by_key(keys, self.namespace)
datastore.Delete(keys)
def _update_entity(self, key):
caching.remove_entity_from_context_cache_by_key(key)
result = datastore.Get(key)
original = copy.deepcopy(result)
instance_kwargs = {field.attname:value for field, param, value in self.values}
instance = MockInstance(**instance_kwargs)
for field, param, value in self.values:
result[field.column] = get_prepared_db_value(self.connection, instance, field, raw=True)
#Add special indexed fields
for index in special_indexes_for_column(self.model, field.column):
indexer = REQUIRES_SPECIAL_INDEXES[index]
result[indexer.indexed_column_name(field.column)] = indexer.prep_value_for_database(value)
if not constraints.constraint_checks_enabled(self.model):
#The fast path, no constraint checking
datastore.Put(result)
caching.add_entity_to_context_cache(self.model, result)
else:
to_acquire, to_release = constraints.get_markers_for_update(self.model, original, result)
#Acquire first, because if that fails then we don't want to alter what's already there
constraints.acquire_identifiers(to_acquire, result.key())
try:
datastore.Put(result)
caching.add_entity_to_context_cache(self.model, result)
except:
constraints.release_identifiers(to_acquire)
raise
else:
#Now we release the ones we don't want anymore
constraints.release_identifiers(to_release)
def _update_entity(self, key):
caching.remove_entity_from_cache_by_key(key)
try:
result = datastore.Get(key)
except datastore_errors.EntityNotFoundError:
# Return false to indicate update failure
return False
original = copy.deepcopy(result)
instance_kwargs = {
field.attname: value
for field, param, value in self.values
}
# Note: If you replace MockInstance with self.model, you'll find that some delete
# tests fail in the test app. This is because any unspecified fields would then call
# get_default (even though we aren't going to use them) which may run a query which
# fails inside this transaction. Given as we are just using MockInstance so that we can
# call django_instance_to_entity it on it with the subset of fields we pass in,
# what we have is fine.
instance = MockInstance(**instance_kwargs)
# Update the entity we read above with the new values
result.update(
django_instance_to_entity(
self.connection,
self.model,
[x[0]
for x in self.values], # Pass in the fields that were updated
True,
instance))
if not constraints.constraint_checks_enabled(self.model):
# The fast path, no constraint checking
datastore.Put(result)
caching.add_entity_to_cache(self.model, result,
caching.CachingSituation.DATASTORE_PUT)
else:
to_acquire, to_release = constraints.get_markers_for_update(
self.model, original, result)
# Acquire first, because if that fails then we don't want to alter what's already there
constraints.acquire_identifiers(to_acquire, result.key())
try:
datastore.Put(result)
caching.add_entity_to_cache(
self.model, result, caching.CachingSituation.DATASTORE_PUT)
except:
constraints.release_identifiers(to_acquire)
raise
else:
# Now we release the ones we don't want anymore
constraints.release_identifiers(to_release)
# Return true to indicate update success
return True
def _update_entity(self, key):
caching.remove_entity_from_cache_by_key(key)
try:
result = datastore.Get(key)
except datastore_errors.EntityNotFoundError:
# Return false to indicate update failure
return False
if (
isinstance(self.select.gae_query, (Query, UniqueQuery)) # ignore QueryByKeys and NoOpQuery
and not utils.entity_matches_query(result, self.select.gae_query)
):
# Due to eventual consistency they query may have returned an entity which no longer
# matches the query
return False
original = copy.deepcopy(result)
instance_kwargs = {field.attname:value for field, param, value in self.values}
# Note: If you replace MockInstance with self.model, you'll find that some delete
# tests fail in the test app. This is because any unspecified fields would then call
# get_default (even though we aren't going to use them) which may run a query which
# fails inside this transaction. Given as we are just using MockInstance so that we can
# call django_instance_to_entity it on it with the subset of fields we pass in,
# what we have is fine.
instance = MockInstance(**instance_kwargs)
# Update the entity we read above with the new values
result.update(django_instance_to_entity(
self.connection, self.model,
[ x[0] for x in self.values], # Pass in the fields that were updated
True, instance)
)
if not constraints.constraint_checks_enabled(self.model):
# The fast path, no constraint checking
datastore.Put(result)
caching.add_entity_to_cache(self.model, result, caching.CachingSituation.DATASTORE_PUT)
else:
to_acquire, to_release = constraints.get_markers_for_update(self.model, original, result)
# Acquire first, because if that fails then we don't want to alter what's already there
constraints.acquire_identifiers(to_acquire, result.key())
try:
datastore.Put(result)
caching.add_entity_to_cache(self.model, result, caching.CachingSituation.DATASTORE_PUT)
except:
constraints.release_identifiers(to_acquire)
raise
else:
# Now we release the ones we don't want anymore
constraints.release_identifiers(to_release)
# Return true to indicate update success
return True
def execute(self):
self.select.execute()
#This is a little bit more inefficient than just doing a keys_only query and
#sending it to delete, but I think this is the sacrifice to make for the unique caching layer
keys = []
for entity in QueryByKeys(
Query(self.select.model._meta.db_table),
[ x.key() for x in self.select.results ],
[]
).Run():
keys.append(entity.key())
##Delete constraints if that's enabled
if constraints.constraint_checks_enabled(self.select.model):
constraints.release(self.select.model, entity)
caching.remove_entity_from_context_cache_by_key(entity.key())
datastore.Delete(keys)
def txn():
if key is not None:
if utils.key_exists(key):
raise IntegrityError("Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name, basestring) and id_or_name.startswith("__"):
raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name)
if not constraints.constraint_checks_enabled(self.model):
#Fast path, just insert
results.append(datastore.Put(ent))
else:
markers = constraints.acquire(self.model, ent)
try:
results.append(datastore.Put(ent))
caching.add_entity_to_context_cache(self.model, ent)
except:
#Make sure we delete any created markers before we re-raise
constraints.release_markers(markers)
raise
def execute(self):
if self.has_pk and not has_concrete_parents(self.model):
results = []
# We are inserting, but we specified an ID, we need to check for existence before we Put()
# We do it in a loop so each check/put is transactional - because it's an ancestor query it shouldn't
# cost any entity groups
was_in_transaction = datastore.IsInTransaction()
for key, ent in zip(self.included_keys, self.entities):
@db.transactional
def txn():
if key is not None:
if utils.key_exists(key):
raise IntegrityError("Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name, basestring) and id_or_name.startswith("__"):
raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name)
if not constraints.constraint_checks_enabled(self.model):
# Fast path, just insert
results.append(datastore.Put(ent))
else:
markers = constraints.acquire(self.model, ent)
try:
results.append(datastore.Put(ent))
if not was_in_transaction:
# We can cache if we weren't in a transaction before this little nested one
caching.add_entity_to_cache(self.model, ent, caching.CachingSituation.DATASTORE_GET_PUT)
except:
# Make sure we delete any created markers before we re-raise
constraints.release_markers(markers)
raise
# Make sure we notify app engine that we are using this ID
# FIXME: Copy ancestor across to the template key
reserve_id(key.kind(), key.id_or_name())
txn()
return results
else:
if not constraints.constraint_checks_enabled(self.model):
# Fast path, just bulk insert
results = datastore.Put(self.entities)
for entity in self.entities:
caching.add_entity_to_cache(self.model, entity, caching.CachingSituation.DATASTORE_PUT)
return results
else:
markers = []
try:
#FIXME: We should rearrange this so that each entity is handled individually like above. We'll
# lose insert performance, but gain consistency on errors which is more important
markers = constraints.acquire_bulk(self.model, self.entities)
results = datastore.Put(self.entities)
for entity in self.entities:
caching.add_entity_to_cache(self.model, entity, caching.CachingSituation.DATASTORE_PUT)
except:
to_delete = chain(*markers)
constraints.release_markers(to_delete)
raise
for ent, k, m in zip(self.entities, results, markers):
ent.__key = k
constraints.update_instance_on_markers(ent, m)
return results
def _update_entity(self, key):
caching.remove_entity_from_cache_by_key(key)
try:
result = datastore.Get(key)
except datastore_errors.EntityNotFoundError:
# Return false to indicate update failure
return False
if (isinstance(
self.select.gae_query,
(Query, UniqueQuery)) # ignore QueryByKeys and NoOpQuery
and
not utils.entity_matches_query(result, self.select.gae_query)):
# Due to eventual consistency they query may have returned an entity which no longer
# matches the query
return False
original = copy.deepcopy(result)
instance_kwargs = {
field.attname: value
for field, param, value in self.values
}
# Note: If you replace MockInstance with self.model, you'll find that some delete
# tests fail in the test app. This is because any unspecified fields would then call
# get_default (even though we aren't going to use them) which may run a query which
# fails inside this transaction. Given as we are just using MockInstance so that we can
# call django_instance_to_entity it on it with the subset of fields we pass in,
# what we have is fine.
instance = MockInstance(**instance_kwargs)
# We need to add to the class attribute, rather than replace it!
original_class = result.get(POLYMODEL_CLASS_ATTRIBUTE, [])
# Update the entity we read above with the new values
result.update(
django_instance_to_entity(
self.connection,
self.model,
[x[0]
for x in self.values], # Pass in the fields that were updated
True,
instance))
# Make sure we keep all classes in the inheritence tree!
if original_class:
if result[POLYMODEL_CLASS_ATTRIBUTE] is not None:
result[POLYMODEL_CLASS_ATTRIBUTE].extend(original_class)
# Make sure we don't add duplicates
else:
result[POLYMODEL_CLASS_ATTRIBUTE] = original_class
if POLYMODEL_CLASS_ATTRIBUTE in result:
result[POLYMODEL_CLASS_ATTRIBUTE] = list(
set(result[POLYMODEL_CLASS_ATTRIBUTE]))
if not constraints.constraint_checks_enabled(self.model):
# The fast path, no constraint checking
datastore.Put(result)
caching.add_entity_to_cache(self.model, result,
caching.CachingSituation.DATASTORE_PUT)
else:
to_acquire, to_release = constraints.get_markers_for_update(
self.model, original, result)
# Acquire first, because if that fails then we don't want to alter what's already there
constraints.acquire_identifiers(to_acquire, result.key())
try:
datastore.Put(result)
caching.add_entity_to_cache(
self.model, result, caching.CachingSituation.DATASTORE_PUT)
except:
constraints.release_identifiers(to_acquire)
raise
else:
# Now we release the ones we don't want anymore
constraints.release_identifiers(to_release)
# Return true to indicate update success
return True
def _update_entity(self, key):
caching.remove_entity_from_cache_by_key(key)
try:
result = datastore.Get(key)
except datastore_errors.EntityNotFoundError:
reserve_id(key.kind(), key.id_or_name())
result = datastore.Entity(key.kind(), id=key.id_or_name())
original = copy.deepcopy(result)
instance_kwargs = {
field.attname: value
for field, param, value in self.values
}
instance = MockInstance(**instance_kwargs)
for field, param, value in self.values:
column_value = get_prepared_db_value(self.connection,
instance,
field,
raw=True)
result[field.column] = column_value
# Add special indexed fields
for index in special_indexes_for_column(self.model, field.column):
indexer = REQUIRES_SPECIAL_INDEXES[index]
values = indexer.prep_value_for_database(column_value)
if values is None:
continue
if not hasattr(values, "__iter__"):
values = [values]
for value in values:
column = indexer.indexed_column_name(field.column, value)
if column in result:
if not isinstance(result[column], list):
result[column] = [result[column], value]
else:
result[column].append(value)
else:
result[column] = value
if not constraints.constraint_checks_enabled(self.model):
# The fast path, no constraint checking
datastore.Put(result)
caching.add_entity_to_cache(self.model, result,
caching.CachingSituation.DATASTORE_PUT)
else:
to_acquire, to_release = constraints.get_markers_for_update(
self.model, original, result)
# Acquire first, because if that fails then we don't want to alter what's already there
constraints.acquire_identifiers(to_acquire, result.key())
try:
datastore.Put(result)
caching.add_entity_to_cache(
self.model, result, caching.CachingSituation.DATASTORE_PUT)
except:
constraints.release_identifiers(to_acquire)
raise
else:
# Now we release the ones we don't want anymore
constraints.release_identifiers(to_release)
def execute(self):
"""
Ideally we'd just be able to tell appengine to delete all the entities
which match the query, that would be nice wouldn't it?
Except we can't. Firstly Delete() only accepts keys so we first have to
execute a keys_only query to find the entities that match the query, then send
those keys to Delete(), except it's not as easy as that either because the
query might be eventually consistent and so we might delete entities which
were updated in another request and no-longer match the query. Bugger.
And then there might be constraints... in which case we need to grab the entity
in its entirety, release any constraints and then delete the entity.
And then there are polymodels (model inheritence) which means we might not even be
deleting the entity after all, only deleting some of the fields from it.
What we do then is do a keys_only query, then iterate the entities in batches of
25 (well _MAX_EG_PER_TXN), each entity in the batch has its polymodel fields wiped out
(if necessary) and then we do either a PutAsync or DeleteAsync all inside a transaction.
Oh, and we wipe out memcache and delete the constraints in an independent transaction.
Things to improve:
- Delete the constraints in a background thread. We don't need to wait for them, and really,
we don't want the non-deletion of them to affect the deletion of the entity. Lingering markers
are handled automatically they just case a small performance hit on write.
- Check the entity matches the query still (there's a fixme there)
"""
self.select.execute()
constraints_enabled = constraints.constraint_checks_enabled(self.model)
keys = [x.key() for x in self.select.results]
def wipe_polymodel_from_entity(entity, db_table):
"""
Wipes out the fields associated with the specified polymodel table
"""
polymodel_value = entity.get('class', [])
if polymodel_value and self.table_to_delete in polymodel_value:
# Remove any local fields from this model from the entity
model = utils.get_model_from_db_table(self.table_to_delete)
for field in model._meta.local_fields:
col = field.column
if col in entity:
del entity[col]
# Then remove this model from the polymodel heirarchy
polymodel_value.remove(self.table_to_delete)
if polymodel_value:
entity['class'] = polymodel_value
@db.transactional(
xg=True, propagation=datastore_rpc.TransactionOptions.INDEPENDENT)
def release_constraints(entity):
if constraints_enabled:
constraints.release(self.model, entity)
@db.transactional(xg=True)
def delete_batch(key_slice):
entities = datastore.Get(key_slice)
#FIXME: We need to make sure the entity still matches the query!
# entities = (x for x in entities if utils.entity_matches_query(x, self.select.gae_query))
to_delete = []
to_update = []
updated_keys = []
# Go through the entities
for entity in entities:
wipe_polymodel_from_entity(entity, self.table_to_delete)
if not entity.get('class'):
to_delete.append(entity)
release_constraints(entity)
else:
to_update.append(entity)
updated_keys.append(entity.key())
datastore.DeleteAsync([x.key() for x in to_delete])
datastore.PutAsync(to_update)
caching.remove_entities_from_cache_by_key(updated_keys,
self.namespace)
return len(updated_keys)
deleted = 0
while keys:
deleted += delete_batch(keys[:datastore_stub_util._MAX_EG_PER_TXN])
keys = keys[datastore_stub_util._MAX_EG_PER_TXN:]
return deleted
def execute(self):
if self.has_pk and not has_concrete_parents(self.model):
results = []
# We are inserting, but we specified an ID, we need to check for existence before we Put()
# We do it in a loop so each check/put is transactional - because it's an ancestor query it shouldn't
# cost any entity groups
for key, ent in zip(self.included_keys, self.entities):
@db.transactional
def txn():
if key is not None:
if utils.key_exists(key):
raise IntegrityError("Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name, basestring) and id_or_name.startswith("__"):
raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name)
if not constraints.constraint_checks_enabled(self.model):
# Fast path, just insert
results.append(datastore.Put(ent))
else:
markers = constraints.acquire(self.model, ent)
try:
results.append(datastore.Put(ent))
caching.add_entity_to_context_cache(self.model, ent)
except:
# Make sure we delete any created markers before we re-raise
constraints.release_markers(markers)
raise
# Make sure we notify app engine that we are using this ID
# FIXME: Copy ancestor across to the template key
id_or_name = key.id_or_name()
if isinstance(id_or_name, (int, long)):
try:
db.allocate_id_range(datastore.Key.from_path(key.kind(), 1), id_or_name, id_or_name)
except:
# We don't re-raise because it's not terminal, but if this happens we need to know why
logging.exception("An error occurred when notifying app engine that an ID has been used. Please report.")
txn()
return results
else:
if not constraints.constraint_checks_enabled(self.model):
# Fast path, just bulk insert
results = datastore.Put(self.entities)
for entity in self.entities:
caching.add_entity_to_context_cache(self.model, entity)
return results
else:
markers = []
try:
#FIXME: We should rearrange this so that each entity is handled individually like above. We'll
# lose insert performance, but gain consistency on errors which is more important
markers = constraints.acquire_bulk(self.model, self.entities)
results = datastore.Put(self.entities)
for entity in self.entities:
caching.add_entity_to_context_cache(self.model, entity)
except:
to_delete = chain(*markers)
constraints.release_markers(to_delete)
raise
for ent, m in zip(self.entities, markers):
constraints.update_instance_on_markers(ent, m)
return results
