def txn(): if key is not None: if utils.key_exists(key): raise IntegrityError( "Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, basestring) and id_or_name.startswith("__"): raise NotSupportedError( "Datastore ids cannot start with __. Id was %s" % id_or_name) if not constraints.constraint_checks_enabled(self.model): # Fast path, just insert results.append(datastore.Put(ent)) else: markers = constraints.acquire(self.model, ent) try: results.append(datastore.Put(ent)) if not was_in_transaction: # We can cache if we weren't in a transaction before this little nested one caching.add_entity_to_cache( self.model, ent, caching.CachingSituation.DATASTORE_GET_PUT) except: # Make sure we delete any created markers before we re-raise constraints.release_markers(markers) raise
def execute(self): self.select.execute() # This is a little bit more inefficient than just doing a keys_only query and # sending it to delete, but I think this is the sacrifice to make for the unique caching layer keys = [] def spawn_query(kind, key): qry = Query(kind) qry["__key__ ="] = key return qry queries = [spawn_query(self.select.db_table, x.key()) for x in self.select.results] if not queries: return for entity in QueryByKeys(self.select.model, queries, []).Run(): keys.append(entity.key()) # Delete constraints if that's enabled if constraints.constraint_checks_enabled(self.select.model): constraints.release(self.select.model, entity) caching.remove_entity_from_cache_by_key(entity.key()) datastore.Delete(keys)
def txn(): if key is not None: if utils.key_exists(key): raise IntegrityError("Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, basestring) and id_or_name.startswith("__"): raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name) if not constraints.constraint_checks_enabled(self.model): # Fast path, just insert results.append(datastore.Put(ent)) else: markers = constraints.acquire(self.model, ent) try: results.append(datastore.Put(ent)) if not was_in_transaction: # We can cache if we weren't in a transaction before this little nested one caching.add_entities_to_cache( self.model, [ent], caching.CachingSituation.DATASTORE_GET_PUT, self.namespace, ) except: # Make sure we delete any created markers before we re-raise constraints.release_markers(markers) raise
def execute(self): self.select.execute() # This is a little bit more inefficient than just doing a keys_only query and # sending it to delete, but I think this is the sacrifice to make for the unique caching layer keys = [] def spawn_query(kind, key): qry = Query( kind, namespace=key.namespace() or None ) # TODO: is the namespace necessary if we're passing the key? qry["__key__ ="] = key return qry queries = [ spawn_query(x.key().kind(), x.key()) for x in self.select.results ] if not queries: return for entity in QueryByKeys(self.model, queries, [], self.namespace).Run(): keys.append(entity.key()) # Delete constraints if that's enabled if constraints.constraint_checks_enabled(self.model): constraints.release(self.model, entity) caching.remove_entities_from_cache_by_key(keys, self.namespace) datastore.Delete(keys)
def execute(self): self.select.execute() # This is a little bit more inefficient than just doing a keys_only query and # sending it to delete, but I think this is the sacrifice to make for the unique caching layer keys = [] def spawn_query(kind, key): qry = Query(kind, namespace=key.namespace() or None) # TODO: is the namespace necessary if we're passing the key? qry["__key__ ="] = key return qry queries = [spawn_query(x.key().kind(), x.key()) for x in self.select.results] if not queries: return for entity in QueryByKeys(self.model, queries, [], self.namespace).Run(): keys.append(entity.key()) # Delete constraints if that's enabled if constraints.constraint_checks_enabled(self.model): constraints.release(self.model, entity) caching.remove_entities_from_cache_by_key(keys, self.namespace) datastore.Delete(keys)
def _update_entity(self, key): caching.remove_entity_from_context_cache_by_key(key) result = datastore.Get(key) original = copy.deepcopy(result) instance_kwargs = {field.attname:value for field, param, value in self.values} instance = MockInstance(**instance_kwargs) for field, param, value in self.values: result[field.column] = get_prepared_db_value(self.connection, instance, field, raw=True) #Add special indexed fields for index in special_indexes_for_column(self.model, field.column): indexer = REQUIRES_SPECIAL_INDEXES[index] result[indexer.indexed_column_name(field.column)] = indexer.prep_value_for_database(value) if not constraints.constraint_checks_enabled(self.model): #The fast path, no constraint checking datastore.Put(result) caching.add_entity_to_context_cache(self.model, result) else: to_acquire, to_release = constraints.get_markers_for_update(self.model, original, result) #Acquire first, because if that fails then we don't want to alter what's already there constraints.acquire_identifiers(to_acquire, result.key()) try: datastore.Put(result) caching.add_entity_to_context_cache(self.model, result) except: constraints.release_identifiers(to_acquire) raise else: #Now we release the ones we don't want anymore constraints.release_identifiers(to_release)
def _update_entity(self, key): caching.remove_entity_from_cache_by_key(key) try: result = datastore.Get(key) except datastore_errors.EntityNotFoundError: # Return false to indicate update failure return False original = copy.deepcopy(result) instance_kwargs = { field.attname: value for field, param, value in self.values } # Note: If you replace MockInstance with self.model, you'll find that some delete # tests fail in the test app. This is because any unspecified fields would then call # get_default (even though we aren't going to use them) which may run a query which # fails inside this transaction. Given as we are just using MockInstance so that we can # call django_instance_to_entity it on it with the subset of fields we pass in, # what we have is fine. instance = MockInstance(**instance_kwargs) # Update the entity we read above with the new values result.update( django_instance_to_entity( self.connection, self.model, [x[0] for x in self.values], # Pass in the fields that were updated True, instance)) if not constraints.constraint_checks_enabled(self.model): # The fast path, no constraint checking datastore.Put(result) caching.add_entity_to_cache(self.model, result, caching.CachingSituation.DATASTORE_PUT) else: to_acquire, to_release = constraints.get_markers_for_update( self.model, original, result) # Acquire first, because if that fails then we don't want to alter what's already there constraints.acquire_identifiers(to_acquire, result.key()) try: datastore.Put(result) caching.add_entity_to_cache( self.model, result, caching.CachingSituation.DATASTORE_PUT) except: constraints.release_identifiers(to_acquire) raise else: # Now we release the ones we don't want anymore constraints.release_identifiers(to_release) # Return true to indicate update success return True
def _update_entity(self, key): caching.remove_entity_from_cache_by_key(key) try: result = datastore.Get(key) except datastore_errors.EntityNotFoundError: # Return false to indicate update failure return False if ( isinstance(self.select.gae_query, (Query, UniqueQuery)) # ignore QueryByKeys and NoOpQuery and not utils.entity_matches_query(result, self.select.gae_query) ): # Due to eventual consistency they query may have returned an entity which no longer # matches the query return False original = copy.deepcopy(result) instance_kwargs = {field.attname:value for field, param, value in self.values} # Note: If you replace MockInstance with self.model, you'll find that some delete # tests fail in the test app. This is because any unspecified fields would then call # get_default (even though we aren't going to use them) which may run a query which # fails inside this transaction. Given as we are just using MockInstance so that we can # call django_instance_to_entity it on it with the subset of fields we pass in, # what we have is fine. instance = MockInstance(**instance_kwargs) # Update the entity we read above with the new values result.update(django_instance_to_entity( self.connection, self.model, [ x[0] for x in self.values], # Pass in the fields that were updated True, instance) ) if not constraints.constraint_checks_enabled(self.model): # The fast path, no constraint checking datastore.Put(result) caching.add_entity_to_cache(self.model, result, caching.CachingSituation.DATASTORE_PUT) else: to_acquire, to_release = constraints.get_markers_for_update(self.model, original, result) # Acquire first, because if that fails then we don't want to alter what's already there constraints.acquire_identifiers(to_acquire, result.key()) try: datastore.Put(result) caching.add_entity_to_cache(self.model, result, caching.CachingSituation.DATASTORE_PUT) except: constraints.release_identifiers(to_acquire) raise else: # Now we release the ones we don't want anymore constraints.release_identifiers(to_release) # Return true to indicate update success return True
def execute(self): self.select.execute() #This is a little bit more inefficient than just doing a keys_only query and #sending it to delete, but I think this is the sacrifice to make for the unique caching layer keys = [] for entity in QueryByKeys( Query(self.select.model._meta.db_table), [ x.key() for x in self.select.results ], [] ).Run(): keys.append(entity.key()) ##Delete constraints if that's enabled if constraints.constraint_checks_enabled(self.select.model): constraints.release(self.select.model, entity) caching.remove_entity_from_context_cache_by_key(entity.key()) datastore.Delete(keys)
def txn(): if key is not None: if utils.key_exists(key): raise IntegrityError("Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, basestring) and id_or_name.startswith("__"): raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name) if not constraints.constraint_checks_enabled(self.model): #Fast path, just insert results.append(datastore.Put(ent)) else: markers = constraints.acquire(self.model, ent) try: results.append(datastore.Put(ent)) caching.add_entity_to_context_cache(self.model, ent) except: #Make sure we delete any created markers before we re-raise constraints.release_markers(markers) raise
def execute(self): if self.has_pk and not has_concrete_parents(self.model): results = [] # We are inserting, but we specified an ID, we need to check for existence before we Put() # We do it in a loop so each check/put is transactional - because it's an ancestor query it shouldn't # cost any entity groups was_in_transaction = datastore.IsInTransaction() for key, ent in zip(self.included_keys, self.entities): @db.transactional def txn(): if key is not None: if utils.key_exists(key): raise IntegrityError("Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, basestring) and id_or_name.startswith("__"): raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name) if not constraints.constraint_checks_enabled(self.model): # Fast path, just insert results.append(datastore.Put(ent)) else: markers = constraints.acquire(self.model, ent) try: results.append(datastore.Put(ent)) if not was_in_transaction: # We can cache if we weren't in a transaction before this little nested one caching.add_entity_to_cache(self.model, ent, caching.CachingSituation.DATASTORE_GET_PUT) except: # Make sure we delete any created markers before we re-raise constraints.release_markers(markers) raise # Make sure we notify app engine that we are using this ID # FIXME: Copy ancestor across to the template key reserve_id(key.kind(), key.id_or_name()) txn() return results else: if not constraints.constraint_checks_enabled(self.model): # Fast path, just bulk insert results = datastore.Put(self.entities) for entity in self.entities: caching.add_entity_to_cache(self.model, entity, caching.CachingSituation.DATASTORE_PUT) return results else: markers = [] try: #FIXME: We should rearrange this so that each entity is handled individually like above. We'll # lose insert performance, but gain consistency on errors which is more important markers = constraints.acquire_bulk(self.model, self.entities) results = datastore.Put(self.entities) for entity in self.entities: caching.add_entity_to_cache(self.model, entity, caching.CachingSituation.DATASTORE_PUT) except: to_delete = chain(*markers) constraints.release_markers(to_delete) raise for ent, k, m in zip(self.entities, results, markers): ent.__key = k constraints.update_instance_on_markers(ent, m) return results
def _update_entity(self, key): caching.remove_entity_from_cache_by_key(key) try: result = datastore.Get(key) except datastore_errors.EntityNotFoundError: # Return false to indicate update failure return False if (isinstance( self.select.gae_query, (Query, UniqueQuery)) # ignore QueryByKeys and NoOpQuery and not utils.entity_matches_query(result, self.select.gae_query)): # Due to eventual consistency they query may have returned an entity which no longer # matches the query return False original = copy.deepcopy(result) instance_kwargs = { field.attname: value for field, param, value in self.values } # Note: If you replace MockInstance with self.model, you'll find that some delete # tests fail in the test app. This is because any unspecified fields would then call # get_default (even though we aren't going to use them) which may run a query which # fails inside this transaction. Given as we are just using MockInstance so that we can # call django_instance_to_entity it on it with the subset of fields we pass in, # what we have is fine. instance = MockInstance(**instance_kwargs) # We need to add to the class attribute, rather than replace it! original_class = result.get(POLYMODEL_CLASS_ATTRIBUTE, []) # Update the entity we read above with the new values result.update( django_instance_to_entity( self.connection, self.model, [x[0] for x in self.values], # Pass in the fields that were updated True, instance)) # Make sure we keep all classes in the inheritence tree! if original_class: if result[POLYMODEL_CLASS_ATTRIBUTE] is not None: result[POLYMODEL_CLASS_ATTRIBUTE].extend(original_class) # Make sure we don't add duplicates else: result[POLYMODEL_CLASS_ATTRIBUTE] = original_class if POLYMODEL_CLASS_ATTRIBUTE in result: result[POLYMODEL_CLASS_ATTRIBUTE] = list( set(result[POLYMODEL_CLASS_ATTRIBUTE])) if not constraints.constraint_checks_enabled(self.model): # The fast path, no constraint checking datastore.Put(result) caching.add_entity_to_cache(self.model, result, caching.CachingSituation.DATASTORE_PUT) else: to_acquire, to_release = constraints.get_markers_for_update( self.model, original, result) # Acquire first, because if that fails then we don't want to alter what's already there constraints.acquire_identifiers(to_acquire, result.key()) try: datastore.Put(result) caching.add_entity_to_cache( self.model, result, caching.CachingSituation.DATASTORE_PUT) except: constraints.release_identifiers(to_acquire) raise else: # Now we release the ones we don't want anymore constraints.release_identifiers(to_release) # Return true to indicate update success return True
def _update_entity(self, key): caching.remove_entity_from_cache_by_key(key) try: result = datastore.Get(key) except datastore_errors.EntityNotFoundError: reserve_id(key.kind(), key.id_or_name()) result = datastore.Entity(key.kind(), id=key.id_or_name()) original = copy.deepcopy(result) instance_kwargs = { field.attname: value for field, param, value in self.values } instance = MockInstance(**instance_kwargs) for field, param, value in self.values: column_value = get_prepared_db_value(self.connection, instance, field, raw=True) result[field.column] = column_value # Add special indexed fields for index in special_indexes_for_column(self.model, field.column): indexer = REQUIRES_SPECIAL_INDEXES[index] values = indexer.prep_value_for_database(column_value) if values is None: continue if not hasattr(values, "__iter__"): values = [values] for value in values: column = indexer.indexed_column_name(field.column, value) if column in result: if not isinstance(result[column], list): result[column] = [result[column], value] else: result[column].append(value) else: result[column] = value if not constraints.constraint_checks_enabled(self.model): # The fast path, no constraint checking datastore.Put(result) caching.add_entity_to_cache(self.model, result, caching.CachingSituation.DATASTORE_PUT) else: to_acquire, to_release = constraints.get_markers_for_update( self.model, original, result) # Acquire first, because if that fails then we don't want to alter what's already there constraints.acquire_identifiers(to_acquire, result.key()) try: datastore.Put(result) caching.add_entity_to_cache( self.model, result, caching.CachingSituation.DATASTORE_PUT) except: constraints.release_identifiers(to_acquire) raise else: # Now we release the ones we don't want anymore constraints.release_identifiers(to_release)
def execute(self): """ Ideally we'd just be able to tell appengine to delete all the entities which match the query, that would be nice wouldn't it? Except we can't. Firstly Delete() only accepts keys so we first have to execute a keys_only query to find the entities that match the query, then send those keys to Delete(), except it's not as easy as that either because the query might be eventually consistent and so we might delete entities which were updated in another request and no-longer match the query. Bugger. And then there might be constraints... in which case we need to grab the entity in its entirety, release any constraints and then delete the entity. And then there are polymodels (model inheritence) which means we might not even be deleting the entity after all, only deleting some of the fields from it. What we do then is do a keys_only query, then iterate the entities in batches of 25 (well _MAX_EG_PER_TXN), each entity in the batch has its polymodel fields wiped out (if necessary) and then we do either a PutAsync or DeleteAsync all inside a transaction. Oh, and we wipe out memcache and delete the constraints in an independent transaction. Things to improve: - Delete the constraints in a background thread. We don't need to wait for them, and really, we don't want the non-deletion of them to affect the deletion of the entity. Lingering markers are handled automatically they just case a small performance hit on write. - Check the entity matches the query still (there's a fixme there) """ self.select.execute() constraints_enabled = constraints.constraint_checks_enabled(self.model) keys = [x.key() for x in self.select.results] def wipe_polymodel_from_entity(entity, db_table): """ Wipes out the fields associated with the specified polymodel table """ polymodel_value = entity.get('class', []) if polymodel_value and self.table_to_delete in polymodel_value: # Remove any local fields from this model from the entity model = utils.get_model_from_db_table(self.table_to_delete) for field in model._meta.local_fields: col = field.column if col in entity: del entity[col] # Then remove this model from the polymodel heirarchy polymodel_value.remove(self.table_to_delete) if polymodel_value: entity['class'] = polymodel_value @db.transactional( xg=True, propagation=datastore_rpc.TransactionOptions.INDEPENDENT) def release_constraints(entity): if constraints_enabled: constraints.release(self.model, entity) @db.transactional(xg=True) def delete_batch(key_slice): entities = datastore.Get(key_slice) #FIXME: We need to make sure the entity still matches the query! # entities = (x for x in entities if utils.entity_matches_query(x, self.select.gae_query)) to_delete = [] to_update = [] updated_keys = [] # Go through the entities for entity in entities: wipe_polymodel_from_entity(entity, self.table_to_delete) if not entity.get('class'): to_delete.append(entity) release_constraints(entity) else: to_update.append(entity) updated_keys.append(entity.key()) datastore.DeleteAsync([x.key() for x in to_delete]) datastore.PutAsync(to_update) caching.remove_entities_from_cache_by_key(updated_keys, self.namespace) return len(updated_keys) deleted = 0 while keys: deleted += delete_batch(keys[:datastore_stub_util._MAX_EG_PER_TXN]) keys = keys[datastore_stub_util._MAX_EG_PER_TXN:] return deleted
def execute(self): if self.has_pk and not has_concrete_parents(self.model): results = [] # We are inserting, but we specified an ID, we need to check for existence before we Put() # We do it in a loop so each check/put is transactional - because it's an ancestor query it shouldn't # cost any entity groups for key, ent in zip(self.included_keys, self.entities): @db.transactional def txn(): if key is not None: if utils.key_exists(key): raise IntegrityError("Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, basestring) and id_or_name.startswith("__"): raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name) if not constraints.constraint_checks_enabled(self.model): # Fast path, just insert results.append(datastore.Put(ent)) else: markers = constraints.acquire(self.model, ent) try: results.append(datastore.Put(ent)) caching.add_entity_to_context_cache(self.model, ent) except: # Make sure we delete any created markers before we re-raise constraints.release_markers(markers) raise # Make sure we notify app engine that we are using this ID # FIXME: Copy ancestor across to the template key id_or_name = key.id_or_name() if isinstance(id_or_name, (int, long)): try: db.allocate_id_range(datastore.Key.from_path(key.kind(), 1), id_or_name, id_or_name) except: # We don't re-raise because it's not terminal, but if this happens we need to know why logging.exception("An error occurred when notifying app engine that an ID has been used. Please report.") txn() return results else: if not constraints.constraint_checks_enabled(self.model): # Fast path, just bulk insert results = datastore.Put(self.entities) for entity in self.entities: caching.add_entity_to_context_cache(self.model, entity) return results else: markers = [] try: #FIXME: We should rearrange this so that each entity is handled individually like above. We'll # lose insert performance, but gain consistency on errors which is more important markers = constraints.acquire_bulk(self.model, self.entities) results = datastore.Put(self.entities) for entity in self.entities: caching.add_entity_to_context_cache(self.model, entity) except: to_delete = chain(*markers) constraints.release_markers(to_delete) raise for ent, m in zip(self.entities, markers): constraints.update_instance_on_markers(ent, m) return results