def Run(self, limit, offset): opts = self._gae_query._Query__query_options if opts.keys_only or opts.projection: return self._gae_query.Run(limit=limit, offset=offset) ret = caching.get_from_cache(self._identifier, self._namespace) if ret is not None and not utils.entity_matches_query( ret, self._gae_query): ret = None if ret is None: # We do a fast keys_only query to get the result keys_query = rpc.Query(self._gae_query._Query__kind, keys_only=True, namespace=self._namespace) keys_query.update(self._gae_query) keys = keys_query.Run(limit=limit, offset=offset) # Do a consistent get so we don't cache stale data, and recheck the result matches the query ret = [ x for x in rpc.Get(keys) if x and utils.entity_matches_query(x, self._gae_query) ] if len(ret) == 1: caching.add_entities_to_cache( self._model, [ret[0]], caching.CachingSituation.DATASTORE_GET, self._namespace, ) return iter(ret) return iter([ret])
def iter_results(results): returned = 0 # This is safe, because Django is fetching all results any way :( sorted_results = sorted(results, cmp=partial(utils.django_ordering_comparison, self.ordering)) sorted_results = [result for result in sorted_results if result is not None] if cache and sorted_results: caching.add_entities_to_cache(self.model, sorted_results, caching.CachingSituation.DATASTORE_GET) for result in sorted_results: if not any([ utils.entity_matches_query(result, qry) for qry in self.queries_by_key[result.key()]]): continue if offset and returned < offset: # Skip entities based on offset returned += 1 continue else: yield _convert_entity_based_on_query_options(result, opts) returned += 1 # If there is a limit, we might be done! if limit is not None and returned == (offset or 0) + limit: break
def txn(): for key in keys: if check_existence and key is not None: if utils.key_exists(key): raise IntegrityError("Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, basestring) and id_or_name.startswith("__"): raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name) # Notify App Engine of any keys we're specifying intentionally reserve_id(key.kind(), key.id_or_name(), self.namespace) results = datastore.Put(entities) for entity in entities: markers.extend(constraints.acquire(self.model, entity)) caching.add_entities_to_cache( self.model, entities, caching.CachingSituation.DATASTORE_GET_PUT, self.namespace, skip_memcache=True ) return results
def txn(): for key in keys: if check_existence and key is not None: if utils.key_exists(key): raise IntegrityError("Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, six.string_types) and id_or_name.startswith("__"): raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name) # Notify App Engine of any keys we're specifying intentionally reserve_id(key.kind(), key.id_or_name(), self.namespace) results = perform_insert(entities) for entity, _ in entities: markers.extend(constraints.acquire(self.model, entity)) caching.add_entities_to_cache( self.model, [x[0] for x in entities], caching.CachingSituation.DATASTORE_GET_PUT, self.namespace, skip_memcache=True ) return results
def txn(): if key is not None: if utils.key_exists(key): raise IntegrityError( "Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, basestring) and id_or_name.startswith("__"): raise NotSupportedError( "Datastore ids cannot start with __. Id was %s" % id_or_name) if not constraints.constraint_checks_enabled(self.model): # Fast path, just insert results.append(datastore.Put(ent)) else: markers = constraints.acquire(self.model, ent) try: results.append(datastore.Put(ent)) if not was_in_transaction: # We can cache if we weren't in a transaction before this little nested one caching.add_entities_to_cache( self.model, [ent], caching.CachingSituation.DATASTORE_GET_PUT) except: # Make sure we delete any created markers before we re-raise constraints.release_markers(markers) raise
def txn(): if key is not None: if utils.key_exists(key): raise IntegrityError("Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, basestring) and id_or_name.startswith("__"): raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name) if not constraints.constraint_checks_enabled(self.model): # Fast path, just insert results.append(datastore.Put(ent)) else: markers = constraints.acquire(self.model, ent) try: results.append(datastore.Put(ent)) if not was_in_transaction: # We can cache if we weren't in a transaction before this little nested one caching.add_entities_to_cache( self.model, [ent], caching.CachingSituation.DATASTORE_GET_PUT, self.namespace, ) except: # Make sure we delete any created markers before we re-raise constraints.release_markers(markers) raise
def Run(self, limit, offset): opts = self._gae_query._Query__query_options if opts.keys_only or opts.projection: return self._gae_query.Run(limit=limit, offset=offset) ret = caching.get_from_cache(self._identifier, self._namespace) if ret is not None and not utils.entity_matches_query(ret, self._gae_query): ret = None if ret is None: # We do a fast keys_only query to get the result keys_query = Query(self._gae_query._Query__kind, keys_only=True, namespace=self._namespace) keys_query.update(self._gae_query) keys = keys_query.Run(limit=limit, offset=offset) # Do a consistent get so we don't cache stale data, and recheck the result matches the query ret = [x for x in datastore.Get(keys) if x and utils.entity_matches_query(x, self._gae_query)] if len(ret) == 1: caching.add_entities_to_cache( self._model, [ret[0]], caching.CachingSituation.DATASTORE_GET, self._namespace, ) return iter(ret) return iter([ret])
def iter_results(results): returned = 0 # This is safe, because Django is fetching all results any way :( sorted_results = sorted(results, cmp=partial( utils.django_ordering_comparison, self.ordering)) sorted_results = [ result for result in sorted_results if result is not None ] if cache_results and sorted_results: caching.add_entities_to_cache( self.model, sorted_results[:max_cache_count], caching.CachingSituation.DATASTORE_GET, self.namespace, ) for result in sorted_results: if is_projection: entity_matches_query = True else: entity_matches_query = any( utils.entity_matches_query(result, qry) for qry in self.queries_by_key[result.key()]) if not entity_matches_query: continue if offset and returned < offset: # Skip entities based on offset returned += 1 continue else: yield _convert_entity_based_on_query_options(result, opts) returned += 1 # If there is a limit, we might be done! if limit is not None and returned == (offset or 0) + limit: break
def execute(self): if self.has_pk and not has_concrete_parents(self.model): results = [] # We are inserting, but we specified an ID, we need to check for existence before we Put() # We do it in a loop so each check/put is transactional - because it's an ancestor query it shouldn't # cost any entity groups was_in_transaction = datastore.IsInTransaction() for key, ent in zip(self.included_keys, self.entities): @db.transactional def txn(): if key is not None: if utils.key_exists(key): raise IntegrityError("Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, basestring) and id_or_name.startswith("__"): raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name) if not constraints.constraint_checks_enabled(self.model): # Fast path, just insert results.append(datastore.Put(ent)) else: markers = constraints.acquire(self.model, ent) try: results.append(datastore.Put(ent)) if not was_in_transaction: # We can cache if we weren't in a transaction before this little nested one caching.add_entities_to_cache( self.model, [ent], caching.CachingSituation.DATASTORE_GET_PUT, self.namespace, ) except: # Make sure we delete any created markers before we re-raise constraints.release_markers(markers) raise # Make sure we notify app engine that we are using this ID # FIXME: Copy ancestor across to the template key reserve_id(key.kind(), key.id_or_name(), self.namespace) txn() return results else: if not constraints.constraint_checks_enabled(self.model): # Fast path, just bulk insert results = datastore.Put(self.entities) caching.add_entities_to_cache( self.model, self.entities, caching.CachingSituation.DATASTORE_PUT, self.namespace ) return results else: markers = [] try: #FIXME: We should rearrange this so that each entity is handled individually like above. We'll # lose insert performance, but gain consistency on errors which is more important markers = constraints.acquire_bulk(self.model, self.entities) results = datastore.Put(self.entities) caching.add_entities_to_cache( self.model, self.entities, caching.CachingSituation.DATASTORE_PUT, self.namespace, ) except: to_delete = chain(*markers) constraints.release_markers(to_delete) raise for ent, k, m in zip(self.entities, results, markers): ent.__key = k constraints.update_instance_on_markers(ent, m) return results
def _update_entity(self, key): caching.remove_entities_from_cache_by_key([key], self.namespace) try: result = datastore.Get(key) except datastore_errors.EntityNotFoundError: # Return false to indicate update failure return False if ( isinstance(self.select.gae_query, (Query, UniqueQuery)) # ignore QueryByKeys and NoOpQuery and not utils.entity_matches_query(result, self.select.gae_query) ): # Due to eventual consistency they query may have returned an entity which no longer # matches the query return False original = copy.deepcopy(result) instance_kwargs = {field.attname:value for field, param, value in self.values} # Note: If you replace MockInstance with self.model, you'll find that some delete # tests fail in the test app. This is because any unspecified fields would then call # get_default (even though we aren't going to use them) which may run a query which # fails inside this transaction. Given as we are just using MockInstance so that we can # call django_instance_to_entity it on it with the subset of fields we pass in, # what we have is fine. meta = self.model._meta instance = MockInstance( _original=MockInstance(_meta=meta, **result), _meta=meta, **instance_kwargs ) # We need to add to the class attribute, rather than replace it! original_class = result.get(POLYMODEL_CLASS_ATTRIBUTE, []) # Update the entity we read above with the new values result.update(django_instance_to_entity( self.connection, self.model, [ x[0] for x in self.values], # Pass in the fields that were updated True, instance) ) # Make sure we keep all classes in the inheritence tree! if original_class: if result[POLYMODEL_CLASS_ATTRIBUTE] is not None: result[POLYMODEL_CLASS_ATTRIBUTE].extend(original_class) # Make sure we don't add duplicates else: result[POLYMODEL_CLASS_ATTRIBUTE] = original_class if POLYMODEL_CLASS_ATTRIBUTE in result: result[POLYMODEL_CLASS_ATTRIBUTE] = list(set(result[POLYMODEL_CLASS_ATTRIBUTE])) if not constraints.constraint_checks_enabled(self.model): # The fast path, no constraint checking datastore.Put(result) caching.add_entities_to_cache( self.model, [result], caching.CachingSituation.DATASTORE_PUT, self.namespace, ) else: to_acquire, to_release = constraints.get_markers_for_update(self.model, original, result) # Acquire first, because if that fails then we don't want to alter what's already there constraints.acquire_identifiers(to_acquire, result.key()) try: datastore.Put(result) caching.add_entities_to_cache( self.model, [result], caching.CachingSituation.DATASTORE_PUT, self.namespace, ) except: constraints.release_identifiers(to_acquire, namespace=self.namespace) raise else: # Now we release the ones we don't want anymore constraints.release_identifiers(to_release, self.namespace) # Return true to indicate update success return True
def execute(self): if self.has_pk and not has_concrete_parents(self.model): results = [] # We are inserting, but we specified an ID, we need to check for existence before we Put() # We do it in a loop so each check/put is transactional - because it's an ancestor query it shouldn't # cost any entity groups was_in_transaction = datastore.IsInTransaction() for key, ent in zip(self.included_keys, self.entities): @db.transactional def txn(): if key is not None: if utils.key_exists(key): raise IntegrityError( "Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, basestring) and id_or_name.startswith("__"): raise NotSupportedError( "Datastore ids cannot start with __. Id was %s" % id_or_name) if not constraints.constraint_checks_enabled(self.model): # Fast path, just insert results.append(datastore.Put(ent)) else: markers = constraints.acquire(self.model, ent) try: results.append(datastore.Put(ent)) if not was_in_transaction: # We can cache if we weren't in a transaction before this little nested one caching.add_entities_to_cache( self.model, [ent], caching.CachingSituation.DATASTORE_GET_PUT) except: # Make sure we delete any created markers before we re-raise constraints.release_markers(markers) raise # Make sure we notify app engine that we are using this ID # FIXME: Copy ancestor across to the template key reserve_id(key.kind(), key.id_or_name()) txn() return results else: if not constraints.constraint_checks_enabled(self.model): # Fast path, just bulk insert results = datastore.Put(self.entities) caching.add_entities_to_cache( self.model, self.entities, caching.CachingSituation.DATASTORE_PUT) return results else: markers = [] try: #FIXME: We should rearrange this so that each entity is handled individually like above. We'll # lose insert performance, but gain consistency on errors which is more important markers = constraints.acquire_bulk(self.model, self.entities) results = datastore.Put(self.entities) caching.add_entities_to_cache( self.model, self.entities, caching.CachingSituation.DATASTORE_PUT) except: to_delete = chain(*markers) constraints.release_markers(to_delete) raise for ent, k, m in zip(self.entities, results, markers): ent.__key = k constraints.update_instance_on_markers(ent, m) return results
def txn(): caching.remove_entities_from_cache_by_key([key], self.namespace) try: result = datastore.Get(key) except datastore_errors.EntityNotFoundError: # Return false to indicate update failure return False if ( isinstance(self.select.gae_query, (Query, UniqueQuery)) # ignore QueryByKeys and NoOpQuery and not utils.entity_matches_query(result, self.select.gae_query) ): # Due to eventual consistency they query may have returned an entity which no longer # matches the query return False original = copy.deepcopy(result) instance_kwargs = {field.attname:value for field, param, value in self.values} # Note: If you replace MockInstance with self.model, you'll find that some delete # tests fail in the test app. This is because any unspecified fields would then call # get_default (even though we aren't going to use them) which may run a query which # fails inside this transaction. Given as we are just using MockInstance so that we can # call django_instance_to_entity it on it with the subset of fields we pass in, # what we have is fine. meta = self.model._meta instance = MockInstance( _original=MockInstance(_meta=meta, **result), _meta=meta, **instance_kwargs ) # We need to add to the class attribute, rather than replace it! original_class = result.get(POLYMODEL_CLASS_ATTRIBUTE, []) # Update the entity we read above with the new values result.update(django_instance_to_entity( self.connection, self.model, [ x[0] for x in self.values], # Pass in the fields that were updated True, instance) ) # Make sure we keep all classes in the inheritence tree! if original_class: if result[POLYMODEL_CLASS_ATTRIBUTE] is not None: result[POLYMODEL_CLASS_ATTRIBUTE].extend(original_class) # Make sure we don't add duplicates else: result[POLYMODEL_CLASS_ATTRIBUTE] = original_class if POLYMODEL_CLASS_ATTRIBUTE in result: result[POLYMODEL_CLASS_ATTRIBUTE] = list(set(result[POLYMODEL_CLASS_ATTRIBUTE])) if not constraints.has_active_unique_constraints(self.model): # The fast path, no constraint checking datastore.Put(result) caching.add_entities_to_cache( self.model, [result], caching.CachingSituation.DATASTORE_PUT, self.namespace, skip_memcache=True, ) else: markers_to_acquire[:], markers_to_release[:] = constraints.get_markers_for_update( self.model, original, result ) datastore.Put(result) constraints.update_identifiers(markers_to_acquire, markers_to_release, result.key()) # If the datastore.Put() fails then the exception will only be raised when the # transaction applies, which means that we will still get to here and will still have # applied the marker changes (because they're in a nested, independent transaction). # Hence we set this flag to tell us that we got this far and that we should roll them back. rollback_markers[0] = True # If something dies between here and the `return` statement then we'll have stale unique markers try: # Update the cache before dealing with unique markers, as CachingSituation.DATASTORE_PUT # will only update the context cache caching.add_entities_to_cache( self.model, [result], caching.CachingSituation.DATASTORE_PUT, self.namespace, skip_memcache=True, ) except: # We ignore the exception because raising will rollback the transaction causing # an inconsistent state logger.exception("Unable to update the context cache") pass # Return true to indicate update success return True
def txn(): caching.remove_entities_from_cache_by_key([key], self.namespace) try: result = rpc.Get(key) except datastore_errors.EntityNotFoundError: # Return false to indicate update failure return False if ( isinstance(self.select.gae_query, (Query, meta_queries.UniqueQuery)) # ignore QueryByKeys and NoOpQuery and not utils.entity_matches_query(result, self.select.gae_query) ): # Due to eventual consistency they query may have returned an entity which no longer # matches the query return False original = copy.deepcopy(result) instance_kwargs = {field.attname: value for field, param, value in self.values} # Note: If you replace MockInstance with self.model, you'll find that some delete # tests fail in the test app. This is because any unspecified fields would then call # get_default (even though we aren't going to use them) which may run a query which # fails inside this transaction. Given as we are just using MockInstance so that we can # call django_instance_to_entities it on it with the subset of fields we pass in, # what we have is fine. meta = self.model._meta instance = MockInstance( _original=MockInstance(_meta=meta, **result), _meta=meta, **instance_kwargs ) # Convert the instance to an entity primary, descendents = django_instance_to_entities( self.connection, [x[0] for x in self.values], # Pass in the fields that were updated True, instance, model=self.model ) # Update the entity we read above with the new values result.update(primary) # Remove fields which have been marked to be unindexed for col in getattr(primary, "_properties_to_remove", []): if col in result: del result[col] # Make sure that any polymodel classes which were in the original entity are kept, # as django_instance_to_entities may have wiped them as well as added them. polymodel_classes = list(set( original.get(POLYMODEL_CLASS_ATTRIBUTE, []) + result.get(POLYMODEL_CLASS_ATTRIBUTE, []) )) if polymodel_classes: result[POLYMODEL_CLASS_ATTRIBUTE] = polymodel_classes def perform_insert(): """ Inserts result, and any descendents with their ancestor value set """ inserted_key = rpc.Put(result) if descendents: for i, descendent in enumerate(descendents): descendents[i] = Entity( descendent.kind(), parent=inserted_key, namespace=inserted_key.namespace(), id=descendent.key().id() or None, name=descendent.key().name() or None ) descendents[i].update(descendent) rpc.Put(descendents) if not constraints.has_active_unique_constraints(self.model): # The fast path, no constraint checking perform_insert() caching.add_entities_to_cache( self.model, [result], caching.CachingSituation.DATASTORE_PUT, self.namespace, skip_memcache=True, ) else: markers_to_acquire[:], markers_to_release[:] = constraints.get_markers_for_update( self.model, original, result ) perform_insert() constraints.update_identifiers(markers_to_acquire, markers_to_release, result.key()) # If the rpc.Put() fails then the exception will only be raised when the # transaction applies, which means that we will still get to here and will still have # applied the marker changes (because they're in a nested, independent transaction). # Hence we set this flag to tell us that we got this far and that we should roll them back. rollback_markers[0] = True # If something dies between here and the `return` statement then we'll have stale unique markers try: # Update the cache before dealing with unique markers, as CachingSituation.DATASTORE_PUT # will only update the context cache caching.add_entities_to_cache( self.model, [result], caching.CachingSituation.DATASTORE_PUT, self.namespace, skip_memcache=True, ) except: # We ignore the exception because raising will rollback the transaction causing # an inconsistent state logger.exception("Unable to update the context cache") pass # Return true to indicate update success return True
def execute(self): check_existence = self.has_pk and not has_concrete_parents(self.model) def perform_insert(entities): results = [] for primary, descendents in entities: new_key = rpc.Put(primary) if descendents: for i, descendent in enumerate(descendents): descendents[i] = Entity( descendent.kind(), parent=new_key, namespace=new_key.namespace(), id=descendent.key().id() or None, name=descendent.key().name() or None ) descendents[i].update(descendent) rpc.Put(descendents) results.append(new_key) return results if not constraints.has_active_unique_constraints(self.model) and not check_existence: # Fast path, no constraint checks and no keys mean we can just do a normal rpc.Put # which isn't limited to 25 results = perform_insert(self.entities) # This modifies self.entities and sets their keys caching.add_entities_to_cache( self.model, [x[0] for x in self.entities], caching.CachingSituation.DATASTORE_GET_PUT, self.namespace, skip_memcache=True ) return results entity_group_count = len(self.entities) def insert_chunk(keys, entities): # Note that this is limited to a maximum of 25 entities. markers = [] @db.transactional(xg=entity_group_count > 1) def txn(): for key in keys: if check_existence and key is not None: if utils.key_exists(key): raise IntegrityError("Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, six.string_types) and id_or_name.startswith("__"): raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name) # Notify App Engine of any keys we're specifying intentionally reserve_id(key.kind(), key.id_or_name(), self.namespace) results = perform_insert(entities) for entity, _ in entities: markers.extend(constraints.acquire(self.model, entity)) caching.add_entities_to_cache( self.model, [x[0] for x in entities], caching.CachingSituation.DATASTORE_GET_PUT, self.namespace, skip_memcache=True ) return results try: return txn() except: # There are 3 possible reasons why we've ended up here: # 1. The rpc.Put() failed, but note that because it's a transaction, the # exception isn't raised until the END of the transaction block. # 2. Some of the markers were acquired, but then we hit a unique constraint # conflict and so the outer transaction was rolled back. # 3. Something else went wrong after we'd acquired markers, e.g. the # caching.add_entities_to_cache call got hit by a metaphorical bus. # In any of these cases, we (may) have acquired markers via (a) nested, independent # transaction(s), and so we need to release them again. constraints.release_markers(markers) raise # We can't really support this and maintain expected behaviour. If we chunked the insert and one of the # chunks fails it will mean some of the data would be saved and rather than trying to communicate that back # to the user it's better that they chunk the data themselves as they can deal with the failure better if entity_group_count > datastore_stub_util._MAX_EG_PER_TXN: raise BulkInsertError("Bulk inserts with unique constraints, or pre-defined keys are limited to {} instances on the datastore".format( datastore_stub_util._MAX_EG_PER_TXN )) return insert_chunk(self.included_keys, self.entities)
def txn(): caching.remove_entities_from_cache_by_key([key], self.namespace) try: result = datastore.Get(key) except datastore_errors.EntityNotFoundError: # Return false to indicate update failure return False if (isinstance( self.select.gae_query, (Query, UniqueQuery)) # ignore QueryByKeys and NoOpQuery and not utils.entity_matches_query(result, self.select.gae_query)): # Due to eventual consistency they query may have returned an entity which no longer # matches the query return False original = copy.deepcopy(result) instance_kwargs = { field.attname: value for field, param, value in self.values } # Note: If you replace MockInstance with self.model, you'll find that some delete # tests fail in the test app. This is because any unspecified fields would then call # get_default (even though we aren't going to use them) which may run a query which # fails inside this transaction. Given as we are just using MockInstance so that we can # call django_instance_to_entity it on it with the subset of fields we pass in, # what we have is fine. meta = self.model._meta instance = MockInstance(_original=MockInstance(_meta=meta, **result), _meta=meta, **instance_kwargs) # We need to add to the class attribute, rather than replace it! original_class = result.get(POLYMODEL_CLASS_ATTRIBUTE, []) # Update the entity we read above with the new values result.update( django_instance_to_entity( self.connection, self.model, [x[0] for x in self.values ], # Pass in the fields that were updated True, instance)) # Make sure we keep all classes in the inheritence tree! if original_class: if result[POLYMODEL_CLASS_ATTRIBUTE] is not None: result[POLYMODEL_CLASS_ATTRIBUTE].extend(original_class) # Make sure we don't add duplicates else: result[POLYMODEL_CLASS_ATTRIBUTE] = original_class if POLYMODEL_CLASS_ATTRIBUTE in result: result[POLYMODEL_CLASS_ATTRIBUTE] = list( set(result[POLYMODEL_CLASS_ATTRIBUTE])) if not constraints.has_active_unique_constraints(self.model): # The fast path, no constraint checking datastore.Put(result) caching.add_entities_to_cache( self.model, [result], caching.CachingSituation.DATASTORE_PUT, self.namespace, skip_memcache=True, ) else: markers_to_acquire[:], markers_to_release[:] = constraints.get_markers_for_update( self.model, original, result) datastore.Put(result) constraints.update_identifiers(markers_to_acquire, markers_to_release, result.key()) # If the datastore.Put() fails then the exception will only be raised when the # transaction applies, which means that we will still get to here and will still have # applied the marker changes (because they're in a nested, independent transaction). # Hence we set this flag to tell us that we got this far and that we should roll them back. rollback_markers[0] = True # If something dies between here and the `return` statement then we'll have stale unique markers try: # Update the cache before dealing with unique markers, as CachingSituation.DATASTORE_PUT # will only update the context cache caching.add_entities_to_cache( self.model, [result], caching.CachingSituation.DATASTORE_PUT, self.namespace, skip_memcache=True, ) except: # We ignore the exception because raising will rollback the transaction causing # an inconsistent state logging.exception("Unable to update the context cache") pass # Return true to indicate update success return True
def execute(self): check_existence = self.has_pk and not has_concrete_parents(self.model) if not constraints.has_active_unique_constraints(self.model) and not check_existence: # Fast path, no constraint checks and no keys mean we can just do a normal datastore.Put # which isn't limited to 25 results = datastore.Put(self.entities) # This modifies self.entities and sets their keys caching.add_entities_to_cache( self.model, self.entities, caching.CachingSituation.DATASTORE_GET_PUT, self.namespace, skip_memcache=True ) return results def insert_chunk(keys, entities): # Note that this is limited to a maximum of 25 entities. markers = [] @db.transactional(xg=len(entities) > 1) def txn(): for key in keys: if check_existence and key is not None: if utils.key_exists(key): raise IntegrityError("Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, basestring) and id_or_name.startswith("__"): raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name) # Notify App Engine of any keys we're specifying intentionally reserve_id(key.kind(), key.id_or_name(), self.namespace) results = datastore.Put(entities) for entity in entities: markers.extend(constraints.acquire(self.model, entity)) caching.add_entities_to_cache( self.model, entities, caching.CachingSituation.DATASTORE_GET_PUT, self.namespace, skip_memcache=True ) return results try: return txn() except: # There are 3 possible reasons why we've ended up here: # 1. The datastore.Put() failed, but note that because it's a transaction, the # exception isn't raised until the END of the transaction block. # 2. Some of the markers were acquired, but then we hit a unique constraint # conflict and so the outer transaction was rolled back. # 3. Something else went wrong after we'd acquired markers, e.g. the # caching.add_entities_to_cache call got hit by a metaphorical bus. # In any of these cases, we (may) have acquired markers via (a) nested, independent # transaction(s), and so we need to release them again. constraints.release_markers(markers) raise # We can't really support this and maintain expected behaviour. If we chunked the insert and one of the # chunks fails it will mean some of the data would be saved and rather than trying to communicate that back # to the user it's better that they chunk the data themselves as they can deal with the failure better if len(self.entities) > datastore_stub_util._MAX_EG_PER_TXN: raise BulkInsertError("Bulk inserts with unique constraints, or pre-defined keys are limited to {} instances on the datastore".format( datastore_stub_util._MAX_EG_PER_TXN )) return insert_chunk(self.included_keys, self.entities)
def _update_entity(self, key): caching.remove_entities_from_cache_by_key([key]) try: result = datastore.Get(key) except datastore_errors.EntityNotFoundError: # Return false to indicate update failure return False if (isinstance( self.select.gae_query, (Query, UniqueQuery)) # ignore QueryByKeys and NoOpQuery and not utils.entity_matches_query(result, self.select.gae_query)): # Due to eventual consistency they query may have returned an entity which no longer # matches the query return False original = copy.deepcopy(result) # Note: If you replace MockInstance with self.model, you'll find that some delete # tests fail in the test app. This is because any unspecified fields would then call # get_default (even though we aren't going to use them) which may run a query which # fails inside this transaction. Given as we are just using MockInstance so that we can # call django_instance_to_entity it on it with the subset of fields we pass in, # what we have is fine. meta = self.model._meta instance_kwargs = { field.attname: value for field, param, value in self.values } instance = MockInstance(_original=MockInstance(_meta=meta, **result), _meta=meta, **instance_kwargs) # We need to add to the class attribute, rather than replace it! original_class = result.get(POLYMODEL_CLASS_ATTRIBUTE, []) # Update the entity we read above with the new values result.update( django_instance_to_entity( self.connection, self.model, [x[0] for x in self.values], # Pass in the fields that were updated True, instance)) # Make sure we keep all classes in the inheritence tree! if original_class: if result[POLYMODEL_CLASS_ATTRIBUTE] is not None: result[POLYMODEL_CLASS_ATTRIBUTE].extend(original_class) # Make sure we don't add duplicates else: result[POLYMODEL_CLASS_ATTRIBUTE] = original_class if POLYMODEL_CLASS_ATTRIBUTE in result: result[POLYMODEL_CLASS_ATTRIBUTE] = list( set(result[POLYMODEL_CLASS_ATTRIBUTE])) if not constraints.constraint_checks_enabled(self.model): # The fast path, no constraint checking datastore.Put(result) caching.add_entities_to_cache( self.model, [result], caching.CachingSituation.DATASTORE_PUT) else: to_acquire, to_release = constraints.get_markers_for_update( self.model, original, result) # Acquire first, because if that fails then we don't want to alter what's already there constraints.acquire_identifiers(to_acquire, result.key()) try: datastore.Put(result) caching.add_entities_to_cache( self.model, [result], caching.CachingSituation.DATASTORE_PUT) except: constraints.release_identifiers(to_acquire) raise else: # Now we release the ones we don't want anymore constraints.release_identifiers(to_release) # Return true to indicate update success return True