def _do_enter(cls, state, decorator_args): _init_storage() mandatory = decorator_args.get("mandatory", False) independent = decorator_args.get("independent", False) xg = decorator_args.get("xg", False) options = CreateTransactionOptions( xg=xg, propagation=TransactionOptions.INDEPENDENT if independent else None ) new_transaction = None if independent: new_transaction = IndependentTransaction(options) elif in_atomic_block(): new_transaction = NestedTransaction(None) elif mandatory: raise TransactionFailedError( "You've specified that an outer transaction is mandatory, but one doesn't exist" ) else: new_transaction = NormalTransaction(options) _STORAGE.transaction_stack.append(new_transaction) _STORAGE.transaction_stack[-1].enter() if isinstance(new_transaction, (IndependentTransaction, NormalTransaction)): caching.get_context().stack.push() # We may have created a new transaction, we may not. current_transaction() returns # the actual active transaction (highest NormalTransaction or lowest IndependentTransaction) # or None if we're in a non_atomic, or there are no transactions return current_transaction()
def execute(self): table = self.table query = datastore.Query(table, keys_only=True, namespace=self.namespace) while query.Count(): datastore.Delete(query.Run()) # Delete the markers we need to from djangae.db.constraints import UniqueMarker query = datastore.Query(UniqueMarker.kind(), keys_only=True, namespace=self.namespace) query["__key__ >="] = datastore.Key.from_path(UniqueMarker.kind(), self.table, namespace=self.namespace) query["__key__ <"] = datastore.Key.from_path(UniqueMarker.kind(), u"{}{}".format( self.table, u'\ufffd'), namespace=self.namespace) while query.Count(): datastore.Delete(query.Run()) # TODO: ideally we would only clear the cached objects for the table that was flushed, but # we have no way of doing that memcache.flush_all() caching.get_context().reset()
def test_consistent_read_updates_cache_outside_transaction(self): """ A read inside a transaction shouldn't update the context cache outside that transaction """ entity_data = { "field1": "Apple", "comb1": 1, "comb2": "Cherry" } original = CachingTestModel.objects.create(**entity_data) caching.get_context().reset(keep_disabled_flags=True) CachingTestModel.objects.get(pk=original.pk) # Should update the cache with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get: CachingTestModel.objects.get(pk=original.pk) self.assertFalse(datastore_get.called) caching.get_context().reset(keep_disabled_flags=True) with transaction.atomic(): with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get: CachingTestModel.objects.get(pk=original.pk) # Should *not* update the cache self.assertTrue(datastore_get.called) with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get: CachingTestModel.objects.get(pk=original.pk) self.assertTrue(datastore_get.called)
def _create_test_db(self, verbosity, autoclobber, *args): if args: logger.warning( "'keepdb' argument is not currently supported on the AppEngine backend" ) get_context().reset()
def _do_exit(cls, state, decorator_args, exception): independent = decorator_args.get("independent", False) try: if state.transaction_started: if exception: _GetConnection().rollback() else: if not _GetConnection().commit(): raise TransactionFailedError() finally: if state.transaction_started: _PopConnection() # Clear the context cache at the end of a transaction if exception: caching.get_context().stack.pop(discard=True) else: caching.get_context().stack.pop(apply_staged=True, clear_staged=True) # If we were in an independent transaction, put everything back # the way it was! if independent: while state.conn_stack: _PushConnection(state.conn_stack.pop()) # Restore the in-context cache as it was caching.get_context().stack = state.original_stack
def _create_test_db(self, verbosity, autoclobber, *args): from google.appengine.ext import testbed # Imported lazily to prevent warnings on GAE assert not self.testbed if args: logger.warning( "'keepdb' argument is not currently supported on the AppEngine backend" ) # We allow users to disable scattered IDs in tests. This primarily for running Django tests that # assume implicit ordering (yeah, annoying) use_scattered = not getattr(settings, "DJANGAE_SEQUENTIAL_IDS_IN_TESTS", False) kwargs = { "use_sqlite": True, "auto_id_policy": testbed.AUTO_ID_POLICY_SCATTERED if use_scattered else testbed.AUTO_ID_POLICY_SEQUENTIAL, "consistency_policy": datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1) } self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_datastore_v3_stub(**kwargs) self.testbed.init_memcache_stub() get_context().reset()
def test_consistent_read_updates_cache_outside_transaction(self): """ A read inside a transaction shouldn't update the context cache outside that transaction """ entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"} original = CachingTestModel.objects.create(**entity_data) caching.get_context().reset(keep_disabled_flags=True) CachingTestModel.objects.get(pk=original.pk) # Should update the cache with sleuth.watch( "google.appengine.api.datastore.Get") as datastore_get: CachingTestModel.objects.get(pk=original.pk) self.assertFalse(datastore_get.called) caching.get_context().reset(keep_disabled_flags=True) with transaction.atomic(): with sleuth.watch( "google.appengine.api.datastore.Get") as datastore_get: CachingTestModel.objects.get( pk=original.pk) # Should *not* update the cache self.assertTrue(datastore_get.called) with sleuth.watch( "google.appengine.api.datastore.Get") as datastore_get: CachingTestModel.objects.get(pk=original.pk) self.assertTrue(datastore_get.called)
def init_testbed(): IGNORED_STUBS = [] # We allow users to disable scattered IDs in tests. This primarily for running Django tests that # assume implicit ordering (yeah, annoying) use_scattered = not getattr(settings, "DJANGAE_SEQUENTIAL_IDS_IN_TESTS", False) stub_kwargs = { "init_taskqueue_stub": { "root_path": environment.get_application_root() }, "init_datastore_v3_stub": { "use_sqlite": True, "auto_id_policy": testbed.AUTO_ID_POLICY_SCATTERED if use_scattered else testbed.AUTO_ID_POLICY_SEQUENTIAL, "consistency_policy": datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1) } } get_context().reset() # Reset any context caching bed = testbed.Testbed() bed.activate() for init_name in testbed.INIT_STUB_METHOD_NAMES.values(): if init_name in IGNORED_STUBS: continue getattr(bed, init_name)(**stub_kwargs.get(init_name, {})) return bed
def init_testbed(): try: import PIL IGNORED_STUBS = [] except ImportError: logger.warning("Unable to initialize the images stub as Pillow is unavailable") IGNORED_STUBS = [ "init_images_stub" ] # We allow users to disable scattered IDs in tests. This primarily for running Django tests that # assume implicit ordering (yeah, annoying) use_scattered = not getattr(settings, "DJANGAE_SEQUENTIAL_IDS_IN_TESTS", False) stub_kwargs = { "init_taskqueue_stub": { "root_path": environment.get_application_root() }, "init_datastore_v3_stub": { "use_sqlite": True, "auto_id_policy": testbed.AUTO_ID_POLICY_SCATTERED if use_scattered else testbed.AUTO_ID_POLICY_SEQUENTIAL, "consistency_policy": datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1) } } get_context().reset(); # Reset any context caching bed = testbed.Testbed() bed.activate() for init_name in testbed.INIT_STUB_METHOD_NAMES.values(): if init_name in IGNORED_STUBS: continue getattr(bed, init_name)(**stub_kwargs.get(init_name, {})) return bed
def test_inconsistent_read_doesnt_update_cache(self): entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"} original = CachingTestModel.objects.create(**entity_data) caching.get_context().reset(keep_disabled_flags=True) CachingTestModel.objects.all() # Inconsistent with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get: CachingTestModel.objects.get(pk=original.pk) self.assertTrue(datastore_get.called)
def test_inconsistent_read_doesnt_update_cache(self): entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"} original = CachingTestModel.objects.create(**entity_data) caching.get_context().reset(keep_disabled_flags=True) CachingTestModel.objects.all() # Inconsistent with sleuth.watch( "google.appengine.api.datastore.Get") as datastore_get: CachingTestModel.objects.get(pk=original.pk) self.assertTrue(datastore_get.called)
def _do_exit(cls, state, decorator_args, exception): context = caching.get_context() transaction = _STORAGE.transaction_stack.pop() transaction._exit() # Restore the context stack as it was context.stack.stack = context.stack.stack + state.original_stack
def __enter__(self): ctx = caching.get_context() self.orig_memcache = ctx.memcache_enabled self.orig_context = ctx.context_enabled ctx.memcache_enabled = not self.memcache ctx.context_enabled = not self.context
def _do_exit(cls, state, decorator_args, exception): if not state.conn_stack: return # Restore the connection stack while state.conn_stack: _PushConnection(state.conn_stack.pop()) caching.get_context().stack = state.original_stack
def _do_enter(cls, state, decorator_args): state.conn_stack = [] # We aren't in a transaction, do nothing! if not in_atomic_block(): return # Store the current in-context stack state.original_stack = copy.deepcopy(caching.get_context().stack) # Similar to independent transactions, unwind the connection statck # until we aren't in a transaction while in_atomic_block(): state.conn_stack.append(_PopConnection()) # Unwind the in-context stack while len(caching.get_context().stack.stack) > 1: caching.get_context().stack.pop(discard=True)
def execute(self): table = self.table query = datastore.Query(table, keys_only=True, namespace=self.namespace) while query.Count(): datastore.Delete(query.Run()) # Delete the markers we need to from djangae.db.constraints import UniqueMarker query = datastore.Query(UniqueMarker.kind(), keys_only=True, namespace=self.namespace) query["__key__ >="] = datastore.Key.from_path(UniqueMarker.kind(), self.table, namespace=self.namespace) query["__key__ <"] = datastore.Key.from_path( UniqueMarker.kind(), u"{}{}".format(self.table, u'\ufffd'), namespace=self.namespace ) while query.Count(): datastore.Delete(query.Run()) # TODO: ideally we would only clear the cached objects for the table that was flushed, but # we have no way of doing that memcache.flush_all() caching.get_context().reset()
def _do_enter(cls, state, decorator_args): mandatory = decorator_args.get("mandatory", False) independent = decorator_args.get("independent", False) xg = decorator_args.get("xg", False) # Reset the state state.conn_stack = [] state.transaction_started = False state.original_stack = None if independent: # Unwind the connection stack and store it on the state so that # we can replace it on exit while in_atomic_block(): state.conn_stack.append(_PopConnection()) state.original_stack = copy.deepcopy(caching.get_context().stack) elif in_atomic_block(): # App Engine doesn't support nested transactions, so if there is a nested # atomic() call we just don't do anything. This is how RunInTransaction does it return elif mandatory: raise TransactionFailedError( "You've specified that an outer transaction is mandatory, but one doesn't exist" ) options = CreateTransactionOptions( xg=xg, propagation=TransactionOptions.INDEPENDENT if independent else None) conn = _GetConnection() new_conn = conn.new_transaction(options) _PushConnection(new_conn) assert (_GetConnection()) # Clear the context cache at the start of a transaction caching.ensure_context() caching.get_context().stack.push() state.transaction_started = True
def _do_enter(cls, state, decorator_args): mandatory = decorator_args.get("mandatory", False) independent = decorator_args.get("independent", False) xg = decorator_args.get("xg", False) # Reset the state state.conn_stack = [] state.transaction_started = False state.original_stack = None if independent: # Unwind the connection stack and store it on the state so that # we can replace it on exit while in_atomic_block(): state.conn_stack.append(_PopConnection()) state.original_stack = copy.deepcopy(caching.get_context().stack) elif in_atomic_block(): # App Engine doesn't support nested transactions, so if there is a nested # atomic() call we just don't do anything. This is how RunInTransaction does it return elif mandatory: raise TransactionFailedError("You've specified that an outer transaction is mandatory, but one doesn't exist") options = CreateTransactionOptions( xg=xg, propagation=TransactionOptions.INDEPENDENT if independent else None ) conn = _GetConnection() new_conn = conn.new_transaction(options) _PushConnection(new_conn) assert(_GetConnection()) # Clear the context cache at the start of a transaction caching.ensure_context() caching.get_context().stack.push() state.transaction_started = True
def _create_test_db(self, verbosity, autoclobber, *args): from google.appengine.ext import testbed # Imported lazily to prevent warnings on GAE assert not self.testbed if args: logging.warning("'keepdb' argument is not currently supported on the AppEngine backend") # We allow users to disable scattered IDs in tests. This primarily for running Django tests that # assume implicit ordering (yeah, annoying) use_scattered = not getattr(settings, "DJANGAE_SEQUENTIAL_IDS_IN_TESTS", False) kwargs = { "use_sqlite": True, "auto_id_policy": testbed.AUTO_ID_POLICY_SCATTERED if use_scattered else testbed.AUTO_ID_POLICY_SEQUENTIAL, "consistency_policy": datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1) } self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_datastore_v3_stub(**kwargs) self.testbed.init_memcache_stub() get_context().reset()
def _do_enter(cls, state, decorator_args): _init_storage() context = caching.get_context() new_transaction = NoTransaction(None) _STORAGE.transaction_stack.append(new_transaction) _STORAGE.transaction_stack[-1]._enter() # Store the current state of the stack (aside from the first entry) state.original_stack = copy.deepcopy(context.stack.stack[1:]) # Unwind the in-context stack leaving just the first entry while len(context.stack.stack) > 1: context.stack.pop(discard=True)
def test_nested_transactions_dont_get_their_own_context(self): """ The datastore doesn't support nested transactions, so when there is a nested atomic block which isn't marked as independent, the atomic is a no-op. Therefore we shouldn't push a context here, and we shouldn't pop it at the end either. """ context = caching.get_context() self.assertEqual(1, context.stack.size) with transaction.atomic(): self.assertEqual(2, context.stack.size) with transaction.atomic(): self.assertEqual(2, context.stack.size) with transaction.atomic(): self.assertEqual(2, context.stack.size) self.assertEqual(2, context.stack.size) self.assertEqual(2, context.stack.size) self.assertEqual(1, context.stack.size)
def _do_exit(cls, state, decorator_args, exception): _init_storage() context = caching.get_context() transaction = _STORAGE.transaction_stack.pop() try: if transaction._connection: if exception: transaction._connection.rollback() else: if not transaction._connection.commit(): raise TransactionFailedError() finally: if isinstance(transaction, (IndependentTransaction, NormalTransaction)): # Clear the context cache at the end of a transaction if exception: context.stack.pop(discard=True) else: context.stack.pop(apply_staged=True, clear_staged=True) transaction.exit() transaction._connection = None
def _destroy_test_db(self, name, verbosity): if self.testbed: get_context().reset() self.testbed.deactivate() self.testbed = None
def __exit__(self, *args, **kwargs): ctx = caching.get_context() ctx.memcache_enabled = self.orig_memcache ctx.context_enabled = self.orig_context
def _destroy_test_db(self, name, verbosity): get_context().reset()
def _create_test_db(self, verbosity, autoclobber, *args): if args: logger.warning("'keepdb' argument is not currently supported on the AppEngine backend") get_context().reset()