def _do_enter(cls, state, decorator_args):
_init_storage()
mandatory = decorator_args.get("mandatory", False)
independent = decorator_args.get("independent", False)
xg = decorator_args.get("xg", False)
options = CreateTransactionOptions(
xg=xg,
propagation=TransactionOptions.INDEPENDENT if independent else None
)
new_transaction = None
if independent:
new_transaction = IndependentTransaction(options)
elif in_atomic_block():
new_transaction = NestedTransaction(None)
elif mandatory:
raise TransactionFailedError(
"You've specified that an outer transaction is mandatory, but one doesn't exist"
)
else:
new_transaction = NormalTransaction(options)
_STORAGE.transaction_stack.append(new_transaction)
_STORAGE.transaction_stack[-1].enter()
if isinstance(new_transaction, (IndependentTransaction, NormalTransaction)):
caching.get_context().stack.push()
# We may have created a new transaction, we may not. current_transaction() returns
# the actual active transaction (highest NormalTransaction or lowest IndependentTransaction)
# or None if we're in a non_atomic, or there are no transactions
return current_transaction()
def execute(self):
table = self.table
query = datastore.Query(table,
keys_only=True,
namespace=self.namespace)
while query.Count():
datastore.Delete(query.Run())
# Delete the markers we need to
from djangae.db.constraints import UniqueMarker
query = datastore.Query(UniqueMarker.kind(),
keys_only=True,
namespace=self.namespace)
query["__key__ >="] = datastore.Key.from_path(UniqueMarker.kind(),
self.table,
namespace=self.namespace)
query["__key__ <"] = datastore.Key.from_path(UniqueMarker.kind(),
u"{}{}".format(
self.table,
u'\ufffd'),
namespace=self.namespace)
while query.Count():
datastore.Delete(query.Run())
# TODO: ideally we would only clear the cached objects for the table that was flushed, but
# we have no way of doing that
memcache.flush_all()
caching.get_context().reset()
def test_consistent_read_updates_cache_outside_transaction(self):
"""
A read inside a transaction shouldn't update the context cache outside that
transaction
"""
entity_data = {
"field1": "Apple",
"comb1": 1,
"comb2": "Cherry"
}
original = CachingTestModel.objects.create(**entity_data)
caching.get_context().reset(keep_disabled_flags=True)
CachingTestModel.objects.get(pk=original.pk) # Should update the cache
with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk)
self.assertFalse(datastore_get.called)
caching.get_context().reset(keep_disabled_flags=True)
with transaction.atomic():
with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk) # Should *not* update the cache
self.assertTrue(datastore_get.called)
with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk)
self.assertTrue(datastore_get.called)
def _create_test_db(self, verbosity, autoclobber, *args):
if args:
logger.warning(
"'keepdb' argument is not currently supported on the AppEngine backend"
)
get_context().reset()
def _do_exit(cls, state, decorator_args, exception):
independent = decorator_args.get("independent", False)
try:
if state.transaction_started:
if exception:
_GetConnection().rollback()
else:
if not _GetConnection().commit():
raise TransactionFailedError()
finally:
if state.transaction_started:
_PopConnection()
# Clear the context cache at the end of a transaction
if exception:
caching.get_context().stack.pop(discard=True)
else:
caching.get_context().stack.pop(apply_staged=True,
clear_staged=True)
# If we were in an independent transaction, put everything back
# the way it was!
if independent:
while state.conn_stack:
_PushConnection(state.conn_stack.pop())
# Restore the in-context cache as it was
caching.get_context().stack = state.original_stack
def _do_exit(cls, state, decorator_args, exception):
independent = decorator_args.get("independent", False)
try:
if state.transaction_started:
if exception:
_GetConnection().rollback()
else:
if not _GetConnection().commit():
raise TransactionFailedError()
finally:
if state.transaction_started:
_PopConnection()
# Clear the context cache at the end of a transaction
if exception:
caching.get_context().stack.pop(discard=True)
else:
caching.get_context().stack.pop(apply_staged=True, clear_staged=True)
# If we were in an independent transaction, put everything back
# the way it was!
if independent:
while state.conn_stack:
_PushConnection(state.conn_stack.pop())
# Restore the in-context cache as it was
caching.get_context().stack = state.original_stack
def _create_test_db(self, verbosity, autoclobber, *args):
from google.appengine.ext import testbed # Imported lazily to prevent warnings on GAE
assert not self.testbed
if args:
logger.warning(
"'keepdb' argument is not currently supported on the AppEngine backend"
)
# We allow users to disable scattered IDs in tests. This primarily for running Django tests that
# assume implicit ordering (yeah, annoying)
use_scattered = not getattr(settings,
"DJANGAE_SEQUENTIAL_IDS_IN_TESTS", False)
kwargs = {
"use_sqlite":
True,
"auto_id_policy":
testbed.AUTO_ID_POLICY_SCATTERED
if use_scattered else testbed.AUTO_ID_POLICY_SEQUENTIAL,
"consistency_policy":
datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1)
}
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_datastore_v3_stub(**kwargs)
self.testbed.init_memcache_stub()
get_context().reset()
def test_consistent_read_updates_cache_outside_transaction(self):
"""
A read inside a transaction shouldn't update the context cache outside that
transaction
"""
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
original = CachingTestModel.objects.create(**entity_data)
caching.get_context().reset(keep_disabled_flags=True)
CachingTestModel.objects.get(pk=original.pk) # Should update the cache
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk)
self.assertFalse(datastore_get.called)
caching.get_context().reset(keep_disabled_flags=True)
with transaction.atomic():
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(
pk=original.pk) # Should *not* update the cache
self.assertTrue(datastore_get.called)
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk)
self.assertTrue(datastore_get.called)
def _do_enter(cls, state, decorator_args):
_init_storage()
mandatory = decorator_args.get("mandatory", False)
independent = decorator_args.get("independent", False)
xg = decorator_args.get("xg", False)
options = CreateTransactionOptions(
xg=xg,
propagation=TransactionOptions.INDEPENDENT if independent else None
)
new_transaction = None
if independent:
new_transaction = IndependentTransaction(options)
elif in_atomic_block():
new_transaction = NestedTransaction(None)
elif mandatory:
raise TransactionFailedError(
"You've specified that an outer transaction is mandatory, but one doesn't exist"
)
else:
new_transaction = NormalTransaction(options)
_STORAGE.transaction_stack.append(new_transaction)
_STORAGE.transaction_stack[-1].enter()
if isinstance(new_transaction, (IndependentTransaction, NormalTransaction)):
caching.get_context().stack.push()
# We may have created a new transaction, we may not. current_transaction() returns
# the actual active transaction (highest NormalTransaction or lowest IndependentTransaction)
# or None if we're in a non_atomic, or there are no transactions
return current_transaction()
def init_testbed():
IGNORED_STUBS = []
# We allow users to disable scattered IDs in tests. This primarily for running Django tests that
# assume implicit ordering (yeah, annoying)
use_scattered = not getattr(settings, "DJANGAE_SEQUENTIAL_IDS_IN_TESTS",
False)
stub_kwargs = {
"init_taskqueue_stub": {
"root_path": environment.get_application_root()
},
"init_datastore_v3_stub": {
"use_sqlite":
True,
"auto_id_policy":
testbed.AUTO_ID_POLICY_SCATTERED
if use_scattered else testbed.AUTO_ID_POLICY_SEQUENTIAL,
"consistency_policy":
datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1)
}
}
get_context().reset()
# Reset any context caching
bed = testbed.Testbed()
bed.activate()
for init_name in testbed.INIT_STUB_METHOD_NAMES.values():
if init_name in IGNORED_STUBS:
continue
getattr(bed, init_name)(**stub_kwargs.get(init_name, {}))
return bed
def init_testbed():
try:
import PIL
IGNORED_STUBS = []
except ImportError:
logger.warning("Unable to initialize the images stub as Pillow is unavailable")
IGNORED_STUBS = [
"init_images_stub"
]
# We allow users to disable scattered IDs in tests. This primarily for running Django tests that
# assume implicit ordering (yeah, annoying)
use_scattered = not getattr(settings, "DJANGAE_SEQUENTIAL_IDS_IN_TESTS", False)
stub_kwargs = {
"init_taskqueue_stub": {
"root_path": environment.get_application_root()
},
"init_datastore_v3_stub": {
"use_sqlite": True,
"auto_id_policy": testbed.AUTO_ID_POLICY_SCATTERED if use_scattered else testbed.AUTO_ID_POLICY_SEQUENTIAL,
"consistency_policy": datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1)
}
}
get_context().reset(); # Reset any context caching
bed = testbed.Testbed()
bed.activate()
for init_name in testbed.INIT_STUB_METHOD_NAMES.values():
if init_name in IGNORED_STUBS:
continue
getattr(bed, init_name)(**stub_kwargs.get(init_name, {}))
return bed
def test_inconsistent_read_doesnt_update_cache(self):
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
original = CachingTestModel.objects.create(**entity_data)
caching.get_context().reset(keep_disabled_flags=True)
CachingTestModel.objects.all() # Inconsistent
with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk)
self.assertTrue(datastore_get.called)
def test_inconsistent_read_doesnt_update_cache(self):
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
original = CachingTestModel.objects.create(**entity_data)
caching.get_context().reset(keep_disabled_flags=True)
CachingTestModel.objects.all() # Inconsistent
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk)
self.assertTrue(datastore_get.called)
def _do_exit(cls, state, decorator_args, exception):
context = caching.get_context()
transaction = _STORAGE.transaction_stack.pop()
transaction._exit()
# Restore the context stack as it was
context.stack.stack = context.stack.stack + state.original_stack
def _do_exit(cls, state, decorator_args, exception):
context = caching.get_context()
transaction = _STORAGE.transaction_stack.pop()
transaction._exit()
# Restore the context stack as it was
context.stack.stack = context.stack.stack + state.original_stack
def __enter__(self):
ctx = caching.get_context()
self.orig_memcache = ctx.memcache_enabled
self.orig_context = ctx.context_enabled
ctx.memcache_enabled = not self.memcache
ctx.context_enabled = not self.context
def __enter__(self):
ctx = caching.get_context()
self.orig_memcache = ctx.memcache_enabled
self.orig_context = ctx.context_enabled
ctx.memcache_enabled = not self.memcache
ctx.context_enabled = not self.context
def _do_exit(cls, state, decorator_args, exception):
if not state.conn_stack:
return
# Restore the connection stack
while state.conn_stack:
_PushConnection(state.conn_stack.pop())
caching.get_context().stack = state.original_stack
def _do_enter(cls, state, decorator_args):
state.conn_stack = []
# We aren't in a transaction, do nothing!
if not in_atomic_block():
return
# Store the current in-context stack
state.original_stack = copy.deepcopy(caching.get_context().stack)
# Similar to independent transactions, unwind the connection statck
# until we aren't in a transaction
while in_atomic_block():
state.conn_stack.append(_PopConnection())
# Unwind the in-context stack
while len(caching.get_context().stack.stack) > 1:
caching.get_context().stack.pop(discard=True)
def _do_exit(cls, state, decorator_args, exception):
if not state.conn_stack:
return
# Restore the connection stack
while state.conn_stack:
_PushConnection(state.conn_stack.pop())
caching.get_context().stack = state.original_stack
def _do_enter(cls, state, decorator_args):
state.conn_stack = []
# We aren't in a transaction, do nothing!
if not in_atomic_block():
return
# Store the current in-context stack
state.original_stack = copy.deepcopy(caching.get_context().stack)
# Similar to independent transactions, unwind the connection statck
# until we aren't in a transaction
while in_atomic_block():
state.conn_stack.append(_PopConnection())
# Unwind the in-context stack
while len(caching.get_context().stack.stack) > 1:
caching.get_context().stack.pop(discard=True)
def execute(self):
table = self.table
query = datastore.Query(table, keys_only=True, namespace=self.namespace)
while query.Count():
datastore.Delete(query.Run())
# Delete the markers we need to
from djangae.db.constraints import UniqueMarker
query = datastore.Query(UniqueMarker.kind(), keys_only=True, namespace=self.namespace)
query["__key__ >="] = datastore.Key.from_path(UniqueMarker.kind(), self.table, namespace=self.namespace)
query["__key__ <"] = datastore.Key.from_path(
UniqueMarker.kind(), u"{}{}".format(self.table, u'\ufffd'), namespace=self.namespace
)
while query.Count():
datastore.Delete(query.Run())
# TODO: ideally we would only clear the cached objects for the table that was flushed, but
# we have no way of doing that
memcache.flush_all()
caching.get_context().reset()
def _do_enter(cls, state, decorator_args):
mandatory = decorator_args.get("mandatory", False)
independent = decorator_args.get("independent", False)
xg = decorator_args.get("xg", False)
# Reset the state
state.conn_stack = []
state.transaction_started = False
state.original_stack = None
if independent:
# Unwind the connection stack and store it on the state so that
# we can replace it on exit
while in_atomic_block():
state.conn_stack.append(_PopConnection())
state.original_stack = copy.deepcopy(caching.get_context().stack)
elif in_atomic_block():
# App Engine doesn't support nested transactions, so if there is a nested
# atomic() call we just don't do anything. This is how RunInTransaction does it
return
elif mandatory:
raise TransactionFailedError(
"You've specified that an outer transaction is mandatory, but one doesn't exist"
)
options = CreateTransactionOptions(
xg=xg,
propagation=TransactionOptions.INDEPENDENT
if independent else None)
conn = _GetConnection()
new_conn = conn.new_transaction(options)
_PushConnection(new_conn)
assert (_GetConnection())
# Clear the context cache at the start of a transaction
caching.ensure_context()
caching.get_context().stack.push()
state.transaction_started = True
def _do_enter(cls, state, decorator_args):
mandatory = decorator_args.get("mandatory", False)
independent = decorator_args.get("independent", False)
xg = decorator_args.get("xg", False)
# Reset the state
state.conn_stack = []
state.transaction_started = False
state.original_stack = None
if independent:
# Unwind the connection stack and store it on the state so that
# we can replace it on exit
while in_atomic_block():
state.conn_stack.append(_PopConnection())
state.original_stack = copy.deepcopy(caching.get_context().stack)
elif in_atomic_block():
# App Engine doesn't support nested transactions, so if there is a nested
# atomic() call we just don't do anything. This is how RunInTransaction does it
return
elif mandatory:
raise TransactionFailedError("You've specified that an outer transaction is mandatory, but one doesn't exist")
options = CreateTransactionOptions(
xg=xg,
propagation=TransactionOptions.INDEPENDENT if independent else None
)
conn = _GetConnection()
new_conn = conn.new_transaction(options)
_PushConnection(new_conn)
assert(_GetConnection())
# Clear the context cache at the start of a transaction
caching.ensure_context()
caching.get_context().stack.push()
state.transaction_started = True
def _create_test_db(self, verbosity, autoclobber, *args):
from google.appengine.ext import testbed # Imported lazily to prevent warnings on GAE
assert not self.testbed
if args:
logging.warning("'keepdb' argument is not currently supported on the AppEngine backend")
# We allow users to disable scattered IDs in tests. This primarily for running Django tests that
# assume implicit ordering (yeah, annoying)
use_scattered = not getattr(settings, "DJANGAE_SEQUENTIAL_IDS_IN_TESTS", False)
kwargs = {
"use_sqlite": True,
"auto_id_policy": testbed.AUTO_ID_POLICY_SCATTERED if use_scattered else testbed.AUTO_ID_POLICY_SEQUENTIAL,
"consistency_policy": datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1)
}
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_datastore_v3_stub(**kwargs)
self.testbed.init_memcache_stub()
get_context().reset()
def _do_enter(cls, state, decorator_args):
_init_storage()
context = caching.get_context()
new_transaction = NoTransaction(None)
_STORAGE.transaction_stack.append(new_transaction)
_STORAGE.transaction_stack[-1]._enter()
# Store the current state of the stack (aside from the first entry)
state.original_stack = copy.deepcopy(context.stack.stack[1:])
# Unwind the in-context stack leaving just the first entry
while len(context.stack.stack) > 1:
context.stack.pop(discard=True)
def _do_enter(cls, state, decorator_args):
_init_storage()
context = caching.get_context()
new_transaction = NoTransaction(None)
_STORAGE.transaction_stack.append(new_transaction)
_STORAGE.transaction_stack[-1]._enter()
# Store the current state of the stack (aside from the first entry)
state.original_stack = copy.deepcopy(context.stack.stack[1:])
# Unwind the in-context stack leaving just the first entry
while len(context.stack.stack) > 1:
context.stack.pop(discard=True)
def test_nested_transactions_dont_get_their_own_context(self):
"""
The datastore doesn't support nested transactions, so when there is a nested
atomic block which isn't marked as independent, the atomic is a no-op. Therefore
we shouldn't push a context here, and we shouldn't pop it at the end either.
"""
context = caching.get_context()
self.assertEqual(1, context.stack.size)
with transaction.atomic():
self.assertEqual(2, context.stack.size)
with transaction.atomic():
self.assertEqual(2, context.stack.size)
with transaction.atomic():
self.assertEqual(2, context.stack.size)
self.assertEqual(2, context.stack.size)
self.assertEqual(2, context.stack.size)
self.assertEqual(1, context.stack.size)
def test_nested_transactions_dont_get_their_own_context(self):
"""
The datastore doesn't support nested transactions, so when there is a nested
atomic block which isn't marked as independent, the atomic is a no-op. Therefore
we shouldn't push a context here, and we shouldn't pop it at the end either.
"""
context = caching.get_context()
self.assertEqual(1, context.stack.size)
with transaction.atomic():
self.assertEqual(2, context.stack.size)
with transaction.atomic():
self.assertEqual(2, context.stack.size)
with transaction.atomic():
self.assertEqual(2, context.stack.size)
self.assertEqual(2, context.stack.size)
self.assertEqual(2, context.stack.size)
self.assertEqual(1, context.stack.size)
def _do_exit(cls, state, decorator_args, exception):
_init_storage()
context = caching.get_context()
transaction = _STORAGE.transaction_stack.pop()
try:
if transaction._connection:
if exception:
transaction._connection.rollback()
else:
if not transaction._connection.commit():
raise TransactionFailedError()
finally:
if isinstance(transaction, (IndependentTransaction, NormalTransaction)):
# Clear the context cache at the end of a transaction
if exception:
context.stack.pop(discard=True)
else:
context.stack.pop(apply_staged=True, clear_staged=True)
transaction.exit()
transaction._connection = None
def _do_exit(cls, state, decorator_args, exception):
_init_storage()
context = caching.get_context()
transaction = _STORAGE.transaction_stack.pop()
try:
if transaction._connection:
if exception:
transaction._connection.rollback()
else:
if not transaction._connection.commit():
raise TransactionFailedError()
finally:
if isinstance(transaction, (IndependentTransaction, NormalTransaction)):
# Clear the context cache at the end of a transaction
if exception:
context.stack.pop(discard=True)
else:
context.stack.pop(apply_staged=True, clear_staged=True)
transaction.exit()
transaction._connection = None
def _destroy_test_db(self, name, verbosity):
if self.testbed:
get_context().reset()
self.testbed.deactivate()
self.testbed = None
def _destroy_test_db(self, name, verbosity):
if self.testbed:
get_context().reset()
self.testbed.deactivate()
self.testbed = None
def __exit__(self, *args, **kwargs):
ctx = caching.get_context()
ctx.memcache_enabled = self.orig_memcache
ctx.context_enabled = self.orig_context
def _destroy_test_db(self, name, verbosity):
get_context().reset()
def _destroy_test_db(self, name, verbosity):
get_context().reset()
def _create_test_db(self, verbosity, autoclobber, *args):
if args:
logger.warning("'keepdb' argument is not currently supported on the AppEngine backend")
get_context().reset()
def __exit__(self, *args, **kwargs):
ctx = caching.get_context()
ctx.memcache_enabled = self.orig_memcache
ctx.context_enabled = self.orig_context