def _update_or_create_shard(self, step):
""" Find or create a random shard and alter its `count` by the given step. """
shard_index = random.randint(0, self.field.shard_count - 1)
# Converting the set to a list introduces some randomness in the ordering, but that's fine
shard_pks = list(self.field.value_from_object(
self.instance)) # needs to be indexable
try:
shard_pk = shard_pks[shard_index]
except IndexError:
# We don't have this many shards yet, so create a new one
with transaction.atomic(xg=True):
# We must re-fetch the instance to ensure that we do this atomically, but we must
# also update self.instance so that the calling code which is referencing
# self.instance also gets the updated list of shard PKs
new_shard = self._create_shard(count=step)
new_instance = self.instance._default_manager.get(
pk=self.instance.pk)
new_instance_shard_pks = getattr(new_instance,
self.field.attname, set())
new_instance_shard_pks.add(new_shard.pk)
setattr(self.instance, self.field.attname,
new_instance_shard_pks)
models.Model.save(
new_instance
) # avoid custom save method, which might do DB lookups
else:
with transaction.atomic():
from djangae.models import CounterShard
shard = CounterShard.objects.get(pk=shard_pk)
shard.count += step
shard.save()
def _update_or_create_shard(self, step, is_reset=False):
""" Find or create a random shard and alter its `count` by the given step. """
shard_index = random.randint(0, self.field.shard_count - 1)
# Converting the set to a list introduces some randomness in the ordering, but that's fine
shard_pks = list(self.field.value_from_object(self.instance)) # needs to be indexable
try:
shard_pk = shard_pks[shard_index]
except IndexError:
# We don't have this many shards yet, so create a new one
with transaction.atomic(xg=True):
# We must re-fetch the instance to ensure that we do this atomically, but we must
# also update self.instance so that the calling code which is referencing
# self.instance also gets the updated list of shard PKs
new_shard = self._create_shard(count=step)
new_instance = self.instance.__class__._default_manager.get(pk=self.instance.pk)
new_instance_shard_pks = getattr(new_instance, self.field.attname, set())
new_instance_shard_pks.add(new_shard.pk)
setattr(self.instance, self.field.attname, new_instance_shard_pks)
models.Model.save(new_instance) # avoid custom save method, which might do DB lookups
shard = new_shard
else:
with transaction.atomic():
from djangae.models import CounterShard
shard = CounterShard.objects.get(pk=shard_pk)
shard.count += step
shard.save()
# if the ShardedCounter has on_change callback, run it now
if self.field.on_change:
self.field.on_change(self.instance, step, is_reset=is_reset)
def test_independent_transaction_applies_to_outer_context(self):
"""
When a transaction commits successfully, we can apply its cache to the outer stack. This
alters the behaviour of transactions a little but in a positive way. Things that change are:
1. If you run an independent transaction inside another transaction, a subsequent Get for an entity
updated there will return the updated instance from the cache. Due to serialization of transactions
it's possible that this would have happened anyway (the outer transaction wouldn't start until the independent
one had finished). It makes this behaviour consistent even when serialization isn't possible.
2. Due to the fact the context cache is hit within a transaction, you can now Put, then Get an entity and it
will return its current state (as the transaction would see it), rather than the state at the beginning of the
transaction. This behaviour is nicer than the default.
"""
entity_data = {
"field1": "Apple",
"comb1": 1,
"comb2": "Cherry"
}
original = CachingTestModel.objects.create(**entity_data)
with transaction.atomic():
with transaction.atomic(independent=True):
inner = CachingTestModel.objects.get(pk=original.pk)
inner.field1 = "Banana"
inner.save()
outer = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual("Banana", outer.field1)
outer.field1 = "Apple"
outer.save()
original = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual("Apple", original.field1)
def test_independent_transaction_applies_to_outer_context(self):
"""
When a transaction commits successfully, we can apply its cache to the outer stack. This
alters the behaviour of transactions a little but in a positive way. Things that change are:
1. If you run an independent transaction inside another transaction, a subsequent Get for an entity
updated there will return the updated instance from the cache. Due to serialization of transactions
it's possible that this would have happened anyway (the outer transaction wouldn't start until the independent
one had finished). It makes this behaviour consistent even when serialization isn't possible.
2. Due to the fact the context cache is hit within a transaction, you can now Put, then Get an entity and it
will return its current state (as the transaction would see it), rather than the state at the beginning of the
transaction. This behaviour is nicer than the default.
"""
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
original = CachingTestModel.objects.create(**entity_data)
with transaction.atomic():
with transaction.atomic(independent=True):
inner = CachingTestModel.objects.get(pk=original.pk)
inner.field1 = "Banana"
inner.save()
outer = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual("Banana", outer.field1)
outer.field1 = "Apple"
outer.save()
original = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual("Apple", original.field1)
def test_repeated_usage_in_a_loop(self):
pk = TestUser.objects.create(username="******").pk
for i in xrange(4):
with transaction.atomic(xg=True):
TestUser.objects.get(pk=pk)
continue
with transaction.atomic(xg=True):
TestUser.objects.get(pk=pk)
def test_repeated_usage_in_a_loop(self):
pk = TestUser.objects.create(username="******").pk
for i in xrange(4):
with transaction.atomic(xg=True):
TestUser.objects.get(pk=pk)
continue
with transaction.atomic(xg=True):
TestUser.objects.get(pk=pk)
def test_non_atomic_context_manager(self):
from .test_connector import TestUser
existing = TestUser.objects.create(username="******",
field2="exists")
with transaction.atomic():
self.assertTrue(transaction.in_atomic_block())
user = TestUser.objects.create(username="******", field2="bar")
with transaction.non_atomic():
# We're outside the transaction, so the user should not exist
self.assertRaises(TestUser.DoesNotExist,
TestUser.objects.get,
pk=user.pk)
self.assertFalse(transaction.in_atomic_block())
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
TestUser.objects.get(
pk=existing.pk
) #Should hit the cache, not the datastore
self.assertFalse(datastore_get.called)
with transaction.atomic(independent=True):
user2 = TestUser.objects.create(username="******", field2="bar2")
self.assertTrue(transaction.in_atomic_block())
with transaction.non_atomic():
self.assertFalse(transaction.in_atomic_block())
self.assertRaises(TestUser.DoesNotExist,
TestUser.objects.get,
pk=user2.pk)
with transaction.non_atomic():
self.assertFalse(transaction.in_atomic_block())
self.assertRaises(TestUser.DoesNotExist,
TestUser.objects.get,
pk=user2.pk)
with sleuth.watch("google.appengine.api.datastore.Get"
) as datastore_get:
TestUser.objects.get(
pk=existing.pk
) #Should hit the cache, not the datastore
self.assertFalse(transaction.in_atomic_block())
self.assertRaises(TestUser.DoesNotExist,
TestUser.objects.get,
pk=user2.pk)
self.assertTrue(TestUser.objects.filter(pk=user2.pk).exists())
self.assertTrue(transaction.in_atomic_block())
def test_nested_transaction_doesnt_apply_to_outer_context(self):
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
original = CachingTestModel.objects.create(**entity_data)
with transaction.atomic():
with transaction.atomic(independent=True):
inner = CachingTestModel.objects.get(pk=original.pk)
inner.field1 = "Banana"
inner.save()
outer = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual("Apple", outer.field1)
original = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual("Banana", original.field1)
def increment(self):
idx = random.randint(0, len(self) - 1)
with transaction.atomic():
shard = CounterShard.objects.get(pk=self[idx])
shard.count += 1
shard.save()
def test_consistent_read_updates_cache_outside_transaction(self):
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
original = CachingTestModel.objects.create(**entity_data)
clear_context_cache()
CachingTestModel.objects.get(pk=original.pk) # Should update the cache
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk)
self.assertFalse(datastore_get.called)
clear_context_cache()
with transaction.atomic():
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(
pk=original.pk) # Should *not* update the cache
self.assertTrue(datastore_get.called)
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk)
self.assertTrue(datastore_get.called)
def test_consistent_read_updates_cache_outside_transaction(self):
"""
A read inside a transaction shouldn't update the context cache outside that
transaction
"""
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
original = CachingTestModel.objects.create(**entity_data)
caching.get_context().reset(keep_disabled_flags=True)
CachingTestModel.objects.get(pk=original.pk) # Should update the cache
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk)
self.assertFalse(datastore_get.called)
caching.get_context().reset(keep_disabled_flags=True)
with transaction.atomic():
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(
pk=original.pk) # Should *not* update the cache
self.assertTrue(datastore_get.called)
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk)
self.assertTrue(datastore_get.called)
def refresh_tokens(self):
"""
Refreshes self.access_token and updates self.token_expiry.
Returns True if refresh was successful, False otherwise.
"""
lock_id = "refresh_tokens:%s" % self.pk
with lock(lock_id, steal_after_ms=6000):
self.refresh_from_db()
if self._oauth_tokens_are_valid(): # pragma: no cover
return
credentials = OAuth2Credentials(
self.access_token, settings.GOOGLE_OAUTH2_CLIENT_ID,
settings.GOOGLE_OAUTH2_CLIENT_SECRET, self.refresh_token,
self.token_expiry,
'https://accounts.google.com/o/oauth2/token',
'YouJudge Client')
# Do a refresh - this should update access_token and token_expiry
try:
credentials.refresh(httplib2.Http(proxy_info=None, timeout=5))
except AccessTokenRefreshError as e:
logging.error("Couldn't refresh oauth tokens for user {}. "
"Error was {}".format(self, e))
return
with transaction.atomic():
self.refresh_from_db()
if credentials.access_token != self.access_token:
self.access_token = credentials.access_token
self.token_expiry = credentials.token_expiry.replace(
tzinfo=pytz.UTC)
self.save()
def increment(self):
idx = random.randint(0, len(self) - 1)
with transaction.atomic():
shard = CounterShard.objects.get(pk=self[idx])
shard.count += 1
shard.save()
def save(self, *args, **kwargs):
assert self.text
assert self.language_code
# Generate the appropriate key on creation
if self._state.adding:
self.pk = MasterTranslation.generate_key(
self.text, self.hint, self.language_code
)
# If there was no plural text specified, just use the default text
if not self.plural_text:
self.plural_text = self.text
# If we are adding for the first time, then create a counterpart
# translation for the master language
if self._state.adding:
with transaction.atomic(xg=True):
new_trans = Translation.objects.create(
master_translation_id=self.pk,
language_code=self.language_code,
text=self.text,
denorm_master_text=self.text,
denorm_master_hint=self.hint
)
self.translations_by_language_code[self.language_code] = new_trans.pk
self.translations.add(new_trans)
return super(MasterTranslation, self).save(*args, **kwargs)
else:
# Otherwise just do a normal save
return super(MasterTranslation, self).save(*args, **kwargs)
def test_save_inside_transaction_evicts_cache(self):
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
identifiers = caching._apply_namespace(
unique_utils.unique_identifiers_from_entity(
CachingTestModel, FakeEntity(entity_data, id=222)),
DEFAULT_NAMESPACE,
)
instance = CachingTestModel.objects.create(id=222, **entity_data)
instance.refresh_from_db() # Adds to memcache (consistent Get)
for identifier in identifiers:
self.assertEqual(entity_data, cache.get(identifier))
with transaction.atomic():
instance.field1 = "Banana"
instance.save()
# Make sure that altering inside the transaction evicted the item from the cache
# and that a get then hits the datastore (which then in turn caches)
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
self.assertEqual(
"Banana",
CachingTestModel.objects.get(pk=instance.pk).field1)
self.assertTrue(datastore_get.called)
def test_save_caches_outside_transaction_only(self):
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
identifiers = caching._apply_namespace(
unique_utils.unique_identifiers_from_entity(
CachingTestModel, FakeEntity(entity_data, id=222)),
DEFAULT_NAMESPACE,
)
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
instance = CachingTestModel.objects.create(id=222, **entity_data)
instance.refresh_from_db()
for identifier in identifiers:
self.assertEqual(entity_data, cache.get(identifier))
instance.delete()
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
with transaction.atomic():
instance = CachingTestModel.objects.create(**entity_data)
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
def unindex_instance(instance):
indexes = Index.objects.filter(instance_db_table=instance._meta.db_table, instance_pk=instance.pk).all()
for index in indexes:
try:
while True:
try:
with transaction.atomic(xg=True):
try:
index = Index.objects.get(pk=index.pk)
except Index.DoesNotExist:
return
count = GlobalOccuranceCount.objects.get(pk=index.iexact)
count.count -= index.occurances
count.save()
index.delete()
if count.count < 0:
logging.error("The GOC of {} was negative ({}) while unindexing {}", count.pk, count.count, index.pk)
break
except transaction.TransactionFailedError:
logging.warning("Transaction collision, retrying!")
time.sleep(1)
continue
except GlobalOccuranceCount.DoesNotExist:
logging.warning("A GlobalOccuranceCount for Index: %s does not exist, ignoring", index.pk)
continue
def populate(self):
""" Create all the CounterShard objects which will be used by this field. Useful to prevent
additional saves being caused when you call increment() or decrement() due to having to
update the list of shard PKs on the instance.
"""
total_to_create = self.field.shard_count - len(self)
while total_to_create:
with transaction.atomic(xg=True):
# We must re-fetch the instance to ensure that we do this atomically, but we must
# also update self.instance so that the calling code which is referencing
# self.instance also gets the updated list of shard PKs
new_instance = self.instance._default_manager.get(
pk=self.instance.pk)
new_instance_shard_pks = getattr(new_instance,
self.field.attname, set())
# Re-check / update the number to create based on the refreshed instance from the DB
total_to_create = self.field.shard_count - len(
new_instance_shard_pks)
num_to_create = min(total_to_create,
MAX_SHARDS_PER_TRANSACTION)
new_shard_pks = set()
for x in xrange(num_to_create):
new_shard_pks.add(self._create_shard(count=0).pk)
new_instance_shard_pks.update(new_shard_pks)
setattr(self.instance, self.field.attname,
new_instance_shard_pks)
models.Model.save(
new_instance
) # avoid custom save method, which might do DB lookups
total_to_create -= num_to_create
def test_save_caches_outside_transaction_only(self):
entity_data = {
"field1": "Apple",
"comb1": 1,
"comb2": "Cherry"
}
identifiers = unique_utils.unique_identifiers_from_entity(CachingTestModel, FakeEntity(entity_data, id=222))
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
instance = CachingTestModel.objects.create(id=222, **entity_data)
for identifier in identifiers:
self.assertEqual(entity_data, cache.get(identifier))
instance.delete()
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
with transaction.atomic():
instance = CachingTestModel.objects.create(**entity_data)
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
def test_save_inside_transaction_evicts_cache(self):
entity_data = {
"field1": "Apple",
"comb1": 1,
"comb2": "Cherry"
}
identifiers = caching._apply_namespace(
unique_utils.unique_identifiers_from_entity(CachingTestModel, FakeEntity(entity_data, id=222)),
DEFAULT_NAMESPACE,
)
instance = CachingTestModel.objects.create(id=222, **entity_data)
for identifier in identifiers:
self.assertEqual(entity_data, cache.get(identifier))
with transaction.atomic():
instance.field1 = "Banana"
instance.save()
# Make sure that altering inside the transaction evicted the item from the cache
# and that a get then hits the datastore (which then in turn caches)
with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get:
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
self.assertEqual("Banana", CachingTestModel.objects.get(pk=instance.pk).field1)
self.assertTrue(datastore_get.called)
def save(self, *args, **kwargs):
if not isinstance(self.amount_paid, Decimal):
self.amount_paid = Decimal(self.amount_paid or "0")
if not isinstance(self.amount_left, Decimal):
self.amount_left = Decimal(self.amount_left or "0")
with transaction.atomic(xg=True):
delta = 0
if self.pk:
old_state = Person.objects.get(pk=self.pk)
delta = self.amount_paid - Decimal(old_state.amount_paid
or "0")
else:
delta = self.amount_paid
super(Person, self).save(*args, **kwargs)
if delta > 0:
Receipt.objects.create(user=self.user, person=self, amount=delta)
try:
dc = DentalChart.objects.get(person_id=self.pk)
except DentalChart.DoesNotExist:
dc = DentalChart(person_id=self.pk)
dc.dental_chart_type = self.dental_chart_type
dc.dental_chart = self.dental_chart
dc.save()
def test_save_wipes_entity_from_cache_inside_transaction(self):
entity_data = {
"field1": "Apple",
"comb1": 1,
"comb2": "Cherry"
}
identifiers = caching._apply_namespace(
unique_utils.unique_identifiers_from_entity(CachingTestModel, FakeEntity(entity_data, id=222)),
DEFAULT_NAMESPACE,
)
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
instance = CachingTestModel.objects.create(id=222, **entity_data)
for identifier in identifiers:
self.assertEqual(entity_data, cache.get(identifier))
with transaction.atomic():
instance.save()
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
def test_consistent_read_updates_cache_outside_transaction(self):
"""
A read inside a transaction shouldn't update the context cache outside that
transaction
"""
entity_data = {
"field1": "Apple",
"comb1": 1,
"comb2": "Cherry"
}
original = CachingTestModel.objects.create(**entity_data)
clear_context_cache()
CachingTestModel.objects.get(pk=original.pk) # Should update the cache
with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk)
self.assertFalse(datastore_get.called)
clear_context_cache()
with transaction.atomic():
with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk) # Should *not* update the cache
self.assertTrue(datastore_get.called)
with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=original.pk)
self.assertTrue(datastore_get.called)
def test_transactional_save_wipes_the_cache_only_after_its_result_is_consistently_available(
self):
entity_data = {
"field1": "old",
}
identifiers = caching._apply_namespace(
unique_utils.unique_identifiers_from_entity(
CachingTestModel, FakeEntity(entity_data, id=222)),
DEFAULT_NAMESPACE,
)
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
instance = CachingTestModel.objects.create(id=222, **entity_data)
instance.refresh_from_db() # Add to memcache (consistent Get)
for identifier in identifiers:
self.assertEqual("old", cache.get(identifier)["field1"])
@non_transactional
def non_transactional_read(instance_pk):
CachingTestModel.objects.get(pk=instance_pk)
with transaction.atomic():
instance.field1 = "new"
instance.save()
non_transactional_read(
instance.pk) # could potentially recache the old object
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
def test_transactional_save_wipes_the_cache_only_after_its_result_is_consistently_available(self):
entity_data = {
"field1": "old",
}
identifiers = caching._apply_namespace(
unique_utils.unique_identifiers_from_entity(
CachingTestModel, FakeEntity(entity_data, id=222)
),
DEFAULT_NAMESPACE,
)
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
instance = CachingTestModel.objects.create(id=222, **entity_data)
for identifier in identifiers:
self.assertEqual("old", cache.get(identifier)["field1"])
@non_transactional
def non_transactional_read(instance_pk):
CachingTestModel.objects.get(pk=instance_pk)
with transaction.atomic():
instance.field1 = "new"
instance.save()
non_transactional_read(instance.pk) # could potentially recache the old object
for identifier in identifiers:
self.assertIsNone(cache.get(identifier))
def populate(self):
""" Create all the CounterShard objects which will be used by this field. Useful to prevent
additional saves being caused when you call increment() or decrement() due to having to
update the list of shard PKs on the instance.
"""
total_to_create = self.field.shard_count - len(self)
while total_to_create:
with transaction.atomic(xg=True):
# We must re-fetch the instance to ensure that we do this atomically, but we must
# also update self.instance so that the calling code which is referencing
# self.instance also gets the updated list of shard PKs
new_instance = self.instance.__class__._default_manager.get(pk=self.instance.pk)
new_instance_shard_pks = getattr(new_instance, self.field.attname, set())
# Re-check / update the number to create based on the refreshed instance from the DB
total_to_create = self.field.shard_count - len(new_instance_shard_pks)
num_to_create = min(total_to_create, MAX_SHARDS_PER_TRANSACTION)
new_shard_pks = set()
for x in range(num_to_create):
new_shard_pks.add(self._create_shard(count=0).pk)
new_instance_shard_pks.update(new_shard_pks)
setattr(self.instance, self.field.attname, new_instance_shard_pks)
models.Model.save(new_instance) # avoid custom save method, which might do DB lookups
total_to_create -= num_to_create
def callback(instance, text, term):
with transaction.atomic(xg=True):
term_count = text.lower().count(term)
index_pk = InstanceIndex.calc_id(term, instance)
try:
index = InstanceIndex.objects.get(pk=index_pk)
if index.count == term_count:
return
else:
# OK, something weird happened and we indexed twice
# we need to just need to figure out what difference we need to add to
# the global count!
term_count = term_count - index.count
except InstanceIndex.DoesNotExist:
index = InstanceIndex.objects.create(
pk=index_pk,
**{
"count": term_count,
"iexact": term,
"instance_db_table": instance._meta.db_table,
"instance_pk": instance.pk,
})
counter, created = TermCount.objects.get_or_create(pk=term)
counter.count += term_count
counter.save()
def callback(instance, text, term):
with transaction.atomic(xg=True):
term_count = text.lower().count(term)
index_pk = InstanceIndex.calc_id(term, instance)
try:
index = InstanceIndex.objects.get(pk=index_pk)
if index.count == term_count:
return
else:
# OK, something weird happened and we indexed twice
# we need to just need to figure out what difference we need to add to
# the global count!
term_count = term_count - index.count
except InstanceIndex.DoesNotExist:
index = InstanceIndex.objects.create(
pk=index_pk,
**{
"count": term_count,
"iexact": term,
"instance_db_table": instance._meta.db_table,
"instance_pk": instance.pk,
}
)
counter, created = TermCount.objects.get_or_create(
pk=term
)
counter.count += term_count
counter.save()
def test_nested_transactions_dont_get_their_own_context(self):
"""
The datastore doesn't support nested transactions, so when there is a nested
atomic block which isn't marked as independent, the atomic is a no-op. Therefore
we shouldn't push a context here, and we shouldn't pop it at the end either.
"""
context = caching.get_context()
self.assertEqual(1, context.stack.size)
with transaction.atomic():
self.assertEqual(2, context.stack.size)
with transaction.atomic():
self.assertEqual(2, context.stack.size)
with transaction.atomic():
self.assertEqual(2, context.stack.size)
self.assertEqual(2, context.stack.size)
self.assertEqual(2, context.stack.size)
self.assertEqual(1, context.stack.size)
def reset(self):
""" Reset the counter to 0. """
with transaction.atomic(xg=True):
value = self.value()
if value > 0:
self.decrement(value)
elif value < 0:
self.increment(value)
def reset(self):
""" Reset the counter to 0. """
with transaction.atomic(xg=True):
value = self.value()
if value > 0:
self.decrement(value)
elif value < 0:
self.increment(value)
def delete(self, *args, **kwargs):
""" Ensure that the count field on the Speech is decremented when this is deleted. """
with transaction.atomic(xg=True):
speech = Speech.objects.get(pk=self.speech_id) # reload inside transaction
value = getattr(speech, self.speech_model_count_field)
setattr(speech, self.speech_model_count_field, value - 1)
speech.save()
return super(FlagBaseModel, self).delete(*args, **kwargs)
def test_interaction_with_datastore_txn(self):
from google.appengine.ext import db
from google.appengine.datastore.datastore_rpc import TransactionOptions
from .test_connector import TestUser
@db.transactional(propagation=TransactionOptions.INDEPENDENT)
def some_indie_txn(_username):
TestUser.objects.create(username=_username)
@db.transactional()
def some_non_indie_txn(_username):
TestUser.objects.create(username=_username)
@db.transactional()
def double_nested_transactional():
@db.transactional(propagation=TransactionOptions.INDEPENDENT)
def do_stuff():
TestUser.objects.create(username="******")
raise ValueError()
try:
return do_stuff
except:
return
with transaction.atomic():
double_nested_transactional()
@db.transactional()
def something_containing_atomic():
with transaction.atomic():
TestUser.objects.create(username="******")
something_containing_atomic()
with transaction.atomic():
with transaction.atomic():
some_non_indie_txn("Bob1")
some_indie_txn("Bob2")
some_indie_txn("Bob3")
with transaction.atomic(independent=True):
some_non_indie_txn("Fred1")
some_indie_txn("Fred2")
some_indie_txn("Fred3")
def test_transactions_get_their_own_context(self):
with sleuth.watch(
"djangae.db.backends.appengine.context.ContextStack.push"
) as context_push:
with transaction.atomic():
pass
self.assertTrue(context_push.called)
def test_nested_transactions_dont_get_their_own_context(self):
"""
The datastore doesn't support nested transactions, so when there is a nested
atomic block which isn't marked as independent, the atomic is a no-op. Therefore
we shouldn't push a context here, and we shouldn't pop it at the end either.
"""
self.assertEqual(1, caching._context.stack.size)
with transaction.atomic():
self.assertEqual(2, caching._context.stack.size)
with transaction.atomic():
self.assertEqual(2, caching._context.stack.size)
with transaction.atomic():
self.assertEqual(2, caching._context.stack.size)
self.assertEqual(2, caching._context.stack.size)
self.assertEqual(2, caching._context.stack.size)
self.assertEqual(1, caching._context.stack.size)
def test_atomic_context_manager(self):
with self.assertRaises(ValueError):
with transaction.atomic():
TestUser.objects.create(username="******", field2="bar")
raise ValueError()
self.assertEqual(0, TestUser.objects.count())
def test_atomic_context_manager(self):
with self.assertRaises(ValueError):
with transaction.atomic():
TestUser.objects.create(username="******", field2="bar")
raise ValueError()
self.assertEqual(0, TestUser.objects.count())
def test_interaction_with_datastore_txn(self):
from google.appengine.ext import db
from google.appengine.datastore.datastore_rpc import TransactionOptions
from .test_connector import TestUser
@db.transactional(propagation=TransactionOptions.INDEPENDENT)
def some_indie_txn(_username):
TestUser.objects.create(username=_username)
@db.transactional()
def some_non_indie_txn(_username):
TestUser.objects.create(username=_username)
@db.transactional()
def double_nested_transactional():
@db.transactional(propagation=TransactionOptions.INDEPENDENT)
def do_stuff():
TestUser.objects.create(username="******")
raise ValueError()
try:
return do_stuff
except Exception:
return
with transaction.atomic():
double_nested_transactional()
@db.transactional()
def something_containing_atomic():
with transaction.atomic():
TestUser.objects.create(username="******")
something_containing_atomic()
with transaction.atomic():
with transaction.atomic():
some_non_indie_txn("Bob1")
some_indie_txn("Bob2")
some_indie_txn("Bob3")
with transaction.atomic(independent=True):
some_non_indie_txn("Fred1")
some_indie_txn("Fred2")
some_indie_txn("Fred3")
def test_transactions_clear_the_context_cache(self):
UniqueModel.objects.create(unique_field="test") #Create an instance
with transaction.atomic():
self.assertFalse(caching.context.cache)
UniqueModel.objects.create(unique_field="test2", unique_combo_one=1) #Create an instance
self.assertTrue(caching.context.cache)
self.assertFalse(caching.context.cache)
def begin_scan(marshall):
""" Trigger tasks to scan template files and python files for translatable strings and to
create corresponding MasterTranslation objects for them.
"""
try:
marshall.refresh_from_db()
except ScanMarshall.DoesNotExist:
logger.warn("Not starting scan as scanmarshall was missing")
return
scan_id = uuid.uuid4()
files_to_scan = []
def walk_dir(root, dirs, files):
for f in files:
filename = os.path.normpath(os.path.join(root, f))
if os.path.splitext(filename)[1] not in (".py", ".html"):
continue
files_to_scan.append(filename)
for app in settings.INSTALLED_APPS:
module_path = os.path.dirname(
apps.get_app_config(app.split(".")[-1]).module.__file__)
for root, dirs, files in os.walk(module_path, followlinks=True):
walk_dir(root, dirs, files)
# Scan the django directory
for root, dirs, files in os.walk(os.path.dirname(django.__file__),
followlinks=True):
walk_dir(root, dirs, files)
# Update the ScanMarshall object with the total number of files to scan. We must do this
# before we defer any of the tasks, as if we go for a gradual increment(), defer(),
# incrememnt(), defer() approach, then the first task could finish and decrement
# `files_left_to_process` back to 0 (thereby marking the scan as done) before we've deferred
# the second task.
# We can't defer the tasks inside the transaction because App Engine only lets us defer a max
# of 5 tasks inside a single transaction.
with transaction.atomic(xg=True):
marshall.refresh_from_db()
marshall.files_left_to_process += len(files_to_scan)
marshall.save()
for offset in xrange(0, len(files_to_scan), 100):
files = files_to_scan[offset:offset + 100]
# Defer with a random delay of between 0 and 10 seconds, just to avoid transaction
# collisions caused by the tasks all finishing and updating the marshall at the same time.
defer(_scan_list,
marshall,
scan_id,
files,
_countdown=random.randint(0, 10))
logger.info("Deferred tasks to scan %d files", len(files_to_scan))
def delete(self, *args, **kwargs):
""" Ensure that the count field on the Speech is decremented when this is deleted. """
with transaction.atomic(xg=True):
speech = Speech.objects.get(
pk=self.speech_id) # reload inside transaction
value = getattr(speech, self.speech_model_count_field)
setattr(speech, self.speech_model_count_field, value - 1)
speech.save()
return super(FlagBaseModel, self).delete(*args, **kwargs)
def _scan_list(marshall, scan_id, filenames):
# FIXME: Need to clean up the translations which aren't in use anymore
for filename in filenames:
# Redeploying to a new version can cause this
if not os.path.exists(filename):
continue
with open(filename) as f:
content = f.read()
results = parse_file(content, os.path.splitext(filename)[-1])
for text, plural, hint, group in results:
if not text:
logger.warn("Empty translation discovered: '{}', '{}', '{}', '{}'".format(text, plural, hint, group))
continue
with transaction.atomic(xg=True):
key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE)
try:
mt = MasterTranslation.objects.get(pk=key)
except MasterTranslation.DoesNotExist:
mt = MasterTranslation(
pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE
)
# By the very act of getting here, this is true
mt.used_in_code_or_templates = True
# If we last updated during this scan, then append, otherwise replace
if mt.last_updated_by_scan_uuid == scan_id:
mt.used_by_groups_in_code_or_templates.add(group)
else:
mt.used_by_groups_in_code_or_templates = { group }
mt.last_updated_by_scan_uuid = scan_id
mt.save()
with transaction.atomic():
marshall.refresh_from_db()
marshall.files_left_to_process -= len(filenames)
marshall.save()
def _scan_list(marshall, scan_id, filenames):
# FIXME: Need to clean up the translations which aren't in use anymore
for filename in filenames:
# Redeploying to a new version can cause this
if not os.path.exists(filename):
continue
with open(filename) as f:
content = unicode(f.read(), settings.DEFAULT_CHARSET)
results = parse_file(content, os.path.splitext(filename)[-1])
for text, plural, hint, group in results:
if not text:
logger.warn("Empty translation discovered: '{}', '{}', '{}', '{}'".format(text, plural, hint, group))
continue
with transaction.atomic(xg=True):
key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE)
try:
mt = MasterTranslation.objects.get(pk=key)
except MasterTranslation.DoesNotExist:
mt = MasterTranslation(
pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE
)
# By the very act of getting here, this is true
mt.used_in_code_or_templates = True
# If we last updated during this scan, then append, otherwise replace
if mt.last_updated_by_scan_uuid == scan_id:
mt.used_by_groups_in_code_or_templates.add(group)
else:
mt.used_by_groups_in_code_or_templates = { group }
mt.last_updated_by_scan_uuid = scan_id
mt.save()
with transaction.atomic():
marshall.refresh_from_db()
marshall.files_left_to_process -= len(filenames)
marshall.save()
def test_nested_rollback_doesnt_apply_on_outer_commit(self):
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
original = CachingTestModel.objects.create(**entity_data)
with transaction.atomic():
try:
with transaction.atomic(independent=True):
inner = CachingTestModel.objects.get(pk=original.pk)
inner.field1 = "Banana"
inner.save()
raise ValueError() # Will rollback the transaction
except ValueError:
pass
outer = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual("Apple", outer.field1)
original = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual("Apple", original.field1) # Shouldn't have changed
def append_file(filename, files_to_scan):
files_to_scan.append(filename)
if len(files_to_scan) == 100:
with transaction.atomic(xg=True):
marshall.refresh_from_db()
marshall.files_left_to_process += 100
marshall.save()
defer(_scan_list, marshall, scan_id, files_to_scan, _transactional=True)
files_to_scan = []
def append_file(filename, files_to_scan):
files_to_scan.append(filename)
if len(files_to_scan) == 100:
with transaction.atomic(xg=True):
marshall.refresh_from_db()
marshall.files_left_to_process += 100
marshall.save()
defer(_scan_list, marshall, scan_id, files_to_scan, _transactional=True)
files_to_scan = []
def test_nested_transaction_doesnt_apply_to_outer_context(self):
entity_data = {
"field1": "Apple",
"comb1": 1,
"comb2": "Cherry"
}
original = CachingTestModel.objects.create(**entity_data)
with transaction.atomic():
with transaction.atomic(independent=True):
inner = CachingTestModel.objects.get(pk=original.pk)
inner.field1 = "Banana"
inner.save()
outer = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual("Apple", outer.field1)
original = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual("Banana", original.field1)
def test_outermost_transaction_applies_all_contexts_on_commit(self):
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
with transaction.atomic():
with transaction.atomic(independent=True):
instance = CachingTestModel.objects.create(**entity_data)
# At this point the instance should be unretrievable, even though we just created it
try:
CachingTestModel.objects.get(pk=instance.pk)
self.fail("Unexpectedly was able to retrieve instance")
except CachingTestModel.DoesNotExist:
pass
# Should now exist in the cache
with sleuth.switch(
"google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=instance.pk)
self.assertFalse(datastore_get.called)
def test_nested_rollback_doesnt_apply_on_outer_commit(self):
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
original = CachingTestModel.objects.create(**entity_data)
with transaction.atomic():
try:
with transaction.atomic(independent=True):
inner = CachingTestModel.objects.get(pk=original.pk)
inner.field1 = "Banana"
inner.save()
raise ValueError() # Will rollback the transaction
except ValueError:
pass
outer = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual("Apple", outer.field1)
original = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual("Apple", original.field1) # Shouldn't have changed
def test_refresh_if_unread_for_created_objects(self):
""" refresh_if_unread should not refresh objects which have been *created* within the
transaction, as at the DB level they will not exist.
"""
from .test_connector import TestFruit
# With caching
with transaction.atomic() as txn:
apple = TestFruit.objects.create(name="Apple", color="Red")
apple.color = "Pink" # Deliberately don't save
txn.refresh_if_unread(apple)
self.assertEqual(apple.color, "Pink")
# Without caching
with DisableCache():
with transaction.atomic() as txn:
apple = TestFruit.objects.create(name="Radish", color="Red")
apple.color = "Pink" # Deliberately don't save
txn.refresh_if_unread(apple)
self.assertEqual(apple.color, "Pink")
def test_get_by_key_hits_datastore_inside_transaction(self):
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
original = CachingTestModel.objects.create(**entity_data)
with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get:
with transaction.atomic():
instance = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual(original, instance)
self.assertTrue(datastore_get.called)
def test_refresh_if_unread_for_created_objects(self):
""" refresh_if_unread should not refresh objects which have been *created* within the
transaction, as at the DB level they will not exist.
"""
from .test_connector import TestFruit
# With caching
with transaction.atomic() as txn:
apple = TestFruit.objects.create(name="Apple", color="Red")
apple.color = "Pink" # Deliberately don't save
txn.refresh_if_unread(apple)
self.assertEqual(apple.color, "Pink")
# Without caching
with DisableCache():
with transaction.atomic() as txn:
apple = TestFruit.objects.create(name="Radish", color="Red")
apple.color = "Pink" # Deliberately don't save
txn.refresh_if_unread(apple)
self.assertEqual(apple.color, "Pink")
def test_get_by_key_hits_datastore_inside_transaction(self):
entity_data = {"field1": "Apple", "comb1": 1, "comb2": "Cherry"}
original = CachingTestModel.objects.create(**entity_data)
with sleuth.watch(
"google.appengine.api.datastore.Get") as datastore_get:
with transaction.atomic():
instance = CachingTestModel.objects.get(pk=original.pk)
self.assertEqual(original, instance)
self.assertTrue(datastore_get.called)
def test_prevent_read(self):
from .test_connector import TestFruit
apple = TestFruit.objects.create(name="Apple", color="Red")
# Don't allow reading apple within the transaction
with transaction.atomic() as txn:
txn.prevent_read(TestFruit, apple.pk)
self.assertRaises(transaction.PreventedReadError,
TestFruit.objects.get,
pk=apple.pk)
def callback(instance, text, term):
try:
with transaction.atomic(xg=True):
index = InstanceIndex.objects.get(pk=InstanceIndex.calc_id(term, instance))
counter = TermCount.objects.get(pk=term)
counter.count -= index.count
counter.save()
index.delete()
except (InstanceIndex.DoesNotExist, TermCount.DoesNotExist):
pass
def test_non_atomic_context_manager(self):
from .test_connector import TestUser
existing = TestUser.objects.create(username="******", field2="exists")
with transaction.atomic():
self.assertTrue(transaction.in_atomic_block())
user = TestUser.objects.create(username="******", field2="bar")
with transaction.non_atomic():
# We're outside the transaction, so the user should not exist
self.assertRaises(TestUser.DoesNotExist, TestUser.objects.get, pk=user.pk)
self.assertFalse(transaction.in_atomic_block())
with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get:
TestUser.objects.get(pk=existing.pk) # Should hit the cache, not the datastore
self.assertFalse(datastore_get.called)
with transaction.atomic(independent=True):
user2 = TestUser.objects.create(username="******", field2="bar2")
self.assertTrue(transaction.in_atomic_block())
with transaction.non_atomic():
self.assertFalse(transaction.in_atomic_block())
self.assertRaises(TestUser.DoesNotExist, TestUser.objects.get, pk=user2.pk)
with transaction.non_atomic():
self.assertFalse(transaction.in_atomic_block())
self.assertRaises(TestUser.DoesNotExist, TestUser.objects.get, pk=user2.pk)
with sleuth.watch("google.appengine.api.datastore.Get") as datastore_get:
# Should hit the cache, not the Datastore
TestUser.objects.get(pk=existing.pk)
self.assertFalse(transaction.in_atomic_block())
self.assertRaises(TestUser.DoesNotExist, TestUser.objects.get, pk=user2.pk)
self.assertTrue(TestUser.objects.filter(pk=user2.pk).exists())
self.assertTrue(transaction.in_atomic_block())
def test_outermost_transaction_applies_all_contexts_on_commit(self):
entity_data = {
"field1": "Apple",
"comb1": 1,
"comb2": "Cherry"
}
with transaction.atomic():
with transaction.atomic(independent=True):
instance = CachingTestModel.objects.create(**entity_data)
# At this point the instance should be unretrievable, even though we just created it
try:
CachingTestModel.objects.get(pk=instance.pk)
self.fail("Unexpectedly was able to retrieve instance")
except CachingTestModel.DoesNotExist:
pass
# Should now exist in the cache
with sleuth.switch("google.appengine.api.datastore.Get") as datastore_get:
CachingTestModel.objects.get(pk=instance.pk)
self.assertFalse(datastore_get.called)
def test_refresh_if_unread_for_resaved_objects(self):
""" refresh_if_unread should not refresh objects which have been re-saved within the
transaction.
"""
from .test_connector import TestFruit
# With caching
apple = TestFruit.objects.create(name="Apple", color="Red")
with transaction.atomic() as txn:
apple.save()
apple.color = "Pink" # Deliberately don't save
txn.refresh_if_unread(apple)
self.assertEqual(apple.color, "Pink")
# Without caching
radish = TestFruit.objects.create(name="Radish", color="Red")
with DisableCache():
with transaction.atomic() as txn:
radish.save()
radish.color = "Pink" # Deliberately don't save
txn.refresh_if_unread(radish)
self.assertEqual(radish.color, "Pink")
def decrement(self):
# Find a non-empty shard and decrement it
shards = self[:]
random.shuffle(shards)
for shard_id in shards:
with transaction.atomic():
shard = CounterShard.objects.get(pk=shard_id)
if not shard.count:
continue
else:
shard.count -= 1
shard.save()
break
def callback(instance, text, term):
try:
with transaction.atomic(xg=True):
index = InstanceIndex.objects.get(
pk=InstanceIndex.calc_id(term, instance))
counter = TermCount.objects.get(pk=term)
counter.count -= index.count
counter.save()
index.delete()
except (InstanceIndex.DoesNotExist, TermCount.DoesNotExist):
pass