def test_thread_locality(self):
store = localstore.LocalStore()
store['name'] = "Hi"
q = Queue()
def do_test():
sleep(0.1)
t = current_thread()
name = t.name
store[name] = 1
store['name'] = name
q.put(dict(store))
threads = []
for x in range(5):
t = Thread(target=do_test, name='thread%x' % x)
t.start()
threads.append(t)
for t in threads:
t.join()
# assert none of the thread stuff touched our queue
self.assertEqual(store['name'], 'Hi')
self.assertEqual(q.qsize(), 5)
qcontents = []
while not q.empty():
qcontents.append(q.get())
self.assertEqual(len(qcontents), 5)
for d in qcontents:
self.assertEqual(len(d), 2)
self.assertNotEqual(d['name'], 'Hi')
self.assertEqual(d[d['name']], 1)
def test_thread_locality(self):
store = localstore.LocalStore()
store['name'] = "Hi"
q = Queue()
def do_test():
sleep(0.1)
t = current_thread()
name = t.name
store[name] = 1
store['name'] = name
q.put(dict(store))
threads = []
for x in range(5):
t = Thread(target=do_test, name='thread%x' % x)
t.start()
threads.append(t)
for t in threads:
t.join()
# assert none of the thread stuff touched our queue
self.assertEqual(store['name'], 'Hi')
self.assertEqual(q.qsize(), 5)
qcontents = []
while not q.empty():
qcontents.append(q.get())
self.assertEqual(len(qcontents), 5)
for d in qcontents:
self.assertEqual(len(d), 2)
self.assertNotEqual(d['name'], 'Hi')
self.assertEqual(d[d['name']], 1)
def test_transaction_rollback(self):
"""Tests johnny's handling of transaction rollbacks.
Similar to the commit, this sets up a write to a db in a transaction,
reads from it (to force a cache write of sometime), then rolls back."""
if not is_multithreading_safe(db_using='default'):
print("\n Skipping test requiring multiple threads.")
return
self.assertFalse(is_managed())
self.assertFalse(transaction.is_dirty())
cache.local.clear()
q = Queue()
other = lambda x: self._run_threaded(x, q)
# load some data
start = Genre.objects.get(id=1)
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
# these should be the same and should have hit cache
self.assertTrue(hit)
self.assertEqual(ostart, start)
# enter manual transaction management
transaction.enter_transaction_management()
managed()
start.title = 'Jackie Chan Novels'
# local invalidation, this key should hit the localstore!
nowlen = len(cache.local)
start.save()
self.assertNotEqual(nowlen, len(cache.local))
# perform a read OUTSIDE this transaction... it should still see the
# old gen key, and should still find the "old" data
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
self.assertTrue(hit)
self.assertNotEqual(ostart.title, start.title)
# perform a READ inside the transaction; this should hit the localstore
# but not the outside!
nowlen = len(cache.local)
start2 = Genre.objects.get(id=1)
self.assertEqual(start2.title, start.title)
self.assertTrue(len(cache.local) > nowlen)
transaction.rollback()
# we rollback, and flush all johnny keys related to this transaction
# subsequent gets should STILL hit the cache in the other thread
# and indeed, in this thread.
self.assertFalse(transaction.is_dirty())
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
self.assertTrue(hit)
start = Genre.objects.get(id=1)
self.assertEqual(ostart.title, start.title)
managed(False)
transaction.leave_transaction_management()
def test_transaction_rollback(self):
"""Tests johnny's handling of transaction rollbacks.
Similar to the commit, this sets up a write to a db in a transaction,
reads from it (to force a cache write of sometime), then rolls back."""
if not is_multithreading_safe(db_using='default'):
print("\n Skipping test requiring multiple threads.")
return
self.assertFalse(is_managed())
self.assertFalse(transaction.is_dirty())
cache.local.clear()
q = Queue()
other = lambda x: self._run_threaded(x, q)
# load some data
start = Genre.objects.get(id=1)
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
# these should be the same and should have hit cache
self.assertTrue(hit)
self.assertEqual(ostart, start)
# enter manual transaction management
transaction.enter_transaction_management()
managed()
start.title = 'Jackie Chan Novels'
# local invalidation, this key should hit the localstore!
nowlen = len(cache.local)
start.save()
self.assertNotEqual(nowlen, len(cache.local))
# perform a read OUTSIDE this transaction... it should still see the
# old gen key, and should still find the "old" data
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
self.assertTrue(hit)
self.assertNotEqual(ostart.title, start.title)
# perform a READ inside the transaction; this should hit the localstore
# but not the outside!
nowlen = len(cache.local)
start2 = Genre.objects.get(id=1)
self.assertEqual(start2.title, start.title)
self.assertTrue(len(cache.local) > nowlen)
transaction.rollback()
# we rollback, and flush all johnny keys related to this transaction
# subsequent gets should STILL hit the cache in the other thread
# and indeed, in this thread.
self.assertFalse(transaction.is_dirty())
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
self.assertTrue(hit)
start = Genre.objects.get(id=1)
self.assertEqual(ostart.title, start.title)
managed(False)
transaction.leave_transaction_management()
def test_transaction_commit(self):
"""Test transaction support in Johnny."""
if not is_multithreading_safe(db_using='default'):
print("\n Skipping test requiring multiple threads.")
return
self.assertFalse(is_managed())
self.assertFalse(transaction.is_dirty())
cache.local.clear()
q = Queue()
other = lambda x: self._run_threaded(x, q)
# load some data
start = Genre.objects.get(id=1)
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
# these should be the same and should have hit cache
self.assertTrue(hit)
self.assertEqual(ostart, start)
# enter manual transaction management
transaction.enter_transaction_management()
managed()
start.title = 'Jackie Chan Novels'
# local invalidation, this key should hit the localstore!
nowlen = len(cache.local)
start.save()
self.assertNotEqual(nowlen, len(cache.local))
# perform a read OUTSIDE this transaction... it should still see the
# old gen key, and should still find the "old" data
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
self.assertTrue(hit)
self.assertNotEqual(ostart.title, start.title)
transaction.commit()
# now that we commit, we push the localstore keys out; this should be
# a cache miss, because we never read it inside the previous transaction
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
self.assertFalse(hit)
self.assertEqual(ostart.title, start.title)
managed(False)
transaction.leave_transaction_management()
def test_transaction_commit(self):
"""Test transaction support in Johnny."""
if not is_multithreading_safe(db_using='default'):
print("\n Skipping test requiring multiple threads.")
return
self.assertFalse(is_managed())
self.assertFalse(transaction.is_dirty())
cache.local.clear()
q = Queue()
other = lambda x: self._run_threaded(x, q)
# load some data
start = Genre.objects.get(id=1)
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
# these should be the same and should have hit cache
self.assertTrue(hit)
self.assertEqual(ostart, start)
# enter manual transaction management
transaction.enter_transaction_management()
managed()
start.title = 'Jackie Chan Novels'
# local invalidation, this key should hit the localstore!
nowlen = len(cache.local)
start.save()
self.assertNotEqual(nowlen, len(cache.local))
# perform a read OUTSIDE this transaction... it should still see the
# old gen key, and should still find the "old" data
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
self.assertTrue(hit)
self.assertNotEqual(ostart.title, start.title)
transaction.commit()
# now that we commit, we push the localstore keys out; this should be
# a cache miss, because we never read it inside the previous transaction
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
self.assertFalse(hit)
self.assertEqual(ostart.title, start.title)
managed(False)
transaction.leave_transaction_management()
def test_savepoints(self):
"""tests savepoints for multiple db's"""
q = Queue()
other = lambda x: self._run_threaded(x, q, {'Genre': Genre})
if len(getattr(settings, "DATABASES", [])) <= 1:
print("\n Skipping multi database tests")
return
if not is_multithreading_safe():
print("\n Skipping test requiring multiple threads.")
return
for name, db in settings.DATABASES.items():
con = connections[name]
if not con.features.uses_savepoints:
print("\n Skipping test requiring savepoints.")
return
# sanity check
self.assertFalse(is_managed())
self.assertFalse(transaction.is_dirty())
self.assertTrue("default" in getattr(settings, "DATABASES"))
self.assertTrue("second" in getattr(settings, "DATABASES"))
g1 = Genre.objects.using("default").get(pk=1)
start_g1 = g1.title
g2 = Genre.objects.using("second").get(pk=1)
transaction.enter_transaction_management(using='default')
managed(using='default')
transaction.enter_transaction_management(using='second')
managed(using='second')
g1.title = "Rollback savepoint"
g1.save()
g2.title = "Committed savepoint"
g2.save(using="second")
sid2 = transaction.savepoint(using="second")
sid = transaction.savepoint(using="default")
g1.title = "Dirty text"
g1.save()
#other thread should see the original key and cache object from memcache,
#not the local cache version
other("Genre.objects.using('default').get(pk=1)")
hit, ostart = q.get()
self.assertTrue(hit)
self.assertEqual(ostart.title, start_g1)
#should not be a hit due to rollback
transaction.savepoint_rollback(sid, using="default")
g1 = Genre.objects.using("default").get(pk=1)
# i think it should be "Rollback Savepoint" here
self.assertEqual(g1.title, start_g1)
#will be pushed to dirty in commit
g2 = Genre.objects.using("second").get(pk=1)
self.assertEqual(g2.title, "Committed savepoint")
transaction.savepoint_commit(sid2, using="second")
#other thread should still see original version even
#after savepoint commit
other("Genre.objects.using('second').get(pk=1)")
hit, ostart = q.get()
self.assertTrue(hit)
self.assertEqual(ostart.title, start_g1)
with self.assertNumQueries(0, using='second'):
g2 = Genre.objects.using("second").get(pk=1)
transaction.commit(using="second")
managed(False, using='second')
with self.assertNumQueries(0, using='second'):
g2 = Genre.objects.using("second").get(pk=1)
self.assertEqual(g2.title, "Committed savepoint")
#now committed and cached, other thread should reflect new title
#without a hit to the db
other("Genre.objects.using('second').get(pk=1)")
hit, ostart = q.get()
self.assertEqual(ostart.title, g2.title)
self.assertTrue(hit)
managed(False, 'default')
transaction.leave_transaction_management("default")
transaction.leave_transaction_management("second")
def test_transactions(self):
"""Tests transaction rollbacks and local cache for multiple dbs"""
if len(getattr(settings, "DATABASES", [])) <= 1:
print("\n Skipping multi database tests")
return
if not is_multithreading_safe():
print("\n Skipping test requiring multiple threads.")
return
for conname in connections:
con = connections[conname]
if not base.supports_transactions(con):
print("\n Skipping test requiring transactions.")
return
q = Queue()
other = lambda x: self._run_threaded(x, q, {'Genre': Genre})
# sanity check
self.assertFalse(is_managed())
self.assertFalse(transaction.is_dirty())
self.assertTrue("default" in getattr(settings, "DATABASES"))
self.assertTrue("second" in getattr(settings, "DATABASES"))
# this should seed this fetch in the global cache
g1 = Genre.objects.using("default").get(pk=1)
g2 = Genre.objects.using("second").get(pk=1)
start_g1 = g1.title
transaction.enter_transaction_management(using='default')
managed(using='default')
transaction.enter_transaction_management(using='second')
managed(using='second')
g1.title = "Testing a rollback"
g2.title = "Testing a commit"
g1.save()
g2.save()
# test outside of transaction, should be cache hit and
# not contain the local changes
other("Genre.objects.using('default').get(pk=1)")
hit, ostart = q.get()
self.assertEqual(ostart.title, start_g1)
self.assertTrue(hit)
transaction.rollback(using='default')
transaction.commit(using='second')
managed(False, using='default')
managed(False, using='second')
#other thread should have seen rollback
other("Genre.objects.using('default').get(pk=1)")
hit, ostart = q.get()
self.assertEqual(ostart.title, start_g1)
self.assertTrue(hit)
#should be a cache hit due to rollback
with self.assertNumQueries(0, using='default'):
g1 = Genre.objects.using("default").get(pk=1)
#should be a db hit due to commit
with self.assertNumQueries(1, using='second'):
g2 = Genre.objects.using("second").get(pk=1)
#other thread sould now be accessing the cache after the get
#from the commit.
other("Genre.objects.using('second').get(pk=1)")
hit, ostart = q.get()
self.assertEqual(ostart.title, g2.title)
self.assertTrue(hit)
self.assertEqual(g1.title, start_g1)
self.assertEqual(g2.title, "Testing a commit")
transaction.leave_transaction_management("default")
transaction.leave_transaction_management("second")
def __init__(self):
self.q = Queue()
qc_hit.connect(self._hit)
qc_miss.connect(self._miss)
qc_skip.connect(self._skip)
class message_queue(object):
"""Return a message queue that gets 'hit' or 'miss' messages. The signal
handlers use weakrefs, so if we don't save references to this object they
will get gc'd pretty fast."""
def __init__(self):
self.q = Queue()
qc_hit.connect(self._hit)
qc_miss.connect(self._miss)
qc_skip.connect(self._skip)
def _hit(self, *a, **k): self.q.put(True)
def _miss(self, *a, **k): self.q.put(False)
def _skip(self, *a, **k): self.q.put(False)
def clear(self):
while not self.q.empty():
self.q.get_nowait()
def get(self): return self.q.get()
def get_nowait(self): return self.q.get_nowait()
def qsize(self): return self.q.qsize()
def empty(self): return self.q.empty()
def __init__(self):
self.q = Queue()
qc_hit.connect(self._hit)
qc_miss.connect(self._miss)
qc_skip.connect(self._skip)
class message_queue(object):
"""Return a message queue that gets 'hit' or 'miss' messages. The signal
handlers use weakrefs, so if we don't save references to this object they
will get gc'd pretty fast."""
def __init__(self):
self.q = Queue()
qc_hit.connect(self._hit)
qc_miss.connect(self._miss)
qc_skip.connect(self._skip)
def _hit(self, *a, **k):
self.q.put(True)
def _miss(self, *a, **k):
self.q.put(False)
def _skip(self, *a, **k):
self.q.put(False)
def clear(self):
while not self.q.empty():
self.q.get_nowait()
def get(self):
return self.q.get()
def get_nowait(self):
return self.q.get_nowait()
def qsize(self):
return self.q.qsize()
def empty(self):
return self.q.empty()
def test_savepoints(self):
"""tests savepoints for multiple db's"""
q = Queue()
other = lambda x: self._run_threaded(x, q, {'Genre': Genre})
if len(getattr(settings, "DATABASES", [])) <= 1:
print("\n Skipping multi database tests")
return
if not is_multithreading_safe():
print("\n Skipping test requiring multiple threads.")
return
for name, db in settings.DATABASES.items():
con = connections[name]
if not con.features.uses_savepoints:
print("\n Skipping test requiring savepoints.")
return
# sanity check
self.assertFalse(is_managed())
self.assertFalse(transaction.is_dirty())
self.assertTrue("default" in getattr(settings, "DATABASES"))
self.assertTrue("second" in getattr(settings, "DATABASES"))
g1 = Genre.objects.using("default").get(pk=1)
start_g1 = g1.title
g2 = Genre.objects.using("second").get(pk=1)
start_g2 = g2.title
transaction.enter_transaction_management(using='default')
managed(using='default')
transaction.enter_transaction_management(using='second')
managed(using='second')
g1.title = "Rollback savepoint"
g1.save()
g2.title = "Committed savepoint"
g2.save(using="second")
sid2 = transaction.savepoint(using="second")
sid = transaction.savepoint(using="default")
g1.title = "Dirty text"
g1.save()
#other thread should see the original key and cache object from memcache,
#not the local cache version
other("Genre.objects.using('default').get(pk=1)")
hit, ostart = q.get()
self.assertTrue(hit)
self.assertEqual(ostart.title, start_g1)
#should not be a hit due to rollback
transaction.savepoint_rollback(sid, using="default")
g1 = Genre.objects.using("default").get(pk=1)
# i think it should be "Rollback Savepoint" here
self.assertEqual(g1.title, start_g1)
#will be pushed to dirty in commit
g2 = Genre.objects.using("second").get(pk=1)
self.assertEqual(g2.title, "Committed savepoint")
transaction.savepoint_commit(sid2, using="second")
#other thread should still see original version even
#after savepoint commit
other("Genre.objects.using('second').get(pk=1)")
hit, ostart = q.get()
self.assertTrue(hit)
self.assertEqual(ostart.title, start_g2)
with self.assertNumQueries(0, using='second'):
g2 = Genre.objects.using("second").get(pk=1)
transaction.commit(using="second")
managed(False, using='second')
with self.assertNumQueries(0, using='second'):
g2 = Genre.objects.using("second").get(pk=1)
self.assertEqual(g2.title, "Committed savepoint")
#now committed and cached, other thread should reflect new title
#without a hit to the db
other("Genre.objects.using('second').get(pk=1)")
hit, ostart = q.get()
self.assertEqual(ostart.title, g2.title)
self.assertTrue(hit)
transaction.commit(using="default")
managed(False, 'default')
transaction.leave_transaction_management("default")
transaction.leave_transaction_management("second")
def test_transactions(self):
"""Tests transaction rollbacks and local cache for multiple dbs"""
if len(getattr(settings, "DATABASES", [])) <= 1:
print("\n Skipping multi database tests")
return
if not is_multithreading_safe():
print("\n Skipping test requiring multiple threads.")
return
for conname in connections:
con = connections[conname]
if not base.supports_transactions(con):
print("\n Skipping test requiring transactions.")
return
q = Queue()
other = lambda x: self._run_threaded(x, q, {'Genre': Genre})
# sanity check
self.assertFalse(is_managed())
self.assertFalse(transaction.is_dirty())
self.assertTrue("default" in getattr(settings, "DATABASES"))
self.assertTrue("second" in getattr(settings, "DATABASES"))
# this should seed this fetch in the global cache
g1 = Genre.objects.using("default").get(pk=1)
g2 = Genre.objects.using("second").get(pk=1)
start_g1 = g1.title
transaction.enter_transaction_management(using='default')
managed(using='default')
transaction.enter_transaction_management(using='second')
managed(using='second')
g1.title = "Testing a rollback"
g2.title = "Testing a commit"
g1.save()
g2.save()
# test outside of transaction, should be cache hit and
# not contain the local changes
other("Genre.objects.using('default').get(pk=1)")
hit, ostart = q.get()
self.assertEqual(ostart.title, start_g1)
self.assertTrue(hit)
transaction.rollback(using='default')
transaction.commit(using='second')
managed(False, using='default')
managed(False, using='second')
#other thread should have seen rollback
other("Genre.objects.using('default').get(pk=1)")
hit, ostart = q.get()
self.assertEqual(ostart.title, start_g1)
self.assertTrue(hit)
#should be a cache hit due to rollback
with self.assertNumQueries(0, using='default'):
g1 = Genre.objects.using("default").get(pk=1)
#should be a db hit due to commit
with self.assertNumQueries(1, using='second'):
g2 = Genre.objects.using("second").get(pk=1)
#other thread sould now be accessing the cache after the get
#from the commit.
other("Genre.objects.using('second').get(pk=1)")
hit, ostart = q.get()
self.assertEqual(ostart.title, g2.title)
self.assertTrue(hit)
self.assertEqual(g1.title, start_g1)
self.assertEqual(g2.title, "Testing a commit")
transaction.leave_transaction_management("default")
transaction.leave_transaction_management("second")
