def test_mvccadapterNewTransactionVsInvalidations(self):
"""
Check that polled invalidations are consistent with the TID at which
the transaction operates. Otherwise, it's like we miss invalidations.
"""
db = databaseFromString("<zodb>\n<mappingstorage/>\n</zodb>")
try:
t1 = transaction.TransactionManager()
c1 = db.open(t1)
r1 = c1.root()
r1['a'] = 1
t1.commit()
t2 = transaction.TransactionManager()
c2 = db.open(t2)
c2.root()['b'] = 1
s1 = c1._storage
l1 = s1._lock
@contextmanager
def beforeLock1():
s1._lock = l1
t2.commit()
with l1:
yield
s1._lock = beforeLock1()
t1.begin()
self.assertIs(s1._lock, l1)
self.assertIn('b', r1)
finally:
db.close()
def testThreeEmptyBucketsNoSegfault(self):
self.openDB()
tm1 = transaction.TransactionManager()
r1 = self.db.open(transaction_manager=tm1).root()
self.assertEqual(len(self.t), 0)
r1["t"] = b = self.t # an empty tree
tm1.commit()
tm2 = transaction.TransactionManager()
r2 = self.db.open(transaction_manager=tm2).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
# In one transaction, add and delete a key.
b[2] = 2
del b[2]
tm1.commit()
# In the other transaction, also add and delete a key.
b = copy
b[1] = 1
del b[1]
# If the commit() segfaults, the C code is still wrong for this case.
self.assertRaises(ConflictError, tm2.commit)
def setUp(self):
self.mappers = setup_mappers()
metadata.drop_all(engine)
metadata.create_all(engine)
self.tm1 = transaction.TransactionManager()
self.tm2 = transaction.TransactionManager()
# With psycopg2 you might supply isolation_level='SERIALIZABLE' here,
# unfortunately that is not supported by cx_Oracle.
e1 = sa.create_engine(TEST_DSN)
e2 = sa.create_engine(TEST_DSN)
self.s1 = orm.sessionmaker(
bind=e1,
extension=tx.ZopeTransactionExtension(
transaction_manager=self.tm1),
twophase=TEST_TWOPHASE,
)()
self.s2 = orm.sessionmaker(
bind=e2,
extension=tx.ZopeTransactionExtension(
transaction_manager=self.tm2),
twophase=TEST_TWOPHASE,
)()
self.tm1.begin()
self.s1.add(User(id=1, firstname='udo', lastname='juergens'))
self.tm1.commit()
def doreadconflict(self, db, mvcc):
tm1 = transaction.TransactionManager()
conn = db.open(mvcc=mvcc, transaction_manager=tm1)
r1 = conn.root()
obj = MinPO('root')
r1["p"] = obj
obj = r1["p"]
obj.child1 = MinPO('child1')
tm1.get().commit()
# start a new transaction with a new connection
tm2 = transaction.TransactionManager()
cn2 = db.open(mvcc=mvcc, transaction_manager=tm2)
r2 = cn2.root()
self.assertEqual(r1._p_serial, r2._p_serial)
obj.child2 = MinPO('child2')
tm1.get().commit()
# resume the transaction using cn2
obj = r2["p"]
# An attempt to access obj.child1 should fail with an RCE
# below if conn isn't using mvcc, because r2 was read earlier
# in the transaction and obj was modified by the other
# transaction.
obj.child1
return obj
def test_write_conflict(self):
# Attempting to change a blob simultaneously from two different
# connections should result in a write conflict error::
self._make_and_commit_blob()
tm1 = transaction.TransactionManager()
tm2 = transaction.TransactionManager()
database = self.database
conn3 = database.open(transaction_manager=tm1)
conn4 = database.open(transaction_manager=tm2)
root3 = conn3.root()
root4 = conn4.root()
blob1c3 = root3['blob1']
blob1c4 = root4['blob1']
conn3_data = b'this is from connection 3'
conn4_data = b'this is from connection 4'
with blob1c3.open('w') as blob1c3fh1:
blob1c3fh1.write(conn3_data)
with blob1c4.open('w') as blob1c4fh1:
blob1c4fh1.write(conn4_data)
tm1.commit()
self._check_blob_contents(root3['blob1'], conn3_data)
with self.assertRaises(ConflictError):
tm2.commit()
# After the conflict, the winning transaction's result is visible on both
# connections::
tm2.abort()
self._check_blob_contents(root4['blob1'], conn3_data)
conn3.close()
conn4.close()
def checkCachePolling(self):
# NOTE: This test still sets poll_interval, a deprecated
# option that does nothing. We keep it around to verify that
# this scenario still works either way.
self._storage = self.make_storage(poll_interval=3600,
share_local_cache=False)
db = DB(self._storage)
try:
# Set up the database.
tm1 = transaction.TransactionManager()
c1 = db.open(transaction_manager=tm1)
r1 = c1.root()
r1['obj'] = obj1 = PersistentMapping({'change': 0})
tm1.commit()
# Load and change the object in an independent connection.
tm2 = transaction.TransactionManager()
c2 = db.open(transaction_manager=tm2)
r2 = c2.root()
r2['obj']['change'] = 1
tm2.commit()
# Now c2 has delta_after0.
self.assertEqual(len(c2._storage._cache.delta_after0), 1)
c2.close()
# Change the object in the original connection.
c1.sync()
obj1['change'] = 2
tm1.commit()
# Close the database connection to c2.
c2._storage._drop_load_connection()
# Make the database connection to c2 reopen without polling.
c2._storage.load(b'\0' * 8, '')
self.assertTrue(c2._storage._load_transaction_open)
# Open a connection, which should be the same connection
# as c2.
c3 = db.open(transaction_manager=tm2)
self.assertTrue(c3 is c2)
self.assertEqual(len(c2._storage._cache.delta_after0), 1)
# Clear the caches (but not delta_after*)
c3._resetCache()
for client in c3._storage._cache.clients_local_first:
client.flush_all()
obj3 = c3.root()['obj']
# Should have loaded the new object.
self.assertEqual(obj3['change'], 2)
finally:
db.close()
def checkPollInterval(self, shared_cache=True):
# Verify the poll_interval parameter causes RelStorage to
# delay invalidation polling.
self._storage = self.make_storage(poll_interval=3600,
share_local_cache=shared_cache)
db = DB(self._storage)
try:
tm1 = transaction.TransactionManager()
c1 = db.open(transaction_manager=tm1)
r1 = c1.root()
r1['alpha'] = 1
tm1.commit()
tm2 = transaction.TransactionManager()
c2 = db.open(transaction_manager=tm2)
r2 = c2.root()
self.assertEqual(r2['alpha'], 1)
self.assertFalse(c2._storage.need_poll())
self.assertTrue(c2._storage._poll_at > 0)
r1['alpha'] = 2
# commit c1 without committing c2.
tm1.commit()
if shared_cache:
# The cache reveals that a poll is needed even though
# the poll timeout has not expired.
self.assertTrue(c2._storage.need_poll())
tm2.commit()
r2 = c2.root()
self.assertEqual(r2['alpha'], 2)
self.assertFalse(c2._storage.need_poll())
else:
# The poll timeout has not expired, so no poll should occur
# yet, even after a commit.
self.assertFalse(c2._storage.need_poll())
tm2.commit()
r2 = c2.root()
self.assertEqual(r2['alpha'], 1)
# expire the poll timer and verify c2 sees the change
c2._storage._poll_at -= 3601
tm2.commit()
r2 = c2.root()
self.assertEqual(r2['alpha'], 2)
c2.close()
c1.close()
finally:
db.close()
def test_deghostification_of_persistent_adapter_registries(self):
# Note that this test duplicates one from zope.component.tests.
# We should be able to get rid of this one when we get rid of
# __setstate__ implementation we have in back35.
# We want to make sure that we see updates correctly.
from ZODB.MappingStorage import DB
db = DB()
tm1 = transaction.TransactionManager()
c1 = db.open(transaction_manager=tm1)
r1 = zope.site.site._LocalAdapterRegistry((base, ))
r2 = zope.site.site._LocalAdapterRegistry((r1, ))
c1.root()[1] = r1
c1.root()[2] = r2
tm1.commit()
r1._p_deactivate()
r2._p_deactivate()
tm2 = transaction.TransactionManager()
c2 = db.open(transaction_manager=tm2)
r1 = c2.root()[1]
r2 = c2.root()[2]
self.assertIsNone(r1.lookup((), IFoo, ''))
foo_blank = Foo('')
base.register((), IFoo, '', foo_blank)
self.assertEqual(r1.lookup((), IFoo, ''), foo_blank)
self.assertIsNone(r2.lookup((), IFoo, '1'))
foo_1 = Foo('1')
r1.register((), IFoo, '1', foo_1)
self.assertEqual(r2.lookup((), IFoo, '1'), foo_1)
self.assertIsNone(r1.lookup((), IFoo, '2'))
self.assertIsNone(r2.lookup((), IFoo, '2'))
foo_2 = Foo('2')
base.register((), IFoo, '2', foo_2)
self.assertEqual(r1.lookup((), IFoo, '2'), foo_2)
self.assertEqual(r2.lookup((), IFoo, '2'), foo_2)
# Cleanup:
db.close()
clear_base()
def testConflictWithOneEmptyBucket(self):
# If one transaction empties a bucket, while another adds an item
# to the bucket, all the changes "look resolvable": bucket conflict
# resolution returns a bucket containing (only) the item added by
# the latter transaction, but changes from the former transaction
# removing the bucket are uncontested: the bucket is removed from
# the BTree despite that resolution thinks it's non-empty! This
# was first reported by Dieter Maurer, to zodb-dev on 22 Mar 2005.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Set up database connections to provoke conflict.
self.openDB()
tm1 = transaction.TransactionManager()
r1 = self.db.open(transaction_manager=tm1).root()
r1["t"] = self.t
tm1.commit()
tm2 = transaction.TransactionManager()
r2 = self.db.open(transaction_manager=tm2).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# Now one transaction empties the first bucket, and another adds a
# key to the first bucket.
for k in range(0, 60, 4):
del self.t[k]
tm1.commit()
copy[1] = 1
try:
tm2.commit()
except ConflictError, detail:
self.assert_(str(detail).startswith('database conflict error'))
def testCantResolveBTreeConflict(self):
# Test that a conflict involving two different changes to
# an internal BTree node is unresolvable. An internal node
# only changes when there are enough additions or deletions
# to a child bucket that the bucket is split or removed.
# It's (almost necessarily) a white-box test, and sensitive to
# implementation details.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Set up database connections to provoke conflict.
self.openDB()
tm1 = transaction.TransactionManager()
r1 = self.db.open(transaction_manager=tm1).root()
r1["t"] = self.t
tm1.commit()
tm2 = transaction.TransactionManager()
r2 = self.db.open(transaction_manager=tm2).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# Now one transaction should add enough keys to cause a split,
# and another should remove all the keys in one bucket.
for k in range(200, 300, 4):
self.t[k] = k
tm1.commit()
for k in range(0, 60, 4):
del copy[k]
try:
tm2.commit()
except ConflictError, detail:
self.assert_(str(detail).startswith('database conflict error'))
def readConflict(self, shouldFail=True):
# Two transactions run concurrently. Each reads some object,
# then one commits and the other tries to read an object
# modified by the first. This read should fail with a conflict
# error because the object state read is not necessarily
# consistent with the objects read earlier in the transaction.
tm1 = transaction.TransactionManager()
conn = self._db.open(mvcc=False, transaction_manager=tm1)
r1 = conn.root()
r1["p"] = self.obj
self.obj.child1 = P()
tm1.get().commit()
# start a new transaction with a new connection
tm2 = transaction.TransactionManager()
cn2 = self._db.open(mvcc=False, transaction_manager=tm2)
# start a new transaction with the other connection
r2 = cn2.root()
self.assertEqual(r1._p_serial, r2._p_serial)
self.obj.child2 = P()
tm1.get().commit()
# resume the transaction using cn2
obj = r2["p"]
# An attempt to access obj should fail, because r2 was read
# earlier in the transaction and obj was modified by the othe
# transaction.
if shouldFail:
self.assertRaises(ReadConflictError, lambda: obj.child1)
# And since ReadConflictError was raised, attempting to commit
# the transaction should re-raise it. checkNotIndependent()
# failed this part of the test for a long time.
self.assertRaises(ReadConflictError, tm2.get().commit)
# And since that commit failed, trying to commit again should
# fail again.
self.assertRaises(TransactionFailedError, tm2.get().commit)
# And again.
self.assertRaises(TransactionFailedError, tm2.get().commit)
# Etc.
self.assertRaises(TransactionFailedError, tm2.get().commit)
else:
# make sure that accessing the object succeeds
obj.child1
tm2.get().abort()
def dummy_data(app):
"""
Add some dummy data to the database.
Note that this is a session fixture that commits data to the database.
Think about it similarly to running the ``initialize_db`` script at the
start of the test suite.
This data should not conflict with any other data added throughout the
test suite or there will be issues - so be careful with this pattern!
"""
tm = transaction.TransactionManager(explicit=True)
with tm:
dbsession = models.get_tm_session(app.registry['dbsession_factory'],
tm)
editor = models.User(name='editor', role='editor')
editor.set_password('editor')
basic = models.User(name='basic', role='basic')
basic.set_password('basic')
page1 = models.Page(name='FrontPage', data='This is the front page')
page1.creator = editor
page2 = models.Page(name='BackPage', data='This is the back page')
page2.creator = basic
dbsession.add_all([basic, editor, page1, page2])
def _separate_zodb_connection(self, callback, *args, **kwargs):
main_connection_portal = getUtility(ISiteRoot)
manager = transaction.TransactionManager()
connection = main_connection_portal._p_jar.db().open(
transaction_manager=manager)
try:
for _r in range(self.maximum_retries):
manager.begin()
portal = connection[main_connection_portal._p_oid]
result = callback(portal, *args, **kwargs)
try:
manager.commit()
except ConflictError:
LOG.info('SequenceNumberIncrementer'
' ConflictError; retrying')
manager.abort()
else:
return result
# Maximum retries exceeded, raise conflict to main
# transaction / actual request.
raise
finally:
connection.close()
def _create_initial_state(self):
# Given a set of referencing objects present at the beginning
# of the pre pack:
# 0 1 2 3
# T1: root -> A -> B -> C
#
# If a new transaction is committed such that the graph becomes:
#
# 0 1
# T2: root -> A
# \-> B -> D -> C
# 2 4 3
#
# That is, C is no longer referenced from B but a new object
# D, B is referenced not from A but from the root.
txm = transaction.TransactionManager(explicit=True)
conn = self.main_db.open(txm)
txm.begin()
A = conn.root.A = PersistentMapping() # OID 0x1
B = A['B'] = PersistentMapping() # OID 0x2
C = B['C'] = PersistentMapping() # OID 0x3
txm.commit()
oids = {
'A': A._p_oid,
'B': B._p_oid,
'C': C._p_oid,
}
conn.close()
return oids
def reset_incremental_id(value=0, key="default"):
with _lock:
tm = transaction.TransactionManager()
conn = datastore.db.open(transaction_manager=tm)
try:
while True:
conn.sync()
root = conn.root()
container = root.get(ID_CONTAINER_KEY)
if container is None:
container = PersistentMapping()
root[ID_CONTAINER_KEY] = container
container[key] = value
try:
tm.commit()
except ConflictError:
sleep(RETRY_INTERVAL)
tm.abort()
except:
tm.abort()
raise
else:
break
finally:
conn.close()
_acquired_ids[key] = None
def __set_keys_in_root_to(self,
storage,
new_data,
old_tids,
old_data,
pack=False):
"""
And return the transaction ID and current checkpoints.
Uses an independent transaction.
Closes *storage*.
"""
db1 = self._closing(DB(storage))
tx = transaction.TransactionManager()
c1 = db1.open(tx)
# We've polled and gained checkpoints
self.assert_checkpoints(c1)
root = c1.root()
self.__do_sets(root, new_data, old_tids, old_data)
tx.commit()
checkpoints = self.assert_checkpoints(c1)
self.__do_check_tids(root, old_tids)
tid_int = bytes8_to_int64(c1._storage.lastTransaction())
c1.close()
if pack:
storage.pack(tid_int, referencesf)
db1.close()
return tid_int, checkpoints
def from_settings(cls, settings, **kwargs):
"""Create a site object from a dictionary of settings
"""
tm = transaction.TransactionManager(explicit=True)
db_session = None
redis_instance = None
if settings.get("sqlalchemy.url"):
db_engine = sqlalchemy.engine_from_config(settings, "sqlalchemy.")
db_session_factory = sqlalchemy.orm.sessionmaker()
db_session_factory.configure(bind=db_engine)
db_session = db_session_factory()
zope.sqlalchemy.register(db_session,
transaction_manager=tm,
keep_session=True)
if settings.get("redis_url"):
redis_instance = redis.StrictRedis.from_url(settings["redis_url"],
decode_responses=True)
class MailerTmp(pyramid_mailer.Mailer):
def __init__(self, **kw):
super().__init__(transaction_manager=tm, **kw)
mailer = MailerTmp.from_settings(settings, "mail.")
return cls(
settings=settings,
db_session=db_session,
redis=redis_instance,
mailer=mailer,
transaction_manager=tm,
**kwargs,
)
def __enter__(self):
self.tm = tm = transaction.TransactionManager()
self.conn = self.db.open(self.tm)
t = tm.begin()
if self.note:
t.note(self.note)
return self.conn
def _populate_root_and_mapping(self):
"""
Creates the following structure in ``self._storage``::
root.myobj1 = PersistentMapping()
root.myobj1.key = PersistentMapping()
root.myobj = 3
Does this over several transactions. Returns
the tid of the last time the root changed, and the tid
of ``root.myobj1``, which is later than the root TID and which
is current, and the database opened on the storage.
"""
tx1 = transaction.TransactionManager()
storage1 = self._storage
db1 = self._closing(DB(storage1))
c1 = db1.open(tx1)
root = c1.root
root().myobj1 = root.myobj1 = mapping = PersistentMapping()
root().myobj = root.myobj = 1
tx1.commit()
c1._storage._cache.clear(load_persistent=False)
c1._storage.poll_invalidations()
root().myobj = root.myobj = 2
tx1.commit()
c1._storage._cache.clear(load_persistent=False)
c1._storage.poll_invalidations()
root().myobj = root.myobj = 3
tx1.commit()
root_tid = self.assert_oid_known(ROOT_OID, c1)
c1._storage._cache.clear(load_persistent=False)
# Now, mutate an object that's not the root
# so that we get a new transaction after the root was
# modified. This transaction will be included in
# a persistent cache.
c1._storage.poll_invalidations()
root().myobj1.key = root.myobj1.key = PersistentMapping()
mapping_oid = mapping._p_oid
mapping_oid_int = bytes8_to_int64(mapping_oid)
tx1.commit()
mapping_tid = self.assert_oid_known(mapping_oid_int, c1)
# self.assert_checkpoints(c1, (root_tid, root_tid))
self.assert_oid_current(mapping_oid_int, c1)
# the root is not in a delta
self.assert_oid_not_known(ROOT_OID, c1)
# Nor is it in the cache, because the Connection's
# object cache still had the root and we were never
# asked.
self.assert_oid_not_cached(ROOT_OID, c1)
# So lets get it in the cache with its current TID.
c1._storage.load(z64)
self.assert_cached_exact(ROOT_OID, root_tid, c1)
c1.close()
return root_tid, mapping_tid, db1
def test_alternate_transaction_manager(self):
from repoze.sendmail.delivery import DirectMailDelivery
from email.message import Message
import transaction
mailer = _makeMailerStub()
delivery = DirectMailDelivery(mailer)
tm = transaction.TransactionManager()
delivery.transaction_manager = tm
fromaddr = "Jim <*****@*****.**>"
toaddrs = ('Guido <*****@*****.**>', 'Steve <*****@*****.**>')
message = Message()
message["From"] = fromaddr
message["To"] = ",".join(toaddrs)
message["Date"] = "Date: Mon, 19 May 2003 10:17:36 -0400"
message["Subject"] = "example"
message.set_payload("This is just an example\n")
msgid = delivery.send(fromaddr, toaddrs, message)
transaction.commit()
self.assertEqual(len(mailer.sent_messages), 0)
t = tm.get()
data_manager = t._resources[0]
self.assertTrue(data_manager.transaction_manager is tm)
t.commit()
self.assertEqual(len(mailer.sent_messages), 1)
self.assertEqual(mailer.sent_messages[0][0], fromaddr)
self.assertEqual(mailer.sent_messages[0][1], toaddrs)
self.assertEqual(mailer.sent_messages[0][2].get_payload(),
"This is just an example\n")
mailer.sent_messages = []
msgid = delivery.send(fromaddr, toaddrs, message)
tm.get().abort()
self.assertEqual(len(mailer.sent_messages), 0)
def checkHistoricalConnection(self):
import persistent
import ZODB.POSException
db = DB(self._storage)
conn = db.open()
root = conn.root()
root['first'] = persistent.mapping.PersistentMapping(count=0)
transaction.commit()
time_of_first_transaction = conn._storage.lastTransaction()
root['second'] = persistent.mapping.PersistentMapping()
root['first']['count'] += 1
transaction.commit()
transaction1 = transaction.TransactionManager(explicit=True)
historical_conn = db.open(transaction_manager=transaction1,
at=time_of_first_transaction)
eq = self.assertEqual
# regular connection sees present:
eq(sorted(conn.root().keys()), ['first', 'second'])
eq(conn.root()['first']['count'], 1)
# historical connection sees past:
transaction1.begin()
eq(sorted(historical_conn.root().keys()), ['first'])
eq(historical_conn.root()['first']['count'], 0)
# Can't change history:
historical_conn.root()['first']['count'] += 1
eq(historical_conn.root()['first']['count'], 1)
self.assertRaises(ZODB.POSException.ReadOnlyHistoryError,
transaction1.commit)
transaction1.abort()
eq(historical_conn.root()['first']['count'], 0)
# Making a change in the present
root['third'] = 3
transaction.commit()
# Is also not reflected in the past, even after explicit sync,
transaction1.begin()
eq(sorted(historical_conn.root().keys()), ['first'])
eq(historical_conn.root()['first']['count'], 0)
# Since we cannot change anything, we cannot join a transaction either.
# The afterCompletion call is never invoked.
historical_conn._storage._storage.afterCompletion = lambda: self.fail(
"Not called")
transaction1.commit()
historical_conn.close()
conn.close()
db.close()
def testMinimizeTerminates(self):
# This is tricky. cPickleCache had a case where it could get into
# an infinite loop, but we don't want the test suite to hang
# if this bug reappears. So this test spawns a thread to run the
# dangerous operation, and the main thread complains if the worker
# thread hasn't finished in 30 seconds (arbitrary, but way more
# than enough). In that case, the worker thread will continue
# running forever (until killed externally), but at least the
# test suite will move on.
#
# The bug was triggered by having a persistent object whose __del__
# method references an attribute of the object. An attempt to
# ghostify such an object will clear the attribute, and if the
# cache also releases the last Python reference to the object then
# (due to ghostifying it), the __del__ method gets invoked.
# Referencing the attribute loads the object again, and also
# puts it back into the cPickleCache. If the cache implementation
# isn't looking out for this, it can get into an infinite loop
# then, endlessly trying to ghostify an object that in turn keeps
# unghostifying itself again.
# This test uses threads, so we can't use the default
# transaction manager.
for conn in self.conns:
conn.close()
self.conns[0] = self.db.open(transaction.TransactionManager())
class Worker(threading.Thread):
def __init__(self, testcase):
threading.Thread.__init__(self)
self.testcase = testcase
def run(self):
global make_trouble
# Make CantGetRidOfMe.__del__ dangerous.
make_trouble = True
conn = self.testcase.conns[0]
r = conn.root()
d = r[1]
for i in range(len(d)):
d[i] = CantGetRidOfMe(i)
conn.transaction_manager.commit()
self.testcase.db.cacheMinimize()
# Defang the nasty objects. Else, because they're
# immortal now, they hang around and create trouble
# for subsequent tests.
make_trouble = False
self.testcase.db.cacheMinimize()
w = Worker(self)
w.start()
w.join(30)
if w.is_alive():
self.fail("cacheMinimize still running after 30 seconds -- "
"almost certainly in an infinite loop")
def setUp(self):
self.openDB()
queue = CatalogEventQueue()
tm1 = transaction.TransactionManager()
self.conn1 = self.db.open(transaction_manager=tm1)
r1 = self.conn1.root()
r1["queue"] = queue
del queue
self.queue = r1["queue"]
tm1.commit()
tm2 = transaction.TransactionManager()
self.conn2 = self.db.open(transaction_manager=tm2)
r2 = self.conn2.root()
self.queue2 = r2["queue"]
ignored = dir(self.queue2) # unghostify
def _check_nested(storage):
db4 = DB(storage)
txm4 = transaction.TransactionManager(True)
conn4 = db4.open(txm4)
txm4.begin()
self.assertEqual(conn4.root.myobj1.key['foo'], 'bar')
conn4.close()
db4.close()
def get_request(self):
if not hasattr(self.request, "pyramid_env"):
registry = self.app.pyramid_config.registry
env = pyramid.scripting.prepare(registry=registry)
env["request"].tm = transaction.TransactionManager(explicit=True)
self.request.update(pyramid_env=env)
return self.request.pyramid_env["request"]
def __init__(self):
self._transaction_manager = transaction.TransactionManager()
storage = ZODB.FileStorage.FileStorage("/store/candice.fs")
db = ZODB.DB(storage)
connection = db.open(self._transaction_manager)
self.root = connection.root
def tm():
tm = transaction.TransactionManager(explicit=True)
tm.begin()
tm.doom()
yield tm
tm.abort()
def checkPackWithGCOnDestinationAfterRestore(self):
raises = self.assertRaises
closing = self._closing
__traceback_info__ = self._storage, self._dst
db = closing(DB(self._storage))
conn = closing(db.open())
root = conn.root()
root.obj = obj1 = MinPO(1)
txn = transaction.get()
txn.note(u'root -> obj')
txn.commit()
root.obj.obj = obj2 = MinPO(2)
txn = transaction.get()
txn.note(u'root -> obj -> obj')
txn.commit()
del root.obj
txn = transaction.get()
txn.note(u'root -X->')
txn.commit()
storage_last_tid = conn._storage.lastTransaction()
self.assertEqual(storage_last_tid, root._p_serial)
# Now copy the transactions to the destination
self._dst.copyTransactionsFrom(self._storage)
self.assertEqual(self._dst.lastTransaction(), storage_last_tid)
# If the source storage is a history-free storage, all
# of the transactions are now marked as packed in the
# destination storage. To trigger a pack, we have to
# add another transaction to the destination that is
# not packed.
db2 = closing(DB(self._dst))
tx_manager = transaction.TransactionManager(explicit=True)
conn2 = closing(db2.open(tx_manager))
txn = tx_manager.begin()
root2 = conn2.root()
root2.extra = 0
txn.note(u'root.extra = 0')
txn.commit()
dest_last_tid = conn2._storage.lastTransaction()
self.assertGreater(dest_last_tid, storage_last_tid)
self.assertEqual(dest_last_tid, root2._p_serial)
# Now pack the destination.
from ZODB.utils import u64 as bytes8_to_int64
if IRelStorage.providedBy(self._dst):
packtime = bytes8_to_int64(storage_last_tid)
else:
from persistent.timestamp import TimeStamp
packtime = TimeStamp(dest_last_tid).timeTime() + 2
self._dst.pack(packtime, referencesf)
# And check to see that the root object exists, but not the other
# objects.
__traceback_info__ += (packtime,)
_data, _serial = self._dst.load(root._p_oid, '')
raises(KeyError, self._dst.load, obj1._p_oid, '')
raises(KeyError, self._dst.load, obj2._p_oid, '')
def zodb_aware_poller(url, timeout=None, db_conf=None, zodb_conf=None, **data):
queues = dict(getUtilitiesFor(IReceptionQueue))
receiver = get_reader(url)
tm = transaction.TransactionManager()
db = init_db(open(db_conf, 'r').read())
with ZODBConnection(db, transaction_manager=tm) as zodb:
with tm:
data['db_root'] = zodb
receiver.poll(queues, timeout=timeout, **data)
def test_wal_does_not_grow(self):
# issue 401 https://github.com/zodb/relstorage/issues/401
import transaction
from ZODB import DB
def size_wal():
d = self._storage._adapter.data_dir
wal = os.path.join(d, 'main.sqlite3-wal')
#os.system('ls -lh ' + wal)
size = os.stat(wal).st_size
return size
def run_transaction(db):
tx = transaction.begin()
conn = db.open()
try:
root = conn.root()
root['key'] = 'abcd' * 1000
tx.commit()
except:
tx.abort()
raise
finally:
conn.close()
old_explicit = transaction.manager.explicit
transaction.manager.explicit = True
try:
db = self._closing(DB(self._storage))
# Open an isolated connection. If the transaction
# manager isn't in explicit mode, it really will talk to
# the database now, which will cause WAL to start accumulating.
# Use explicit transaction managers to prevent that from happening
# and keep a tight reign on transaction duration.
isolated_txm = transaction.TransactionManager()
isolated_txm.explicit = True # If this is commented out, the WAL grows without bound
c1 = db.open(isolated_txm)
try:
TX_COUNT = 100
RUNS = 3
# Prime it.
for _ in range(TX_COUNT):
run_transaction(db)
wal_size = size_wal()
for _ in range(RUNS):
for _ in range(TX_COUNT):
run_transaction(db)
self.assertEqual(wal_size, size_wal())
finally:
c1.close()
db.close()
finally:
transaction.manager.explicit = old_explicit
