def checkPackLotsWhileWriting(self):
# This is like the other pack-while-writing tests, except it packs
# repeatedly until the client thread is done. At the time it was
# introduced, it reliably provoked
# CorruptedError: ... transaction with checkpoint flag set
# in the ZEO flavor of the FileStorage tests.
db = DB(self._storage)
conn = db.open()
root = conn.root()
choices = range(10)
for i in choices:
root[i] = MinPO(i)
transaction.commit()
snooze()
packt = time.time()
for dummy in choices:
for i in choices:
root[i].value = MinPO(i)
transaction.commit()
NUM_LOOP_TRIP = 100
timer = ElapsedTimer(time.time())
thread = ClientThread(db, choices, NUM_LOOP_TRIP, timer, 0)
thread.start()
while thread.isAlive():
db.pack(packt)
snooze()
packt = time.time()
thread.join()
self._sanity_check()
def doreadconflict(self, db, mvcc):
tm1 = transaction.TransactionManager()
conn = db.open(mvcc=mvcc, transaction_manager=tm1)
r1 = conn.root()
obj = MinPO('root')
r1["p"] = obj
obj = r1["p"]
obj.child1 = MinPO('child1')
tm1.get().commit()
# start a new transaction with a new connection
tm2 = transaction.TransactionManager()
cn2 = db.open(mvcc=mvcc, transaction_manager=tm2)
r2 = cn2.root()
self.assertEqual(r1._p_serial, r2._p_serial)
obj.child2 = MinPO('child2')
tm1.get().commit()
# resume the transaction using cn2
obj = r2["p"]
# An attempt to access obj.child1 should fail with an RCE
# below if conn isn't using mvcc, because r2 was read earlier
# in the transaction and obj was modified by the other
# transaction.
obj.child1
return obj
def checkConflicts(self):
oid = self._storage.new_oid()
revid1 = self._dostore(oid, data=MinPO(11))
self._dostore(oid, revid=revid1, data=MinPO(12))
self.assertRaises(POSException.ConflictError,
self._dostore,
oid,
revid=revid1,
data=MinPO(13))
def checkSimpleIteration(self):
# Store a bunch of revisions of a single object
self._oid = oid = self._storage.new_oid()
revid1 = self._dostore(oid, data=MinPO(11))
revid2 = self._dostore(oid, revid=revid1, data=MinPO(12))
revid3 = self._dostore(oid, revid=revid2, data=MinPO(13))
# Now iterate over all the transactions and compare carefully
txniter = self._storage.iterator()
self.iter_verify(txniter, [revid1, revid2, revid3], 11)
def dowork(self, version=''):
c = self.db.open(version)
r = c.root()
o = r[time.time()] = MinPO(0)
transaction.commit()
for i in range(25):
o.value = MinPO(i)
transaction.commit()
o = o.value
c.close()
def checkCommitToNonVersion(self):
eq = self.assertEqual
oid, version = self._setup_version()
data, revid = self._storage.load(oid, version)
eq(zodb_unpickle(data), MinPO(54))
data, revid = self._storage.load(oid, '')
eq(zodb_unpickle(data), MinPO(51))
self._commitVersion(version, '')
data, revid = self._storage.load(oid, '')
eq(zodb_unpickle(data), MinPO(54))
def checkVersionLock(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(11))
version = 'test-version'
revid = self._dostore(oid, revid=revid, data=MinPO(12),
version=version)
self.assertRaises(POSException.VersionLockError,
self._dostore,
oid, revid=revid, data=MinPO(14),
version='another-version')
def checkWriteAfterAbort(self):
oid = self._storage.new_oid()
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
# Now abort this transaction
self._storage.tpc_abort(t)
# Now start all over again
oid = self._storage.new_oid()
self._dostore(oid=oid, data=MinPO(6))
def checkLen(self):
# len(storage) reports the number of objects.
# check it is zero when empty
self.assertEqual(len(self._storage), 0)
# check it is correct when the storage contains two object.
# len may also be zero, for storages that do not keep track
# of this number
self._dostore(data=MinPO(22))
self._dostore(data=MinPO(23))
self.assertTrue(len(self._storage) in [0, 2])
def checkCommitToOtherVersion(self):
eq = self.assertEqual
oid1, version1 = self._setup_version('one')
data, revid1 = self._storage.load(oid1, version1)
eq(zodb_unpickle(data), MinPO(54))
oid2, version2 = self._setup_version('two')
data, revid2 = self._storage.load(oid2, version2)
eq(zodb_unpickle(data), MinPO(54))
# make sure we see the non-version data when appropriate
data, revid2 = self._storage.load(oid1, version2)
eq(zodb_unpickle(data), MinPO(51))
data, revid2 = self._storage.load(oid2, version1)
eq(zodb_unpickle(data), MinPO(51))
data, revid2 = self._storage.load(oid1, '')
eq(zodb_unpickle(data), MinPO(51))
# Okay, now let's commit object1 to version2
oids = self._commitVersion(version1, version2)
eq(len(oids), 1)
eq(oids[0], oid1)
data, revid = self._storage.load(oid1, version2)
eq(zodb_unpickle(data), MinPO(54))
data, revid = self._storage.load(oid2, version2)
eq(zodb_unpickle(data), MinPO(54))
# an object can only exist in one version, so a load from
# version1 should now give the non-version data
data, revid2 = self._storage.load(oid1, version1)
eq(zodb_unpickle(data), MinPO(51))
# as should a version that has never been used
data, revid2 = self._storage.load(oid1, 'bela lugosi')
eq(zodb_unpickle(data), MinPO(51))
def checkLoadSerial(self):
oid = self._storage.new_oid()
revid = ZERO
revisions = {}
for i in range(31, 38):
revid = self._dostore(oid, revid=revid, data=MinPO(i))
revisions[revid] = MinPO(i)
# Now make sure all the revisions have the correct value
for revid, value in revisions.items():
data = self._storage.loadSerial(oid, revid)
self.assertEqual(zodb_unpickle(data), value)
def checkCommitVersionSerialno(self):
oid = self._storage.new_oid()
revid1 = self._dostore(oid, data=MinPO(12))
revid2 = self._dostore(oid, revid=revid1, data=MinPO(13),
version="version")
oids = self._commitVersion("version", "")
self.assertEqual([oid], oids)
data, revid3 = self._storage.load(oid, "")
# use repr() to avoid getting binary data in a traceback on error
self.assertNotEqual(`revid1`, `revid3`)
self.assertNotEqual(`revid2`, `revid3`)
def checkPackAfterUndoDeletion(self):
db = DB(self._storage)
cn = db.open()
root = cn.root()
pack_times = []
def set_pack_time():
pack_times.append(time.time())
snooze()
root["key0"] = MinPO(0)
root["key1"] = MinPO(1)
root["key2"] = MinPO(2)
txn = transaction.get()
txn.note("create 3 keys")
txn.commit()
set_pack_time()
del root["key1"]
txn = transaction.get()
txn.note("delete 1 key")
txn.commit()
set_pack_time()
root._p_deactivate()
cn.sync()
self.assert_(listeq(root.keys(), ["key0", "key2"]))
L = db.undoInfo()
db.undo(L[0]["id"])
txn = transaction.get()
txn.note("undo deletion")
txn.commit()
set_pack_time()
root._p_deactivate()
cn.sync()
self.assert_(listeq(root.keys(), ["key0", "key1", "key2"]))
for t in pack_times:
self._storage.pack(t, referencesf)
root._p_deactivate()
cn.sync()
self.assert_(listeq(root.keys(), ["key0", "key1", "key2"]))
for i in range(3):
obj = root["key%d" % i]
self.assertEqual(obj.value, i)
root.items()
self._inter_pack_pause()
def checkUndoAbortVersion(self):
def load_value(oid, version=''):
data, revid = self._storage.load(oid, version)
return zodb_unpickle(data).value
# create a bunch of packable transactions
oid = self._storage.new_oid()
revid = '\000' * 8
for i in range(3):
revid = self._dostore(oid, revid, description='packable%d' % i)
pt = time.time()
time.sleep(1)
oid1 = self._storage.new_oid()
version = 'version'
revid1 = self._dostore(oid1, data=MinPO(0), description='create1')
revid2 = self._dostore(oid1,
data=MinPO(1),
revid=revid1,
version=version,
description='version1')
self._dostore(oid1,
data=MinPO(2),
revid=revid2,
version=version,
description='version2')
self._dostore(description='create2')
self._abortVersion(version)
info = self._storage.undoInfo()
t_id = info[0]['id']
self.assertEqual(load_value(oid1), 0)
# after abort, we should see non-version data
self.assertEqual(load_value(oid1, version), 0)
self._undo(t_id, note="undo abort version")
self.assertEqual(load_value(oid1), 0)
# t undo will re-create the version
self.assertEqual(load_value(oid1, version), 2)
info = self._storage.undoInfo()
t_id = info[0]['id']
self._storage.pack(pt, referencesf)
self._undo(t_id, note="undo undo")
# undo of undo will put as back where we started
self.assertEqual(load_value(oid1), 0)
# after abort, we should see non-version data
self.assertEqual(load_value(oid1, version), 0)
def checkModifyAfterAbortVersion(self):
eq = self.assertEqual
oid, version = self._setup_version()
self._abortVersion(version)
data, revid = self._storage.load(oid, '')
# And modify it a few times
revid = self._dostore(oid, revid=revid, data=MinPO(52))
revid = self._dostore(oid, revid=revid, data=MinPO(53))
revid = self._dostore(oid, revid=revid, data=MinPO(54))
data, newrevid = self._storage.load(oid, '')
eq(newrevid, revid)
eq(zodb_unpickle(data), MinPO(54))
def checkVersionedLoadErrors(self):
oid = self._storage.new_oid()
version = 'test-version'
revid = self._dostore(oid, data=MinPO(11))
revid = self._dostore(oid, revid=revid, data=MinPO(12),
version=version)
# Try to load a bogus oid
self.assertRaises(KeyError,
self._storage.load,
self._storage.new_oid(), '')
data, revid = self._storage.load(oid, 'bogus')
self.assertEqual(zodb_unpickle(data), MinPO(11))
def checkUndoCommitVersion(self):
def load_value(oid, version=''):
data, revid = self._storage.load(oid, version)
return zodb_unpickle(data).value
# create a bunch of packable transactions
oid = self._storage.new_oid()
revid = '\000' * 8
for i in range(4):
revid = self._dostore(oid, revid, description='packable%d' % i)
pt = time.time()
time.sleep(1)
oid1 = self._storage.new_oid()
version = 'version'
revid1 = self._dostore(oid1, data=MinPO(0), description='create1')
revid2 = self._dostore(oid1, data=MinPO(1), revid=revid1,
version=version, description='version1')
self._dostore(oid1, data=MinPO(2), revid=revid2,
version=version, description='version2')
self._dostore(description='create2')
t = transaction.Transaction()
t.description = 'commit version'
self._storage.tpc_begin(t)
self._storage.commitVersion(version, '', t)
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
info = self._storage.undoInfo()
t_id = info[0]['id']
self.assertEqual(load_value(oid1), 2)
self.assertEqual(load_value(oid1, version), 2)
self._storage.pack(pt, referencesf)
self._undo(t_id, note="undo commit version")
self.assertEqual(load_value(oid1), 0)
self.assertEqual(load_value(oid1, version), 2)
data, tid = self._storage.load(oid1, "")
# After undoing the version commit, the non-version data
# once again becomes the non-version data from 'create1'.
self.assertEqual(tid, self._storage.lastTransaction())
# The current version data comes from an undo record, which
# means that it gets data via the backpointer but tid from the
# current txn.
data, tid, ver = loadEx(self._storage, oid1, version)
self.assertEqual(ver, version)
self.assertEqual(tid, self._storage.lastTransaction())
def dowork(self):
c = self.db.open()
r = c.root()
o = r[time.time()] = MinPO(0)
transaction.commit()
for i in range(25):
o.value = MinPO(i)
transaction.commit()
o = o.value
serial = o._p_serial
root_serial = r._p_serial
c.close()
return serial, root_serial
def checkStoreAndLoad(self):
eq = self.assertEqual
oid = self._storage.new_oid()
self._dostore(oid=oid, data=MinPO(7))
data, revid = utils.load_current(self._storage, oid)
value = zodb_unpickle(data)
eq(value, MinPO(7))
# Now do a bunch of updates to an object
for i in range(13, 22):
revid = self._dostore(oid, revid=revid, data=MinPO(i))
# Now get the latest revision of the object
data, revid = utils.load_current(self._storage, oid)
eq(zodb_unpickle(data), MinPO(21))
def checkSerialIsNoneForInitialRevision(self):
eq = self.assertEqual
oid = self._storage.new_oid()
txn = TransactionMetaData()
self._storage.tpc_begin(txn)
# Use None for serial. Don't use _dostore() here because that coerces
# serial=None to serial=ZERO.
self._storage.store(oid, None, zodb_pickle(MinPO(11)), '', txn)
self._storage.tpc_vote(txn)
newrevid = self._storage.tpc_finish(txn)
data, revid = utils.load_current(self._storage, oid)
value = zodb_unpickle(data)
eq(value, MinPO(11))
eq(revid, newrevid)
def checkCommitVersionInvalidation(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(1))
revid = self._dostore(oid, revid=revid, data=MinPO(2))
revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo")
t = Transaction()
self._storage.tpc_begin(t)
self._storage.commitVersion("foo", "bar", t)
self._storage.load(oid, "")
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
data, revid = self._storage.load(oid, "bar")
obj = zodb_unpickle(data)
assert obj == MinPO(3), obj
def checkLoadBeforeOld(self):
# Look for a very old revision. With the BaseStorage implementation
# this should require multple history() calls.
oid = self._storage.new_oid()
revs = []
revid = None
for i in range(50):
revid = self._dostore(oid, revid, data=MinPO(i))
revs.append(revid)
data, start, end = self._storage.loadBefore(oid, revs[12])
self.assertEqual(zodb_unpickle(data), MinPO(11))
self.assertEqual(start, revs[11])
self.assertEqual(end, revs[12])
def checkSerialIsNoneForInitialRevision(self):
eq = self.assertEqual
oid = self._storage.new_oid()
txn = transaction.Transaction()
self._storage.tpc_begin(txn)
# Use None for serial. Don't use _dostore() here because that coerces
# serial=None to serial=ZERO.
r1 = self._storage.store(oid, None, zodb_pickle(MinPO(11)), '', txn)
r2 = self._storage.tpc_vote(txn)
self._storage.tpc_finish(txn)
newrevid = handle_serials(oid, r1, r2)
data, revid = self._storage.load(oid, '')
value = zodb_unpickle(data)
eq(value, MinPO(11))
eq(revid, newrevid)
def _dostore(self,
storage,
oid=None,
revid=None,
data=None,
already_pickled=0,
user=None,
description=None):
# Borrowed from StorageTestBase, to allow passing storage.
"""Do a complete storage transaction. The defaults are:
- oid=None, ask the storage for a new oid
- revid=None, use a revid of ZERO
- data=None, pickle up some arbitrary data (the integer 7)
Returns the object's new revision id.
"""
import transaction
from ZODB.tests.MinPO import MinPO
if oid is None:
oid = storage.new_oid()
if revid is None:
revid = StorageTestBase.ZERO
if data is None:
data = MinPO(7)
if type(data) == int:
data = MinPO(data)
if not already_pickled:
data = StorageTestBase.zodb_pickle(data)
# Begin the transaction
t = transaction.Transaction()
if user is not None:
t.user = user
if description is not None:
t.description = description
try:
storage.tpc_begin(t)
# Store an object
r1 = storage.store(oid, revid, data, '', t)
# Finish the transaction
r2 = storage.tpc_vote(t)
revid = handle_serials(oid, r1, r2)
storage.tpc_finish(t)
except:
storage.tpc_abort(t)
raise
return revid
def _checkHistory(self, data):
start = time()
# Store a couple of revisions of the object
oid = self._storage.new_oid()
self.assertRaises(KeyError,self._storage.history,oid)
revids = [None]
for data in data:
if sys.platform == 'win32':
# time.time() has a precision of 1ms on Windows.
sleep(0.002)
revids.append(self._dostore(oid, revids[-1], MinPO(data)))
revids.reverse()
del revids[-1]
# Now get various snapshots of the object's history
for i in range(1, 1 + len(revids)):
h = self._storage.history(oid, size=i)
self.assertEqual([d['tid'] for d in h], revids[:i])
# Check results are sorted by timestamp, in descending order.
if sys.platform == 'win32':
# Same as above. This is also required in case this method is
# called several times for the same storage.
sleep(0.002)
a = time()
for d in h:
b = a
a = d['time']
self.assertLess(a, b)
self.assertLess(start, a)
def checkUndoZombieNonVersion(self):
if not hasattr(self._storage, 'supportsTransactionalUndo'):
return
if not self._storage.supportsTransactionalUndo():
return
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(94))
# Get the undo information
info = self._storage.undoInfo()
tid = info[0]['id']
# Undo the creation of the object, rendering it a zombie
t = Transaction()
self._storage.tpc_begin(t)
oids = self._storage.undo(tid, t)
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
# Now attempt to iterator over the storage
iter = self._storage.iterator()
for txn in iter:
for rec in txn:
pass
# The last transaction performed an undo of the transaction that
# created object oid. (As Barry points out, the object is now in the
# George Bailey state.) Assert that the final data record contains
# None in the data attribute.
self.assertEqual(rec.oid, oid)
self.assertEqual(rec.data, None)
def checkTimeoutAfterVote(self):
self._storage = storage = self.openClientStorage()
# Assert that the zeo cache is empty
self.assert_(not list(storage._cache.contents()))
# Create the object
oid = storage.new_oid()
obj = MinPO(7)
# Now do a store, sleeping before the finish so as to cause a timeout
t = TransactionMetaData()
old_connection_count = storage.connection_count_for_tests
storage.tpc_begin(t)
revid1 = storage.store(oid, ZERO, zodb_pickle(obj), '', t)
storage.tpc_vote(t)
# Now sleep long enough for the storage to time out
time.sleep(3)
self.assert_(
(not storage.is_connected())
or (storage.connection_count_for_tests > old_connection_count))
storage._wait()
self.assert_(storage.is_connected())
# We expect finish to fail
self.assertRaises(ClientDisconnected, storage.tpc_finish, t)
# The cache should still be empty
self.assert_(not list(storage._cache.contents()))
# Load should fail since the object should not be in either the cache
# or the server.
self.assertRaises(KeyError, storage.load, oid, '')
def checkLoadBefore(self):
# Store 10 revisions of one object and then make sure that we
# can get all the non-current revisions back.
oid = self._storage.new_oid()
revs = []
revid = None
for i in range(10):
# We need to ensure that successive timestamps are at least
# two apart, so that a timestamp exists that's unambiguously
# between successive timestamps. Each call to snooze()
# guarantees that the next timestamp will be at least one
# larger (and probably much more than that) than the previous
# one.
snooze()
snooze()
revid = self._dostore(oid, revid, data=MinPO(i))
revs.append(self._storage.loadEx(oid, ""))
prev = u64(revs[0][1])
for i in range(1, 10):
tid = revs[i][1]
cur = u64(tid)
middle = prev + (cur - prev) // 2
assert prev < middle < cur # else the snooze() trick failed
prev = cur
t = self._storage.loadBefore(oid, p64(middle))
self.assert_(t is not None)
data, start, end = t
self.assertEqual(revs[i - 1][0], data)
self.assertEqual(tid, end)
def checkLoadBeforeConsecutiveTids(self):
eq = self.assertEqual
oid = self._storage.new_oid()
def helper(tid, revid, x):
data = zodb_pickle(MinPO(x))
t = transaction.Transaction()
try:
self._storage.tpc_begin(t, p64(tid))
r1 = self._storage.store(oid, revid, data, '', t)
# Finish the transaction
r2 = self._storage.tpc_vote(t)
newrevid = handle_serials(oid, r1, r2)
self._storage.tpc_finish(t)
except:
self._storage.tpc_abort(t)
raise
return newrevid
revid1 = helper(1, None, 1)
revid2 = helper(2, revid1, 2)
revid3 = helper(3, revid2, 3)
data, start_tid, end_tid = self._storage.loadBefore(oid, p64(2))
eq(zodb_unpickle(data), MinPO(1))
eq(u64(start_tid), 1)
eq(u64(end_tid), 2)
def checkLoadBeforeUndo(self):
# Do several transactions then undo them.
oid = self._storage.new_oid()
revid = None
for i in range(5):
revid = self._dostore(oid, revid, data=MinPO(i))
revs = []
for i in range(4):
info = self._storage.undoInfo()
tid = info[0]["id"]
# Always undo the most recent txn, so the value will
# alternate between 3 and 4.
self._undo(tid, [oid], note="undo %d" % i)
revs.append(self._storage.loadEx(oid, ""))
prev_tid = None
for i, (data, tid, ver) in enumerate(revs):
t = self._storage.loadBefore(oid, p64(u64(tid) + 1))
self.assertEqual(data, t[0])
self.assertEqual(tid, t[1])
if prev_tid:
self.assert_(prev_tid < t[1])
prev_tid = t[1]
if i < 3:
self.assertEqual(revs[i + 1][1], t[2])
else:
self.assertEqual(None, t[2])
def testLRU(self):
# verify the LRU behavior of the cache
dataset_size = 5
CACHE_SIZE = dataset_size * 2 + 1
# a cache big enough to hold the objects added in two
# transactions, plus the root object
self.db.setCacheSize(CACHE_SIZE)
c = self.db.open()
r = c.root()
l = {}
# the root is the only thing in the cache, because all the
# other objects are new
self.assertEqual(len(c._cache), 1)
# run several transactions
for t in range(5):
for i in range(dataset_size):
l[(t, i)] = r[i] = MinPO(i)
transaction.commit()
# commit() will register the objects, placing them in the
# cache. at the end of commit, the cache will be reduced
# down to CACHE_SIZE items
if len(l) > CACHE_SIZE:
self.assertEqual(c._cache.ringlen(), CACHE_SIZE)
for i in range(dataset_size):
# Check objects added in the first two transactions.
# They must all be ghostified.
self.assertEqual(l[(0, i)]._p_changed, None)
self.assertEqual(l[(1, i)]._p_changed, None)
# Check objects added in the last two transactions.
# They must all still exist in memory, but have
# had their changes flushed
self.assertEqual(l[(3, i)]._p_changed, 0)
self.assertEqual(l[(4, i)]._p_changed, 0)
def test_ConflictErrorDoesntImport(self):
from ZODB.serialize import ObjectWriter
from ZODB.POSException import ConflictError
from ZODB.tests.MinPO import MinPO
obj = MinPO()
data = ObjectWriter().serialize(obj)
# The pickle contains a GLOBAL ('c') opcode resolving to MinPO's
# module and class.
self.assertTrue(b'cZODB.tests.MinPO\nMinPO\n' in data)
# Fiddle the pickle so it points to something "impossible" instead.
data = data.replace(
b'cZODB.tests.MinPO\nMinPO\n',
b'cpath.that.does.not.exist\nlikewise.the.class\n')
# Pickle can't resolve that GLOBAL opcode -- gets ImportError.
self.assertRaises(ImportError, loads, data)
# Verify that building ConflictError doesn't get ImportError.
try:
raise ConflictError(object=obj, data=data)
except ConflictError as detail:
# And verify that the msg names the impossible path.
self.assertTrue(
'path.that.does.not.exist.likewise.the.class' in str(detail))
else:
self.fail("expected ConflictError, but no exception raised")
def __init__(self, value):
MinPO.__init__(self, value)
self.an_attribute = 42
def checkTimeoutProvokingConflicts(self):
self._storage = storage = self.openClientStorage()
# Assert that the zeo cache is empty.
self.assert_(not list(storage._cache.contents()))
# Create the object
oid = storage.new_oid()
obj = MinPO(7)
# We need to successfully commit an object now so we have something to
# conflict about.
t = Transaction()
storage.tpc_begin(t)
revid1a = storage.store(oid, ZERO, zodb_pickle(obj), '', t)
revid1b = storage.tpc_vote(t)
revid1 = handle_serials(oid, revid1a, revid1b)
storage.tpc_finish(t)
# Now do a store, sleeping before the finish so as to cause a timeout.
obj.value = 8
t = Transaction()
old_connection_count = storage.connection_count_for_tests
storage.tpc_begin(t)
revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t)
revid2b = storage.tpc_vote(t)
revid2 = handle_serials(oid, revid2a, revid2b)
# Now sleep long enough for the storage to time out.
# This used to sleep for 3 seconds, and sometimes (but very rarely)
# failed then. Now we try for a minute. It typically succeeds
# on the second time thru the loop, and, since self.timeout is 1,
# it's typically faster now (2/1.8 ~= 1.11 seconds sleeping instead
# of 3).
deadline = time.time() + 60 # wait up to a minute
while time.time() < deadline:
if (storage.is_connected() and
(storage.connection_count_for_tests == old_connection_count)
):
time.sleep(self.timeout / 1.8)
else:
break
self.assert_(
(not storage.is_connected())
or
(storage.connection_count_for_tests > old_connection_count)
)
storage._wait()
self.assert_(storage.is_connected())
# We expect finish to fail.
self.assertRaises(ClientDisconnected, storage.tpc_finish, t)
storage.tpc_abort(t)
# Now we think we've committed the second transaction, but we really
# haven't. A third one should produce a POSKeyError on the server,
# which manifests as a ConflictError on the client.
obj.value = 9
t = Transaction()
storage.tpc_begin(t)
storage.store(oid, revid2, zodb_pickle(obj), '', t)
self.assertRaises(ConflictError, storage.tpc_vote, t)
# Even aborting won't help.
storage.tpc_abort(t)
self.assertRaises(ZODB.POSException.StorageTransactionError,
storage.tpc_finish, t)
# Try again.
obj.value = 10
t = Transaction()
storage.tpc_begin(t)
storage.store(oid, revid2, zodb_pickle(obj), '', t)
# Even aborting won't help.
self.assertRaises(ConflictError, storage.tpc_vote, t)
# Abort this one and try a transaction that should succeed.
storage.tpc_abort(t)
# Now do a store.
obj.value = 11
t = Transaction()
storage.tpc_begin(t)
revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t)
revid2b = storage.tpc_vote(t)
revid2 = handle_serials(oid, revid2a, revid2b)
storage.tpc_finish(t)
# Now load the object and verify that it has a value of 11.
data, revid = storage.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO(11))
self.assertEqual(revid, revid2)
def _PackWhileWriting(self, pack_now):
# A storage should allow some reading and writing during
# a pack. This test attempts to exercise locking code
# in the storage to test that it is safe. It generates
# a lot of revisions, so that pack takes a long time.
db = DB(self._storage)
conn = db.open()
root = conn.root()
for i in range(10):
root[i] = MinPO(i)
transaction.commit()
snooze()
packt = time.time()
choices = list(range(10))
for dummy in choices:
for i in choices:
root[i].value = MinPO(i)
transaction.commit()
# How many client threads should we run, and how long should we
# wait for them to finish? Hard to say. Running 4 threads and
# waiting 30 seconds too often left a thread still alive on Tim's
# Win98SE box, during ZEO flavors of this test. Those tend to
# run one thread at a time to completion, and take about 10 seconds
# per thread. There doesn't appear to be a compelling reason to
# run that many threads. Running 3 threads and waiting up to a
# minute seems to work well in practice. The ZEO tests normally
# finish faster than that, and the non-ZEO tests very much faster
# than that.
NUM_LOOP_TRIP = 50
timer = ElapsedTimer(time.time())
threads = [ClientThread(db, choices, NUM_LOOP_TRIP, timer, i)
for i in range(3)]
for t in threads:
t.start()
if pack_now:
db.pack(time.time())
else:
db.pack(packt)
for t in threads:
t.join(60)
liveness = [t.isAlive() for t in threads]
if True in liveness:
# They should have finished by now.
print('Liveness:', liveness)
# Combine the outcomes, and sort by start time.
outcomes = []
for t in threads:
outcomes.extend(t.outcomes)
# each outcome list has as many of these as a loop trip got thru:
# thread_id
# elapsed millis at loop top
# elapsed millis at attempt to assign to self.root[index]
# index into self.root getting replaced
# elapsed millis when outcome known
# 'OK' or 'Conflict'
# True if we got beyond this line, False if it raised an
# exception (one possible Conflict cause):
# self.root[index].value = MinPO(j)
def cmp_by_time(a, b):
return cmp((a[1], a[0]), (b[1], b[0]))
outcomes.sort(cmp_by_time)
counts = [0] * 4
for outcome in outcomes:
n = len(outcome)
assert n >= 2
tid = outcome[0]
print('tid:%d top:%5d' % (tid, outcome[1]), end=' ')
if n > 2:
print('commit:%5d' % outcome[2], end=' ')
if n > 3:
print('index:%2d' % outcome[3], end=' ')
if n > 4:
print('known:%5d' % outcome[4], end=' ')
if n > 5:
print('%8s' % outcome[5], end=' ')
if n > 6:
print('assigned:%5s' % outcome[6], end=' ')
counts[tid] += 1
if counts[tid] == NUM_LOOP_TRIP:
print('thread %d done' % tid, end=' ')
print()
self.fail('a thread is still alive')
self._sanity_check()
db.close()