def undo(i):
info = s.undoInfo()
t = TransactionMetaData()
s.tpc_begin(t)
base = i * OBJECTS + i
for j in range(OBJECTS):
tid = info[base + j]['id']
s.undo(tid, t)
s.tpc_vote(t)
s.tpc_finish(t)
def checkWriteMethods(self):
self._make_readonly()
self.assertRaises(ReadOnlyError, self._storage.new_oid)
t = TransactionMetaData()
self.assertRaises(ReadOnlyError, self._storage.tpc_begin, t)
self.assertRaises(ReadOnlyError, self._storage.store, b'\000' * 8,
None, b'', '', t)
self.assertRaises(ReadOnlyError, self._storage.undo, b'\000' * 8, t)
def checkWriteAfterAbort(self):
oid = self._storage.new_oid()
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
# Now abort this transaction
self._storage.tpc_abort(t)
# Now start all over again
oid = self._storage.new_oid()
self._dostore(oid=oid, data=MinPO(6))
def store(storage, oid, value='x', serial=ZODB.utils.z64):
if not isinstance(oid, bytes):
oid = ZODB.utils.p64(oid)
if not isinstance(serial, bytes):
serial = ZODB.utils.p64(serial)
t = TransactionMetaData()
storage.tpc_begin(t)
storage.store(oid, serial, value, '', t)
storage.tpc_vote(t)
storage.tpc_finish(t)
def checkTransactionIdIncreases(self):
import time
from ZODB.utils import newTid
from ZODB.TimeStamp import TimeStamp
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
# Add a fake transaction
transactions = self._storage._transactions
self.assertEqual(1, len(transactions))
fake_timestamp = b'zzzzzzzy' # the year 5735 ;-)
transactions[fake_timestamp] = transactions.values()[0]
# Verify the next transaction comes after the fake transaction
t = TransactionMetaData()
self._storage.tpc_begin(t)
self.assertEqual(self._storage._tid, b'zzzzzzzz')
def __read_current_and_lock(self, storage, read_current_oid, lock_oid, tid,
begin=True, tx=None):
tx = tx if tx is not None else TransactionMetaData()
if begin:
storage.tpc_begin(tx)
if read_current_oid is not None:
storage.checkCurrentSerialInTransaction(read_current_oid, tid, tx)
storage.store(lock_oid, tid, b'bad pickle2', '', tx)
storage.tpc_vote(tx)
return tx
def checkanalyze(self):
import types
from BTrees.OOBTree import OOBTree
from ZODB.scripts import analyze
# Set up a module to act as a broken import
module_name = 'brokenmodule'
module = types.ModuleType(module_name)
sys.modules[module_name] = module
class Broken(MinPO):
__module__ = module_name
module.Broken = Broken
oids = [[self._storage.new_oid(), None] for i in range(3)]
def store(i, data):
oid, revid = oids[i]
self._storage.store(oid, revid, data, "", t)
for i in range(2):
t = TransactionMetaData()
self._storage.tpc_begin(t)
# sometimes data is in this format
store(0, dumps(OOBTree, _protocol))
# and it could be from a broken module
store(1, dumps(Broken, _protocol))
# but mostly it looks like this
store(2, zodb_pickle(MinPO(2)))
self._storage.tpc_vote(t)
tid = self._storage.tpc_finish(t)
for oid_revid in oids:
oid_revid[1] = tid
# now break the import of the Broken class
del sys.modules[module_name]
# from ZODB.scripts.analyze.analyze
fsi = self._storage.iterator()
rep = analyze.Report()
for txn in fsi:
analyze.analyze_trans(rep, txn)
# from ZODB.scripts.analyze.report
typemap = sorted(rep.TYPEMAP.keys())
cumpct = 0.0
for t in typemap:
pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES
cumpct += pct
self.assertAlmostEqual(cumpct, 100.0, 0,
"Failed to analyze some records")
def run(self):
tname = self.getName()
testcase = self.testcase
# Create client connections to each server
clients = self.clients
for i in range(len(testcase.addr)):
c = testcase.openClientStorage(addr=testcase.addr[i])
c.__name = "C%d" % i
clients.append(c)
for i in range(testcase.ntrans):
# Because we want a transaction spanning all storages,
# we can't use _dostore(). This is several _dostore() calls
# expanded in-line (mostly).
# Create oid->serial mappings
for c in clients:
c.__oids = []
c.__serials = {}
# Begin a transaction
t = TransactionMetaData()
for c in clients:
#print("%s.%s.%s begin" % (tname, c.__name, i))
c.tpc_begin(t)
for j in range(testcase.nobj):
for c in clients:
# Create and store a new object on each server
oid = c.new_oid()
c.__oids.append(oid)
data = MinPO("%s.%s.t%d.o%d" % (tname, c.__name, i, j))
#print(data.value)
data = zodb_pickle(data)
c.store(oid, ZERO, data, '', t)
# Vote on all servers and handle serials
for c in clients:
#print("%s.%s.%s vote" % (tname, c.__name, i))
c.tpc_vote(t)
# Finish on all servers
for c in clients:
#print("%s.%s.%s finish\n" % (tname, c.__name, i))
c.tpc_finish(t)
for c in clients:
# Check that we got serials for all oids
for oid in c.__oids:
testcase.failUnless(oid in c.__serials)
# Check that we got serials for no other oids
for oid in c.__serials.keys():
testcase.failUnless(oid in c.__oids)
def checkTwoObjectUndoAtOnce(self):
# Convenience
eq = self.assertEqual
unless = self.assertTrue
p30, p31, p32, p50, p51, p52 = map(
zodb_pickle, map(MinPO, (30, 31, 32, 50, 51, 52)))
oid1 = self._storage.new_oid()
oid2 = self._storage.new_oid()
# Store two objects in the same transaction
tid = self._multi_obj_transaction([
(oid1, ZERO, p30),
(oid2, ZERO, p50),
])
# Update those same two objects
tid = self._multi_obj_transaction([
(oid1, tid, p31),
(oid2, tid, p51),
])
# Update those same two objects
tid = self._multi_obj_transaction([
(oid1, tid, p32),
(oid2, tid, p52),
])
# Make sure the objects have the current value
data, revid1 = load_current(self._storage, oid1)
eq(zodb_unpickle(data), MinPO(32))
data, revid2 = load_current(self._storage, oid2)
eq(zodb_unpickle(data), MinPO(52))
# Now attempt to undo the transaction containing two objects
info = self._storage.undoInfo()
tid = info[0]['id']
tid1 = info[1]['id']
t = TransactionMetaData()
oids = self._begin_undos_vote(t, tid, tid1)
serial = self._storage.tpc_finish(t)
# We may get the finalization stuff called an extra time,
# depending on the implementation.
if serial is None:
self.assertEqual(oids, {oid1, oid2})
data, revid1 = load_current(self._storage, oid1)
eq(zodb_unpickle(data), MinPO(30))
data, revid2 = load_current(self._storage, oid2)
eq(zodb_unpickle(data), MinPO(50))
# Now try to undo the one we just did to undo, whew
info = self._storage.undoInfo()
self._undo(info[0]['id'], [oid1, oid2])
data, revid1 = load_current(self._storage, oid1)
eq(zodb_unpickle(data), MinPO(32))
data, revid2 = load_current(self._storage, oid2)
eq(zodb_unpickle(data), MinPO(52))
self._iterate()
def helper(tid, revid, x):
data = zodb_pickle(MinPO(x))
t = TransactionMetaData()
try:
self._storage.tpc_begin(t, p64(tid))
self._storage.store(oid, revid, data, '', t)
# Finish the transaction
self._storage.tpc_vote(t)
newrevid = self._storage.tpc_finish(t)
except:
self._storage.tpc_abort(t)
raise
return newrevid
def _undo(self, tid, expected_oids=None, note=None):
# Undo a tid that affects a single object (oid).
# This is very specialized.
t = TransactionMetaData()
t.note(note or u"undo")
self._storage.tpc_begin(t)
undo_result = self._storage.undo(tid, t)
vote_result = self._storage.tpc_vote(t)
if expected_oids is not None:
oids = set(undo_result[1]) if undo_result else set()
if vote_result:
oids.update(vote_result)
self.assertEqual(oids, set(expected_oids))
return self._storage.tpc_finish(t)
def checkBasics(self):
self.assertEqual(self._storage.lastTransaction(), ZERO)
t = TransactionMetaData()
self._storage.tpc_begin(t)
self.assertRaises(POSException.StorageTransactionError,
self._storage.tpc_begin, t)
# Aborting is easy
self._storage.tpc_abort(t)
# Test a few expected exceptions when we're doing operations giving a
# different Transaction object than the one we've begun on.
self._storage.tpc_begin(t)
self.assertRaises(POSException.StorageTransactionError,
self._storage.store, ZERO, ZERO, b'', '',
TransactionMetaData())
self.assertRaises(POSException.StorageTransactionError,
self._storage.store, ZERO, 1, b'2', '',
TransactionMetaData())
self.assertRaises(POSException.StorageTransactionError,
self._storage.tpc_vote, TransactionMetaData())
self._storage.tpc_abort(t)
def checkSerialIsNoneForInitialRevision(self):
eq = self.assertEqual
oid = self._storage.new_oid()
txn = TransactionMetaData()
self._storage.tpc_begin(txn)
# Use None for serial. Don't use _dostore() here because that coerces
# serial=None to serial=ZERO.
self._storage.store(oid, None, zodb_pickle(MinPO(11)), '', txn)
self._storage.tpc_vote(txn)
newrevid = self._storage.tpc_finish(txn)
data, revid = utils.load_current(self._storage, oid)
value = zodb_unpickle(data)
eq(value, MinPO(11))
eq(revid, newrevid)
def checkTwoObjectUndo(self):
eq = self.assertEqual
# Convenience
p31, p32, p51, p52 = map(zodb_pickle, map(MinPO, (31, 32, 51, 52)))
oid1 = self._storage.new_oid()
oid2 = self._storage.new_oid()
revid1 = revid2 = ZERO
# Store two objects in the same transaction
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.store(oid1, revid1, p31, '', t)
self._storage.store(oid2, revid2, p51, '', t)
# Finish the transaction
self._storage.tpc_vote(t)
tid = self._storage.tpc_finish(t)
# Update those same two objects
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.store(oid1, tid, p32, '', t)
self._storage.store(oid2, tid, p52, '', t)
# Finish the transaction
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
# Make sure the objects have the current value
data, revid1 = load_current(self._storage, oid1)
eq(zodb_unpickle(data), MinPO(32))
data, revid2 = load_current(self._storage, oid2)
eq(zodb_unpickle(data), MinPO(52))
# Now attempt to undo the transaction containing two objects
info = self._storage.undoInfo()
self._undo(info[0]['id'], [oid1, oid2])
data, revid1 = load_current(self._storage, oid1)
eq(zodb_unpickle(data), MinPO(31))
data, revid2 = load_current(self._storage, oid2)
eq(zodb_unpickle(data), MinPO(51))
self._iterate()
def checkStoreBumpsOid(self):
# If .store() is handed an oid bigger than the storage knows
# about already, it's crucial that the storage bump its notion
# of the largest oid in use.
t = TransactionMetaData()
self._storage.tpc_begin(t)
giant_oid = b'\xee' * 8
# Store an object.
# oid, serial, data, version, transaction
r1 = self._storage.store(giant_oid, b'\0'*8, b'data', b'', t)
# Finish the transaction.
r2 = self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
# Before ZODB 3.2.6, this failed, with ._oid == z64.
self.assertEqual(self._storage._oid, giant_oid)
def _initroot(self):
try:
load_current(self._storage, ZERO)
except KeyError:
from ZODB.Connection import TransactionMetaData
file = BytesIO()
p = Pickler(file, _protocol)
p.dump((PersistentMapping, None))
p.dump({'_container': {}})
t = TransactionMetaData()
t.description = u'initial database creation'
self._storage.tpc_begin(t)
self._storage.store(ZERO, None, file.getvalue(), '', t)
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
def checkDisconnectedAbort(self):
self._storage = self.openClientStorage()
self._dostore()
oids = [self._storage.new_oid() for i in range(5)]
txn = TransactionMetaData()
self._storage.tpc_begin(txn)
for oid in oids:
data = zodb_pickle(MinPO(oid))
self._storage.store(oid, None, data, '', txn)
self.shutdownServer()
with short_timeout(self):
self.assertRaises(ClientDisconnected, self._storage.tpc_vote, txn)
self.startServer(create=0)
self._storage.tpc_abort(txn)
self._dostore()
def checkIteratorGCStorageTPCAborting(self):
# The odd little jig we do below arises from the fact that the
# CS iterator may not be constructed right away if the CS is wrapped.
# We need to actually do some iteration to get the iterator created.
# We do a store to make sure the iterator isn't exhausted right away.
self._dostore()
six.advance_iterator(self._storage.iterator())
iid = list(self._storage._iterator_ids)[0]
t = TransactionMetaData()
self._storage._iterators._last_gc = -1
self._storage.tpc_begin(t)
self._storage.tpc_abort(t)
self._assertIteratorIdsEmpty()
self.assertRaises(KeyError, self._storage._call, 'iterator_next', iid)
def checkRestoreBumpsOid(self):
# As above, if .restore() is handed an oid bigger than the storage
# knows about already, it's crucial that the storage bump its notion
# of the largest oid in use. Because copyTransactionsFrom(), and
# ZRS recovery, use the .restore() method, this is plain critical.
t = TransactionMetaData()
self._storage.tpc_begin(t)
giant_oid = b'\xee' * 8
# Store an object.
# oid, serial, data, version, prev_txn, transaction
r1 = self._storage.restore(giant_oid, b'\0'*8, b'data', b'', None, t)
# Finish the transaction.
r2 = self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
# Before ZODB 3.2.6, this failed, with ._oid == z64.
self.assertEqual(self._storage._oid, giant_oid)
def checkIteratorGCStorageDisconnect(self):
# The odd little jig we do below arises from the fact that the
# CS iterator may not be constructed right away if the CS is wrapped.
# We need to actually do some iteration to get the iterator created.
# We do a store to make sure the iterator isn't exhausted right away.
self._dostore()
six.advance_iterator(self._storage.iterator())
iid = list(self._storage._iterator_ids)[0]
t = TransactionMetaData()
self._storage.tpc_begin(t)
# Show that after disconnecting, the client side GCs the iterators
# as well. I'm calling this directly to avoid accidentally
# calling tpc_abort implicitly.
self._storage.notify_disconnected()
self.assertEquals(0, len(self._storage._iterator_ids))
def checkFlushAfterTruncate(self, fail=False):
r0 = self._dostore(z64)
storage = self._storage
t = TransactionMetaData()
storage.tpc_begin(t)
storage.store(z64, r0, b'foo', b'', t)
storage.tpc_vote(t)
# Read operations are done with separate 'file' objects with their
# own buffers: here, the buffer also includes voted data.
load_current(storage, z64)
# This must invalidate all read buffers.
storage.tpc_abort(t)
self._dostore(z64, r0, b'bar', 1)
# In the case that read buffers were not invalidated, return value
# is based on what was cached during the first load.
self.assertEqual(load_current(storage, z64)[0],
b'foo' if fail else b'bar')
def checkAbortAfterVote(self):
oid1 = self._storage.new_oid()
revid1 = self._dostore(oid=oid1, data=MinPO(-2))
oid = self._storage.new_oid()
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
# Now abort this transaction
self._storage.tpc_vote(t)
self._storage.tpc_abort(t)
# Now start all over again
oid = self._storage.new_oid()
revid = self._dostore(oid=oid, data=MinPO(6))
for oid, revid in [(oid1, revid1), (oid, revid)]:
data, _revid = utils.load_current(self._storage, oid)
self.assertEqual(revid, _revid)
def checkTimeout(self):
self._storage = storage = self.openClientStorage()
txn = TransactionMetaData()
storage.tpc_begin(txn)
storage.tpc_vote(txn)
time.sleep(2)
with short_timeout(self):
self.assertRaises(ClientDisconnected, storage.tpc_finish, txn)
# Make sure it's logged as CRITICAL
for line in open("server.log"):
if (('Transaction timeout after' in line)
and ('CRITICAL ZEO.StorageServer' in line)):
break
else:
self.assert_(False, 'bad logging')
storage.close()
def checkIterationIntraTransaction(self):
# XXX: This test overrides the broken version from
# IteratorStorage; prior to
# https://github.com/zopefoundation/ZODB/pull/281 it passed a
# native str, not bytes, as the previous tid.
oid = self._storage.new_oid()
t = TransactionMetaData()
data = zodb_pickle(MinPO(0))
try:
self._storage.tpc_begin(t)
self._storage.store(oid, RevisionStorage.ZERO, data, '', t)
self._storage.tpc_vote(t)
# Don't do tpc_finish yet
it = self._storage.iterator()
for x in it:
self.assertIsNotNone(x)
finally:
self._storage.tpc_finish(t)
def checkUndoUnresolvable(self):
# This test is based on checkNotUndoable in the
# TransactionalUndoStorage test suite. Except here, conflict
# resolution should allow us to undo the transaction anyway.
obj = PCounter2()
obj.inc()
oid = self._storage.new_oid()
revid_a = self._dostore(oid, data=obj)
obj.inc()
revid_b = self._dostore(oid, revid=revid_a, data=obj)
obj.inc()
revid_c = self._dostore(oid, revid=revid_b, data=obj)
# Start the undo
info = self._storage.undoInfo()
tid = info[1]['id']
t = TransactionMetaData()
self.assertRaises(UndoError, self._begin_undos_vote, t, tid)
self._storage.tpc_abort(t)
def dostore(self, i):
data = zodb_pickle(MinPO((self.getName(), i)))
t = TransactionMetaData()
oid = self.oid()
self.pause()
self.storage.tpc_begin(t)
self.pause()
# Always create a new object, signified by None for revid
self.storage.store(oid, None, data, '', t)
self.pause()
self.storage.tpc_vote(t)
self.pause()
revid = self.storage.tpc_finish(t)
self.pause()
self.oids[oid] = revid
def main():
if len(sys.argv) not in (3, 4):
sys.stderr.write("Usage: timeout.py address delay [storage-name]\n" %
sys.argv[0])
sys.exit(2)
hostport = sys.argv[1]
delay = float(sys.argv[2])
if sys.argv[3:]:
name = sys.argv[3]
else:
name = "1"
if "/" in hostport:
address = hostport
else:
if ":" in hostport:
i = hostport.index(":")
host, port = hostport[:i], hostport[i+1:]
else:
host, port = "", hostport
port = int(port)
address = (host, port)
print("Connecting to %s..." % repr(address))
storage = ClientStorage(address, name)
print("Connected. Now starting a transaction...")
oid = storage.new_oid()
revid = ZERO
data = MinPO("timeout.py")
pickled_data = zodb_pickle(data)
t = TransactionMetaData()
t.user = "******"
storage.tpc_begin(t)
storage.store(oid, revid, pickled_data, '', t)
print("Stored. Now voting...")
storage.tpc_vote(t)
print("Voted; now sleeping %s..." % delay)
time.sleep(delay)
print("Done.")
def _dostore(self,
oid=None,
revid=None,
data=None,
already_pickled=0,
user=None,
description=None):
"""Do a complete storage transaction. The defaults are:
- oid=None, ask the storage for a new oid
- revid=None, use a revid of ZERO
- data=None, pickle up some arbitrary data (the integer 7)
Returns the object's new revision id.
"""
if oid is None:
oid = self._storage.new_oid()
if revid is None:
revid = ZERO
if data is None:
data = MinPO(7)
if type(data) == int:
data = MinPO(data)
if not already_pickled:
data = zodb_pickle(data)
# Begin the transaction
t = TransactionMetaData()
if user is not None:
t.user = user
if description is not None:
t.description = description
try:
self._storage.tpc_begin(t)
# Store an object
r1 = self._storage.store(oid, revid, data, '', t)
# Finish the transaction
r2 = self._storage.tpc_vote(t)
revid = self._storage.tpc_finish(t)
except:
self._storage.tpc_abort(t)
raise
return revid
def checkUndoConflictResolution(self):
# This test is based on checkNotUndoable in the
# TransactionalUndoStorage test suite. Except here, conflict
# resolution should allow us to undo the transaction anyway.
obj = PCounter()
obj.inc()
oid = self._storage.new_oid()
revid_a = self._dostore(oid, data=obj)
obj.inc()
revid_b = self._dostore(oid, revid=revid_a, data=obj)
obj.inc()
revid_c = self._dostore(oid, revid=revid_b, data=obj)
# Start the undo
info = self._storage.undoInfo()
tid = info[1]['id']
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.undo(tid, t)
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
def test_check_refs_missing_after_prepack(self):
from ZODB.Connection import TransactionMetaData
expect_oids = self._create_initial_state()
# 0 1
# T2: root -> A
# \-> B -> D -> C -> E
# 2 4 3 5
def hook(conn):
conn.root.B['D']['C']['E'] = PersistentMapping()
self._mutate_state(expect_oids, save_B=True, before_commit=hook)
# Even if pack_gc is off, we find the references.
self._storage._options.pack_gc = False
self._storage.pack(None, referencesf, prepack_only=True)
# 0 1
# T2: root -> A
# \-> B -> -> C -> E
# 2 4 3 5
_state, tid = self._storage.load(expect_oids['D'], '')
txn_meta = TransactionMetaData()
self._storage.tpc_begin(txn_meta)
self._storage.deleteObject(expect_oids['D'], tid, txn_meta)
self._storage.tpc_vote(txn_meta)
self._storage.tpc_finish(txn_meta)
missing = self._storage.pack(None, referencesf, check_refs=True, skip_prepack=True)
self.assertTrue(missing)
# TODO: Define this data structure.
self.assertEqual([(2, 4)], missing)
missing = self._storage.pack(None, referencesf, check_refs=True)
self.assertTrue(missing)
self.assertEqual([(2, 4)], missing)
