def undo(self, tid, note=None):
t = TransactionMetaData()
if note is not None:
t.note(note)
oids = self._begin_undos_vote(t, tid)
self._storage.tpc_finish(t)
return oids
def checkUndoInvalidation(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(23))
revid = self._dostore(oid, revid=revid, data=MinPO(24))
revid = self._dostore(oid, revid=revid, data=MinPO(25))
info = self._storage.undoInfo()
if not info:
# Preserved this comment, but don't understand it:
# "Perhaps we have an old storage implementation that
# does do the negative nonsense."
info = self._storage.undoInfo(0, 20)
tid = info[0]['id']
# Now start an undo transaction
t = TransactionMetaData()
t.note(u'undo1')
oids = self._begin_undos_vote(t, tid)
# Make sure this doesn't load invalid data into the cache
self._storage.load(oid, '')
self._storage.tpc_finish(t)
[uoid] = oids
assert uoid == oid
data, revid = self._storage.load(oid, '')
obj = zodb_unpickle(data)
assert obj == MinPO(24)
def undo(self, tid, note=None):
t = TransactionMetaData()
if note is not None:
t.note(note)
oids = self._begin_undos_vote(t, tid)
self._storage.tpc_finish(t)
return oids
def checkBasics(self):
self.assertEqual(self._storage.lastTransaction(), ZERO)
t = TransactionMetaData()
self._storage.tpc_begin(t)
self.assertRaises(POSException.StorageTransactionError,
self._storage.tpc_begin, t)
# Aborting is easy
self._storage.tpc_abort(t)
# Test a few expected exceptions when we're doing operations giving a
# different Transaction object than the one we've begun on.
self._storage.tpc_begin(t)
self.assertRaises(
POSException.StorageTransactionError,
self._storage.store,
ZERO, ZERO, b'', '', TransactionMetaData())
self.assertRaises(
POSException.StorageTransactionError,
self._storage.store,
ZERO, 1, b'2', '', TransactionMetaData())
self.assertRaises(
POSException.StorageTransactionError,
self._storage.tpc_vote, TransactionMetaData())
self._storage.tpc_abort(t)
def checkUndoInvalidation(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(23))
revid = self._dostore(oid, revid=revid, data=MinPO(24))
revid = self._dostore(oid, revid=revid, data=MinPO(25))
info = self._storage.undoInfo()
if not info:
# Preserved this comment, but don't understand it:
# "Perhaps we have an old storage implementation that
# does do the negative nonsense."
info = self._storage.undoInfo(0, 20)
tid = info[0]['id']
# Now start an undo transaction
t = TransactionMetaData()
t.note(u'undo1')
oids = self._begin_undos_vote(t, tid)
# Make sure this doesn't load invalid data into the cache
self._storage.load(oid, '')
self._storage.tpc_finish(t)
[uoid] = oids
assert uoid == oid
data, revid = self._storage.load(oid, '')
obj = zodb_unpickle(data)
assert obj == MinPO(24)
def tpc_begin(self, id, user, description, ext, tid=None, status=" "):
if self.read_only:
raise ReadOnlyError()
if self.transaction is not None:
if self.transaction.id == id:
self.log("duplicate tpc_begin(%s)" % repr(id))
return
else:
raise StorageTransactionError("Multiple simultaneous tpc_begin"
" requests from one client.")
t = TransactionMetaData(user, description, ext)
t.id = id
self.serials = []
self.conflicts = {}
self.invalidated = []
self.txnlog = CommitLog()
self.blob_log = []
self.tid = tid
self.status = status
self.stats.active_txns += 1
# Assign the transaction attribute last. This is so we don't
# think we've entered TPC until everything is set. Why?
# Because if we have an error after this, the server will
# think it is in TPC and the client will think it isn't. At
# that point, the client will keep trying to enter TPC and
# server won't let it. Errors *after* the tpc_begin call will
# cause the client to abort the transaction.
# (Also see https://bugs.launchpad.net/zodb/+bug/374737.)
self.transaction = t
def checkNote(self):
oid = self._storage.new_oid()
t = TransactionMetaData()
self._storage.tpc_begin(t)
t.note(u'this is a test')
self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
def checkNote(self):
oid = self._storage.new_oid()
t = TransactionMetaData()
self._storage.tpc_begin(t)
t.note(u'this is a test')
self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
def checkMultipleEmptyTransactions(self):
# There was a bug in handling empty transactions in mapping
# storage that caused the commit lock not to be released. :(
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
t = TransactionMetaData()
self._storage.tpc_begin(t) # Hung here before
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
def _undo(self, tid, expected_oids=None, note=None):
# Undo a tid that affects a single object (oid).
# This is very specialized.
t = TransactionMetaData()
t.note(note or u"undo")
self._storage.tpc_begin(t)
undo_result = self._storage.undo(tid, t)
vote_result = self._storage.tpc_vote(t)
if expected_oids is not None:
oids = set(undo_result[1]) if undo_result else set()
if vote_result:
oids.update(vote_result)
self.assertEqual(oids, set(expected_oids))
return self._storage.tpc_finish(t)
def _undo(self, tid, expected_oids=None, note=None):
# Undo a tid that affects a single object (oid).
# This is very specialized.
t = TransactionMetaData()
t.note(note or u"undo")
self._storage.tpc_begin(t)
undo_result = self._storage.undo(tid, t)
vote_result = self._storage.tpc_vote(t)
if expected_oids is not None:
oids = set(undo_result[1]) if undo_result else set()
if vote_result:
oids.update(vote_result)
self.assertEqual(oids, set(expected_oids))
return self._storage.tpc_finish(t)
def _initroot(self):
try:
load_current(self._storage, ZERO)
except KeyError:
from ZODB.Connection import TransactionMetaData
file = BytesIO()
p = Pickler(file, _protocol)
p.dump((PersistentMapping, None))
p.dump({'_container': {}})
t = TransactionMetaData()
t.description = u'initial database creation'
self._storage.tpc_begin(t)
self._storage.store(ZERO, None, file.getvalue(), '', t)
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
def _initroot(self):
try:
load_current(self._storage, ZERO)
except KeyError:
from ZODB.Connection import TransactionMetaData
file = BytesIO()
p = Pickler(file, _protocol)
p.dump((PersistentMapping, None))
p.dump({'_container': {}})
t = TransactionMetaData()
t.description = u'initial database creation'
self._storage.tpc_begin(t)
self._storage.store(ZERO, None, file.getvalue(), '', t)
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
def checkImplementsIExternalGC(self):
db, conn, obj_oid = self.__setup_checkImplementsIExternalGC()
storage = conn._storage
history = storage.history(obj_oid, size=100)
self.assertEqual(6, len(history))
latest_tid = history[0]['tid']
# We can delete the latest TID for the OID, and the whole
# object goes away on a pack.
t = TransactionMetaData()
storage.tpc_begin(t)
count = storage.deleteObject(obj_oid, latest_tid, t)
self.assertEqual(count, 1)
# Doing it again will do nothing because it's already
# gone.
count = storage.deleteObject(obj_oid, latest_tid, t)
storage.tpc_vote(t)
storage.tpc_finish(t)
# Getting the most recent fails.
with self.assertRaises(POSKeyError):
storage.load(obj_oid)
# But we can load a state before then.
state = storage.loadSerial(obj_oid, history[1]['tid'])
self.assertEqual(len(state), history[1]['size'])
# Length is still 2
storage._adapter.stats.large_database_change()
self.assertEqual(len(storage), 2)
# The most recent size is 0 too
history_after = storage.history(obj_oid)
self.assertEqual(0, history_after[0]['size'])
# Now if we proceed to pack it, *without* doing a GC...
from relstorage.storage.pack import Pack
options = storage._options.copy(pack_gc=False)
self.assertFalse(options.pack_gc)
packer = Pack(options, storage._adapter, storage.blobhelper, storage._cache)
self.assertFalse(packer.options.pack_gc)
packer.pack(storage.lastTransactionInt(), referencesf)
# ... and bring the storage into the current view...
storage.sync()
# ...then the object is gone in all revisions...
with self.assertRaises(POSKeyError):
storage.load(obj_oid)
for history_item in history:
tid = history_item['tid']
with self.assertRaises(POSKeyError):
storage.loadSerial(obj_oid, tid)
# ...and the size is smaller.
self.assertEqual(len(storage), 1)
conn.close()
db.close()
def tpc_begin(self, transaction):
tdata = TransactionMetaData(
transaction.user,
transaction.description,
transaction.extension)
transaction.set_data(self, tdata)
self._storage.tpc_begin(tdata)
def checkImplementsIExternalGC(self):
from zope.interface.verify import verifyObject
import ZODB.interfaces
verifyObject(ZODB.interfaces.IExternalGC, self._storage)
# Now do it.
from ZODB.utils import z64
db = self._closing(ZODB.DB(self._storage)) # create the root
conn = self._closing(db.open())
storage = conn._storage
_state, tid = storage.load(z64)
# copied from zc.zodbdgc
t = TransactionMetaData()
storage.tpc_begin(t)
count = storage.deleteObject(z64, tid, t)
self.assertEqual(count, 1)
# Doing it again will do nothing because it's already
# gone.
count = storage.deleteObject(z64, tid, t)
self.assertEqual(count, 0)
storage.tpc_vote(t)
storage.tpc_finish(t)
with self.assertRaises(POSKeyError):
storage.load(z64)
conn.close()
db.close()
def checkCommitWithEmptyData(self):
"""
Verify that transaction is persisted even if it has no data, or even
both no data and empty metadata.
"""
# verify:
# - commit with empty data but non-empty metadata
# - commit with empty data and empty metadata
# (the fact of commit carries information by itself)
stor = self._storage
for description in (u'commit with empty data', u''):
t = TransactionMetaData(description=description)
stor.tpc_begin(t)
stor.tpc_vote(t)
head = stor.tpc_finish(t)
self.assertEqual(head, stor.lastTransaction())
v = list(stor.iterator(start=head, stop=head))
self.assertEqual(len(v), 1)
trec = v[
0] # FileStorage.TransactionRecord or hexstorage.Transaction
self.assertEqual(trec.tid, head)
self.assertEqual(trec.user, b'')
self.assertEqual(trec.description, description.encode('utf-8'))
self.assertEqual(trec.extension, {})
drecv = list(trec)
self.assertEqual(drecv, [])
def checkTimeoutOnAbort(self):
storage = self.openClientStorage()
txn = TransactionMetaData()
storage.tpc_begin(txn)
storage.tpc_vote(txn)
storage.tpc_abort(txn)
storage.close()
def _multi_obj_transaction(self, objs):
t = TransactionMetaData()
self._storage.tpc_begin(t)
for oid, rev, data in objs:
self._storage.store(oid, rev, data, '', t)
self._storage.tpc_vote(t)
return self._storage.tpc_finish(t)
def checkTimeoutAfterVote(self):
self._storage = storage = self.openClientStorage()
# Assert that the zeo cache is empty
self.assert_(not list(storage._cache.contents()))
# Create the object
oid = storage.new_oid()
obj = MinPO(7)
# Now do a store, sleeping before the finish so as to cause a timeout
t = TransactionMetaData()
old_connection_count = storage.connection_count_for_tests
storage.tpc_begin(t)
revid1 = storage.store(oid, ZERO, zodb_pickle(obj), '', t)
storage.tpc_vote(t)
# Now sleep long enough for the storage to time out
time.sleep(3)
self.assert_(
(not storage.is_connected())
or (storage.connection_count_for_tests > old_connection_count))
storage._wait()
self.assert_(storage.is_connected())
# We expect finish to fail
self.assertRaises(ClientDisconnected, storage.tpc_finish, t)
# The cache should still be empty
self.assert_(not list(storage._cache.contents()))
# Load should fail since the object should not be in either the cache
# or the server.
self.assertRaises(KeyError, storage.load, oid, '')
def checkNotUndoable(self):
eq = self.assertEqual
# Set things up so we've got a transaction that can't be undone
oid = self._storage.new_oid()
revid_a = self._dostore(oid, data=MinPO(51))
revid_b = self._dostore(oid, revid=revid_a, data=MinPO(52))
revid_c = self._dostore(oid, revid=revid_b, data=MinPO(53))
# Start the undo
info = self._storage.undoInfo()
tid = info[1]['id']
t = TransactionMetaData()
self.assertRaises(POSException.UndoError, self._begin_undos_vote, t,
tid)
self._storage.tpc_abort(t)
# Now have more fun: object1 and object2 are in the same transaction,
# which we'll try to undo to, but one of them has since modified in
# different transaction, so the undo should fail.
oid1 = oid
revid1 = revid_c
oid2 = self._storage.new_oid()
revid2 = ZERO
p81, p82, p91, p92 = map(zodb_pickle, map(MinPO, (81, 82, 91, 92)))
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.store(oid1, revid1, p81, '', t)
self._storage.store(oid2, revid2, p91, '', t)
self._storage.tpc_vote(t)
tid = self._storage.tpc_finish(t)
# Make sure the objects have the expected values
data, revid_11 = load_current(self._storage, oid1)
eq(zodb_unpickle(data), MinPO(81))
data, revid_22 = load_current(self._storage, oid2)
eq(zodb_unpickle(data), MinPO(91))
eq(revid_11, tid)
eq(revid_22, tid)
# Now modify oid2
revid2 = self._dostore(oid2, tid, MinPO(92))
self.assertNotEqual(tid, revid2)
info = self._storage.undoInfo()
tid = info[1]['id']
t = TransactionMetaData()
self.assertRaises(POSException.UndoError, self._begin_undos_vote, t,
tid)
self._storage.tpc_abort(t)
self._iterate()
def checkTwoObjectUndoAgain(self):
eq = self.assertEqual
p31, p32, p33, p51, p52, p53 = map(
zodb_pickle, map(MinPO, (31, 32, 33, 51, 52, 53)))
# Like the above, but the first revision of the objects are stored in
# different transactions.
oid1 = self._storage.new_oid()
oid2 = self._storage.new_oid()
revid1 = self._dostore(oid1, data=p31, already_pickled=1)
revid2 = self._dostore(oid2, data=p51, already_pickled=1)
# Update those same two objects
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.store(oid1, revid1, p32, '', t)
self._storage.store(oid2, revid2, p52, '', t)
# Finish the transaction
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
# Now attempt to undo the transaction containing two objects
info = self._storage.undoInfo()
self._undo(info[0]["id"], [oid1, oid2])
data, revid1 = load_current(self._storage, oid1)
eq(zodb_unpickle(data), MinPO(31))
data, revid2 = load_current(self._storage, oid2)
eq(zodb_unpickle(data), MinPO(51))
# Like the above, but this time, the second transaction contains only
# one object.
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.store(oid1, revid1, p33, '', t)
self._storage.store(oid2, revid2, p53, '', t)
# Finish the transaction
self._storage.tpc_vote(t)
tid = self._storage.tpc_finish(t)
# Update in different transactions
revid1 = self._dostore(oid1, revid=tid, data=MinPO(34))
revid2 = self._dostore(oid2, revid=tid, data=MinPO(54))
# Now attempt to undo the transaction containing two objects
info = self._storage.undoInfo()
self.undo(info[1]['id'])
data, revid1 = load_current(self._storage, oid1)
eq(zodb_unpickle(data), MinPO(33))
data, revid2 = load_current(self._storage, oid2)
eq(zodb_unpickle(data), MinPO(54))
self._iterate()
def __init__(self, storage, doNextEvent, threadStartedEvent):
self.storage = storage
self.trans = TransactionMetaData()
self.doNextEvent = doNextEvent
self.threadStartedEvent = threadStartedEvent
self.gotValueError = 0
self.gotDisconnected = 0
threading.Thread.__init__(self)
self.setDaemon(1)
def checkTL_ReadCurrentConflict_DoesNotTakeExclusiveLocks(self):
# Proves that if we try to check an old serial that has already moved on,
# we don't try taking exclusive locks at all.
from relstorage.adapters.interfaces import UnableToLockRowsToModifyError
obj1_oid, obj2_oid, tid, db = self.__store_two_for_read_current_error()
assert obj1_oid, obj1_oid
assert obj2_oid, obj2_oid
assert tid, tid
root_adapter = self._storage._adapter
commit_lock_timeout = self.__tiny_commit_time
root_adapter.locker.commit_lock_timeout = commit_lock_timeout
self._storage._options.commit_lock_timeout = commit_lock_timeout
storageA = self._closing(self._storage.new_instance())
storageC = self._closing(self._storage.new_instance())
# The Begin phase actually calls into the database when readCurrent()
# is called to verify the tid. So we actually need to do that much of it now.
storageB = self._closing(self._storage.new_instance())
txb = TransactionMetaData()
storageB.tpc_begin(txb) # Capture our current tid now
storageB.checkCurrentSerialInTransaction(obj2_oid, tid, txb)
should_ileave = root_adapter.force_lock_objects_and_detect_conflicts_interleavable
for s in storageA, storageB, storageC:
s._adapter.force_lock_objects_and_detect_conflicts_interleavable = should_ileave
# Walk through a full transaction in A so that the tid changes.
conn = db.open()
root = conn.root()
root['object1'].some_attr = 1
root['object2'].some_attr = 1
transaction.commit()
new_tid = root['object1']._p_serial
conn.close()
self.assertGreater(new_tid, tid)
# Tx A takes exclusive lock on object1
txa = self.__read_current_and_lock(storageA, None, obj1_oid, new_tid)
# Prove that it's locked. (This is slow on MySQL 5.7)
with self.assertRaises(UnableToLockRowsToModifyError):
self.__read_current_and_lock(storageC, None, obj1_oid, new_tid)
# Now, try to readCurrent of object2 in the old tid, and take an exclusive lock
# on obj1. We should immediately get a read current error and not conflict with the
# exclusive lock.
with self.assertRaisesRegex(VoteReadConflictError,
"serial this txn started"):
self.__read_current_and_lock(storageB, obj2_oid, obj1_oid, tid)
# Which is still held because we cannot lock it.
with self.assertRaises(UnableToLockRowsToModifyError):
self.__read_current_and_lock(storageC, None, obj1_oid, new_tid)
storageA.tpc_abort(txa)
def __read_current_and_lock(self, storage, read_current_oid, lock_oid,
tid):
tx = TransactionMetaData()
storage.tpc_begin(tx)
if read_current_oid is not None:
storage.checkCurrentSerialInTransaction(read_current_oid, tid, tx)
storage.store(lock_oid, tid, b'bad pickle2', '', tx)
storage.tpc_vote(tx)
return tx
def store(storage, oid, value='x', serial=ZODB.utils.z64):
if not isinstance(oid, bytes):
oid = ZODB.utils.p64(oid)
if not isinstance(serial, bytes):
serial = ZODB.utils.p64(serial)
t = TransactionMetaData()
storage.tpc_begin(t)
storage.store(oid, serial, value, '', t)
storage.tpc_vote(t)
storage.tpc_finish(t)
def checkWriteMethods(self):
self._make_readonly()
self.assertRaises(ReadOnlyError, self._storage.new_oid)
t = TransactionMetaData()
self.assertRaises(ReadOnlyError, self._storage.tpc_begin, t)
self.assertRaises(ReadOnlyError, self._storage.store, b'\000' * 8,
None, b'', '', t)
self.assertRaises(ReadOnlyError, self._storage.undo, b'\000' * 8, t)
def checkTransactionIdIncreases(self):
import time
from ZODB.utils import newTid
from ZODB.TimeStamp import TimeStamp
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
# Add a fake transaction
transactions = self._storage._transactions
self.assertEqual(1, len(transactions))
fake_timestamp = b'zzzzzzzy' # the year 5735 ;-)
transactions[fake_timestamp] = transactions.values()[0]
# Verify the next transaction comes after the fake transaction
t = TransactionMetaData()
self._storage.tpc_begin(t)
self.assertEqual(self._storage._tid, b'zzzzzzzz')
def checkWriteAfterAbort(self):
oid = self._storage.new_oid()
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
# Now abort this transaction
self._storage.tpc_abort(t)
# Now start all over again
oid = self._storage.new_oid()
self._dostore(oid=oid, data=MinPO(6))
def undo(i):
info = s.undoInfo()
t = TransactionMetaData()
s.tpc_begin(t)
base = i * OBJECTS + i
for j in range(OBJECTS):
tid = info[base + j]['id']
s.undo(tid, t)
s.tpc_vote(t)
s.tpc_finish(t)
def checkanalyze(self):
import types
from BTrees.OOBTree import OOBTree
from ZODB.scripts import analyze
# Set up a module to act as a broken import
module_name = 'brokenmodule'
module = types.ModuleType(module_name)
sys.modules[module_name] = module
class Broken(MinPO):
__module__ = module_name
module.Broken = Broken
oids = [[self._storage.new_oid(), None] for i in range(3)]
def store(i, data):
oid, revid = oids[i]
self._storage.store(oid, revid, data, "", t)
for i in range(2):
t = TransactionMetaData()
self._storage.tpc_begin(t)
# sometimes data is in this format
store(0, dumps(OOBTree, _protocol))
# and it could be from a broken module
store(1, dumps(Broken, _protocol))
# but mostly it looks like this
store(2, zodb_pickle(MinPO(2)))
self._storage.tpc_vote(t)
tid = self._storage.tpc_finish(t)
for oid_revid in oids:
oid_revid[1] = tid
# now break the import of the Broken class
del sys.modules[module_name]
# from ZODB.scripts.analyze.analyze
fsi = self._storage.iterator()
rep = analyze.Report()
for txn in fsi:
analyze.analyze_trans(rep, txn)
# from ZODB.scripts.analyze.report
typemap = sorted(rep.TYPEMAP.keys())
cumpct = 0.0
for t in typemap:
pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES
cumpct += pct
self.assertAlmostEqual(cumpct, 100.0, 0,
"Failed to analyze some records")
def _dostore(self,
oid=None,
revid=None,
data=None,
already_pickled=0,
user=None,
description=None):
"""Do a complete storage transaction. The defaults are:
- oid=None, ask the storage for a new oid
- revid=None, use a revid of ZERO
- data=None, pickle up some arbitrary data (the integer 7)
Returns the object's new revision id.
"""
if oid is None:
oid = self._storage.new_oid()
if revid is None:
revid = ZERO
if data is None:
data = MinPO(7)
if type(data) == int:
data = MinPO(data)
if not already_pickled:
data = zodb_pickle(data)
# Begin the transaction
t = TransactionMetaData()
if user is not None:
t.user = user
if description is not None:
t.description = description
try:
self._storage.tpc_begin(t)
# Store an object
r1 = self._storage.store(oid, revid, data, '', t)
# Finish the transaction
r2 = self._storage.tpc_vote(t)
revid = self._storage.tpc_finish(t)
except:
self._storage.tpc_abort(t)
raise
return revid
def main():
if len(sys.argv) not in (3, 4):
sys.stderr.write("Usage: timeout.py address delay [storage-name]\n" %
sys.argv[0])
sys.exit(2)
hostport = sys.argv[1]
delay = float(sys.argv[2])
if sys.argv[3:]:
name = sys.argv[3]
else:
name = "1"
if "/" in hostport:
address = hostport
else:
if ":" in hostport:
i = hostport.index(":")
host, port = hostport[:i], hostport[i+1:]
else:
host, port = "", hostport
port = int(port)
address = (host, port)
print("Connecting to %s..." % repr(address))
storage = ClientStorage(address, name)
print("Connected. Now starting a transaction...")
oid = storage.new_oid()
revid = ZERO
data = MinPO("timeout.py")
pickled_data = zodb_pickle(data)
t = TransactionMetaData()
t.user = "******"
storage.tpc_begin(t)
storage.store(oid, revid, pickled_data, '', t)
print("Stored. Now voting...")
storage.tpc_vote(t)
print("Voted; now sleeping %s..." % delay)
time.sleep(delay)
print("Done.")
def run(self):
tname = self.getName()
testcase = self.testcase
# Create client connections to each server
clients = self.clients
for i in range(len(testcase.addr)):
c = testcase.openClientStorage(addr=testcase.addr[i])
c.__name = "C%d" % i
clients.append(c)
for i in range(testcase.ntrans):
# Because we want a transaction spanning all storages,
# we can't use _dostore(). This is several _dostore() calls
# expanded in-line (mostly).
# Create oid->serial mappings
for c in clients:
c.__oids = []
c.__serials = {}
# Begin a transaction
t = TransactionMetaData()
for c in clients:
#print("%s.%s.%s begin" % (tname, c.__name, i))
c.tpc_begin(t)
for j in range(testcase.nobj):
for c in clients:
# Create and store a new object on each server
oid = c.new_oid()
c.__oids.append(oid)
data = MinPO("%s.%s.t%d.o%d" % (tname, c.__name, i, j))
#print(data.value)
data = zodb_pickle(data)
c.store(oid, ZERO, data, '', t)
# Vote on all servers and handle serials
for c in clients:
#print("%s.%s.%s vote" % (tname, c.__name, i))
c.tpc_vote(t)
# Finish on all servers
for c in clients:
#print("%s.%s.%s finish\n" % (tname, c.__name, i))
c.tpc_finish(t)
for c in clients:
# Check that we got serials for all oids
for oid in c.__oids:
testcase.failUnless(oid in c.__serials)
# Check that we got serials for no other oids
for oid in c.__serials.keys():
testcase.failUnless(oid in c.__oids)
def checkTwoObjectUndoAtOnce(self):
# Convenience
eq = self.assertEqual
unless = self.assertTrue
p30, p31, p32, p50, p51, p52 = map(
zodb_pickle, map(MinPO, (30, 31, 32, 50, 51, 52)))
oid1 = self._storage.new_oid()
oid2 = self._storage.new_oid()
# Store two objects in the same transaction
tid = self._multi_obj_transaction([
(oid1, ZERO, p30),
(oid2, ZERO, p50),
])
# Update those same two objects
tid = self._multi_obj_transaction([
(oid1, tid, p31),
(oid2, tid, p51),
])
# Update those same two objects
tid = self._multi_obj_transaction([
(oid1, tid, p32),
(oid2, tid, p52),
])
# Make sure the objects have the current value
data, revid1 = load_current(self._storage, oid1)
eq(zodb_unpickle(data), MinPO(32))
data, revid2 = load_current(self._storage, oid2)
eq(zodb_unpickle(data), MinPO(52))
# Now attempt to undo the transaction containing two objects
info = self._storage.undoInfo()
tid = info[0]['id']
tid1 = info[1]['id']
t = TransactionMetaData()
oids = self._begin_undos_vote(t, tid, tid1)
serial = self._storage.tpc_finish(t)
# We may get the finalization stuff called an extra time,
# depending on the implementation.
if serial is None:
self.assertEqual(oids, {oid1, oid2})
data, revid1 = load_current(self._storage, oid1)
eq(zodb_unpickle(data), MinPO(30))
data, revid2 = load_current(self._storage, oid2)
eq(zodb_unpickle(data), MinPO(50))
# Now try to undo the one we just did to undo, whew
info = self._storage.undoInfo()
self._undo(info[0]['id'], [oid1, oid2])
data, revid1 = load_current(self._storage, oid1)
eq(zodb_unpickle(data), MinPO(32))
data, revid2 = load_current(self._storage, oid2)
eq(zodb_unpickle(data), MinPO(52))
self._iterate()
def _dostore(self, oid=None, revid=None, data=None, already_pickled=0, user=None, description=None):
"""Do a complete storage transaction. The defaults are:
- oid=None, ask the storage for a new oid
- revid=None, use a revid of ZERO
- data=None, pickle up some arbitrary data (the integer 7)
Returns the object's new revision id.
"""
if oid is None:
oid = self._storage.new_oid()
if revid is None:
revid = ZERO
if data is None:
data = MinPO(7)
if type(data) == int:
data = MinPO(data)
if not already_pickled:
data = zodb_pickle(data)
# Begin the transaction
t = TransactionMetaData()
if user is not None:
t.user = user
if description is not None:
t.description = description
try:
self._storage.tpc_begin(t)
# Store an object
r1 = self._storage.store(oid, revid, data, "", t)
# Finish the transaction
r2 = self._storage.tpc_vote(t)
revid = self._storage.tpc_finish(t)
except:
self._storage.tpc_abort(t)
raise
return revid
def check_tid_ordering_w_commit(self):
# It's important that storages always give a consistent
# ordering for revisions, tids. This is most likely to fail
# around commit. Here we'll do some basic tests to check this.
# We'll use threads to arrange for ordering to go wrong and
# verify that a storage gets it right.
# First, some initial data.
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.store(ZERO, ZERO, b'x', '', t)
self._storage.tpc_vote(t)
tids = []
self._storage.tpc_finish(t, lambda tid: tids.append(tid))
# OK, now we'll start a new transaction, take it to finish,
# and then block finish while we do some other operations.
t = TransactionMetaData()
self._storage.tpc_begin(t)
self._storage.store(ZERO, tids[0], b'y', '', t)
self._storage.tpc_vote(t)
to_join = []
def run_in_thread(func):
t = threading.Thread(target=func)
t.setDaemon(True)
t.start()
to_join.append(t)
started = threading.Event()
finish = threading.Event()
@run_in_thread
def commit():
def callback(tid):
started.set()
tids.append(tid)
finish.wait()
self._storage.tpc_finish(t, callback)
results = {}
started.wait()
attempts = []
attempts_cond = utils.Condition()
def update_attempts():
with attempts_cond:
attempts.append(1)
attempts_cond.notifyAll()
@run_in_thread
def lastTransaction():
update_attempts()
results['lastTransaction'] = self._storage.lastTransaction()
@run_in_thread
def load():
update_attempts()
results['load'] = utils.load_current(self._storage, ZERO)[1]
expected_attempts = 2
if hasattr(self._storage, 'getTid'):
expected_attempts += 1
@run_in_thread
def getTid():
update_attempts()
results['getTid'] = self._storage.getTid(ZERO)
if hasattr(self._storage, 'lastInvalidations'):
expected_attempts += 1
@run_in_thread
def lastInvalidations():
update_attempts()
invals = self._storage.lastInvalidations(1)
if invals:
results['lastInvalidations'] = invals[0][0]
with attempts_cond:
while len(attempts) < expected_attempts:
attempts_cond.wait()
time.sleep(.01) # for good measure :)
finish.set()
for t in to_join:
t.join(1)
self.assertEqual(results.pop('load'), tids[1])
self.assertEqual(results.pop('lastTransaction'), tids[1])
for m, tid in results.items():
self.assertEqual(tid, tids[1])
