def checkanalyze(self):
import types
from BTrees.OOBTree import OOBTree
from ZODB.scripts import analyze
# Set up a module to act as a broken import
module_name = 'brokenmodule'
module = types.ModuleType(module_name)
sys.modules[module_name] = module
class Broken(MinPO):
__module__ = module_name
module.Broken = Broken
oids = [[self._storage.new_oid(), None] for i in range(3)]
def store(i, data):
oid, revid = oids[i]
self._storage.store(oid, revid, data, "", t)
for i in range(2):
t = TransactionMetaData()
self._storage.tpc_begin(t)
# sometimes data is in this format
store(0, dumps(OOBTree, _protocol))
# and it could be from a broken module
store(1, dumps(Broken, _protocol))
# but mostly it looks like this
store(2, zodb_pickle(MinPO(2)))
self._storage.tpc_vote(t)
tid = self._storage.tpc_finish(t)
for oid_revid in oids:
oid_revid[1] = tid
# now break the import of the Broken class
del sys.modules[module_name]
# from ZODB.scripts.analyze.analyze
fsi = self._storage.iterator()
rep = analyze.Report()
for txn in fsi:
analyze.analyze_trans(rep, txn)
# from ZODB.scripts.analyze.report
typemap = sorted(rep.TYPEMAP.keys())
cumpct = 0.0
for t in typemap:
pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES
cumpct += pct
self.assertAlmostEqual(cumpct, 100.0, 0,
"Failed to analyze some records")
def checkanalyze(self):
import types
from BTrees.OOBTree import OOBTree
from ZODB.scripts import analyze
# Set up a module to act as a broken import
module_name = 'brokenmodule'
module = types.ModuleType(module_name)
sys.modules[module_name] = module
class Broken(MinPO):
__module__ = module_name
module.Broken = Broken
oids = [[self._storage.new_oid(), None] for i in range(3)]
def store(i, data):
oid, revid = oids[i]
self._storage.store(oid, revid, data, "", t)
for i in range(2):
t = TransactionMetaData()
self._storage.tpc_begin(t)
# sometimes data is in this format
store(0, dumps(OOBTree, _protocol))
# and it could be from a broken module
store(1, dumps(Broken, _protocol))
# but mostly it looks like this
store(2, zodb_pickle(MinPO(2)))
self._storage.tpc_vote(t)
tid = self._storage.tpc_finish(t)
for oid_revid in oids:
oid_revid[1] = tid
# now break the import of the Broken class
del sys.modules[module_name]
# from ZODB.scripts.analyze.analyze
fsi = self._storage.iterator()
rep = analyze.Report()
for txn in fsi:
analyze.analyze_trans(rep, txn)
# from ZODB.scripts.analyze.report
typemap = sorted(rep.TYPEMAP.keys())
cumpct = 0.0
for t in typemap:
pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES
cumpct += pct
self.assertAlmostEqual(cumpct, 100.0, 0,
"Failed to analyze some records")
def test_get_pickle_metadata_w_protocol_3_class_pickle(self):
from ZODB.utils import get_pickle_metadata
from ZODB._compat import dumps
from ZODB._compat import HIGHEST_PROTOCOL
if HIGHEST_PROTOCOL >= 3:
pickle = dumps(ExampleClass, protocol=3)
self.assertEqual(get_pickle_metadata(pickle),
(__name__, ExampleClass.__name__))
def test_get_pickle_metadata_w_protocol_3_class_pickle(self):
from ZODB.utils import get_pickle_metadata
from ZODB._compat import dumps
from ZODB._compat import HIGHEST_PROTOCOL
if HIGHEST_PROTOCOL >= 3:
pickle = dumps(ExampleClass, protocol=3)
self.assertEqual(get_pickle_metadata(pickle),
(__name__, ExampleClass.__name__))
def test_zodbcommit(zext):
tmpd = mkdtemp('', 'zodbcommit.')
defer(lambda: rmtree(tmpd))
stor = storageFromURL('%s/2.fs' % tmpd)
defer(stor.close)
head = stor.lastTransaction()
# commit some transactions via zodbcommit and verify if storage dump gives
# what is expected.
t1 = Transaction(z64, ' ', b'user name', b'description ...', zext(dumps({'a': 'b'}, _protocol)), [
ObjectData(p64(1), b'data1', 'sha1', sha1(b'data1')),
ObjectData(p64(2), b'data2', 'sha1', sha1(b'data2'))])
t1.tid = zodbcommit(stor, head, t1)
t2 = Transaction(z64, ' ', b'user2', b'desc2', b'', [
ObjectDelete(p64(2))])
t2.tid = zodbcommit(stor, t1.tid, t2)
buf = BytesIO()
zodbdump(stor, p64(u64(head)+1), None, out=buf)
dumped = buf.getvalue()
assert dumped == b''.join([_.zdump() for _ in (t1, t2)])
# ObjectCopy. XXX zodbcommit handled ObjectCopy by actually copying data,
# not referencing previous transaction via backpointer.
t3 = Transaction(z64, ' ', b'user3', b'desc3', b'', [
ObjectCopy(p64(1), t1.tid)])
t3.tid = zodbcommit(stor, t2.tid, t3)
data1_1, _, _ = stor.loadBefore(p64(1), p64(u64(t1.tid)+1))
data1_3, _, _ = stor.loadBefore(p64(1), p64(u64(t3.tid)+1))
assert data1_1 == data1_3
assert data1_1 == b'data1' # just in case
def tpc_begin(self, transaction, tid=None, status=' '):
if self._is_read_only:
raise POSException.ReadOnlyError()
self._lock_acquire()
try:
if self._transaction is transaction:
raise POSException.StorageTransactionError(
"Duplicate tpc_begin calls for same transaction")
self._lock_release()
self._commit_lock_acquire()
self._lock_acquire()
self._transaction = transaction
self._clear_temp()
user = transaction.user
desc = transaction.description
ext = transaction._extension
if ext:
ext = dumps(ext, _protocol)
else:
ext = ""
self._ude = user, desc, ext
if tid is None:
now = time.time()
t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60, )))
self._ts = t = t.laterThan(self._ts)
self._tid = t.raw()
else:
self._ts = TimeStamp(tid)
self._tid = tid
self._tstatus = status
self._begin(self._tid, user, desc, ext)
finally:
self._lock_release()
def tpc_begin(self, transaction, tid=None, status=' '):
if self._is_read_only:
raise POSException.ReadOnlyError()
self._lock_acquire()
try:
if self._transaction is transaction:
raise POSException.StorageTransactionError(
"Duplicate tpc_begin calls for same transaction")
self._lock_release()
self._commit_lock_acquire()
self._lock_acquire()
self._transaction = transaction
self._clear_temp()
user = transaction.user
desc = transaction.description
ext = transaction._extension
if ext:
ext = dumps(ext, _protocol)
else:
ext = ""
self._ude = user, desc, ext
if tid is None:
now = time.time()
t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,)))
self._ts = t = t.laterThan(self._ts)
self._tid = t.raw()
else:
self._ts = TimeStamp(tid)
self._tid = tid
self._tstatus = status
self._begin(self._tid, user, desc, ext)
finally:
self._lock_release()
def transactionAsTuple(txn):
ext = txn.extension
return (txn.user, txn.description, dumps(ext, _protocol) if ext else '',
txn.status == 'p', txn.tid)
def test_get_pickle_metadata_w_protocol_0_class_pickle(self):
from ZODB.utils import get_pickle_metadata
from ZODB._compat import dumps
pickle = dumps(ExampleClass, protocol=0)
self.assertEqual(get_pickle_metadata(pickle),
(__name__, ExampleClass.__name__))
def test_get_pickle_metadata_w_protocol_2_class_pickle(self):
from ZODB.utils import get_pickle_metadata
from ZODB._compat import dumps
pickle = dumps(ExampleClass, protocol=2)
self.assertEqual(get_pickle_metadata(pickle),
(__name__, ExampleClass.__name__))
def tpc_vote(self, transaction):
"""Store current transaction."""
txn_context = self._txn_container.get(transaction)
self.waitStoreResponses(txn_context)
ttid = txn_context.ttid
ext = transaction._extension
ext = dumps(ext, _protocol) if ext else ''
# user and description are cast to str in case they're unicode.
# BBB: This is not required anymore with recent ZODB.
packet = Packets.AskStoreTransaction(ttid, str(transaction.user),
str(transaction.description), ext, list(txn_context.cache_dict))
queue = txn_context.queue
conn_dict = txn_context.conn_dict
# Ask in parallel all involved storage nodes to commit object metadata.
# Nodes that store the transaction metadata get a special packet.
trans_nodes = txn_context.write(self, packet, ttid)
packet = Packets.AskVoteTransaction(ttid)
for uuid in conn_dict:
if uuid not in trans_nodes:
self._askStorageForWrite(txn_context, uuid, packet)
self.waitStoreResponses(txn_context)
if None in conn_dict.itervalues(): # unlikely
# If some writes failed, we must first check whether
# all oids have been locked by at least one node.
failed = {node.getUUID(): node.isRunning()
for node in self.nm.getStorageList()
if conn_dict.get(node.getUUID(), 0) is None}
if txn_context.lockless_dict:
getCellList = self.pt.getCellList
for offset, uuid_set in txn_context.lockless_dict.iteritems():
for cell in getCellList(offset):
uuid = cell.getUUID()
if not (uuid in failed or uuid in uuid_set):
break
else:
# Very unlikely. Instead of raising, we could recover
# the transaction by doing something similar to
# deadlock avoidance; that would be done before voting.
# But it's not worth the complexity.
raise NEOStorageError(
'partition %s not fully write-locked' % offset)
failed = [uuid for uuid, running in failed.iteritems() if running]
# If there are running nodes for which some writes failed, ask the
# master whether they can be disconnected while keeping the cluster
# operational. If possible, this will happen during tpc_finish.
if failed:
try:
self._askPrimary(Packets.FailedVote(ttid, failed))
except ConnectionClosed:
pass
txn_context.voted = True
# We must not go further if connection to master was lost since
# tpc_begin, to lower the probability of failing during tpc_finish.
# IDEA: We can improve in 2 opposite directions:
# - In the case of big transactions, it would be useful to
# also detect failures earlier.
# - If possible, recover from master failure.
if txn_context.error:
raise NEOStorageError(txn_context.error)
if OLD_ZODB:
return [(oid, ResolvedSerial)
for oid in txn_context.resolved_dict]
return txn_context.resolved_dict