def tpc_begin(self, transaction, tid=None, status=' '):
if self._is_read_only:
raise POSException.ReadOnlyError()
with self._lock:
if self._transaction is transaction:
raise POSException.StorageTransactionError(
"Duplicate tpc_begin calls for same transaction")
self._commit_lock.acquire()
with self._lock:
self._transaction = transaction
self._clear_temp()
user = transaction.user
desc = transaction.description
ext = transaction.extension_bytes
self._ude = user, desc, ext
if tid is None:
now = time.time()
t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60, )))
self._ts = t = t.laterThan(self._ts)
self._tid = t.raw()
else:
self._ts = TimeStamp(tid)
self._tid = tid
del self._resolved[:]
self._tstatus = status
self._begin(self._tid, user, desc, ext)
def copy(source, dest, verbose=0):
"""Copy transactions from a source to a destination storage
This is typically used for converting data from one storage to
another. `source` must have an .iterator() method.
"""
_ts = None
ok = 1
preindex = {}
preget = preindex.get
# restore() is a new storage API method which has an identical
# signature to store() except that it does not return anything.
# Semantically, restore() is also identical to store() except that it
# doesn't do the ConflictError or VersionLockError consistency
# checks. The reason to use restore() over store() in this method is
# that store() cannot be used to copy transactions spanning a version
# commit or abort, or over transactional undos.
#
# We'll use restore() if it's available, otherwise we'll fall back to
# using store(). However, if we use store, then
# copyTransactionsFrom() may fail with VersionLockError or
# ConflictError.
restoring = py2_hasattr(dest, 'restore')
fiter = source.iterator()
for transaction in fiter:
tid = transaction.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok: print(('Time stamps out of order %s, %s' % (_ts, t)))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print(('Time stamps back in order %s' % (t)))
ok = 1
if verbose:
print(_ts)
dest.tpc_begin(transaction, tid, transaction.status)
for r in transaction:
oid = r.oid
if verbose:
print(oid_repr(oid), r.version, len(r.data))
if restoring:
dest.restore(oid, r.tid, r.data, r.version, r.data_txn,
transaction)
else:
pre = preget(oid, None)
dest.store(oid, pre, r.data, r.version, transaction)
preindex[oid] = tid
dest.tpc_vote(transaction)
dest.tpc_finish(transaction)
def copy(source, dest, verbose=0):
"""Copy transactions from a source to a destination storage
This is typically used for converting data from one storage to
another. `source` must have an .iterator() method.
"""
_ts = None
ok = 1
preindex = {};
preget = preindex.get
# restore() is a new storage API method which has an identical
# signature to store() except that it does not return anything.
# Semantically, restore() is also identical to store() except that it
# doesn't do the ConflictError or VersionLockError consistency
# checks. The reason to use restore() over store() in this method is
# that store() cannot be used to copy transactions spanning a version
# commit or abort, or over transactional undos.
#
# We'll use restore() if it's available, otherwise we'll fall back to
# using store(). However, if we use store, then
# copyTransactionsFrom() may fail with VersionLockError or
# ConflictError.
restoring = py2_hasattr(dest, 'restore')
fiter = source.iterator()
for transaction in fiter:
tid = transaction.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok: print(('Time stamps out of order %s, %s' % (_ts, t)))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print(('Time stamps back in order %s' % (t)))
ok = 1
if verbose:
print(_ts)
dest.tpc_begin(transaction, tid, transaction.status)
for r in transaction:
oid = r.oid
if verbose:
print(oid_repr(oid), r.version, len(r.data))
if restoring:
dest.restore(oid, r.tid, r.data, r.version,
r.data_txn, transaction)
else:
pre = preget(oid, None)
s = dest.store(oid, pre, r.data, r.version, transaction)
preindex[oid] = s
dest.tpc_vote(transaction)
dest.tpc_finish(transaction)
def testMTime(self):
obj = P()
self.assertEqual(obj._p_mtime, None)
t = int(time.time())
ts = TimeStamp(*time.gmtime(t)[:6])
obj._p_serial = ts.raw()
self.assertEqual(obj._p_mtime, t)
self.assertIsInstance(obj._p_mtime, float)
def testMTime(self):
obj = P()
self.assertEqual(obj._p_mtime, None)
t = int(time.time())
ts = TimeStamp(*time.gmtime(t)[:6])
obj._p_serial = ts.raw()
self.assertEqual(obj._p_mtime, t)
self.assertIsInstance(obj._p_mtime, float)
def patchedNewTid(old): # noqa
if getattr(time.time, 'previous_time_function', False):
t = time.time.previous_time_function()
ts = TimeStamp(*time.gmtime.previous_gmtime_function(t)[:5] +
(t % 60, ))
else:
t = time.time()
ts = TimeStamp(*time.gmtime(t)[:5] + (t % 60, ))
if old is not None:
ts = ts.laterThan(TimeStamp(old))
return ts.raw()
def test_OFSFileLastModified_File(self):
from OFS.Image import File
dummy = File('dummy', 'Dummy', b'data')
self.assertIsNone(ILastModified(dummy)())
timestamp = 987654321.0 # time stamp (in UTC)
ts = TimeStamp(*time.gmtime(timestamp)[:6]) # corresponding TimeStamp
# equivalent in local time, which is what the last-modified adapter
# should return
mod = datetime.datetime.fromtimestamp(timestamp, tzlocal())
dummy._p_jar = FauxDataManager()
dummy._p_serial = ts.raw()
self.assertEqual(mod, ILastModified(dummy)())
def patchedNewTid(old): # noqa
"""Make sure ZODB.utils.newTid always uses the real time functions
instead of the ones possibly patched by freezegun.
This is necessary because ZODB seems to be relying on time being monotonic
for its transaction IDs, and freezing time results in
POSException.ReadConflictErrors.
"""
from persistent.TimeStamp import TimeStamp # noqa
import freezegun # noqa
t = freezegun.api.real_time()
ts = TimeStamp(*freezegun.api.real_gmtime(t)[:5] + (t % 60, ))
if old is not None:
ts = ts.laterThan(TimeStamp(old))
return ts.raw()
def tpc_begin(self, transaction, tid=None, status=' '):
if self._is_read_only:
raise POSException.ReadOnlyError()
self._lock_acquire()
try:
if self._transaction is transaction:
raise POSException.StorageTransactionError(
"Duplicate tpc_begin calls for same transaction")
self._lock_release()
self._commit_lock_acquire()
self._lock_acquire()
self._transaction = transaction
self._clear_temp()
user = transaction.user
desc = transaction.description
ext = transaction._extension
if ext:
ext = dumps(ext, _protocol)
else:
ext = ""
self._ude = user, desc, ext
if tid is None:
now = time.time()
t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60, )))
self._ts = t = t.laterThan(self._ts)
self._tid = t.raw()
else:
self._ts = TimeStamp(tid)
self._tid = tid
self._tstatus = status
self._begin(self._tid, user, desc, ext)
finally:
self._lock_release()
def tpc_begin(self, transaction, tid=None, status=' '):
if self._is_read_only:
raise POSException.ReadOnlyError()
self._lock_acquire()
try:
if self._transaction is transaction:
raise POSException.StorageTransactionError(
"Duplicate tpc_begin calls for same transaction")
self._lock_release()
self._commit_lock_acquire()
self._lock_acquire()
self._transaction = transaction
self._clear_temp()
user = transaction.user
desc = transaction.description
ext = transaction._extension
if ext:
ext = dumps(ext, _protocol)
else:
ext = ""
self._ude = user, desc, ext
if tid is None:
now = time.time()
t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,)))
self._ts = t = t.laterThan(self._ts)
self._tid = t.raw()
else:
self._ts = TimeStamp(tid)
self._tid = tid
self._tstatus = status
self._begin(self._tid, user, desc, ext)
finally:
self._lock_release()
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
print "Recovering", inp, "into", outp
if os.path.exists(outp) and not force:
die("%s exists" % outp)
f = open(inp, "rb")
if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
f.seek(0,2)
file_size = f.tell()
ofs = ZODB.FileStorage.FileStorage(outp, create=1)
_ts = None
ok = 1
prog1 = 0
undone = 0
pos = 4L
ltid = None
while pos:
try:
npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
except EOFError:
break
except (KeyboardInterrupt, SystemExit):
raise
except Exception, err:
print "error reading txn header:", err
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print "looking for valid txn header at", pos
continue
ltid = tid
if txn is None:
undone = undone + npos - pos
pos = npos
continue
else:
pos = npos
tid = txn.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok:
print ("Time stamps out of order %s, %s" % (_ts, t))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print ("Time stamps back in order %s" % (t))
ok = 1
ofs.tpc_begin(txn, tid, txn.status)
if verbose:
print "begin", pos, _ts,
if verbose > 1:
print
sys.stdout.flush()
nrec = 0
try:
for r in txn:
if verbose > 1:
if r.data is None:
l = "bp"
else:
l = len(r.data)
print "%7d %s %s" % (u64(r.oid), l)
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,
txn)
nrec += 1
except (KeyboardInterrupt, SystemExit):
raise
except Exception, err:
if partial and nrec:
ofs._status = "p"
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print "partial"
else:
ofs.tpc_abort(txn)
print "error copying transaction:", err
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print "looking for valid txn header at", pos
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
print("Recovering", inp, "into", outp)
if os.path.exists(outp) and not force:
die("%s exists" % outp)
f = open(inp, "rb")
if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
f.seek(0, 2)
file_size = f.tell()
ofs = ZODB.FileStorage.FileStorage(outp, create=1)
_ts = None
ok = 1
prog1 = 0
undone = 0
pos = 4
ltid = None
while pos:
try:
npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
except EOFError:
break
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
print("error reading txn header:", err)
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print("looking for valid txn header at", pos)
continue
ltid = tid
if txn is None:
undone = undone + npos - pos
pos = npos
continue
else:
pos = npos
tid = txn.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok:
print(("Time stamps out of order %s, %s" % (_ts, t)))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print(("Time stamps back in order %s" % (t)))
ok = 1
ofs.tpc_begin(txn, tid, txn.status)
if verbose:
print("begin", pos, _ts, end=' ')
if verbose > 1:
print()
sys.stdout.flush()
nrec = 0
try:
for r in txn:
if verbose > 1:
if r.data is None:
l = "bp"
else:
l = len(r.data)
print("%7d %s %s" % (u64(r.oid), l))
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn, txn)
nrec += 1
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
if partial and nrec:
ofs._status = "p"
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print("partial")
else:
ofs.tpc_abort(txn)
print("error copying transaction:", err)
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print("looking for valid txn header at", pos)
else:
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print("finish")
sys.stdout.flush()
if not verbose:
prog = pos * 20 / file_size
while prog > prog1:
prog1 = prog1 + 1
iprogress(prog1)
bad = file_size - undone - ofs._pos
print("\n%s bytes removed during recovery" % bad)
if undone:
print("%s bytes of undone transaction data were skipped" % undone)
if pack is not None:
print("Packing ...")
from ZODB.serialize import referencesf
ofs.pack(pack, referencesf)
ofs.close()
f.close()
def newTid(old):
t = time.time()
ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
if old is not None:
ts = ts.laterThan(TimeStamp(old))
return ts.raw()
def newTid(old):
t = time.time()
ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
if old is not None:
ts = ts.laterThan(TimeStamp(old))
return ts.raw()
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
print("Recovering", inp, "into", outp)
if os.path.exists(outp) and not force:
die("%s exists" % outp)
f = open(inp, "rb")
if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
f.seek(0,2)
file_size = f.tell()
ofs = ZODB.FileStorage.FileStorage(outp, create=1)
_ts = None
ok = 1
prog1 = 0
undone = 0
pos = 4
ltid = None
while pos:
try:
npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
except EOFError:
break
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
print("error reading txn header:", err)
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print("looking for valid txn header at", pos)
continue
ltid = tid
if txn is None:
undone = undone + npos - pos
pos = npos
continue
else:
pos = npos
tid = txn.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok:
print(("Time stamps out of order %s, %s" % (_ts, t)))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print(("Time stamps back in order %s" % (t)))
ok = 1
ofs.tpc_begin(txn, tid, txn.status)
if verbose:
print("begin", pos, _ts, end=' ')
if verbose > 1:
print()
sys.stdout.flush()
nrec = 0
try:
for r in txn:
if verbose > 1:
if r.data is None:
l = "bp"
else:
l = len(r.data)
print("%7d %s %s" % (u64(r.oid), l))
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,
txn)
nrec += 1
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
if partial and nrec:
ofs._status = "p"
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print("partial")
else:
ofs.tpc_abort(txn)
print("error copying transaction:", err)
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print("looking for valid txn header at", pos)
else:
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print("finish")
sys.stdout.flush()
if not verbose:
prog = pos * 20 / file_size
while prog > prog1:
prog1 = prog1 + 1
iprogress(prog1)
bad = file_size - undone - ofs._pos
print("\n%s bytes removed during recovery" % bad)
if undone:
print("%s bytes of undone transaction data were skipped" % undone)
if pack is not None:
print("Packing ...")
from ZODB.serialize import referencesf
ofs.pack(pack, referencesf)
ofs.close()
f.close()
