def checkTimeStamp(self):
# Alternate test suite
t = TimeStamp(2002, 1, 23, 10, 48, 5) # GMT
self.assertEquals(str(t), '2002-01-23 10:48:05.000000')
self.assertEquals(repr(t), '\x03B9H\x15UUU')
self.assertEquals(TimeStamp('\x03B9H\x15UUU'), t)
self.assertEquals(t.year(), 2002)
self.assertEquals(t.month(), 1)
self.assertEquals(t.day(), 23)
self.assertEquals(t.hour(), 10)
self.assertEquals(t.minute(), 48)
self.assertEquals(round(t.second()), 5)
self.assertEquals(t.timeTime(), 1011782885)
t1 = TimeStamp(2002, 1, 23, 10, 48, 10)
self.assertEquals(str(t1), '2002-01-23 10:48:10.000000')
self.assert_(t == t)
self.assert_(t != t1)
self.assert_(t < t1)
self.assert_(t <= t1)
self.assert_(t1 >= t)
self.assert_(t1 > t)
self.failIf(t == t1)
self.failIf(t != t)
self.failIf(t > t1)
self.failIf(t >= t1)
self.failIf(t1 < t)
self.failIf(t1 <= t)
self.assertEquals(cmp(t, t), 0)
self.assertEquals(cmp(t, t1), -1)
self.assertEquals(cmp(t1, t), 1)
self.assertEquals(t1.laterThan(t), t1)
self.assert_(t.laterThan(t1) > t1)
self.assertEquals(TimeStamp(2002,1,23), TimeStamp(2002,1,23,0,0,0))
def tpc_begin(self, transaction, tid=None, status=' '):
if self._is_read_only:
raise POSException.ReadOnlyError()
self._lock_acquire()
try:
if self._transaction is transaction:
return
self._lock_release()
self._commit_lock_acquire()
self._lock_acquire()
self._transaction = transaction
self._clear_temp()
user = transaction.user
desc = transaction.description
ext = transaction._extension
if ext:
ext = cPickle.dumps(ext, 1)
else:
ext = ""
self._ude = user, desc, ext
if tid is None:
now = time.time()
t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,)))
self._ts = t = t.laterThan(self._ts)
self._tid = `t`
else:
self._ts = TimeStamp(tid)
self._tid = tid
self._tstatus = status
self._begin(self._tid, user, desc, ext)
finally:
self._lock_release()
def copy(source, dest, verbose=0):
"""Copy transactions from a source to a destination storage
This is typically used for converting data from one storage to
another. `source` must have an .iterator() method.
"""
_ts = None
ok = 1
preindex = {};
preget = preindex.get
# restore() is a new storage API method which has an identical
# signature to store() except that it does not return anything.
# Semantically, restore() is also identical to store() except that it
# doesn't do the ConflictError or VersionLockError consistency
# checks. The reason to use restore() over store() in this method is
# that store() cannot be used to copy transactions spanning a version
# commit or abort, or over transactional undos.
#
# We'll use restore() if it's available, otherwise we'll fall back to
# using store(). However, if we use store, then
# copyTransactionsFrom() may fail with VersionLockError or
# ConflictError.
restoring = py2_hasattr(dest, 'restore')
fiter = source.iterator()
for transaction in fiter:
tid = transaction.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok: print(('Time stamps out of order %s, %s' % (_ts, t)))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print(('Time stamps back in order %s' % (t)))
ok = 1
if verbose:
print(_ts)
dest.tpc_begin(transaction, tid, transaction.status)
for r in transaction:
oid = r.oid
if verbose:
print(oid_repr(oid), r.version, len(r.data))
if restoring:
dest.restore(oid, r.tid, r.data, r.version,
r.data_txn, transaction)
else:
pre = preget(oid, None)
s = dest.store(oid, pre, r.data, r.version, transaction)
preindex[oid] = s
dest.tpc_vote(transaction)
dest.tpc_finish(transaction)
def newTimeStamp(old=None,
TimeStamp=TimeStamp,
time=time.time, gmtime=time.gmtime):
t = time()
ts = TimeStamp(gmtime(t)[:5]+(t%60,))
if old is not None:
return ts.laterThan(old)
return ts
def get_timestamp(prev_ts=None):
"""Internal helper to return a unique TimeStamp instance.
If the optional argument is not None, it must be a TimeStamp; the
return value is then guaranteed to be at least 1 microsecond later
the argument.
"""
t = time.time()
t = TimeStamp(*time.gmtime(t)[:5] + (t % 60,))
if prev_ts is not None:
t = t.laterThan(prev_ts)
return t
def getTID(at, before):
if at is not None:
if before is not None:
raise ValueError('can only pass zero or one of `at` and `before`')
if isinstance(at, datetime.datetime):
at = toTimeStamp(at)
else:
at = TimeStamp(at)
before = at.laterThan(at).raw()
elif before is not None:
if isinstance(before, datetime.datetime):
before = toTimeStamp(before).raw()
else:
before = TimeStamp(before).raw()
return before
def new_tid(self):
#
# Probably better off using a counter as a tid
#
now = time.time()
t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,)))
#get the latest timestamp from memcache
key = '%s,tid' % self._prefix
self.tid_lock.acquire()
try:
result = self._mc.get(key)
if result:
t = t.laterThan(result)
tid = repr(t)
result = self._mc.replace(key, tid)
if not result:
raise MemcacheError
return tid
finally:
self.tid_lock.release()
def tpc_begin(self, transaction, tid=None, status=' '):
if self._is_read_only:
raise POSException.ReadOnlyError()
self._lock_acquire()
try:
if self._transaction is transaction:
raise POSException.StorageTransactionError(
"Duplicate tpc_begin calls for same transaction")
self._lock_release()
self._commit_lock_acquire()
self._lock_acquire()
self._transaction = transaction
self._clear_temp()
user = transaction.user
desc = transaction.description
ext = transaction._extension
if ext:
ext = dumps(ext, _protocol)
else:
ext = ""
self._ude = user, desc, ext
if tid is None:
now = time.time()
t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,)))
self._ts = t = t.laterThan(self._ts)
self._tid = t.raw()
else:
self._ts = TimeStamp(tid)
self._tid = tid
self._tstatus = status
self._begin(self._tid, user, desc, ext)
finally:
self._lock_release()
def copyTransactionsFrom(self, other, verbose=0):
"""Copy transactions from another storage.
This is typically used for converting data from one storage to
another. `other` must have an .iterator() method.
"""
_ts = None
ok = 1
preindex = {}
preget = preindex.get # waaaa
# restore() is a new storage API method which has an identical
# signature to store() except that it does not return anything.
# Semantically, restore() is also identical to store() except that it
# doesn't do the ConflictError or VersionLockError consistency
# checks. The reason to use restore() over store() in this method is
# that store() cannot be used to copy transactions spanning a version
# commit or abort, or over transactional undos.
#
# We'll use restore() if it's available, otherwise we'll fall back to
# using store(). However, if we use store, then
# copyTransactionsFrom() may fail with VersionLockError or
# ConflictError.
restoring = hasattr(self, 'restore')
fiter = other.iterator()
for transaction in fiter:
tid = transaction.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok: print('Time stamps out of order %s, %s' % (_ts, t))
ok = 0
_ts = t.laterThan(_ts)
tid = ` _ts `
else:
_ts = t
if not ok:
print('Time stamps back in order %s' % (t))
ok = 1
if verbose:
print _ts
self.tpc_begin(transaction, tid, transaction.status)
for r in transaction:
oid = r.oid
if verbose:
print oid_repr(oid), r.version, len(r.data)
if restoring:
self.restore(oid, r.tid, r.data, r.version, r.data_txn,
transaction)
else:
pre = preget(oid, None)
s = self.store(oid, pre, r.data, r.version, transaction)
preindex[oid] = s
self.tpc_vote(transaction)
self.tpc_finish(transaction)
fiter.close()
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
print "Recovering", inp, "into", outp
if os.path.exists(outp) and not force:
die("%s exists" % outp)
f = open(inp, "rb")
if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
f.seek(0,2)
file_size = f.tell()
ofs = ZODB.FileStorage.FileStorage(outp, create=1)
_ts = None
ok = 1
prog1 = 0
undone = 0
pos = 4L
ltid = None
while pos:
try:
npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
except EOFError:
break
except (KeyboardInterrupt, SystemExit):
raise
except Exception, err:
print "error reading txn header:", err
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print "looking for valid txn header at", pos
continue
ltid = tid
if txn is None:
undone = undone + npos - pos
pos = npos
continue
else:
pos = npos
tid = txn.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok:
print ("Time stamps out of order %s, %s" % (_ts, t))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print ("Time stamps back in order %s" % (t))
ok = 1
ofs.tpc_begin(txn, tid, txn.status)
if verbose:
print "begin", pos, _ts,
if verbose > 1:
print
sys.stdout.flush()
nrec = 0
try:
for r in txn:
if verbose > 1:
if r.data is None:
l = "bp"
else:
l = len(r.data)
print "%7d %s %s" % (u64(r.oid), l)
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,
txn)
nrec += 1
except (KeyboardInterrupt, SystemExit):
raise
except Exception, err:
if partial and nrec:
ofs._status = "p"
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print "partial"
else:
ofs.tpc_abort(txn)
print "error copying transaction:", err
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print "looking for valid txn header at", pos
def checkLaterThan(self):
t = time.gmtime()
ts = TimeStamp(*t[:6])
ts2 = ts.laterThan(ts)
self.assert_(ts2 > ts)
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
print("Recovering", inp, "into", outp)
if os.path.exists(outp) and not force:
die("%s exists" % outp)
f = open(inp, "rb")
if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
f.seek(0, 2)
file_size = f.tell()
ofs = ZODB.FileStorage.FileStorage(outp, create=1)
_ts = None
ok = 1
prog1 = 0
undone = 0
pos = 4
ltid = None
while pos:
try:
npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
except EOFError:
break
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
print("error reading txn header:", err)
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print("looking for valid txn header at", pos)
continue
ltid = tid
if txn is None:
undone = undone + npos - pos
pos = npos
continue
else:
pos = npos
tid = txn.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok:
print(("Time stamps out of order %s, %s" % (_ts, t)))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print(("Time stamps back in order %s" % (t)))
ok = 1
ofs.tpc_begin(txn, tid, txn.status)
if verbose:
print("begin", pos, _ts, end=' ')
if verbose > 1:
print()
sys.stdout.flush()
nrec = 0
try:
for r in txn:
if verbose > 1:
if r.data is None:
l = "bp"
else:
l = len(r.data)
print("%7d %s %s" % (u64(r.oid), l))
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn, txn)
nrec += 1
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
if partial and nrec:
ofs._status = "p"
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print("partial")
else:
ofs.tpc_abort(txn)
print("error copying transaction:", err)
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print("looking for valid txn header at", pos)
else:
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print("finish")
sys.stdout.flush()
if not verbose:
prog = pos * 20 / file_size
while prog > prog1:
prog1 = prog1 + 1
iprogress(prog1)
bad = file_size - undone - ofs._pos
print("\n%s bytes removed during recovery" % bad)
if undone:
print("%s bytes of undone transaction data were skipped" % undone)
if pack is not None:
print("Packing ...")
from ZODB.serialize import referencesf
ofs.pack(pack, referencesf)
ofs.close()
f.close()
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
print "Recovering", inp, "into", outp
if os.path.exists(outp) and not force:
die("%s exists" % outp)
f = open(inp, "rb")
if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
f.seek(0, 2)
file_size = f.tell()
ofs = ZODB.FileStorage.FileStorage(outp, create=1)
_ts = None
ok = 1
prog1 = 0
undone = 0
pos = 4L
ltid = None
while pos:
try:
npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
except EOFError:
break
except (KeyboardInterrupt, SystemExit):
raise
except Exception, err:
print "error reading txn header:", err
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print "looking for valid txn header at", pos
continue
ltid = tid
if txn is None:
undone = undone + npos - pos
pos = npos
continue
else:
pos = npos
tid = txn.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok:
print("Time stamps out of order %s, %s" % (_ts, t))
ok = 0
_ts = t.laterThan(_ts)
tid = ` _ts `
else:
_ts = t
if not ok:
print("Time stamps back in order %s" % (t))
ok = 1
ofs.tpc_begin(txn, tid, txn.status)
if verbose:
print "begin", pos, _ts,
if verbose > 1:
print
sys.stdout.flush()
nrec = 0
try:
for r in txn:
if verbose > 1:
if r.data is None:
l = "bp"
else:
l = len(r.data)
print "%7d %s %s" % (u64(r.oid), l)
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn, txn)
nrec += 1
except (KeyboardInterrupt, SystemExit):
raise
except Exception, err:
if partial and nrec:
ofs._status = "p"
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print "partial"
else:
ofs.tpc_abort(txn)
print "error copying transaction:", err
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print "looking for valid txn header at", pos
def newTid(old):
t = time.time()
ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
if old is not None:
ts = ts.laterThan(TimeStamp(old))
return ts.raw()
def newTid(old):
t = time.time()
ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
if old is not None:
ts = ts.laterThan(TimeStamp(old))
return ts.raw()
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
print("Recovering", inp, "into", outp)
if os.path.exists(outp) and not force:
die("%s exists" % outp)
f = open(inp, "rb")
if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
f.seek(0,2)
file_size = f.tell()
ofs = ZODB.FileStorage.FileStorage(outp, create=1)
_ts = None
ok = 1
prog1 = 0
undone = 0
pos = 4
ltid = None
while pos:
try:
npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
except EOFError:
break
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
print("error reading txn header:", err)
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print("looking for valid txn header at", pos)
continue
ltid = tid
if txn is None:
undone = undone + npos - pos
pos = npos
continue
else:
pos = npos
tid = txn.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok:
print(("Time stamps out of order %s, %s" % (_ts, t)))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print(("Time stamps back in order %s" % (t)))
ok = 1
ofs.tpc_begin(txn, tid, txn.status)
if verbose:
print("begin", pos, _ts, end=' ')
if verbose > 1:
print()
sys.stdout.flush()
nrec = 0
try:
for r in txn:
if verbose > 1:
if r.data is None:
l = "bp"
else:
l = len(r.data)
print("%7d %s %s" % (u64(r.oid), l))
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,
txn)
nrec += 1
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
if partial and nrec:
ofs._status = "p"
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print("partial")
else:
ofs.tpc_abort(txn)
print("error copying transaction:", err)
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print("looking for valid txn header at", pos)
else:
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print("finish")
sys.stdout.flush()
if not verbose:
prog = pos * 20 / file_size
while prog > prog1:
prog1 = prog1 + 1
iprogress(prog1)
bad = file_size - undone - ofs._pos
print("\n%s bytes removed during recovery" % bad)
if undone:
print("%s bytes of undone transaction data were skipped" % undone)
if pack is not None:
print("Packing ...")
from ZODB.serialize import referencesf
ofs.pack(pack, referencesf)
ofs.close()
f.close()
def checkLaterThan(self):
t = time.gmtime()
ts = TimeStamp(*t[:6])
ts2 = ts.laterThan(ts)
self.assert_(ts2 > ts)
