def checkTimeStamp(self): # Alternate test suite t = TimeStamp(2002, 1, 23, 10, 48, 5) # GMT self.assertEquals(str(t), '2002-01-23 10:48:05.000000') self.assertEquals(repr(t), '\x03B9H\x15UUU') self.assertEquals(TimeStamp('\x03B9H\x15UUU'), t) self.assertEquals(t.year(), 2002) self.assertEquals(t.month(), 1) self.assertEquals(t.day(), 23) self.assertEquals(t.hour(), 10) self.assertEquals(t.minute(), 48) self.assertEquals(round(t.second()), 5) self.assertEquals(t.timeTime(), 1011782885) t1 = TimeStamp(2002, 1, 23, 10, 48, 10) self.assertEquals(str(t1), '2002-01-23 10:48:10.000000') self.assert_(t == t) self.assert_(t != t1) self.assert_(t < t1) self.assert_(t <= t1) self.assert_(t1 >= t) self.assert_(t1 > t) self.failIf(t == t1) self.failIf(t != t) self.failIf(t > t1) self.failIf(t >= t1) self.failIf(t1 < t) self.failIf(t1 <= t) self.assertEquals(cmp(t, t), 0) self.assertEquals(cmp(t, t1), -1) self.assertEquals(cmp(t1, t), 1) self.assertEquals(t1.laterThan(t), t1) self.assert_(t.laterThan(t1) > t1) self.assertEquals(TimeStamp(2002,1,23), TimeStamp(2002,1,23,0,0,0))
def tpc_begin(self, transaction, tid=None, status=' '): if self._is_read_only: raise POSException.ReadOnlyError() self._lock_acquire() try: if self._transaction is transaction: return self._lock_release() self._commit_lock_acquire() self._lock_acquire() self._transaction = transaction self._clear_temp() user = transaction.user desc = transaction.description ext = transaction._extension if ext: ext = cPickle.dumps(ext, 1) else: ext = "" self._ude = user, desc, ext if tid is None: now = time.time() t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,))) self._ts = t = t.laterThan(self._ts) self._tid = `t` else: self._ts = TimeStamp(tid) self._tid = tid self._tstatus = status self._begin(self._tid, user, desc, ext) finally: self._lock_release()
def copy(source, dest, verbose=0): """Copy transactions from a source to a destination storage This is typically used for converting data from one storage to another. `source` must have an .iterator() method. """ _ts = None ok = 1 preindex = {}; preget = preindex.get # restore() is a new storage API method which has an identical # signature to store() except that it does not return anything. # Semantically, restore() is also identical to store() except that it # doesn't do the ConflictError or VersionLockError consistency # checks. The reason to use restore() over store() in this method is # that store() cannot be used to copy transactions spanning a version # commit or abort, or over transactional undos. # # We'll use restore() if it's available, otherwise we'll fall back to # using store(). However, if we use store, then # copyTransactionsFrom() may fail with VersionLockError or # ConflictError. restoring = py2_hasattr(dest, 'restore') fiter = source.iterator() for transaction in fiter: tid = transaction.tid if _ts is None: _ts = TimeStamp(tid) else: t = TimeStamp(tid) if t <= _ts: if ok: print(('Time stamps out of order %s, %s' % (_ts, t))) ok = 0 _ts = t.laterThan(_ts) tid = _ts.raw() else: _ts = t if not ok: print(('Time stamps back in order %s' % (t))) ok = 1 if verbose: print(_ts) dest.tpc_begin(transaction, tid, transaction.status) for r in transaction: oid = r.oid if verbose: print(oid_repr(oid), r.version, len(r.data)) if restoring: dest.restore(oid, r.tid, r.data, r.version, r.data_txn, transaction) else: pre = preget(oid, None) s = dest.store(oid, pre, r.data, r.version, transaction) preindex[oid] = s dest.tpc_vote(transaction) dest.tpc_finish(transaction)
def newTimeStamp(old=None, TimeStamp=TimeStamp, time=time.time, gmtime=time.gmtime): t = time() ts = TimeStamp(gmtime(t)[:5]+(t%60,)) if old is not None: return ts.laterThan(old) return ts
def get_timestamp(prev_ts=None): """Internal helper to return a unique TimeStamp instance. If the optional argument is not None, it must be a TimeStamp; the return value is then guaranteed to be at least 1 microsecond later the argument. """ t = time.time() t = TimeStamp(*time.gmtime(t)[:5] + (t % 60,)) if prev_ts is not None: t = t.laterThan(prev_ts) return t
def getTID(at, before): if at is not None: if before is not None: raise ValueError('can only pass zero or one of `at` and `before`') if isinstance(at, datetime.datetime): at = toTimeStamp(at) else: at = TimeStamp(at) before = at.laterThan(at).raw() elif before is not None: if isinstance(before, datetime.datetime): before = toTimeStamp(before).raw() else: before = TimeStamp(before).raw() return before
def new_tid(self): # # Probably better off using a counter as a tid # now = time.time() t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,))) #get the latest timestamp from memcache key = '%s,tid' % self._prefix self.tid_lock.acquire() try: result = self._mc.get(key) if result: t = t.laterThan(result) tid = repr(t) result = self._mc.replace(key, tid) if not result: raise MemcacheError return tid finally: self.tid_lock.release()
def tpc_begin(self, transaction, tid=None, status=' '): if self._is_read_only: raise POSException.ReadOnlyError() self._lock_acquire() try: if self._transaction is transaction: raise POSException.StorageTransactionError( "Duplicate tpc_begin calls for same transaction") self._lock_release() self._commit_lock_acquire() self._lock_acquire() self._transaction = transaction self._clear_temp() user = transaction.user desc = transaction.description ext = transaction._extension if ext: ext = dumps(ext, _protocol) else: ext = "" self._ude = user, desc, ext if tid is None: now = time.time() t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,))) self._ts = t = t.laterThan(self._ts) self._tid = t.raw() else: self._ts = TimeStamp(tid) self._tid = tid self._tstatus = status self._begin(self._tid, user, desc, ext) finally: self._lock_release()
def copyTransactionsFrom(self, other, verbose=0): """Copy transactions from another storage. This is typically used for converting data from one storage to another. `other` must have an .iterator() method. """ _ts = None ok = 1 preindex = {} preget = preindex.get # waaaa # restore() is a new storage API method which has an identical # signature to store() except that it does not return anything. # Semantically, restore() is also identical to store() except that it # doesn't do the ConflictError or VersionLockError consistency # checks. The reason to use restore() over store() in this method is # that store() cannot be used to copy transactions spanning a version # commit or abort, or over transactional undos. # # We'll use restore() if it's available, otherwise we'll fall back to # using store(). However, if we use store, then # copyTransactionsFrom() may fail with VersionLockError or # ConflictError. restoring = hasattr(self, 'restore') fiter = other.iterator() for transaction in fiter: tid = transaction.tid if _ts is None: _ts = TimeStamp(tid) else: t = TimeStamp(tid) if t <= _ts: if ok: print('Time stamps out of order %s, %s' % (_ts, t)) ok = 0 _ts = t.laterThan(_ts) tid = ` _ts ` else: _ts = t if not ok: print('Time stamps back in order %s' % (t)) ok = 1 if verbose: print _ts self.tpc_begin(transaction, tid, transaction.status) for r in transaction: oid = r.oid if verbose: print oid_repr(oid), r.version, len(r.data) if restoring: self.restore(oid, r.tid, r.data, r.version, r.data_txn, transaction) else: pre = preget(oid, None) s = self.store(oid, pre, r.data, r.version, transaction) preindex[oid] = s self.tpc_vote(transaction) self.tpc_finish(transaction) fiter.close()
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None): print "Recovering", inp, "into", outp if os.path.exists(outp) and not force: die("%s exists" % outp) f = open(inp, "rb") if f.read(4) != ZODB.FileStorage.packed_version: die("input is not a file storage") f.seek(0,2) file_size = f.tell() ofs = ZODB.FileStorage.FileStorage(outp, create=1) _ts = None ok = 1 prog1 = 0 undone = 0 pos = 4L ltid = None while pos: try: npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid) except EOFError: break except (KeyboardInterrupt, SystemExit): raise except Exception, err: print "error reading txn header:", err if not verbose: progress(prog1) pos = scan(f, pos) if verbose > 1: print "looking for valid txn header at", pos continue ltid = tid if txn is None: undone = undone + npos - pos pos = npos continue else: pos = npos tid = txn.tid if _ts is None: _ts = TimeStamp(tid) else: t = TimeStamp(tid) if t <= _ts: if ok: print ("Time stamps out of order %s, %s" % (_ts, t)) ok = 0 _ts = t.laterThan(_ts) tid = _ts.raw() else: _ts = t if not ok: print ("Time stamps back in order %s" % (t)) ok = 1 ofs.tpc_begin(txn, tid, txn.status) if verbose: print "begin", pos, _ts, if verbose > 1: print sys.stdout.flush() nrec = 0 try: for r in txn: if verbose > 1: if r.data is None: l = "bp" else: l = len(r.data) print "%7d %s %s" % (u64(r.oid), l) ofs.restore(r.oid, r.tid, r.data, '', r.data_txn, txn) nrec += 1 except (KeyboardInterrupt, SystemExit): raise except Exception, err: if partial and nrec: ofs._status = "p" ofs.tpc_vote(txn) ofs.tpc_finish(txn) if verbose: print "partial" else: ofs.tpc_abort(txn) print "error copying transaction:", err if not verbose: progress(prog1) pos = scan(f, pos) if verbose > 1: print "looking for valid txn header at", pos
def checkLaterThan(self): t = time.gmtime() ts = TimeStamp(*t[:6]) ts2 = ts.laterThan(ts) self.assert_(ts2 > ts)
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None): print("Recovering", inp, "into", outp) if os.path.exists(outp) and not force: die("%s exists" % outp) f = open(inp, "rb") if f.read(4) != ZODB.FileStorage.packed_version: die("input is not a file storage") f.seek(0, 2) file_size = f.tell() ofs = ZODB.FileStorage.FileStorage(outp, create=1) _ts = None ok = 1 prog1 = 0 undone = 0 pos = 4 ltid = None while pos: try: npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid) except EOFError: break except (KeyboardInterrupt, SystemExit): raise except Exception as err: print("error reading txn header:", err) if not verbose: progress(prog1) pos = scan(f, pos) if verbose > 1: print("looking for valid txn header at", pos) continue ltid = tid if txn is None: undone = undone + npos - pos pos = npos continue else: pos = npos tid = txn.tid if _ts is None: _ts = TimeStamp(tid) else: t = TimeStamp(tid) if t <= _ts: if ok: print(("Time stamps out of order %s, %s" % (_ts, t))) ok = 0 _ts = t.laterThan(_ts) tid = _ts.raw() else: _ts = t if not ok: print(("Time stamps back in order %s" % (t))) ok = 1 ofs.tpc_begin(txn, tid, txn.status) if verbose: print("begin", pos, _ts, end=' ') if verbose > 1: print() sys.stdout.flush() nrec = 0 try: for r in txn: if verbose > 1: if r.data is None: l = "bp" else: l = len(r.data) print("%7d %s %s" % (u64(r.oid), l)) ofs.restore(r.oid, r.tid, r.data, '', r.data_txn, txn) nrec += 1 except (KeyboardInterrupt, SystemExit): raise except Exception as err: if partial and nrec: ofs._status = "p" ofs.tpc_vote(txn) ofs.tpc_finish(txn) if verbose: print("partial") else: ofs.tpc_abort(txn) print("error copying transaction:", err) if not verbose: progress(prog1) pos = scan(f, pos) if verbose > 1: print("looking for valid txn header at", pos) else: ofs.tpc_vote(txn) ofs.tpc_finish(txn) if verbose: print("finish") sys.stdout.flush() if not verbose: prog = pos * 20 / file_size while prog > prog1: prog1 = prog1 + 1 iprogress(prog1) bad = file_size - undone - ofs._pos print("\n%s bytes removed during recovery" % bad) if undone: print("%s bytes of undone transaction data were skipped" % undone) if pack is not None: print("Packing ...") from ZODB.serialize import referencesf ofs.pack(pack, referencesf) ofs.close() f.close()
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None): print "Recovering", inp, "into", outp if os.path.exists(outp) and not force: die("%s exists" % outp) f = open(inp, "rb") if f.read(4) != ZODB.FileStorage.packed_version: die("input is not a file storage") f.seek(0, 2) file_size = f.tell() ofs = ZODB.FileStorage.FileStorage(outp, create=1) _ts = None ok = 1 prog1 = 0 undone = 0 pos = 4L ltid = None while pos: try: npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid) except EOFError: break except (KeyboardInterrupt, SystemExit): raise except Exception, err: print "error reading txn header:", err if not verbose: progress(prog1) pos = scan(f, pos) if verbose > 1: print "looking for valid txn header at", pos continue ltid = tid if txn is None: undone = undone + npos - pos pos = npos continue else: pos = npos tid = txn.tid if _ts is None: _ts = TimeStamp(tid) else: t = TimeStamp(tid) if t <= _ts: if ok: print("Time stamps out of order %s, %s" % (_ts, t)) ok = 0 _ts = t.laterThan(_ts) tid = ` _ts ` else: _ts = t if not ok: print("Time stamps back in order %s" % (t)) ok = 1 ofs.tpc_begin(txn, tid, txn.status) if verbose: print "begin", pos, _ts, if verbose > 1: print sys.stdout.flush() nrec = 0 try: for r in txn: if verbose > 1: if r.data is None: l = "bp" else: l = len(r.data) print "%7d %s %s" % (u64(r.oid), l) ofs.restore(r.oid, r.tid, r.data, '', r.data_txn, txn) nrec += 1 except (KeyboardInterrupt, SystemExit): raise except Exception, err: if partial and nrec: ofs._status = "p" ofs.tpc_vote(txn) ofs.tpc_finish(txn) if verbose: print "partial" else: ofs.tpc_abort(txn) print "error copying transaction:", err if not verbose: progress(prog1) pos = scan(f, pos) if verbose > 1: print "looking for valid txn header at", pos
def newTid(old): t = time.time() ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,)) if old is not None: ts = ts.laterThan(TimeStamp(old)) return ts.raw()
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None): print("Recovering", inp, "into", outp) if os.path.exists(outp) and not force: die("%s exists" % outp) f = open(inp, "rb") if f.read(4) != ZODB.FileStorage.packed_version: die("input is not a file storage") f.seek(0,2) file_size = f.tell() ofs = ZODB.FileStorage.FileStorage(outp, create=1) _ts = None ok = 1 prog1 = 0 undone = 0 pos = 4 ltid = None while pos: try: npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid) except EOFError: break except (KeyboardInterrupt, SystemExit): raise except Exception as err: print("error reading txn header:", err) if not verbose: progress(prog1) pos = scan(f, pos) if verbose > 1: print("looking for valid txn header at", pos) continue ltid = tid if txn is None: undone = undone + npos - pos pos = npos continue else: pos = npos tid = txn.tid if _ts is None: _ts = TimeStamp(tid) else: t = TimeStamp(tid) if t <= _ts: if ok: print(("Time stamps out of order %s, %s" % (_ts, t))) ok = 0 _ts = t.laterThan(_ts) tid = _ts.raw() else: _ts = t if not ok: print(("Time stamps back in order %s" % (t))) ok = 1 ofs.tpc_begin(txn, tid, txn.status) if verbose: print("begin", pos, _ts, end=' ') if verbose > 1: print() sys.stdout.flush() nrec = 0 try: for r in txn: if verbose > 1: if r.data is None: l = "bp" else: l = len(r.data) print("%7d %s %s" % (u64(r.oid), l)) ofs.restore(r.oid, r.tid, r.data, '', r.data_txn, txn) nrec += 1 except (KeyboardInterrupt, SystemExit): raise except Exception as err: if partial and nrec: ofs._status = "p" ofs.tpc_vote(txn) ofs.tpc_finish(txn) if verbose: print("partial") else: ofs.tpc_abort(txn) print("error copying transaction:", err) if not verbose: progress(prog1) pos = scan(f, pos) if verbose > 1: print("looking for valid txn header at", pos) else: ofs.tpc_vote(txn) ofs.tpc_finish(txn) if verbose: print("finish") sys.stdout.flush() if not verbose: prog = pos * 20 / file_size while prog > prog1: prog1 = prog1 + 1 iprogress(prog1) bad = file_size - undone - ofs._pos print("\n%s bytes removed during recovery" % bad) if undone: print("%s bytes of undone transaction data were skipped" % undone) if pack is not None: print("Packing ...") from ZODB.serialize import referencesf ofs.pack(pack, referencesf) ofs.close() f.close()