def newTid(old): t = time.time() ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,)) if old is not None: ts = ts.laterThan(TimeStamp(old)) return ts.raw()
def doit(srcdb, dstdb, options): outfp = options.outfp profilep = options.profilep verbose = options.verbose # some global information largest_pickle = 0 largest_txn_in_size = 0 largest_txn_in_objects = 0 total_pickle_size = 0 total_object_count = 0 # Ripped from BaseStorage.copyTransactionsFrom() ts = None ok = True prevrevids = {} counter = 0 skipper = 0 if options.timestamps: print("%4s. %26s %6s %8s %5s %5s %5s %5s %5s" % ( "NUM", "TID AS TIMESTAMP", "OBJS", "BYTES", # Does anybody know what these times mean? "t4-t0", "t1-t0", "t2-t1", "t3-t2", "t4-t3")) else: print("%4s. %20s %6s %8s %6s %6s %6s %6s %6s" % ( "NUM", "TRANSACTION ID", "OBJS", "BYTES", # Does anybody know what these times mean? "t4-t0", "t1-t0", "t2-t1", "t3-t2", "t4-t3")) for txn in srcdb.iterator(): skipper += 1 if skipper <= options.skiptxn: continue counter += 1 if counter > options.maxtxn >= 0: break tid = txn.tid if ts is None: ts = TimeStamp(tid) else: t = TimeStamp(tid) if t <= ts: if ok: print(( 'Time stamps are out of order %s, %s' % (ts, t)), file=sys.stderr) ok = False ts = t.laterThan(ts) tid = ts.raw() else: ts = t if not ok: print(( 'Time stamps are back in order %s' % t), file=sys.stderr) ok = True if verbose > 1: print(ts) prof = None if profilep and (counter % 100) == 0: prof = profile.Profile() objects = 0 size = 0 newrevids = RevidAccumulator() t0 = time.time() dstdb.tpc_begin(txn, tid, txn.status) t1 = time.time() for r in txn: oid = r.oid objects += 1 thissize = len(r.data) size += thissize if thissize > largest_pickle: largest_pickle = thissize if verbose > 1: if not r.version: vstr = 'norev' else: vstr = r.version print(utils.U64(oid), vstr, len(r.data)) oldrevid = prevrevids.get(oid, utils.z64) result = dstdb.store(oid, oldrevid, r.data, r.version, txn) newrevids.store(oid, result) t2 = time.time() result = dstdb.tpc_vote(txn) t3 = time.time() newrevids.tpc_vote(result) prevrevids.update(newrevids.get_dict()) # Profile every 100 transactions if prof: prof.runcall(dstdb.tpc_finish, txn) else: dstdb.tpc_finish(txn) t4 = time.time() # record the results if objects > largest_txn_in_objects: largest_txn_in_objects = objects if size > largest_txn_in_size: largest_txn_in_size = size if options.timestamps: tidstr = str(TimeStamp(tid)) format = "%4d. %26s %6d %8d %5.3f %5.3f %5.3f %5.3f %5.3f" else: tidstr = utils.U64(tid) format = "%4d. %20s %6d %8d %6.4f %6.4f %6.4f %6.4f %6.4f" print(format % (skipper, tidstr, objects, size, t4-t0, t1-t0, t2-t1, t3-t2, t4-t3), file=outfp) total_pickle_size += size total_object_count += objects if prof: prof.create_stats() fp = open('profile-%02d.txt' % (counter / 100), 'wb') marshal.dump(prof.stats, fp) fp.close() print("Largest pickle: %8d" % largest_pickle, file=outfp) print("Largest transaction: %8d" % largest_txn_in_size, file=outfp) print("Largest object count: %8d" % largest_txn_in_objects, file=outfp) print("Total pickle size: %14d" % total_pickle_size, file=outfp) print("Total object count: %8d" % total_object_count, file=outfp)