def test__p_mtime_w_serial(self): from persistent.timestamp import TimeStamp WHEN_TUPLE = (2011, 2, 15, 13, 33, 27.5) ts = TimeStamp(*WHEN_TUPLE) inst, jar, OID = self._makeOneWithJar() inst._p_serial = ts.raw() self.assertEqual(inst._p_mtime, ts.timeTime())
def newTid(old): t = time.time() ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,)) if old is not None: ts = ts.laterThan(TimeStamp(old)) return ts.raw()
def doit(srcdb, dstdb, options): outfp = options.outfp profilep = options.profilep verbose = options.verbose # some global information largest_pickle = 0 largest_txn_in_size = 0 largest_txn_in_objects = 0 total_pickle_size = 0 total_object_count = 0 # Ripped from BaseStorage.copyTransactionsFrom() ts = None ok = True prevrevids = {} counter = 0 skipper = 0 if options.timestamps: print("%4s. %26s %6s %8s %5s %5s %5s %5s %5s" % ( "NUM", "TID AS TIMESTAMP", "OBJS", "BYTES", # Does anybody know what these times mean? "t4-t0", "t1-t0", "t2-t1", "t3-t2", "t4-t3")) else: print("%4s. %20s %6s %8s %6s %6s %6s %6s %6s" % ( "NUM", "TRANSACTION ID", "OBJS", "BYTES", # Does anybody know what these times mean? "t4-t0", "t1-t0", "t2-t1", "t3-t2", "t4-t3")) for txn in srcdb.iterator(): skipper += 1 if skipper <= options.skiptxn: continue counter += 1 if counter > options.maxtxn >= 0: break tid = txn.tid if ts is None: ts = TimeStamp(tid) else: t = TimeStamp(tid) if t <= ts: if ok: print(( 'Time stamps are out of order %s, %s' % (ts, t)), file=sys.stderr) ok = False ts = t.laterThan(ts) tid = ts.raw() else: ts = t if not ok: print(( 'Time stamps are back in order %s' % t), file=sys.stderr) ok = True if verbose > 1: print(ts) prof = None if profilep and (counter % 100) == 0: prof = profile.Profile() objects = 0 size = 0 newrevids = RevidAccumulator() t0 = time.time() dstdb.tpc_begin(txn, tid, txn.status) t1 = time.time() for r in txn: oid = r.oid objects += 1 thissize = len(r.data) size += thissize if thissize > largest_pickle: largest_pickle = thissize if verbose > 1: if not r.version: vstr = 'norev' else: vstr = r.version print(utils.U64(oid), vstr, len(r.data)) oldrevid = prevrevids.get(oid, utils.z64) result = dstdb.store(oid, oldrevid, r.data, r.version, txn) newrevids.store(oid, result) t2 = time.time() result = dstdb.tpc_vote(txn) t3 = time.time() newrevids.tpc_vote(result) prevrevids.update(newrevids.get_dict()) # Profile every 100 transactions if prof: prof.runcall(dstdb.tpc_finish, txn) else: dstdb.tpc_finish(txn) t4 = time.time() # record the results if objects > largest_txn_in_objects: largest_txn_in_objects = objects if size > largest_txn_in_size: largest_txn_in_size = size if options.timestamps: tidstr = str(TimeStamp(tid)) format = "%4d. %26s %6d %8d %5.3f %5.3f %5.3f %5.3f %5.3f" else: tidstr = utils.U64(tid) format = "%4d. %20s %6d %8d %6.4f %6.4f %6.4f %6.4f %6.4f" print(format % (skipper, tidstr, objects, size, t4-t0, t1-t0, t2-t1, t3-t2, t4-t3), file=outfp) total_pickle_size += size total_object_count += objects if prof: prof.create_stats() fp = open('profile-%02d.txt' % (counter / 100), 'wb') marshal.dump(prof.stats, fp) fp.close() print("Largest pickle: %8d" % largest_pickle, file=outfp) print("Largest transaction: %8d" % largest_txn_in_size, file=outfp) print("Largest object count: %8d" % largest_txn_in_objects, file=outfp) print("Total pickle size: %14d" % total_pickle_size, file=outfp) print("Total object count: %8d" % total_object_count, file=outfp)
def __pre_pack(self, t, referencesf): logger.info("pack: beginning pre-pack") # In 2019, Unix timestamps look like # 1564006806.0 # While 64-bit integer TIDs for the same timestamp look like # 275085010696509852 # # Multiple TIDs can map to a single Unix timestamp. # For example, the 9 integers between 275085010624927044 and # 275085010624927035 all map to 1564006804.9999998. # # Therefore, Unix timestamps are ambiguous, especially if we're committing # multiple transactions rapidly (within the resolution of the underlying TID # clock). # This ambiguity mostly matters for unit tests, where we do commit rapidly. # # To help them out, we accept 64-bit integer TIDs to specify an exact # transaction to pack to. # We also allow None or a negative number to mean "current committed transaction". if t is None: t = -1 if t > 275085010696509852: # Must be a TID. # Turn it back into a time.time() for later logging ts = TimeStamp(int64_to_8bytes(t)) logger.debug( "Treating requested pack time %s as TID meaning %s", t, ts ) best_pack_tid_int = t t = ts.timeTime() elif t < 0 or t >= time.time(): # Packing for the current time or in the future means to pack # to the lastest commit in the database. This matters if not all # machine clocks are synchronized. best_pack_tid_int = MAX_TID - 1 else: # Find the latest commit before or at the pack time. # Note that several TIDs will fit in the resolution of a time.time(), # so this is slightly ambiguous. requested_pack_ts = TimeStamp(*time.gmtime(t)[:5] + (t % 60,)) requested_pack_tid = requested_pack_ts.raw() requested_pack_tid_int = bytes8_to_int64(requested_pack_tid) best_pack_tid_int = requested_pack_tid_int tid_int = self.packundo.choose_pack_transaction(best_pack_tid_int) if tid_int is None: logger.debug("all transactions before %s have already " "been packed", time.ctime(t)) return s = time.ctime(TimeStamp(int64_to_8bytes(tid_int)).timeTime()) logger.info("Analyzing transactions committed %s or before (TID %d)", s, tid_int) # In pre_pack, the adapter fills tables with # information about what to pack. The adapter # must not actually pack anything yet. def get_references(state): """Return an iterable of the set of OIDs the given state refers to.""" if not state: return () return {bytes8_to_int64(oid) for oid in referencesf(state)} self.packundo.pre_pack(tid_int, get_references) logger.info("pack: pre-pack complete") return tid_int