def tpc_begin(self, transaction, tid=None, status=' '):
if self._is_read_only:
raise POSException.ReadOnlyError()
self._lock_acquire()
try:
if self._transaction is transaction:
return
self._lock_release()
self._commit_lock_acquire()
self._lock_acquire()
self._transaction = transaction
self._clear_temp()
user = transaction.user
desc = transaction.description
ext = transaction._extension
if ext:
ext = cPickle.dumps(ext, 1)
else:
ext = ""
self._ude = user, desc, ext
if tid is None:
now = time.time()
t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,)))
self._ts = t = t.laterThan(self._ts)
self._tid = `t`
else:
self._ts = TimeStamp(tid)
self._tid = tid
self._tstatus = status
self._begin(self._tid, user, desc, ext)
finally:
self._lock_release()
def run(path, days, notPacked):
f = open(path, "rb")
f.seek(0, 2)
now = datetime.date.today()
#day->size
stats = {}
th = prev_txn(f)
bool = True
while bool:
ts = TimeStamp(th.tid)
then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day()))
delta = timedelta(days=int(days))
if( not(now - then < delta)):
bool = False
th = th.prev_txn()
reader = Reader()
iterator = FileIterator(path, pos=th._pos)
for i in iterator:
object_types = {}
for o in i:
ot = reader.getIdentity(o.data)
try:
stats[ot] = stats[ot] + 1
except KeyError:
stats[ot] = 1
f.close()
for (o,n) in sorted(stats.items(), key=lambda (k,v): v, reverse=True):
print "%6d: %s" % (n,o)
def copy(source, dest, verbose=0):
"""Copy transactions from a source to a destination storage
This is typically used for converting data from one storage to
another. `source` must have an .iterator() method.
"""
_ts = None
ok = 1
preindex = {};
preget = preindex.get
# restore() is a new storage API method which has an identical
# signature to store() except that it does not return anything.
# Semantically, restore() is also identical to store() except that it
# doesn't do the ConflictError or VersionLockError consistency
# checks. The reason to use restore() over store() in this method is
# that store() cannot be used to copy transactions spanning a version
# commit or abort, or over transactional undos.
#
# We'll use restore() if it's available, otherwise we'll fall back to
# using store(). However, if we use store, then
# copyTransactionsFrom() may fail with VersionLockError or
# ConflictError.
restoring = py2_hasattr(dest, 'restore')
fiter = source.iterator()
for transaction in fiter:
tid = transaction.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok: print(('Time stamps out of order %s, %s' % (_ts, t)))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print(('Time stamps back in order %s' % (t)))
ok = 1
if verbose:
print(_ts)
dest.tpc_begin(transaction, tid, transaction.status)
for r in transaction:
oid = r.oid
if verbose:
print(oid_repr(oid), r.version, len(r.data))
if restoring:
dest.restore(oid, r.tid, r.data, r.version,
r.data_txn, transaction)
else:
pre = preget(oid, None)
s = dest.store(oid, pre, r.data, r.version, transaction)
preindex[oid] = s
dest.tpc_vote(transaction)
dest.tpc_finish(transaction)
def newTimeStamp(old=None,
TimeStamp=TimeStamp,
time=time.time, gmtime=time.gmtime):
t = time()
ts = TimeStamp(gmtime(t)[:5]+(t%60,))
if old is not None:
return ts.laterThan(old)
return ts
def testMTime(self):
obj = P()
self.assertEqual(obj._p_mtime, None)
t = int(time.time())
ts = TimeStamp(*time.gmtime(t)[:6])
obj._p_serial = ts.raw()
self.assertEqual(obj._p_mtime, t)
self.assertIsInstance(obj._p_mtime, float)
def get_timestamp(prev_ts=None):
"""Internal helper to return a unique TimeStamp instance.
If the optional argument is not None, it must be a TimeStamp; the
return value is then guaranteed to be at least 1 microsecond later
the argument.
"""
t = time.time()
t = TimeStamp(*time.gmtime(t)[:5] + (t % 60,))
if prev_ts is not None:
t = t.laterThan(prev_ts)
return t
def checkFullTimeStamp(self):
native_ts = int(time.time()) # fractional seconds get in the way
t = time.gmtime(native_ts) # the corresponding GMT struct tm
ts = TimeStamp(*t[:6])
# Seconds are stored internally via (conceptually) multiplying by
# 2**32 then dividing by 60, ending up with a 32-bit integer.
# While this gives a lot of room for cramming many distinct
# TimeStamps into a second, it's not good at roundtrip accuracy.
# For example, 1 second is stored as int(2**32/60) == 71582788.
# Converting back gives 71582788*60.0/2**32 == 0.9999999962747097.
# In general, we can lose up to 0.999... to truncation during
# storing, creating an absolute error up to about 1*60.0/2**32 ==
# 0.000000014 on the seconds value we get back. This is so even
# when we have an exact integral second value going in (as we
# do in this test), so we can't expect equality in any comparison
# involving seconds. Minutes (etc) are stored exactly, so we
# can expect equality for those.
self.assert_(abs(ts.timeTime() - native_ts) < EPSILON)
self.assertEqual(ts.year(), t[0])
self.assertEqual(ts.month(), t[1])
self.assertEqual(ts.day(), t[2])
self.assertEquals(ts.hour(), t[3])
self.assertEquals(ts.minute(), t[4])
self.assert_(abs(ts.second() - t[5]) < EPSILON)
def getTID(at, before):
if at is not None:
if before is not None:
raise ValueError('can only pass zero or one of `at` and `before`')
if isinstance(at, datetime.datetime):
at = toTimeStamp(at)
else:
at = TimeStamp(at)
before = at.laterThan(at).raw()
elif before is not None:
if isinstance(before, datetime.datetime):
before = toTimeStamp(before).raw()
else:
before = TimeStamp(before).raw()
return before
def new_tid(self):
#
# Probably better off using a counter as a tid
#
now = time.time()
t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,)))
#get the latest timestamp from memcache
key = '%s,tid' % self._prefix
self.tid_lock.acquire()
try:
result = self._mc.get(key)
if result:
t = t.laterThan(result)
tid = repr(t)
result = self._mc.replace(key, tid)
if not result:
raise MemcacheError
return tid
finally:
self.tid_lock.release()
def _check_ymd(self, yr, mo, dy):
ts = TimeStamp(yr, mo, dy)
self.assertEqual(ts.year(), yr)
self.assertEqual(ts.month(), mo)
self.assertEqual(ts.day(), dy)
self.assertEquals(ts.hour(), 0)
self.assertEquals(ts.minute(), 0)
self.assertEquals(ts.second(), 0)
t = time.gmtime(ts.timeTime())
self.assertEquals(yr, t[0])
self.assertEquals(mo, t[1])
self.assertEquals(dy, t[2])
def tpc_begin(self, transaction, tid=None, status=' '):
if self._is_read_only:
raise POSException.ReadOnlyError()
self._lock_acquire()
try:
if self._transaction is transaction:
raise POSException.StorageTransactionError(
"Duplicate tpc_begin calls for same transaction")
self._lock_release()
self._commit_lock_acquire()
self._lock_acquire()
self._transaction = transaction
self._clear_temp()
user = transaction.user
desc = transaction.description
ext = transaction._extension
if ext:
ext = dumps(ext, _protocol)
else:
ext = ""
self._ude = user, desc, ext
if tid is None:
now = time.time()
t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,)))
self._ts = t = t.laterThan(self._ts)
self._tid = t.raw()
else:
self._ts = TimeStamp(tid)
self._tid = tid
self._tstatus = status
self._begin(self._tid, user, desc, ext)
finally:
self._lock_release()
def checkTimeStamp(self):
# Alternate test suite
t = TimeStamp(2002, 1, 23, 10, 48, 5) # GMT
self.assertEquals(str(t), '2002-01-23 10:48:05.000000')
self.assertEquals(repr(t), '\x03B9H\x15UUU')
self.assertEquals(TimeStamp('\x03B9H\x15UUU'), t)
self.assertEquals(t.year(), 2002)
self.assertEquals(t.month(), 1)
self.assertEquals(t.day(), 23)
self.assertEquals(t.hour(), 10)
self.assertEquals(t.minute(), 48)
self.assertEquals(round(t.second()), 5)
self.assertEquals(t.timeTime(), 1011782885)
t1 = TimeStamp(2002, 1, 23, 10, 48, 10)
self.assertEquals(str(t1), '2002-01-23 10:48:10.000000')
self.assert_(t == t)
self.assert_(t != t1)
self.assert_(t < t1)
self.assert_(t <= t1)
self.assert_(t1 >= t)
self.assert_(t1 > t)
self.failIf(t == t1)
self.failIf(t != t)
self.failIf(t > t1)
self.failIf(t >= t1)
self.failIf(t1 < t)
self.failIf(t1 <= t)
self.assertEquals(cmp(t, t), 0)
self.assertEquals(cmp(t, t1), -1)
self.assertEquals(cmp(t1, t), 1)
self.assertEquals(t1.laterThan(t), t1)
self.assert_(t.laterThan(t1) > t1)
self.assertEquals(TimeStamp(2002, 1, 23),
TimeStamp(2002, 1, 23, 0, 0, 0))
def checkLaterThan(self):
t = time.gmtime()
ts = TimeStamp(*t[:6])
ts2 = ts.laterThan(ts)
self.assert_(ts2 > ts)
def checkRawTimestamp(self):
t = time.gmtime()
ts1 = TimeStamp(*t[:6])
ts2 = TimeStamp(`ts1`)
self.assertEquals(ts1, ts2)
self.assertEquals(ts1.timeTime(), ts2.timeTime())
self.assertEqual(ts1.year(), ts2.year())
self.assertEqual(ts1.month(), ts2.month())
self.assertEqual(ts1.day(), ts2.day())
self.assertEquals(ts1.hour(), ts2.hour())
self.assertEquals(ts1.minute(), ts2.minute())
self.assert_(abs(ts1.second() - ts2.second()) < EPSILON)
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
print "Recovering", inp, "into", outp
if os.path.exists(outp) and not force:
die("%s exists" % outp)
f = open(inp, "rb")
if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
f.seek(0,2)
file_size = f.tell()
ofs = ZODB.FileStorage.FileStorage(outp, create=1)
_ts = None
ok = 1
prog1 = 0
undone = 0
pos = 4L
ltid = None
while pos:
try:
npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
except EOFError:
break
except (KeyboardInterrupt, SystemExit):
raise
except Exception, err:
print "error reading txn header:", err
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print "looking for valid txn header at", pos
continue
ltid = tid
if txn is None:
undone = undone + npos - pos
pos = npos
continue
else:
pos = npos
tid = txn.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok:
print ("Time stamps out of order %s, %s" % (_ts, t))
ok = 0
_ts = t.laterThan(_ts)
tid = `_ts`
else:
_ts = t
if not ok:
print ("Time stamps back in order %s" % (t))
ok = 1
ofs.tpc_begin(txn, tid, txn.status)
if verbose:
print "begin", pos, _ts,
if verbose > 1:
print
sys.stdout.flush()
nrec = 0
try:
for r in txn:
if verbose > 1:
if r.data is None:
l = "bp"
else:
l = len(r.data)
print "%7d %s %s" % (u64(r.oid), l)
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,
txn)
nrec += 1
except (KeyboardInterrupt, SystemExit):
raise
except Exception, err:
if partial and nrec:
ofs._status = "p"
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print "partial"
else:
ofs.tpc_abort(txn)
print "error copying transaction:", err
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print "looking for valid txn header at", pos
def get_timestamp(self):
return TimeStamp(self.tid)
def _get_timestamp(self):
t = time.time()
t = TimeStamp(*time.gmtime(t)[:5] + (t % 60, ))
return ` t `
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
print("Recovering", inp, "into", outp)
if os.path.exists(outp) and not force:
die("%s exists" % outp)
f = open(inp, "rb")
if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
f.seek(0,2)
file_size = f.tell()
ofs = ZODB.FileStorage.FileStorage(outp, create=1)
_ts = None
ok = 1
prog1 = 0
undone = 0
pos = 4
ltid = None
while pos:
try:
npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
except EOFError:
break
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
print("error reading txn header:", err)
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print("looking for valid txn header at", pos)
continue
ltid = tid
if txn is None:
undone = undone + npos - pos
pos = npos
continue
else:
pos = npos
tid = txn.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok:
print(("Time stamps out of order %s, %s" % (_ts, t)))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print(("Time stamps back in order %s" % (t)))
ok = 1
ofs.tpc_begin(txn, tid, txn.status)
if verbose:
print("begin", pos, _ts, end=' ')
if verbose > 1:
print()
sys.stdout.flush()
nrec = 0
try:
for r in txn:
if verbose > 1:
if r.data is None:
l = "bp"
else:
l = len(r.data)
print("%7d %s %s" % (u64(r.oid), l))
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,
txn)
nrec += 1
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
if partial and nrec:
ofs._status = "p"
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print("partial")
else:
ofs.tpc_abort(txn)
print("error copying transaction:", err)
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print("looking for valid txn header at", pos)
else:
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print("finish")
sys.stdout.flush()
if not verbose:
prog = pos * 20 / file_size
while prog > prog1:
prog1 = prog1 + 1
iprogress(prog1)
bad = file_size - undone - ofs._pos
print("\n%s bytes removed during recovery" % bad)
if undone:
print("%s bytes of undone transaction data were skipped" % undone)
if pack is not None:
print("Packing ...")
from ZODB.serialize import referencesf
ofs.pack(pack, referencesf)
ofs.close()
f.close()
def readable_tid_repr(tid):
result = tid_repr(tid)
if isinstance(tid, str) and len(tid) == 8:
result = "%s %s" % (result, TimeStamp(tid))
return result
def main():
usage = "usage: %prog [options] filename"
parser = OptionParser(usage=usage)
parser.add_option("-n",
"--number",
dest="num",
help="display only the 'n' busiest days",
default=20,
type="int")
parser.add_option("-f",
"--file",
dest="filename",
action="store",
type="string",
help="your FileStorage")
parser.add_option(
"-d",
"--date",
dest="date",
action="store",
type="string",
help="show the stats only for the date d (format dd-mm-yyyy)")
parser.add_option("-a",
"--days",
dest="days",
action="store",
default="0",
type="string",
help="show the stats only for the last 'a' days")
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_false",
help="show percentage and time remaining")
(options, args) = parser.parse_args()
objectsToDisplay = options.num
VERBOSE = False
if options.filename:
fname = options.filename
else:
print "You have to enter the filename, see --help for details"
return 2
if options.verbose != None:
VERBOSE = True
stats = {}
start = time.time()
size = os.stat(fname).st_size
it = ZODB.FileStorage.FileIterator(fname)
lastPercent = 0.0
recordsCounter = 0
interval = 0.005
dataFound = False
now = datetime.date.today()
try:
for t in it:
#Format the date of the current transaction following dd-mm-yyyy
ts = TimeStamp(t.tid)
then = datetime.date(int(ts.year()), int(ts.month()),
int(ts.day()))
delta = timedelta(days=int(options.days))
if ((not int(options.days)) or (now - then < delta)):
dateT = strftime("%d-%m-%Y", [
int(ts.year()),
int(ts.month()),
int(ts.day()), 1, 1, 1, 1, 1, 1
])
percent = float(it._file.tell()) / float(size) * 100
#Check if we found the searched date
if options.date:
if str(dateT) == str(options.date):
dataFound = True
elif dataFound:
break
#Show the percentage of the work completed and the remaining time
if (percent - lastPercent > interval):
spentTime = time.time() - start
remainingTime = spentTime / float(
it._file.tell()) * (float(size)) - spentTime
if VERBOSE:
sys.stderr.write(
"\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d"
% (percent, GetInHMS(time.time() - start, True),
GetInHMS(remainingTime, False), recordsCounter))
lastPercent = percent
stat = stats.get(dateT)
if stat is None:
stat = stats[dateT] = Stat()
stat.n = 1
else:
stat.n += 1
for r in t:
#need to reduce the time of the dictionary stats from time to time
if recordsCounter % (objectsToDisplay * 100) == 0:
tmp = {}
for date, s in sorted(
stats.items(),
key=lambda (k, v): v.n,
reverse=True)[0:objectsToDisplay]:
tmp[date] = s
try:
tmp[dateT] = stats[dateT]
except KeyError:
pass
stats = tmp
if r.data:
mod, klass = get_pickle_metadata(r.data)
l = len(r.data)
stat = stats.get(dateT)
stat.records += 1
recordsCounter += 1
stat = stats.get(dateT)
if stat is not None:
stat.mean.append(TimeStamp(t.tid).timeTime())
except KeyboardInterrupt:
pass
print "\n"
print "%-15s %17s %17s %22s" % ("Date", "Transactions", "Records Changed",
"Average interval")
print "%s" % "_" * 74
if options.date:
for date, s in sorted(stats.items(),
key=lambda (k, v): v.n,
reverse=True):
meanTime = 0
for i in range(1, len(s.mean)):
meanTime += s.mean[i] - s.mean[i - 1]
if str(date) == str(options.date):
print "%-15s | %15d | %15d | %15f secs" % (date,
(s.n), s.records,
meanTime / s.n)
else:
for date, s in sorted(stats.items(),
key=lambda (k, v): v.n,
reverse=True)[0:objectsToDisplay]:
meanTime = 0
for i in range(1, len(s.mean)):
meanTime += s.mean[i] - s.mean[i - 1]
print "%-15s | %15d | %15d | %15f secs" % (date, (s.n), s.records,
meanTime / s.n)
def newTid(old):
t = time.time()
ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
if old is not None:
ts = ts.laterThan(TimeStamp(old))
return `ts`
def main(argv=sys.argv):
parser = optparse.OptionParser(description=__doc__,
usage="%prog [options] config_file")
parser.add_option(
"--dry-run",
dest="dry_run",
action="store_true",
help="Attempt to open the storages, then explain what would be done")
parser.add_option(
"--clear",
dest="clear",
action="store_true",
help="Clear the contents of the destination storage before copying")
parser.add_option(
"--incremental",
dest="incremental",
action="store_true",
help="Assume the destination contains a partial copy of the source "
"and resume copying from the last transaction. WARNING: no "
"effort is made to verify that the destination holds the same "
"transaction data before this point! Use at your own risk. "
"Currently only supports RelStorage destinations.")
parser.set_defaults(dry_run=False, clear=False)
options, args = parser.parse_args(argv[1:])
if len(args) != 1:
parser.error("The name of one configuration file is required.")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(BytesIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, args[0])
source = config.source.open()
destination = config.destination.open()
log.info("Storages opened successfully.")
if options.incremental:
if not hasattr(destination, '_adapter'):
msg = ("Error: no API is known for determining the last committed "
"transaction of the destination storage. Aborting "
"conversion.")
sys.exit(msg)
if not storage_has_data(destination):
log.warning(
"Destination empty, start conversion from the beginning.")
else:
last_tid = destination._adapter.txncontrol.get_tid(
destination._load_cursor)
next_tid = p64(last_tid + 1)
source = source.iterator(start=next_tid)
log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid))
if options.dry_run:
log.info("Dry run mode: not changing the destination.")
if storage_has_data(destination):
log.warning("The destination storage has data.")
count = 0
for txn in source.iterator():
log.info('%s user=%s description=%s' %
(TimeStamp(txn.tid), txn.user, txn.description))
count += 1
log.info("Would copy %d transactions.", count)
else:
if options.clear:
log.info("Clearing old data...")
if hasattr(destination, 'zap_all'):
destination.zap_all()
else:
msg = ("Error: no API is known for clearing this type "
"of storage. Use another method.")
sys.exit(msg)
log.info("Done clearing old data.")
if storage_has_data(destination):
msg = "Error: the destination storage has data. Try --clear."
sys.exit(msg)
destination.copyTransactionsFrom(source)
source.close()
destination.close()
def toTimeStamp(dt):
utc_struct = dt.utctimetuple()
# if this is a leapsecond, this will probably fail. That may be a good
# thing: leapseconds are not really accounted for with serials.
args = utc_struct[:5] + (utc_struct[5] + dt.microsecond / 1000000.0, )
return TimeStamp(*args)
def checkCompare(self):
ts1 = TimeStamp(1972, 6, 27)
ts2 = TimeStamp(1971, 12, 12)
self.assert_(ts1 > ts2)
self.assert_(ts2 <= ts1)
def main():
usage = "usage: %prog [options] filename"
parser = OptionParser(usage=usage)
parser.add_option("-n", "--number", dest="num",
help="display only the n biggest objects", default=-1, type="int")
parser.add_option("-f", "--output", dest="filename", action="store", type="string",
help="the FileStorage")
parser.add_option("-v", "--verbose", dest="verbose", action="store_false",
help="show percentage and time remaining")
(options, args) = parser.parse_args()
VERBOSE = False
if options.filename:
fname = options.filename
else:
print "You have to enter the FileStorage filename, see --help for details"
return 2
if options.verbose != None:
VERBOSE = True
objectsToDisplay = options.num
stats = {}
start = time.time()
size = os.stat(fname).st_size
it = ZODB.FileStorage.FileIterator(fname)
lastPercent = 0.0
recordsCounter = 0
interval = 0.005
now = datetime.date.today()
try:
for t in it:
percent = float(it._file.tell())/float(size) * 100
#Show the percentage of the work completed and the remaining time
if(percent - lastPercent > interval):
spentTime = time.time() - start
remainingTime = spentTime / float(it._file.tell()) * (float(size)) - spentTime
if VERBOSE:
sys.stderr.write("\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d" % (percent,GetInHMS(time.time() - start), GetInHMS(remainingTime), recordsCounter))
sys.stdout.flush()
lastPercent = percent
for r in t:
#need to reduce the time of the dictionary stats from time to time
ts = TimeStamp(t.tid)
then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day()))
delta = timedelta(days=3)
#don't reduce the size of the dictionary when analysing last 3 days transactions
if recordsCounter % (objectsToDisplay*100) == 0 and (now - then > delta):
tmp = {}
for class_name, s in sorted(
stats.items(), key=lambda (k,v): v.size, reverse=True)[0: objectsToDisplay]:
tmp[class_name] = s
stats = tmp
if r.data:
mod, klass = get_pickle_metadata(r.data)
l = len(r.data)
class_name = mod + "." + klass + " oid: " + oid_repr(r.oid).strip()
stat = stats.get(class_name)
if stat is None:
stat = stats[class_name] = Stat()
stat.size = stat.min = stat.max = l
stat.oid = oid_repr(r.oid).strip()
stat.className = mod + "." + klass
stat.number = 1
else:
stat.min = min(stat.min, l)
stat.max = max(stat.max, l)
stat.number = stat.number + 1
stat.size = stat.size + l
recordsCounter += 1
except KeyboardInterrupt:
pass
print "\n"
print "%-41s %9s %15s %15s %9s %9s %9s" % ("Module.ClassName", "Oid", "Percentage", "Total Size", "Min", "Max", "Copies")
print "%s" % "_" * 114
for class_name, s in sorted(
stats.items(), key=lambda (k,v): v.size, reverse=True)[0: objectsToDisplay]:
class_name = s.className
if len(class_name) > 40:
class_name = class_name[::-1][0:35][::-1]
class_name = "[..]" + class_name
print "%-40s | %8s | %13f%% | %13s | %7s | %7s | %7s" % (class_name, s.oid, (s.size*100.0/size) , pretty_size(s.size), pretty_size(s.min), pretty_size(s.max), s.number)
def run(path, days, notPacked):
f = open(path, "rb")
f.seek(0, 2)
size = os.path.getsize(path)
now = datetime.date.today()
notPackedDays = []
for day in range(notPacked):
notPackedDays.append(str(now - timedelta(days=day + 1)))
# day->size
stats = {}
th = prev_txn(f)
bool = True
while bool:
ts = TimeStamp(th.tid)
then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day()))
delta = timedelta(days=int(days))
if now - then < delta:
dateT = strftime("%Y-%m-%d", [int(ts.year()), int(ts.month()), int(ts.day()), 1, 1, 1, 1, 1, 1])
try:
stats[dateT] = stats[dateT] + th.length
except KeyError:
stats[dateT] = th.length
else:
bool = False
th = th.prev_txn()
f.close()
total = 0
totalPacked = 0
daysPacked = 0
for (d, s) in sorted(stats.items(), key=lambda (k, v): v, reverse=True):
print d, "size:", pretty_size(s),
date = str(d)
if date in notPackedDays or date == str(now):
print "(not yet packed)"
else:
totalPacked = totalPacked + s
daysPacked = daysPacked + 1
print
total = total + s
if int(totalPacked):
average = totalPacked / int(daysPacked)
else:
average = 0
print "\n-- ALREADY PACKED DAYS--"
print "The amount of data added in", daysPacked, "days is", pretty_size(totalPacked)
print "Average", pretty_size(average), "per day"
print "Following this trend, the size of the database will be:"
print "\t", pretty_size(average * 365 + size), " in 1 year"
print "\t", pretty_size(average * 365 * 2 + size), " in 2 years"
print "\t", pretty_size(average * 365 * 10 + size), " in 10 years"
print "\n-- ALL DAYS --"
print "The amount of data added in", days, "days is", pretty_size(total)
if int(total):
print "Average", pretty_size(total / int(days)), "per day"
else:
print "Average 0bytes per day"
def timestamp(minutes):
import time
from persistent.TimeStamp import TimeStamp
t = time.time() + 60 * minutes
return TimeStamp(*time.gmtime(t)[:5] + (t % 60, ))
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
print("Recovering", inp, "into", outp)
if os.path.exists(outp) and not force:
die("%s exists" % outp)
f = open(inp, "rb")
if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
f.seek(0,2)
file_size = f.tell()
ofs = ZODB.FileStorage.FileStorage(outp, create=1)
_ts = None
ok = 1
prog1 = 0
undone = 0
pos = 4
ltid = None
while pos:
try:
npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
except EOFError:
break
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
print("error reading txn header:", err)
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print("looking for valid txn header at", pos)
continue
ltid = tid
if txn is None:
undone = undone + npos - pos
pos = npos
continue
else:
pos = npos
tid = txn.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok:
print(("Time stamps out of order %s, %s" % (_ts, t)))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print(("Time stamps back in order %s" % (t)))
ok = 1
ofs.tpc_begin(txn, tid, txn.status)
if verbose:
print("begin", pos, _ts, end=' ')
if verbose > 1:
print()
sys.stdout.flush()
nrec = 0
try:
for r in txn:
if verbose > 1:
if r.data is None:
l = "bp"
else:
l = len(r.data)
print("%7d %s %s" % (u64(r.oid), l))
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,
txn)
nrec += 1
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
if partial and nrec:
ofs._status = "p"
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print("partial")
else:
ofs.tpc_abort(txn)
print("error copying transaction:", err)
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print("looking for valid txn header at", pos)
else:
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print("finish")
sys.stdout.flush()
if not verbose:
prog = pos * 20 / file_size
while prog > prog1:
prog1 = prog1 + 1
iprogress(prog1)
bad = file_size - undone - ofs._pos
print("\n%s bytes removed during recovery" % bad)
if undone:
print("%s bytes of undone transaction data were skipped" % undone)
if pack is not None:
print("Packing ...")
from ZODB.serialize import referencesf
ofs.pack(pack, referencesf)
ofs.close()
f.close()
def copy(source, dest, verbose=0):
"""Copy transactions from a source to a destination storage
This is typically used for converting data from one storage to
another. `source` must have an .iterator() method.
"""
_ts = None
ok = 1
preindex = {}
preget = preindex.get
# restore() is a new storage API method which has an identical
# signature to store() except that it does not return anything.
# Semantically, restore() is also identical to store() except that it
# doesn't do the ConflictError or VersionLockError consistency
# checks. The reason to use restore() over store() in this method is
# that store() cannot be used to copy transactions spanning a version
# commit or abort, or over transactional undos.
#
# We'll use restore() if it's available, otherwise we'll fall back to
# using store(). However, if we use store, then
# copyTransactionsFrom() may fail with VersionLockError or
# ConflictError.
restoring = hasattr(dest, 'restore')
fiter = source.iterator()
for transaction in fiter:
tid = transaction.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok: print('Time stamps out of order %s, %s' % (_ts, t))
ok = 0
_ts = t.laterThan(_ts)
tid = ` _ts `
else:
_ts = t
if not ok:
print('Time stamps back in order %s' % (t))
ok = 1
if verbose:
print _ts
dest.tpc_begin(transaction, tid, transaction.status)
for r in transaction:
oid = r.oid
if verbose:
print oid_repr(oid), r.version, len(r.data)
if restoring:
dest.restore(oid, r.tid, r.data, r.version, r.data_txn,
transaction)
else:
pre = preget(oid, None)
s = dest.store(oid, pre, r.data, r.version, transaction)
preindex[oid] = s
dest.tpc_vote(transaction)
dest.tpc_finish(transaction)
fiter.close()
def newTid(old):
t = time.time()
ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
if old is not None:
ts = ts.laterThan(TimeStamp(old))
return ts.raw()
def pack(self, t, referencesf):
# Packing is hard, at least when undo is supported.
# Even for a simple storage like this one, packing
# is pretty complex.
self._lock_acquire()
try:
stop=`TimeStamp(*time.gmtime(t)[:5]+(t%60,))`
# Build indexes up to the pack time:
index, vindex = self._build_indexes(stop)
# TODO: This packing algorithm is flawed. It ignores
# references from non-current records after the pack
# time.
# Now build an index of *only* those objects reachable
# from the root.
rootl = [z64]
pindex = {}
while rootl:
oid = rootl.pop()
if oid in pindex:
continue
# Scan non-version pickle for references
r = index.get(oid, None)
if r is None:
if self._base:
p, s = self._base.load(oid, '')
referencesf(p, rootl)
else:
pindex[oid] = r
oid, pre, vdata, p, tid = r
referencesf(p, rootl)
if vdata:
nv = vdata[1]
if nv:
oid, pre, vdata, p, tid = nv
referencesf(p, rootl)
# Now we're ready to do the actual packing.
# We'll simply edit the transaction data in place.
# We'll defer deleting transactions till the end
# to avoid messing up the BTree items.
deleted = []
for tid, (p, u, d, e, records) in self._data.items():
if tid >= stop:
break
o = []
for r in records:
c = pindex.get(r[0])
if c is None:
# GC this record, no longer referenced
continue
if c == r:
# This is the most recent revision.
o.append(r)
else:
# This record is not the indexed record,
# so it may not be current. Let's see.
vdata = r[3]
if vdata:
# Version record are current *only* if they
# are indexed
continue
else:
# OK, this isn't a version record, so it may be the
# non-version record for the indexed record.
vdata = c[3]
if vdata:
if vdata[1] != r:
# This record is not the non-version
# record for the indexed record
continue
else:
# The indexed record is not a version record,
# so this record can not be the non-version
# record for it.
continue
o.append(r)
if o:
if len(o) != len(records):
self._data[tid] = 1, u, d, e, tuple(o) # Reset data
else:
deleted.append(tid)
# Now delete empty transactions
for tid in deleted:
del self._data[tid]
# Now reset previous pointers for "current" records:
for r in pindex.values():
r[1] = None # Previous record
if r[2] and r[2][1]: # vdata
# If this record contains version data and
# non-version data, then clear it out.
r[2][1][2] = None
# Finally, rebuild indexes from transaction data:
self._index, self._vindex = self._build_indexes()
finally:
self._lock_release()
self.getSize()
def _tidToTimestamp(self, tid):
if isinstance(tid, bytes) and len(tid) == 8:
return str(TimeStamp(tid))
return tid_repr(tid)
def main():
usage = "usage: %prog [options] filename"
parser = OptionParser(usage=usage)
parser.add_option("-n",
"--number",
dest="num",
help="display only the n biggest objects",
default=-1,
type="int")
parser.add_option("-f",
"--output",
dest="filename",
action="store",
type="string",
help="the FileStorage")
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_false",
help="show percentage and time remaining")
(options, args) = parser.parse_args()
VERBOSE = False
if options.filename:
fname = options.filename
else:
print "You have to enter the FileStorage filename, see --help for details"
return 2
if options.verbose != None:
VERBOSE = True
objectsToDisplay = options.num
stats = {}
start = time.time()
size = os.stat(fname).st_size
it = ZODB.FileStorage.FileIterator(fname)
lastPercent = 0.0
recordsCounter = 0
interval = 0.005
now = datetime.date.today()
try:
for t in it:
percent = float(it._file.tell()) / float(size) * 100
#Show the percentage of the work completed and the remaining time
if (percent - lastPercent > interval):
spentTime = time.time() - start
remainingTime = spentTime / float(
it._file.tell()) * (float(size)) - spentTime
if VERBOSE:
sys.stderr.write(
"\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d"
% (percent, GetInHMS(time.time() - start),
GetInHMS(remainingTime), recordsCounter))
sys.stdout.flush()
lastPercent = percent
for r in t:
#need to reduce the time of the dictionary stats from time to time
ts = TimeStamp(t.tid)
then = datetime.date(int(ts.year()), int(ts.month()),
int(ts.day()))
delta = timedelta(days=3)
#don't reduce the size of the dictionary when analysing last 3 days transactions
if recordsCounter % (objectsToDisplay * 100) == 0 and (
now - then > delta):
tmp = {}
for class_name, s in sorted(
stats.items(), key=lambda (k, v): v.size,
reverse=True)[0:objectsToDisplay]:
tmp[class_name] = s
stats = tmp
if r.data:
mod, klass = get_pickle_metadata(r.data)
l = len(r.data)
class_name = mod + "." + klass + " oid: " + oid_repr(
r.oid).strip()
stat = stats.get(class_name)
if stat is None:
stat = stats[class_name] = Stat()
stat.size = stat.min = stat.max = l
stat.oid = oid_repr(r.oid).strip()
stat.className = mod + "." + klass
stat.number = 1
else:
stat.min = min(stat.min, l)
stat.max = max(stat.max, l)
stat.number = stat.number + 1
stat.size = stat.size + l
recordsCounter += 1
except KeyboardInterrupt:
pass
print "\n"
print "%-41s %9s %15s %15s %9s %9s %9s" % ("Module.ClassName", "Oid",
"Percentage", "Total Size",
"Min", "Max", "Copies")
print "%s" % "_" * 114
for class_name, s in sorted(stats.items(),
key=lambda (k, v): v.size,
reverse=True)[0:objectsToDisplay]:
class_name = s.className
if len(class_name) > 40:
class_name = class_name[::-1][0:35][::-1]
class_name = "[..]" + class_name
print "%-40s | %8s | %13f%% | %13s | %7s | %7s | %7s" % (
class_name, s.oid, (s.size * 100.0 / size), pretty_size(
s.size), pretty_size(s.min), pretty_size(s.max), s.number)
def main():
usage = "usage: %prog [options] filename"
parser = OptionParser(usage=usage)
parser.add_option("-n", "--number", dest="num",
help="display only the 'n' busiest days", default=20, type="int")
parser.add_option("-f", "--file", dest="filename", action="store", type="string",
help="your FileStorage")
parser.add_option("-d", "--date", dest="date", action="store", type="string",
help="show the stats only for the date d (format dd-mm-yyyy)")
parser.add_option("-a", "--days", dest="days", action="store", default="0", type="string",
help="show the stats only for the last 'a' days")
parser.add_option("-v", "--verbose", dest="verbose", action="store_false",
help="show percentage and time remaining")
(options, args) = parser.parse_args()
objectsToDisplay = options.num
VERBOSE = False
if options.filename:
fname = options.filename
else:
print "You have to enter the filename, see --help for details"
return 2
if options.verbose != None:
VERBOSE = True
stats = {}
start = time.time()
size = os.stat(fname).st_size
it = ZODB.FileStorage.FileIterator(fname)
lastPercent = 0.0
recordsCounter = 0
interval = 0.005
dataFound = False
now = datetime.date.today()
try:
for t in it:
#Format the date of the current transaction following dd-mm-yyyy
ts = TimeStamp(t.tid)
then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day()))
delta = timedelta(days=int(options.days))
if((not int(options.days)) or (now - then < delta)):
dateT = strftime("%d-%m-%Y", [int(ts.year()), int(ts.month()), int(ts.day()),1,1,1,1,1,1] )
percent = float(it._file.tell())/float(size) * 100
#Check if we found the searched date
if options.date:
if str(dateT) == str(options.date):
dataFound = True
elif dataFound:
break
#Show the percentage of the work completed and the remaining time
if(percent - lastPercent > interval):
spentTime = time.time() - start
remainingTime = spentTime / float(it._file.tell()) * (float(size)) - spentTime
if VERBOSE:
sys.stderr.write("\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d" % (percent,GetInHMS(time.time() - start, True), GetInHMS(remainingTime, False), recordsCounter))
lastPercent = percent
stat = stats.get(dateT)
if stat is None:
stat = stats[dateT] = Stat()
stat.n = 1
else:
stat.n += 1
for r in t:
#need to reduce the time of the dictionary stats from time to time
if recordsCounter % (objectsToDisplay*100) == 0:
tmp = {}
for date, s in sorted(
stats.items(), key=lambda (k,v): v.n, reverse=True)[0: objectsToDisplay]:
tmp[date] = s
try:
tmp[dateT] = stats[dateT]
except KeyError:
pass
stats = tmp
if r.data:
mod, klass = get_pickle_metadata(r.data)
l = len(r.data)
stat = stats.get(dateT)
stat.records += 1
recordsCounter += 1
stat = stats.get(dateT)
if stat is not None:
stat.mean.append(TimeStamp(t.tid).timeTime())
except KeyboardInterrupt:
pass
print "\n"
print "%-15s %17s %17s %22s" % ("Date", "Transactions","Records Changed", "Average interval")
print "%s" % "_" * 74
if options.date:
for date, s in sorted(
stats.items(), key=lambda (k,v): v.n, reverse=True):
meanTime = 0
for i in range(1,len(s.mean)):
meanTime += s.mean[i] - s.mean[i-1]
if str(date) == str(options.date):
print "%-15s | %15d | %15d | %15f secs" % (date, (s.n),s.records, meanTime/s.n)
else:
for date, s in sorted(
stats.items(), key=lambda (k,v): v.n, reverse=True)[0: objectsToDisplay]:
meanTime = 0
for i in range(1,len(s.mean)):
meanTime += s.mean[i] - s.mean[i-1]
print "%-15s | %15d | %15d | %15f secs" % (date, (s.n), s.records, meanTime/s.n)
def listHistory(self):
self.debug_mark('- listing history')
if 'tid' in self.request:
requested_tid = p64(int(self.request['tid'], 0))
else:
requested_tid = None
results = []
for n, d in enumerate(self.history[self.first_idx:self.last_idx]):
utid = u64(d.tid)
ts = TimeStamp(d.tid).timeTime()
utc_timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(ts))
local_timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts))
try:
user_location, user_id = d.user.split()
except ValueError:
user_location = None
user_id = d.user
if isinstance(user_location, bytes):
user_location = user_location.decode('UTF-8', 'replace')
if isinstance(user_id, bytes):
user_id = user_id.decode('UTF-8', 'replace')
description = d.description
if isinstance(description, bytes):
description = description.decode('UTF-8', 'replace')
try:
size = d._tend - d._tpos
except AttributeError:
size = None
self.debug_mark('- listing 0x%x (%s)' % (utid, formatSize(size)))
ext = d.extension if isinstance(d.extension, dict) else {}
objects = []
for idx, record in enumerate(d):
url = "@@zodbbrowser?oid=0x%x&tid=0x%x" % (u64(record.oid),
utid)
if 'fast' in self.request or (
self.time_elapsed() > 10
and idx > 10
and 'full' not in self.request):
objects.append(dict(
oid=u64(record.oid),
path='0x%x' % u64(record.oid),
oid_repr=oid_repr(record.oid),
class_repr='',
url=url,
repr='(view object)',
))
else:
obj = self.jar.get(record.oid)
objects.append(dict(
oid=u64(record.oid),
path=getObjectPath(obj, d.tid),
oid_repr=oid_repr(record.oid),
class_repr=getObjectType(obj),
url=url,
repr=IValueRenderer(obj).render(d.tid),
))
if len(objects) == 1:
summary = '1 object record'
else:
summary = '%d object records' % len(objects)
if size is not None:
summary += ' (%s)' % formatSize(size)
results.append(dict(
index=(self.first_idx + n + 1),
utc_timestamp=utc_timestamp,
local_timestamp=local_timestamp,
user_id=user_id,
user_location=user_location,
description=description,
utid=utid,
current=(d.tid == requested_tid),
href=self.getUrl(tid=utid),
size=size,
summary=summary,
hidden=(len(objects) > 5),
objects=objects,
**ext
))
if results and not requested_tid and self.page == 0:
results[-1]['current'] = True
self.debug_mark('- back to rendering')
return results[::-1]
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
print "Recovering", inp, "into", outp
if os.path.exists(outp) and not force:
die("%s exists" % outp)
f = open(inp, "rb")
if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
f.seek(0,2)
file_size = f.tell()
ofs = ZODB.FileStorage.FileStorage(outp, create=1)
_ts = None
ok = 1
prog1 = 0
undone = 0
pos = 4L
ltid = None
while pos:
try:
npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
except EOFError:
break
except (KeyboardInterrupt, SystemExit):
raise
except Exception, err:
print "error reading txn header:", err
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print "looking for valid txn header at", pos
continue
ltid = tid
if txn is None:
undone = undone + npos - pos
pos = npos
continue
else:
pos = npos
tid = txn.tid
if _ts is None:
_ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= _ts:
if ok:
print ("Time stamps out of order %s, %s" % (_ts, t))
ok = 0
_ts = t.laterThan(_ts)
tid = _ts.raw()
else:
_ts = t
if not ok:
print ("Time stamps back in order %s" % (t))
ok = 1
ofs.tpc_begin(txn, tid, txn.status)
if verbose:
print "begin", pos, _ts,
if verbose > 1:
print
sys.stdout.flush()
nrec = 0
try:
for r in txn:
if verbose > 1:
if r.data is None:
l = "bp"
else:
l = len(r.data)
print "%7d %s %s" % (u64(r.oid), l)
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,
txn)
nrec += 1
except (KeyboardInterrupt, SystemExit):
raise
except Exception, err:
if partial and nrec:
ofs._status = "p"
ofs.tpc_vote(txn)
ofs.tpc_finish(txn)
if verbose:
print "partial"
else:
ofs.tpc_abort(txn)
print "error copying transaction:", err
if not verbose:
progress(prog1)
pos = scan(f, pos)
if verbose > 1:
print "looking for valid txn header at", pos
def checkLaterThan(self):
t = time.gmtime()
ts = TimeStamp(*t[:6])
ts2 = ts.laterThan(ts)
self.assert_(ts2 > ts)
def checkTimeStamp(self):
# Alternate test suite
t = TimeStamp(2002, 1, 23, 10, 48, 5) # GMT
self.assertEquals(str(t), '2002-01-23 10:48:05.000000')
self.assertEquals(repr(t), '\x03B9H\x15UUU')
self.assertEquals(TimeStamp('\x03B9H\x15UUU'), t)
self.assertEquals(t.year(), 2002)
self.assertEquals(t.month(), 1)
self.assertEquals(t.day(), 23)
self.assertEquals(t.hour(), 10)
self.assertEquals(t.minute(), 48)
self.assertEquals(round(t.second()), 5)
self.assertEquals(t.timeTime(), 1011782885)
t1 = TimeStamp(2002, 1, 23, 10, 48, 10)
self.assertEquals(str(t1), '2002-01-23 10:48:10.000000')
self.assert_(t == t)
self.assert_(t != t1)
self.assert_(t < t1)
self.assert_(t <= t1)
self.assert_(t1 >= t)
self.assert_(t1 > t)
self.failIf(t == t1)
self.failIf(t != t)
self.failIf(t > t1)
self.failIf(t >= t1)
self.failIf(t1 < t)
self.failIf(t1 <= t)
self.assertEquals(cmp(t, t), 0)
self.assertEquals(cmp(t, t1), -1)
self.assertEquals(cmp(t1, t), 1)
self.assertEquals(t1.laterThan(t), t1)
self.assert_(t.laterThan(t1) > t1)
self.assertEquals(TimeStamp(2002,1,23), TimeStamp(2002,1,23,0,0,0))
def run(path, days, notPacked):
f = open(path, "rb")
f.seek(0, 2)
size = os.path.getsize(path)
now = datetime.date.today()
notPackedDays = []
for day in range(notPacked):
notPackedDays.append(str(now - timedelta(days=day + 1)))
#day->size
stats = {}
th = prev_txn(f)
bool = True
while bool:
ts = TimeStamp(th.tid)
then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day()))
delta = timedelta(days=int(days))
if (now - then < delta):
dateT = strftime("%Y-%m-%d", [
int(ts.year()),
int(ts.month()),
int(ts.day()), 1, 1, 1, 1, 1, 1
])
try:
stats[dateT] = stats[dateT] + th.length
except KeyError:
stats[dateT] = th.length
else:
bool = False
th = th.prev_txn()
f.close()
total = 0
totalPacked = 0
daysPacked = 0
for (d, s) in sorted(stats.items(), key=lambda (k, v): v, reverse=True):
print d, "size:", pretty_size(s),
date = str(d)
if (date in notPackedDays or date == str(now)):
print "(not yet packed)"
else:
totalPacked = totalPacked + s
daysPacked = daysPacked + 1
print
total = total + s
if int(totalPacked):
average = totalPacked / int(daysPacked)
else:
average = 0
print "\n-- ALREADY PACKED DAYS--"
print "The amount of data added in", daysPacked, "days is", pretty_size(
totalPacked)
print "Average", pretty_size(average), "per day"
print "Following this trend, the size of the database will be:"
print "\t", pretty_size(average * 365 + size), " in 1 year"
print "\t", pretty_size(average * 365 * 2 + size), " in 2 years"
print "\t", pretty_size(average * 365 * 10 + size), " in 10 years"
print "\n-- ALL DAYS --"
print "The amount of data added in", days, "days is", pretty_size(total)
if int(total):
print "Average", pretty_size(total / int(days)), "per day"
else:
print "Average 0bytes per day"
