def fsdump(path, file=None, with_offset=1):
iter = FileIterator(path)
for i, trans in enumerate(iter):
if with_offset:
print(("Trans #%05d tid=%016x time=%s offset=%d" %
(i, u64(trans.tid), TimeStamp(trans.tid), trans._pos)), file=file)
else:
print(("Trans #%05d tid=%016x time=%s" %
(i, u64(trans.tid), TimeStamp(trans.tid))), file=file)
print((" status=%r user=%r description=%r" %
(trans.status, trans.user, trans.description)), file=file)
for j, rec in enumerate(trans):
if rec.data is None:
fullclass = "undo or abort of object creation"
size = ""
else:
modname, classname = get_pickle_metadata(rec.data)
size = " size=%d" % len(rec.data)
fullclass = "%s.%s" % (modname, classname)
if rec.data_txn:
# It would be nice to print the transaction number
# (i) but it would be expensive to keep track of.
bp = " bp=%016x" % u64(rec.data_txn)
else:
bp = ""
print((" data #%05d oid=%016x%s class=%s%s" %
(j, u64(rec.oid), size, fullclass, bp)), file=file)
iter.close()
def __init__(self,
message=None,
object=None,
oid=None,
serials=None,
data=None):
if message is None:
self.message = "database conflict error"
else:
self.message = message
if object is None:
self.oid = None
self.class_name = None
else:
self.oid = object._p_oid
klass = object.__class__
self.class_name = klass.__module__ + "." + klass.__name__
if oid is not None:
assert self.oid is None
self.oid = oid
if data is not None:
# avoid circular import chain
from ZODB.utils import get_pickle_metadata
self.class_name = '.'.join(get_pickle_metadata(data))
self.serials = serials
def __init__(self, message=None, object=None, oid=None, serials=None,
data=None):
if message is None:
self.message = "database conflict error"
else:
self.message = message
if object is None:
self.oid = None
self.class_name = None
else:
self.oid = object._p_oid
klass = object.__class__
self.class_name = klass.__module__ + "." + klass.__name__
if oid is not None:
assert self.oid is None
self.oid = oid
if data is not None:
# avoid circular import chain
from ZODB.utils import get_pickle_metadata
self.class_name = "%s.%s" % get_pickle_metadata(data)
## else:
## if message != "data read conflict error":
## raise RuntimeError
self.serials = serials
def buildRefmap(fs):
'''build a refmap from a filestorage. look in every record of every
transaction. build a dict of oid -> list((referenced oids, mod.klass))
'''
refmap = {}
fsi = fs.iterator()
size = os.stat(fs.__name__).st_size
start = time.time()
lastPercent = 0.0
interval = 0.005
print "[1/3] Computing the Reference Map"
for txn in fsi:
percent = float(fsi._file.tell())/float(size) * 100
if(percent - lastPercent > interval):
spentTime = time.time() - start
remainingTime = spentTime / float(fsi._file.tell()) * (float(size)) - spentTime
sys.stdout.write("\r%f%% complete, time spent %s, remaining time: %s" % (percent,GetInHMS(time.time() - start, True), GetInHMS(remainingTime, False)))
lastPercent = percent
for rec in txn:
if rec.data:
mod, klass = get_pickle_metadata(rec.data)
refs = referencesf(rec.data)
refmap[rec.oid] = (refs, mod+"."+klass)
print "\n",
return refmap
def fsdump(path, file=None, with_offset=1):
iter = FileIterator(path)
for i, trans in enumerate(iter):
if with_offset:
print(("Trans #%05d tid=%016x time=%s offset=%d" %
(i, u64(trans.tid), TimeStamp(trans.tid), trans._pos)),
file=file)
else:
print(("Trans #%05d tid=%016x time=%s" %
(i, u64(trans.tid), TimeStamp(trans.tid))),
file=file)
print((" status=%r user=%r description=%r" %
(trans.status, trans.user, trans.description)),
file=file)
for j, rec in enumerate(trans):
if rec.data is None:
fullclass = "undo or abort of object creation"
size = ""
else:
modname, classname = get_pickle_metadata(rec.data)
size = " size=%d" % len(rec.data)
fullclass = "%s.%s" % (modname, classname)
if rec.data_txn:
# It would be nice to print the transaction number
# (i) but it would be expensive to keep track of.
bp = " bp=%016x" % u64(rec.data_txn)
else:
bp = ""
print((" data #%05d oid=%016x%s class=%s%s" %
(j, u64(rec.oid), size, fullclass, bp)),
file=file)
iter.close()
def buildRefmap(fs):
'''build a refmap from a filestorage. look in every record of every
transaction. build a dict of oid -> list((referenced oids, mod.klass))
'''
refmap = {}
fsi = fs.iterator()
size = os.stat(fs.__name__).st_size
start = time.time()
lastPercent = 0.0
interval = 0.005
print "[1/3] Computing the Reference Map"
for txn in fsi:
percent = float(fsi._file.tell()) / float(size) * 100
if (percent - lastPercent > interval):
spentTime = time.time() - start
remainingTime = spentTime / float(
fsi._file.tell()) * (float(size)) - spentTime
sys.stdout.write(
"\r%f%% complete, time spent %s, remaining time: %s" %
(percent, GetInHMS(time.time() - start,
True), GetInHMS(remainingTime, False)))
lastPercent = percent
for rec in txn:
if rec.data:
mod, klass = get_pickle_metadata(rec.data)
refs = referencesf(rec.data)
refmap[rec.oid] = (refs, mod + "." + klass)
print "\n",
return refmap
def test_get_pickle_metadata_w_protocol_3_class_pickle(self):
from ZODB.utils import get_pickle_metadata
from ZODB._compat import dumps
from ZODB._compat import HIGHEST_PROTOCOL
if HIGHEST_PROTOCOL >= 3:
pickle = dumps(ExampleClass, protocol=3)
self.assertEqual(get_pickle_metadata(pickle),
(__name__, ExampleClass.__name__))
def test_get_pickle_metadata_w_protocol_3_class_pickle(self):
from ZODB.utils import get_pickle_metadata
from ZODB._compat import dumps
from ZODB._compat import HIGHEST_PROTOCOL
if HIGHEST_PROTOCOL >= 3:
pickle = dumps(ExampleClass, protocol=3)
self.assertEqual(get_pickle_metadata(pickle),
(__name__, ExampleClass.__name__))
def main(path):
fs = FileStorage(path, read_only=1)
if PACK:
fs.pack()
db = ZODB.DB(fs)
rt = db.open().root()
paths = find_paths(rt, 3)
def total_size(oid):
cache = {}
cache_size = 1000
def _total_size(oid, seen):
v = cache.get(oid)
if v is not None:
return v
data, serialno = fs.load(oid, '')
size = len(data)
for suboid in referencesf(data):
if seen.has_key(suboid):
continue
seen[suboid] = 1
size += _total_size(suboid, seen)
cache[oid] = size
if len(cache) == cache_size:
cache.popitem()
return size
return _total_size(oid, {})
keys = fs._index.keys()
keys.sort()
keys.reverse()
if not VERBOSE:
# If not running verbosely, don't print an entry for an object
# unless it has an entry in paths.
keys = filter(paths.has_key, keys)
fmt = "%8s %5d %8d %s %s.%s"
for oid in keys:
data, serialno = fs.load(oid, '')
mod, klass = get_pickle_metadata(data)
refs = referencesf(data)
path = paths.get(oid, '-')
print fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)
def main(path):
fs = FileStorage(path, read_only=1)
if PACK:
fs.pack()
db = ZODB.DB(fs)
rt = db.open().root()
paths = find_paths(rt, 3)
def total_size(oid):
cache = {}
cache_size = 1000
def _total_size(oid, seen):
v = cache.get(oid)
if v is not None:
return v
data, serialno = fs.load(oid, "")
size = len(data)
for suboid in referencesf(data):
if seen.has_key(suboid):
continue
seen[suboid] = 1
size += _total_size(suboid, seen)
cache[oid] = size
if len(cache) == cache_size:
cache.popitem()
return size
return _total_size(oid, {})
keys = fs._index.keys()
keys.sort()
keys.reverse()
if not VERBOSE:
# If not running verbosely, don't print an entry for an object
# unless it has an entry in paths.
keys = filter(paths.has_key, keys)
fmt = "%8s %5d %8d %s %s.%s"
for oid in keys:
data, serialno = fs.load(oid, "")
mod, klass = get_pickle_metadata(data)
refs = referencesf(data)
path = paths.get(oid, "-")
print fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)
def report(oid, data, serial, missing):
from_mod, from_class = get_pickle_metadata(data)
if len(missing) > 1:
plural = "s"
else:
plural = ""
ts = TimeStamp(serial)
print("oid %s %s.%s" % (hex(u64(oid)), from_mod, from_class))
print("last updated: %s, tid=%s" % (ts, hex(u64(serial))))
print("refers to invalid object%s:" % plural)
for oid, info, reason in missing:
if isinstance(info, tuple):
description = "%s.%s" % info
else:
description = str(info)
print("\toid %s %s: %r" % (oid_repr(oid), reason, description))
print()
def report(oid, data, serial, missing):
from_mod, from_class = get_pickle_metadata(data)
if len(missing) > 1:
plural = "s"
else:
plural = ""
ts = TimeStamp(serial)
print "oid %s %s.%s" % (hex(u64(oid)), from_mod, from_class)
print "last updated: %s, tid=%s" % (ts, hex(u64(serial)))
print "refers to invalid object%s:" % plural
for oid, info, reason in missing:
if isinstance(info, types.TupleType):
description = "%s.%s" % info
else:
description = str(info)
print "\toid %s %s: %r" % (oid_repr(oid), reason, description)
print
def run(path, v=0):
fs = FileStorage(path, read_only=1)
# break into the file implementation
if hasattr(fs._index, 'iterkeys'):
iter = fs._index.iterkeys()
else:
iter = fs._index.keys()
totals = {}
for oid in iter:
data, serialno = fs.load(oid, '')
mod, klass = get_pickle_metadata(data)
key = "%s.%s" % (mod, klass)
bytes, count = totals.get(key, (0, 0))
bytes += len(data)
count += 1
totals[key] = bytes, count
if v:
print "%8s %5d %s" % (U64(oid), len(data), key)
L = totals.items()
L.sort(lambda a, b: cmp(a[1], b[1]))
L.reverse()
print "Totals per object class:"
for key, (bytes, count) in L:
print "%8d %8d %s" % (count, bytes, key)
def run(path, v=0):
fs = FileStorage(path, read_only=1)
# break into the file implementation
if hasattr(fs._index, 'iterkeys'):
iter = six.iterkeys(fs._index)
else:
iter = fs._index.keys()
totals = {}
for oid in iter:
data, serialno = fs.load(oid, '')
mod, klass = get_pickle_metadata(data)
key = "%s.%s" % (mod, klass)
bytes, count = totals.get(key, (0, 0))
bytes += len(data)
count += 1
totals[key] = bytes, count
if v:
print("%8s %5d %s" % (U64(oid), len(data), key))
L = totals.items()
L.sort(lambda a, b: cmp(a[1], b[1]))
L.reverse()
print("Totals per object class:")
for key, (bytes, count) in L:
print("%8d %8d %s" % (count, bytes, key))
def test_get_pickle_metadata_w_protocol_2_class_pickle(self):
from ZODB.utils import get_pickle_metadata
from ZODB._compat import dumps
pickle = dumps(ExampleClass, protocol=2)
self.assertEqual(get_pickle_metadata(pickle),
(__name__, ExampleClass.__name__))
def main():
usage = "usage: %prog [options] filename"
parser = OptionParser(usage=usage)
parser.add_option("-n", "--number", dest="num",
help="display only the 'n' busiest days", default=20, type="int")
parser.add_option("-f", "--file", dest="filename", action="store", type="string",
help="your FileStorage")
parser.add_option("-d", "--date", dest="date", action="store", type="string",
help="show the stats only for the date d (format dd-mm-yyyy)")
parser.add_option("-a", "--days", dest="days", action="store", default="0", type="string",
help="show the stats only for the last 'a' days")
parser.add_option("-v", "--verbose", dest="verbose", action="store_false",
help="show percentage and time remaining")
(options, args) = parser.parse_args()
objectsToDisplay = options.num
VERBOSE = False
if options.filename:
fname = options.filename
else:
print "You have to enter the filename, see --help for details"
return 2
if options.verbose != None:
VERBOSE = True
stats = {}
start = time.time()
size = os.stat(fname).st_size
it = ZODB.FileStorage.FileIterator(fname)
lastPercent = 0.0
recordsCounter = 0
interval = 0.005
dataFound = False
now = datetime.date.today()
try:
for t in it:
#Format the date of the current transaction following dd-mm-yyyy
ts = TimeStamp(t.tid)
then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day()))
delta = timedelta(days=int(options.days))
if((not int(options.days)) or (now - then < delta)):
dateT = strftime("%d-%m-%Y", [int(ts.year()), int(ts.month()), int(ts.day()),1,1,1,1,1,1] )
percent = float(it._file.tell())/float(size) * 100
#Check if we found the searched date
if options.date:
if str(dateT) == str(options.date):
dataFound = True
elif dataFound:
break
#Show the percentage of the work completed and the remaining time
if(percent - lastPercent > interval):
spentTime = time.time() - start
remainingTime = spentTime / float(it._file.tell()) * (float(size)) - spentTime
if VERBOSE:
sys.stderr.write("\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d" % (percent,GetInHMS(time.time() - start, True), GetInHMS(remainingTime, False), recordsCounter))
lastPercent = percent
stat = stats.get(dateT)
if stat is None:
stat = stats[dateT] = Stat()
stat.n = 1
else:
stat.n += 1
for r in t:
#need to reduce the time of the dictionary stats from time to time
if recordsCounter % (objectsToDisplay*100) == 0:
tmp = {}
for date, s in sorted(
stats.items(), key=lambda (k,v): v.n, reverse=True)[0: objectsToDisplay]:
tmp[date] = s
try:
tmp[dateT] = stats[dateT]
except KeyError:
pass
stats = tmp
if r.data:
mod, klass = get_pickle_metadata(r.data)
l = len(r.data)
stat = stats.get(dateT)
stat.records += 1
recordsCounter += 1
stat = stats.get(dateT)
if stat is not None:
stat.mean.append(TimeStamp(t.tid).timeTime())
except KeyboardInterrupt:
pass
print "\n"
print "%-15s %17s %17s %22s" % ("Date", "Transactions","Records Changed", "Average interval")
print "%s" % "_" * 74
if options.date:
for date, s in sorted(
stats.items(), key=lambda (k,v): v.n, reverse=True):
meanTime = 0
for i in range(1,len(s.mean)):
meanTime += s.mean[i] - s.mean[i-1]
if str(date) == str(options.date):
print "%-15s | %15d | %15d | %15f secs" % (date, (s.n),s.records, meanTime/s.n)
else:
for date, s in sorted(
stats.items(), key=lambda (k,v): v.n, reverse=True)[0: objectsToDisplay]:
meanTime = 0
for i in range(1,len(s.mean)):
meanTime += s.mean[i] - s.mean[i-1]
print "%-15s | %15d | %15d | %15f secs" % (date, (s.n), s.records, meanTime/s.n)
def main():
usage = "usage: %prog [options] filename"
parser = OptionParser(usage=usage)
parser.add_option("-n",
"--number",
dest="num",
help="display only the n biggest objects",
default=-1,
type="int")
parser.add_option("-f",
"--output",
dest="filename",
action="store",
type="string",
help="the FileStorage")
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_false",
help="show percentage and time remaining")
(options, args) = parser.parse_args()
VERBOSE = False
if options.filename:
fname = options.filename
else:
print "You have to enter the FileStorage filename, see --help for details"
return 2
if options.verbose != None:
VERBOSE = True
objectsToDisplay = options.num
stats = {}
start = time.time()
size = os.stat(fname).st_size
it = ZODB.FileStorage.FileIterator(fname)
lastPercent = 0.0
recordsCounter = 0
interval = 0.005
now = datetime.date.today()
try:
for t in it:
percent = float(it._file.tell()) / float(size) * 100
#Show the percentage of the work completed and the remaining time
if (percent - lastPercent > interval):
spentTime = time.time() - start
remainingTime = spentTime / float(
it._file.tell()) * (float(size)) - spentTime
if VERBOSE:
sys.stderr.write(
"\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d"
% (percent, GetInHMS(time.time() - start),
GetInHMS(remainingTime), recordsCounter))
sys.stdout.flush()
lastPercent = percent
for r in t:
#need to reduce the time of the dictionary stats from time to time
ts = TimeStamp(t.tid)
then = datetime.date(int(ts.year()), int(ts.month()),
int(ts.day()))
delta = timedelta(days=3)
#don't reduce the size of the dictionary when analysing last 3 days transactions
if recordsCounter % (objectsToDisplay * 100) == 0 and (
now - then > delta):
tmp = {}
for class_name, s in sorted(
stats.items(), key=lambda (k, v): v.size,
reverse=True)[0:objectsToDisplay]:
tmp[class_name] = s
stats = tmp
if r.data:
mod, klass = get_pickle_metadata(r.data)
l = len(r.data)
class_name = mod + "." + klass + " oid: " + oid_repr(
r.oid).strip()
stat = stats.get(class_name)
if stat is None:
stat = stats[class_name] = Stat()
stat.size = stat.min = stat.max = l
stat.oid = oid_repr(r.oid).strip()
stat.className = mod + "." + klass
stat.number = 1
else:
stat.min = min(stat.min, l)
stat.max = max(stat.max, l)
stat.number = stat.number + 1
stat.size = stat.size + l
recordsCounter += 1
except KeyboardInterrupt:
pass
print "\n"
print "%-41s %9s %15s %15s %9s %9s %9s" % ("Module.ClassName", "Oid",
"Percentage", "Total Size",
"Min", "Max", "Copies")
print "%s" % "_" * 114
for class_name, s in sorted(stats.items(),
key=lambda (k, v): v.size,
reverse=True)[0:objectsToDisplay]:
class_name = s.className
if len(class_name) > 40:
class_name = class_name[::-1][0:35][::-1]
class_name = "[..]" + class_name
print "%-40s | %8s | %13f%% | %13s | %7s | %7s | %7s" % (
class_name, s.oid, (s.size * 100.0 / size), pretty_size(
s.size), pretty_size(s.min), pretty_size(s.max), s.number)
def main():
usage = "usage: %prog [options] filename"
parser = OptionParser(usage=usage)
parser.add_option("-n", "--number", dest="num",
help="display only the n biggest objects", default=-1, type="int")
parser.add_option("-f", "--output", dest="filename", action="store", type="string",
help="the FileStorage")
parser.add_option("-v", "--verbose", dest="verbose", action="store_false",
help="show percentage and time remaining")
(options, args) = parser.parse_args()
VERBOSE = False
if options.filename:
fname = options.filename
else:
print "You have to enter the FileStorage filename, see --help for details"
return 2
if options.verbose != None:
VERBOSE = True
objectsToDisplay = options.num
stats = {}
start = time.time()
size = os.stat(fname).st_size
it = ZODB.FileStorage.FileIterator(fname)
lastPercent = 0.0
recordsCounter = 0
interval = 0.005
now = datetime.date.today()
try:
for t in it:
percent = float(it._file.tell())/float(size) * 100
#Show the percentage of the work completed and the remaining time
if(percent - lastPercent > interval):
spentTime = time.time() - start
remainingTime = spentTime / float(it._file.tell()) * (float(size)) - spentTime
if VERBOSE:
sys.stderr.write("\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d" % (percent,GetInHMS(time.time() - start), GetInHMS(remainingTime), recordsCounter))
sys.stdout.flush()
lastPercent = percent
for r in t:
#need to reduce the time of the dictionary stats from time to time
ts = TimeStamp(t.tid)
then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day()))
delta = timedelta(days=3)
#don't reduce the size of the dictionary when analysing last 3 days transactions
if recordsCounter % (objectsToDisplay*100) == 0 and (now - then > delta):
tmp = {}
for class_name, s in sorted(
stats.items(), key=lambda (k,v): v.size, reverse=True)[0: objectsToDisplay]:
tmp[class_name] = s
stats = tmp
if r.data:
mod, klass = get_pickle_metadata(r.data)
l = len(r.data)
class_name = mod + "." + klass + " oid: " + oid_repr(r.oid).strip()
stat = stats.get(class_name)
if stat is None:
stat = stats[class_name] = Stat()
stat.size = stat.min = stat.max = l
stat.oid = oid_repr(r.oid).strip()
stat.className = mod + "." + klass
stat.number = 1
else:
stat.min = min(stat.min, l)
stat.max = max(stat.max, l)
stat.number = stat.number + 1
stat.size = stat.size + l
recordsCounter += 1
except KeyboardInterrupt:
pass
print "\n"
print "%-41s %9s %15s %15s %9s %9s %9s" % ("Module.ClassName", "Oid", "Percentage", "Total Size", "Min", "Max", "Copies")
print "%s" % "_" * 114
for class_name, s in sorted(
stats.items(), key=lambda (k,v): v.size, reverse=True)[0: objectsToDisplay]:
class_name = s.className
if len(class_name) > 40:
class_name = class_name[::-1][0:35][::-1]
class_name = "[..]" + class_name
print "%-40s | %8s | %13f%% | %13s | %7s | %7s | %7s" % (class_name, s.oid, (s.size*100.0/size) , pretty_size(s.size), pretty_size(s.min), pretty_size(s.max), s.number)
def main():
usage = "usage: %prog [options] filename"
parser = OptionParser(usage=usage)
parser.add_option("-n",
"--number",
dest="num",
help="display only the n biggest classes",
default=-1,
type="int")
parser.add_option("-f",
"--file",
dest="filename",
action="store",
type="string",
help="your FileStorage")
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_false",
help="show percentage and time remaining")
(options, args) = parser.parse_args()
VERBOSE = False
if options.filename:
fname = options.filename
else:
print "You have to enter the filename, see --help for details"
return 2
if options.verbose != None:
VERBOSE = True
objectsToDisplay = options.num
stats = {}
start = time.time()
size = os.stat(fname).st_size
it = ZODB.FileStorage.FileIterator(fname)
lastPercent = 0.0
recordsCounter = 0
interval = 0.005
try:
for t in it:
percent = float(it._file.tell()) / float(size) * 100
#Show the percentage of the work completed and the remaining time
if (percent - lastPercent > interval):
spentTime = time.time() - start
remainingTime = spentTime / float(
it._file.tell()) * (float(size)) - spentTime
if VERBOSE:
sys.stderr.write(
"\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d"
% (percent, GetInHMS(time.time() - start),
GetInHMS(remainingTime), recordsCounter))
lastPercent = percent
for r in t:
if r.data:
mod, klass = get_pickle_metadata(r.data)
l = len(r.data)
class_name = mod + "." + klass
stat = stats.get(class_name)
if stat is None:
stat = stats[class_name] = Stat()
stat.min = stat.max = stat.total = l
else:
stat.min = min(stat.min, l)
stat.max = max(stat.max, l)
stat.total += l
stat.n += 1
recordsCounter += 1
except KeyboardInterrupt:
pass
print "\n"
print "%-51s %15s %15s %15s %15s %15s" % (
"Module.ClassName", "Percentage", "Min", "Max", "Size", "Instances")
print "%s" % "_" * 134
for class_name, s in sorted(stats.items(),
key=lambda (k, v): v.total,
reverse=True)[0:objectsToDisplay]:
#Truncate the string if necessary
if len(class_name) > 50:
class_name = class_name[::-1][0:45][::-1]
class_name = "[..]" + class_name
print "%-50s | %15s%% | %13s | %13s | %13s | %13s" % (class_name, (
s.total * 100.0 / size), pretty_size(s.min), pretty_size(
s.max), pretty_size(s.total), s.n)
def get_type(record):
mod, klass = get_pickle_metadata(record.data)
return "%s.%s" % (mod, klass)
def get_class(pickle):
return "%s.%s" % get_pickle_metadata(pickle)
def get_class(pickle):
return "%s.%s" % get_pickle_metadata(pickle)
def main():
usage = "usage: %prog [options] filename"
parser = OptionParser(usage=usage)
parser.add_option("-n", "--number", dest="num",
help="display only the 'n' busiest days", default=20, type="int")
parser.add_option("-f", "--file", dest="filename", action="store", type="string",
help="your FileStorage")
parser.add_option("-d", "--date", dest="date", action="store", type="string",
help="show the stats only for the date d (format dd-mm-yyyy)")
parser.add_option("-a", "--days", dest="days", action="store", default="0", type="string",
help="show the stats only for the last 'a' days")
parser.add_option("-v", "--verbose", dest="verbose", action="store_false",
help="show percentage and time remaining")
(options, args) = parser.parse_args()
objectsToDisplay = options.num
VERBOSE = False
if options.filename:
fname = options.filename
else:
print "You have to enter the filename, see --help for details"
return 2
if options.verbose != None:
VERBOSE = True
stats = {}
start = time.time()
size = os.stat(fname).st_size
it = ZODB.FileStorage.FileIterator(fname)
lastPercent = 0.0
recordsCounter = 0
interval = 0.005
dataFound = False
now = datetime.date.today()
try:
for t in it:
#Format the date of the current transaction following dd-mm-yyyy
ts = TimeStamp(t.tid)
then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day()))
delta = timedelta(days=int(options.days))
if((not int(options.days)) or (now - then < delta)):
dateT = strftime("%d-%m-%Y", [int(ts.year()), int(ts.month()), int(ts.day()),1,1,1,1,1,1] )
percent = float(it._file.tell())/float(size) * 100
#Check if we found the searched date
if options.date:
if str(dateT) == str(options.date):
dataFound = True
elif dataFound:
break
#Show the percentage of the work completed and the remaining time
if(percent - lastPercent > interval):
spentTime = time.time() - start
remainingTime = spentTime / float(it._file.tell()) * (float(size)) - spentTime
if VERBOSE:
sys.stderr.write("\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d" % (percent,GetInHMS(time.time() - start, True), GetInHMS(remainingTime, False), recordsCounter))
lastPercent = percent
stat = stats.get(dateT)
if stat is None:
stat = stats[dateT] = Stat()
stat.n = 1
else:
stat.n += 1
for r in t:
#need to reduce the time of the dictionary stats from time to time
if recordsCounter % (objectsToDisplay*100) == 0:
tmp = {}
for date, s in sorted(
stats.items(), key=lambda (k,v): v.n, reverse=True)[0: objectsToDisplay]:
tmp[date] = s
try:
tmp[dateT] = stats[dateT]
except KeyError:
pass
stats = tmp
if r.data:
mod, klass = get_pickle_metadata(r.data)
l = len(r.data)
stat = stats.get(dateT)
stat.records += 1
recordsCounter += 1
stat = stats.get(dateT)
if stat is not None:
stat.mean.append(TimeStamp(t.tid).timeTime())
except KeyboardInterrupt:
pass
print "\n"
print "%-15s %17s %17s %22s" % ("Date", "Transactions","Records Changed", "Average interval")
print "%s" % "_" * 74
if options.date:
for date, s in sorted(
stats.items(), key=lambda (k,v): v.n, reverse=True):
meanTime = 0
for i in range(1,len(s.mean)):
meanTime += s.mean[i] - s.mean[i-1]
if str(date) == str(options.date):
print "%-15s | %15d | %15d | %15f secs" % (date, (s.n),s.records, meanTime/s.n)
else:
for date, s in sorted(
stats.items(), key=lambda (k,v): v.n, reverse=True)[0: objectsToDisplay]:
meanTime = 0
for i in range(1,len(s.mean)):
meanTime += s.mean[i] - s.mean[i-1]
print "%-15s | %15d | %15d | %15f secs" % (date, (s.n), s.records, meanTime/s.n)
def main():
usage = "usage: %prog [options] filename"
parser = OptionParser(usage=usage)
parser.add_option("-n", "--number", dest="num",
help="display only the n biggest classes", default=-1, type="int")
parser.add_option("-f", "--file", dest="filename", action="store", type="string",
help="your FileStorage")
parser.add_option("-v", "--verbose", dest="verbose", action="store_false",
help="show percentage and time remaining")
(options, args) = parser.parse_args()
VERBOSE = False
if options.filename:
fname = options.filename
else:
print "You have to enter the filename, see --help for details"
return 2
if options.verbose != None:
VERBOSE = True
objectsToDisplay = options.num
stats = {}
start = time.time()
size = os.stat(fname).st_size
it = ZODB.FileStorage.FileIterator(fname)
lastPercent = 0.0
recordsCounter = 0
interval = 0.005
try:
for t in it:
percent = float(it._file.tell())/float(size) * 100
#Show the percentage of the work completed and the remaining time
if(percent - lastPercent > interval):
spentTime = time.time() - start
remainingTime = spentTime / float(it._file.tell()) * (float(size)) - spentTime
if VERBOSE:
sys.stderr.write("\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d" % (percent,GetInHMS(time.time() - start), GetInHMS(remainingTime), recordsCounter))
lastPercent = percent
for r in t:
if r.data:
mod, klass = get_pickle_metadata(r.data)
l = len(r.data)
class_name = mod + "." + klass
stat = stats.get(class_name)
if stat is None:
stat = stats[class_name] = Stat()
stat.min = stat.max = stat.total = l
else:
stat.min = min(stat.min, l)
stat.max = max(stat.max, l)
stat.total += l
stat.n += 1
recordsCounter += 1
except KeyboardInterrupt:
pass
print "\n"
print "%-51s %15s %15s %15s %15s %15s" % ("Module.ClassName", "Percentage", "Min", "Max", "Size", "Instances")
print "%s" % "_" * 134
for class_name, s in sorted(
stats.items(), key=lambda (k,v): v.total, reverse=True)[0: objectsToDisplay]:
#Truncate the string if necessary
if len(class_name) > 50:
class_name = class_name[::-1][0:45][::-1]
class_name = "[..]" + class_name
print "%-50s | %15s%% | %13s | %13s | %13s | %13s" % (class_name, (s.total*100.0/size), pretty_size(s.min), pretty_size(s.max), pretty_size(s.total), s.n)
def test_get_pickle_metadata_w_protocol_0_class_pickle(self):
from ZODB.utils import get_pickle_metadata
from ZODB._compat import dumps
pickle = dumps(ExampleClass, protocol=0)
self.assertEqual(get_pickle_metadata(pickle),
(__name__, ExampleClass.__name__))
def get_type(record):
mod, klass = get_pickle_metadata(record.data)
return "%s.%s" % (mod, klass)
