def checkRawTimestamp(self): t = time.gmtime() ts1 = TimeStamp(*t[:6]) ts2 = TimeStamp(`ts1`) self.assertEquals(ts1, ts2) self.assertEquals(ts1.timeTime(), ts2.timeTime()) self.assertEqual(ts1.year(), ts2.year()) self.assertEqual(ts1.month(), ts2.month()) self.assertEqual(ts1.day(), ts2.day()) self.assertEquals(ts1.hour(), ts2.hour()) self.assertEquals(ts1.minute(), ts2.minute()) self.assert_(abs(ts1.second() - ts2.second()) < EPSILON)
def run(path, days, notPacked): f = open(path, "rb") f.seek(0, 2) now = datetime.date.today() #day->size stats = {} th = prev_txn(f) bool = True while bool: ts = TimeStamp(th.tid) then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day())) delta = timedelta(days=int(days)) if( not(now - then < delta)): bool = False th = th.prev_txn() reader = Reader() iterator = FileIterator(path, pos=th._pos) for i in iterator: object_types = {} for o in i: ot = reader.getIdentity(o.data) try: stats[ot] = stats[ot] + 1 except KeyError: stats[ot] = 1 f.close() for (o,n) in sorted(stats.items(), key=lambda (k,v): v, reverse=True): print "%6d: %s" % (n,o)
def checkFullTimeStamp(self): native_ts = int(time.time()) # fractional seconds get in the way t = time.gmtime(native_ts) # the corresponding GMT struct tm ts = TimeStamp(*t[:6]) # Seconds are stored internally via (conceptually) multiplying by # 2**32 then dividing by 60, ending up with a 32-bit integer. # While this gives a lot of room for cramming many distinct # TimeStamps into a second, it's not good at roundtrip accuracy. # For example, 1 second is stored as int(2**32/60) == 71582788. # Converting back gives 71582788*60.0/2**32 == 0.9999999962747097. # In general, we can lose up to 0.999... to truncation during # storing, creating an absolute error up to about 1*60.0/2**32 == # 0.000000014 on the seconds value we get back. This is so even # when we have an exact integral second value going in (as we # do in this test), so we can't expect equality in any comparison # involving seconds. Minutes (etc) are stored exactly, so we # can expect equality for those. self.assert_(abs(ts.timeTime() - native_ts) < EPSILON) self.assertEqual(ts.year(), t[0]) self.assertEqual(ts.month(), t[1]) self.assertEqual(ts.day(), t[2]) self.assertEquals(ts.hour(), t[3]) self.assertEquals(ts.minute(), t[4]) self.assert_(abs(ts.second() - t[5]) < EPSILON)
def checkTimeStamp(self): # Alternate test suite t = TimeStamp(2002, 1, 23, 10, 48, 5) # GMT self.assertEquals(str(t), '2002-01-23 10:48:05.000000') self.assertEquals(repr(t), '\x03B9H\x15UUU') self.assertEquals(TimeStamp('\x03B9H\x15UUU'), t) self.assertEquals(t.year(), 2002) self.assertEquals(t.month(), 1) self.assertEquals(t.day(), 23) self.assertEquals(t.hour(), 10) self.assertEquals(t.minute(), 48) self.assertEquals(round(t.second()), 5) self.assertEquals(t.timeTime(), 1011782885) t1 = TimeStamp(2002, 1, 23, 10, 48, 10) self.assertEquals(str(t1), '2002-01-23 10:48:10.000000') self.assert_(t == t) self.assert_(t != t1) self.assert_(t < t1) self.assert_(t <= t1) self.assert_(t1 >= t) self.assert_(t1 > t) self.failIf(t == t1) self.failIf(t != t) self.failIf(t > t1) self.failIf(t >= t1) self.failIf(t1 < t) self.failIf(t1 <= t) self.assertEquals(cmp(t, t), 0) self.assertEquals(cmp(t, t1), -1) self.assertEquals(cmp(t1, t), 1) self.assertEquals(t1.laterThan(t), t1) self.assert_(t.laterThan(t1) > t1) self.assertEquals(TimeStamp(2002, 1, 23), TimeStamp(2002, 1, 23, 0, 0, 0))
def checkTimeStamp(self): # Alternate test suite t = TimeStamp(2002, 1, 23, 10, 48, 5) # GMT self.assertEquals(str(t), '2002-01-23 10:48:05.000000') self.assertEquals(repr(t), '\x03B9H\x15UUU') self.assertEquals(TimeStamp('\x03B9H\x15UUU'), t) self.assertEquals(t.year(), 2002) self.assertEquals(t.month(), 1) self.assertEquals(t.day(), 23) self.assertEquals(t.hour(), 10) self.assertEquals(t.minute(), 48) self.assertEquals(round(t.second()), 5) self.assertEquals(t.timeTime(), 1011782885) t1 = TimeStamp(2002, 1, 23, 10, 48, 10) self.assertEquals(str(t1), '2002-01-23 10:48:10.000000') self.assert_(t == t) self.assert_(t != t1) self.assert_(t < t1) self.assert_(t <= t1) self.assert_(t1 >= t) self.assert_(t1 > t) self.failIf(t == t1) self.failIf(t != t) self.failIf(t > t1) self.failIf(t >= t1) self.failIf(t1 < t) self.failIf(t1 <= t) self.assertEquals(cmp(t, t), 0) self.assertEquals(cmp(t, t1), -1) self.assertEquals(cmp(t1, t), 1) self.assertEquals(t1.laterThan(t), t1) self.assert_(t.laterThan(t1) > t1) self.assertEquals(TimeStamp(2002,1,23), TimeStamp(2002,1,23,0,0,0))
def checkRawTimestamp(self): t = time.gmtime() ts1 = TimeStamp(*t[:6]) ts2 = TimeStamp( ` ts1 `) self.assertEquals(ts1, ts2) self.assertEquals(ts1.timeTime(), ts2.timeTime()) self.assertEqual(ts1.year(), ts2.year()) self.assertEqual(ts1.month(), ts2.month()) self.assertEqual(ts1.day(), ts2.day()) self.assertEquals(ts1.hour(), ts2.hour()) self.assertEquals(ts1.minute(), ts2.minute()) self.assert_(abs(ts1.second() - ts2.second()) < EPSILON)
def _check_ymd(self, yr, mo, dy): ts = TimeStamp(yr, mo, dy) self.assertEqual(ts.year(), yr) self.assertEqual(ts.month(), mo) self.assertEqual(ts.day(), dy) self.assertEquals(ts.hour(), 0) self.assertEquals(ts.minute(), 0) self.assertEquals(ts.second(), 0) t = time.gmtime(ts.timeTime()) self.assertEquals(yr, t[0]) self.assertEquals(mo, t[1]) self.assertEquals(dy, t[2])
def main(): usage = "usage: %prog [options] filename" parser = OptionParser(usage=usage) parser.add_option("-n", "--number", dest="num", help="display only the 'n' busiest days", default=20, type="int") parser.add_option("-f", "--file", dest="filename", action="store", type="string", help="your FileStorage") parser.add_option("-d", "--date", dest="date", action="store", type="string", help="show the stats only for the date d (format dd-mm-yyyy)") parser.add_option("-a", "--days", dest="days", action="store", default="0", type="string", help="show the stats only for the last 'a' days") parser.add_option("-v", "--verbose", dest="verbose", action="store_false", help="show percentage and time remaining") (options, args) = parser.parse_args() objectsToDisplay = options.num VERBOSE = False if options.filename: fname = options.filename else: print "You have to enter the filename, see --help for details" return 2 if options.verbose != None: VERBOSE = True stats = {} start = time.time() size = os.stat(fname).st_size it = ZODB.FileStorage.FileIterator(fname) lastPercent = 0.0 recordsCounter = 0 interval = 0.005 dataFound = False now = datetime.date.today() try: for t in it: #Format the date of the current transaction following dd-mm-yyyy ts = TimeStamp(t.tid) then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day())) delta = timedelta(days=int(options.days)) if((not int(options.days)) or (now - then < delta)): dateT = strftime("%d-%m-%Y", [int(ts.year()), int(ts.month()), int(ts.day()),1,1,1,1,1,1] ) percent = float(it._file.tell())/float(size) * 100 #Check if we found the searched date if options.date: if str(dateT) == str(options.date): dataFound = True elif dataFound: break #Show the percentage of the work completed and the remaining time if(percent - lastPercent > interval): spentTime = time.time() - start remainingTime = spentTime / float(it._file.tell()) * (float(size)) - spentTime if VERBOSE: sys.stderr.write("\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d" % (percent,GetInHMS(time.time() - start, True), GetInHMS(remainingTime, False), recordsCounter)) lastPercent = percent stat = stats.get(dateT) if stat is None: stat = stats[dateT] = Stat() stat.n = 1 else: stat.n += 1 for r in t: #need to reduce the time of the dictionary stats from time to time if recordsCounter % (objectsToDisplay*100) == 0: tmp = {} for date, s in sorted( stats.items(), key=lambda (k,v): v.n, reverse=True)[0: objectsToDisplay]: tmp[date] = s try: tmp[dateT] = stats[dateT] except KeyError: pass stats = tmp if r.data: mod, klass = get_pickle_metadata(r.data) l = len(r.data) stat = stats.get(dateT) stat.records += 1 recordsCounter += 1 stat = stats.get(dateT) if stat is not None: stat.mean.append(TimeStamp(t.tid).timeTime()) except KeyboardInterrupt: pass print "\n" print "%-15s %17s %17s %22s" % ("Date", "Transactions","Records Changed", "Average interval") print "%s" % "_" * 74 if options.date: for date, s in sorted( stats.items(), key=lambda (k,v): v.n, reverse=True): meanTime = 0 for i in range(1,len(s.mean)): meanTime += s.mean[i] - s.mean[i-1] if str(date) == str(options.date): print "%-15s | %15d | %15d | %15f secs" % (date, (s.n),s.records, meanTime/s.n) else: for date, s in sorted( stats.items(), key=lambda (k,v): v.n, reverse=True)[0: objectsToDisplay]: meanTime = 0 for i in range(1,len(s.mean)): meanTime += s.mean[i] - s.mean[i-1] print "%-15s | %15d | %15d | %15f secs" % (date, (s.n), s.records, meanTime/s.n)
def run(path, days, notPacked): f = open(path, "rb") f.seek(0, 2) size = os.path.getsize(path) now = datetime.date.today() notPackedDays = [] for day in range(notPacked): notPackedDays.append(str(now - timedelta(days=day + 1))) # day->size stats = {} th = prev_txn(f) bool = True while bool: ts = TimeStamp(th.tid) then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day())) delta = timedelta(days=int(days)) if now - then < delta: dateT = strftime("%Y-%m-%d", [int(ts.year()), int(ts.month()), int(ts.day()), 1, 1, 1, 1, 1, 1]) try: stats[dateT] = stats[dateT] + th.length except KeyError: stats[dateT] = th.length else: bool = False th = th.prev_txn() f.close() total = 0 totalPacked = 0 daysPacked = 0 for (d, s) in sorted(stats.items(), key=lambda (k, v): v, reverse=True): print d, "size:", pretty_size(s), date = str(d) if date in notPackedDays or date == str(now): print "(not yet packed)" else: totalPacked = totalPacked + s daysPacked = daysPacked + 1 print total = total + s if int(totalPacked): average = totalPacked / int(daysPacked) else: average = 0 print "\n-- ALREADY PACKED DAYS--" print "The amount of data added in", daysPacked, "days is", pretty_size(totalPacked) print "Average", pretty_size(average), "per day" print "Following this trend, the size of the database will be:" print "\t", pretty_size(average * 365 + size), " in 1 year" print "\t", pretty_size(average * 365 * 2 + size), " in 2 years" print "\t", pretty_size(average * 365 * 10 + size), " in 10 years" print "\n-- ALL DAYS --" print "The amount of data added in", days, "days is", pretty_size(total) if int(total): print "Average", pretty_size(total / int(days)), "per day" else: print "Average 0bytes per day"
def main(): usage = "usage: %prog [options] filename" parser = OptionParser(usage=usage) parser.add_option("-n", "--number", dest="num", help="display only the n biggest objects", default=-1, type="int") parser.add_option("-f", "--output", dest="filename", action="store", type="string", help="the FileStorage") parser.add_option("-v", "--verbose", dest="verbose", action="store_false", help="show percentage and time remaining") (options, args) = parser.parse_args() VERBOSE = False if options.filename: fname = options.filename else: print "You have to enter the FileStorage filename, see --help for details" return 2 if options.verbose != None: VERBOSE = True objectsToDisplay = options.num stats = {} start = time.time() size = os.stat(fname).st_size it = ZODB.FileStorage.FileIterator(fname) lastPercent = 0.0 recordsCounter = 0 interval = 0.005 now = datetime.date.today() try: for t in it: percent = float(it._file.tell()) / float(size) * 100 #Show the percentage of the work completed and the remaining time if (percent - lastPercent > interval): spentTime = time.time() - start remainingTime = spentTime / float( it._file.tell()) * (float(size)) - spentTime if VERBOSE: sys.stderr.write( "\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d" % (percent, GetInHMS(time.time() - start), GetInHMS(remainingTime), recordsCounter)) sys.stdout.flush() lastPercent = percent for r in t: #need to reduce the time of the dictionary stats from time to time ts = TimeStamp(t.tid) then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day())) delta = timedelta(days=3) #don't reduce the size of the dictionary when analysing last 3 days transactions if recordsCounter % (objectsToDisplay * 100) == 0 and ( now - then > delta): tmp = {} for class_name, s in sorted( stats.items(), key=lambda (k, v): v.size, reverse=True)[0:objectsToDisplay]: tmp[class_name] = s stats = tmp if r.data: mod, klass = get_pickle_metadata(r.data) l = len(r.data) class_name = mod + "." + klass + " oid: " + oid_repr( r.oid).strip() stat = stats.get(class_name) if stat is None: stat = stats[class_name] = Stat() stat.size = stat.min = stat.max = l stat.oid = oid_repr(r.oid).strip() stat.className = mod + "." + klass stat.number = 1 else: stat.min = min(stat.min, l) stat.max = max(stat.max, l) stat.number = stat.number + 1 stat.size = stat.size + l recordsCounter += 1 except KeyboardInterrupt: pass print "\n" print "%-41s %9s %15s %15s %9s %9s %9s" % ("Module.ClassName", "Oid", "Percentage", "Total Size", "Min", "Max", "Copies") print "%s" % "_" * 114 for class_name, s in sorted(stats.items(), key=lambda (k, v): v.size, reverse=True)[0:objectsToDisplay]: class_name = s.className if len(class_name) > 40: class_name = class_name[::-1][0:35][::-1] class_name = "[..]" + class_name print "%-40s | %8s | %13f%% | %13s | %7s | %7s | %7s" % ( class_name, s.oid, (s.size * 100.0 / size), pretty_size( s.size), pretty_size(s.min), pretty_size(s.max), s.number)
def main(): usage = "usage: %prog [options] filename" parser = OptionParser(usage=usage) parser.add_option("-n", "--number", dest="num", help="display only the n biggest objects", default=-1, type="int") parser.add_option("-f", "--output", dest="filename", action="store", type="string", help="the FileStorage") parser.add_option("-v", "--verbose", dest="verbose", action="store_false", help="show percentage and time remaining") (options, args) = parser.parse_args() VERBOSE = False if options.filename: fname = options.filename else: print "You have to enter the FileStorage filename, see --help for details" return 2 if options.verbose != None: VERBOSE = True objectsToDisplay = options.num stats = {} start = time.time() size = os.stat(fname).st_size it = ZODB.FileStorage.FileIterator(fname) lastPercent = 0.0 recordsCounter = 0 interval = 0.005 now = datetime.date.today() try: for t in it: percent = float(it._file.tell())/float(size) * 100 #Show the percentage of the work completed and the remaining time if(percent - lastPercent > interval): spentTime = time.time() - start remainingTime = spentTime / float(it._file.tell()) * (float(size)) - spentTime if VERBOSE: sys.stderr.write("\r%f%% complete, time spent %s, remaining time: %s, recordsCounter %d" % (percent,GetInHMS(time.time() - start), GetInHMS(remainingTime), recordsCounter)) sys.stdout.flush() lastPercent = percent for r in t: #need to reduce the time of the dictionary stats from time to time ts = TimeStamp(t.tid) then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day())) delta = timedelta(days=3) #don't reduce the size of the dictionary when analysing last 3 days transactions if recordsCounter % (objectsToDisplay*100) == 0 and (now - then > delta): tmp = {} for class_name, s in sorted( stats.items(), key=lambda (k,v): v.size, reverse=True)[0: objectsToDisplay]: tmp[class_name] = s stats = tmp if r.data: mod, klass = get_pickle_metadata(r.data) l = len(r.data) class_name = mod + "." + klass + " oid: " + oid_repr(r.oid).strip() stat = stats.get(class_name) if stat is None: stat = stats[class_name] = Stat() stat.size = stat.min = stat.max = l stat.oid = oid_repr(r.oid).strip() stat.className = mod + "." + klass stat.number = 1 else: stat.min = min(stat.min, l) stat.max = max(stat.max, l) stat.number = stat.number + 1 stat.size = stat.size + l recordsCounter += 1 except KeyboardInterrupt: pass print "\n" print "%-41s %9s %15s %15s %9s %9s %9s" % ("Module.ClassName", "Oid", "Percentage", "Total Size", "Min", "Max", "Copies") print "%s" % "_" * 114 for class_name, s in sorted( stats.items(), key=lambda (k,v): v.size, reverse=True)[0: objectsToDisplay]: class_name = s.className if len(class_name) > 40: class_name = class_name[::-1][0:35][::-1] class_name = "[..]" + class_name print "%-40s | %8s | %13f%% | %13s | %7s | %7s | %7s" % (class_name, s.oid, (s.size*100.0/size) , pretty_size(s.size), pretty_size(s.min), pretty_size(s.max), s.number)
def run(path, days, notPacked): f = open(path, "rb") f.seek(0, 2) size = os.path.getsize(path) now = datetime.date.today() notPackedDays = [] for day in range(notPacked): notPackedDays.append(str(now - timedelta(days=day + 1))) #day->size stats = {} th = prev_txn(f) bool = True while bool: ts = TimeStamp(th.tid) then = datetime.date(int(ts.year()), int(ts.month()), int(ts.day())) delta = timedelta(days=int(days)) if (now - then < delta): dateT = strftime("%Y-%m-%d", [ int(ts.year()), int(ts.month()), int(ts.day()), 1, 1, 1, 1, 1, 1 ]) try: stats[dateT] = stats[dateT] + th.length except KeyError: stats[dateT] = th.length else: bool = False th = th.prev_txn() f.close() total = 0 totalPacked = 0 daysPacked = 0 for (d, s) in sorted(stats.items(), key=lambda (k, v): v, reverse=True): print d, "size:", pretty_size(s), date = str(d) if (date in notPackedDays or date == str(now)): print "(not yet packed)" else: totalPacked = totalPacked + s daysPacked = daysPacked + 1 print total = total + s if int(totalPacked): average = totalPacked / int(daysPacked) else: average = 0 print "\n-- ALREADY PACKED DAYS--" print "The amount of data added in", daysPacked, "days is", pretty_size( totalPacked) print "Average", pretty_size(average), "per day" print "Following this trend, the size of the database will be:" print "\t", pretty_size(average * 365 + size), " in 1 year" print "\t", pretty_size(average * 365 * 2 + size), " in 2 years" print "\t", pretty_size(average * 365 * 10 + size), " in 10 years" print "\n-- ALL DAYS --" print "The amount of data added in", days, "days is", pretty_size(total) if int(total): print "Average", pretty_size(total / int(days)), "per day" else: print "Average 0bytes per day"