def __init__(self, verbosity): self.verbosity = verbosity self.pfmt = PathFormatter(120) t = time.time() self.stats = { 'startTime': t, 'resetTime': t, 'flushTime': t, 'statsTime': t, 'evtps': 0.0, 'evtpg': 0.0, 'totalEvents': 0, 'expectedEvents':None, 'isWarmingUp': True, } self.events = {} self.totalEvents = {} self.settings = { 'resetTime': 30, 'flushTime': 1, }
parser.add_argument('--stopOnFirstFail',dest='stopOnFirstFail', action='store', type=int, default=1, help='TODO') parser.add_argument('--verbose', dest='verbose', action='store', type=int, default='',help='TODO') parser.add_argument('--fast', dest='fast', action='store', type=int, default=0,help='TODO') parser.add_argument('--Hasher', dest='Hasher', action='store', type=str, default='FastAttributeHashV1.Cached', help='TODO') args = vars(parser.parse_args()) if args['verbose']>=4: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%Y%m%d %I:%M:%S') elif args['verbose']>=2: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.WARNING, datefmt='%Y%m%d %I:%M:%S') elif args['verbose']>=1: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.ERROR, datefmt='%Y%m%d %I:%M:%S') else: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.ERROR, datefmt='%Y%m%d %I:%M:%S') pfmt = PathFormatter(120) cache = sqlite.sqlite(); cache.setCacheLocation(args['cacheOld']) cache.initialize() analyzer = BackupAnalyzer() analyzer.setCache(cache) analyzer.initialize() logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount())) hh = Hasher.FullContentHashV1.Base() hh.initialize() fh = None
import math import sys """ FIX: the backup analizer should use some coordinated sql's through the cache class (bases on sqlite3), not double-connecting to the sale DB same for SimpleComparer """ parser = argparse.ArgumentParser(description='Create the sqlite DB') parser.add_argument('--cache', dest='cache', action='store', type=str, default='',help='TODO') parser.add_argument('--data', dest='data', action='store', type=str, default='',help='TODO') parser.add_argument('--percent', dest='percent', action='store', type=float, default='',help='TODO') parser.add_argument('--min', dest='min', action='store', type=int, default='',help='TODO') args = vars(parser.parse_args()) pfmt = PathFormatter(120) cache = sqlite.sqlite(); cache.setCacheLocation(args['cache']) cache.initialize() analyzer = BackupAnalyzer() analyzer.setCache(cache) analyzer.initialize() print "files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount()) print "files without full hashes: %s files" % (analyzer.getFilesWithoutFullHashesCount()) hh = FullContentHashV1.FullContentHashV1() hh.initialize()
""" parser = argparse.ArgumentParser(description='Create the sqlite DB') parser.add_argument('--cache', dest='cache', action='store', type=str, default='',help='TODO') parser.add_argument('--data', dest='data', action='store', type=str, default='',help='TODO') parser.add_argument('--percent', dest='percent', action='store', type=float, default='',help='TODO') parser.add_argument('--min', dest='min', action='store', type=int, default='',help='TODO') parser.add_argument('--verbose', dest='verbose', action='store', type=int, default='',help='TODO') args = vars(parser.parse_args()) if args['verbose']>=4: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%Y%m%d %I:%M:%S') else: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.WARNING, datefmt='%Y%m%d %I:%M:%S') pfmt = PathFormatter(120) cache = sqlite.sqlite(); cache.setCacheLocation(args['cache']) cache.initialize() analyzer = BackupAnalyzer() analyzer.setCache(cache) analyzer.initialize() logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount())) logging.info("files without full hashes: %s files" % (analyzer.getFilesWithoutFullHashesCount())) hh = FullContentHashV1.FullContentHashV1() hh.initialize()
import sys import logging """ FIX: the backup analizer should use some coordinated sql's through the cache class (bases on sqlite3), not double-connecting to the sale DB same for SimpleComparer """ parser = argparse.ArgumentParser(description='Create the sqlite DB') parser.add_argument('--cacheOld', dest='cacheOld', action='store', type=str, default='',help='TODO') parser.add_argument('--source', dest='source', action='store', type=str, default='',help='TODO') parser.add_argument('--destination', dest='destination', action='store', type=str, default='',help='TODO') parser.add_argument('--percent', dest='percent', action='store', type=float, default='',help='TODO') parser.add_argument('--min', dest='min', action='store', type=int, default='',help='TODO') args = vars(parser.parse_args()) pfmt = PathFormatter(120) cache = sqlite.sqlite(); cache.setCacheLocation(args['cacheOld']) cache.initialize() analyzer = BackupAnalyzer() analyzer.setCache(cache) analyzer.initialize() logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount())) hh = FullContentHashV1.FullContentHashV1() hh.initialize() wrt = Writer(args['destination'], args['source'])
parser.add_argument('--percent', dest='percent', action='store', type=float, default='',help='TODO') parser.add_argument('--min', dest='min', action='store', type=int, default='',help='TODO') parser.add_argument('--stopOnFirstFail',dest='stopOnFirstFail', action='store', type=int, default=1, help='TODO') parser.add_argument('--verbose', dest='verbose', action='store', type=int, default='',help='TODO') args = vars(parser.parse_args()) if args['verbose']>=4: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%Y%m%d %I:%M:%S') elif args['verbose']>=2: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.WARNING, datefmt='%Y%m%d %I:%M:%S') elif args['verbose']>=1: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.CRITICAL, datefmt='%Y%m%d %I:%M:%S') else: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.CRITICAL, datefmt='%Y%m%d %I:%M:%S') pfmt = PathFormatter(120) cache = sqlite.sqlite(); cache.setCacheLocation(args['cacheOld']) cache.initialize() analyzer = BackupAnalyzer() analyzer.setCache(cache) analyzer.initialize() logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount())) hh = FullContentHashV1.FullContentHashV1() hh.initialize() wrt = Writer(args['destination'], args['source'])
class ScriptStatusTracker(object): def __init__(self, verbosity): self.verbosity = verbosity self.pfmt = PathFormatter(120) t = time.time() self.stats = { 'startTime': t, 'resetTime': t, 'flushTime': t, 'statsTime': t, 'evtps': 0.0, 'evtpg': 0.0, 'totalEvents': 0, 'expectedEvents':None, 'isWarmingUp': True, } self.events = {} self.totalEvents = {} self.settings = { 'resetTime': 30, 'flushTime': 1, } def storeEvent(self, tm, event, data): if self.verbosity==0: return if not event in self.events: self.events[event] = 0 self.events[event]+= 1 if not event in self.totalEvents: self.totalEvents[event] = 0 self.totalEvents[event]+= 1 self.stats['totalEvents']+= 1 def resetStats(self, tm): self.stats['resetTime'] = tm #stats['evtps'] = 0 self.stats['isWarmingUp'] = True self.events = {} def calcStats(self, tm): pass def composeOutputStr(self, statusStr, event, data): return "%s %8s: %s" % (statusStr, event, self.pfmt.format(data['p'].path).ljust(120)) def logEvent(self, tm, event, data): if self.verbosity==0: return doHandle = False if self.verbosity>=3: doHandle = True elif self.verbosity>=2: doHandle = True elif self.verbosity>=1: if tm - self.stats['statsTime'] > 0.1: doHandle = True if doHandle: self.stats['statsTime'] = tm evtps = " ....e/s," if not self.stats['isWarmingUp']: evtps = "%5.1fe/s," % (min(self.stats['evtps'], 9999.9)) pgpc = "--.-%," if not self.stats['expectedEvents'] is None: pgpc = "%4.1f%%," % (99.9*self.stats['evtpg']) self.calcStats(tm) self.printEvent(tm, self.composeOutputStr("%s%s" % (pgpc, evtps), tm, event, data)) if tm - self.stats['resetTime'] > self.settings['resetTime']: self.resetStats(tm) def printEvent(self, tm, str): if self.verbosity>=3: sys.stdout.write("\n%s" % (str)) sys.stdout.flush() else: sys.stdout.write("\r%s" % (str)) if tm - self.stats['flushTime'] > self.settings['flushTime']: self.stats['flushTime'] = tm sys.stdout.flush() def printStr(self, str): sys.stdout.write("\n%s" % (str))
level=logging.DEBUG, datefmt='%Y%m%d %I:%M:%S') elif args['verbose'] >= 2: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.WARNING, datefmt='%Y%m%d %I:%M:%S') elif args['verbose'] >= 1: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.ERROR, datefmt='%Y%m%d %I:%M:%S') else: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.ERROR, datefmt='%Y%m%d %I:%M:%S') pfmt = PathFormatter(120) cache = sqlite.sqlite() cache.setCacheLocation(args['cacheOld']) cache.initialize() analyzer = BackupAnalyzer() analyzer.setCache(cache) analyzer.initialize() logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount())) hh = Hasher.FullContentHashV1.Base() hh.initialize()
""" parser = argparse.ArgumentParser(description='Create the sqlite DB') parser.add_argument('--cache', dest='cache', action='store', type=str, default='',help='TODO') parser.add_argument('--data', dest='data', action='store', type=str, default='',help='TODO') parser.add_argument('--percent', dest='percent', action='store', type=float, default='',help='TODO') parser.add_argument('--min', dest='min', action='store', type=int, default='',help='TODO') parser.add_argument('--verbose', dest='verbose', action='store', type=int, default='',help='TODO') args = vars(parser.parse_args()) if args['verbose']>=4: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%Y%m%d %I:%M:%S') else: logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.WARNING, datefmt='%Y%m%d %I:%M:%S') pfmt = PathFormatter(120) cache = sqlite.sqlite(); cache.setCacheLocation(args['cache']) cache.initialize() analyzer = BackupAnalyzer() analyzer.setCache(cache) analyzer.initialize() logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount())) logging.info("files without full hashes: %s files" % (analyzer.getFilesWithoutFullHashesCount())) hh = Hasher.FullContentHashV1.Base() hh.initialize()