def __init__(self, verbosity):
self.verbosity = verbosity
self.pfmt = PathFormatter(120)
t = time.time()
self.stats = {
'startTime': t,
'resetTime': t,
'flushTime': t,
'statsTime': t,
'evtps': 0.0,
'evtpg': 0.0,
'totalEvents': 0,
'expectedEvents':None,
'isWarmingUp': True,
}
self.events = {}
self.totalEvents = {}
self.settings = {
'resetTime': 30,
'flushTime': 1,
}
parser.add_argument('--stopOnFirstFail',dest='stopOnFirstFail', action='store', type=int, default=1, help='TODO')
parser.add_argument('--verbose', dest='verbose', action='store', type=int, default='',help='TODO')
parser.add_argument('--fast', dest='fast', action='store', type=int, default=0,help='TODO')
parser.add_argument('--Hasher', dest='Hasher', action='store', type=str, default='FastAttributeHashV1.Cached', help='TODO')
args = vars(parser.parse_args())
if args['verbose']>=4:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%Y%m%d %I:%M:%S')
elif args['verbose']>=2:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.WARNING, datefmt='%Y%m%d %I:%M:%S')
elif args['verbose']>=1:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.ERROR, datefmt='%Y%m%d %I:%M:%S')
else:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.ERROR, datefmt='%Y%m%d %I:%M:%S')
pfmt = PathFormatter(120)
cache = sqlite.sqlite();
cache.setCacheLocation(args['cacheOld'])
cache.initialize()
analyzer = BackupAnalyzer()
analyzer.setCache(cache)
analyzer.initialize()
logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount()))
hh = Hasher.FullContentHashV1.Base()
hh.initialize()
fh = None
import math
import sys
"""
FIX: the backup analizer should use some coordinated sql's through the cache class (bases on sqlite3), not double-connecting to the sale DB
same for SimpleComparer
"""
parser = argparse.ArgumentParser(description='Create the sqlite DB')
parser.add_argument('--cache', dest='cache', action='store', type=str, default='',help='TODO')
parser.add_argument('--data', dest='data', action='store', type=str, default='',help='TODO')
parser.add_argument('--percent', dest='percent', action='store', type=float, default='',help='TODO')
parser.add_argument('--min', dest='min', action='store', type=int, default='',help='TODO')
args = vars(parser.parse_args())
pfmt = PathFormatter(120)
cache = sqlite.sqlite();
cache.setCacheLocation(args['cache'])
cache.initialize()
analyzer = BackupAnalyzer()
analyzer.setCache(cache)
analyzer.initialize()
print "files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount())
print "files without full hashes: %s files" % (analyzer.getFilesWithoutFullHashesCount())
hh = FullContentHashV1.FullContentHashV1()
hh.initialize()
"""
parser = argparse.ArgumentParser(description='Create the sqlite DB')
parser.add_argument('--cache', dest='cache', action='store', type=str, default='',help='TODO')
parser.add_argument('--data', dest='data', action='store', type=str, default='',help='TODO')
parser.add_argument('--percent', dest='percent', action='store', type=float, default='',help='TODO')
parser.add_argument('--min', dest='min', action='store', type=int, default='',help='TODO')
parser.add_argument('--verbose', dest='verbose', action='store', type=int, default='',help='TODO')
args = vars(parser.parse_args())
if args['verbose']>=4:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%Y%m%d %I:%M:%S')
else:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.WARNING, datefmt='%Y%m%d %I:%M:%S')
pfmt = PathFormatter(120)
cache = sqlite.sqlite();
cache.setCacheLocation(args['cache'])
cache.initialize()
analyzer = BackupAnalyzer()
analyzer.setCache(cache)
analyzer.initialize()
logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount()))
logging.info("files without full hashes: %s files" % (analyzer.getFilesWithoutFullHashesCount()))
hh = FullContentHashV1.FullContentHashV1()
hh.initialize()
import sys
import logging
"""
FIX: the backup analizer should use some coordinated sql's through the cache class (bases on sqlite3), not double-connecting to the sale DB
same for SimpleComparer
"""
parser = argparse.ArgumentParser(description='Create the sqlite DB')
parser.add_argument('--cacheOld', dest='cacheOld', action='store', type=str, default='',help='TODO')
parser.add_argument('--source', dest='source', action='store', type=str, default='',help='TODO')
parser.add_argument('--destination', dest='destination', action='store', type=str, default='',help='TODO')
parser.add_argument('--percent', dest='percent', action='store', type=float, default='',help='TODO')
parser.add_argument('--min', dest='min', action='store', type=int, default='',help='TODO')
args = vars(parser.parse_args())
pfmt = PathFormatter(120)
cache = sqlite.sqlite();
cache.setCacheLocation(args['cacheOld'])
cache.initialize()
analyzer = BackupAnalyzer()
analyzer.setCache(cache)
analyzer.initialize()
logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount()))
hh = FullContentHashV1.FullContentHashV1()
hh.initialize()
wrt = Writer(args['destination'], args['source'])
parser.add_argument('--percent', dest='percent', action='store', type=float, default='',help='TODO')
parser.add_argument('--min', dest='min', action='store', type=int, default='',help='TODO')
parser.add_argument('--stopOnFirstFail',dest='stopOnFirstFail', action='store', type=int, default=1, help='TODO')
parser.add_argument('--verbose', dest='verbose', action='store', type=int, default='',help='TODO')
args = vars(parser.parse_args())
if args['verbose']>=4:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%Y%m%d %I:%M:%S')
elif args['verbose']>=2:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.WARNING, datefmt='%Y%m%d %I:%M:%S')
elif args['verbose']>=1:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.CRITICAL, datefmt='%Y%m%d %I:%M:%S')
else:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.CRITICAL, datefmt='%Y%m%d %I:%M:%S')
pfmt = PathFormatter(120)
cache = sqlite.sqlite();
cache.setCacheLocation(args['cacheOld'])
cache.initialize()
analyzer = BackupAnalyzer()
analyzer.setCache(cache)
analyzer.initialize()
logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount()))
hh = FullContentHashV1.FullContentHashV1()
hh.initialize()
wrt = Writer(args['destination'], args['source'])
class ScriptStatusTracker(object):
def __init__(self, verbosity):
self.verbosity = verbosity
self.pfmt = PathFormatter(120)
t = time.time()
self.stats = {
'startTime': t,
'resetTime': t,
'flushTime': t,
'statsTime': t,
'evtps': 0.0,
'evtpg': 0.0,
'totalEvents': 0,
'expectedEvents':None,
'isWarmingUp': True,
}
self.events = {}
self.totalEvents = {}
self.settings = {
'resetTime': 30,
'flushTime': 1,
}
def storeEvent(self, tm, event, data):
if self.verbosity==0:
return
if not event in self.events:
self.events[event] = 0
self.events[event]+= 1
if not event in self.totalEvents:
self.totalEvents[event] = 0
self.totalEvents[event]+= 1
self.stats['totalEvents']+= 1
def resetStats(self, tm):
self.stats['resetTime'] = tm
#stats['evtps'] = 0
self.stats['isWarmingUp'] = True
self.events = {}
def calcStats(self, tm):
pass
def composeOutputStr(self, statusStr, event, data):
return "%s %8s: %s" % (statusStr, event, self.pfmt.format(data['p'].path).ljust(120))
def logEvent(self, tm, event, data):
if self.verbosity==0:
return
doHandle = False
if self.verbosity>=3:
doHandle = True
elif self.verbosity>=2:
doHandle = True
elif self.verbosity>=1:
if tm - self.stats['statsTime'] > 0.1:
doHandle = True
if doHandle:
self.stats['statsTime'] = tm
evtps = " ....e/s,"
if not self.stats['isWarmingUp']:
evtps = "%5.1fe/s," % (min(self.stats['evtps'], 9999.9))
pgpc = "--.-%,"
if not self.stats['expectedEvents'] is None:
pgpc = "%4.1f%%," % (99.9*self.stats['evtpg'])
self.calcStats(tm)
self.printEvent(tm, self.composeOutputStr("%s%s" % (pgpc, evtps), tm, event, data))
if tm - self.stats['resetTime'] > self.settings['resetTime']:
self.resetStats(tm)
def printEvent(self, tm, str):
if self.verbosity>=3:
sys.stdout.write("\n%s" % (str))
sys.stdout.flush()
else:
sys.stdout.write("\r%s" % (str))
if tm - self.stats['flushTime'] > self.settings['flushTime']:
self.stats['flushTime'] = tm
sys.stdout.flush()
def printStr(self, str):
sys.stdout.write("\n%s" % (str))
level=logging.DEBUG,
datefmt='%Y%m%d %I:%M:%S')
elif args['verbose'] >= 2:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
level=logging.WARNING,
datefmt='%Y%m%d %I:%M:%S')
elif args['verbose'] >= 1:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
level=logging.ERROR,
datefmt='%Y%m%d %I:%M:%S')
else:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
level=logging.ERROR,
datefmt='%Y%m%d %I:%M:%S')
pfmt = PathFormatter(120)
cache = sqlite.sqlite()
cache.setCacheLocation(args['cacheOld'])
cache.initialize()
analyzer = BackupAnalyzer()
analyzer.setCache(cache)
analyzer.initialize()
logging.info("files with full hashes: %s files" %
(analyzer.getFilesWithFullHashesCount()))
hh = Hasher.FullContentHashV1.Base()
hh.initialize()
"""
parser = argparse.ArgumentParser(description='Create the sqlite DB')
parser.add_argument('--cache', dest='cache', action='store', type=str, default='',help='TODO')
parser.add_argument('--data', dest='data', action='store', type=str, default='',help='TODO')
parser.add_argument('--percent', dest='percent', action='store', type=float, default='',help='TODO')
parser.add_argument('--min', dest='min', action='store', type=int, default='',help='TODO')
parser.add_argument('--verbose', dest='verbose', action='store', type=int, default='',help='TODO')
args = vars(parser.parse_args())
if args['verbose']>=4:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%Y%m%d %I:%M:%S')
else:
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.WARNING, datefmt='%Y%m%d %I:%M:%S')
pfmt = PathFormatter(120)
cache = sqlite.sqlite();
cache.setCacheLocation(args['cache'])
cache.initialize()
analyzer = BackupAnalyzer()
analyzer.setCache(cache)
analyzer.initialize()
logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount()))
logging.info("files without full hashes: %s files" % (analyzer.getFilesWithoutFullHashesCount()))
hh = Hasher.FullContentHashV1.Base()
hh.initialize()
