path = Path(wrt.getDestinationFilePathToContent(op), False)
path.size = np_size
hashMatches = False
if args['fast']:
pathSrc = Path(wrt.getSourceFilePath(op), False)
pathSrc.ctime = np_ctime
pathSrc.mtime = np_mtime
pathSrc.atime = np_atime
try:
pathSrc.size = getsize(op)
except (OSError) as e:
pathSrc.size = -1
logging.info(" fast check: %s" % (pfmt.format(op).ljust(120)))
try:
fastHash = fh._hash(pathSrc)
except (OSError, IOError) as e:
fastHash = 'exception'
#print(" fastHash: %s" % (fastHash))
#print(" fhash: %s" % (fhash))
if fastHash==fhash:
hashMatches = True
else:
# do a "slow" hashing also... to be able to log stuff correctly, leave hashMatches=False as-is
logging.info(" full check: %s" % (pfmt.format(op).ljust(120)))
hash = hh.hash(path)
hashMatches = False
if hash==fullHash:
cache.setCacheLocation(args['cache'])
cache.initialize()
analyzer = BackupAnalyzer()
analyzer.setCache(cache)
analyzer.initialize()
logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount()))
logging.info("files without full hashes: %s files" % (analyzer.getFilesWithoutFullHashesCount()))
hh = FullContentHashV1.FullContentHashV1()
hh.initialize()
files = analyzer.getFilesWithoutFullHashes('random',
max(args['min'], math.ceil(analyzer.getFilesCount()*(args['percent']/100)))
)
for (p, fhash, sz) in files:
logging.info(" hash: %s" % (pfmt.format(p).ljust(120)))
path = Path(p, False)
path.size = sz
hash = hh.hash(path)
#print " hash: %s" % (hash)
cache.updateFileFullHashIntoFiles(path, hash)
cache.commit()
hh.destroy()
analyzer.destroy()
cache.destroy()
analyzer = BackupAnalyzer()
analyzer.setCache(cache)
analyzer.initialize()
print "files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount())
print "files without full hashes: %s files" % (analyzer.getFilesWithoutFullHashesCount())
hh = FullContentHashV1.FullContentHashV1()
hh.initialize()
files = analyzer.getFilesWithoutFullHashes('random',
max(args['min'], math.ceil(analyzer.getFilesCount()*(args['percent']/100)))
)
sys.stdout.write("\n")
for (p, fhash, sz) in files:
sys.stdout.write("\r hash: %s" % (pfmt.format(p).ljust(120)))
path = Path(p, False)
path.size = sz
hash = hh.hash(path)
#print " hash: %s" % (hash)
cache.updateFileFullHashIntoFiles(path, hash)
cache.commit()
sys.stdout.write("\n")
hh.destroy()
analyzer.destroy()
cache.destroy()
hh = FullContentHashV1.FullContentHashV1()
hh.initialize()
wrt = Writer(args['destination'], args['source'])
wrt.initialize()
files = analyzer.getFilesWithFullHashes('random',
max(args['min'], math.ceil(analyzer.getFilesWithFullHashesCount()*(args['percent']/100)))
)
sys.stdout.write("\n")
for (np, fhash, sz, fullHash) in files:
op = wrt.getDestinationFilePath(np)
sys.stdout.write("\r check: %s" % (pfmt.format(op).ljust(120)))
path = Path(wrt.getDestinationFilePathToContent(op), False)
path.size = sz
hash = hh.hash(path)
if hash!=fullHash:
logging.error("!"*80)
logging.error("! fullHash check failed!")
logging.error("! fullHash: %s, expected: %s" % (hash, fullHash))
logging.error("!"*80)
cache.commit()
hh.destroy()
analyzer.destroy()
hh = FullContentHashV1.FullContentHashV1()
hh.initialize()
wrt = Writer(args['destination'], args['source'])
wrt.initialize()
files = analyzer.getFilesWithFullHashes('random',
max(args['min'], math.ceil(analyzer.getFilesWithFullHashesCount()*(args['percent']/100)))
)
failedChecks = []
for (np, fhash, sz, fullHash) in files:
op = wrt.getDestinationFilePath(np)
logging.info(" check: %s" % (pfmt.format(op).ljust(120)))
path = Path(wrt.getDestinationFilePathToContent(op), False)
path.size = sz
hash = hh.hash(path)
if hash!=fullHash:
if args['stopOnFirstFail']:
logging.error("!"*80)
logging.error("! fullHash check failed!")
logging.error("! path: %s" % (np))
logging.error("! fullHash: %s" % (hash))
logging.error("! expected: %s" % (fullHash))
logging.error("!"*80)
sys.exit(1)
logging.error(" fullHash check failed!")
class ScriptStatusTracker(object):
def __init__(self, verbosity):
self.verbosity = verbosity
self.pfmt = PathFormatter(120)
t = time.time()
self.stats = {
'startTime': t,
'resetTime': t,
'flushTime': t,
'statsTime': t,
'evtps': 0.0,
'evtpg': 0.0,
'totalEvents': 0,
'expectedEvents':None,
'isWarmingUp': True,
}
self.events = {}
self.totalEvents = {}
self.settings = {
'resetTime': 30,
'flushTime': 1,
}
def storeEvent(self, tm, event, data):
if self.verbosity==0:
return
if not event in self.events:
self.events[event] = 0
self.events[event]+= 1
if not event in self.totalEvents:
self.totalEvents[event] = 0
self.totalEvents[event]+= 1
self.stats['totalEvents']+= 1
def resetStats(self, tm):
self.stats['resetTime'] = tm
#stats['evtps'] = 0
self.stats['isWarmingUp'] = True
self.events = {}
def calcStats(self, tm):
pass
def composeOutputStr(self, statusStr, event, data):
return "%s %8s: %s" % (statusStr, event, self.pfmt.format(data['p'].path).ljust(120))
def logEvent(self, tm, event, data):
if self.verbosity==0:
return
doHandle = False
if self.verbosity>=3:
doHandle = True
elif self.verbosity>=2:
doHandle = True
elif self.verbosity>=1:
if tm - self.stats['statsTime'] > 0.1:
doHandle = True
if doHandle:
self.stats['statsTime'] = tm
evtps = " ....e/s,"
if not self.stats['isWarmingUp']:
evtps = "%5.1fe/s," % (min(self.stats['evtps'], 9999.9))
pgpc = "--.-%,"
if not self.stats['expectedEvents'] is None:
pgpc = "%4.1f%%," % (99.9*self.stats['evtpg'])
self.calcStats(tm)
self.printEvent(tm, self.composeOutputStr("%s%s" % (pgpc, evtps), tm, event, data))
if tm - self.stats['resetTime'] > self.settings['resetTime']:
self.resetStats(tm)
def printEvent(self, tm, str):
if self.verbosity>=3:
sys.stdout.write("\n%s" % (str))
sys.stdout.flush()
else:
sys.stdout.write("\r%s" % (str))
if tm - self.stats['flushTime'] > self.settings['flushTime']:
self.stats['flushTime'] = tm
sys.stdout.flush()
def printStr(self, str):
sys.stdout.write("\n%s" % (str))
path = Path(wrt.getDestinationFilePathToContent(op), False)
path.size = np_size
hashMatches = False
if args['fast']:
pathSrc = Path(wrt.getSourceFilePath(op), False)
pathSrc.ctime = np_ctime
pathSrc.mtime = np_mtime
pathSrc.atime = np_atime
try:
pathSrc.size = getsize(op)
except (OSError) as e:
pathSrc.size = -1
logging.info(" fast check: %s" % (pfmt.format(op).ljust(120)))
try:
fastHash = fh._hash(pathSrc)
except (OSError, IOError) as e:
fastHash = 'exception'
#print(" fastHash: %s" % (fastHash))
#print(" fhash: %s" % (fhash))
if fastHash == fhash:
hashMatches = True
else:
# do a "slow" hashing also... to be able to log stuff correctly, leave hashMatches=False as-is
logging.info(" full check: %s" % (pfmt.format(op).ljust(120)))
hash = hh.hash(path)
hashMatches = False
if hash == fullHash:
cache.setCacheLocation(args['cache'])
cache.initialize()
analyzer = BackupAnalyzer()
analyzer.setCache(cache)
analyzer.initialize()
logging.info("files with full hashes: %s files" % (analyzer.getFilesWithFullHashesCount()))
logging.info("files without full hashes: %s files" % (analyzer.getFilesWithoutFullHashesCount()))
hh = Hasher.FullContentHashV1.Base()
hh.initialize()
files = analyzer.getFilesWithoutFullHashes('random',
max(args['min'], math.ceil(analyzer.getFilesCount()*(args['percent']/100)))
)
for (p, fhash, sz) in files:
logging.info(" hash: %s" % (pfmt.format(p).ljust(120)))
path = Path(p, False)
path.size = sz
hash = hh.hash(path)
#print " hash: %s" % (hash)
cache.updateFileFullHashIntoFiles(path, hash)
cache.commit()
hh.destroy()
analyzer.destroy()
cache.destroy()
