def Validate(): """ Check all packets to be valid. """ printlog('Validate %r' % time.strftime("%a, %d %b %Y %H:%M:%S +0000")) customers_dir = settings.getCustomersFilesDir() if not os.path.exists(customers_dir): return False for customer_filename in os.listdir(customers_dir): onecustdir = os.path.join(customers_dir, customer_filename) if not os.path.isdir(onecustdir): continue for key_alias_filename in os.listdir(onecustdir): onekeydir = os.path.join(onecustdir, key_alias_filename) if not os.path.isdir(onekeydir): continue def cb(path, subpath, name): if not os.path.isfile(path): return True packetsrc = bpio.ReadBinaryFile(path) if not packetsrc: try: os.remove( path) # if is is no good it is of no use to anyone printlog('Validate %r removed (empty file)' % path) except: printlog('Validate ERROR removing %r' % path) return False p = signed.Unserialize(packetsrc) if p is None: try: os.remove( path) # if is is no good it is of no use to anyone printlog('Validate %r removed (unserialize error)' % path) except: printlog('Validate ERROR removing %r') return False result = p.Valid() packetsrc = '' del p if not result: try: os.remove( path) # if is is no good it is of no use to anyone printlog('Validate %r removed (invalid packet)' % path) except: printlog('Validate ERROR removing %r' % path) return False time.sleep(0.1) return False bpio.traverse_dir_recursive(cb, onekeydir) return True
def Validate(): """ Check all packets to be valid. """ printlog('Validate ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000"))) contactsdb.init() commands.init() customers_dir = settings.getCustomersFilesDir() if not os.path.exists(customers_dir): return for customer_filename in os.listdir(customers_dir): onecustdir = os.path.join(customers_dir, customer_filename) if not os.path.isdir(onecustdir): continue for key_alias_filename in os.listdir(onecustdir): onekeydir = os.path.join(onecustdir, key_alias_filename) if not os.path.isdir(onekeydir): continue def cb(path, subpath, name): # if not os.access(path, os.R_OK | os.W_OK): # return False if not os.path.isfile(path): return True # if name in [settings.BackupIndexFileName(),]: # return False packetsrc = bpio.ReadBinaryFile(path) if not packetsrc: try: os.remove(path) # if is is no good it is of no use to anyone printlog('Validate ' + path + ' removed (empty file)') except: printlog('Validate ERROR removing ' + path) return False p = signed.Unserialize(packetsrc) if p is None: try: os.remove(path) # if is is no good it is of no use to anyone printlog('Validate ' + path + ' removed (unserialize error)') except: printlog('Validate ERROR removing ' + path) return False result = p.Valid() packetsrc = '' del p if not result: try: os.remove(path) # if is is no good it is of no use to anyone printlog('Validate ' + path + ' removed (invalid packet)') except: printlog('Validate ERROR removing ' + path) return False time.sleep(0.1) return False bpio.traverse_dir_recursive(cb, onekeydir)
def ReadLocalFiles(): """ This method scans local backups and build the whole "local" matrix. """ global _LocalFilesNotifyCallback local_files().clear() local_max_block_numbers().clear() local_backup_size().clear() _counter = [0] def visit(realpath, subpath, name): # subpath is something like 0/0/1/0/F20131120053803PM/0-1-Data if not os.path.isfile(realpath): return True if realpath.startswith("newblock-"): return False if subpath in [ settings.BackupIndexFileName(), settings.BackupInfoFileName(), settings.BackupInfoFileNameOld(), settings.BackupInfoEncryptedFileName(), ]: return False try: version = subpath.split("/")[-2] except: return False if not packetid.IsCanonicalVersion(version): return True LocalFileReport(packetID=subpath) _counter[0] += 1 return False bpio.traverse_dir_recursive(visit, settings.getLocalBackupsDir()) lg.out(8, "backup_matrix.ReadLocalFiles %d files indexed" % _counter[0]) if lg.is_debug(8): try: if sys.version_info >= (2, 6): # localSZ = sys.getsizeof(local_files()) # remoteSZ = sys.getsizeof(remote_files()) import lib.getsizeof localSZ = lib.getsizeof.total_size(local_files()) remoteSZ = lib.getsizeof.total_size(remote_files()) indexByName = lib.getsizeof.total_size(backup_fs.fs()) indexByID = lib.getsizeof.total_size(backup_fs.fsID()) lg.out(10, " all local info uses %d bytes in the memory" % localSZ) lg.out(10, " all remote info uses %d bytes in the memory" % remoteSZ) lg.out(10, " index by name takes %d bytes in the memory" % indexByName) lg.out(10, " index by ID takes %d bytes in the memory" % indexByID) except: lg.exc() if _LocalFilesNotifyCallback is not None: _LocalFilesNotifyCallback()
def TraverseLocalFileSystem(basedir, expanded_dirs, callback): def cb(realpath, subpath, name): if not os.access(realpath, os.R_OK): return False if os.path.isfile(realpath): callback(realpath, subpath, name) return False if subpath not in expanded_dirs: callback(realpath, subpath, name) return False bpio.traverse_dir_recursive(cb, basedir)
def ListCustomerFiles(customer_idurl): filename = nameurl.UrlFilename(customer_idurl) customer_dir = os.path.join(settings.getCustomersFilesDir(), filename) result = cStringIO.StringIO() def cb(realpath, subpath, name): if os.path.isdir(realpath): result.write("D%s\n" % subpath) else: result.write("F%s\n" % subpath) return True bpio.traverse_dir_recursive(cb, customer_dir) src = result.getvalue() result.close() return src
def ListCustomerFiles(customer_idurl): filename = nameurl.UrlFilename(customer_idurl) customer_dir = os.path.join(settings.getCustomersFilesDir(), filename) result = cStringIO.StringIO() def cb(realpath, subpath, name): if os.path.isdir(realpath): result.write('D%s\n' % subpath) else: result.write('F%s\n' % subpath) return True bpio.traverse_dir_recursive(cb, customer_dir) src = result.getvalue() result.close() return src
def TreeSummary(ownerdir, key_alias): out = StringIO() out.write('K%s\n' % key_alias) def cb(result, realpath, subpath, name): if not os.access(realpath, os.R_OK): return False if os.path.isfile(realpath): try: filesz = os.path.getsize(realpath) except: filesz = -1 result.write('F%s %d\n' % (subpath, filesz)) return False if not packetid.IsCanonicalVersion(name): result.write('D%s\n' % subpath) return True maxBlock = -1 versionSize = {} dataBlocks = {} parityBlocks = {} dataMissing = {} parityMissing = {} for filename in os.listdir(realpath): packetID = subpath + '/' + filename pth = os.path.join(realpath, filename) try: filesz = os.path.getsize(pth) except: filesz = -1 if os.path.isdir(pth): result.write('D%s\n' % packetID) continue if not packetid.Valid(packetID): result.write('F%s %d\n' % (packetID, filesz)) continue customer, pathID, versionName, blockNum, supplierNum, dataORparity = packetid.SplitFull( packetID) if None in [ pathID, versionName, blockNum, supplierNum, dataORparity ]: result.write('F%s %d\n' % (packetID, filesz)) continue if dataORparity != 'Data' and dataORparity != 'Parity': result.write('F%s %d\n' % (packetID, filesz)) continue if maxBlock < blockNum: maxBlock = blockNum if supplierNum not in versionSize: versionSize[supplierNum] = 0 if supplierNum not in dataBlocks: dataBlocks[supplierNum] = {} if supplierNum not in parityBlocks: parityBlocks[supplierNum] = {} if dataORparity == 'Data': dataBlocks[supplierNum][blockNum] = filesz elif dataORparity == 'Parity': parityBlocks[supplierNum][blockNum] = filesz for supplierNum in versionSize.keys(): dataMissing[supplierNum] = set(range(maxBlock + 1)) parityMissing[supplierNum] = set(range(maxBlock + 1)) for blockNum in range(maxBlock + 1): if blockNum in list(dataBlocks[supplierNum].keys()): versionSize[supplierNum] += dataBlocks[supplierNum][ blockNum] dataMissing[supplierNum].discard(blockNum) if blockNum in list(parityBlocks[supplierNum].keys()): versionSize[supplierNum] += parityBlocks[supplierNum][ blockNum] parityMissing[supplierNum].discard(blockNum) suppliers = set(list(dataBlocks.keys()) + list(parityBlocks.keys())) for supplierNum in suppliers: versionString = '%s %d 0-%d %d' % (subpath, supplierNum, maxBlock, versionSize[supplierNum]) if len(dataMissing[supplierNum]) > 0 or len( parityMissing[supplierNum]) > 0: versionString += ' missing' if len(dataMissing[supplierNum]) > 0: versionString += ' Data:' + (','.join( map(str, dataMissing[supplierNum]))) if len(parityMissing[supplierNum]) > 0: versionString += ' Parity:' + (','.join( map(str, parityMissing[supplierNum]))) result.write('V%s\n' % versionString) del dataBlocks del parityBlocks del dataMissing del parityMissing return False bpio.traverse_dir_recursive( lambda realpath, subpath, name: cb(out, realpath, subpath, name), ownerdir) src = out.getvalue() out.close() return src
def TreeSummary(ownerdir): out = cStringIO.StringIO() def cb(result, realpath, subpath, name): if not os.access(realpath, os.R_OK): return False if os.path.isfile(realpath): try: filesz = os.path.getsize(realpath) except: filesz = -1 result.write("F%s %d\n" % (subpath, filesz)) return False if not packetid.IsCanonicalVersion(name): result.write("D%s\n" % subpath) return True maxBlock = -1 versionSize = {} dataBlocks = {} parityBlocks = {} dataMissing = {} parityMissing = {} for filename in os.listdir(realpath): packetID = subpath + "/" + filename pth = os.path.join(realpath, filename) try: filesz = os.path.getsize(pth) except: filesz = -1 if os.path.isdir(pth): result.write("D%s\n" % packetID) continue if not packetid.Valid(packetID): result.write("F%s %d\n" % (packetID, filesz)) continue pathID, versionName, blockNum, supplierNum, dataORparity = packetid.SplitFull(packetID) if None in [pathID, versionName, blockNum, supplierNum, dataORparity]: result.write("F%s %d\n" % (packetID, filesz)) continue if dataORparity != "Data" and dataORparity != "Parity": result.write("F%s %d\n" % (packetID, filesz)) continue if maxBlock < blockNum: maxBlock = blockNum if supplierNum not in versionSize: versionSize[supplierNum] = 0 if supplierNum not in dataBlocks: dataBlocks[supplierNum] = {} if supplierNum not in parityBlocks: parityBlocks[supplierNum] = {} if dataORparity == "Data": dataBlocks[supplierNum][blockNum] = filesz elif dataORparity == "Parity": parityBlocks[supplierNum][blockNum] = filesz for supplierNum in versionSize.keys(): dataMissing[supplierNum] = set(range(maxBlock + 1)) parityMissing[supplierNum] = set(range(maxBlock + 1)) for blockNum in range(maxBlock + 1): if blockNum in dataBlocks[supplierNum].keys(): versionSize[supplierNum] += dataBlocks[supplierNum][blockNum] dataMissing[supplierNum].discard(blockNum) if blockNum in parityBlocks[supplierNum].keys(): versionSize[supplierNum] += parityBlocks[supplierNum][blockNum] parityMissing[supplierNum].discard(blockNum) suppliers = set(dataBlocks.keys() + parityBlocks.keys()) for supplierNum in suppliers: versionString = "%s %d 0-%d %d" % (subpath, supplierNum, maxBlock, versionSize[supplierNum]) if len(dataMissing[supplierNum]) > 0 or len(parityMissing[supplierNum]) > 0: versionString += " missing" if len(dataMissing[supplierNum]) > 0: versionString += " Data:" + (",".join(map(str, dataMissing[supplierNum]))) if len(parityMissing[supplierNum]) > 0: versionString += " Parity:" + (",".join(map(str, parityMissing[supplierNum]))) result.write("V%s\n" % versionString) del dataBlocks del parityBlocks del dataMissing del parityMissing return False bpio.traverse_dir_recursive(lambda realpath, subpath, name: cb(out, realpath, subpath, name), ownerdir) src = out.getvalue() out.close() return src
def SpaceTime(): """ Test all packets for each customer. Check if he use more space than we gave him and if packets is too old. """ printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000"))) space = accounting.read_customers_quotas() if space is None: printlog( 'SpaceTime ERROR customers quotas file can not be read or it is empty, skip' ) return customers_dir = settings.getCustomersFilesDir() if not os.path.exists(customers_dir): printlog('SpaceTime ERROR customers folder not exist') return remove_list = {} used_space = accounting.read_customers_usage() for customer_filename in os.listdir(customers_dir): onecustdir = os.path.join(customers_dir, customer_filename) if not os.path.isdir(onecustdir): remove_list[onecustdir] = 'is not a folder' continue # idurl = nameurl.FilenameUrl(customer_filename) idurl = global_id.GlobalUserToIDURL(customer_filename) if idurl is None: remove_list[onecustdir] = 'wrong folder name' continue curspace = space.get(idurl, None) if curspace is None: remove_list[onecustdir] = 'not found in space file' continue try: maxspaceV = int(curspace) except: remove_list[onecustdir] = 'wrong space value' continue timedict = {} sizedict = {} def cb(path, subpath, name): if not os.path.isfile(path): return True stats = os.stat(path) timedict[path] = stats.st_ctime sizedict[path] = stats.st_size for key_alias in os.listdir(onecustdir): if not misc.ValidKeyAlias(key_alias): remove_list[onecustdir] = 'invalid key alias' continue okekeydir = os.path.join(onecustdir, key_alias) bpio.traverse_dir_recursive(cb, okekeydir) currentV = 0 for path in sorted(list(timedict.keys()), key=lambda x: timedict[x], reverse=True): filesize = sizedict.get(path, 0) currentV += filesize if currentV < maxspaceV: continue try: os.remove(path) printlog('SpaceTime ' + path + ' file removed (cur:%s, max: %s)' % (str(currentV), str(maxspaceV))) except: printlog('SpaceTime ERROR removing ' + path) # time.sleep(0.01) used_space[idurl] = str(currentV) timedict.clear() sizedict.clear() for path in remove_list.keys(): if not os.path.exists(path): continue if os.path.isdir(path): try: bpio._dir_remove(path) printlog('SpaceTime ' + path + ' dir removed (%s)' % (remove_list[path])) except: printlog('SpaceTime ERROR removing ' + path) continue try: if not os.access(path, os.W_OK): os.chmod(path, 0o600) except: pass try: os.remove(path) printlog('SpaceTime ' + path + ' file removed (%s)' % (remove_list[path])) except: printlog('SpaceTime ERROR removing ' + path) del remove_list accounting.update_customers_usage(used_space)
def SpaceTime(): """ Test all packets for each customer. Check if he use more space than we gave him and if packets is too old. """ printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000"))) space = bpio._read_dict(settings.CustomersSpaceFile()) if space is None: printlog('SpaceTime ERROR can not read file ' + settings.CustomersSpaceFile()) return customers_dir = settings.getCustomersFilesDir() if not os.path.exists(customers_dir): printlog('SpaceTime ERROR customers folder not exist') return remove_list = {} used_space = {} for customer_filename in os.listdir(customers_dir): onecustdir = os.path.join(customers_dir, customer_filename) if not os.path.isdir(onecustdir): remove_list[onecustdir] = 'is not a folder' continue idurl = nameurl.FilenameUrl(customer_filename) if idurl is None: remove_list[onecustdir] = 'wrong folder name' continue curspace = space.get(idurl, None) if curspace is None: remove_list[onecustdir] = 'not found in space file' continue try: maxspaceV = int(curspace) except: remove_list[onecustdir] = 'wrong space value' continue timedict = {} sizedict = {} def cb(path, subpath, name): # if not os.access(path, os.R_OK | os.W_OK): # return False if not os.path.isfile(path): return True # if name in [settings.BackupIndexFileName(),]: # return False stats = os.stat(path) timedict[path] = stats.st_ctime sizedict[path] = stats.st_size bpio.traverse_dir_recursive(cb, onecustdir) currentV = 0 for path in sorted(timedict.keys(), key=lambda x: timedict[x], reverse=True): filesize = sizedict.get(path, 0) currentV += filesize if currentV < maxspaceV: continue try: os.remove(path) printlog('SpaceTime ' + path + ' file removed (cur:%s, max: %s)' % (str(currentV), str(maxspaceV))) except: printlog('SpaceTime ERROR removing ' + path) # time.sleep(0.01) used_space[idurl] = str(currentV) timedict.clear() sizedict.clear() for path in remove_list.keys(): if not os.path.exists(path): continue if os.path.isdir(path): try: bpio._dir_remove(path) printlog('SpaceTime ' + path + ' dir removed (%s)' % (remove_list[path])) except: printlog('SpaceTime ERROR removing ' + path) continue try: if not os.access(path, os.W_OK): os.chmod(path, 0o600) except: pass try: os.remove(path) printlog('SpaceTime ' + path + ' file removed (%s)' % (remove_list[path])) except: printlog('SpaceTime ERROR removing ' + path) del remove_list bpio._write_dict(settings.CustomersUsedSpaceFile(), used_space)