def ListCustomerFiles(customer_idurl): filename = nameurl.UrlFilename(customer_idurl) customer_dir = os.path.join(settings.getCustomersFilesDir(), filename) result = cStringIO.StringIO() def cb(realpath, subpath, name): if os.path.isdir(realpath): result.write('D%s\n' % subpath) else: result.write('F%s\n' % subpath) return True dhnio.traverse_dir_recursive(cb, customer_dir) src = result.getvalue() result.close() return src
def Validate(): printlog("Validate " + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000"))) contacts_init() commands_init() customers_dir = getCustomersFilesDir() if not os.path.exists(customers_dir): return for customer_filename in os.listdir(customers_dir): onecustdir = os.path.join(customers_dir, customer_filename) if not os.path.isdir(onecustdir): continue def cb(path, subpath, name): if not os.access(path, os.R_OK | os.W_OK): return False if not os.path.isfile(path): return True if name in [BackupIndexFileName()]: return False packetsrc = dhnio.ReadBinaryFile(path) if not packetsrc: try: os.remove(path) # if is is no good it is of no use to anyone printlog("Validate " + path + " removed (empty file)") except: printlog("Validate ERROR removing " + path) return False packet = Unserialize(packetsrc) if packet is None: try: os.remove(path) # if is is no good it is of no use to anyone printlog("Validate " + path + " removed (unserialize error)") except: printlog("Validate ERROR removing " + path) return False result = packet.Valid() packetsrc = "" del packet if not result: try: os.remove(path) # if is is no good it is of no use to anyone printlog("Validate " + path + " removed (invalid packet)") except: printlog("Validate ERROR removing " + path) return False time.sleep(0.1) return False dhnio.traverse_dir_recursive(cb, onecustdir)
def ReadLocalFiles(): """ This method scans local backups and build the whole "local" matrix. """ global _LocalFilesNotifyCallback local_files().clear() local_max_block_numbers().clear() local_backup_size().clear() _counter = [0,] def visit(realpath, subpath, name): # subpath is something like 0/0/1/0/F20131120053803PM/0-1-Data if not os.path.isfile(realpath): return True if realpath.startswith('newblock-'): return False if subpath in [ settings.BackupIndexFileName(), settings.BackupInfoFileName(), settings.BackupInfoFileNameOld(), settings.BackupInfoEncryptedFileName() ]: return False try: version = subpath.split('/')[-2] except: return False if not packetid.IsCanonicalVersion(version): return True LocalFileReport(packetID=subpath) _counter[0] += 1 return False dhnio.traverse_dir_recursive(visit, settings.getLocalBackupsDir()) dhnio.Dprint(8, 'backup_matrix.ReadLocalFiles %d files indexed' % _counter[0]) if dhnio.Debug(8): try: if sys.version_info >= (2, 6): #localSZ = sys.getsizeof(local_files()) #remoteSZ = sys.getsizeof(remote_files()) import lib.getsizeof localSZ = lib.getsizeof.total_size(local_files()) remoteSZ = lib.getsizeof.total_size(remote_files()) indexByName = lib.getsizeof.total_size(backup_fs.fs()) indexByID = lib.getsizeof.total_size(backup_fs.fsID()) dhnio.Dprint(10, ' all local info uses %d bytes in the memory' % localSZ) dhnio.Dprint(10, ' all remote info uses %d bytes in the memory' % remoteSZ) dhnio.Dprint(10, ' index by name takes %d bytes in the memory' % indexByName) dhnio.Dprint(10, ' index by ID takes %d bytes in the memory' % indexByID) except: dhnio.DprintException() if _LocalFilesNotifyCallback is not None: _LocalFilesNotifyCallback()
def TreeSummary(ownerdir): out = cStringIO.StringIO() def cb(result, realpath, subpath, name): if not os.access(realpath, os.R_OK): return False if os.path.isfile(realpath): result.write('F%s\n' % subpath) return False if not packetid.IsCanonicalVersion(name): result.write('D%s\n' % subpath) return True maxBlock = -1 dataBlocks = {} parityBlocks = {} dataMissing = {} parityMissing = {} for filename in os.listdir(realpath): packetID = subpath + '/' + filename pth = os.path.join(realpath, filename) if os.path.isdir(pth): result.write('D%s\n' % packetID) continue if not packetid.Valid(packetID): result.write('F%s\n' % packetID) continue pathID, versionName, blockNum, supplierNum, dataORparity = packetid.SplitFull(packetID) if None in [pathID, versionName, blockNum, supplierNum, dataORparity]: result.write('F%s\n' % packetID) continue if dataORparity == 'Data': if not dataBlocks.has_key(supplierNum): dataBlocks[supplierNum] = set() dataMissing[supplierNum] = [] dataBlocks[supplierNum].add(blockNum) elif dataORparity == 'Parity': if not parityBlocks.has_key(supplierNum): parityBlocks[supplierNum] = set() parityMissing[supplierNum] = [] parityBlocks[supplierNum].add(blockNum) else: result.write('F%s\n' % packetID) continue if maxBlock < blockNum: maxBlock = blockNum for blockNum in range(maxBlock+1): for supplierNum in dataBlocks.keys(): if not blockNum in dataBlocks[supplierNum]: dataMissing[supplierNum].append(str(blockNum)) for supplierNum in parityBlocks.keys(): if not blockNum in parityBlocks[supplierNum]: parityMissing[supplierNum].append(str(blockNum)) suppliers = set(dataBlocks.keys() + parityBlocks.keys()) for supplierNum in suppliers: versionString = '%s %d 0-%d' % (subpath, supplierNum, maxBlock) dataMiss = [] parityMiss = [] if dataMissing.has_key(supplierNum): dataMiss = dataMissing[supplierNum] if parityMissing.has_key(supplierNum): parityMiss = parityMissing[supplierNum] if len(dataMiss) > 0 or len(parityMiss) > 0: versionString += ' missing' if len(dataMiss) > 0: versionString += ' Data:' + (','.join(dataMiss)) if len(parityMiss) > 0: versionString += ' Parity:' + (','.join(parityMiss)) del dataMiss del parityMiss result.write('V%s\n' % versionString) del dataBlocks del parityBlocks del dataMissing del parityMissing return False dhnio.traverse_dir_recursive(lambda realpath, subpath, name: cb(out, realpath, subpath, name), ownerdir) src = out.getvalue() out.close() return src
def SpaceTime(): printlog("SpaceTime " + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000"))) space = dhnio._read_dict(CustomersSpaceFile()) if space is None: printlog("SpaceTime ERROR can not read file " + CustomersSpaceFile()) return customers_dir = getCustomersFilesDir() if not os.path.exists(customers_dir): printlog("SpaceTime ERROR customers folder not exist") return remove_list = {} for customer_filename in os.listdir(customers_dir): onecustdir = os.path.join(customers_dir, customer_filename) if not os.path.isdir(onecustdir): remove_list[onecustdir] = "is not a folder" continue idurl = FilenameUrl(customer_filename) if idurl is None: remove_list[onecustdir] = "wrong folder name" continue curspace = space.get(idurl, None) if curspace is None: continue try: maxspaceV = int(float(curspace) * 1024 * 1024) # in bytes except: remove_list[onecustdir] = "wrong space value" continue timedict = {} sizedict = {} def cb(path, subpath, name): if not os.access(path, os.R_OK | os.W_OK): return False if not os.path.isfile(path): return True if name in [BackupIndexFileName()]: return False stats = os.stat(path) timedict[path] = stats.st_ctime sizedict[path] = stats.st_size dhnio.traverse_dir_recursive(cb, onecustdir) currentV = 0 for path in sorted(timedict.keys(), key=lambda x: timedict[x], reverse=True): currentV += sizedict.get(path, 0) if currentV < maxspaceV: continue try: os.remove(path) printlog("SpaceTime " + path + " file removed (cur:%s, max: %s)" % (str(currentV), str(maxspaceV))) except: printlog("SpaceTime ERROR removing " + path) time.sleep(0.1) timedict.clear() sizedict.clear() for path in remove_list.keys(): if not os.path.exists(path): continue if os.path.isdir(path): try: dhnio._dir_remove(path) printlog("SpaceTime " + path + " dir removed (%s)" % (remove_list[path])) except: printlog("SpaceTime ERROR removing " + path) continue if not os.access(path, os.W_OK): os.chmod(path, 0600) try: os.remove(path) printlog("SpaceTime " + path + " file removed (%s)" % (remove_list[path])) except: printlog("SpaceTime ERROR removing " + path) del remove_list