def ListCustomerFiles(customer_idurl):
filename = nameurl.UrlFilename(customer_idurl)
customer_dir = os.path.join(settings.getCustomersFilesDir(), filename)
result = cStringIO.StringIO()
def cb(realpath, subpath, name):
if os.path.isdir(realpath):
result.write('D%s\n' % subpath)
else:
result.write('F%s\n' % subpath)
return True
dhnio.traverse_dir_recursive(cb, customer_dir)
src = result.getvalue()
result.close()
return src
def Validate():
printlog("Validate " + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
contacts_init()
commands_init()
customers_dir = getCustomersFilesDir()
if not os.path.exists(customers_dir):
return
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
continue
def cb(path, subpath, name):
if not os.access(path, os.R_OK | os.W_OK):
return False
if not os.path.isfile(path):
return True
if name in [BackupIndexFileName()]:
return False
packetsrc = dhnio.ReadBinaryFile(path)
if not packetsrc:
try:
os.remove(path) # if is is no good it is of no use to anyone
printlog("Validate " + path + " removed (empty file)")
except:
printlog("Validate ERROR removing " + path)
return False
packet = Unserialize(packetsrc)
if packet is None:
try:
os.remove(path) # if is is no good it is of no use to anyone
printlog("Validate " + path + " removed (unserialize error)")
except:
printlog("Validate ERROR removing " + path)
return False
result = packet.Valid()
packetsrc = ""
del packet
if not result:
try:
os.remove(path) # if is is no good it is of no use to anyone
printlog("Validate " + path + " removed (invalid packet)")
except:
printlog("Validate ERROR removing " + path)
return False
time.sleep(0.1)
return False
dhnio.traverse_dir_recursive(cb, onecustdir)
def ReadLocalFiles():
"""
This method scans local backups and build the whole "local" matrix.
"""
global _LocalFilesNotifyCallback
local_files().clear()
local_max_block_numbers().clear()
local_backup_size().clear()
_counter = [0,]
def visit(realpath, subpath, name):
# subpath is something like 0/0/1/0/F20131120053803PM/0-1-Data
if not os.path.isfile(realpath):
return True
if realpath.startswith('newblock-'):
return False
if subpath in [ settings.BackupIndexFileName(), settings.BackupInfoFileName(), settings.BackupInfoFileNameOld(), settings.BackupInfoEncryptedFileName() ]:
return False
try:
version = subpath.split('/')[-2]
except:
return False
if not packetid.IsCanonicalVersion(version):
return True
LocalFileReport(packetID=subpath)
_counter[0] += 1
return False
dhnio.traverse_dir_recursive(visit, settings.getLocalBackupsDir())
dhnio.Dprint(8, 'backup_matrix.ReadLocalFiles %d files indexed' % _counter[0])
if dhnio.Debug(8):
try:
if sys.version_info >= (2, 6):
#localSZ = sys.getsizeof(local_files())
#remoteSZ = sys.getsizeof(remote_files())
import lib.getsizeof
localSZ = lib.getsizeof.total_size(local_files())
remoteSZ = lib.getsizeof.total_size(remote_files())
indexByName = lib.getsizeof.total_size(backup_fs.fs())
indexByID = lib.getsizeof.total_size(backup_fs.fsID())
dhnio.Dprint(10, ' all local info uses %d bytes in the memory' % localSZ)
dhnio.Dprint(10, ' all remote info uses %d bytes in the memory' % remoteSZ)
dhnio.Dprint(10, ' index by name takes %d bytes in the memory' % indexByName)
dhnio.Dprint(10, ' index by ID takes %d bytes in the memory' % indexByID)
except:
dhnio.DprintException()
if _LocalFilesNotifyCallback is not None:
_LocalFilesNotifyCallback()
def TreeSummary(ownerdir):
out = cStringIO.StringIO()
def cb(result, realpath, subpath, name):
if not os.access(realpath, os.R_OK):
return False
if os.path.isfile(realpath):
result.write('F%s\n' % subpath)
return False
if not packetid.IsCanonicalVersion(name):
result.write('D%s\n' % subpath)
return True
maxBlock = -1
dataBlocks = {}
parityBlocks = {}
dataMissing = {}
parityMissing = {}
for filename in os.listdir(realpath):
packetID = subpath + '/' + filename
pth = os.path.join(realpath, filename)
if os.path.isdir(pth):
result.write('D%s\n' % packetID)
continue
if not packetid.Valid(packetID):
result.write('F%s\n' % packetID)
continue
pathID, versionName, blockNum, supplierNum, dataORparity = packetid.SplitFull(packetID)
if None in [pathID, versionName, blockNum, supplierNum, dataORparity]:
result.write('F%s\n' % packetID)
continue
if dataORparity == 'Data':
if not dataBlocks.has_key(supplierNum):
dataBlocks[supplierNum] = set()
dataMissing[supplierNum] = []
dataBlocks[supplierNum].add(blockNum)
elif dataORparity == 'Parity':
if not parityBlocks.has_key(supplierNum):
parityBlocks[supplierNum] = set()
parityMissing[supplierNum] = []
parityBlocks[supplierNum].add(blockNum)
else:
result.write('F%s\n' % packetID)
continue
if maxBlock < blockNum:
maxBlock = blockNum
for blockNum in range(maxBlock+1):
for supplierNum in dataBlocks.keys():
if not blockNum in dataBlocks[supplierNum]:
dataMissing[supplierNum].append(str(blockNum))
for supplierNum in parityBlocks.keys():
if not blockNum in parityBlocks[supplierNum]:
parityMissing[supplierNum].append(str(blockNum))
suppliers = set(dataBlocks.keys() + parityBlocks.keys())
for supplierNum in suppliers:
versionString = '%s %d 0-%d' % (subpath, supplierNum, maxBlock)
dataMiss = []
parityMiss = []
if dataMissing.has_key(supplierNum):
dataMiss = dataMissing[supplierNum]
if parityMissing.has_key(supplierNum):
parityMiss = parityMissing[supplierNum]
if len(dataMiss) > 0 or len(parityMiss) > 0:
versionString += ' missing'
if len(dataMiss) > 0:
versionString += ' Data:' + (','.join(dataMiss))
if len(parityMiss) > 0:
versionString += ' Parity:' + (','.join(parityMiss))
del dataMiss
del parityMiss
result.write('V%s\n' % versionString)
del dataBlocks
del parityBlocks
del dataMissing
del parityMissing
return False
dhnio.traverse_dir_recursive(lambda realpath, subpath, name: cb(out, realpath, subpath, name), ownerdir)
src = out.getvalue()
out.close()
return src
def SpaceTime():
printlog("SpaceTime " + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
space = dhnio._read_dict(CustomersSpaceFile())
if space is None:
printlog("SpaceTime ERROR can not read file " + CustomersSpaceFile())
return
customers_dir = getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog("SpaceTime ERROR customers folder not exist")
return
remove_list = {}
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = "is not a folder"
continue
idurl = FilenameUrl(customer_filename)
if idurl is None:
remove_list[onecustdir] = "wrong folder name"
continue
curspace = space.get(idurl, None)
if curspace is None:
continue
try:
maxspaceV = int(float(curspace) * 1024 * 1024) # in bytes
except:
remove_list[onecustdir] = "wrong space value"
continue
timedict = {}
sizedict = {}
def cb(path, subpath, name):
if not os.access(path, os.R_OK | os.W_OK):
return False
if not os.path.isfile(path):
return True
if name in [BackupIndexFileName()]:
return False
stats = os.stat(path)
timedict[path] = stats.st_ctime
sizedict[path] = stats.st_size
dhnio.traverse_dir_recursive(cb, onecustdir)
currentV = 0
for path in sorted(timedict.keys(), key=lambda x: timedict[x], reverse=True):
currentV += sizedict.get(path, 0)
if currentV < maxspaceV:
continue
try:
os.remove(path)
printlog("SpaceTime " + path + " file removed (cur:%s, max: %s)" % (str(currentV), str(maxspaceV)))
except:
printlog("SpaceTime ERROR removing " + path)
time.sleep(0.1)
timedict.clear()
sizedict.clear()
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
dhnio._dir_remove(path)
printlog("SpaceTime " + path + " dir removed (%s)" % (remove_list[path]))
except:
printlog("SpaceTime ERROR removing " + path)
continue
if not os.access(path, os.W_OK):
os.chmod(path, 0600)
try:
os.remove(path)
printlog("SpaceTime " + path + " file removed (%s)" % (remove_list[path]))
except:
printlog("SpaceTime ERROR removing " + path)
del remove_list
