def Validate():
"""
Check all packets to be valid.
"""
printlog('Validate %r' % time.strftime("%a, %d %b %Y %H:%M:%S +0000"))
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
return False
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
continue
for key_alias_filename in os.listdir(onecustdir):
onekeydir = os.path.join(onecustdir, key_alias_filename)
if not os.path.isdir(onekeydir):
continue
def cb(path, subpath, name):
if not os.path.isfile(path):
return True
packetsrc = bpio.ReadBinaryFile(path)
if not packetsrc:
try:
os.remove(
path) # if is is no good it is of no use to anyone
printlog('Validate %r removed (empty file)' % path)
except:
printlog('Validate ERROR removing %r' % path)
return False
p = signed.Unserialize(packetsrc)
if p is None:
try:
os.remove(
path) # if is is no good it is of no use to anyone
printlog('Validate %r removed (unserialize error)' %
path)
except:
printlog('Validate ERROR removing %r')
return False
result = p.Valid()
packetsrc = ''
del p
if not result:
try:
os.remove(
path) # if is is no good it is of no use to anyone
printlog('Validate %r removed (invalid packet)' % path)
except:
printlog('Validate ERROR removing %r' % path)
return False
time.sleep(0.1)
return False
bpio.traverse_dir_recursive(cb, onekeydir)
return True
def Validate():
"""
Check all packets to be valid.
"""
printlog('Validate ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
contactsdb.init()
commands.init()
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
return
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
continue
for key_alias_filename in os.listdir(onecustdir):
onekeydir = os.path.join(onecustdir, key_alias_filename)
if not os.path.isdir(onekeydir):
continue
def cb(path, subpath, name):
# if not os.access(path, os.R_OK | os.W_OK):
# return False
if not os.path.isfile(path):
return True
# if name in [settings.BackupIndexFileName(),]:
# return False
packetsrc = bpio.ReadBinaryFile(path)
if not packetsrc:
try:
os.remove(path) # if is is no good it is of no use to anyone
printlog('Validate ' + path + ' removed (empty file)')
except:
printlog('Validate ERROR removing ' + path)
return False
p = signed.Unserialize(packetsrc)
if p is None:
try:
os.remove(path) # if is is no good it is of no use to anyone
printlog('Validate ' + path + ' removed (unserialize error)')
except:
printlog('Validate ERROR removing ' + path)
return False
result = p.Valid()
packetsrc = ''
del p
if not result:
try:
os.remove(path) # if is is no good it is of no use to anyone
printlog('Validate ' + path + ' removed (invalid packet)')
except:
printlog('Validate ERROR removing ' + path)
return False
time.sleep(0.1)
return False
bpio.traverse_dir_recursive(cb, onekeydir)
def ReadLocalFiles():
"""
This method scans local backups and build the whole "local" matrix.
"""
global _LocalFilesNotifyCallback
local_files().clear()
local_max_block_numbers().clear()
local_backup_size().clear()
_counter = [0]
def visit(realpath, subpath, name):
# subpath is something like 0/0/1/0/F20131120053803PM/0-1-Data
if not os.path.isfile(realpath):
return True
if realpath.startswith("newblock-"):
return False
if subpath in [
settings.BackupIndexFileName(),
settings.BackupInfoFileName(),
settings.BackupInfoFileNameOld(),
settings.BackupInfoEncryptedFileName(),
]:
return False
try:
version = subpath.split("/")[-2]
except:
return False
if not packetid.IsCanonicalVersion(version):
return True
LocalFileReport(packetID=subpath)
_counter[0] += 1
return False
bpio.traverse_dir_recursive(visit, settings.getLocalBackupsDir())
lg.out(8, "backup_matrix.ReadLocalFiles %d files indexed" % _counter[0])
if lg.is_debug(8):
try:
if sys.version_info >= (2, 6):
# localSZ = sys.getsizeof(local_files())
# remoteSZ = sys.getsizeof(remote_files())
import lib.getsizeof
localSZ = lib.getsizeof.total_size(local_files())
remoteSZ = lib.getsizeof.total_size(remote_files())
indexByName = lib.getsizeof.total_size(backup_fs.fs())
indexByID = lib.getsizeof.total_size(backup_fs.fsID())
lg.out(10, " all local info uses %d bytes in the memory" % localSZ)
lg.out(10, " all remote info uses %d bytes in the memory" % remoteSZ)
lg.out(10, " index by name takes %d bytes in the memory" % indexByName)
lg.out(10, " index by ID takes %d bytes in the memory" % indexByID)
except:
lg.exc()
if _LocalFilesNotifyCallback is not None:
_LocalFilesNotifyCallback()
def TraverseLocalFileSystem(basedir, expanded_dirs, callback):
def cb(realpath, subpath, name):
if not os.access(realpath, os.R_OK):
return False
if os.path.isfile(realpath):
callback(realpath, subpath, name)
return False
if subpath not in expanded_dirs:
callback(realpath, subpath, name)
return False
bpio.traverse_dir_recursive(cb, basedir)
def ListCustomerFiles(customer_idurl):
filename = nameurl.UrlFilename(customer_idurl)
customer_dir = os.path.join(settings.getCustomersFilesDir(), filename)
result = cStringIO.StringIO()
def cb(realpath, subpath, name):
if os.path.isdir(realpath):
result.write("D%s\n" % subpath)
else:
result.write("F%s\n" % subpath)
return True
bpio.traverse_dir_recursive(cb, customer_dir)
src = result.getvalue()
result.close()
return src
def ListCustomerFiles(customer_idurl):
filename = nameurl.UrlFilename(customer_idurl)
customer_dir = os.path.join(settings.getCustomersFilesDir(), filename)
result = cStringIO.StringIO()
def cb(realpath, subpath, name):
if os.path.isdir(realpath):
result.write('D%s\n' % subpath)
else:
result.write('F%s\n' % subpath)
return True
bpio.traverse_dir_recursive(cb, customer_dir)
src = result.getvalue()
result.close()
return src
def TreeSummary(ownerdir, key_alias):
out = StringIO()
out.write('K%s\n' % key_alias)
def cb(result, realpath, subpath, name):
if not os.access(realpath, os.R_OK):
return False
if os.path.isfile(realpath):
try:
filesz = os.path.getsize(realpath)
except:
filesz = -1
result.write('F%s %d\n' % (subpath, filesz))
return False
if not packetid.IsCanonicalVersion(name):
result.write('D%s\n' % subpath)
return True
maxBlock = -1
versionSize = {}
dataBlocks = {}
parityBlocks = {}
dataMissing = {}
parityMissing = {}
for filename in os.listdir(realpath):
packetID = subpath + '/' + filename
pth = os.path.join(realpath, filename)
try:
filesz = os.path.getsize(pth)
except:
filesz = -1
if os.path.isdir(pth):
result.write('D%s\n' % packetID)
continue
if not packetid.Valid(packetID):
result.write('F%s %d\n' % (packetID, filesz))
continue
customer, pathID, versionName, blockNum, supplierNum, dataORparity = packetid.SplitFull(
packetID)
if None in [
pathID, versionName, blockNum, supplierNum, dataORparity
]:
result.write('F%s %d\n' % (packetID, filesz))
continue
if dataORparity != 'Data' and dataORparity != 'Parity':
result.write('F%s %d\n' % (packetID, filesz))
continue
if maxBlock < blockNum:
maxBlock = blockNum
if supplierNum not in versionSize:
versionSize[supplierNum] = 0
if supplierNum not in dataBlocks:
dataBlocks[supplierNum] = {}
if supplierNum not in parityBlocks:
parityBlocks[supplierNum] = {}
if dataORparity == 'Data':
dataBlocks[supplierNum][blockNum] = filesz
elif dataORparity == 'Parity':
parityBlocks[supplierNum][blockNum] = filesz
for supplierNum in versionSize.keys():
dataMissing[supplierNum] = set(range(maxBlock + 1))
parityMissing[supplierNum] = set(range(maxBlock + 1))
for blockNum in range(maxBlock + 1):
if blockNum in list(dataBlocks[supplierNum].keys()):
versionSize[supplierNum] += dataBlocks[supplierNum][
blockNum]
dataMissing[supplierNum].discard(blockNum)
if blockNum in list(parityBlocks[supplierNum].keys()):
versionSize[supplierNum] += parityBlocks[supplierNum][
blockNum]
parityMissing[supplierNum].discard(blockNum)
suppliers = set(list(dataBlocks.keys()) + list(parityBlocks.keys()))
for supplierNum in suppliers:
versionString = '%s %d 0-%d %d' % (subpath, supplierNum, maxBlock,
versionSize[supplierNum])
if len(dataMissing[supplierNum]) > 0 or len(
parityMissing[supplierNum]) > 0:
versionString += ' missing'
if len(dataMissing[supplierNum]) > 0:
versionString += ' Data:' + (','.join(
map(str, dataMissing[supplierNum])))
if len(parityMissing[supplierNum]) > 0:
versionString += ' Parity:' + (','.join(
map(str, parityMissing[supplierNum])))
result.write('V%s\n' % versionString)
del dataBlocks
del parityBlocks
del dataMissing
del parityMissing
return False
bpio.traverse_dir_recursive(
lambda realpath, subpath, name: cb(out, realpath, subpath, name),
ownerdir)
src = out.getvalue()
out.close()
return src
def TreeSummary(ownerdir):
out = cStringIO.StringIO()
def cb(result, realpath, subpath, name):
if not os.access(realpath, os.R_OK):
return False
if os.path.isfile(realpath):
try:
filesz = os.path.getsize(realpath)
except:
filesz = -1
result.write("F%s %d\n" % (subpath, filesz))
return False
if not packetid.IsCanonicalVersion(name):
result.write("D%s\n" % subpath)
return True
maxBlock = -1
versionSize = {}
dataBlocks = {}
parityBlocks = {}
dataMissing = {}
parityMissing = {}
for filename in os.listdir(realpath):
packetID = subpath + "/" + filename
pth = os.path.join(realpath, filename)
try:
filesz = os.path.getsize(pth)
except:
filesz = -1
if os.path.isdir(pth):
result.write("D%s\n" % packetID)
continue
if not packetid.Valid(packetID):
result.write("F%s %d\n" % (packetID, filesz))
continue
pathID, versionName, blockNum, supplierNum, dataORparity = packetid.SplitFull(packetID)
if None in [pathID, versionName, blockNum, supplierNum, dataORparity]:
result.write("F%s %d\n" % (packetID, filesz))
continue
if dataORparity != "Data" and dataORparity != "Parity":
result.write("F%s %d\n" % (packetID, filesz))
continue
if maxBlock < blockNum:
maxBlock = blockNum
if supplierNum not in versionSize:
versionSize[supplierNum] = 0
if supplierNum not in dataBlocks:
dataBlocks[supplierNum] = {}
if supplierNum not in parityBlocks:
parityBlocks[supplierNum] = {}
if dataORparity == "Data":
dataBlocks[supplierNum][blockNum] = filesz
elif dataORparity == "Parity":
parityBlocks[supplierNum][blockNum] = filesz
for supplierNum in versionSize.keys():
dataMissing[supplierNum] = set(range(maxBlock + 1))
parityMissing[supplierNum] = set(range(maxBlock + 1))
for blockNum in range(maxBlock + 1):
if blockNum in dataBlocks[supplierNum].keys():
versionSize[supplierNum] += dataBlocks[supplierNum][blockNum]
dataMissing[supplierNum].discard(blockNum)
if blockNum in parityBlocks[supplierNum].keys():
versionSize[supplierNum] += parityBlocks[supplierNum][blockNum]
parityMissing[supplierNum].discard(blockNum)
suppliers = set(dataBlocks.keys() + parityBlocks.keys())
for supplierNum in suppliers:
versionString = "%s %d 0-%d %d" % (subpath, supplierNum, maxBlock, versionSize[supplierNum])
if len(dataMissing[supplierNum]) > 0 or len(parityMissing[supplierNum]) > 0:
versionString += " missing"
if len(dataMissing[supplierNum]) > 0:
versionString += " Data:" + (",".join(map(str, dataMissing[supplierNum])))
if len(parityMissing[supplierNum]) > 0:
versionString += " Parity:" + (",".join(map(str, parityMissing[supplierNum])))
result.write("V%s\n" % versionString)
del dataBlocks
del parityBlocks
del dataMissing
del parityMissing
return False
bpio.traverse_dir_recursive(lambda realpath, subpath, name: cb(out, realpath, subpath, name), ownerdir)
src = out.getvalue()
out.close()
return src
def SpaceTime():
"""
Test all packets for each customer.
Check if he use more space than we gave him and if packets is too
old.
"""
printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
space = accounting.read_customers_quotas()
if space is None:
printlog(
'SpaceTime ERROR customers quotas file can not be read or it is empty, skip'
)
return
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog('SpaceTime ERROR customers folder not exist')
return
remove_list = {}
used_space = accounting.read_customers_usage()
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
# idurl = nameurl.FilenameUrl(customer_filename)
idurl = global_id.GlobalUserToIDURL(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl, None)
if curspace is None:
remove_list[onecustdir] = 'not found in space file'
continue
try:
maxspaceV = int(curspace)
except:
remove_list[onecustdir] = 'wrong space value'
continue
timedict = {}
sizedict = {}
def cb(path, subpath, name):
if not os.path.isfile(path):
return True
stats = os.stat(path)
timedict[path] = stats.st_ctime
sizedict[path] = stats.st_size
for key_alias in os.listdir(onecustdir):
if not misc.ValidKeyAlias(key_alias):
remove_list[onecustdir] = 'invalid key alias'
continue
okekeydir = os.path.join(onecustdir, key_alias)
bpio.traverse_dir_recursive(cb, okekeydir)
currentV = 0
for path in sorted(list(timedict.keys()),
key=lambda x: timedict[x],
reverse=True):
filesize = sizedict.get(path, 0)
currentV += filesize
if currentV < maxspaceV:
continue
try:
os.remove(path)
printlog('SpaceTime ' + path +
' file removed (cur:%s, max: %s)' %
(str(currentV), str(maxspaceV)))
except:
printlog('SpaceTime ERROR removing ' + path)
# time.sleep(0.01)
used_space[idurl] = str(currentV)
timedict.clear()
sizedict.clear()
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
printlog('SpaceTime ' + path + ' dir removed (%s)' %
(remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
printlog('SpaceTime ' + path + ' file removed (%s)' %
(remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
del remove_list
accounting.update_customers_usage(used_space)
def SpaceTime():
"""
Test all packets for each customer.
Check if he use more space than we gave him and if packets is too
old.
"""
printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
space = bpio._read_dict(settings.CustomersSpaceFile())
if space is None:
printlog('SpaceTime ERROR can not read file ' + settings.CustomersSpaceFile())
return
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog('SpaceTime ERROR customers folder not exist')
return
remove_list = {}
used_space = {}
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
idurl = nameurl.FilenameUrl(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl, None)
if curspace is None:
remove_list[onecustdir] = 'not found in space file'
continue
try:
maxspaceV = int(curspace)
except:
remove_list[onecustdir] = 'wrong space value'
continue
timedict = {}
sizedict = {}
def cb(path, subpath, name):
# if not os.access(path, os.R_OK | os.W_OK):
# return False
if not os.path.isfile(path):
return True
# if name in [settings.BackupIndexFileName(),]:
# return False
stats = os.stat(path)
timedict[path] = stats.st_ctime
sizedict[path] = stats.st_size
bpio.traverse_dir_recursive(cb, onecustdir)
currentV = 0
for path in sorted(timedict.keys(), key=lambda x: timedict[x], reverse=True):
filesize = sizedict.get(path, 0)
currentV += filesize
if currentV < maxspaceV:
continue
try:
os.remove(path)
printlog('SpaceTime ' + path + ' file removed (cur:%s, max: %s)' % (str(currentV), str(maxspaceV)))
except:
printlog('SpaceTime ERROR removing ' + path)
# time.sleep(0.01)
used_space[idurl] = str(currentV)
timedict.clear()
sizedict.clear()
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
printlog('SpaceTime ' + path + ' dir removed (%s)' % (remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
printlog('SpaceTime ' + path + ' file removed (%s)' % (remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
del remove_list
bpio._write_dict(settings.CustomersUsedSpaceFile(), used_space)
