def _stats(params):
from contacts import contactsdb
from p2p import contact_status
from lib import diskspace
result = {}
result['suppliers'] = contactsdb.num_suppliers()
result['max_suppliers'] = settings.getSuppliersNumberDesired()
result['online_suppliers'] = contact_status.countOnlineAmong(
contactsdb.suppliers())
result['customers'] = contactsdb.num_customers()
result['bytes_donated'] = settings.getDonatedBytes()
result['value_donated'] = diskspace.MakeStringFromBytes(
settings.getDonatedBytes())
result['bytes_needed'] = settings.getNeededBytes()
result['value_needed'] = diskspace.MakeStringFromBytes(
settings.getNeededBytes())
result['bytes_used_total'] = backup_fs.sizebackups()
result['value_used_total'] = diskspace.MakeStringFromBytes(
backup_fs.sizebackups())
result['bytes_used_supplier'] = 0 if (
contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() /
contactsdb.num_suppliers()))
result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders()
result['files_count'] = backup_fs.numberfiles()
result['folders_count'] = backup_fs.numberfolders()
result['items_count'] = backup_fs.counter()
result['timestamp'] = time.time()
return {
'result': result,
}
def report_consumed_storage():
result = {}
result['suppliers_num'] = contactsdb.num_suppliers()
result['needed'] = settings.getNeededBytes()
# result['needed_str'] = diskspace.MakeStringFromBytes(result['needed'])
result['used'] = int(backup_fs.sizebackups() / 2)
# result['used_str'] = diskspace.MakeStringFromBytes(result['used'])
result['available'] = result['needed'] - result['used']
# result['available_str'] = diskspace.MakeStringFromBytes(result['available'])
result['needed_per_supplier'] = 0
result['used_per_supplier'] = 0
result['available_per_supplier'] = 0
if result['suppliers_num'] > 0:
result['needed_per_supplier'] = int(
math.ceil(2.0 * result['needed'] / result['suppliers_num']))
result['used_per_supplier'] = int(
math.ceil(2.0 * result['used'] / result['suppliers_num']))
result['available_per_supplier'] = result[
'needed_per_supplier'] - result['used_per_supplier']
# result['needed_per_supplier_str'] = diskspace.MakeStringFromBytes(result['needed_per_supplier'])
# result['used_per_supplier_str'] = diskspace.MakeStringFromBytes(result['used_per_supplier'])
# result['available_per_supplier_str'] = diskspace.MakeStringFromBytes(result['available_per_supplier'])
try:
result['used_percent'] = misc.value2percent(float(result['used']),
float(result['needed']))
except:
result['used_percent'] = '0%'
return result
def doCleanUpBackups(self, arg):
# here we check all backups we have and remove the old one
# user can set how many versions of that file or folder to keep
# other versions (older) will be removed here
versionsToKeep = settings.getBackupsMaxCopies()
bytesUsed = backup_fs.sizebackups() / contactsdb.num_suppliers()
bytesNeeded = diskspace.GetBytesFromString(settings.getNeededString(), 0)
lg.out(6, 'backup_monitor.doCleanUpBackups backupsToKeep=%d used=%d needed=%d' % (versionsToKeep, bytesUsed, bytesNeeded))
delete_count = 0
if versionsToKeep > 0:
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
if backup_control.IsPathInProcess(pathID):
continue
versions = itemInfo.list_versions()
# TODO: do we need to sort the list? it comes from a set, so must be sorted may be
while len(versions) > versionsToKeep:
backupID = pathID + '/' + versions.pop(0)
lg.out(6, 'backup_monitor.doCleanUpBackups %d of %d backups for %s, so remove older %s' % (len(versions), versionsToKeep, localPath, backupID))
backup_control.DeleteBackup(backupID, saveDB=False, calculate=False)
delete_count += 1
# we need also to fit used space into needed space (given from other users)
# they trust us - do not need to take extra space from our friends
# so remove oldest backups, but keep at least one for every folder - at least locally!
# still our suppliers will remove our "extra" files by their "local_tester"
if bytesNeeded <= bytesUsed:
sizeOk = False
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
if sizeOk:
break
versions = itemInfo.list_versions(True, False)
if len(versions) <= 1:
continue
for version in versions[1:]:
backupID = pathID + '/' + version
versionInfo = itemInfo.get_version_info(version)
if versionInfo[1] > 0:
lg.out(6, 'backup_monitor.doCleanUpBackups over use %d of %d, so remove %s of %s' % (
bytesUsed, bytesNeeded, backupID, localPath))
backup_control.DeleteBackup(backupID, saveDB=False, calculate=False)
delete_count += 1
bytesUsed -= versionInfo[1]
if bytesNeeded > bytesUsed:
sizeOk = True
break
if delete_count > 0:
backup_fs.Scan()
backup_fs.Calculate()
backup_control.Save()
from web import control
control.request_update()
collected = gc.collect()
lg.out(6, 'backup_monitor.doCleanUpBackups collected %d objects' % collected)
def _stats(params):
from contacts import contactsdb
from p2p import contact_status
from lib import diskspace
result = {}
result['suppliers'] = contactsdb.num_suppliers()
result['max_suppliers'] = settings.getSuppliersNumberDesired()
result['online_suppliers'] = contact_status.countOnlineAmong(contactsdb.suppliers())
result['customers'] = contactsdb.num_customers()
result['bytes_donated'] = settings.getDonatedBytes()
result['value_donated'] = diskspace.MakeStringFromBytes(settings.getDonatedBytes())
result['bytes_needed'] = settings.getNeededBytes()
result['value_needed'] = diskspace.MakeStringFromBytes(settings.getNeededBytes())
result['bytes_used_total'] = backup_fs.sizebackups()
result['value_used_total'] = diskspace.MakeStringFromBytes(backup_fs.sizebackups())
result['bytes_used_supplier'] = 0 if (contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() / contactsdb.num_suppliers()))
result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders()
result['files_count'] = backup_fs.numberfiles()
result['folders_count'] = backup_fs.numberfolders()
result['items_count'] = backup_fs.counter()
result['timestamp'] = time.time()
return {'result': result, }
def report_consumed_storage():
result = {}
result['suppliers_num'] = contactsdb.num_suppliers()
result['needed'] = settings.getNeededBytes()
result['needed_str'] = diskspace.MakeStringFromBytes(result['needed'])
result['used'] = int(backup_fs.sizebackups() / 2)
result['used_str'] = diskspace.MakeStringFromBytes(result['used'])
result['available'] = result['needed'] - result['used']
result['available_str'] = diskspace.MakeStringFromBytes(result['available'])
result['needed_per_supplier'] = 0
result['used_per_supplier'] = 0
result['available_per_supplier'] = 0
if result['suppliers_num'] > 0:
result['needed_per_supplier'] = int(math.ceil(2.0 * result['needed'] / result['suppliers_num']))
result['used_per_supplier'] = int(math.ceil(2.0 * result['used'] / result['suppliers_num']))
result['available_per_supplier'] = result['needed_per_supplier'] - result['used_per_supplier']
result['needed_per_supplier_str'] = diskspace.MakeStringFromBytes(result['needed_per_supplier'])
result['used_per_supplier_str'] = diskspace.MakeStringFromBytes(result['used_per_supplier'])
result['available_per_supplier_str'] = diskspace.MakeStringFromBytes(result['available_per_supplier'])
try:
result['used_percent'] = misc.percent2string(float(result['used']) / float(result['needed']))
except:
result['used_percent'] = '0%'
return result
def doCleanUpBackups(self, *args, **kwargs):
# here we check all backups we have and remove the old one
# user can set how many versions of that file or folder to keep
# other versions (older) will be removed here
from storage import backup_rebuilder
try:
self.backups_progress_last_iteration = len(
backup_rebuilder.A().backupsWasRebuilt)
except:
self.backups_progress_last_iteration = 0
versionsToKeep = settings.getBackupsMaxCopies()
if not contactsdb.num_suppliers():
bytesUsed = 0
else:
bytesUsed = backup_fs.sizebackups() / contactsdb.num_suppliers()
bytesNeeded = diskspace.GetBytesFromString(settings.getNeededString(),
0)
customerGlobID = my_id.getGlobalID()
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups backupsToKeep=%d used=%d needed=%d'
% (versionsToKeep, bytesUsed, bytesNeeded))
delete_count = 0
if versionsToKeep > 0:
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
pathID = global_id.CanonicalID(pathID)
if backup_control.IsPathInProcess(pathID):
continue
versions = itemInfo.list_versions()
# TODO: do we need to sort the list? it comes from a set, so must be sorted may be
while len(versions) > versionsToKeep:
backupID = packetid.MakeBackupID(customerGlobID, pathID,
versions.pop(0))
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups %d of %d backups for %s, so remove older %s'
% (len(versions), versionsToKeep, localPath,
backupID))
backup_control.DeleteBackup(backupID,
saveDB=False,
calculate=False)
delete_count += 1
# we need also to fit used space into needed space (given from other users)
# they trust us - do not need to take extra space from our friends
# so remove oldest backups, but keep at least one for every folder - at least locally!
# still our suppliers will remove our "extra" files by their "local_tester"
if bytesNeeded <= bytesUsed:
sizeOk = False
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
if sizeOk:
break
pathID = global_id.CanonicalID(pathID)
versions = itemInfo.list_versions(True, False)
if len(versions) <= 1:
continue
for version in versions[1:]:
backupID = packetid.MakeBackupID(customerGlobID, pathID,
version)
versionInfo = itemInfo.get_version_info(version)
if versionInfo[1] > 0:
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups over use %d of %d, so remove %s of %s'
%
(bytesUsed, bytesNeeded, backupID, localPath))
backup_control.DeleteBackup(backupID,
saveDB=False,
calculate=False)
delete_count += 1
bytesUsed -= versionInfo[1]
if bytesNeeded > bytesUsed:
sizeOk = True
break
if delete_count > 0:
backup_fs.Scan()
backup_fs.Calculate()
backup_control.Save()
from main import control
control.request_update()
collected = gc.collect()
if self.backups_progress_last_iteration > 0:
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups sending "restart", backups_progress_last_iteration=%s'
% self.backups_progress_last_iteration)
reactor.callLater(1, self.automat, 'restart') # @UndefinedVariable
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups collected %d objects' %
collected)
