def OnJobDone(backupID, result):
"""
A callback method fired when backup is finished.
Here we need to save the index data base.
"""
from storage import backup_rebuilder
# from customer import io_throttle
lg.info('job done [%s] with result "%s", %d more tasks' %
(backupID, result, len(tasks())))
jobs().pop(backupID)
customerGlobalID, remotePath, version = packetid.SplitBackupID(backupID)
customer_idurl = global_id.GlobalUserToIDURL(customerGlobalID)
if result == 'done':
maxBackupsNum = settings.getBackupsMaxCopies()
if maxBackupsNum:
item = backup_fs.GetByID(remotePath,
iterID=backup_fs.fsID(customer_idurl))
if item:
versions = item.list_versions(sorted=True, reverse=True)
if len(versions) > maxBackupsNum:
for version in versions[maxBackupsNum:]:
item.delete_version(version)
backupID = packetid.MakeBackupID(
customerGlobalID, remotePath, version)
backup_rebuilder.RemoveBackupToWork(backupID)
# io_throttle.DeleteBackupRequests(backupID)
# io_throttle.DeleteBackupSendings(backupID)
# callback.delete_backup_interest(backupID)
backup_fs.DeleteLocalBackup(
settings.getLocalBackupsDir(), backupID)
backup_matrix.EraseBackupLocalInfo(backupID)
backup_matrix.EraseBackupLocalInfo(backupID)
backup_fs.ScanID(remotePath)
backup_fs.Calculate()
Save()
control.request_update([
('pathID', remotePath),
])
# TODO: check used space, if we have over use - stop all tasks immediately
backup_matrix.RepaintBackup(backupID)
elif result == 'abort':
DeleteBackup(backupID)
if len(tasks()) == 0:
# do we really need to restart backup_monitor after each backup?
# if we have a lot tasks started this will produce a lot unneeded actions
# will be smarter to restart it once we finish all tasks
# because user will probably leave BitDust working after starting a long running operations
from storage import backup_monitor
if _Debug:
lg.out(
_DebugLevel,
'backup_control.OnJobDone restarting backup_monitor() machine because no tasks left'
)
backup_monitor.A('restart')
reactor.callLater(0, RunTask) # @UndefinedVariable
reactor.callLater(0, FireTaskFinishedCallbacks, remotePath, version,
result) # @UndefinedVariable
def doCleanUpBackups(self, arg):
# here we check all backups we have and remove the old one
# user can set how many versions of that file or folder to keep
# other versions (older) will be removed here
versionsToKeep = settings.getBackupsMaxCopies()
bytesUsed = backup_fs.sizebackups() / contactsdb.num_suppliers()
bytesNeeded = diskspace.GetBytesFromString(settings.getNeededString(), 0)
lg.out(6, 'backup_monitor.doCleanUpBackups backupsToKeep=%d used=%d needed=%d' % (versionsToKeep, bytesUsed, bytesNeeded))
delete_count = 0
if versionsToKeep > 0:
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
if backup_control.IsPathInProcess(pathID):
continue
versions = itemInfo.list_versions()
# TODO: do we need to sort the list? it comes from a set, so must be sorted may be
while len(versions) > versionsToKeep:
backupID = pathID + '/' + versions.pop(0)
lg.out(6, 'backup_monitor.doCleanUpBackups %d of %d backups for %s, so remove older %s' % (len(versions), versionsToKeep, localPath, backupID))
backup_control.DeleteBackup(backupID, saveDB=False, calculate=False)
delete_count += 1
# we need also to fit used space into needed space (given from other users)
# they trust us - do not need to take extra space from our friends
# so remove oldest backups, but keep at least one for every folder - at least locally!
# still our suppliers will remove our "extra" files by their "local_tester"
if bytesNeeded <= bytesUsed:
sizeOk = False
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
if sizeOk:
break
versions = itemInfo.list_versions(True, False)
if len(versions) <= 1:
continue
for version in versions[1:]:
backupID = pathID + '/' + version
versionInfo = itemInfo.get_version_info(version)
if versionInfo[1] > 0:
lg.out(6, 'backup_monitor.doCleanUpBackups over use %d of %d, so remove %s of %s' % (
bytesUsed, bytesNeeded, backupID, localPath))
backup_control.DeleteBackup(backupID, saveDB=False, calculate=False)
delete_count += 1
bytesUsed -= versionInfo[1]
if bytesNeeded > bytesUsed:
sizeOk = True
break
if delete_count > 0:
backup_fs.Scan()
backup_fs.Calculate()
backup_control.Save()
from web import control
control.request_update()
collected = gc.collect()
lg.out(6, 'backup_monitor.doCleanUpBackups collected %d objects' % collected)
def OnJobDone(backupID, result):
"""
A callback method fired when backup is finished.
Here we need to save the index data base.
"""
import backup_rebuilder
from customer import io_throttle
lg.out(4, '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
lg.out(4, 'backup_control.OnJobDone [%s] %s, %d more tasks' % (backupID, result, len(tasks())))
jobs().pop(backupID)
pathID, version = packetid.SplitBackupID(backupID)
if result == 'done':
maxBackupsNum = settings.getBackupsMaxCopies()
if maxBackupsNum:
item = backup_fs.GetByID(pathID)
if item:
versions = item.list_versions(sorted=True, reverse=True)
if len(versions) > maxBackupsNum:
for version in versions[maxBackupsNum:]:
item.delete_version(version)
backupID = pathID + '/' + version
backup_rebuilder.RemoveBackupToWork(backupID)
io_throttle.DeleteBackupRequests(backupID)
io_throttle.DeleteBackupSendings(backupID)
callback.delete_backup_interest(backupID)
backup_fs.DeleteLocalBackup(settings.getLocalBackupsDir(), backupID)
backup_matrix.EraseBackupLocalInfo(backupID)
backup_matrix.EraseBackupLocalInfo(backupID)
backup_fs.ScanID(pathID)
backup_fs.Calculate()
Save()
control.request_update([('pathID', pathID), ])
# TODO: check used space, if we have over use - stop all tasks immediately
backup_matrix.RepaintBackup(backupID)
elif result == 'abort':
DeleteBackup(backupID)
if len(tasks()) == 0:
# do we really need to restart backup_monitor after each backup?
# if we have a lot tasks started this will produce a lot unneeded actions
# will be smarter to restart it once we finish all tasks
# because user will probably leave BitDust working after starting a long running operations
from storage import backup_monitor
backup_monitor.A('restart')
RunTasks()
reactor.callLater(0, FireTaskFinishedCallbacks, pathID, version, result)
def doCleanUpBackups(self, *args, **kwargs):
# here we check all backups we have and remove the old one
# user can set how many versions of that file or folder to keep
# other versions (older) will be removed here
from storage import backup_rebuilder
try:
self.backups_progress_last_iteration = len(
backup_rebuilder.A().backupsWasRebuilt)
except:
self.backups_progress_last_iteration = 0
versionsToKeep = settings.getBackupsMaxCopies()
if not contactsdb.num_suppliers():
bytesUsed = 0
else:
bytesUsed = backup_fs.sizebackups() / contactsdb.num_suppliers()
bytesNeeded = diskspace.GetBytesFromString(settings.getNeededString(),
0)
customerGlobID = my_id.getGlobalID()
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups backupsToKeep=%d used=%d needed=%d'
% (versionsToKeep, bytesUsed, bytesNeeded))
delete_count = 0
if versionsToKeep > 0:
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
pathID = global_id.CanonicalID(pathID)
if backup_control.IsPathInProcess(pathID):
continue
versions = itemInfo.list_versions()
# TODO: do we need to sort the list? it comes from a set, so must be sorted may be
while len(versions) > versionsToKeep:
backupID = packetid.MakeBackupID(customerGlobID, pathID,
versions.pop(0))
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups %d of %d backups for %s, so remove older %s'
% (len(versions), versionsToKeep, localPath,
backupID))
backup_control.DeleteBackup(backupID,
saveDB=False,
calculate=False)
delete_count += 1
# we need also to fit used space into needed space (given from other users)
# they trust us - do not need to take extra space from our friends
# so remove oldest backups, but keep at least one for every folder - at least locally!
# still our suppliers will remove our "extra" files by their "local_tester"
if bytesNeeded <= bytesUsed:
sizeOk = False
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
if sizeOk:
break
pathID = global_id.CanonicalID(pathID)
versions = itemInfo.list_versions(True, False)
if len(versions) <= 1:
continue
for version in versions[1:]:
backupID = packetid.MakeBackupID(customerGlobID, pathID,
version)
versionInfo = itemInfo.get_version_info(version)
if versionInfo[1] > 0:
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups over use %d of %d, so remove %s of %s'
%
(bytesUsed, bytesNeeded, backupID, localPath))
backup_control.DeleteBackup(backupID,
saveDB=False,
calculate=False)
delete_count += 1
bytesUsed -= versionInfo[1]
if bytesNeeded > bytesUsed:
sizeOk = True
break
if delete_count > 0:
backup_fs.Scan()
backup_fs.Calculate()
backup_control.Save()
from main import control
control.request_update()
collected = gc.collect()
if self.backups_progress_last_iteration > 0:
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups sending "restart", backups_progress_last_iteration=%s'
% self.backups_progress_last_iteration)
reactor.callLater(1, self.automat, 'restart') # @UndefinedVariable
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups collected %d objects' %
collected)
