def doRemoveTempFile(self, *args, **kwargs):
"""
Action method.
"""
if not args or not len(args) > 1:
return
filename = args[1]
if filename:
tmpfile.throw_out(filename, 'block restored')
if settings.getBackupsKeepLocalCopies():
return
from storage import backup_matrix
from storage import backup_rebuilder
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == self.backup_id:
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRemoveTempFile SKIP because rebuilding in process'
)
return
count = 0
for supplierNum in range(
contactsdb.num_suppliers(customer_idurl=self.customer_idurl)):
supplierIDURL = contactsdb.supplier(
supplierNum, customer_idurl=self.customer_idurl)
if not supplierIDURL:
continue
for dataORparity in [
'Data',
'Parity',
]:
packetID = packetid.MakePacketID(self.backup_id,
self.block_number,
supplierNum, dataORparity)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, remotePath)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
backup_matrix.LocalBlockReport(self.backup_id, self.block_number,
*args, **kwargs)
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRemoveTempFile %d files were removed' %
count)
def doRemoveTempFile(self, arg):
try:
filename = arg[1]
except:
return
tmpfile.throw_out(filename, 'block restored')
if settings.getBackupsKeepLocalCopies():
return
import backup_rebuilder
import backup_matrix
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == self.BackupID:
lg.out(
6,
'restore.doRemoveTempFile SKIP because rebuilding in process'
)
return
count = 0
for supplierNum in xrange(
contactsdb.num_suppliers(customer_idurl=self.CustomerIDURL)):
supplierIDURL = contactsdb.supplier(
supplierNum, customer_idurl=self.CustomerIDURL)
if not supplierIDURL:
continue
for dataORparity in ['Data', 'Parity']:
packetID = packetid.MakePacketID(self.BackupID,
self.BlockNumber, supplierNum,
dataORparity)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, remotePath)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
backup_matrix.LocalBlockReport(self.BackupID, self.BlockNumber, arg)
lg.out(6, 'restore.doRemoveTempFile %d files were removed' % count)
def doRemoveUnusedFiles(self, arg):
# we want to remove files for this block
# because we only need them during rebuilding
if settings.getBackupsKeepLocalCopies() is True:
# if user set this in settings - he want to keep the local files
return
# ... user do not want to keep local backups
if settings.getGeneralWaitSuppliers() is True:
from customer import fire_hire
# but he want to be sure - all suppliers are green for a long time
if len(contact_status.listOfflineSuppliers(
)) > 0 or time.time() - fire_hire.GetLastFireTime() < 24 * 60 * 60:
# some people are not there or we do not have stable team yet
# do not remove the files because we need it to rebuild
return
count = 0
from storage import backup_matrix
from storage import restore_monitor
from storage import backup_rebuilder
if _Debug:
lg.out(_DebugLevel, 'data_sender.doRemoveUnusedFiles')
for backupID in misc.sorted_backup_ids(
backup_matrix.local_files().keys()):
if restore_monitor.IsWorking(backupID):
if _Debug:
lg.out(_DebugLevel,
' %s : SKIP, because restoring' % backupID)
continue
if backup_rebuilder.IsBackupNeedsWork(backupID):
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because needs rebuilding' %
backupID)
continue
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == backupID:
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because rebuilding is in process'
% backupID)
continue
packets = backup_matrix.ScanBlocksToRemove(
backupID, settings.getGeneralWaitSuppliers())
for packetID in packets:
customer, pathID = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, pathID)
if os.path.isfile(filename):
try:
os.remove(filename)
# lg.out(6, ' ' + os.path.basename(filename))
except:
lg.exc()
continue
count += 1
if _Debug:
lg.out(_DebugLevel, ' %d files were removed' % count)
backup_matrix.ReadLocalFiles()
def doRemoveUnusedFiles(self, *args, **kwargs):
"""
Action method.
"""
if not list_files_orator.is_synchronized():
# always make sure we have a very fresh info about remote files before take any actions
return
# we want to remove files for this block
# because we only need them during rebuilding
if settings.getBackupsKeepLocalCopies() is True:
# if user set this in settings - he want to keep the local files
return
# ... user do not want to keep local backups
if settings.getGeneralWaitSuppliers() is True:
from customer import fire_hire
# but he want to be sure - all suppliers are green for a long time
if len(online_status.listOfflineSuppliers()) > 0 or (
time.time() - fire_hire.GetLastFireTime() < 24 * 60 * 60):
# some people are not there or we do not have stable team yet
# do not remove the files because we need it to rebuild
return
count = 0
from storage import backup_matrix
from storage import restore_monitor
from storage import backup_rebuilder
if _Debug:
lg.out(_DebugLevel, 'data_sender.doRemoveUnusedFiles')
for backupID in misc.sorted_backup_ids(
list(backup_matrix.local_files().keys())):
if restore_monitor.IsWorking(backupID):
if _Debug:
lg.out(_DebugLevel,
' %s : SKIP, because restoring' % backupID)
continue
if backup_rebuilder.IsBackupNeedsWork(backupID):
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because needs rebuilding' %
backupID)
continue
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == backupID:
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because rebuilding is in process'
% backupID)
continue
if backupID not in backup_matrix.remote_files():
if _Debug:
lg.out(
_DebugLevel,
' going to erase %s because not found in remote files'
% backupID)
customer, pathID, version = packetid.SplitBackupID(backupID)
dirpath = os.path.join(settings.getLocalBackupsDir(), customer,
pathID, version)
if os.path.isdir(dirpath):
try:
count += bpio.rmdir_recursive(dirpath,
ignore_errors=True)
except:
lg.exc()
continue
packets = backup_matrix.ScanBlocksToRemove(
backupID,
check_all_suppliers=settings.getGeneralWaitSuppliers())
for packetID in packets:
customer, pathID = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, pathID)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
if _Debug:
lg.out(_DebugLevel, ' %d files were removed' % count)
backup_matrix.ReadLocalFiles()
