def doPrepareListBackups(self, *args, **kwargs):
from storage import backup_rebuilder
if backup_control.HasRunningBackup():
# if some backups are running right now no need to rebuild something - too much use of CPU
backup_rebuilder.RemoveAllBackupsToWork()
lg.out(6, 'backup_monitor.doPrepareListBackups skip all rebuilds')
self.automat('list-backups-done')
return
# take remote and local backups and get union from it
allBackupIDs = set(
list(backup_matrix.local_files().keys()) +
list(backup_matrix.remote_files().keys()))
# take only backups from data base
allBackupIDs.intersection_update(backup_fs.ListAllBackupIDs())
# remove running backups
allBackupIDs.difference_update(backup_control.ListRunningBackups())
# sort it in reverse order - newer backups should be repaired first
allBackupIDs = misc.sorted_backup_ids(list(allBackupIDs), True)
# add backups to the queue
backup_rebuilder.AddBackupsToWork(allBackupIDs)
lg.out(
6, 'backup_monitor.doPrepareListBackups %d items:' %
len(allBackupIDs))
lg.out(6, ' %s' % allBackupIDs)
self.automat('list-backups-done', allBackupIDs)
def doRemoveUnusedFiles(self, arg):
# we want to remove files for this block
# because we only need them during rebuilding
if settings.getGeneralLocalBackups() is True:
# if user set this in settings - he want to keep the local files
return
# ... user do not want to keep local backups
if settings.getGeneralWaitSuppliers() is True:
# but he want to be sure - all suppliers are green for a long time
if contact_status.hasOfflineSuppliers() or time.time() - fire_hire.GetLastFireTime() < 24*60*60:
# some people are not there or we do not have stable team yet
# do not remove the files because we need it to rebuild
return
count = 0
for backupID in misc.sorted_backup_ids(backup_matrix.local_files().keys()):
packets = backup_matrix.ScanBlocksToRemove(backupID, settings.getGeneralWaitSuppliers())
for packetID in packets:
filename = os.path.join(settings.getLocalBackupsDir(), packetID)
if os.path.isfile(filename):
try:
os.remove(filename)
# dhnio.Dprint(6, ' ' + os.path.basename(filename))
except:
dhnio.DprintException()
continue
count += 1
dhnio.Dprint(8, 'data_sender.doRemoveUnusedFiles %d files were removed' % count)
backup_matrix.ReadLocalFiles()
def doScanAndQueue(self, arg):
global _ShutdownFlag
dhnio.Dprint(10, 'data_sender.doScanAndQueue')
log = open(os.path.join(settings.LogsDir(), 'data_sender.log'), 'w')
log.write('doScanAndQueue %s\n' % time.asctime())
if _ShutdownFlag:
log.write('doScanAndQueue _ShutdownFlag is True\n')
self.automat('scan-done')
log.flush()
log.close()
return
if '' not in contacts.getSupplierIDs():
for backupID in misc.sorted_backup_ids(backup_matrix.local_files().keys(), True):
packetsBySupplier = backup_matrix.ScanBlocksToSend(backupID)
log.write('%s\n' % packetsBySupplier)
for supplierNum in packetsBySupplier.keys():
supplier_idurl = contacts.getSupplierID(supplierNum)
if not supplier_idurl:
dhnio.Dprint(2, 'data_sender.doScanAndQueue WARNING ?supplierNum? %s for %s' % (supplierNum, backupID))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, blockNum, supplierNum_, dataORparity = packetid.BidBnSnDp(packetID)
if backupID_ != backupID:
dhnio.Dprint(2, 'data_sender.doScanAndQueue WARNING ?backupID? %s for %s' % (packetID, backupID))
continue
if supplierNum_ != supplierNum:
dhnio.Dprint(2, 'data_sender.doScanAndQueue WARNING ?supplierNum? %s for %s' % (packetID, backupID))
continue
if io_throttle.HasPacketInSendQueue(supplier_idurl, packetID):
log.write('%s in the send queue to %s\n' % (packetID, supplier_idurl))
continue
if not io_throttle.OkToSend(supplier_idurl):
log.write('ok to send %s ? - NO!\n' % supplier_idurl)
continue
tranByiID = transport_control.transfers_by_idurl(supplier_idurl)
if len(tranByiID) > 3:
log.write('transfers by %s: %d\n' % (supplier_idurl, len(tranByiID)))
continue
filename = os.path.join(settings.getLocalBackupsDir(), packetID)
if not os.path.isfile(filename):
log.write('%s is not file\n' % filename)
continue
io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
misc.getLocalID(),
self._packetAcked,
self._packetFailed)
log.write('io_throttle.QueueSendFile %s\n' % packetID)
# dhnio.Dprint(6, ' %s for %s' % (packetID, backupID))
self.automat('scan-done')
log.flush()
log.close()
def GetBackupIDs(remote=True, local=False, sorted_ids=False):
"""
Return a list of backup IDs which is present in the matrixes.
You can choose which matrix to use.
"""
s = set()
if remote:
s.update(remote_files().keys())
if local:
s.update(local_files().keys())
if sorted_ids:
return misc.sorted_backup_ids(list(s))
return list(s)
def doPrepareListBackups(self, arg):
if backup_control.HasRunningBackup():
# if some backups are running right now no need to rebuild something - too much use of CPU
backup_rebuilder.RemoveAllBackupsToWork()
dhnio.Dprint(6, 'backup_monitor.doPrepareListBackups skip all rebuilds')
self.automat('list-backups-done')
return
# take remote and local backups and get union from it
allBackupIDs = set(backup_matrix.local_files().keys() + backup_matrix.remote_files().keys())
# take only backups from data base
allBackupIDs.intersection_update(backup_fs.ListAllBackupIDs())
# remove running backups
allBackupIDs.difference_update(backup_control.ListRunningBackups())
# sort it in reverse order - newer backups should be repaired first
allBackupIDs = misc.sorted_backup_ids(list(allBackupIDs), True)
# add backups to the queue
backup_rebuilder.AddBackupsToWork(allBackupIDs)
dhnio.Dprint(6, 'backup_monitor.doPrepareListBackups %d items' % len(allBackupIDs))
self.automat('list-backups-done')
def doRemoveUnusedFiles(self, arg):
# we want to remove files for this block
# because we only need them during rebuilding
if settings.getBackupsKeepLocalCopies() is True:
# if user set this in settings - he want to keep the local files
return
# ... user do not want to keep local backups
if settings.getGeneralWaitSuppliers() is True:
from customer import fire_hire
# but he want to be sure - all suppliers are green for a long time
if len(contact_status.listOfflineSuppliers(
)) > 0 or time.time() - fire_hire.GetLastFireTime() < 24 * 60 * 60:
# some people are not there or we do not have stable team yet
# do not remove the files because we need it to rebuild
return
count = 0
from storage import backup_matrix
from storage import restore_monitor
from storage import backup_rebuilder
if _Debug:
lg.out(_DebugLevel, 'data_sender.doRemoveUnusedFiles')
for backupID in misc.sorted_backup_ids(
backup_matrix.local_files().keys()):
if restore_monitor.IsWorking(backupID):
if _Debug:
lg.out(_DebugLevel,
' %s : SKIP, because restoring' % backupID)
continue
if backup_rebuilder.IsBackupNeedsWork(backupID):
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because needs rebuilding' %
backupID)
continue
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == backupID:
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because rebuilding is in process'
% backupID)
continue
packets = backup_matrix.ScanBlocksToRemove(
backupID, settings.getGeneralWaitSuppliers())
for packetID in packets:
customer, pathID = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, pathID)
if os.path.isfile(filename):
try:
os.remove(filename)
# lg.out(6, ' ' + os.path.basename(filename))
except:
lg.exc()
continue
count += 1
if _Debug:
lg.out(_DebugLevel, ' %d files were removed' % count)
backup_matrix.ReadLocalFiles()
def doScanAndQueue(self, arg):
global _ShutdownFlag
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue _ShutdownFlag=%r' % _ShutdownFlag)
if _Debug:
log = open(os.path.join(settings.LogsDir(), 'data_sender.log'),
'w')
log.write('doScanAndQueue %s\n' % time.asctime())
if _ShutdownFlag:
if _Debug:
log.write('doScanAndQueue _ShutdownFlag is True\n')
self.automat('scan-done')
if _Debug:
log.flush()
log.close()
return
for customer_idurl in contactsdb.known_customers():
if '' not in contactsdb.suppliers(customer_idurl):
from storage import backup_matrix
for backupID in misc.sorted_backup_ids(
backup_matrix.local_files().keys(), True):
packetsBySupplier = backup_matrix.ScanBlocksToSend(
backupID)
if _Debug:
log.write('%s\n' % packetsBySupplier)
for supplierNum in packetsBySupplier.keys():
supplier_idurl = contactsdb.supplier(
supplierNum, customer_idurl=customer_idurl)
if not supplier_idurl:
lg.warn('?supplierNum? %s for %s' %
(supplierNum, backupID))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, _, supplierNum_, _ = packetid.BidBnSnDp(
packetID)
if backupID_ != backupID:
lg.warn('?backupID? %s for %s' %
(packetID, backupID))
continue
if supplierNum_ != supplierNum:
lg.warn('?supplierNum? %s for %s' %
(packetID, backupID))
continue
if io_throttle.HasPacketInSendQueue(
supplier_idurl, packetID):
if _Debug:
log.write(
'%s already in sending queue for %s\n'
% (packetID, supplier_idurl))
continue
if not io_throttle.OkToSend(supplier_idurl):
if _Debug:
log.write('ok to send %s ? - NO!\n' %
supplier_idurl)
continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
# tranByID = gate.transfers_out_by_idurl().get(supplier_idurl, [])
# if len(tranByID) > 3:
# log.write('transfers by %s: %d\n' % (supplier_idurl, len(tranByID)))
# continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customerGlobalID,
pathID,
)
if not os.path.isfile(filename):
if _Debug:
log.write('%s is not a file\n' % filename)
continue
if io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
my_id.getLocalID(),
self._packetAcked,
self._packetFailed,
):
if _Debug:
log.write(
'io_throttle.QueueSendFile %s\n' %
packetID)
else:
if _Debug:
log.write(
'io_throttle.QueueSendFile FAILED %s\n'
% packetID)
# lg.out(6, ' %s for %s' % (packetID, backupID))
# DEBUG
# break
self.automat('scan-done')
if _Debug:
log.flush()
log.close()
def doRemoveUnusedFiles(self, *args, **kwargs):
"""
Action method.
"""
if not list_files_orator.is_synchronized():
# always make sure we have a very fresh info about remote files before take any actions
return
# we want to remove files for this block
# because we only need them during rebuilding
if settings.getBackupsKeepLocalCopies() is True:
# if user set this in settings - he want to keep the local files
return
# ... user do not want to keep local backups
if settings.getGeneralWaitSuppliers() is True:
from customer import fire_hire
# but he want to be sure - all suppliers are green for a long time
if len(online_status.listOfflineSuppliers()) > 0 or (
time.time() - fire_hire.GetLastFireTime() < 24 * 60 * 60):
# some people are not there or we do not have stable team yet
# do not remove the files because we need it to rebuild
return
count = 0
from storage import backup_matrix
from storage import restore_monitor
from storage import backup_rebuilder
if _Debug:
lg.out(_DebugLevel, 'data_sender.doRemoveUnusedFiles')
for backupID in misc.sorted_backup_ids(
list(backup_matrix.local_files().keys())):
if restore_monitor.IsWorking(backupID):
if _Debug:
lg.out(_DebugLevel,
' %s : SKIP, because restoring' % backupID)
continue
if backup_rebuilder.IsBackupNeedsWork(backupID):
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because needs rebuilding' %
backupID)
continue
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == backupID:
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because rebuilding is in process'
% backupID)
continue
if backupID not in backup_matrix.remote_files():
if _Debug:
lg.out(
_DebugLevel,
' going to erase %s because not found in remote files'
% backupID)
customer, pathID, version = packetid.SplitBackupID(backupID)
dirpath = os.path.join(settings.getLocalBackupsDir(), customer,
pathID, version)
if os.path.isdir(dirpath):
try:
count += bpio.rmdir_recursive(dirpath,
ignore_errors=True)
except:
lg.exc()
continue
packets = backup_matrix.ScanBlocksToRemove(
backupID,
check_all_suppliers=settings.getGeneralWaitSuppliers())
for packetID in packets:
customer, pathID = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, pathID)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
if _Debug:
lg.out(_DebugLevel, ' %d files were removed' % count)
backup_matrix.ReadLocalFiles()
def doScanAndQueue(self, *args, **kwargs):
"""
Action method.
"""
global _ShutdownFlag
if _ShutdownFlag:
if _Debug:
lg.out(_DebugLevel,
'data_sender.doScanAndQueue _ShutdownFlag is True\n')
self.automat('scan-done', 0)
return
from storage import backup_matrix
from storage import backup_fs
backup_matrix.ReadLocalFiles()
progress = 0
# if _Debug:
# lg.out(_DebugLevel, 'data_sender.doScanAndQueue with %d known customers' % len(contactsdb.known_customers()))
for customer_idurl in contactsdb.known_customers():
if customer_idurl != my_id.getIDURL():
# TODO: check that later
if _Debug:
lg.out(
_DebugLevel + 2,
'data_sender.doScanAndQueue skip sending to another customer: %r'
% customer_idurl)
continue
known_suppliers = contactsdb.suppliers(customer_idurl)
if not known_suppliers or id_url.is_some_empty(known_suppliers):
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue found empty supplier(s) for customer %r, SKIP'
% customer_idurl)
continue
known_backups = misc.sorted_backup_ids(
list(backup_matrix.local_files().keys()), True)
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue found %d known suppliers for customer %r with %d backups'
%
(len(known_suppliers), customer_idurl, len(known_backups)))
for backupID in known_backups:
this_customer_idurl = packetid.CustomerIDURL(backupID)
if this_customer_idurl != customer_idurl:
continue
customerGlobalID, pathID, _ = packetid.SplitBackupID(
backupID, normalize_key_alias=True)
keyAlias = packetid.KeyAlias(customerGlobalID)
item = backup_fs.GetByID(pathID,
iterID=backup_fs.fsID(
customer_idurl, keyAlias))
if not item:
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue skip sending backup %r path not exist in catalog'
% backupID)
continue
if item.key_id and customerGlobalID and customerGlobalID != item.key_id:
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue skip sending backup %r key is different in the catalog: %r ~ %r'
% (
backupID,
customerGlobalID,
item.key_id,
))
continue
packetsBySupplier = backup_matrix.ScanBlocksToSend(
backupID, limit_per_supplier=None)
total_for_customer = sum(
[len(v) for v in packetsBySupplier.values()])
if total_for_customer:
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue sending %r for customer %r with %d pieces'
%
(item.name(), customer_idurl, total_for_customer))
for supplierNum in packetsBySupplier.keys():
# supplier_idurl = contactsdb.supplier(supplierNum, customer_idurl=customer_idurl)
if supplierNum >= 0 and supplierNum < len(
known_suppliers):
supplier_idurl = known_suppliers[supplierNum]
else:
supplier_idurl = None
if not supplier_idurl:
lg.warn(
'skip sending, unknown supplier_idurl supplierNum=%s for %s, customer_idurl=%r'
% (supplierNum, backupID, customer_idurl))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, _, supplierNum_, _ = packetid.BidBnSnDp(
packetID)
if backupID_ != backupID:
lg.warn(
'skip sending, unexpected backupID supplierNum=%s for %s, customer_idurl=%r'
% (packetID, backupID, customer_idurl))
continue
if supplierNum_ != supplierNum:
lg.warn(
'skip sending, unexpected supplierNum %s for %s, customer_idurl=%r'
% (packetID, backupID, customer_idurl))
continue
if io_throttle.HasPacketInSendQueue(
supplier_idurl, packetID):
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue %s already in sending queue for %r'
% (packetID, supplier_idurl))
continue
latest_progress = self.statistic.get(
supplier_idurl, {}).get('latest', '')
if len(latest_progress
) >= 3 and latest_progress.endswith('---'):
if _Debug:
lg.out(
_DebugLevel + 2,
'data_sender.doScanAndQueue skip sending to supplier %r because multiple packets already failed'
% supplier_idurl)
continue
if not io_throttle.OkToSend(supplier_idurl):
if _Debug:
lg.out(
_DebugLevel + 2,
'data_sender.doScanAndQueue skip sending, queue is busy for %r'
% supplier_idurl)
continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customerGlobalID,
pathID,
)
if not os.path.isfile(filename):
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue %s is not a file'
% filename)
continue
itemInfo = item.to_json()
if io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
my_id.getIDURL(),
lambda packet, ownerID,
packetID: self._packetAcked(
packet, ownerID, packetID, itemInfo),
lambda remoteID, packetID,
why: self._packetFailed(
remoteID, packetID, why, itemInfo),
):
progress += 1
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue for %r put %s in the queue progress=%d'
% (
item.name(),
packetID,
progress,
))
else:
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue io_throttle.QueueSendFile FAILED %s'
% packetID)
if _Debug:
lg.out(_DebugLevel,
'data_sender.doScanAndQueue progress=%s' % progress)
self.automat('scan-done', progress)
def doScanAndQueue(self, *args, **kwargs):
"""
Action method.
"""
global _ShutdownFlag
if _ShutdownFlag:
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue _ShutdownFlag is True\n')
self.automat('scan-done', 0)
return
from storage import backup_matrix
from storage import backup_fs
backup_matrix.ReadLocalFiles()
progress = 0
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue with %d known customers' % len(contactsdb.known_customers()))
for customer_idurl in contactsdb.known_customers():
if customer_idurl != my_id.getLocalID():
# TODO: check that later
if _Debug:
lg.out(_DebugLevel + 6, 'data_sender.doScanAndQueue skip sending to another customer: %r' % customer_idurl)
continue
known_suppliers = contactsdb.suppliers(customer_idurl)
if not known_suppliers or id_url.is_some_empty(known_suppliers):
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue found empty supplier(s) for customer %r, SKIP' % customer_idurl)
continue
known_backups = misc.sorted_backup_ids(list(backup_matrix.local_files().keys()), True)
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue found %d known suppliers for customer %r with %d backups' % (
len(known_suppliers), customer_idurl, len(known_backups)))
for backupID in known_backups:
this_customer_idurl = packetid.CustomerIDURL(backupID)
if this_customer_idurl != customer_idurl:
continue
customerGlobalID, pathID, _ = packetid.SplitBackupID(backupID, normalize_key_alias=True)
item = backup_fs.GetByID(pathID, iterID=backup_fs.fsID(customer_idurl=customer_idurl))
if not item:
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue skip sending backup %r path not exist in catalog' % backupID)
continue
if item.key_id and customerGlobalID and customerGlobalID != item.key_id:
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue skip sending backup %r key is different in the catalog' % backupID)
continue
packetsBySupplier = backup_matrix.ScanBlocksToSend(backupID, limit_per_supplier=None)
total_for_customer = sum([len(v) for v in packetsBySupplier.values()])
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue to be delivered for customer %r : %d' % (customer_idurl, total_for_customer))
for supplierNum in packetsBySupplier.keys():
# supplier_idurl = contactsdb.supplier(supplierNum, customer_idurl=customer_idurl)
if supplierNum >= 0 and supplierNum < len(known_suppliers):
supplier_idurl = known_suppliers[supplierNum]
else:
supplier_idurl = None
if not supplier_idurl:
lg.warn('skip sending, unknown supplier_idurl supplierNum=%s for %s, customer_idurl=%r' % (
supplierNum, backupID, customer_idurl))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, _, supplierNum_, _ = packetid.BidBnSnDp(packetID)
if backupID_ != backupID:
lg.warn('skip sending, unexpected backupID supplierNum=%s for %s, customer_idurl=%r' % (
packetID, backupID, customer_idurl))
continue
if supplierNum_ != supplierNum:
lg.warn('skip sending, unexpected supplierNum %s for %s, customer_idurl=%r' % (
packetID, backupID, customer_idurl))
continue
if io_throttle.HasPacketInSendQueue(supplier_idurl, packetID):
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue %s already in sending queue for %r' % (packetID, supplier_idurl))
continue
if not io_throttle.OkToSend(supplier_idurl):
if _Debug:
lg.out(_DebugLevel + 6, 'data_sender.doScanAndQueue skip sending, queue is busy for %r\n' % supplier_idurl)
continue
# customerGlobalID, pathID = packetid.SplitPacketID(packetID)
# tranByID = gate.transfers_out_by_idurl().get(supplier_idurl, [])
# if len(tranByID) > 3:
# log.write(u'transfers by %s: %d\n' % (supplier_idurl, len(tranByID)))
# continue
customerGlobalID, pathID = packetid.SplitPacketID(packetID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customerGlobalID,
pathID,
)
if not os.path.isfile(filename):
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue %s is not a file\n' % filename)
continue
if io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
my_id.getIDURL(),
self._packetAcked,
self._packetFailed,
):
progress += 1
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue put %s in the queue progress=%d' % (packetID, progress, ))
else:
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue io_throttle.QueueSendFile FAILED %s' % packetID)
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue progress=%s' % progress)
self.automat('scan-done', progress)
def doScanAndQueue(self, *args, **kwargs):
"""
Action method.
"""
global _ShutdownFlag
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue _ShutdownFlag=%r' % _ShutdownFlag)
if _ShutdownFlag:
if _Debug:
lg.out(_DebugLevel, ' _ShutdownFlag is True\n')
self.automat('scan-done')
return
from storage import backup_matrix
backup_matrix.ReadLocalFiles()
progress = 0
for customer_idurl in contactsdb.known_customers():
if customer_idurl != my_id.getLocalIDURL():
# TODO: check that later
if _Debug:
lg.out(
_DebugLevel + 6,
' skip sending to another customer: %r' %
customer_idurl)
continue
known_suppliers = contactsdb.suppliers(customer_idurl)
if b'' in known_suppliers or '' in known_suppliers:
if _Debug:
lg.out(
_DebugLevel,
' found empty supplier for customer %r, SKIP' %
customer_idurl)
continue
known_backups = misc.sorted_backup_ids(
list(backup_matrix.local_files().keys()), True)
if _Debug:
lg.out(
_DebugLevel,
' found %d known suppliers for customer %r with %d backups'
%
(len(known_suppliers), customer_idurl, len(known_backups)))
for backupID in known_backups:
this_customer_idurl = packetid.CustomerIDURL(backupID)
if this_customer_idurl != customer_idurl:
continue
packetsBySupplier = backup_matrix.ScanBlocksToSend(backupID)
if _Debug:
lg.out(
_DebugLevel, ' packets for customer %r : %s' %
(customer_idurl, packetsBySupplier))
for supplierNum in packetsBySupplier.keys():
# supplier_idurl = contactsdb.supplier(supplierNum, customer_idurl=customer_idurl)
try:
supplier_idurl = known_suppliers[supplierNum]
except:
lg.exc()
continue
if not supplier_idurl:
lg.warn(
'unknown supplier_idurl supplierNum=%s for %s, customer_idurl=%r'
% (supplierNum, backupID, customer_idurl))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, _, supplierNum_, _ = packetid.BidBnSnDp(
packetID)
if backupID_ != backupID:
lg.warn(
'unexpected backupID supplierNum=%s for %s, customer_idurl=%r'
% (packetID, backupID, customer_idurl))
continue
if supplierNum_ != supplierNum:
lg.warn(
'unexpected supplierNum %s for %s, customer_idurl=%r'
% (packetID, backupID, customer_idurl))
continue
if io_throttle.HasPacketInSendQueue(
supplier_idurl, packetID):
if _Debug:
lg.out(
_DebugLevel,
' %s already in sending queue for %r'
% (packetID, supplier_idurl))
continue
if not io_throttle.OkToSend(supplier_idurl):
if _Debug:
lg.out(
_DebugLevel + 6,
' skip, not ok to send %s\n' %
supplier_idurl)
continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
# tranByID = gate.transfers_out_by_idurl().get(supplier_idurl, [])
# if len(tranByID) > 3:
# log.write(u'transfers by %s: %d\n' % (supplier_idurl, len(tranByID)))
# continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customerGlobalID,
pathID,
)
if not os.path.isfile(filename):
if _Debug:
lg.out(_DebugLevel,
' %s is not a file\n' % filename)
continue
if io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
my_id.getLocalID(),
self._packetAcked,
self._packetFailed,
):
progress += 1
if _Debug:
lg.out(
_DebugLevel,
' io_throttle.QueueSendFile %s' %
packetID)
else:
if _Debug:
lg.out(
_DebugLevel,
' io_throttle.QueueSendFile FAILED %s'
% packetID)
if _Debug:
lg.out(_DebugLevel,
'data_sender.doScanAndQueue progress=%s' % progress)
self.automat('scan-done')
