def doPrepareListBackups(self, *args, **kwargs):
from storage import backup_rebuilder
if backup_control.HasRunningBackup():
# if some backups are running right now no need to rebuild something - too much use of CPU
backup_rebuilder.RemoveAllBackupsToWork()
lg.out(6, 'backup_monitor.doPrepareListBackups skip all rebuilds')
self.automat('list-backups-done')
return
# take remote and local backups and get union from it
allBackupIDs = set(
list(backup_matrix.local_files().keys()) +
list(backup_matrix.remote_files().keys()))
# take only backups from data base
allBackupIDs.intersection_update(backup_fs.ListAllBackupIDs())
# remove running backups
allBackupIDs.difference_update(backup_control.ListRunningBackups())
# sort it in reverse order - newer backups should be repaired first
allBackupIDs = misc.sorted_backup_ids(list(allBackupIDs), True)
# add backups to the queue
backup_rebuilder.AddBackupsToWork(allBackupIDs)
lg.out(
6, 'backup_monitor.doPrepareListBackups %d items:' %
len(allBackupIDs))
lg.out(6, ' %s' % allBackupIDs)
self.automat('list-backups-done', allBackupIDs)
def IncomingSupplierListFiles(newpacket, list_files_global_id):
"""
Called when command "Files" were received from one of my suppliers.
This is an answer from given supplier (after my request) to get a
list of our files stored on his machine.
"""
supplier_idurl = newpacket.OwnerID
# incoming_key_id = newpacket.PacketID.strip().split(':')[0]
customer_idurl = list_files_global_id['idurl']
num = contactsdb.supplier_position(supplier_idurl, customer_idurl=customer_idurl)
if num < -1:
lg.warn('unknown supplier: %s' % supplier_idurl)
return False
from supplier import list_files
from customer import list_files_orator
target_key_id = my_keys.latest_key_id(list_files_global_id['key_id'])
if not my_keys.is_key_private(target_key_id):
lg.warn('key %r not registered, not possible to decrypt ListFiles() packet from %r' % (target_key_id, supplier_idurl, ))
return False
try:
block = encrypted.Unserialize(newpacket.Payload, decrypt_key=target_key_id, )
input_data = block.Data()
except:
lg.err('failed decrypting data from packet %r received from %r' % (newpacket, supplier_idurl))
return False
list_files_raw = list_files.UnpackListFiles(input_data, settings.ListFilesFormat())
remote_files_changed, backups2remove, paths2remove, missed_backups = backup_matrix.process_raw_list_files(
supplier_num=num,
list_files_text_body=list_files_raw,
customer_idurl=None,
is_in_sync=None,
auto_create=False,
)
list_files_orator.IncomingListFiles(newpacket)
if remote_files_changed:
backup_matrix.SaveLatestRawListFiles(supplier_idurl, list_files_raw)
if _Debug:
lg.args(_DebugLevel, supplier=nameurl.GetName(supplier_idurl), customer=nameurl.GetName(customer_idurl),
backups2remove=len(backups2remove), paths2remove=len(paths2remove),
files_changed=remote_files_changed, missed_backups=len(missed_backups), )
if len(backups2remove) > 0:
p2p_service.RequestDeleteListBackups(backups2remove)
if _Debug:
lg.out(_DebugLevel, ' also sent requests to remove %d backups' % len(backups2remove))
if len(paths2remove) > 0:
p2p_service.RequestDeleteListPaths(paths2remove)
if _Debug:
lg.out(_DebugLevel, ' also sent requests to remove %d paths' % len(paths2remove))
if len(missed_backups) > 0:
from storage import backup_rebuilder
backup_rebuilder.AddBackupsToWork(missed_backups)
backup_rebuilder.A('start')
if _Debug:
lg.out(_DebugLevel, ' also triggered service_rebuilding with %d missed backups' % len(missed_backups))
del backups2remove
del paths2remove
del missed_backups
return True
def IncomingSupplierListFiles(newpacket, list_files_global_id):
"""
Called by ``p2p.p2p_service`` when command "Files" were received from one
of our suppliers.
This is an answer from given supplier (after our request) to get a
list of our files stored on his machine.
"""
from p2p import p2p_service
supplier_idurl = newpacket.OwnerID
# incoming_key_id = newpacket.PacketID.strip().split(':')[0]
customer_idurl = list_files_global_id['idurl']
num = contactsdb.supplier_position(supplier_idurl, customer_idurl=customer_idurl)
if num < -1:
lg.warn('unknown supplier: %s' % supplier_idurl)
return False
from supplier import list_files
from customer import list_files_orator
try:
block = encrypted.Unserialize(
newpacket.Payload,
decrypt_key=my_keys.make_key_id(alias='customer', creator_idurl=my_id.getLocalIDURL(), ),
)
input_data = block.Data()
except:
lg.out(2, 'backup_control.IncomingSupplierListFiles ERROR decrypting data from %s' % newpacket)
return False
src = list_files.UnpackListFiles(input_data, settings.ListFilesFormat())
backups2remove, paths2remove, missed_backups = backup_matrix.ReadRawListFiles(num, src)
list_files_orator.IncomingListFiles(newpacket)
backup_matrix.SaveLatestRawListFiles(supplier_idurl, src)
if _Debug:
lg.out(_DebugLevel, 'backup_control.IncomingSupplierListFiles from [%s]: paths2remove=%d, backups2remove=%d missed_backups=%d' % (
nameurl.GetName(supplier_idurl), len(paths2remove), len(backups2remove), len(missed_backups)))
if len(backups2remove) > 0:
p2p_service.RequestDeleteListBackups(backups2remove)
if _Debug:
lg.out(_DebugLevel, ' also sent requests to remove %d backups' % len(backups2remove))
if len(paths2remove) > 0:
p2p_service.RequestDeleteListPaths(paths2remove)
if _Debug:
lg.out(_DebugLevel, ' also sent requests to remove %d paths' % len(paths2remove))
if len(missed_backups) > 0:
from storage import backup_rebuilder
backup_rebuilder.AddBackupsToWork(missed_backups)
backup_rebuilder.A('start')
if _Debug:
lg.out(_DebugLevel, ' also triggered service_rebuilding with %d missed backups' % len(missed_backups))
del backups2remove
del paths2remove
del missed_backups
return True
def IncomingSupplierListFiles(newpacket):
"""
Called by ``p2p.p2p_service`` when command "Files" were received from one
of our suppliers.
This is an answer from given supplier (after our request) to get a
list of our files stored on his machine.
"""
from p2p import p2p_service
supplier_idurl = newpacket.OwnerID
customer_idurl = my_id.getLocalID()
if newpacket.PacketID.count(':') and newpacket.PacketID.count('@'):
try:
customer_idurl = global_id.GlobalUserToIDURL(newpacket.PacketID.split(':')[0])
except:
lg.exc()
num = contactsdb.supplier_position(supplier_idurl, customer_idurl=customer_idurl)
if num < -1:
lg.out(2, 'backup_control.IncomingSupplierListFiles ERROR unknown supplier: %s' % supplier_idurl)
return False
from supplier import list_files
from customer import list_files_orator
src = list_files.UnpackListFiles(newpacket.Payload, settings.ListFilesFormat())
backups2remove, paths2remove, missed_backups = backup_matrix.ReadRawListFiles(num, src)
list_files_orator.IncomingListFiles(newpacket)
backup_matrix.SaveLatestRawListFiles(supplier_idurl, src)
if _Debug:
lg.out(_DebugLevel, 'backup_control.IncomingSupplierListFiles from [%s]: paths2remove=%d, backups2remove=%d missed_backups=%d' % (
nameurl.GetName(supplier_idurl), len(paths2remove), len(backups2remove), len(missed_backups)))
if len(backups2remove) > 0:
p2p_service.RequestDeleteListBackups(backups2remove)
if _Debug:
lg.out(_DebugLevel, ' also sent requests to remove %d backups' % len(backups2remove))
if len(paths2remove) > 0:
p2p_service.RequestDeleteListPaths(paths2remove)
if _Debug:
lg.out(_DebugLevel, ' also sent requests to remove %d paths' % len(paths2remove))
if len(missed_backups) > 0:
from storage import backup_rebuilder
backup_rebuilder.AddBackupsToWork(missed_backups)
backup_rebuilder.A('start')
if _Debug:
lg.out(_DebugLevel, ' also triggered service_rebuilding with %d missed backups' % len(missed_backups))
del backups2remove
del paths2remove
del missed_backups
return True
