def _on_files_received(self, newpacket, info):
from logs import lg
from lib import serialization
from main import settings
from main import events
from p2p import p2p_service
from storage import backup_fs
from storage import backup_control
from crypt import encrypted
from crypt import my_keys
from userid import my_id
from userid import global_id
from storage import backup_matrix
from supplier import list_files
from contacts import contactsdb
list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID)
if not list_files_global_id['idurl']:
lg.warn('invalid PacketID: %s' % newpacket.PacketID)
return False
trusted_customer_idurl = list_files_global_id['idurl']
incoming_key_id = list_files_global_id['key_id']
if trusted_customer_idurl == my_id.getGlobalID():
lg.warn('skip %s packet which seems to came from my own supplier' %
newpacket)
# only process list Files() from other users who granted me access
return False
if not my_keys.is_valid_key_id(incoming_key_id):
lg.warn('ignore, invalid key id in packet %s' % newpacket)
return False
if not my_keys.is_key_private(incoming_key_id):
lg.warn('private key is not registered : %s' % incoming_key_id)
p2p_service.SendFail(newpacket, 'private key is not registered')
return False
try:
block = encrypted.Unserialize(
newpacket.Payload,
decrypt_key=incoming_key_id,
)
except:
lg.exc(newpacket.Payload)
return False
if block is None:
lg.warn('failed reading data from %s' % newpacket.RemoteID)
return False
# if block.CreatorID != trusted_customer_idurl:
# lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % (
# block.CreatorID, list_files_global_id['idurl'], ))
# return False
try:
raw_files = block.Data()
except:
lg.exc()
return False
if block.CreatorID == trusted_customer_idurl:
# this is a trusted guy sending some shared files to me
try:
json_data = serialization.BytesToDict(raw_files,
keys_to_text=True)
json_data['items']
except:
lg.exc()
return False
count = backup_fs.Unserialize(
raw_data=json_data,
iter=backup_fs.fs(trusted_customer_idurl),
iterID=backup_fs.fsID(trusted_customer_idurl),
from_json=True,
)
p2p_service.SendAck(newpacket)
events.send(
'shared-list-files-received',
dict(
customer_idurl=trusted_customer_idurl,
new_items=count,
))
if count == 0:
lg.warn('no files were imported during file sharing')
else:
backup_control.Save()
lg.info('imported %d shared files from %s, key_id=%s' % (
count,
trusted_customer_idurl,
incoming_key_id,
))
return True
# otherwise this must be an external supplier sending us a files he stores for trusted customer
external_supplier_idurl = block.CreatorID
try:
supplier_raw_list_files = list_files.UnpackListFiles(
raw_files, settings.ListFilesFormat())
backup_matrix.SaveLatestRawListFiles(
supplier_idurl=external_supplier_idurl,
raw_data=supplier_raw_list_files,
customer_idurl=trusted_customer_idurl,
)
except:
lg.exc()
return False
# need to detect supplier position from the list of packets
# and place that supplier on the correct position in contactsdb
real_supplier_pos = backup_matrix.DetectSupplierPosition(
supplier_raw_list_files)
known_supplier_pos = contactsdb.supplier_position(
external_supplier_idurl, trusted_customer_idurl)
if real_supplier_pos >= 0:
if known_supplier_pos >= 0 and known_supplier_pos != real_supplier_pos:
lg.warn(
'external supplier %s position is not matching to list files, rewriting for customer %s'
% (external_supplier_idurl, trusted_customer_idurl))
contactsdb.erase_supplier(
idurl=external_supplier_idurl,
customer_idurl=trusted_customer_idurl,
)
contactsdb.add_supplier(
idurl=external_supplier_idurl,
position=real_supplier_pos,
customer_idurl=trusted_customer_idurl,
)
contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl)
else:
lg.warn(
'not possible to detect external supplier position for customer %s'
% trusted_customer_idurl)
# finally send ack packet back
p2p_service.SendAck(newpacket)
lg.info(
'received list of packets from external supplier %s for customer %s'
% (external_supplier_idurl, trusted_customer_idurl))
return True
def doCleanUpBackups(self, arg):
# here we check all backups we have and remove the old one
# user can set how many versions of that file or folder to keep
# other versions (older) will be removed here
from storage import backup_rebuilder
try:
self.backups_progress_last_iteration = len(backup_rebuilder.A().backupsWasRebuilt)
except:
self.backups_progress_last_iteration = 0
versionsToKeep = settings.getBackupsMaxCopies()
if not contactsdb.num_suppliers():
bytesUsed = 0
else:
bytesUsed = backup_fs.sizebackups() / contactsdb.num_suppliers()
bytesNeeded = diskspace.GetBytesFromString(settings.getNeededString(), 0)
customerGlobID = my_id.getGlobalID()
if _Debug:
lg.out(_DebugLevel, 'backup_monitor.doCleanUpBackups backupsToKeep=%d used=%d needed=%d' % (versionsToKeep, bytesUsed, bytesNeeded))
delete_count = 0
if versionsToKeep > 0:
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
pathID = global_id.CanonicalID(pathID)
if backup_control.IsPathInProcess(pathID):
continue
versions = itemInfo.list_versions()
# TODO: do we need to sort the list? it comes from a set, so must be sorted may be
while len(versions) > versionsToKeep:
backupID = packetid.MakeBackupID(customerGlobID, pathID, versions.pop(0))
if _Debug:
lg.out(_DebugLevel, 'backup_monitor.doCleanUpBackups %d of %d backups for %s, so remove older %s' % (
len(versions), versionsToKeep, localPath, backupID))
backup_control.DeleteBackup(backupID, saveDB=False, calculate=False)
delete_count += 1
# we need also to fit used space into needed space (given from other users)
# they trust us - do not need to take extra space from our friends
# so remove oldest backups, but keep at least one for every folder - at least locally!
# still our suppliers will remove our "extra" files by their "local_tester"
if bytesNeeded <= bytesUsed:
sizeOk = False
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
if sizeOk:
break
pathID = global_id.CanonicalID(pathID)
versions = itemInfo.list_versions(True, False)
if len(versions) <= 1:
continue
for version in versions[1:]:
backupID = packetid.MakeBackupID(customerGlobID, pathID, version)
versionInfo = itemInfo.get_version_info(version)
if versionInfo[1] > 0:
if _Debug:
lg.out(_DebugLevel, 'backup_monitor.doCleanUpBackups over use %d of %d, so remove %s of %s' % (
bytesUsed, bytesNeeded, backupID, localPath))
backup_control.DeleteBackup(backupID, saveDB=False, calculate=False)
delete_count += 1
bytesUsed -= versionInfo[1]
if bytesNeeded > bytesUsed:
sizeOk = True
break
if delete_count > 0:
backup_fs.Scan()
backup_fs.Calculate()
backup_control.Save()
from main import control
control.request_update()
collected = gc.collect()
if self.backups_progress_last_iteration > 0:
if _Debug:
lg.out(_DebugLevel, 'backup_monitor.doCleanUpBackups sending "restart", backups_progress_last_iteration=%s' %
self.backups_progress_last_iteration)
reactor.callLater(1, self.automat, 'restart')
if _Debug:
lg.out(_DebugLevel, 'backup_monitor.doCleanUpBackups collected %d objects' % collected)
def on_files_received(newpacket, info):
list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID)
if not list_files_global_id['idurl']:
lg.warn('invalid PacketID: %s' % newpacket.PacketID)
return False
trusted_customer_idurl = list_files_global_id['idurl']
incoming_key_id = list_files_global_id['key_id']
if trusted_customer_idurl == my_id.getLocalID():
if _Debug:
lg.dbg(_DebugLevel, 'ignore %s packet which seems to came from my own supplier' % newpacket)
# only process list Files() from other customers who granted me access to their files
return False
if not my_keys.is_valid_key_id(incoming_key_id):
lg.warn('ignore, invalid key id in packet %s' % newpacket)
return False
if not my_keys.is_key_private(incoming_key_id):
lg.warn('private key is not registered : %s' % incoming_key_id)
p2p_service.SendFail(newpacket, 'private key is not registered')
return False
try:
block = encrypted.Unserialize(
newpacket.Payload,
decrypt_key=incoming_key_id,
)
except:
lg.exc(newpacket.Payload)
return False
if block is None:
lg.warn('failed reading data from %s' % newpacket.RemoteID)
return False
# if block.CreatorID != trusted_customer_idurl:
# lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % (
# block.CreatorID, list_files_global_id['idurl'], ))
# return False
try:
raw_files = block.Data()
except:
lg.exc()
return False
if block.CreatorID == trusted_customer_idurl:
# this is a trusted guy sending some shared files to me
try:
json_data = serialization.BytesToDict(raw_files, keys_to_text=True, encoding='utf-8')
json_data['items']
except:
lg.exc()
return False
count = backup_fs.Unserialize(
raw_data=json_data,
iter=backup_fs.fs(trusted_customer_idurl),
iterID=backup_fs.fsID(trusted_customer_idurl),
from_json=True,
)
p2p_service.SendAck(newpacket)
if count == 0:
lg.warn('no files were imported during file sharing')
else:
backup_control.Save()
lg.info('imported %d shared files from %s, key_id=%s' % (
count, trusted_customer_idurl, incoming_key_id, ))
events.send('shared-list-files-received', dict(
customer_idurl=trusted_customer_idurl,
new_items=count,
))
return True
# otherwise this must be an external supplier sending us a files he stores for trusted customer
external_supplier_idurl = block.CreatorID
try:
supplier_raw_list_files = list_files.UnpackListFiles(raw_files, settings.ListFilesFormat())
except:
lg.exc()
return False
# need to detect supplier position from the list of packets
# and place that supplier on the correct position in contactsdb
supplier_pos = backup_matrix.DetectSupplierPosition(supplier_raw_list_files)
known_supplier_pos = contactsdb.supplier_position(external_supplier_idurl, trusted_customer_idurl)
if _Debug:
lg.args(_DebugLevel, supplier_pos=supplier_pos, known_supplier_pos=known_supplier_pos, external_supplier=external_supplier_idurl,
trusted_customer=trusted_customer_idurl, key_id=incoming_key_id)
if supplier_pos >= 0:
if known_supplier_pos >= 0 and known_supplier_pos != supplier_pos:
lg.err('known external supplier %r position %d is not matching to received list files position %d for customer %s' % (
external_supplier_idurl, known_supplier_pos, supplier_pos, trusted_customer_idurl))
# TODO: we should remove that bellow because we do not need it
# service_customer_family() should take care of suppliers list for trusted customer
# so we need to just read that list from DHT
# contactsdb.erase_supplier(
# idurl=external_supplier_idurl,
# customer_idurl=trusted_customer_idurl,
# )
# contactsdb.add_supplier(
# idurl=external_supplier_idurl,
# position=supplier_pos,
# customer_idurl=trusted_customer_idurl,
# )
# contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl)
else:
lg.warn('not possible to detect external supplier position for customer %s from received list files, known position is %s' % (
trusted_customer_idurl, known_supplier_pos))
supplier_pos = known_supplier_pos
remote_files_changed, _, _, _ = backup_matrix.process_raw_list_files(
supplier_num=supplier_pos,
list_files_text_body=supplier_raw_list_files,
customer_idurl=trusted_customer_idurl,
is_in_sync=True,
auto_create=True,
)
if remote_files_changed:
backup_matrix.SaveLatestRawListFiles(
supplier_idurl=external_supplier_idurl,
raw_data=supplier_raw_list_files,
customer_idurl=trusted_customer_idurl,
)
# finally sending Ack() packet back
p2p_service.SendAck(newpacket)
if remote_files_changed:
lg.info('received updated list of files from external supplier %s for customer %s' % (external_supplier_idurl, trusted_customer_idurl))
return True