def doSendHisFiles(self, *args, **kwargs):
"""
Action method.
"""
customer_key_id = my_keys.make_key_id(
alias='customer', creator_idurl=self.customer_idurl)
if my_keys.is_key_registered(customer_key_id):
list_files.send(
customer_idurl=self.customer_idurl,
packet_id='%s:%s' % (
customer_key_id,
packetid.UniqueID(),
),
format_type=settings.ListFilesFormat(),
key_id=customer_key_id,
remote_idurl=self.customer_idurl, # send to the customer
)
else:
# if "customer" key is not delivered to me yet, use his "master" key
list_files.send(
customer_idurl=self.customer_idurl,
packet_id='%s:%s' % (
customer_key_id,
packetid.UniqueID(),
),
format_type=settings.ListFilesFormat(),
key_id=my_keys.make_key_id(alias='master',
creator_idurl=self.customer_idurl),
remote_idurl=self.customer_idurl, # send to the customer
)
lg.err('key %s is not registered, not able to send his files' %
customer_key_id)
def _on_list_files(self, newpacket):
from main import settings
if newpacket.Payload != settings.ListFilesFormat():
return False
# TODO: perform validations before sending back list of files
from supplier import list_files
from crypt import my_keys
from userid import global_id
list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID)
if list_files_global_id['key_id']:
# customer id and data id can be recognized from packet id
# return back list of files according to the request
customer_idurl = list_files_global_id['idurl']
key_id = list_files_global_id['key_id']
else:
# packet id format is unknown
# by default returning back all files from that recipient if he is a customer
customer_idurl = newpacket.OwnerID
key_id = my_keys.make_key_id(alias='customer',
creator_idurl=customer_idurl)
list_files.send(
customer_idurl=customer_idurl,
packet_id=newpacket.PacketID,
format_type=settings.ListFilesFormat(),
key_id=key_id,
remote_idurl=newpacket.OwnerID, # send back to the requestor
)
return True
def _on_list_files(self, newpacket):
from main import settings
if newpacket.Payload != settings.ListFilesFormat():
return False
# TODO: perform validations before sending back list of files
from supplier import list_files
list_files.send(newpacket.OwnerID, newpacket.PacketID,
settings.ListFilesFormat())
return True
def SendListFiles(target_supplier, customer_idurl=None, key_id=None, wide=False, callbacks={}):
"""
This is used as a request method from your supplier : if you send him a ListFiles() packet
he will reply you with a list of stored files in a Files() packet.
"""
MyID = my_id.getLocalID()
if not customer_idurl:
customer_idurl = MyID
if not str(target_supplier).isdigit():
RemoteID = target_supplier
else:
RemoteID = contactsdb.supplier(target_supplier, customer_idurl=customer_idurl)
if not RemoteID:
lg.warn("RemoteID is empty target_supplier=%s" % str(target_supplier))
return None
if _Debug:
lg.out(_DebugLevel, "p2p_service.SendListFiles to %s" % nameurl.GetName(RemoteID))
if not key_id:
key_id = global_id.MakeGlobalID(idurl=customer_idurl, key_alias='customer')
PacketID = "%s:%s" % (key_id, packetid.UniqueID(), )
Payload = settings.ListFilesFormat()
result = signed.Packet(
Command=commands.ListFiles(),
OwnerID=MyID,
CreatorID=MyID,
PacketID=PacketID,
Payload=Payload,
RemoteID=RemoteID,
)
gateway.outbox(result, wide=wide, callbacks=callbacks)
return result
def IncomingSupplierListFiles(newpacket, list_files_global_id):
"""
Called when command "Files" were received from one of my suppliers.
This is an answer from given supplier (after my request) to get a
list of our files stored on his machine.
"""
supplier_idurl = newpacket.OwnerID
# incoming_key_id = newpacket.PacketID.strip().split(':')[0]
customer_idurl = list_files_global_id['idurl']
num = contactsdb.supplier_position(supplier_idurl, customer_idurl=customer_idurl)
if num < -1:
lg.warn('unknown supplier: %s' % supplier_idurl)
return False
from supplier import list_files
from customer import list_files_orator
target_key_id = my_keys.latest_key_id(list_files_global_id['key_id'])
if not my_keys.is_key_private(target_key_id):
lg.warn('key %r not registered, not possible to decrypt ListFiles() packet from %r' % (target_key_id, supplier_idurl, ))
return False
try:
block = encrypted.Unserialize(newpacket.Payload, decrypt_key=target_key_id, )
input_data = block.Data()
except:
lg.err('failed decrypting data from packet %r received from %r' % (newpacket, supplier_idurl))
return False
list_files_raw = list_files.UnpackListFiles(input_data, settings.ListFilesFormat())
remote_files_changed, backups2remove, paths2remove, missed_backups = backup_matrix.process_raw_list_files(
supplier_num=num,
list_files_text_body=list_files_raw,
customer_idurl=None,
is_in_sync=None,
auto_create=False,
)
list_files_orator.IncomingListFiles(newpacket)
if remote_files_changed:
backup_matrix.SaveLatestRawListFiles(supplier_idurl, list_files_raw)
if _Debug:
lg.args(_DebugLevel, supplier=nameurl.GetName(supplier_idurl), customer=nameurl.GetName(customer_idurl),
backups2remove=len(backups2remove), paths2remove=len(paths2remove),
files_changed=remote_files_changed, missed_backups=len(missed_backups), )
if len(backups2remove) > 0:
p2p_service.RequestDeleteListBackups(backups2remove)
if _Debug:
lg.out(_DebugLevel, ' also sent requests to remove %d backups' % len(backups2remove))
if len(paths2remove) > 0:
p2p_service.RequestDeleteListPaths(paths2remove)
if _Debug:
lg.out(_DebugLevel, ' also sent requests to remove %d paths' % len(paths2remove))
if len(missed_backups) > 0:
from storage import backup_rebuilder
backup_rebuilder.AddBackupsToWork(missed_backups)
backup_rebuilder.A('start')
if _Debug:
lg.out(_DebugLevel, ' also triggered service_rebuilding with %d missed backups' % len(missed_backups))
del backups2remove
del paths2remove
del missed_backups
return True
def doSendHisFiles(self, arg):
"""
Action method.
"""
packet_id = '%s:%s' % (
global_id.UrlToGlobalID(self.customer_idurl),
packetid.UniqueID(),
)
list_files.send(self.customer_idurl, packet_id,
settings.ListFilesFormat())
def on_list_files(newpacket):
json_query = {}
try:
j = serialization.BytesToDict(newpacket.Payload,
keys_to_text=True,
values_to_text=True)
j['items'][0]
json_query = j
except:
if strng.to_text(newpacket.Payload) == settings.ListFilesFormat():
json_query = {
'items': [
'*',
],
}
if json_query is None:
lg.exc('unrecognized ListFiles() query received')
return False
# TODO: perform validations before sending back list of files
list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID)
if list_files_global_id['key_id']:
# customer id and data id can be recognized from packet id
# return back list of files according to the request
customer_idurl = list_files_global_id['idurl']
key_id = list_files_global_id['key_id']
else:
# packet id format is unknown
# by default returning back all files from that recipient if he is a customer
customer_idurl = newpacket.OwnerID
key_id = my_keys.make_key_id(alias='customer',
creator_idurl=customer_idurl)
key_id = my_keys.latest_key_id(key_id)
list_files.send(
customer_idurl=customer_idurl,
packet_id=newpacket.PacketID,
format_type=settings.ListFilesFormat(),
key_id=key_id,
remote_idurl=newpacket.OwnerID, # send back to the requesting node
query_items=json_query['items'],
)
return True
def IncomingSupplierListFiles(newpacket, list_files_global_id):
"""
Called by ``p2p.p2p_service`` when command "Files" were received from one
of our suppliers.
This is an answer from given supplier (after our request) to get a
list of our files stored on his machine.
"""
from p2p import p2p_service
supplier_idurl = newpacket.OwnerID
# incoming_key_id = newpacket.PacketID.strip().split(':')[0]
customer_idurl = list_files_global_id['idurl']
num = contactsdb.supplier_position(supplier_idurl, customer_idurl=customer_idurl)
if num < -1:
lg.warn('unknown supplier: %s' % supplier_idurl)
return False
from supplier import list_files
from customer import list_files_orator
try:
block = encrypted.Unserialize(
newpacket.Payload,
decrypt_key=my_keys.make_key_id(alias='customer', creator_idurl=my_id.getLocalIDURL(), ),
)
input_data = block.Data()
except:
lg.out(2, 'backup_control.IncomingSupplierListFiles ERROR decrypting data from %s' % newpacket)
return False
src = list_files.UnpackListFiles(input_data, settings.ListFilesFormat())
backups2remove, paths2remove, missed_backups = backup_matrix.ReadRawListFiles(num, src)
list_files_orator.IncomingListFiles(newpacket)
backup_matrix.SaveLatestRawListFiles(supplier_idurl, src)
if _Debug:
lg.out(_DebugLevel, 'backup_control.IncomingSupplierListFiles from [%s]: paths2remove=%d, backups2remove=%d missed_backups=%d' % (
nameurl.GetName(supplier_idurl), len(paths2remove), len(backups2remove), len(missed_backups)))
if len(backups2remove) > 0:
p2p_service.RequestDeleteListBackups(backups2remove)
if _Debug:
lg.out(_DebugLevel, ' also sent requests to remove %d backups' % len(backups2remove))
if len(paths2remove) > 0:
p2p_service.RequestDeleteListPaths(paths2remove)
if _Debug:
lg.out(_DebugLevel, ' also sent requests to remove %d paths' % len(paths2remove))
if len(missed_backups) > 0:
from storage import backup_rebuilder
backup_rebuilder.AddBackupsToWork(missed_backups)
backup_rebuilder.A('start')
if _Debug:
lg.out(_DebugLevel, ' also triggered service_rebuilding with %d missed backups' % len(missed_backups))
del backups2remove
del paths2remove
del missed_backups
return True
def doSendHisFiles(self, arg):
"""
Action method.
"""
customer_key_id = my_keys.make_key_id(alias='customer', creator_idurl=self.customer_idurl)
if my_keys.is_key_registered(customer_key_id):
list_files.send(
customer_idurl=self.customer_idurl,
packet_id='%s:%s' % (customer_key_id, packetid.UniqueID(), ),
format_type=settings.ListFilesFormat(),
key_id=customer_key_id,
remote_idurl=self.customer_idurl, # send to the customer
)
else:
lg.warn('key %s is not registered, not able to send his files' % customer_key_id)
def IncomingSupplierListFiles(newpacket):
"""
Called by ``p2p.p2p_service`` when command "Files" were received from one
of our suppliers.
This is an answer from given supplier (after our request) to get a
list of our files stored on his machine.
"""
from p2p import p2p_service
supplier_idurl = newpacket.OwnerID
customer_idurl = my_id.getLocalID()
if newpacket.PacketID.count(':') and newpacket.PacketID.count('@'):
try:
customer_idurl = global_id.GlobalUserToIDURL(newpacket.PacketID.split(':')[0])
except:
lg.exc()
num = contactsdb.supplier_position(supplier_idurl, customer_idurl=customer_idurl)
if num < -1:
lg.out(2, 'backup_control.IncomingSupplierListFiles ERROR unknown supplier: %s' % supplier_idurl)
return False
from supplier import list_files
from customer import list_files_orator
src = list_files.UnpackListFiles(newpacket.Payload, settings.ListFilesFormat())
backups2remove, paths2remove, missed_backups = backup_matrix.ReadRawListFiles(num, src)
list_files_orator.IncomingListFiles(newpacket)
backup_matrix.SaveLatestRawListFiles(supplier_idurl, src)
if _Debug:
lg.out(_DebugLevel, 'backup_control.IncomingSupplierListFiles from [%s]: paths2remove=%d, backups2remove=%d missed_backups=%d' % (
nameurl.GetName(supplier_idurl), len(paths2remove), len(backups2remove), len(missed_backups)))
if len(backups2remove) > 0:
p2p_service.RequestDeleteListBackups(backups2remove)
if _Debug:
lg.out(_DebugLevel, ' also sent requests to remove %d backups' % len(backups2remove))
if len(paths2remove) > 0:
p2p_service.RequestDeleteListPaths(paths2remove)
if _Debug:
lg.out(_DebugLevel, ' also sent requests to remove %d paths' % len(paths2remove))
if len(missed_backups) > 0:
from storage import backup_rebuilder
backup_rebuilder.AddBackupsToWork(missed_backups)
backup_rebuilder.A('start')
if _Debug:
lg.out(_DebugLevel, ' also triggered service_rebuilding with %d missed backups' % len(missed_backups))
del backups2remove
del paths2remove
del missed_backups
return True
def SendRequestListFiles(supplierNumORidurl, customer_idurl=None):
MyID = my_id.getLocalID()
if not customer_idurl:
customer_idurl = MyID
if not str(supplierNumORidurl).isdigit():
RemoteID = supplierNumORidurl
else:
RemoteID = contactsdb.supplier(supplierNumORidurl,
customer_idurl=customer_idurl)
if not RemoteID:
lg.warn("RemoteID is empty supplierNumORidurl=%s" %
str(supplierNumORidurl))
return None
if _Debug:
lg.out(
_DebugLevel, "p2p_service.SendRequestListFiles [%s]" %
nameurl.GetName(RemoteID))
PacketID = "%s:%s" % (global_id.UrlToGlobalID(customer_idurl),
packetid.UniqueID())
Payload = settings.ListFilesFormat()
result = signed.Packet(commands.ListFiles(), MyID, MyID, PacketID, Payload,
RemoteID)
gateway.outbox(result)
return result
def _on_files_received(self, newpacket, info):
from logs import lg
from lib import serialization
from main import settings
from main import events
from p2p import p2p_service
from storage import backup_fs
from storage import backup_control
from crypt import encrypted
from crypt import my_keys
from userid import my_id
from userid import global_id
from storage import backup_matrix
from supplier import list_files
from contacts import contactsdb
list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID)
if not list_files_global_id['idurl']:
lg.warn('invalid PacketID: %s' % newpacket.PacketID)
return False
trusted_customer_idurl = list_files_global_id['idurl']
incoming_key_id = list_files_global_id['key_id']
if trusted_customer_idurl == my_id.getGlobalID():
lg.warn('skip %s packet which seems to came from my own supplier' %
newpacket)
# only process list Files() from other users who granted me access
return False
if not my_keys.is_valid_key_id(incoming_key_id):
lg.warn('ignore, invalid key id in packet %s' % newpacket)
return False
if not my_keys.is_key_private(incoming_key_id):
lg.warn('private key is not registered : %s' % incoming_key_id)
p2p_service.SendFail(newpacket, 'private key is not registered')
return False
try:
block = encrypted.Unserialize(
newpacket.Payload,
decrypt_key=incoming_key_id,
)
except:
lg.exc(newpacket.Payload)
return False
if block is None:
lg.warn('failed reading data from %s' % newpacket.RemoteID)
return False
# if block.CreatorID != trusted_customer_idurl:
# lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % (
# block.CreatorID, list_files_global_id['idurl'], ))
# return False
try:
raw_files = block.Data()
except:
lg.exc()
return False
if block.CreatorID == trusted_customer_idurl:
# this is a trusted guy sending some shared files to me
try:
json_data = serialization.BytesToDict(raw_files,
keys_to_text=True)
json_data['items']
except:
lg.exc()
return False
count = backup_fs.Unserialize(
raw_data=json_data,
iter=backup_fs.fs(trusted_customer_idurl),
iterID=backup_fs.fsID(trusted_customer_idurl),
from_json=True,
)
p2p_service.SendAck(newpacket)
events.send(
'shared-list-files-received',
dict(
customer_idurl=trusted_customer_idurl,
new_items=count,
))
if count == 0:
lg.warn('no files were imported during file sharing')
else:
backup_control.Save()
lg.info('imported %d shared files from %s, key_id=%s' % (
count,
trusted_customer_idurl,
incoming_key_id,
))
return True
# otherwise this must be an external supplier sending us a files he stores for trusted customer
external_supplier_idurl = block.CreatorID
try:
supplier_raw_list_files = list_files.UnpackListFiles(
raw_files, settings.ListFilesFormat())
backup_matrix.SaveLatestRawListFiles(
supplier_idurl=external_supplier_idurl,
raw_data=supplier_raw_list_files,
customer_idurl=trusted_customer_idurl,
)
except:
lg.exc()
return False
# need to detect supplier position from the list of packets
# and place that supplier on the correct position in contactsdb
real_supplier_pos = backup_matrix.DetectSupplierPosition(
supplier_raw_list_files)
known_supplier_pos = contactsdb.supplier_position(
external_supplier_idurl, trusted_customer_idurl)
if real_supplier_pos >= 0:
if known_supplier_pos >= 0 and known_supplier_pos != real_supplier_pos:
lg.warn(
'external supplier %s position is not matching to list files, rewriting for customer %s'
% (external_supplier_idurl, trusted_customer_idurl))
contactsdb.erase_supplier(
idurl=external_supplier_idurl,
customer_idurl=trusted_customer_idurl,
)
contactsdb.add_supplier(
idurl=external_supplier_idurl,
position=real_supplier_pos,
customer_idurl=trusted_customer_idurl,
)
contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl)
else:
lg.warn(
'not possible to detect external supplier position for customer %s'
% trusted_customer_idurl)
# finally send ack packet back
p2p_service.SendAck(newpacket)
lg.info(
'received list of packets from external supplier %s for customer %s'
% (external_supplier_idurl, trusted_customer_idurl))
return True
def on_files_received(newpacket, info):
list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID)
if not list_files_global_id['idurl']:
lg.warn('invalid PacketID: %s' % newpacket.PacketID)
return False
trusted_customer_idurl = list_files_global_id['idurl']
incoming_key_id = list_files_global_id['key_id']
if trusted_customer_idurl == my_id.getLocalID():
if _Debug:
lg.dbg(_DebugLevel, 'ignore %s packet which seems to came from my own supplier' % newpacket)
# only process list Files() from other customers who granted me access to their files
return False
if not my_keys.is_valid_key_id(incoming_key_id):
lg.warn('ignore, invalid key id in packet %s' % newpacket)
return False
if not my_keys.is_key_private(incoming_key_id):
lg.warn('private key is not registered : %s' % incoming_key_id)
p2p_service.SendFail(newpacket, 'private key is not registered')
return False
try:
block = encrypted.Unserialize(
newpacket.Payload,
decrypt_key=incoming_key_id,
)
except:
lg.exc(newpacket.Payload)
return False
if block is None:
lg.warn('failed reading data from %s' % newpacket.RemoteID)
return False
# if block.CreatorID != trusted_customer_idurl:
# lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % (
# block.CreatorID, list_files_global_id['idurl'], ))
# return False
try:
raw_files = block.Data()
except:
lg.exc()
return False
if block.CreatorID == trusted_customer_idurl:
# this is a trusted guy sending some shared files to me
try:
json_data = serialization.BytesToDict(raw_files, keys_to_text=True, encoding='utf-8')
json_data['items']
except:
lg.exc()
return False
count = backup_fs.Unserialize(
raw_data=json_data,
iter=backup_fs.fs(trusted_customer_idurl),
iterID=backup_fs.fsID(trusted_customer_idurl),
from_json=True,
)
p2p_service.SendAck(newpacket)
if count == 0:
lg.warn('no files were imported during file sharing')
else:
backup_control.Save()
lg.info('imported %d shared files from %s, key_id=%s' % (
count, trusted_customer_idurl, incoming_key_id, ))
events.send('shared-list-files-received', dict(
customer_idurl=trusted_customer_idurl,
new_items=count,
))
return True
# otherwise this must be an external supplier sending us a files he stores for trusted customer
external_supplier_idurl = block.CreatorID
try:
supplier_raw_list_files = list_files.UnpackListFiles(raw_files, settings.ListFilesFormat())
except:
lg.exc()
return False
# need to detect supplier position from the list of packets
# and place that supplier on the correct position in contactsdb
supplier_pos = backup_matrix.DetectSupplierPosition(supplier_raw_list_files)
known_supplier_pos = contactsdb.supplier_position(external_supplier_idurl, trusted_customer_idurl)
if _Debug:
lg.args(_DebugLevel, supplier_pos=supplier_pos, known_supplier_pos=known_supplier_pos, external_supplier=external_supplier_idurl,
trusted_customer=trusted_customer_idurl, key_id=incoming_key_id)
if supplier_pos >= 0:
if known_supplier_pos >= 0 and known_supplier_pos != supplier_pos:
lg.err('known external supplier %r position %d is not matching to received list files position %d for customer %s' % (
external_supplier_idurl, known_supplier_pos, supplier_pos, trusted_customer_idurl))
# TODO: we should remove that bellow because we do not need it
# service_customer_family() should take care of suppliers list for trusted customer
# so we need to just read that list from DHT
# contactsdb.erase_supplier(
# idurl=external_supplier_idurl,
# customer_idurl=trusted_customer_idurl,
# )
# contactsdb.add_supplier(
# idurl=external_supplier_idurl,
# position=supplier_pos,
# customer_idurl=trusted_customer_idurl,
# )
# contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl)
else:
lg.warn('not possible to detect external supplier position for customer %s from received list files, known position is %s' % (
trusted_customer_idurl, known_supplier_pos))
supplier_pos = known_supplier_pos
remote_files_changed, _, _, _ = backup_matrix.process_raw_list_files(
supplier_num=supplier_pos,
list_files_text_body=supplier_raw_list_files,
customer_idurl=trusted_customer_idurl,
is_in_sync=True,
auto_create=True,
)
if remote_files_changed:
backup_matrix.SaveLatestRawListFiles(
supplier_idurl=external_supplier_idurl,
raw_data=supplier_raw_list_files,
customer_idurl=trusted_customer_idurl,
)
# finally sending Ack() packet back
p2p_service.SendAck(newpacket)
if remote_files_changed:
lg.info('received updated list of files from external supplier %s for customer %s' % (external_supplier_idurl, trusted_customer_idurl))
return True
