def doSendHisFiles(self, *args, **kwargs): """ Action method. """ customer_key_id = my_keys.make_key_id( alias='customer', creator_idurl=self.customer_idurl) if my_keys.is_key_registered(customer_key_id): list_files.send( customer_idurl=self.customer_idurl, packet_id='%s:%s' % ( customer_key_id, packetid.UniqueID(), ), format_type=settings.ListFilesFormat(), key_id=customer_key_id, remote_idurl=self.customer_idurl, # send to the customer ) else: # if "customer" key is not delivered to me yet, use his "master" key list_files.send( customer_idurl=self.customer_idurl, packet_id='%s:%s' % ( customer_key_id, packetid.UniqueID(), ), format_type=settings.ListFilesFormat(), key_id=my_keys.make_key_id(alias='master', creator_idurl=self.customer_idurl), remote_idurl=self.customer_idurl, # send to the customer ) lg.err('key %s is not registered, not able to send his files' % customer_key_id)
def _on_list_files(self, newpacket): from main import settings if newpacket.Payload != settings.ListFilesFormat(): return False # TODO: perform validations before sending back list of files from supplier import list_files from crypt import my_keys from userid import global_id list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID) if list_files_global_id['key_id']: # customer id and data id can be recognized from packet id # return back list of files according to the request customer_idurl = list_files_global_id['idurl'] key_id = list_files_global_id['key_id'] else: # packet id format is unknown # by default returning back all files from that recipient if he is a customer customer_idurl = newpacket.OwnerID key_id = my_keys.make_key_id(alias='customer', creator_idurl=customer_idurl) list_files.send( customer_idurl=customer_idurl, packet_id=newpacket.PacketID, format_type=settings.ListFilesFormat(), key_id=key_id, remote_idurl=newpacket.OwnerID, # send back to the requestor ) return True
def _on_list_files(self, newpacket): from main import settings if newpacket.Payload != settings.ListFilesFormat(): return False # TODO: perform validations before sending back list of files from supplier import list_files list_files.send(newpacket.OwnerID, newpacket.PacketID, settings.ListFilesFormat()) return True
def SendListFiles(target_supplier, customer_idurl=None, key_id=None, wide=False, callbacks={}): """ This is used as a request method from your supplier : if you send him a ListFiles() packet he will reply you with a list of stored files in a Files() packet. """ MyID = my_id.getLocalID() if not customer_idurl: customer_idurl = MyID if not str(target_supplier).isdigit(): RemoteID = target_supplier else: RemoteID = contactsdb.supplier(target_supplier, customer_idurl=customer_idurl) if not RemoteID: lg.warn("RemoteID is empty target_supplier=%s" % str(target_supplier)) return None if _Debug: lg.out(_DebugLevel, "p2p_service.SendListFiles to %s" % nameurl.GetName(RemoteID)) if not key_id: key_id = global_id.MakeGlobalID(idurl=customer_idurl, key_alias='customer') PacketID = "%s:%s" % (key_id, packetid.UniqueID(), ) Payload = settings.ListFilesFormat() result = signed.Packet( Command=commands.ListFiles(), OwnerID=MyID, CreatorID=MyID, PacketID=PacketID, Payload=Payload, RemoteID=RemoteID, ) gateway.outbox(result, wide=wide, callbacks=callbacks) return result
def IncomingSupplierListFiles(newpacket, list_files_global_id): """ Called when command "Files" were received from one of my suppliers. This is an answer from given supplier (after my request) to get a list of our files stored on his machine. """ supplier_idurl = newpacket.OwnerID # incoming_key_id = newpacket.PacketID.strip().split(':')[0] customer_idurl = list_files_global_id['idurl'] num = contactsdb.supplier_position(supplier_idurl, customer_idurl=customer_idurl) if num < -1: lg.warn('unknown supplier: %s' % supplier_idurl) return False from supplier import list_files from customer import list_files_orator target_key_id = my_keys.latest_key_id(list_files_global_id['key_id']) if not my_keys.is_key_private(target_key_id): lg.warn('key %r not registered, not possible to decrypt ListFiles() packet from %r' % (target_key_id, supplier_idurl, )) return False try: block = encrypted.Unserialize(newpacket.Payload, decrypt_key=target_key_id, ) input_data = block.Data() except: lg.err('failed decrypting data from packet %r received from %r' % (newpacket, supplier_idurl)) return False list_files_raw = list_files.UnpackListFiles(input_data, settings.ListFilesFormat()) remote_files_changed, backups2remove, paths2remove, missed_backups = backup_matrix.process_raw_list_files( supplier_num=num, list_files_text_body=list_files_raw, customer_idurl=None, is_in_sync=None, auto_create=False, ) list_files_orator.IncomingListFiles(newpacket) if remote_files_changed: backup_matrix.SaveLatestRawListFiles(supplier_idurl, list_files_raw) if _Debug: lg.args(_DebugLevel, supplier=nameurl.GetName(supplier_idurl), customer=nameurl.GetName(customer_idurl), backups2remove=len(backups2remove), paths2remove=len(paths2remove), files_changed=remote_files_changed, missed_backups=len(missed_backups), ) if len(backups2remove) > 0: p2p_service.RequestDeleteListBackups(backups2remove) if _Debug: lg.out(_DebugLevel, ' also sent requests to remove %d backups' % len(backups2remove)) if len(paths2remove) > 0: p2p_service.RequestDeleteListPaths(paths2remove) if _Debug: lg.out(_DebugLevel, ' also sent requests to remove %d paths' % len(paths2remove)) if len(missed_backups) > 0: from storage import backup_rebuilder backup_rebuilder.AddBackupsToWork(missed_backups) backup_rebuilder.A('start') if _Debug: lg.out(_DebugLevel, ' also triggered service_rebuilding with %d missed backups' % len(missed_backups)) del backups2remove del paths2remove del missed_backups return True
def doSendHisFiles(self, arg): """ Action method. """ packet_id = '%s:%s' % ( global_id.UrlToGlobalID(self.customer_idurl), packetid.UniqueID(), ) list_files.send(self.customer_idurl, packet_id, settings.ListFilesFormat())
def on_list_files(newpacket): json_query = {} try: j = serialization.BytesToDict(newpacket.Payload, keys_to_text=True, values_to_text=True) j['items'][0] json_query = j except: if strng.to_text(newpacket.Payload) == settings.ListFilesFormat(): json_query = { 'items': [ '*', ], } if json_query is None: lg.exc('unrecognized ListFiles() query received') return False # TODO: perform validations before sending back list of files list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID) if list_files_global_id['key_id']: # customer id and data id can be recognized from packet id # return back list of files according to the request customer_idurl = list_files_global_id['idurl'] key_id = list_files_global_id['key_id'] else: # packet id format is unknown # by default returning back all files from that recipient if he is a customer customer_idurl = newpacket.OwnerID key_id = my_keys.make_key_id(alias='customer', creator_idurl=customer_idurl) key_id = my_keys.latest_key_id(key_id) list_files.send( customer_idurl=customer_idurl, packet_id=newpacket.PacketID, format_type=settings.ListFilesFormat(), key_id=key_id, remote_idurl=newpacket.OwnerID, # send back to the requesting node query_items=json_query['items'], ) return True
def IncomingSupplierListFiles(newpacket, list_files_global_id): """ Called by ``p2p.p2p_service`` when command "Files" were received from one of our suppliers. This is an answer from given supplier (after our request) to get a list of our files stored on his machine. """ from p2p import p2p_service supplier_idurl = newpacket.OwnerID # incoming_key_id = newpacket.PacketID.strip().split(':')[0] customer_idurl = list_files_global_id['idurl'] num = contactsdb.supplier_position(supplier_idurl, customer_idurl=customer_idurl) if num < -1: lg.warn('unknown supplier: %s' % supplier_idurl) return False from supplier import list_files from customer import list_files_orator try: block = encrypted.Unserialize( newpacket.Payload, decrypt_key=my_keys.make_key_id(alias='customer', creator_idurl=my_id.getLocalIDURL(), ), ) input_data = block.Data() except: lg.out(2, 'backup_control.IncomingSupplierListFiles ERROR decrypting data from %s' % newpacket) return False src = list_files.UnpackListFiles(input_data, settings.ListFilesFormat()) backups2remove, paths2remove, missed_backups = backup_matrix.ReadRawListFiles(num, src) list_files_orator.IncomingListFiles(newpacket) backup_matrix.SaveLatestRawListFiles(supplier_idurl, src) if _Debug: lg.out(_DebugLevel, 'backup_control.IncomingSupplierListFiles from [%s]: paths2remove=%d, backups2remove=%d missed_backups=%d' % ( nameurl.GetName(supplier_idurl), len(paths2remove), len(backups2remove), len(missed_backups))) if len(backups2remove) > 0: p2p_service.RequestDeleteListBackups(backups2remove) if _Debug: lg.out(_DebugLevel, ' also sent requests to remove %d backups' % len(backups2remove)) if len(paths2remove) > 0: p2p_service.RequestDeleteListPaths(paths2remove) if _Debug: lg.out(_DebugLevel, ' also sent requests to remove %d paths' % len(paths2remove)) if len(missed_backups) > 0: from storage import backup_rebuilder backup_rebuilder.AddBackupsToWork(missed_backups) backup_rebuilder.A('start') if _Debug: lg.out(_DebugLevel, ' also triggered service_rebuilding with %d missed backups' % len(missed_backups)) del backups2remove del paths2remove del missed_backups return True
def doSendHisFiles(self, arg): """ Action method. """ customer_key_id = my_keys.make_key_id(alias='customer', creator_idurl=self.customer_idurl) if my_keys.is_key_registered(customer_key_id): list_files.send( customer_idurl=self.customer_idurl, packet_id='%s:%s' % (customer_key_id, packetid.UniqueID(), ), format_type=settings.ListFilesFormat(), key_id=customer_key_id, remote_idurl=self.customer_idurl, # send to the customer ) else: lg.warn('key %s is not registered, not able to send his files' % customer_key_id)
def IncomingSupplierListFiles(newpacket): """ Called by ``p2p.p2p_service`` when command "Files" were received from one of our suppliers. This is an answer from given supplier (after our request) to get a list of our files stored on his machine. """ from p2p import p2p_service supplier_idurl = newpacket.OwnerID customer_idurl = my_id.getLocalID() if newpacket.PacketID.count(':') and newpacket.PacketID.count('@'): try: customer_idurl = global_id.GlobalUserToIDURL(newpacket.PacketID.split(':')[0]) except: lg.exc() num = contactsdb.supplier_position(supplier_idurl, customer_idurl=customer_idurl) if num < -1: lg.out(2, 'backup_control.IncomingSupplierListFiles ERROR unknown supplier: %s' % supplier_idurl) return False from supplier import list_files from customer import list_files_orator src = list_files.UnpackListFiles(newpacket.Payload, settings.ListFilesFormat()) backups2remove, paths2remove, missed_backups = backup_matrix.ReadRawListFiles(num, src) list_files_orator.IncomingListFiles(newpacket) backup_matrix.SaveLatestRawListFiles(supplier_idurl, src) if _Debug: lg.out(_DebugLevel, 'backup_control.IncomingSupplierListFiles from [%s]: paths2remove=%d, backups2remove=%d missed_backups=%d' % ( nameurl.GetName(supplier_idurl), len(paths2remove), len(backups2remove), len(missed_backups))) if len(backups2remove) > 0: p2p_service.RequestDeleteListBackups(backups2remove) if _Debug: lg.out(_DebugLevel, ' also sent requests to remove %d backups' % len(backups2remove)) if len(paths2remove) > 0: p2p_service.RequestDeleteListPaths(paths2remove) if _Debug: lg.out(_DebugLevel, ' also sent requests to remove %d paths' % len(paths2remove)) if len(missed_backups) > 0: from storage import backup_rebuilder backup_rebuilder.AddBackupsToWork(missed_backups) backup_rebuilder.A('start') if _Debug: lg.out(_DebugLevel, ' also triggered service_rebuilding with %d missed backups' % len(missed_backups)) del backups2remove del paths2remove del missed_backups return True
def SendRequestListFiles(supplierNumORidurl, customer_idurl=None): MyID = my_id.getLocalID() if not customer_idurl: customer_idurl = MyID if not str(supplierNumORidurl).isdigit(): RemoteID = supplierNumORidurl else: RemoteID = contactsdb.supplier(supplierNumORidurl, customer_idurl=customer_idurl) if not RemoteID: lg.warn("RemoteID is empty supplierNumORidurl=%s" % str(supplierNumORidurl)) return None if _Debug: lg.out( _DebugLevel, "p2p_service.SendRequestListFiles [%s]" % nameurl.GetName(RemoteID)) PacketID = "%s:%s" % (global_id.UrlToGlobalID(customer_idurl), packetid.UniqueID()) Payload = settings.ListFilesFormat() result = signed.Packet(commands.ListFiles(), MyID, MyID, PacketID, Payload, RemoteID) gateway.outbox(result) return result
def _on_files_received(self, newpacket, info): from logs import lg from lib import serialization from main import settings from main import events from p2p import p2p_service from storage import backup_fs from storage import backup_control from crypt import encrypted from crypt import my_keys from userid import my_id from userid import global_id from storage import backup_matrix from supplier import list_files from contacts import contactsdb list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID) if not list_files_global_id['idurl']: lg.warn('invalid PacketID: %s' % newpacket.PacketID) return False trusted_customer_idurl = list_files_global_id['idurl'] incoming_key_id = list_files_global_id['key_id'] if trusted_customer_idurl == my_id.getGlobalID(): lg.warn('skip %s packet which seems to came from my own supplier' % newpacket) # only process list Files() from other users who granted me access return False if not my_keys.is_valid_key_id(incoming_key_id): lg.warn('ignore, invalid key id in packet %s' % newpacket) return False if not my_keys.is_key_private(incoming_key_id): lg.warn('private key is not registered : %s' % incoming_key_id) p2p_service.SendFail(newpacket, 'private key is not registered') return False try: block = encrypted.Unserialize( newpacket.Payload, decrypt_key=incoming_key_id, ) except: lg.exc(newpacket.Payload) return False if block is None: lg.warn('failed reading data from %s' % newpacket.RemoteID) return False # if block.CreatorID != trusted_customer_idurl: # lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % ( # block.CreatorID, list_files_global_id['idurl'], )) # return False try: raw_files = block.Data() except: lg.exc() return False if block.CreatorID == trusted_customer_idurl: # this is a trusted guy sending some shared files to me try: json_data = serialization.BytesToDict(raw_files, keys_to_text=True) json_data['items'] except: lg.exc() return False count = backup_fs.Unserialize( raw_data=json_data, iter=backup_fs.fs(trusted_customer_idurl), iterID=backup_fs.fsID(trusted_customer_idurl), from_json=True, ) p2p_service.SendAck(newpacket) events.send( 'shared-list-files-received', dict( customer_idurl=trusted_customer_idurl, new_items=count, )) if count == 0: lg.warn('no files were imported during file sharing') else: backup_control.Save() lg.info('imported %d shared files from %s, key_id=%s' % ( count, trusted_customer_idurl, incoming_key_id, )) return True # otherwise this must be an external supplier sending us a files he stores for trusted customer external_supplier_idurl = block.CreatorID try: supplier_raw_list_files = list_files.UnpackListFiles( raw_files, settings.ListFilesFormat()) backup_matrix.SaveLatestRawListFiles( supplier_idurl=external_supplier_idurl, raw_data=supplier_raw_list_files, customer_idurl=trusted_customer_idurl, ) except: lg.exc() return False # need to detect supplier position from the list of packets # and place that supplier on the correct position in contactsdb real_supplier_pos = backup_matrix.DetectSupplierPosition( supplier_raw_list_files) known_supplier_pos = contactsdb.supplier_position( external_supplier_idurl, trusted_customer_idurl) if real_supplier_pos >= 0: if known_supplier_pos >= 0 and known_supplier_pos != real_supplier_pos: lg.warn( 'external supplier %s position is not matching to list files, rewriting for customer %s' % (external_supplier_idurl, trusted_customer_idurl)) contactsdb.erase_supplier( idurl=external_supplier_idurl, customer_idurl=trusted_customer_idurl, ) contactsdb.add_supplier( idurl=external_supplier_idurl, position=real_supplier_pos, customer_idurl=trusted_customer_idurl, ) contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl) else: lg.warn( 'not possible to detect external supplier position for customer %s' % trusted_customer_idurl) # finally send ack packet back p2p_service.SendAck(newpacket) lg.info( 'received list of packets from external supplier %s for customer %s' % (external_supplier_idurl, trusted_customer_idurl)) return True
def on_files_received(newpacket, info): list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID) if not list_files_global_id['idurl']: lg.warn('invalid PacketID: %s' % newpacket.PacketID) return False trusted_customer_idurl = list_files_global_id['idurl'] incoming_key_id = list_files_global_id['key_id'] if trusted_customer_idurl == my_id.getLocalID(): if _Debug: lg.dbg(_DebugLevel, 'ignore %s packet which seems to came from my own supplier' % newpacket) # only process list Files() from other customers who granted me access to their files return False if not my_keys.is_valid_key_id(incoming_key_id): lg.warn('ignore, invalid key id in packet %s' % newpacket) return False if not my_keys.is_key_private(incoming_key_id): lg.warn('private key is not registered : %s' % incoming_key_id) p2p_service.SendFail(newpacket, 'private key is not registered') return False try: block = encrypted.Unserialize( newpacket.Payload, decrypt_key=incoming_key_id, ) except: lg.exc(newpacket.Payload) return False if block is None: lg.warn('failed reading data from %s' % newpacket.RemoteID) return False # if block.CreatorID != trusted_customer_idurl: # lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % ( # block.CreatorID, list_files_global_id['idurl'], )) # return False try: raw_files = block.Data() except: lg.exc() return False if block.CreatorID == trusted_customer_idurl: # this is a trusted guy sending some shared files to me try: json_data = serialization.BytesToDict(raw_files, keys_to_text=True, encoding='utf-8') json_data['items'] except: lg.exc() return False count = backup_fs.Unserialize( raw_data=json_data, iter=backup_fs.fs(trusted_customer_idurl), iterID=backup_fs.fsID(trusted_customer_idurl), from_json=True, ) p2p_service.SendAck(newpacket) if count == 0: lg.warn('no files were imported during file sharing') else: backup_control.Save() lg.info('imported %d shared files from %s, key_id=%s' % ( count, trusted_customer_idurl, incoming_key_id, )) events.send('shared-list-files-received', dict( customer_idurl=trusted_customer_idurl, new_items=count, )) return True # otherwise this must be an external supplier sending us a files he stores for trusted customer external_supplier_idurl = block.CreatorID try: supplier_raw_list_files = list_files.UnpackListFiles(raw_files, settings.ListFilesFormat()) except: lg.exc() return False # need to detect supplier position from the list of packets # and place that supplier on the correct position in contactsdb supplier_pos = backup_matrix.DetectSupplierPosition(supplier_raw_list_files) known_supplier_pos = contactsdb.supplier_position(external_supplier_idurl, trusted_customer_idurl) if _Debug: lg.args(_DebugLevel, supplier_pos=supplier_pos, known_supplier_pos=known_supplier_pos, external_supplier=external_supplier_idurl, trusted_customer=trusted_customer_idurl, key_id=incoming_key_id) if supplier_pos >= 0: if known_supplier_pos >= 0 and known_supplier_pos != supplier_pos: lg.err('known external supplier %r position %d is not matching to received list files position %d for customer %s' % ( external_supplier_idurl, known_supplier_pos, supplier_pos, trusted_customer_idurl)) # TODO: we should remove that bellow because we do not need it # service_customer_family() should take care of suppliers list for trusted customer # so we need to just read that list from DHT # contactsdb.erase_supplier( # idurl=external_supplier_idurl, # customer_idurl=trusted_customer_idurl, # ) # contactsdb.add_supplier( # idurl=external_supplier_idurl, # position=supplier_pos, # customer_idurl=trusted_customer_idurl, # ) # contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl) else: lg.warn('not possible to detect external supplier position for customer %s from received list files, known position is %s' % ( trusted_customer_idurl, known_supplier_pos)) supplier_pos = known_supplier_pos remote_files_changed, _, _, _ = backup_matrix.process_raw_list_files( supplier_num=supplier_pos, list_files_text_body=supplier_raw_list_files, customer_idurl=trusted_customer_idurl, is_in_sync=True, auto_create=True, ) if remote_files_changed: backup_matrix.SaveLatestRawListFiles( supplier_idurl=external_supplier_idurl, raw_data=supplier_raw_list_files, customer_idurl=trusted_customer_idurl, ) # finally sending Ack() packet back p2p_service.SendAck(newpacket) if remote_files_changed: lg.info('received updated list of files from external supplier %s for customer %s' % (external_supplier_idurl, trusted_customer_idurl)) return True