def populate_messages(recipient_id=None, sender_id=None, message_types=[], offset=0, limit=100): if recipient_id: if not recipient_id.count('@'): from contacts import contactsdb recipient_idurl = contactsdb.find_correspondent_by_nickname( recipient_id) if not recipient_idurl: lg.err('recipient %r was not found' % recipient_id) return recipient_id = global_id.UrlToGlobalID(recipient_idurl) recipient_glob_id = global_id.ParseGlobalID(recipient_id) if not recipient_glob_id['idurl']: lg.err('wrong recipient_id') return recipient_id = global_id.MakeGlobalID(**recipient_glob_id) if not my_keys.is_valid_key_id(recipient_id): lg.err('invalid recipient_id: %s' % recipient_id) return if sender_id: sender_local_key_id = my_keys.get_local_key_id(sender_id) if sender_local_key_id is None: return if recipient_id: recipient_local_key_id = my_keys.get_local_key_id(recipient_id) if recipient_local_key_id is None: lg.warn('recipient %r local key id was not registered' % recipient_id) return for row in query_messages( sender_id=sender_id, recipient_id=recipient_id, bidirectional=False, message_types=message_types, offset=offset, limit=limit, raw_results=True, ): conversation_id = get_conversation_id(row[0], row[2], int(row[5])) if conversation_id is None: continue snap_id = '{}/{}'.format(conversation_id, row[7]) listeners.push_snapshot('message', snap_id=snap_id, created=row[6], data=build_json_message( sender=row[1], recipient=row[3], direction='in' if row[4] == 0 else 'out', conversation_id=conversation_id, message_type=MESSAGE_TYPE_CODES.get( int(row[5]), 'private_message'), message_time=row[6], message_id=row[7], data=json.loads(row[8]), ))
def on_audit_key_received(newpacket, info, status, error_message): """ Callback will be executed when remote user would like to check if I poses given key locally. """ block = encrypted.Unserialize(newpacket.Payload) if block is None: lg.out( 2, 'key_ring.on_audit_key_received ERROR reading data from %s' % newpacket.RemoteID) return False try: raw_payload = block.Data() json_payload = serialization.BytesToDict(raw_payload, keys_to_text=True, values_to_text=True) key_id = json_payload['key_id'] json_payload['audit'] public_sample = base64.b64decode( json_payload['audit']['public_sample']) private_sample = base64.b64decode( json_payload['audit']['private_sample']) except Exception as exc: lg.exc() p2p_service.SendFail(newpacket, str(exc)) return False if not my_keys.is_valid_key_id(key_id): p2p_service.SendFail(newpacket, 'invalid key id') return False if not my_keys.is_key_registered(key_id, include_master=True): p2p_service.SendFail(newpacket, 'key not registered') return False if public_sample: response_payload = base64.b64encode( my_keys.encrypt(key_id, public_sample)) p2p_service.SendAck(newpacket, response_payload) if _Debug: lg.info('remote user %s requested audit of public key %s' % (newpacket.OwnerID, key_id)) return True if private_sample: if not my_keys.is_key_private(key_id): p2p_service.SendFail(newpacket, 'private key not registered') return False response_payload = base64.b64encode( my_keys.decrypt(key_id, private_sample)) p2p_service.SendAck(newpacket, response_payload) if _Debug: lg.info('remote user %s requested audit of private key %s' % (newpacket.OwnerID, key_id)) return True p2p_service.SendFail(newpacket, 'wrong audit request') return False
def decrypt(self, inp, private_key=None): """ Decrypt `inp` data with private key provided (callable, key_id or openssh format), or by using locally stored "master" key. """ if private_key is None: return key.DecryptLocalPrivateKey(inp) if callable(private_key): return private_key(inp) from crypt import my_keys if my_keys.is_valid_key_id(private_key): return my_keys.decrypt(private_key, inp) return key.DecryptOpenSSHPrivateKey(private_key, inp)
def _on_files_received(self, newpacket, info): import json from logs import lg from p2p import p2p_service from storage import backup_fs from storage import backup_control from crypt import encrypted from crypt import my_keys from userid import my_id from userid import global_id try: user_id = newpacket.PacketID.strip().split(':')[0] if user_id == my_id.getGlobalID(): # skip my own Files() packets which comes from my suppliers # only process list Files() from other users who granted me access return False key_id = user_id if not my_keys.is_valid_key_id(key_id): # ignore, invalid key id in packet id return False if not my_keys.is_key_private(key_id): raise Exception('private key is not registered') except Exception as exc: lg.warn(str(exc)) p2p_service.SendFail(newpacket, str(exc)) return False block = encrypted.Unserialize(newpacket.Payload) if block is None: lg.warn('failed reading data from %s' % newpacket.RemoteID) return False if block.CreatorID != global_id.GlobalUserToIDURL(user_id): lg.warn( 'invalid packet, creator ID must be present in packet ID : %s ~ %s' % ( block.CreatorID, user_id, )) return False try: json_data = json.loads(block.Data(), encoding='utf-8') json_data['items'] customer_idurl = block.CreatorID count = backup_fs.Unserialize( raw_data=json_data, iter=backup_fs.fs(customer_idurl), iterID=backup_fs.fsID(customer_idurl), from_json=True, ) except Exception as exc: lg.exc() p2p_service.SendFail(newpacket, str(exc)) return False p2p_service.SendAck(newpacket) if count == 0: lg.warn('no files were imported during file sharing') else: backup_control.Save() lg.info('imported %d shared files from %s, key_id=%s' % ( count, customer_idurl, key_id, )) return True # from access import shared_access_coordinator # this_share = shared_access_coordinator.get_active_share(key_id) # if not this_share: # lg.warn('share is not opened: %s' % key_id) # p2p_service.SendFail(newpacket, 'share is not opened') # return False # this_share.automat('customer-list-files-received', (newpacket, info, block, )) return True
def request(self, json_payload, newpacket, info): from twisted.internet import reactor # @UnresolvedImport from logs import lg from main import events from crypt import my_keys from p2p import p2p_service from contacts import contactsdb from storage import accounting from userid import global_id customer_idurl = newpacket.OwnerID customer_id = global_id.UrlToGlobalID(customer_idurl) bytes_for_customer = 0 try: bytes_for_customer = int(json_payload['needed_bytes']) except: lg.warn("wrong payload" % newpacket.Payload) return p2p_service.SendFail(newpacket, 'wrong payload') try: customer_public_key = json_payload['customer_public_key'] customer_public_key_id = customer_public_key['key_id'] except: customer_public_key = None customer_public_key_id = None data_owner_idurl = None target_customer_idurl = None family_position = json_payload.get('position') ecc_map = json_payload.get('ecc_map') family_snapshot = json_payload.get('family_snapshot') key_id = json_payload.get('key_id') target_customer_id = json_payload.get('customer_id') if key_id: # this is a request from external user to access shared data stored by one of my customers # this is "second" customer requesting data from "first" customer if not key_id or not my_keys.is_valid_key_id(key_id): lg.warn('missed or invalid key id') return p2p_service.SendFail(newpacket, 'invalid key id') target_customer_idurl = global_id.GlobalUserToIDURL( target_customer_id) if not contactsdb.is_customer(target_customer_idurl): lg.warn("target user %s is not a customer" % target_customer_id) return p2p_service.SendFail(newpacket, 'not a customer') if target_customer_idurl == customer_idurl: lg.warn('customer %s requesting shared access to own files' % customer_idurl) return p2p_service.SendFail(newpacket, 'invalid case') if not my_keys.is_key_registered(key_id): lg.warn('key not registered: %s' % key_id) p2p_service.SendFail(newpacket, 'key not registered') return False data_owner_idurl = my_keys.split_key_id(key_id)[1] if data_owner_idurl != target_customer_idurl and data_owner_idurl != customer_idurl: # pretty complex scenario: # external customer requesting access to data which belongs not to that customer # this is "third" customer accessing data belongs to "second" customer # TODO: for now just stop it lg.warn( 'under construction, key_id=%s customer_idurl=%s target_customer_idurl=%s' % ( key_id, customer_idurl, target_customer_idurl, )) p2p_service.SendFail(newpacket, 'under construction') return False # do not create connection with that customer, only accept the request lg.info( 'external customer %s requested access to shared data at %s' % ( customer_id, key_id, )) return p2p_service.SendAck(newpacket, 'accepted') # key_id is not present in the request: # this is a request to connect new customer (or reconnect existing one) to that supplier if not bytes_for_customer or bytes_for_customer < 0: lg.warn("wrong payload : %s" % newpacket.Payload) return p2p_service.SendFail(newpacket, 'wrong storage value') current_customers = contactsdb.customers() if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.request created a new space file') space_dict = accounting.read_customers_quotas() try: free_bytes = int(space_dict[b'free']) except: lg.exc() return p2p_service.SendFail(newpacket, 'broken space file') if (customer_idurl not in current_customers and customer_idurl in list(space_dict.keys())): lg.warn("broken space file") return p2p_service.SendFail(newpacket, 'broken space file') if (customer_idurl in current_customers and customer_idurl not in list(space_dict.keys())): lg.warn("broken customers file") return p2p_service.SendFail(newpacket, 'broken customers file') if customer_idurl in current_customers: free_bytes += int(space_dict.get(customer_idurl, 0)) space_dict[b'free'] = free_bytes current_customers.remove(customer_idurl) space_dict.pop(customer_idurl) new_customer = False else: new_customer = True lg.out( 8, ' new_customer=%s current_allocated_bytes=%s' % ( new_customer, space_dict.get(customer_idurl), )) from supplier import local_tester if free_bytes <= bytes_for_customer: contactsdb.update_customers(current_customers) contactsdb.remove_customer_meta_info(customer_idurl) contactsdb.save_customers() accounting.write_customers_quotas(space_dict) if customer_public_key_id: my_keys.erase_key(customer_public_key_id) reactor.callLater( 0, local_tester.TestUpdateCustomers) # @UndefinedVariable if new_customer: lg.out( 8, " NEW CUSTOMER: DENIED !!!!!!!!!!! not enough space available" ) events.send('new-customer-denied', dict(idurl=customer_idurl)) else: lg.out( 8, " OLD CUSTOMER: DENIED !!!!!!!!!!! not enough space available" ) events.send('existing-customer-denied', dict(idurl=customer_idurl)) return p2p_service.SendAck(newpacket, 'deny') space_dict[b'free'] = free_bytes - bytes_for_customer current_customers.append(customer_idurl) space_dict[customer_idurl] = bytes_for_customer contactsdb.update_customers(current_customers) contactsdb.save_customers() contactsdb.add_customer_meta_info( customer_idurl, { 'ecc_map': ecc_map, 'position': family_position, 'family_snapshot': family_snapshot, }) accounting.write_customers_quotas(space_dict) if customer_public_key_id: my_keys.erase_key(customer_public_key_id) try: if not my_keys.is_key_registered(customer_public_key_id): key_id, key_object = my_keys.read_key_info( customer_public_key) if not my_keys.register_key(key_id, key_object): lg.err('failed to register customer public key') except: lg.exc() else: lg.warn('customer public key was not provided in the request') reactor.callLater( 0, local_tester.TestUpdateCustomers) # @UndefinedVariable if new_customer: lg.out( 8, " NEW CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s" % (customer_idurl, family_position, ecc_map, bytes_for_customer)) lg.out( 8, " family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot, ) events.send( 'new-customer-accepted', dict( idurl=customer_idurl, allocated_bytes=bytes_for_customer, ecc_map=ecc_map, position=family_position, family_snapshot=family_snapshot, key_id=customer_public_key_id, )) else: lg.out( 8, " OLD CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s" % (customer_idurl, family_position, ecc_map, bytes_for_customer)) lg.out( 8, " family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot) events.send( 'existing-customer-accepted', dict( idurl=customer_idurl, allocated_bytes=bytes_for_customer, ecc_map=ecc_map, position=family_position, key_id=customer_public_key_id, family_snapshot=family_snapshot, )) return p2p_service.SendAck(newpacket, 'accepted')
def _on_files_received(self, newpacket, info): from logs import lg from lib import serialization from main import settings from main import events from p2p import p2p_service from storage import backup_fs from storage import backup_control from crypt import encrypted from crypt import my_keys from userid import my_id from userid import global_id from storage import backup_matrix from supplier import list_files from contacts import contactsdb list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID) if not list_files_global_id['idurl']: lg.warn('invalid PacketID: %s' % newpacket.PacketID) return False trusted_customer_idurl = list_files_global_id['idurl'] incoming_key_id = list_files_global_id['key_id'] if trusted_customer_idurl == my_id.getGlobalID(): lg.warn('skip %s packet which seems to came from my own supplier' % newpacket) # only process list Files() from other users who granted me access return False if not my_keys.is_valid_key_id(incoming_key_id): lg.warn('ignore, invalid key id in packet %s' % newpacket) return False if not my_keys.is_key_private(incoming_key_id): lg.warn('private key is not registered : %s' % incoming_key_id) p2p_service.SendFail(newpacket, 'private key is not registered') return False try: block = encrypted.Unserialize( newpacket.Payload, decrypt_key=incoming_key_id, ) except: lg.exc(newpacket.Payload) return False if block is None: lg.warn('failed reading data from %s' % newpacket.RemoteID) return False # if block.CreatorID != trusted_customer_idurl: # lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % ( # block.CreatorID, list_files_global_id['idurl'], )) # return False try: raw_files = block.Data() except: lg.exc() return False if block.CreatorID == trusted_customer_idurl: # this is a trusted guy sending some shared files to me try: json_data = serialization.BytesToDict(raw_files, keys_to_text=True) json_data['items'] except: lg.exc() return False count = backup_fs.Unserialize( raw_data=json_data, iter=backup_fs.fs(trusted_customer_idurl), iterID=backup_fs.fsID(trusted_customer_idurl), from_json=True, ) p2p_service.SendAck(newpacket) events.send( 'shared-list-files-received', dict( customer_idurl=trusted_customer_idurl, new_items=count, )) if count == 0: lg.warn('no files were imported during file sharing') else: backup_control.Save() lg.info('imported %d shared files from %s, key_id=%s' % ( count, trusted_customer_idurl, incoming_key_id, )) return True # otherwise this must be an external supplier sending us a files he stores for trusted customer external_supplier_idurl = block.CreatorID try: supplier_raw_list_files = list_files.UnpackListFiles( raw_files, settings.ListFilesFormat()) backup_matrix.SaveLatestRawListFiles( supplier_idurl=external_supplier_idurl, raw_data=supplier_raw_list_files, customer_idurl=trusted_customer_idurl, ) except: lg.exc() return False # need to detect supplier position from the list of packets # and place that supplier on the correct position in contactsdb real_supplier_pos = backup_matrix.DetectSupplierPosition( supplier_raw_list_files) known_supplier_pos = contactsdb.supplier_position( external_supplier_idurl, trusted_customer_idurl) if real_supplier_pos >= 0: if known_supplier_pos >= 0 and known_supplier_pos != real_supplier_pos: lg.warn( 'external supplier %s position is not matching to list files, rewriting for customer %s' % (external_supplier_idurl, trusted_customer_idurl)) contactsdb.erase_supplier( idurl=external_supplier_idurl, customer_idurl=trusted_customer_idurl, ) contactsdb.add_supplier( idurl=external_supplier_idurl, position=real_supplier_pos, customer_idurl=trusted_customer_idurl, ) contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl) else: lg.warn( 'not possible to detect external supplier position for customer %s' % trusted_customer_idurl) # finally send ack packet back p2p_service.SendAck(newpacket) lg.info( 'received list of packets from external supplier %s for customer %s' % (external_supplier_idurl, trusted_customer_idurl)) return True
def on_files_received(newpacket, info): list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID) if not list_files_global_id['idurl']: lg.warn('invalid PacketID: %s' % newpacket.PacketID) return False trusted_customer_idurl = list_files_global_id['idurl'] incoming_key_id = list_files_global_id['key_id'] if trusted_customer_idurl == my_id.getLocalID(): if _Debug: lg.dbg(_DebugLevel, 'ignore %s packet which seems to came from my own supplier' % newpacket) # only process list Files() from other customers who granted me access to their files return False if not my_keys.is_valid_key_id(incoming_key_id): lg.warn('ignore, invalid key id in packet %s' % newpacket) return False if not my_keys.is_key_private(incoming_key_id): lg.warn('private key is not registered : %s' % incoming_key_id) p2p_service.SendFail(newpacket, 'private key is not registered') return False try: block = encrypted.Unserialize( newpacket.Payload, decrypt_key=incoming_key_id, ) except: lg.exc(newpacket.Payload) return False if block is None: lg.warn('failed reading data from %s' % newpacket.RemoteID) return False # if block.CreatorID != trusted_customer_idurl: # lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % ( # block.CreatorID, list_files_global_id['idurl'], )) # return False try: raw_files = block.Data() except: lg.exc() return False if block.CreatorID == trusted_customer_idurl: # this is a trusted guy sending some shared files to me try: json_data = serialization.BytesToDict(raw_files, keys_to_text=True, encoding='utf-8') json_data['items'] except: lg.exc() return False count = backup_fs.Unserialize( raw_data=json_data, iter=backup_fs.fs(trusted_customer_idurl), iterID=backup_fs.fsID(trusted_customer_idurl), from_json=True, ) p2p_service.SendAck(newpacket) if count == 0: lg.warn('no files were imported during file sharing') else: backup_control.Save() lg.info('imported %d shared files from %s, key_id=%s' % ( count, trusted_customer_idurl, incoming_key_id, )) events.send('shared-list-files-received', dict( customer_idurl=trusted_customer_idurl, new_items=count, )) return True # otherwise this must be an external supplier sending us a files he stores for trusted customer external_supplier_idurl = block.CreatorID try: supplier_raw_list_files = list_files.UnpackListFiles(raw_files, settings.ListFilesFormat()) except: lg.exc() return False # need to detect supplier position from the list of packets # and place that supplier on the correct position in contactsdb supplier_pos = backup_matrix.DetectSupplierPosition(supplier_raw_list_files) known_supplier_pos = contactsdb.supplier_position(external_supplier_idurl, trusted_customer_idurl) if _Debug: lg.args(_DebugLevel, supplier_pos=supplier_pos, known_supplier_pos=known_supplier_pos, external_supplier=external_supplier_idurl, trusted_customer=trusted_customer_idurl, key_id=incoming_key_id) if supplier_pos >= 0: if known_supplier_pos >= 0 and known_supplier_pos != supplier_pos: lg.err('known external supplier %r position %d is not matching to received list files position %d for customer %s' % ( external_supplier_idurl, known_supplier_pos, supplier_pos, trusted_customer_idurl)) # TODO: we should remove that bellow because we do not need it # service_customer_family() should take care of suppliers list for trusted customer # so we need to just read that list from DHT # contactsdb.erase_supplier( # idurl=external_supplier_idurl, # customer_idurl=trusted_customer_idurl, # ) # contactsdb.add_supplier( # idurl=external_supplier_idurl, # position=supplier_pos, # customer_idurl=trusted_customer_idurl, # ) # contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl) else: lg.warn('not possible to detect external supplier position for customer %s from received list files, known position is %s' % ( trusted_customer_idurl, known_supplier_pos)) supplier_pos = known_supplier_pos remote_files_changed, _, _, _ = backup_matrix.process_raw_list_files( supplier_num=supplier_pos, list_files_text_body=supplier_raw_list_files, customer_idurl=trusted_customer_idurl, is_in_sync=True, auto_create=True, ) if remote_files_changed: backup_matrix.SaveLatestRawListFiles( supplier_idurl=external_supplier_idurl, raw_data=supplier_raw_list_files, customer_idurl=trusted_customer_idurl, ) # finally sending Ack() packet back p2p_service.SendAck(newpacket) if remote_files_changed: lg.info('received updated list of files from external supplier %s for customer %s' % (external_supplier_idurl, trusted_customer_idurl)) return True