def create_archive_folder(group_key_id, force_path_id=None): group_key_alias, group_creator_idurl = my_keys.split_key_id(group_key_id) catalog_path = os.path.join('.archive', group_key_alias) archive_folder_catalog_path = global_id.MakeGlobalID( key_alias=group_key_alias, customer=group_creator_idurl.to_id(), path=catalog_path) res = api.file_exists(archive_folder_catalog_path) if res['status'] != 'OK': lg.err('failed to check archive folder in the catalog: %r' % res) return None if res['result']['exist']: ret = res['result']['path_id'] if force_path_id is not None: if force_path_id != ret: lg.err('archive folder exists, but have different path ID in the catalog: %r' % ret) return None return ret res = api.file_create(archive_folder_catalog_path, as_folder=True, exist_ok=True, force_path_id=force_path_id) if res['status'] != 'OK': lg.err('failed to create archive folder in the catalog: %r' % res) return None if res['result']['created']: lg.info('created new archive folder in the catalog: %r' % res) else: lg.info('archive folder already exist in the catalog: %r' % res) ret = res['result']['path_id'] if force_path_id is not None: if force_path_id != ret: lg.err('archive folder exists, but have different path ID in the catalog: %r' % ret) return None return ret
def audit_private_key(key_id, untrusted_idurl, timeout=10): """ Be sure remote user posses given private key. I need to posses the public key to be able to audit. I will generate a random string, encrypt it with given key public key and send encrypted string to him. He will decrypt and send me back original string. Returns Deferred object. """ if _Debug: lg.out(_DebugLevel, 'key_ring.audit_private_key testing %s from %s' % (key_id, untrusted_idurl)) result = Deferred() recipient_id_obj = identitycache.FromCache(untrusted_idurl) if not recipient_id_obj: lg.warn('not found "%s" in identity cache' % untrusted_idurl) result.errback(Exception('not found "%s" in identity cache' % untrusted_idurl)) return result key_alias, creator_idurl = my_keys.split_key_id(key_id) if not key_alias or not creator_idurl: lg.warn('wrong key_id') result.errback(Exception('wrong key_id')) return result private_test_sample = key.NewSessionKey() if untrusted_idurl == creator_idurl and key_alias == 'master': lg.warn('doing audit of master key (private part) of remote user') private_test_encrypted_sample = recipient_id_obj.encrypt(private_test_sample) else: if not my_keys.is_key_registered(key_id): lg.warn('unknown key: "%s"' % key_id) result.errback(Exception('unknown key: "%s"' % key_id)) return result private_test_encrypted_sample = my_keys.encrypt(key_id, private_test_sample) json_payload = { 'key_id': key_id, 'audit': { 'public_sample': '', 'private_sample': base64.b64encode(private_test_encrypted_sample), } } raw_payload = serialization.DictToBytes(json_payload, values_to_text=True) block = encrypted.Block( BackupID=key_id, Data=raw_payload, SessionKey=key.NewSessionKey(), # encrypt data using public key of recipient EncryptKey=lambda inp: recipient_id_obj.encrypt(inp), ) encrypted_payload = block.Serialize() p2p_service.SendAuditKey( remote_idurl=recipient_id_obj.getIDURL(), encrypted_payload=encrypted_payload, packet_id=key_id, timeout=timeout, callbacks={ commands.Ack(): lambda response, info: _on_audit_private_key_response(response, info, key_id, untrusted_idurl, private_test_sample, result), commands.Fail(): lambda response, info: result.errback(Exception(response)), None: lambda pkt_out: result.errback(Exception('timeout')), # timeout }, ) return result
def _on_audit_public_key_response(response, info, key_id, untrusted_idurl, test_sample, result): try: response_sample = base64.b64decode(response.Payload) except: lg.exc() result.callback(False) return False key_alias, creator_idurl = my_keys.split_key_id(key_id) if creator_idurl == untrusted_idurl and key_alias == 'master': recipient_id_obj = identitycache.FromCache(creator_idurl) if not recipient_id_obj: lg.warn('not found "%s" in identity cache' % creator_idurl) result.errback(Exception('not found "%s" in identity cache' % creator_idurl)) return result orig_sample = recipient_id_obj.encrypt(test_sample) else: orig_sample = my_keys.encrypt(key_id, test_sample) if response_sample == orig_sample: if _Debug: lg.out(_DebugLevel, 'key_ring._on_audit_public_key_response : %s on %s' % (key_id, untrusted_idurl, )) lg.out(_DebugLevel, ' is OK !!!!!!!!!!!!!!!!!!!!!!!!!') result.callback(True) return True lg.warn('key %s on %s is not OK' % (key_id, untrusted_idurl, )) result.callback(False) return False
def transfer_key(key_id, trusted_idurl, include_private=False, include_signature=False, timeout=10, result=None): """ Actually sending given key to remote user. """ if _Debug: lg.out(_DebugLevel, 'key_ring.transfer_key %s -> %s' % (key_id, trusted_idurl)) if not result: result = Deferred() recipient_id_obj = identitycache.FromCache(trusted_idurl) if not recipient_id_obj: lg.warn('not found "%s" in identity cache' % trusted_idurl) result.errback(Exception('not found "%s" in identity cache' % trusted_idurl)) return result key_alias, creator_idurl = my_keys.split_key_id(key_id) if not key_alias or not creator_idurl: lg.warn('wrong key_id') result.errback(Exception('wrong key_id')) return result if not my_keys.is_key_registered(key_id): lg.warn('unknown key: "%s"' % key_id) result.errback(Exception('unknown key: "%s"' % key_id)) return result key_object = my_keys.key_obj(key_id) try: key_json = my_keys.make_key_info( key_object, key_id=key_id, include_private=include_private, include_signature=include_signature, ) except Exception as exc: lg.exc() result.errback(exc) return result key_data = serialization.DictToBytes(key_json, values_to_text=True) block = encrypted.Block( BackupID=key_id, Data=key_data, SessionKey=key.NewSessionKey(session_key_type=key.SessionKeyType()), SessionKeyType=key.SessionKeyType(), # encrypt data using public key of recipient EncryptKey=lambda inp: recipient_id_obj.encrypt(inp), ) encrypted_key_data = block.Serialize() p2p_service.SendKey( remote_idurl=recipient_id_obj.getIDURL(), encrypted_key_data=encrypted_key_data, packet_id=key_id, callbacks={ commands.Ack(): lambda response, info: _on_transfer_key_response(response, info, key_id, result), commands.Fail(): lambda response, info: _on_transfer_key_response(response, info, key_id, result), None: lambda pkt_out: _on_transfer_key_response(None, None, key_id, result), }, timeout=timeout, ) return result
def transfer_private_key(key_id, idurl): if _Debug: lg.out(_DebugLevel, 'key_ring.transfer_private_key %s -> %s' % (key_id, idurl)) result = Deferred() recipient_id_obj = identitycache.FromCache(idurl) if not recipient_id_obj: lg.warn('not found "%s" in identity cache' % idurl) result.errback(Exception('not found "%s" in identity cache' % idurl)) return result key_alias, creator_idurl = my_keys.split_key_id(key_id) if not key_alias or not creator_idurl: lg.warn('wrong key_id') result.errback(Exception('wrong key_id')) return result key_object = my_keys.known_keys().get(key_id) if key_object is None: lg.warn('unknown key: "%s"' % key_id) result.errback(Exception('unknown key: "%s"' % key_id)) return result key_json = { 'key_id': key_id, 'alias': key_alias, 'creator': creator_idurl, 'fingerprint': str(key_object.fingerprint()), 'type': str(key_object.type()), 'ssh_type': str(key_object.sshType()), 'size': str(key_object.size()), 'public': str(key_object.public().toString('openssh')), 'private': str(key_object.toString('openssh')), } key_data = json.dumps(key_json) block = encrypted.Block( BackupID=key_id, Data=key_data, SessionKey=key.NewSessionKey(), # encrypt data using public key of recipient EncryptKey=lambda inp: recipient_id_obj.encrypt(inp), ) encrypted_key_data = block.Serialize() p2p_service.SendKey( remote_idurl=recipient_id_obj.getIDURL(), encrypted_key_data=encrypted_key_data, packet_id=key_id, callbacks={ commands.Ack(): lambda response, info: result.callback(response), commands.Fail(): lambda response, info: result.errback(Exception(response)), }, ) return result
def audit_public_key(key_id, untrusted_idurl, timeout=10): """ Be sure remote user posses given public key. I also need to posses that public key in order to do such audit. I will send him a random string, he needs to encrypt it and send me back. I can compare his encrypted output with mine. Returns Deferred object. """ if _Debug: lg.out( _DebugLevel, 'key_ring.audit_public_key testing %s from %s' % (key_id, untrusted_idurl)) result = Deferred() recipient_id_obj = identitycache.FromCache(untrusted_idurl) if not recipient_id_obj: lg.warn('not found "%s" in identity cache' % untrusted_idurl) result.errback( Exception('not found "%s" in identity cache' % untrusted_idurl)) return result key_alias, creator_idurl = my_keys.split_key_id(key_id) if not key_alias or not creator_idurl: lg.warn('wrong key_id') result.errback(Exception('wrong key_id')) return result if untrusted_idurl == creator_idurl and key_alias == 'master': lg.warn('doing audit of master key (public part) of remote user') else: if not my_keys.is_key_registered(key_id): lg.warn('unknown key: "%s"' % key_id) result.errback(Exception('unknown key: "%s"' % key_id)) return result public_test_sample = key.NewSessionKey() json_payload = { 'key_id': key_id, 'audit': { 'public_sample': base64.b64encode(public_test_sample), 'private_sample': '', } } raw_payload = json.dumps(json_payload) block = encrypted.Block( BackupID=key_id, Data=raw_payload, SessionKey=key.NewSessionKey(), # encrypt data using public key of recipient EncryptKey=lambda inp: recipient_id_obj.encrypt(inp), ) encrypted_payload = block.Serialize() p2p_service.SendAuditKey( remote_idurl=recipient_id_obj.getIDURL(), encrypted_payload=encrypted_payload, packet_id=key_id, timeout=timeout, callbacks={ commands.Ack(): lambda response, info: _on_audit_public_key_response( response, info, key_id, untrusted_idurl, public_test_sample, result), commands.Fail(): lambda response, info: result.errback(Exception(response)), None: lambda pkt_out: result.errback(Exception('timeout')), # timeout }, ) return result
def _do_transfer_key(key_id, idurl, include_private=False, timeout=10, result=None): if _Debug: lg.out(_DebugLevel, 'key_ring.transfer_key %s -> %s' % (key_id, idurl)) if not result: result = Deferred() recipient_id_obj = identitycache.FromCache(idurl) if not recipient_id_obj: lg.warn('not found "%s" in identity cache' % idurl) result.errback(Exception('not found "%s" in identity cache' % idurl)) return result key_alias, creator_idurl = my_keys.split_key_id(key_id) if not key_alias or not creator_idurl: lg.warn('wrong key_id') result.errback(Exception('wrong key_id')) return result key_object = my_keys.known_keys().get(key_id) if key_object is None: lg.warn('unknown key: "%s"' % key_id) result.errback(Exception('unknown key: "%s"' % key_id)) return result try: key_json = my_keys.make_key_info(key_object, key_id=key_id, include_private=include_private) except Exception as exc: lg.exc() result.errback(exc) return result key_data = json.dumps(key_json) block = encrypted.Block( BackupID=key_id, Data=key_data, SessionKey=key.NewSessionKey(), # encrypt data using public key of recipient EncryptKey=lambda inp: recipient_id_obj.encrypt(inp), ) encrypted_key_data = block.Serialize() p2p_service.SendKey( remote_idurl=recipient_id_obj.getIDURL(), encrypted_key_data=encrypted_key_data, packet_id=key_id, callbacks={ commands.Ack(): lambda response, info: _on_transfer_key_response( response, info, key_id, result), commands.Fail(): lambda response, info: _on_transfer_key_response( response, info, key_id, result), # commands.Ack(): lambda response, info: result.callback(response), # commands.Fail(): lambda response, info: result.errback(Exception(response)), None: lambda pkt_out: _on_transfer_key_response(None, None, key_id, result), }, timeout=timeout, ) return result
def request(self, json_payload, newpacket, info): from twisted.internet import reactor # @UnresolvedImport from logs import lg from main import events from crypt import my_keys from p2p import p2p_service from contacts import contactsdb from storage import accounting from userid import global_id customer_idurl = newpacket.OwnerID customer_id = global_id.UrlToGlobalID(customer_idurl) bytes_for_customer = 0 try: bytes_for_customer = int(json_payload['needed_bytes']) except: lg.warn("wrong payload" % newpacket.Payload) return p2p_service.SendFail(newpacket, 'wrong payload') try: customer_public_key = json_payload['customer_public_key'] customer_public_key_id = customer_public_key['key_id'] except: customer_public_key = None customer_public_key_id = None data_owner_idurl = None target_customer_idurl = None family_position = json_payload.get('position') ecc_map = json_payload.get('ecc_map') family_snapshot = json_payload.get('family_snapshot') key_id = json_payload.get('key_id') target_customer_id = json_payload.get('customer_id') if key_id: # this is a request from external user to access shared data stored by one of my customers # this is "second" customer requesting data from "first" customer if not key_id or not my_keys.is_valid_key_id(key_id): lg.warn('missed or invalid key id') return p2p_service.SendFail(newpacket, 'invalid key id') target_customer_idurl = global_id.GlobalUserToIDURL( target_customer_id) if not contactsdb.is_customer(target_customer_idurl): lg.warn("target user %s is not a customer" % target_customer_id) return p2p_service.SendFail(newpacket, 'not a customer') if target_customer_idurl == customer_idurl: lg.warn('customer %s requesting shared access to own files' % customer_idurl) return p2p_service.SendFail(newpacket, 'invalid case') if not my_keys.is_key_registered(key_id): lg.warn('key not registered: %s' % key_id) p2p_service.SendFail(newpacket, 'key not registered') return False data_owner_idurl = my_keys.split_key_id(key_id)[1] if data_owner_idurl != target_customer_idurl and data_owner_idurl != customer_idurl: # pretty complex scenario: # external customer requesting access to data which belongs not to that customer # this is "third" customer accessing data belongs to "second" customer # TODO: for now just stop it lg.warn( 'under construction, key_id=%s customer_idurl=%s target_customer_idurl=%s' % ( key_id, customer_idurl, target_customer_idurl, )) p2p_service.SendFail(newpacket, 'under construction') return False # do not create connection with that customer, only accept the request lg.info( 'external customer %s requested access to shared data at %s' % ( customer_id, key_id, )) return p2p_service.SendAck(newpacket, 'accepted') # key_id is not present in the request: # this is a request to connect new customer (or reconnect existing one) to that supplier if not bytes_for_customer or bytes_for_customer < 0: lg.warn("wrong payload : %s" % newpacket.Payload) return p2p_service.SendFail(newpacket, 'wrong storage value') current_customers = contactsdb.customers() if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.request created a new space file') space_dict = accounting.read_customers_quotas() try: free_bytes = int(space_dict[b'free']) except: lg.exc() return p2p_service.SendFail(newpacket, 'broken space file') if (customer_idurl not in current_customers and customer_idurl in list(space_dict.keys())): lg.warn("broken space file") return p2p_service.SendFail(newpacket, 'broken space file') if (customer_idurl in current_customers and customer_idurl not in list(space_dict.keys())): lg.warn("broken customers file") return p2p_service.SendFail(newpacket, 'broken customers file') if customer_idurl in current_customers: free_bytes += int(space_dict.get(customer_idurl, 0)) space_dict[b'free'] = free_bytes current_customers.remove(customer_idurl) space_dict.pop(customer_idurl) new_customer = False else: new_customer = True lg.out( 8, ' new_customer=%s current_allocated_bytes=%s' % ( new_customer, space_dict.get(customer_idurl), )) from supplier import local_tester if free_bytes <= bytes_for_customer: contactsdb.update_customers(current_customers) contactsdb.remove_customer_meta_info(customer_idurl) contactsdb.save_customers() accounting.write_customers_quotas(space_dict) if customer_public_key_id: my_keys.erase_key(customer_public_key_id) reactor.callLater( 0, local_tester.TestUpdateCustomers) # @UndefinedVariable if new_customer: lg.out( 8, " NEW CUSTOMER: DENIED !!!!!!!!!!! not enough space available" ) events.send('new-customer-denied', dict(idurl=customer_idurl)) else: lg.out( 8, " OLD CUSTOMER: DENIED !!!!!!!!!!! not enough space available" ) events.send('existing-customer-denied', dict(idurl=customer_idurl)) return p2p_service.SendAck(newpacket, 'deny') space_dict[b'free'] = free_bytes - bytes_for_customer current_customers.append(customer_idurl) space_dict[customer_idurl] = bytes_for_customer contactsdb.update_customers(current_customers) contactsdb.save_customers() contactsdb.add_customer_meta_info( customer_idurl, { 'ecc_map': ecc_map, 'position': family_position, 'family_snapshot': family_snapshot, }) accounting.write_customers_quotas(space_dict) if customer_public_key_id: my_keys.erase_key(customer_public_key_id) try: if not my_keys.is_key_registered(customer_public_key_id): key_id, key_object = my_keys.read_key_info( customer_public_key) if not my_keys.register_key(key_id, key_object): lg.err('failed to register customer public key') except: lg.exc() else: lg.warn('customer public key was not provided in the request') reactor.callLater( 0, local_tester.TestUpdateCustomers) # @UndefinedVariable if new_customer: lg.out( 8, " NEW CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s" % (customer_idurl, family_position, ecc_map, bytes_for_customer)) lg.out( 8, " family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot, ) events.send( 'new-customer-accepted', dict( idurl=customer_idurl, allocated_bytes=bytes_for_customer, ecc_map=ecc_map, position=family_position, family_snapshot=family_snapshot, key_id=customer_public_key_id, )) else: lg.out( 8, " OLD CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s" % (customer_idurl, family_position, ecc_map, bytes_for_customer)) lg.out( 8, " family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot) events.send( 'existing-customer-accepted', dict( idurl=customer_idurl, allocated_bytes=bytes_for_customer, ecc_map=ecc_map, position=family_position, key_id=customer_public_key_id, family_snapshot=family_snapshot, )) return p2p_service.SendAck(newpacket, 'accepted')