def _on_request_service_received(self, newpacket, info): from twisted.internet.defer import Deferred from logs import lg from lib import serialization from services import driver from p2p import p2p_service from transport import packet_out if len(newpacket.Payload) > 1024 * 10: lg.warn('too long payload') p2p_service.SendFail(newpacket, 'too long payload') return False try: json_payload = serialization.BytesToDict(newpacket.Payload, keys_to_text=True, values_to_text=True) json_payload['name'] json_payload['payload'] except: lg.warn('json payload invalid') p2p_service.SendFail(newpacket, 'json payload invalid') return False service_name = str(json_payload['name']) lg.out( self.debug_level, "service_p2p_hookups.RequestService {%s} from %s" % ( service_name, newpacket.OwnerID, )) if not driver.is_exist(service_name): lg.warn("got wrong payload in %s" % service_name) p2p_service.SendFail(newpacket, 'service %s not exist' % service_name) return False if not driver.is_on(service_name): p2p_service.SendFail(newpacket, 'service %s is off' % service_name) return False try: result = driver.request(service_name, json_payload['payload'], newpacket, info) except: lg.exc() p2p_service.SendFail(newpacket, 'request processing failed with exception') return False if not result: lg.out( self.debug_level, "service_p2p_hookups._send_request_service SKIP request %s" % service_name) return False if isinstance(result, Deferred): lg.out( self.debug_level, "service_p2p_hookups._send_request_service fired delayed execution" ) elif isinstance(result, packet_out.PacketOut): lg.out( self.debug_level, "service_p2p_hookups._send_request_service outbox packet sent") return True
def on_incoming_message(request, info, status, error_message): """ Message came in for us """ global _IncomingMessageCallbacks if _Debug: lg.out( _DebugLevel, "message.on_incoming_message new PrivateMessage %r from %s" % ( request.PacketID, request.OwnerID, )) private_message_object = PrivateMessage.deserialize(request.Payload) if private_message_object is None: lg.err( "PrivateMessage deserialize failed, can not extract message from request payload of %d bytes" % len(request.Payload)) return False try: decrypted_message = private_message_object.decrypt() json_message = serialization.BytesToDict( decrypted_message, unpack_types=True, encoding='utf-8', ) json_message = jsn.dict_keys_to_text( jsn.dict_values_to_text(json_message)) except: lg.exc() return False if request.PacketID in received_messages_ids(): lg.warn("skip incoming message %s because found in recent history" % request.PacketID) return False received_messages_ids().append(request.PacketID) if len(received_messages_ids()) > 100: received_messages_ids(True) handled = False try: for cb in _IncomingMessageCallbacks: handled = cb(request, private_message_object, json_message) if _Debug: lg.args(_DebugLevel, cb=cb, packet_id=request.PacketID, handled=handled) if handled: break except: lg.exc() if _Debug: lg.args(_DebugLevel, msg=json_message, handled=handled) if handled: return True if config.conf().getBool( 'services/private-messages/acknowledge-unread-messages-enabled'): p2p_service.SendAckNoRequest(request.OwnerID, request.PacketID, response='unread') return True
def Unserialize(data, decrypt_key=None): """ A method to create a ``encrypted.Block`` instance from input string. """ dct = serialization.BytesToDict(data, keys_to_text=True, encoding='utf-8') if _Debug: lg.out(_DebugLevel, 'encrypted.Unserialize %s' % repr(dct)[:100]) try: newobject = Block( CreatorID=id_url.field(dct['c']), BackupID=strng.to_text(dct['b']), BlockNumber=dct['n'], LastBlock=dct['e'], EncryptedSessionKey=base64.b64decode(strng.to_bin(dct['k'])), SessionKeyType=strng.to_text(dct['t']), Length=dct['l'], EncryptedData=dct['p'], Signature=dct['s'], DecryptKey=decrypt_key, ) except: lg.exc() if _Debug: lg.out(_DebugLevel, repr(dct)) return None return newobject
def _request_data(request, mandatory_keys=[], default_value={}): """ Simplify extracting input parameters from request body. """ input_request_data = request.content.getvalue() if not input_request_data: if mandatory_keys: raise Exception('mandatory json input missed: %s' % mandatory_keys) return default_value try: data = serialization.BytesToDict(input_request_data, keys_to_text=True, values_to_text=True) except: raise Exception('invalid json input') for k in mandatory_keys: if isinstance(k, tuple): found = False for f in k: if f in data: found = True if not found: raise Exception('one of mandatory parameters missed: %s' % k) else: if k not in data: raise Exception('one of mandatory parameters missed: %s' % mandatory_keys) return data
def default_nodes(): """ List of DHT nodes currently maintained : (host, UDP port number) """ from system import bpio from system import local_fs from lib import serialization from main import settings from logs import lg networks_json = serialization.BytesToDict( local_fs.ReadBinaryFile( os.path.join(bpio.getExecutableDir(), 'networks.json'))) my_network = local_fs.ReadTextFile(settings.NetworkFileName()).strip() if not my_network: my_network = 'main' if my_network not in networks_json: my_network = 'main' network_info = networks_json[my_network] dht_seeds = [] for dht_seed in network_info['dht-seeds']: dht_seeds.append(( dht_seed['host'], dht_seed['udp_port'], )) lg.info('Active network is [%s] dht_seeds=%s' % ( my_network, dht_seeds, )) return dht_seeds
def default_nodes(): """ A set of identity servers currently maintained, see file networks.json in the root folder. """ from system import bpio from system import local_fs from lib import serialization from lib import strng from main import settings # from logs import lg networks_json = serialization.BytesToDict( local_fs.ReadBinaryFile( os.path.join(bpio.getExecutableDir(), 'networks.json')), keys_to_text=True, values_to_text=True, ) my_network = local_fs.ReadTextFile(settings.NetworkFileName()).strip() if not my_network: my_network = 'main' if my_network not in networks_json: my_network = 'main' network_info = networks_json[my_network] identity_servers = {} for identity_server in network_info['identity-servers']: identity_servers[strng.to_bin(identity_server['host'])] = ( identity_server['http_port'], identity_server['tcp_port'], ) # lg.info('Active network is [%s] identity_servers=%s' % (my_network, identity_servers, )) return identity_servers
def Unserialize(data): """ We expect here a string containing a whole packet object in text form. Will return a real object in the memory from given string. All class fields are loaded, signature can be verified to be sure - it was truly original string. """ if data is None: return None dct = serialization.BytesToDict(data, keys_to_text=True) if _Debug: lg.out(_DebugLevel, 'signed.Unserialize %r' % dct) try: newobject = Packet( Command=strng.to_text(dct['m']), OwnerID=dct['o'], CreatorID=dct['c'], PacketID=strng.to_text(dct['i']), Date=strng.to_text(dct['d']), Payload=dct['p'], RemoteID=dct['r'], KeyID=strng.to_text(dct['k']), Signature=dct['s'], ) except: lg.exc() newobject = None if newobject is None: lg.warn("result is None") return None return newobject
def _load_routes(self): src = config.conf().getData('services/proxy-server/current-routes') if src is None: lg.warn('setting [services/proxy-server/current-routes] not exist') return try: dct = serialization.BytesToDict(strng.to_bin(src), keys_to_text=True, values_to_text=True) except: dct = {} for k, v in dct.items(): self.routes[strng.to_bin(k)] = v ident = identity.identity(xmlsrc=v['identity']) if not self._is_my_contacts_present_in_identity(ident): if _Debug: lg.out(_DebugLevel, ' DO OVERRIDE identity for %s' % k) identitycache.OverrideIdentity(k, v['identity']) else: if _Debug: lg.out(_DebugLevel, ' skip overriding %s' % k) if _Debug: lg.out( _DebugLevel, 'proxy_router._load_routes %d routes total' % len(self.routes))
def _on_extract_done(self, retcode, backupID, source_filename, output_location, backup_index): tmpfile.throw_out(source_filename, 'file extracted') for snapshot_filename in os.listdir(output_location): snapshot_path = os.path.join(output_location, snapshot_filename) snapshot_data = serialization.BytesToDict( local_fs.ReadBinaryFile(snapshot_path), values_to_text=True) for archive_message in snapshot_data.get('items', []): if self.start_sequence_id is not None: if self.start_sequence_id > archive_message['sequence_id']: continue if self.end_sequence_id is not None: if self.end_sequence_id < archive_message['sequence_id']: continue self.extracted_messages.append(archive_message) if _Debug: lg.dbg( _DebugLevel, 'archive snapshot %r extracted successfully to %r, extracted %d archive messages so far' % ( source_filename, output_location, len(self.extracted_messages), )) self._do_restore_next_backup(backup_index + 1) return retcode
def decrypt_json(encrypted_data, secret_bytes_key, cipher_type='AES'): dct = serialization.BytesToDict( encrypted_data, encoding='utf-8', keys_to_text=True, values_to_text=True, ) if cipher_type == 'AES': cipher = AES.new( key=secret_bytes_key, mode=AES.MODE_CBC, iv=base64.b64decode(dct['iv'].encode('utf-8')), ) elif cipher_type == 'DES3': cipher = DES3.new( key=secret_bytes_key, mode=DES3.MODE_CBC, iv=base64.b64decode(dct['iv'].encode('utf-8')), ) else: raise Exception('unsupported cipher type') padded_data = cipher.decrypt(base64.b64decode(dct['ct'].encode('utf-8'))) if cipher_type == 'AES': raw_data = Padding.unpad( padded_data=padded_data, block_size=AES.block_size, ) elif cipher_type == 'DES3': raw_data = Padding.unpad( padded_data=padded_data, block_size=DES3.block_size, ) # TODO: remove salt from raw_data return raw_data
def Contacts(request, info): """ """ if _Debug: lg.out(_DebugLevel, 'p2p_service.Contacts %d bytes in [%s] : %r' % ( len(request.Payload), request.PacketID, serialization.BytesToDict(request.Payload))) lg.out(_DebugLevel, ' from remoteID=%s ownerID=%s creatorID=%s' % ( request.RemoteID, request.OwnerID, request.CreatorID))
def read_network_config_file(): networks_json_path = find_network_config_file() network_info = serialization.BytesToDict( local_fs.ReadBinaryFile(networks_json_path), keys_to_text=True, values_to_text=True, ) return network_info
def Coin(request, info): if _Debug: try: input_coins = serialization.BytesToDict(request.Payload) except: lg.exc() input_coins = [] lg.out(_DebugLevel, "p2p_service.Coin from %s with %d coins" % ( nameurl.GetName(info.sender_idurl), len(input_coins), ))
def dataReceived(self, data): try: json_data = serialization.BytesToDict(data, keys_to_text=True, values_to_text=True) except: lg.exc() return if _Debug: lg.dbg(_DebugLevel, 'received %d bytes from web socket: %r' % (len(data), json_data)) if not do_process_incoming_message(json_data): lg.warn('failed processing incoming message from web socket: %r' % json_data)
def test_serialization(self): data1 = os.urandom(1024) dct1 = { 'd': { 'data': data1, }, } raw = serialization.DictToBytes(dct1, encoding='latin1') dct2 = serialization.BytesToDict(raw, encoding='latin1') data2 = dct2['d']['data'] self.assertEqual(data1, data2)
def CancelService(request, info): try: service_info = serialization.BytesToDict(request.Payload) except: lg.exc() service_info = {} if _Debug: lg.out(_DebugLevel, 'p2p_service.CancelService %s with "%s" in %d bytes' % ( request.PacketID, service_info.get('name', 'unknown service name'), len(request.Payload), )) lg.out(_DebugLevel, ' from remoteID=%s ownerID=%s creatorID=%s' % ( request.RemoteID, request.OwnerID, request.CreatorID))
def test_serialization(self): from lib import serialization data1 = os.urandom(1024) dct1 = { 'd': { 'data': data1, }, } raw = serialization.DictToBytes(dct1) dct2 = serialization.BytesToDict(raw) data2 = dct2['d']['data'] self.assertEqual(data1, data2)
def decrypt_json(encrypted_data, secret_16bytes_key): dct = serialization.BytesToDict(encrypted_data) iv = base64.b64decode(dct['iv']) ct = base64.b64decode(dct['ct']) cipher = AES.new( key=secret_16bytes_key, mode=AES.MODE_CBC, iv=iv, ) result = Padding.unpad(cipher.decrypt(ct), AES.block_size) # TODO: remove salt from raw_data return result
def on_audit_key_received(newpacket, info, status, error_message): """ Callback will be executed when remote user would like to check if I poses given key locally. """ block = encrypted.Unserialize(newpacket.Payload) if block is None: lg.out( 2, 'key_ring.on_audit_key_received ERROR reading data from %s' % newpacket.RemoteID) return False try: raw_payload = block.Data() json_payload = serialization.BytesToDict(raw_payload, keys_to_text=True, values_to_text=True) key_id = json_payload['key_id'] json_payload['audit'] public_sample = base64.b64decode( json_payload['audit']['public_sample']) private_sample = base64.b64decode( json_payload['audit']['private_sample']) except Exception as exc: lg.exc() p2p_service.SendFail(newpacket, str(exc)) return False if not my_keys.is_valid_key_id(key_id): p2p_service.SendFail(newpacket, 'invalid key id') return False if not my_keys.is_key_registered(key_id, include_master=True): p2p_service.SendFail(newpacket, 'key not registered') return False if public_sample: response_payload = base64.b64encode( my_keys.encrypt(key_id, public_sample)) p2p_service.SendAck(newpacket, response_payload) if _Debug: lg.info('remote user %s requested audit of public key %s' % (newpacket.OwnerID, key_id)) return True if private_sample: if not my_keys.is_key_private(key_id): p2p_service.SendFail(newpacket, 'private key not registered') return False response_payload = base64.b64encode( my_keys.decrypt(key_id, private_sample)) p2p_service.SendAck(newpacket, response_payload) if _Debug: lg.info('remote user %s requested audit of private key %s' % (newpacket.OwnerID, key_id)) return True p2p_service.SendFail(newpacket, 'wrong audit request') return False
def Event(request, info): """ """ if _Debug: try: e_json = serialization.BytesToDict(request.Payload) e_json['event_id'] e_json['payload'] except: lg.exc() return lg.out(_DebugLevel, "p2p_service.Event %s from %s with %d bytes in json" % ( e_json['event_id'], info.sender_idurl, len(request.Payload), ))
def deserialize(input_string): try: dct = serialization.BytesToDict(input_string) message_obj = PrivateMessage( recipient_global_id=dct['r'], sender=dct['s'], encrypted_session=dct['k'], encrypted_body=dct['b'], ) except: lg.exc() return None return message_obj
def deserialize(cls, input_string): try: dct = serialization.BytesToDict(input_string, keys_to_text=True, encoding='utf-8') message_obj = cls( recipient=strng.to_text(dct['r']), sender=strng.to_text(dct['s']), encrypted_session=base64.b64decode(strng.to_bin(dct['k'])), encrypted_body=dct['p'], ) except: lg.exc() return None return message_obj
def Unserialize(data, decrypt_key=None): """ A method to create a ``encrypted.Block`` instance from input string. """ dct = serialization.BytesToDict(data) newobject = Block( CreatorID=dct['c'], BackupID=dct['b'], BlockNumber=dct['n'], EncryptedSessionKey=dct['k'], SessionKeyType=dct['t'], Length=dct['l'], EncryptedData=dct['p'], Signature=dct['s'], ) return newobject
def on_event_packet_received(newpacket, info, status, error_message): global _EventPacketReceivedCallbacks try: e_json = serialization.BytesToDict(newpacket.Payload, keys_to_text=True) strng.to_text(e_json['event_id']) e_json['payload'] except: lg.warn("invalid json payload") return False handled = False for cb in _EventPacketReceivedCallbacks: handled = cb(newpacket, e_json) if handled: break return handled
def _write_route(self, user_id): src = config.conf().getData('services/proxy-server/current-routes') try: dct = serialization.BytesToDict(strng.to_bin(src), keys_to_text=True, values_to_text=True) except: dct = {} dct[user_id] = self.routes[user_id] newsrc = strng.to_text( serialization.DictToBytes(dct, keys_to_text=True, values_to_text=True)) config.conf().setData('services/proxy-server/current-routes', newsrc) if _Debug: lg.out(_DebugLevel, 'proxy_router._write_route %d bytes wrote' % len(newsrc))
def pop_signed_message(queue_id, message_id): existing_message = pop_message(queue_id, message_id) if not existing_message: return existing_message try: signed_data = signed.Unserialize(existing_message.payload) except: raise Exception('unserialize message fails') if not signed_data: raise Exception('unserialized message is empty') if not signed_data.Valid(): raise Exception('unserialized message is not valid') try: existing_message.payload = serialization.BytesToDict( signed_data.Payload, keys_to_text=True) except: raise Exception('failed reading message json data') return existing_message
def deserialize(input_string): try: dct = serialization.BytesToDict(input_string, keys_to_text=True, encoding='utf-8') _recipient = dct['r'] _sender = dct['s'] _encrypted_session_key = dct['k'] _encrypted_body = dct['p'] message_obj = PrivateMessage( recipient_global_id=_recipient, sender=_sender, encrypted_session=_encrypted_session_key, encrypted_body=_encrypted_body, ) except: lg.exc() return None return message_obj
def decrypt_json(encrypted_data, secret_16bytes_key): dct = serialization.BytesToDict( encrypted_data, encoding='utf-8', keys_to_text=True, values_to_text=True, ) cipher = AES.new( key=secret_16bytes_key, mode=AES.MODE_CBC, iv=base64.b64decode(dct['iv'].encode('utf-8')), ) padded_data = cipher.decrypt(base64.b64decode(dct['ct'].encode('utf-8'))) raw_data = Padding.unpad( padded_data=padded_data, block_size=AES.block_size, ) # TODO: remove salt from raw_data return raw_data
def _on_supplier_fail(self, response, info): if _Debug: lg.out(_DebugLevel, 'family_member._on_supplier_fail with %r' % response) if response.PacketID in self.suppliers_requests: self.suppliers_requests.remove(response.PacketID) try: json_payload = serialization.BytesToDict(response.Payload) ecc_map = strng.to_text(json_payload['ecc_map']) suppliers_list = list(map(strng.to_bin, json_payload['suppliers'])) except: lg.exc() if not self.suppliers_requests: self.automat('all-suppliers-agree') return None self.automat('one-supplier-not-agree', ecc_map=ecc_map, suppliers_list=suppliers_list, supplier_idurl=response.OwnerID)
def on_list_files(newpacket): json_query = {} try: j = serialization.BytesToDict(newpacket.Payload, keys_to_text=True, values_to_text=True) j['items'][0] json_query = j except: if strng.to_text(newpacket.Payload) == settings.ListFilesFormat(): json_query = { 'items': [ '*', ], } if json_query is None: lg.exc('unrecognized ListFiles() query received') return False # TODO: perform validations before sending back list of files list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID) if list_files_global_id['key_id']: # customer id and data id can be recognized from packet id # return back list of files according to the request customer_idurl = list_files_global_id['idurl'] key_id = list_files_global_id['key_id'] else: # packet id format is unknown # by default returning back all files from that recipient if he is a customer customer_idurl = newpacket.OwnerID key_id = my_keys.make_key_id(alias='customer', creator_idurl=customer_idurl) key_id = my_keys.latest_key_id(key_id) list_files.send( customer_idurl=customer_idurl, packet_id=newpacket.PacketID, format_type=settings.ListFilesFormat(), key_id=key_id, remote_idurl=newpacket.OwnerID, # send back to the requesting node query_items=json_query['items'], ) return True