def _on_request_service_received(self, newpacket, info):
from twisted.internet.defer import Deferred
from logs import lg
from lib import serialization
from services import driver
from p2p import p2p_service
from transport import packet_out
if len(newpacket.Payload) > 1024 * 10:
lg.warn('too long payload')
p2p_service.SendFail(newpacket, 'too long payload')
return False
try:
json_payload = serialization.BytesToDict(newpacket.Payload,
keys_to_text=True,
values_to_text=True)
json_payload['name']
json_payload['payload']
except:
lg.warn('json payload invalid')
p2p_service.SendFail(newpacket, 'json payload invalid')
return False
service_name = str(json_payload['name'])
lg.out(
self.debug_level,
"service_p2p_hookups.RequestService {%s} from %s" % (
service_name,
newpacket.OwnerID,
))
if not driver.is_exist(service_name):
lg.warn("got wrong payload in %s" % service_name)
p2p_service.SendFail(newpacket,
'service %s not exist' % service_name)
return False
if not driver.is_on(service_name):
p2p_service.SendFail(newpacket, 'service %s is off' % service_name)
return False
try:
result = driver.request(service_name, json_payload['payload'],
newpacket, info)
except:
lg.exc()
p2p_service.SendFail(newpacket,
'request processing failed with exception')
return False
if not result:
lg.out(
self.debug_level,
"service_p2p_hookups._send_request_service SKIP request %s" %
service_name)
return False
if isinstance(result, Deferred):
lg.out(
self.debug_level,
"service_p2p_hookups._send_request_service fired delayed execution"
)
elif isinstance(result, packet_out.PacketOut):
lg.out(
self.debug_level,
"service_p2p_hookups._send_request_service outbox packet sent")
return True
def on_incoming_message(request, info, status, error_message):
"""
Message came in for us
"""
global _IncomingMessageCallbacks
if _Debug:
lg.out(
_DebugLevel,
"message.on_incoming_message new PrivateMessage %r from %s" % (
request.PacketID,
request.OwnerID,
))
private_message_object = PrivateMessage.deserialize(request.Payload)
if private_message_object is None:
lg.err(
"PrivateMessage deserialize failed, can not extract message from request payload of %d bytes"
% len(request.Payload))
return False
try:
decrypted_message = private_message_object.decrypt()
json_message = serialization.BytesToDict(
decrypted_message,
unpack_types=True,
encoding='utf-8',
)
json_message = jsn.dict_keys_to_text(
jsn.dict_values_to_text(json_message))
except:
lg.exc()
return False
if request.PacketID in received_messages_ids():
lg.warn("skip incoming message %s because found in recent history" %
request.PacketID)
return False
received_messages_ids().append(request.PacketID)
if len(received_messages_ids()) > 100:
received_messages_ids(True)
handled = False
try:
for cb in _IncomingMessageCallbacks:
handled = cb(request, private_message_object, json_message)
if _Debug:
lg.args(_DebugLevel,
cb=cb,
packet_id=request.PacketID,
handled=handled)
if handled:
break
except:
lg.exc()
if _Debug:
lg.args(_DebugLevel, msg=json_message, handled=handled)
if handled:
return True
if config.conf().getBool(
'services/private-messages/acknowledge-unread-messages-enabled'):
p2p_service.SendAckNoRequest(request.OwnerID,
request.PacketID,
response='unread')
return True
def Unserialize(data, decrypt_key=None):
"""
A method to create a ``encrypted.Block`` instance from input string.
"""
dct = serialization.BytesToDict(data, keys_to_text=True, encoding='utf-8')
if _Debug:
lg.out(_DebugLevel, 'encrypted.Unserialize %s' % repr(dct)[:100])
try:
newobject = Block(
CreatorID=id_url.field(dct['c']),
BackupID=strng.to_text(dct['b']),
BlockNumber=dct['n'],
LastBlock=dct['e'],
EncryptedSessionKey=base64.b64decode(strng.to_bin(dct['k'])),
SessionKeyType=strng.to_text(dct['t']),
Length=dct['l'],
EncryptedData=dct['p'],
Signature=dct['s'],
DecryptKey=decrypt_key,
)
except:
lg.exc()
if _Debug:
lg.out(_DebugLevel, repr(dct))
return None
return newobject
def _request_data(request, mandatory_keys=[], default_value={}):
"""
Simplify extracting input parameters from request body.
"""
input_request_data = request.content.getvalue()
if not input_request_data:
if mandatory_keys:
raise Exception('mandatory json input missed: %s' % mandatory_keys)
return default_value
try:
data = serialization.BytesToDict(input_request_data,
keys_to_text=True,
values_to_text=True)
except:
raise Exception('invalid json input')
for k in mandatory_keys:
if isinstance(k, tuple):
found = False
for f in k:
if f in data:
found = True
if not found:
raise Exception('one of mandatory parameters missed: %s' % k)
else:
if k not in data:
raise Exception('one of mandatory parameters missed: %s' %
mandatory_keys)
return data
def default_nodes():
"""
List of DHT nodes currently maintained : (host, UDP port number)
"""
from system import bpio
from system import local_fs
from lib import serialization
from main import settings
from logs import lg
networks_json = serialization.BytesToDict(
local_fs.ReadBinaryFile(
os.path.join(bpio.getExecutableDir(), 'networks.json')))
my_network = local_fs.ReadTextFile(settings.NetworkFileName()).strip()
if not my_network:
my_network = 'main'
if my_network not in networks_json:
my_network = 'main'
network_info = networks_json[my_network]
dht_seeds = []
for dht_seed in network_info['dht-seeds']:
dht_seeds.append((
dht_seed['host'],
dht_seed['udp_port'],
))
lg.info('Active network is [%s] dht_seeds=%s' % (
my_network,
dht_seeds,
))
return dht_seeds
def default_nodes():
"""
A set of identity servers currently maintained, see file networks.json in the root folder.
"""
from system import bpio
from system import local_fs
from lib import serialization
from lib import strng
from main import settings
# from logs import lg
networks_json = serialization.BytesToDict(
local_fs.ReadBinaryFile(
os.path.join(bpio.getExecutableDir(), 'networks.json')),
keys_to_text=True,
values_to_text=True,
)
my_network = local_fs.ReadTextFile(settings.NetworkFileName()).strip()
if not my_network:
my_network = 'main'
if my_network not in networks_json:
my_network = 'main'
network_info = networks_json[my_network]
identity_servers = {}
for identity_server in network_info['identity-servers']:
identity_servers[strng.to_bin(identity_server['host'])] = (
identity_server['http_port'],
identity_server['tcp_port'],
)
# lg.info('Active network is [%s] identity_servers=%s' % (my_network, identity_servers, ))
return identity_servers
def Unserialize(data):
"""
We expect here a string containing a whole packet object in text form.
Will return a real object in the memory from given string.
All class fields are loaded, signature can be verified to be sure - it was truly original string.
"""
if data is None:
return None
dct = serialization.BytesToDict(data, keys_to_text=True)
if _Debug:
lg.out(_DebugLevel, 'signed.Unserialize %r' % dct)
try:
newobject = Packet(
Command=strng.to_text(dct['m']),
OwnerID=dct['o'],
CreatorID=dct['c'],
PacketID=strng.to_text(dct['i']),
Date=strng.to_text(dct['d']),
Payload=dct['p'],
RemoteID=dct['r'],
KeyID=strng.to_text(dct['k']),
Signature=dct['s'],
)
except:
lg.exc()
newobject = None
if newobject is None:
lg.warn("result is None")
return None
return newobject
def _load_routes(self):
src = config.conf().getData('services/proxy-server/current-routes')
if src is None:
lg.warn('setting [services/proxy-server/current-routes] not exist')
return
try:
dct = serialization.BytesToDict(strng.to_bin(src),
keys_to_text=True,
values_to_text=True)
except:
dct = {}
for k, v in dct.items():
self.routes[strng.to_bin(k)] = v
ident = identity.identity(xmlsrc=v['identity'])
if not self._is_my_contacts_present_in_identity(ident):
if _Debug:
lg.out(_DebugLevel, ' DO OVERRIDE identity for %s' % k)
identitycache.OverrideIdentity(k, v['identity'])
else:
if _Debug:
lg.out(_DebugLevel, ' skip overriding %s' % k)
if _Debug:
lg.out(
_DebugLevel,
'proxy_router._load_routes %d routes total' % len(self.routes))
def _on_extract_done(self, retcode, backupID, source_filename,
output_location, backup_index):
tmpfile.throw_out(source_filename, 'file extracted')
for snapshot_filename in os.listdir(output_location):
snapshot_path = os.path.join(output_location, snapshot_filename)
snapshot_data = serialization.BytesToDict(
local_fs.ReadBinaryFile(snapshot_path), values_to_text=True)
for archive_message in snapshot_data.get('items', []):
if self.start_sequence_id is not None:
if self.start_sequence_id > archive_message['sequence_id']:
continue
if self.end_sequence_id is not None:
if self.end_sequence_id < archive_message['sequence_id']:
continue
self.extracted_messages.append(archive_message)
if _Debug:
lg.dbg(
_DebugLevel,
'archive snapshot %r extracted successfully to %r, extracted %d archive messages so far'
% (
source_filename,
output_location,
len(self.extracted_messages),
))
self._do_restore_next_backup(backup_index + 1)
return retcode
def decrypt_json(encrypted_data, secret_bytes_key, cipher_type='AES'):
dct = serialization.BytesToDict(
encrypted_data,
encoding='utf-8',
keys_to_text=True,
values_to_text=True,
)
if cipher_type == 'AES':
cipher = AES.new(
key=secret_bytes_key,
mode=AES.MODE_CBC,
iv=base64.b64decode(dct['iv'].encode('utf-8')),
)
elif cipher_type == 'DES3':
cipher = DES3.new(
key=secret_bytes_key,
mode=DES3.MODE_CBC,
iv=base64.b64decode(dct['iv'].encode('utf-8')),
)
else:
raise Exception('unsupported cipher type')
padded_data = cipher.decrypt(base64.b64decode(dct['ct'].encode('utf-8')))
if cipher_type == 'AES':
raw_data = Padding.unpad(
padded_data=padded_data,
block_size=AES.block_size,
)
elif cipher_type == 'DES3':
raw_data = Padding.unpad(
padded_data=padded_data,
block_size=DES3.block_size,
)
# TODO: remove salt from raw_data
return raw_data
def Contacts(request, info):
"""
"""
if _Debug:
lg.out(_DebugLevel, 'p2p_service.Contacts %d bytes in [%s] : %r' % (
len(request.Payload), request.PacketID, serialization.BytesToDict(request.Payload)))
lg.out(_DebugLevel, ' from remoteID=%s ownerID=%s creatorID=%s' % (
request.RemoteID, request.OwnerID, request.CreatorID))
def read_network_config_file():
networks_json_path = find_network_config_file()
network_info = serialization.BytesToDict(
local_fs.ReadBinaryFile(networks_json_path),
keys_to_text=True,
values_to_text=True,
)
return network_info
def Coin(request, info):
if _Debug:
try:
input_coins = serialization.BytesToDict(request.Payload)
except:
lg.exc()
input_coins = []
lg.out(_DebugLevel, "p2p_service.Coin from %s with %d coins" % (
nameurl.GetName(info.sender_idurl), len(input_coins), ))
def dataReceived(self, data):
try:
json_data = serialization.BytesToDict(data, keys_to_text=True, values_to_text=True)
except:
lg.exc()
return
if _Debug:
lg.dbg(_DebugLevel, 'received %d bytes from web socket: %r' % (len(data), json_data))
if not do_process_incoming_message(json_data):
lg.warn('failed processing incoming message from web socket: %r' % json_data)
def test_serialization(self):
data1 = os.urandom(1024)
dct1 = {
'd': {
'data': data1,
},
}
raw = serialization.DictToBytes(dct1, encoding='latin1')
dct2 = serialization.BytesToDict(raw, encoding='latin1')
data2 = dct2['d']['data']
self.assertEqual(data1, data2)
def CancelService(request, info):
try:
service_info = serialization.BytesToDict(request.Payload)
except:
lg.exc()
service_info = {}
if _Debug:
lg.out(_DebugLevel, 'p2p_service.CancelService %s with "%s" in %d bytes' % (
request.PacketID, service_info.get('name', 'unknown service name'), len(request.Payload), ))
lg.out(_DebugLevel, ' from remoteID=%s ownerID=%s creatorID=%s' % (
request.RemoteID, request.OwnerID, request.CreatorID))
def test_serialization(self):
from lib import serialization
data1 = os.urandom(1024)
dct1 = {
'd': {
'data': data1,
},
}
raw = serialization.DictToBytes(dct1)
dct2 = serialization.BytesToDict(raw)
data2 = dct2['d']['data']
self.assertEqual(data1, data2)
def decrypt_json(encrypted_data, secret_16bytes_key):
dct = serialization.BytesToDict(encrypted_data)
iv = base64.b64decode(dct['iv'])
ct = base64.b64decode(dct['ct'])
cipher = AES.new(
key=secret_16bytes_key,
mode=AES.MODE_CBC,
iv=iv,
)
result = Padding.unpad(cipher.decrypt(ct), AES.block_size)
# TODO: remove salt from raw_data
return result
def on_audit_key_received(newpacket, info, status, error_message):
"""
Callback will be executed when remote user would like to check if I poses given key locally.
"""
block = encrypted.Unserialize(newpacket.Payload)
if block is None:
lg.out(
2, 'key_ring.on_audit_key_received ERROR reading data from %s' %
newpacket.RemoteID)
return False
try:
raw_payload = block.Data()
json_payload = serialization.BytesToDict(raw_payload,
keys_to_text=True,
values_to_text=True)
key_id = json_payload['key_id']
json_payload['audit']
public_sample = base64.b64decode(
json_payload['audit']['public_sample'])
private_sample = base64.b64decode(
json_payload['audit']['private_sample'])
except Exception as exc:
lg.exc()
p2p_service.SendFail(newpacket, str(exc))
return False
if not my_keys.is_valid_key_id(key_id):
p2p_service.SendFail(newpacket, 'invalid key id')
return False
if not my_keys.is_key_registered(key_id, include_master=True):
p2p_service.SendFail(newpacket, 'key not registered')
return False
if public_sample:
response_payload = base64.b64encode(
my_keys.encrypt(key_id, public_sample))
p2p_service.SendAck(newpacket, response_payload)
if _Debug:
lg.info('remote user %s requested audit of public key %s' %
(newpacket.OwnerID, key_id))
return True
if private_sample:
if not my_keys.is_key_private(key_id):
p2p_service.SendFail(newpacket, 'private key not registered')
return False
response_payload = base64.b64encode(
my_keys.decrypt(key_id, private_sample))
p2p_service.SendAck(newpacket, response_payload)
if _Debug:
lg.info('remote user %s requested audit of private key %s' %
(newpacket.OwnerID, key_id))
return True
p2p_service.SendFail(newpacket, 'wrong audit request')
return False
def Event(request, info):
"""
"""
if _Debug:
try:
e_json = serialization.BytesToDict(request.Payload)
e_json['event_id']
e_json['payload']
except:
lg.exc()
return
lg.out(_DebugLevel, "p2p_service.Event %s from %s with %d bytes in json" % (
e_json['event_id'], info.sender_idurl, len(request.Payload), ))
def deserialize(input_string):
try:
dct = serialization.BytesToDict(input_string)
message_obj = PrivateMessage(
recipient_global_id=dct['r'],
sender=dct['s'],
encrypted_session=dct['k'],
encrypted_body=dct['b'],
)
except:
lg.exc()
return None
return message_obj
def deserialize(cls, input_string):
try:
dct = serialization.BytesToDict(input_string, keys_to_text=True, encoding='utf-8')
message_obj = cls(
recipient=strng.to_text(dct['r']),
sender=strng.to_text(dct['s']),
encrypted_session=base64.b64decode(strng.to_bin(dct['k'])),
encrypted_body=dct['p'],
)
except:
lg.exc()
return None
return message_obj
def Unserialize(data, decrypt_key=None):
"""
A method to create a ``encrypted.Block`` instance from input string.
"""
dct = serialization.BytesToDict(data)
newobject = Block(
CreatorID=dct['c'],
BackupID=dct['b'],
BlockNumber=dct['n'],
EncryptedSessionKey=dct['k'],
SessionKeyType=dct['t'],
Length=dct['l'],
EncryptedData=dct['p'],
Signature=dct['s'],
)
return newobject
def on_event_packet_received(newpacket, info, status, error_message):
global _EventPacketReceivedCallbacks
try:
e_json = serialization.BytesToDict(newpacket.Payload,
keys_to_text=True)
strng.to_text(e_json['event_id'])
e_json['payload']
except:
lg.warn("invalid json payload")
return False
handled = False
for cb in _EventPacketReceivedCallbacks:
handled = cb(newpacket, e_json)
if handled:
break
return handled
def _write_route(self, user_id):
src = config.conf().getData('services/proxy-server/current-routes')
try:
dct = serialization.BytesToDict(strng.to_bin(src),
keys_to_text=True,
values_to_text=True)
except:
dct = {}
dct[user_id] = self.routes[user_id]
newsrc = strng.to_text(
serialization.DictToBytes(dct,
keys_to_text=True,
values_to_text=True))
config.conf().setData('services/proxy-server/current-routes', newsrc)
if _Debug:
lg.out(_DebugLevel,
'proxy_router._write_route %d bytes wrote' % len(newsrc))
def pop_signed_message(queue_id, message_id):
existing_message = pop_message(queue_id, message_id)
if not existing_message:
return existing_message
try:
signed_data = signed.Unserialize(existing_message.payload)
except:
raise Exception('unserialize message fails')
if not signed_data:
raise Exception('unserialized message is empty')
if not signed_data.Valid():
raise Exception('unserialized message is not valid')
try:
existing_message.payload = serialization.BytesToDict(
signed_data.Payload, keys_to_text=True)
except:
raise Exception('failed reading message json data')
return existing_message
def deserialize(input_string):
try:
dct = serialization.BytesToDict(input_string,
keys_to_text=True,
encoding='utf-8')
_recipient = dct['r']
_sender = dct['s']
_encrypted_session_key = dct['k']
_encrypted_body = dct['p']
message_obj = PrivateMessage(
recipient_global_id=_recipient,
sender=_sender,
encrypted_session=_encrypted_session_key,
encrypted_body=_encrypted_body,
)
except:
lg.exc()
return None
return message_obj
def decrypt_json(encrypted_data, secret_16bytes_key):
dct = serialization.BytesToDict(
encrypted_data,
encoding='utf-8',
keys_to_text=True,
values_to_text=True,
)
cipher = AES.new(
key=secret_16bytes_key,
mode=AES.MODE_CBC,
iv=base64.b64decode(dct['iv'].encode('utf-8')),
)
padded_data = cipher.decrypt(base64.b64decode(dct['ct'].encode('utf-8')))
raw_data = Padding.unpad(
padded_data=padded_data,
block_size=AES.block_size,
)
# TODO: remove salt from raw_data
return raw_data
def _on_supplier_fail(self, response, info):
if _Debug:
lg.out(_DebugLevel,
'family_member._on_supplier_fail with %r' % response)
if response.PacketID in self.suppliers_requests:
self.suppliers_requests.remove(response.PacketID)
try:
json_payload = serialization.BytesToDict(response.Payload)
ecc_map = strng.to_text(json_payload['ecc_map'])
suppliers_list = list(map(strng.to_bin, json_payload['suppliers']))
except:
lg.exc()
if not self.suppliers_requests:
self.automat('all-suppliers-agree')
return None
self.automat('one-supplier-not-agree',
ecc_map=ecc_map,
suppliers_list=suppliers_list,
supplier_idurl=response.OwnerID)
def on_list_files(newpacket):
json_query = {}
try:
j = serialization.BytesToDict(newpacket.Payload,
keys_to_text=True,
values_to_text=True)
j['items'][0]
json_query = j
except:
if strng.to_text(newpacket.Payload) == settings.ListFilesFormat():
json_query = {
'items': [
'*',
],
}
if json_query is None:
lg.exc('unrecognized ListFiles() query received')
return False
# TODO: perform validations before sending back list of files
list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID)
if list_files_global_id['key_id']:
# customer id and data id can be recognized from packet id
# return back list of files according to the request
customer_idurl = list_files_global_id['idurl']
key_id = list_files_global_id['key_id']
else:
# packet id format is unknown
# by default returning back all files from that recipient if he is a customer
customer_idurl = newpacket.OwnerID
key_id = my_keys.make_key_id(alias='customer',
creator_idurl=customer_idurl)
key_id = my_keys.latest_key_id(key_id)
list_files.send(
customer_idurl=customer_idurl,
packet_id=newpacket.PacketID,
format_type=settings.ListFilesFormat(),
key_id=key_id,
remote_idurl=newpacket.OwnerID, # send back to the requesting node
query_items=json_query['items'],
)
return True