def populate_messages(recipient_id=None,
sender_id=None,
message_types=[],
offset=0,
limit=100):
if recipient_id:
if not recipient_id.count('@'):
from contacts import contactsdb
recipient_idurl = contactsdb.find_correspondent_by_nickname(
recipient_id)
if not recipient_idurl:
lg.err('recipient %r was not found' % recipient_id)
return
recipient_id = global_id.UrlToGlobalID(recipient_idurl)
recipient_glob_id = global_id.ParseGlobalID(recipient_id)
if not recipient_glob_id['idurl']:
lg.err('wrong recipient_id')
return
recipient_id = global_id.MakeGlobalID(**recipient_glob_id)
if not my_keys.is_valid_key_id(recipient_id):
lg.err('invalid recipient_id: %s' % recipient_id)
return
if sender_id:
sender_local_key_id = my_keys.get_local_key_id(sender_id)
if sender_local_key_id is None:
return
if recipient_id:
recipient_local_key_id = my_keys.get_local_key_id(recipient_id)
if recipient_local_key_id is None:
lg.warn('recipient %r local key id was not registered' %
recipient_id)
return
for row in query_messages(
sender_id=sender_id,
recipient_id=recipient_id,
bidirectional=False,
message_types=message_types,
offset=offset,
limit=limit,
raw_results=True,
):
conversation_id = get_conversation_id(row[0], row[2], int(row[5]))
if conversation_id is None:
continue
snap_id = '{}/{}'.format(conversation_id, row[7])
listeners.push_snapshot('message',
snap_id=snap_id,
created=row[6],
data=build_json_message(
sender=row[1],
recipient=row[3],
direction='in' if row[4] == 0 else 'out',
conversation_id=conversation_id,
message_type=MESSAGE_TYPE_CODES.get(
int(row[5]), 'private_message'),
message_time=row[6],
message_id=row[7],
data=json.loads(row[8]),
))
def on_audit_key_received(newpacket, info, status, error_message):
"""
Callback will be executed when remote user would like to check if I poses given key locally.
"""
block = encrypted.Unserialize(newpacket.Payload)
if block is None:
lg.out(
2, 'key_ring.on_audit_key_received ERROR reading data from %s' %
newpacket.RemoteID)
return False
try:
raw_payload = block.Data()
json_payload = serialization.BytesToDict(raw_payload,
keys_to_text=True,
values_to_text=True)
key_id = json_payload['key_id']
json_payload['audit']
public_sample = base64.b64decode(
json_payload['audit']['public_sample'])
private_sample = base64.b64decode(
json_payload['audit']['private_sample'])
except Exception as exc:
lg.exc()
p2p_service.SendFail(newpacket, str(exc))
return False
if not my_keys.is_valid_key_id(key_id):
p2p_service.SendFail(newpacket, 'invalid key id')
return False
if not my_keys.is_key_registered(key_id, include_master=True):
p2p_service.SendFail(newpacket, 'key not registered')
return False
if public_sample:
response_payload = base64.b64encode(
my_keys.encrypt(key_id, public_sample))
p2p_service.SendAck(newpacket, response_payload)
if _Debug:
lg.info('remote user %s requested audit of public key %s' %
(newpacket.OwnerID, key_id))
return True
if private_sample:
if not my_keys.is_key_private(key_id):
p2p_service.SendFail(newpacket, 'private key not registered')
return False
response_payload = base64.b64encode(
my_keys.decrypt(key_id, private_sample))
p2p_service.SendAck(newpacket, response_payload)
if _Debug:
lg.info('remote user %s requested audit of private key %s' %
(newpacket.OwnerID, key_id))
return True
p2p_service.SendFail(newpacket, 'wrong audit request')
return False
def decrypt(self, inp, private_key=None):
"""
Decrypt `inp` data with private key provided (callable, key_id or openssh format),
or by using locally stored "master" key.
"""
if private_key is None:
return key.DecryptLocalPrivateKey(inp)
if callable(private_key):
return private_key(inp)
from crypt import my_keys
if my_keys.is_valid_key_id(private_key):
return my_keys.decrypt(private_key, inp)
return key.DecryptOpenSSHPrivateKey(private_key, inp)
def _on_files_received(self, newpacket, info):
import json
from logs import lg
from p2p import p2p_service
from storage import backup_fs
from storage import backup_control
from crypt import encrypted
from crypt import my_keys
from userid import my_id
from userid import global_id
try:
user_id = newpacket.PacketID.strip().split(':')[0]
if user_id == my_id.getGlobalID():
# skip my own Files() packets which comes from my suppliers
# only process list Files() from other users who granted me access
return False
key_id = user_id
if not my_keys.is_valid_key_id(key_id):
# ignore, invalid key id in packet id
return False
if not my_keys.is_key_private(key_id):
raise Exception('private key is not registered')
except Exception as exc:
lg.warn(str(exc))
p2p_service.SendFail(newpacket, str(exc))
return False
block = encrypted.Unserialize(newpacket.Payload)
if block is None:
lg.warn('failed reading data from %s' % newpacket.RemoteID)
return False
if block.CreatorID != global_id.GlobalUserToIDURL(user_id):
lg.warn(
'invalid packet, creator ID must be present in packet ID : %s ~ %s'
% (
block.CreatorID,
user_id,
))
return False
try:
json_data = json.loads(block.Data(), encoding='utf-8')
json_data['items']
customer_idurl = block.CreatorID
count = backup_fs.Unserialize(
raw_data=json_data,
iter=backup_fs.fs(customer_idurl),
iterID=backup_fs.fsID(customer_idurl),
from_json=True,
)
except Exception as exc:
lg.exc()
p2p_service.SendFail(newpacket, str(exc))
return False
p2p_service.SendAck(newpacket)
if count == 0:
lg.warn('no files were imported during file sharing')
else:
backup_control.Save()
lg.info('imported %d shared files from %s, key_id=%s' % (
count,
customer_idurl,
key_id,
))
return True
# from access import shared_access_coordinator
# this_share = shared_access_coordinator.get_active_share(key_id)
# if not this_share:
# lg.warn('share is not opened: %s' % key_id)
# p2p_service.SendFail(newpacket, 'share is not opened')
# return False
# this_share.automat('customer-list-files-received', (newpacket, info, block, ))
return True
def request(self, json_payload, newpacket, info):
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from main import events
from crypt import my_keys
from p2p import p2p_service
from contacts import contactsdb
from storage import accounting
from userid import global_id
customer_idurl = newpacket.OwnerID
customer_id = global_id.UrlToGlobalID(customer_idurl)
bytes_for_customer = 0
try:
bytes_for_customer = int(json_payload['needed_bytes'])
except:
lg.warn("wrong payload" % newpacket.Payload)
return p2p_service.SendFail(newpacket, 'wrong payload')
try:
customer_public_key = json_payload['customer_public_key']
customer_public_key_id = customer_public_key['key_id']
except:
customer_public_key = None
customer_public_key_id = None
data_owner_idurl = None
target_customer_idurl = None
family_position = json_payload.get('position')
ecc_map = json_payload.get('ecc_map')
family_snapshot = json_payload.get('family_snapshot')
key_id = json_payload.get('key_id')
target_customer_id = json_payload.get('customer_id')
if key_id:
# this is a request from external user to access shared data stored by one of my customers
# this is "second" customer requesting data from "first" customer
if not key_id or not my_keys.is_valid_key_id(key_id):
lg.warn('missed or invalid key id')
return p2p_service.SendFail(newpacket, 'invalid key id')
target_customer_idurl = global_id.GlobalUserToIDURL(
target_customer_id)
if not contactsdb.is_customer(target_customer_idurl):
lg.warn("target user %s is not a customer" %
target_customer_id)
return p2p_service.SendFail(newpacket, 'not a customer')
if target_customer_idurl == customer_idurl:
lg.warn('customer %s requesting shared access to own files' %
customer_idurl)
return p2p_service.SendFail(newpacket, 'invalid case')
if not my_keys.is_key_registered(key_id):
lg.warn('key not registered: %s' % key_id)
p2p_service.SendFail(newpacket, 'key not registered')
return False
data_owner_idurl = my_keys.split_key_id(key_id)[1]
if data_owner_idurl != target_customer_idurl and data_owner_idurl != customer_idurl:
# pretty complex scenario:
# external customer requesting access to data which belongs not to that customer
# this is "third" customer accessing data belongs to "second" customer
# TODO: for now just stop it
lg.warn(
'under construction, key_id=%s customer_idurl=%s target_customer_idurl=%s'
% (
key_id,
customer_idurl,
target_customer_idurl,
))
p2p_service.SendFail(newpacket, 'under construction')
return False
# do not create connection with that customer, only accept the request
lg.info(
'external customer %s requested access to shared data at %s' %
(
customer_id,
key_id,
))
return p2p_service.SendAck(newpacket, 'accepted')
# key_id is not present in the request:
# this is a request to connect new customer (or reconnect existing one) to that supplier
if not bytes_for_customer or bytes_for_customer < 0:
lg.warn("wrong payload : %s" % newpacket.Payload)
return p2p_service.SendFail(newpacket, 'wrong storage value')
current_customers = contactsdb.customers()
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.request created a new space file')
space_dict = accounting.read_customers_quotas()
try:
free_bytes = int(space_dict[b'free'])
except:
lg.exc()
return p2p_service.SendFail(newpacket, 'broken space file')
if (customer_idurl not in current_customers
and customer_idurl in list(space_dict.keys())):
lg.warn("broken space file")
return p2p_service.SendFail(newpacket, 'broken space file')
if (customer_idurl in current_customers
and customer_idurl not in list(space_dict.keys())):
lg.warn("broken customers file")
return p2p_service.SendFail(newpacket, 'broken customers file')
if customer_idurl in current_customers:
free_bytes += int(space_dict.get(customer_idurl, 0))
space_dict[b'free'] = free_bytes
current_customers.remove(customer_idurl)
space_dict.pop(customer_idurl)
new_customer = False
else:
new_customer = True
lg.out(
8, ' new_customer=%s current_allocated_bytes=%s' % (
new_customer,
space_dict.get(customer_idurl),
))
from supplier import local_tester
if free_bytes <= bytes_for_customer:
contactsdb.update_customers(current_customers)
contactsdb.remove_customer_meta_info(customer_idurl)
contactsdb.save_customers()
accounting.write_customers_quotas(space_dict)
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
reactor.callLater(
0, local_tester.TestUpdateCustomers) # @UndefinedVariable
if new_customer:
lg.out(
8,
" NEW CUSTOMER: DENIED !!!!!!!!!!! not enough space available"
)
events.send('new-customer-denied', dict(idurl=customer_idurl))
else:
lg.out(
8,
" OLD CUSTOMER: DENIED !!!!!!!!!!! not enough space available"
)
events.send('existing-customer-denied',
dict(idurl=customer_idurl))
return p2p_service.SendAck(newpacket, 'deny')
space_dict[b'free'] = free_bytes - bytes_for_customer
current_customers.append(customer_idurl)
space_dict[customer_idurl] = bytes_for_customer
contactsdb.update_customers(current_customers)
contactsdb.save_customers()
contactsdb.add_customer_meta_info(
customer_idurl, {
'ecc_map': ecc_map,
'position': family_position,
'family_snapshot': family_snapshot,
})
accounting.write_customers_quotas(space_dict)
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
try:
if not my_keys.is_key_registered(customer_public_key_id):
key_id, key_object = my_keys.read_key_info(
customer_public_key)
if not my_keys.register_key(key_id, key_object):
lg.err('failed to register customer public key')
except:
lg.exc()
else:
lg.warn('customer public key was not provided in the request')
reactor.callLater(
0, local_tester.TestUpdateCustomers) # @UndefinedVariable
if new_customer:
lg.out(
8,
" NEW CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s"
%
(customer_idurl, family_position, ecc_map, bytes_for_customer))
lg.out(
8,
" family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot,
)
events.send(
'new-customer-accepted',
dict(
idurl=customer_idurl,
allocated_bytes=bytes_for_customer,
ecc_map=ecc_map,
position=family_position,
family_snapshot=family_snapshot,
key_id=customer_public_key_id,
))
else:
lg.out(
8,
" OLD CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s"
%
(customer_idurl, family_position, ecc_map, bytes_for_customer))
lg.out(
8,
" family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot)
events.send(
'existing-customer-accepted',
dict(
idurl=customer_idurl,
allocated_bytes=bytes_for_customer,
ecc_map=ecc_map,
position=family_position,
key_id=customer_public_key_id,
family_snapshot=family_snapshot,
))
return p2p_service.SendAck(newpacket, 'accepted')
def _on_files_received(self, newpacket, info):
from logs import lg
from lib import serialization
from main import settings
from main import events
from p2p import p2p_service
from storage import backup_fs
from storage import backup_control
from crypt import encrypted
from crypt import my_keys
from userid import my_id
from userid import global_id
from storage import backup_matrix
from supplier import list_files
from contacts import contactsdb
list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID)
if not list_files_global_id['idurl']:
lg.warn('invalid PacketID: %s' % newpacket.PacketID)
return False
trusted_customer_idurl = list_files_global_id['idurl']
incoming_key_id = list_files_global_id['key_id']
if trusted_customer_idurl == my_id.getGlobalID():
lg.warn('skip %s packet which seems to came from my own supplier' %
newpacket)
# only process list Files() from other users who granted me access
return False
if not my_keys.is_valid_key_id(incoming_key_id):
lg.warn('ignore, invalid key id in packet %s' % newpacket)
return False
if not my_keys.is_key_private(incoming_key_id):
lg.warn('private key is not registered : %s' % incoming_key_id)
p2p_service.SendFail(newpacket, 'private key is not registered')
return False
try:
block = encrypted.Unserialize(
newpacket.Payload,
decrypt_key=incoming_key_id,
)
except:
lg.exc(newpacket.Payload)
return False
if block is None:
lg.warn('failed reading data from %s' % newpacket.RemoteID)
return False
# if block.CreatorID != trusted_customer_idurl:
# lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % (
# block.CreatorID, list_files_global_id['idurl'], ))
# return False
try:
raw_files = block.Data()
except:
lg.exc()
return False
if block.CreatorID == trusted_customer_idurl:
# this is a trusted guy sending some shared files to me
try:
json_data = serialization.BytesToDict(raw_files,
keys_to_text=True)
json_data['items']
except:
lg.exc()
return False
count = backup_fs.Unserialize(
raw_data=json_data,
iter=backup_fs.fs(trusted_customer_idurl),
iterID=backup_fs.fsID(trusted_customer_idurl),
from_json=True,
)
p2p_service.SendAck(newpacket)
events.send(
'shared-list-files-received',
dict(
customer_idurl=trusted_customer_idurl,
new_items=count,
))
if count == 0:
lg.warn('no files were imported during file sharing')
else:
backup_control.Save()
lg.info('imported %d shared files from %s, key_id=%s' % (
count,
trusted_customer_idurl,
incoming_key_id,
))
return True
# otherwise this must be an external supplier sending us a files he stores for trusted customer
external_supplier_idurl = block.CreatorID
try:
supplier_raw_list_files = list_files.UnpackListFiles(
raw_files, settings.ListFilesFormat())
backup_matrix.SaveLatestRawListFiles(
supplier_idurl=external_supplier_idurl,
raw_data=supplier_raw_list_files,
customer_idurl=trusted_customer_idurl,
)
except:
lg.exc()
return False
# need to detect supplier position from the list of packets
# and place that supplier on the correct position in contactsdb
real_supplier_pos = backup_matrix.DetectSupplierPosition(
supplier_raw_list_files)
known_supplier_pos = contactsdb.supplier_position(
external_supplier_idurl, trusted_customer_idurl)
if real_supplier_pos >= 0:
if known_supplier_pos >= 0 and known_supplier_pos != real_supplier_pos:
lg.warn(
'external supplier %s position is not matching to list files, rewriting for customer %s'
% (external_supplier_idurl, trusted_customer_idurl))
contactsdb.erase_supplier(
idurl=external_supplier_idurl,
customer_idurl=trusted_customer_idurl,
)
contactsdb.add_supplier(
idurl=external_supplier_idurl,
position=real_supplier_pos,
customer_idurl=trusted_customer_idurl,
)
contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl)
else:
lg.warn(
'not possible to detect external supplier position for customer %s'
% trusted_customer_idurl)
# finally send ack packet back
p2p_service.SendAck(newpacket)
lg.info(
'received list of packets from external supplier %s for customer %s'
% (external_supplier_idurl, trusted_customer_idurl))
return True
def on_files_received(newpacket, info):
list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID)
if not list_files_global_id['idurl']:
lg.warn('invalid PacketID: %s' % newpacket.PacketID)
return False
trusted_customer_idurl = list_files_global_id['idurl']
incoming_key_id = list_files_global_id['key_id']
if trusted_customer_idurl == my_id.getLocalID():
if _Debug:
lg.dbg(_DebugLevel, 'ignore %s packet which seems to came from my own supplier' % newpacket)
# only process list Files() from other customers who granted me access to their files
return False
if not my_keys.is_valid_key_id(incoming_key_id):
lg.warn('ignore, invalid key id in packet %s' % newpacket)
return False
if not my_keys.is_key_private(incoming_key_id):
lg.warn('private key is not registered : %s' % incoming_key_id)
p2p_service.SendFail(newpacket, 'private key is not registered')
return False
try:
block = encrypted.Unserialize(
newpacket.Payload,
decrypt_key=incoming_key_id,
)
except:
lg.exc(newpacket.Payload)
return False
if block is None:
lg.warn('failed reading data from %s' % newpacket.RemoteID)
return False
# if block.CreatorID != trusted_customer_idurl:
# lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % (
# block.CreatorID, list_files_global_id['idurl'], ))
# return False
try:
raw_files = block.Data()
except:
lg.exc()
return False
if block.CreatorID == trusted_customer_idurl:
# this is a trusted guy sending some shared files to me
try:
json_data = serialization.BytesToDict(raw_files, keys_to_text=True, encoding='utf-8')
json_data['items']
except:
lg.exc()
return False
count = backup_fs.Unserialize(
raw_data=json_data,
iter=backup_fs.fs(trusted_customer_idurl),
iterID=backup_fs.fsID(trusted_customer_idurl),
from_json=True,
)
p2p_service.SendAck(newpacket)
if count == 0:
lg.warn('no files were imported during file sharing')
else:
backup_control.Save()
lg.info('imported %d shared files from %s, key_id=%s' % (
count, trusted_customer_idurl, incoming_key_id, ))
events.send('shared-list-files-received', dict(
customer_idurl=trusted_customer_idurl,
new_items=count,
))
return True
# otherwise this must be an external supplier sending us a files he stores for trusted customer
external_supplier_idurl = block.CreatorID
try:
supplier_raw_list_files = list_files.UnpackListFiles(raw_files, settings.ListFilesFormat())
except:
lg.exc()
return False
# need to detect supplier position from the list of packets
# and place that supplier on the correct position in contactsdb
supplier_pos = backup_matrix.DetectSupplierPosition(supplier_raw_list_files)
known_supplier_pos = contactsdb.supplier_position(external_supplier_idurl, trusted_customer_idurl)
if _Debug:
lg.args(_DebugLevel, supplier_pos=supplier_pos, known_supplier_pos=known_supplier_pos, external_supplier=external_supplier_idurl,
trusted_customer=trusted_customer_idurl, key_id=incoming_key_id)
if supplier_pos >= 0:
if known_supplier_pos >= 0 and known_supplier_pos != supplier_pos:
lg.err('known external supplier %r position %d is not matching to received list files position %d for customer %s' % (
external_supplier_idurl, known_supplier_pos, supplier_pos, trusted_customer_idurl))
# TODO: we should remove that bellow because we do not need it
# service_customer_family() should take care of suppliers list for trusted customer
# so we need to just read that list from DHT
# contactsdb.erase_supplier(
# idurl=external_supplier_idurl,
# customer_idurl=trusted_customer_idurl,
# )
# contactsdb.add_supplier(
# idurl=external_supplier_idurl,
# position=supplier_pos,
# customer_idurl=trusted_customer_idurl,
# )
# contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl)
else:
lg.warn('not possible to detect external supplier position for customer %s from received list files, known position is %s' % (
trusted_customer_idurl, known_supplier_pos))
supplier_pos = known_supplier_pos
remote_files_changed, _, _, _ = backup_matrix.process_raw_list_files(
supplier_num=supplier_pos,
list_files_text_body=supplier_raw_list_files,
customer_idurl=trusted_customer_idurl,
is_in_sync=True,
auto_create=True,
)
if remote_files_changed:
backup_matrix.SaveLatestRawListFiles(
supplier_idurl=external_supplier_idurl,
raw_data=supplier_raw_list_files,
customer_idurl=trusted_customer_idurl,
)
# finally sending Ack() packet back
p2p_service.SendAck(newpacket)
if remote_files_changed:
lg.info('received updated list of files from external supplier %s for customer %s' % (external_supplier_idurl, trusted_customer_idurl))
return True
