def cancel(self, request, info):
from main import events
from p2p import p2p_service
if not contactsdb.is_customer(request.OwnerID):
lg.warn(
"got packet from %s, but he is not a customer" %
request.OwnerID)
return p2p_service.SendFail(request, 'not a customer')
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.cancel created a new space file')
space_dict = accounting.read_customers_quotas()
if request.OwnerID not in space_dict.keys():
lg.warn(
"got packet from %s, but not found him in space dictionary" %
request.OwnerID)
return p2p_service.SendFail(request, 'not a customer')
try:
free_bytes = int(space_dict['free'])
space_dict['free'] = free_bytes + int(space_dict[request.OwnerID])
except:
lg.exc()
return p2p_service.SendFail(request, 'broken space file')
new_customers = list(contactsdb.customers())
new_customers.remove(request.OwnerID)
contactsdb.update_customers(new_customers)
contactsdb.save_customers()
space_dict.pop(request.OwnerID)
accounting.write_customers_quotas(space_dict)
from supplier import local_tester
reactor.callLater(0, local_tester.TestUpdateCustomers)
lg.out(8, " OLD CUSTOMER: TERMINATED !!!!!!!!!!!!!!")
events.send('existing-customer-terminated', dict(idurl=request.OwnerID))
return p2p_service.SendAck(request, 'accepted')
def cancel(self, json_payload, newpacket, info):
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from main import events
from p2p import p2p_service
from contacts import contactsdb
from storage import accounting
customer_idurl = newpacket.OwnerID
if not contactsdb.is_customer(customer_idurl):
lg.warn("got packet from %s, but he is not a customer" % customer_idurl)
return p2p_service.SendFail(newpacket, 'not a customer')
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.cancel created a new space file')
space_dict = accounting.read_customers_quotas()
if customer_idurl not in list(space_dict.keys()):
lg.warn("got packet from %s, but not found him in space dictionary" % customer_idurl)
return p2p_service.SendFail(newpacket, 'not a customer')
try:
free_bytes = int(space_dict[b'free'])
space_dict[b'free'] = free_bytes + int(space_dict[customer_idurl])
except:
lg.exc()
return p2p_service.SendFail(newpacket, 'broken space file')
new_customers = list(contactsdb.customers())
new_customers.remove(customer_idurl)
contactsdb.update_customers(new_customers)
contactsdb.remove_customer_meta_info(customer_idurl)
contactsdb.save_customers()
space_dict.pop(customer_idurl)
accounting.write_customers_quotas(space_dict)
from supplier import local_tester
reactor.callLater(0, local_tester.TestUpdateCustomers) # @UndefinedVariable
lg.out(8, " OLD CUSTOMER: TERMINATED !!!!!!!!!!!!!!")
events.send('existing-customer-terminated', dict(idurl=customer_idurl))
return p2p_service.SendAck(newpacket, 'accepted')
def make_valid_filename(customerIDURL, glob_path):
"""
Must be a customer, and then we make full path filename for where this
packet is stored locally.
"""
keyAlias = glob_path['key_alias'] or 'master'
packetID = glob_path['path']
customerGlobID = glob_path['customer']
if not customerGlobID:
lg.warn("customer id is empty")
return ''
if not packetid.Valid(packetID): # SECURITY
if packetID not in [
settings.BackupInfoFileName(),
settings.BackupInfoFileNameOld(),
settings.BackupInfoEncryptedFileName(),
settings.BackupIndexFileName()
]:
lg.warn('invalid file path')
return ''
if not contactsdb.is_customer(customerIDURL): # SECURITY
lg.warn("%s is not my customer" % (customerIDURL))
if customerGlobID:
if glob_path['idurl'] != customerIDURL:
lg.warn('making filename for another customer: %s != %s' %
(glob_path['idurl'], customerIDURL))
filename = make_filename(customerGlobID, packetID, keyAlias)
return filename
def cancel(self, request, info):
from p2p import p2p_service
if not contactsdb.is_customer(request.OwnerID):
lg.warn(
"got packet from %s, but he is not a customer" %
request.OwnerID)
return p2p_service.SendFail(request, 'not a customer')
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.cancel created a new space file')
space_dict = accounting.read_customers_quotas()
if request.OwnerID not in space_dict.keys():
lg.warn(
"got packet from %s, but not found him in space dictionary" %
request.OwnerID)
return p2p_service.SendFail(request, 'not a customer')
try:
free_bytes = int(space_dict['free'])
space_dict['free'] = free_bytes + int(space_dict[request.OwnerID])
except:
lg.exc()
return p2p_service.SendFail(request, 'broken space file')
new_customers = list(contactsdb.customers())
new_customers.remove(request.OwnerID)
contactsdb.update_customers(new_customers)
contactsdb.save_customers()
space_dict.pop(request.OwnerID)
accounting.write_customers_quotas(space_dict)
from supplier import local_tester
reactor.callLater(0, local_tester.TestUpdateCustomers)
return p2p_service.SendAck(request, 'accepted')
def _outbox_packet_sent(self, pkt_out):
from p2p import commands
from contacts import contactsdb
from supplier import customer_assistant
if pkt_out.outpacket.Command == commands.Identity():
if contactsdb.is_customer(pkt_out.outpacket.RemoteID):
ca = customer_assistant.by_idurl(pkt_out.outpacket.RemoteID)
if ca:
ca.automat('propagate', pkt_out)
def cancel(self, json_payload, newpacket, info):
from logs import lg
from contacts import contactsdb
from p2p import p2p_service
customer_idurl = newpacket.OwnerID
if not contactsdb.is_customer(customer_idurl):
lg.warn("got packet from %s, but he is not a customer" %
customer_idurl)
from dht import dht_relations
dht_relations.close_customer_supplier_relation(customer_idurl)
return p2p_service.SendAck(newpacket, 'accepted')
def _on_inbox_packet_received(self, newpacket, info, status, error_message):
from p2p import commands
from contacts import contactsdb
from supplier import customer_assistant
if newpacket.Command in [commands.Ack(), commands.Fail()]:
if contactsdb.is_customer(newpacket.OwnerID):
ca = customer_assistant.by_idurl(newpacket.OwnerID)
if ca:
ca.automat(newpacket.Command.lower(), newpacket)
return True
return False
def _on_outbox_packet_sent(self, pkt_out):
from twisted.internet import reactor # @UnresolvedImport
from p2p import commands
from contacts import contactsdb
from supplier import customer_assistant
if pkt_out.outpacket.Command == commands.Identity():
if contactsdb.is_customer(pkt_out.outpacket.RemoteID):
ca = customer_assistant.by_idurl(pkt_out.outpacket.RemoteID)
if ca:
reactor.callLater(0, ca.automat, 'propagate',
pkt_out) # @UndefinedVariable
return False
def cancel(self, json_payload, newpacket, info):
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from main import events
from p2p import p2p_service
from contacts import contactsdb
from storage import accounting
from crypt import my_keys
customer_idurl = newpacket.OwnerID
try:
customer_public_key = json_payload['customer_public_key']
customer_public_key_id = customer_public_key['key_id']
except:
customer_public_key = None
customer_public_key_id = None
customer_ecc_map = json_payload.get('ecc_map')
if not contactsdb.is_customer(customer_idurl):
lg.warn("got packet from %s, but he is not a customer" %
customer_idurl)
return p2p_service.SendFail(newpacket, 'not a customer')
if accounting.check_create_customers_quotas():
lg.info('created a new space file')
space_dict, free_space = accounting.read_customers_quotas()
if customer_idurl.to_bin() not in list(space_dict.keys()):
lg.warn(
"got packet from %s, but not found him in space dictionary" %
customer_idurl)
return p2p_service.SendFail(newpacket, 'not a customer')
try:
free_bytes = int(free_space)
free_space = free_bytes + int(space_dict[customer_idurl.to_bin()])
except:
lg.exc()
return p2p_service.SendFail(newpacket, 'broken space file')
new_customers = list(contactsdb.customers())
new_customers.remove(customer_idurl)
space_dict.pop(customer_idurl.to_bin())
accounting.write_customers_quotas(space_dict, free_space)
contactsdb.remove_customer_meta_info(customer_idurl)
contactsdb.update_customers(new_customers)
contactsdb.save_customers()
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
# TODO: erase customer's groups keys also
from supplier import local_tester
reactor.callLater(
0, local_tester.TestUpdateCustomers) # @UndefinedVariable
lg.info("OLD CUSTOMER TERMINATED %r" % customer_idurl)
events.send('existing-customer-terminated',
data=dict(idurl=customer_idurl, ecc_map=customer_ecc_map))
return p2p_service.SendAck(newpacket, 'accepted')
def _on_inbox_packet_received(self, newpacket, info, status,
error_message):
from twisted.internet import reactor # @UnresolvedImport
from p2p import commands
from contacts import contactsdb
from supplier import customer_assistant
if newpacket.Command in [commands.Ack(), commands.Fail()]:
if contactsdb.is_customer(newpacket.OwnerID):
ca = customer_assistant.by_idurl(newpacket.OwnerID)
if ca:
reactor.callLater(0, ca.automat, newpacket.Command.lower(),
newpacket) # @UndefinedVariable
return True
return False
def Retrieve(request):
"""
Customer is asking us for data he previously stored with us.
We send with ``outboxNoAck()`` method because he will ask again if
he does not get it
"""
# TODO: rename to RetreiveData()
if not driver.is_started("service_supplier"):
return SendFail(request, "supplier service is off")
if not contactsdb.is_customer(request.OwnerID):
lg.warn("had unknown customer " + request.OwnerID)
SendFail(request, "not a customer")
return
filename = makeFilename(request.OwnerID, request.PacketID)
if filename == "":
lg.warn("had empty filename")
SendFail(request, "empty filename")
return
if not os.path.exists(filename):
lg.warn("did not find requested file locally " + filename)
SendFail(request, "did not find requested file locally")
return
if not os.access(filename, os.R_OK):
lg.warn("no read access to requested packet " + filename)
SendFail(request, "no read access to requested packet")
return
data = bpio.ReadBinaryFile(filename)
if not data:
lg.warn("empty data on disk " + filename)
SendFail(request, "empty data on disk")
return
outpacket = signed.Unserialize(data)
del data
if outpacket is None:
lg.warn("Unserialize fails, not Valid packet " + filename)
SendFail(request, "unserialize fails")
return
if not outpacket.Valid():
lg.warn("unserialized packet is not Valid " + filename)
SendFail(request, "unserialized packet is not Valid")
return
if _Debug:
lg.out(
_DebugLevel,
"p2p_service.Retrieve sending %r back to %s" % (outpacket, nameurl.GetName(outpacket.CreatorID)),
)
gateway.outbox(outpacket, target=outpacket.CreatorID)
def makeFilename(customerID, packetID):
"""
Must be a customer, and then we make full path filename for where this
packet is stored locally.
"""
if not packetid.Valid(packetID): # SECURITY
if packetID not in [
settings.BackupInfoFileName(),
settings.BackupInfoFileNameOld(),
settings.BackupInfoEncryptedFileName(),
settings.BackupIndexFileName(),
]:
# lg.out(1, "p2p_service.makeFilename ERROR failed packetID format: " + packetID )
return ""
if not contactsdb.is_customer(customerID): # SECURITY
lg.warn("%s is not a customer" % (customerID))
return ""
return constructFilename(customerID, packetID)
def makeFilename(customerIDURL, packetID):
"""
Must be a customer, and then we make full path filename for where this
packet is stored locally.
"""
customerGlobID, packetID = packetid.SplitPacketID(packetID)
if not packetid.Valid(packetID): # SECURITY
if packetID not in [
settings.BackupInfoFileName(),
settings.BackupInfoFileNameOld(),
settings.BackupInfoEncryptedFileName(),
settings.BackupIndexFileName()
]:
# lg.out(1, "p2p_service.makeFilename ERROR failed packetID format: " + packetID )
return ''
if not contactsdb.is_customer(customerIDURL): # SECURITY
lg.warn("%s is not a customer" % (customerIDURL))
return ''
if customerGlobID:
customerIDURL_packet = global_id.GlobalUserToIDURL(customerGlobID)
if customerIDURL_packet != customerIDURL:
lg.warn('making filename for another customer: %s != %s' %
(customerIDURL_packet, customerIDURL))
return constructFilename(customerIDURL, packetID)
def Data(request):
"""
This is when we 1) save my requested data to restore the backup 2) or save
the customer file on our local HDD.
"""
# 1. this is our Data!
if request.OwnerID == my_id.getLocalID():
if _Debug:
lg.out(_DebugLevel, "p2p_service.Data %r for us from %s" % (request, nameurl.GetName(request.RemoteID)))
if driver.is_started("service_backups"):
if request.PacketID in [settings.BackupIndexFileName()]:
from storage import backup_control
backup_control.IncomingSupplierBackupIndex(request)
return True
return False
# 2. this Data is not belong to us
if not driver.is_started("service_supplier"):
return SendFail(request, "supplier service is off")
if not contactsdb.is_customer(request.OwnerID): # SECURITY
lg.warn("%s not a customer, packetID=%s" % (request.OwnerID, request.PacketID))
SendFail(request, "not a customer")
return
filename = makeFilename(request.OwnerID, request.PacketID)
if filename == "":
lg.warn("got empty filename, bad customer or wrong packetID? ")
SendFail(request, "empty filename")
return
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.warn("ERROR can not create sub dir " + dirname)
SendFail(request, "write error")
return
data = request.Serialize()
donated_bytes = settings.getDonatedBytes()
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(), {"free": donated_bytes})
if _Debug:
lg.out(_DebugLevel, "p2p_service.Data created a new space file")
space_dict = bpio._read_dict(settings.CustomersSpaceFile())
if request.OwnerID not in space_dict.keys():
lg.warn("no info about donated space for %s" % request.OwnerID)
SendFail(request, "no info about donated space")
return
used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {})
if request.OwnerID in used_space_dict.keys():
try:
bytes_used_by_customer = int(used_space_dict[request.OwnerID])
bytes_donated_to_customer = int(space_dict[request.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(data):
lg.warn("no free space for %s" % request.OwnerID)
SendFail(request, "no free space")
return
except:
lg.exc()
if not bpio.WriteFile(filename, data):
lg.warn("ERROR can not write to " + str(filename))
SendFail(request, "write error")
return
SendAck(request, str(len(request.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime)
del data
if _Debug:
lg.out(
_DebugLevel,
"p2p_service.Data saved from [%s/%s] to %s"
% (nameurl.GetName(request.OwnerID), nameurl.GetName(request.CreatorID), filename),
)
def verify_packet_ownership(newpacket, raise_exception=False):
"""
At that point packet creator is already verified via signature,
but creator could be not authorized to store data on that node.
So based on owner ID decision must be made what to do with the packet.
Returns IDURL of the user who should receive and Ack() or None if not authorized.
"""
# SECURITY
owner_idurl = newpacket.OwnerID
creator_idurl = newpacket.CreatorID
owner_id = owner_idurl.to_id()
creator_id = creator_idurl.to_id()
packet_key_alias, packet_owner_id, _ = packetid.SplitKeyOwnerData(
newpacket.PacketID)
packet_key_id = my_keys.latest_key_id(
my_keys.make_key_id(packet_key_alias,
creator_idurl,
creator_glob_id=packet_owner_id))
if _Debug:
lg.args(_DebugLevel,
owner_id=owner_id,
creator_id=creator_id,
packet_id=newpacket.PacketID,
key_id_registered=my_keys.is_key_registered(packet_key_id))
if newpacket.Command == commands.Data():
if owner_idurl.to_bin() == creator_idurl.to_bin():
if contactsdb.is_customer(creator_idurl):
if _Debug:
lg.dbg(
_DebugLevel,
'OK, scenario 1: customer is sending own data to own supplier'
)
return owner_idurl
lg.err(
'FAIL, scenario 6: user is not my customer but trying to store data'
)
if raise_exception:
raise Exception(
'non-authorized user is trying to store data on the supplier'
)
return None
if contactsdb.is_customer(creator_idurl):
if _Debug:
lg.dbg(
_DebugLevel,
'OK, scenario 2: customer wants to store data for someone else on own supplier'
)
# TODO: check that, why do we need that?
return creator_idurl
if packet_owner_id == owner_id:
if contactsdb.is_customer(owner_idurl):
if my_keys.is_key_registered(packet_key_id):
if _Debug:
lg.dbg(
_DebugLevel,
'OK, scenario 3: another authorized user is sending data to customer to be stored on the supplier'
)
return creator_idurl
lg.err('non-authorized user is trying to store data on the supplier')
return None
if newpacket.Command in [
commands.DeleteFile(),
commands.DeleteBackup(),
]:
if owner_idurl == creator_idurl:
if contactsdb.is_customer(creator_idurl):
if _Debug:
lg.dbg(
_DebugLevel,
'OK, scenario 4: customer wants to remove already stored data on own supplier'
)
return owner_idurl
lg.err(
'FAIL, scenario 7: non-authorized user is trying to erase data owned by customer from the supplier'
)
if raise_exception:
raise Exception(
'non-authorized user is trying to erase data owned by customer from the supplier'
)
return None
if contactsdb.is_customer(creator_idurl):
# TODO: check that, why do we need that?
if _Debug:
lg.dbg(
_DebugLevel,
'OK, scenario 8: customer wants to erase existing data that belongs to someone else but stored on the supplier'
)
return creator_idurl
if packet_owner_id == owner_id:
if contactsdb.is_customer(owner_idurl):
if my_keys.is_key_registered(packet_key_id):
if _Debug:
lg.dbg(
_DebugLevel,
'OK, scenario 5: another authorized user wants to remove already stored data from the supplier'
)
return creator_idurl
lg.err('non-authorized user is trying to erase data on the supplier')
return None
if driver.is_enabled('service_proxy_server'):
if _Debug:
lg.dbg(
_DebugLevel,
'IGNORE, scenario 9: received Data() not authorized, but proxy router service was enabled'
)
return None
# TODO:
# scenario 9: make possible to set "active" flag True/False for any key
# this way customer can make virtual location available for other user but in read-only mode
raise Exception('scenario not implemented yet, received %r' % newpacket)
def _on_data(self, newpacket):
import os
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from lib import jsn
from system import bpio
from main import settings
from userid import my_id
from userid import global_id
from contacts import contactsdb
from p2p import p2p_service
from storage import accounting
if newpacket.OwnerID == my_id.getLocalID():
# this Data belong to us, SKIP
return False
if not contactsdb.is_customer(newpacket.OwnerID):
# SECURITY
# TODO: process files from another customer : glob_path['idurl']
lg.warn("skip, %s not a customer, packetID=%s" %
(newpacket.OwnerID, newpacket.PacketID))
# p2p_service.SendFail(newpacket, 'not a customer')
return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
p2p_service.SendFail(newpacket, 'incorrect path')
return False
filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID?")
p2p_service.SendFail(newpacket, 'empty filename')
return False
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.err("can not create sub dir %s" % dirname)
p2p_service.SendFail(newpacket, 'write error')
return False
data = newpacket.Serialize()
donated_bytes = settings.getDonatedBytes()
accounting.check_create_customers_quotas(donated_bytes)
space_dict = accounting.read_customers_quotas()
if newpacket.OwnerID not in list(space_dict.keys()):
lg.err("no info about donated space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no info about donated space')
return False
used_space_dict = accounting.read_customers_usage()
if newpacket.OwnerID in list(used_space_dict.keys()):
try:
bytes_used_by_customer = int(
used_space_dict[newpacket.OwnerID])
bytes_donated_to_customer = int(space_dict[newpacket.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(
data):
lg.warn("no free space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no free space')
return False
except:
lg.exc()
if not bpio.WriteBinaryFile(filename, data):
lg.err("can not write to %s" % str(filename))
p2p_service.SendFail(newpacket, 'write error')
return False
# Here Data() packet was stored as it is on supplier node (current machine)
sz = len(data)
del data
lg.out(self.debug_level, "service_supplier._on_data %r" % newpacket)
lg.out(
self.debug_level, " from [ %s | %s ]" % (
newpacket.OwnerID,
newpacket.CreatorID,
))
lg.out(self.debug_level, " saved with %d bytes to %s" % (
sz,
filename,
))
p2p_service.SendAck(newpacket, str(len(newpacket.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime) # @UndefinedVariable
if self.publish_event_supplier_file_modified:
from main import events
events.send('supplier-file-modified',
data=dict(
action='write',
glob_path=glob_path['path'],
owner_id=newpacket.OwnerID,
))
return True
def request(self, json_payload, newpacket, info):
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from main import events
from crypt import my_keys
from p2p import p2p_service
from contacts import contactsdb
from storage import accounting
from userid import global_id
customer_idurl = newpacket.OwnerID
customer_id = global_id.UrlToGlobalID(customer_idurl)
bytes_for_customer = 0
try:
bytes_for_customer = int(json_payload['needed_bytes'])
except:
lg.warn("wrong payload" % newpacket.Payload)
return p2p_service.SendFail(newpacket, 'wrong payload')
try:
customer_public_key = json_payload['customer_public_key']
customer_public_key_id = customer_public_key['key_id']
except:
customer_public_key = None
customer_public_key_id = None
data_owner_idurl = None
target_customer_idurl = None
family_position = json_payload.get('position')
ecc_map = json_payload.get('ecc_map')
family_snapshot = json_payload.get('family_snapshot')
key_id = json_payload.get('key_id')
target_customer_id = json_payload.get('customer_id')
if key_id:
# this is a request from external user to access shared data stored by one of my customers
# this is "second" customer requesting data from "first" customer
if not key_id or not my_keys.is_valid_key_id(key_id):
lg.warn('missed or invalid key id')
return p2p_service.SendFail(newpacket, 'invalid key id')
target_customer_idurl = global_id.GlobalUserToIDURL(
target_customer_id)
if not contactsdb.is_customer(target_customer_idurl):
lg.warn("target user %s is not a customer" %
target_customer_id)
return p2p_service.SendFail(newpacket, 'not a customer')
if target_customer_idurl == customer_idurl:
lg.warn('customer %s requesting shared access to own files' %
customer_idurl)
return p2p_service.SendFail(newpacket, 'invalid case')
if not my_keys.is_key_registered(key_id):
lg.warn('key not registered: %s' % key_id)
p2p_service.SendFail(newpacket, 'key not registered')
return False
data_owner_idurl = my_keys.split_key_id(key_id)[1]
if data_owner_idurl != target_customer_idurl and data_owner_idurl != customer_idurl:
# pretty complex scenario:
# external customer requesting access to data which belongs not to that customer
# this is "third" customer accessing data belongs to "second" customer
# TODO: for now just stop it
lg.warn(
'under construction, key_id=%s customer_idurl=%s target_customer_idurl=%s'
% (
key_id,
customer_idurl,
target_customer_idurl,
))
p2p_service.SendFail(newpacket, 'under construction')
return False
# do not create connection with that customer, only accept the request
lg.info(
'external customer %s requested access to shared data at %s' %
(
customer_id,
key_id,
))
return p2p_service.SendAck(newpacket, 'accepted')
# key_id is not present in the request:
# this is a request to connect new customer (or reconnect existing one) to that supplier
if not bytes_for_customer or bytes_for_customer < 0:
lg.warn("wrong payload : %s" % newpacket.Payload)
return p2p_service.SendFail(newpacket, 'wrong storage value')
current_customers = contactsdb.customers()
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.request created a new space file')
space_dict = accounting.read_customers_quotas()
try:
free_bytes = int(space_dict[b'free'])
except:
lg.exc()
return p2p_service.SendFail(newpacket, 'broken space file')
if (customer_idurl not in current_customers
and customer_idurl in list(space_dict.keys())):
lg.warn("broken space file")
return p2p_service.SendFail(newpacket, 'broken space file')
if (customer_idurl in current_customers
and customer_idurl not in list(space_dict.keys())):
lg.warn("broken customers file")
return p2p_service.SendFail(newpacket, 'broken customers file')
if customer_idurl in current_customers:
free_bytes += int(space_dict.get(customer_idurl, 0))
space_dict[b'free'] = free_bytes
current_customers.remove(customer_idurl)
space_dict.pop(customer_idurl)
new_customer = False
else:
new_customer = True
lg.out(
8, ' new_customer=%s current_allocated_bytes=%s' % (
new_customer,
space_dict.get(customer_idurl),
))
from supplier import local_tester
if free_bytes <= bytes_for_customer:
contactsdb.update_customers(current_customers)
contactsdb.remove_customer_meta_info(customer_idurl)
contactsdb.save_customers()
accounting.write_customers_quotas(space_dict)
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
reactor.callLater(
0, local_tester.TestUpdateCustomers) # @UndefinedVariable
if new_customer:
lg.out(
8,
" NEW CUSTOMER: DENIED !!!!!!!!!!! not enough space available"
)
events.send('new-customer-denied', dict(idurl=customer_idurl))
else:
lg.out(
8,
" OLD CUSTOMER: DENIED !!!!!!!!!!! not enough space available"
)
events.send('existing-customer-denied',
dict(idurl=customer_idurl))
return p2p_service.SendAck(newpacket, 'deny')
space_dict[b'free'] = free_bytes - bytes_for_customer
current_customers.append(customer_idurl)
space_dict[customer_idurl] = bytes_for_customer
contactsdb.update_customers(current_customers)
contactsdb.save_customers()
contactsdb.add_customer_meta_info(
customer_idurl, {
'ecc_map': ecc_map,
'position': family_position,
'family_snapshot': family_snapshot,
})
accounting.write_customers_quotas(space_dict)
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
try:
if not my_keys.is_key_registered(customer_public_key_id):
key_id, key_object = my_keys.read_key_info(
customer_public_key)
if not my_keys.register_key(key_id, key_object):
lg.err('failed to register customer public key')
except:
lg.exc()
else:
lg.warn('customer public key was not provided in the request')
reactor.callLater(
0, local_tester.TestUpdateCustomers) # @UndefinedVariable
if new_customer:
lg.out(
8,
" NEW CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s"
%
(customer_idurl, family_position, ecc_map, bytes_for_customer))
lg.out(
8,
" family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot,
)
events.send(
'new-customer-accepted',
dict(
idurl=customer_idurl,
allocated_bytes=bytes_for_customer,
ecc_map=ecc_map,
position=family_position,
family_snapshot=family_snapshot,
key_id=customer_public_key_id,
))
else:
lg.out(
8,
" OLD CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s"
%
(customer_idurl, family_position, ecc_map, bytes_for_customer))
lg.out(
8,
" family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot)
events.send(
'existing-customer-accepted',
dict(
idurl=customer_idurl,
allocated_bytes=bytes_for_customer,
ecc_map=ecc_map,
position=family_position,
key_id=customer_public_key_id,
family_snapshot=family_snapshot,
))
return p2p_service.SendAck(newpacket, 'accepted')
def _on_data(self, newpacket):
import os
from twisted.internet import reactor
from logs import lg
from system import bpio
from main import settings
from userid import my_id
from userid import global_id
from contacts import contactsdb
from p2p import p2p_service
if newpacket.OwnerID == my_id.getLocalID():
# this Data belong to us, SKIP
return False
if not contactsdb.is_customer(newpacket.OwnerID): # SECURITY
lg.err("%s not a customer, packetID=%s" %
(newpacket.OwnerID, newpacket.PacketID))
p2p_service.SendFail(newpacket, 'not a customer')
return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
p2p_service.SendFail(newpacket, 'incorrect path')
return False
# TODO: process files from another customer : glob_path['idurl']
filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID?")
p2p_service.SendFail(newpacket, 'empty filename')
return False
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.err("can not create sub dir %s" % dirname)
p2p_service.SendFail(newpacket, 'write error')
return False
data = newpacket.Serialize()
donated_bytes = settings.getDonatedBytes()
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(), {
'free': donated_bytes,
})
lg.warn('created a new space file: %s' %
settings.CustomersSpaceFile())
space_dict = bpio._read_dict(settings.CustomersSpaceFile())
if newpacket.OwnerID not in space_dict.keys():
lg.err("no info about donated space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no info about donated space')
return False
used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(),
{})
if newpacket.OwnerID in used_space_dict.keys():
try:
bytes_used_by_customer = int(
used_space_dict[newpacket.OwnerID])
bytes_donated_to_customer = int(space_dict[newpacket.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(
data):
lg.warn("no free space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no free space')
return False
except:
lg.exc()
if not bpio.WriteFile(filename, data):
lg.err("can not write to %s" % str(filename))
p2p_service.SendFail(newpacket, 'write error')
return False
# Here Data() packet was stored as it is on supplier node (current machine)
sz = len(data)
del data
lg.out(
self.debug_level,
"service_supplier._on_data %r saved from [%s | %s] to %s with %d bytes"
% (
newpacket,
newpacket.OwnerID,
newpacket.CreatorID,
filename,
sz,
))
p2p_service.SendAck(newpacket, str(len(newpacket.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime)
# temporary disabled
# from main import events
# events.send('supplier-file-modified', data=dict(
# action='write',
# glob_path=glob_path['path'],
# owner_id=newpacket.OwnerID,
# ))
return True
def _on_retreive(self, newpacket):
import os
from logs import lg
from system import bpio
from userid import my_id
from userid import global_id
from crypt import signed
from contacts import contactsdb
from transport import gateway
from p2p import p2p_service
from p2p import commands
if not contactsdb.is_customer(newpacket.OwnerID):
lg.err("had unknown customer %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'not a customer')
return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
p2p_service.SendFail(newpacket, 'incorrect path')
return False
if glob_path['idurl']:
if newpacket.CreatorID == glob_path['idurl']:
pass # same customer, based on CreatorID : OK!
else:
lg.warn(
'one of customers requesting a Data from another customer!'
)
else:
lg.warn('no customer global id found in PacketID: %s' %
newpacket.PacketID)
# TODO: process requests from another customer : glob_path['idurl']
filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
if True:
# TODO: settings.getCustomersDataSharingEnabled() and
# SECURITY
filename = self._do_make_valid_filename(
glob_path['idurl'], glob_path)
if not filename:
lg.warn("had empty filename")
p2p_service.SendFail(newpacket, 'empty filename')
return False
if not os.path.exists(filename):
lg.warn("did not find requested file locally : %s" % filename)
p2p_service.SendFail(newpacket,
'did not find requested file locally')
return False
if not os.access(filename, os.R_OK):
lg.warn("no read access to requested packet %s" % filename)
p2p_service.SendFail(newpacket,
'no read access to requested packet')
return False
data = bpio.ReadBinaryFile(filename)
if not data:
lg.warn("empty data on disk %s" % filename)
p2p_service.SendFail(newpacket, 'empty data on disk')
return False
outpacket = signed.Unserialize(data)
del data
if outpacket is None:
lg.warn("Unserialize fails, not Valid packet %s" % filename)
p2p_service.SendFail(newpacket, 'unserialize fails')
return False
if not outpacket.Valid():
lg.warn("unserialized packet is not Valid %s" % filename)
p2p_service.SendFail(newpacket, 'unserialized packet is not Valid')
return False
if outpacket.Command != commands.Data():
lg.warn('sending back packet which is not a Data')
# here Data() packet is sent back as it is...
# that means outpacket.RemoteID=my_id.getLocalID() - it was addressed to that node and stored as it is
# need to take that in account every time you receive Data() packet
# it can be not a new Data(), but the old data returning back as a response to Retreive() packet
lg.warn('from request %r : sending %r back to %s' %
(newpacket, outpacket, outpacket.CreatorID))
gateway.outbox(outpacket, target=outpacket.CreatorID)
return True
def Data(request):
"""
This is when we 1) save my requested data to restore the backup 2) or save
the customer file on our local HDD.
"""
if _Debug:
lg.out(
_DebugLevel, 'p2p_service.Data %d bytes in [%s] by %s | %s' %
(len(request.Payload), request.PacketID, request.OwnerID,
request.CreatorID))
# 1. this is our Data!
if request.OwnerID == my_id.getLocalID():
if _Debug:
lg.out(
_DebugLevel, "p2p_service.Data %r for us from %s" %
(request, nameurl.GetName(request.RemoteID)))
if driver.is_on('service_backups'):
# TODO: move this into callback
settings.BackupIndexFileName()
indexPacketID = global_id.MakeGlobalID(
idurl=my_id.getLocalID(), path=settings.BackupIndexFileName())
if request.PacketID == indexPacketID:
from storage import backup_control
backup_control.IncomingSupplierBackupIndex(request)
return True
return False
# 2. this Data is not belong to us
if not driver.is_on('service_supplier'):
return SendFail(request, 'supplier service is off')
if not contactsdb.is_customer(request.OwnerID): # SECURITY
lg.warn("%s not a customer, packetID=%s" %
(request.OwnerID, request.PacketID))
SendFail(request, 'not a customer')
return
glob_path = global_id.ParseGlobalID(request.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(my_id.getGlobalID() + ':' +
request.PacketID)
if not glob_path['path']:
lg.warn("got incorrect PacketID")
SendFail(request, 'incorrect PacketID')
return
# TODO: process files from another customer : glob_path['idurl']
filename = makeFilename(request.OwnerID, glob_path['path'])
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID? ")
SendFail(request, 'empty filename')
return
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.warn("ERROR can not create sub dir " + dirname)
SendFail(request, 'write error')
return
data = request.Serialize()
donated_bytes = settings.getDonatedBytes()
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(),
{'free': donated_bytes})
if _Debug:
lg.out(_DebugLevel, 'p2p_service.Data created a new space file')
space_dict = bpio._read_dict(settings.CustomersSpaceFile())
if request.OwnerID not in space_dict.keys():
lg.warn("no info about donated space for %s" % request.OwnerID)
SendFail(request, 'no info about donated space')
return
used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {})
if request.OwnerID in used_space_dict.keys():
try:
bytes_used_by_customer = int(used_space_dict[request.OwnerID])
bytes_donated_to_customer = int(space_dict[request.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(data):
lg.warn("no free space for %s" % request.OwnerID)
SendFail(request, 'no free space')
return
except:
lg.exc()
if not bpio.WriteFile(filename, data):
lg.warn("ERROR can not write to " + str(filename))
SendFail(request, 'write error')
return
SendAck(request, str(len(request.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime)
del data
if _Debug:
lg.out(
_DebugLevel, "p2p_service.Data saved from [%s | %s] to %s" % (
request.OwnerID,
request.CreatorID,
filename,
))
def Retrieve(request):
"""
Customer is asking us for data he previously stored with us.
We send with ``outboxNoAck()`` method because he will ask again if
he does not get it
"""
# TODO: move to storage folder
# TODO: rename to RetrieveData()
if _Debug:
lg.out(
_DebugLevel, 'p2p_service.Retrieve [%s] by %s | %s' %
(request.PacketID, request.OwnerID, request.CreatorID))
if not driver.is_on('service_supplier'):
return SendFail(request, 'supplier service is off')
if not contactsdb.is_customer(request.OwnerID):
lg.warn("had unknown customer " + request.OwnerID)
SendFail(request, 'not a customer')
return
glob_path = global_id.ParseGlobalID(request.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(my_id.getGlobalID() + ':' +
request.PacketID)
if not glob_path['path']:
lg.warn("got incorrect PacketID")
SendFail(request, 'incorrect PacketID')
return
if glob_path['idurl']:
if request.CreatorID == glob_path['idurl']:
if _Debug:
lg.out(_DebugLevel, ' same customer CreatorID')
else:
lg.warn(
'one of customers requesting a Data from another customer!')
else:
lg.warn('no customer global id found in PacketID: %s' %
request.PacketID)
# TODO: process requests from another customer : glob_path['idurl']
filename = makeFilename(request.OwnerID, glob_path['path'])
if filename == '':
if True:
# TODO: settings.getCustomersDataSharingEnabled() and
# driver.services()['service_supplier'].has_permissions(request.CreatorID, )
filename = makeFilename(glob_path['idurl'], glob_path['path'])
if filename == '':
lg.warn("had empty filename")
SendFail(request, 'empty filename')
return
if not os.path.exists(filename):
lg.warn("did not find requested file locally " + filename)
SendFail(request, 'did not find requested file locally')
return
if not os.access(filename, os.R_OK):
lg.warn("no read access to requested packet " + filename)
SendFail(request, 'no read access to requested packet')
return
data = bpio.ReadBinaryFile(filename)
if not data:
lg.warn("empty data on disk " + filename)
SendFail(request, 'empty data on disk')
return
outpacket = signed.Unserialize(data)
del data
if outpacket is None:
lg.warn("Unserialize fails, not Valid packet " + filename)
SendFail(request, 'unserialize fails')
return
if not outpacket.Valid():
lg.warn("unserialized packet is not Valid " + filename)
SendFail(request, 'unserialized packet is not Valid')
return
if _Debug:
lg.out(
_DebugLevel, "p2p_service.Retrieve sending %r back to %s" %
(outpacket, nameurl.GetName(outpacket.CreatorID)))
gateway.outbox(outpacket, target=outpacket.CreatorID)
