def cancel(self, request, info): from main import events from p2p import p2p_service if not contactsdb.is_customer(request.OwnerID): lg.warn( "got packet from %s, but he is not a customer" % request.OwnerID) return p2p_service.SendFail(request, 'not a customer') if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.cancel created a new space file') space_dict = accounting.read_customers_quotas() if request.OwnerID not in space_dict.keys(): lg.warn( "got packet from %s, but not found him in space dictionary" % request.OwnerID) return p2p_service.SendFail(request, 'not a customer') try: free_bytes = int(space_dict['free']) space_dict['free'] = free_bytes + int(space_dict[request.OwnerID]) except: lg.exc() return p2p_service.SendFail(request, 'broken space file') new_customers = list(contactsdb.customers()) new_customers.remove(request.OwnerID) contactsdb.update_customers(new_customers) contactsdb.save_customers() space_dict.pop(request.OwnerID) accounting.write_customers_quotas(space_dict) from supplier import local_tester reactor.callLater(0, local_tester.TestUpdateCustomers) lg.out(8, " OLD CUSTOMER: TERMINATED !!!!!!!!!!!!!!") events.send('existing-customer-terminated', dict(idurl=request.OwnerID)) return p2p_service.SendAck(request, 'accepted')
def cancel(self, json_payload, newpacket, info): from twisted.internet import reactor # @UnresolvedImport from logs import lg from main import events from p2p import p2p_service from contacts import contactsdb from storage import accounting customer_idurl = newpacket.OwnerID if not contactsdb.is_customer(customer_idurl): lg.warn("got packet from %s, but he is not a customer" % customer_idurl) return p2p_service.SendFail(newpacket, 'not a customer') if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.cancel created a new space file') space_dict = accounting.read_customers_quotas() if customer_idurl not in list(space_dict.keys()): lg.warn("got packet from %s, but not found him in space dictionary" % customer_idurl) return p2p_service.SendFail(newpacket, 'not a customer') try: free_bytes = int(space_dict[b'free']) space_dict[b'free'] = free_bytes + int(space_dict[customer_idurl]) except: lg.exc() return p2p_service.SendFail(newpacket, 'broken space file') new_customers = list(contactsdb.customers()) new_customers.remove(customer_idurl) contactsdb.update_customers(new_customers) contactsdb.remove_customer_meta_info(customer_idurl) contactsdb.save_customers() space_dict.pop(customer_idurl) accounting.write_customers_quotas(space_dict) from supplier import local_tester reactor.callLater(0, local_tester.TestUpdateCustomers) # @UndefinedVariable lg.out(8, " OLD CUSTOMER: TERMINATED !!!!!!!!!!!!!!") events.send('existing-customer-terminated', dict(idurl=customer_idurl)) return p2p_service.SendAck(newpacket, 'accepted')
def make_valid_filename(customerIDURL, glob_path): """ Must be a customer, and then we make full path filename for where this packet is stored locally. """ keyAlias = glob_path['key_alias'] or 'master' packetID = glob_path['path'] customerGlobID = glob_path['customer'] if not customerGlobID: lg.warn("customer id is empty") return '' if not packetid.Valid(packetID): # SECURITY if packetID not in [ settings.BackupInfoFileName(), settings.BackupInfoFileNameOld(), settings.BackupInfoEncryptedFileName(), settings.BackupIndexFileName() ]: lg.warn('invalid file path') return '' if not contactsdb.is_customer(customerIDURL): # SECURITY lg.warn("%s is not my customer" % (customerIDURL)) if customerGlobID: if glob_path['idurl'] != customerIDURL: lg.warn('making filename for another customer: %s != %s' % (glob_path['idurl'], customerIDURL)) filename = make_filename(customerGlobID, packetID, keyAlias) return filename
def cancel(self, request, info): from p2p import p2p_service if not contactsdb.is_customer(request.OwnerID): lg.warn( "got packet from %s, but he is not a customer" % request.OwnerID) return p2p_service.SendFail(request, 'not a customer') if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.cancel created a new space file') space_dict = accounting.read_customers_quotas() if request.OwnerID not in space_dict.keys(): lg.warn( "got packet from %s, but not found him in space dictionary" % request.OwnerID) return p2p_service.SendFail(request, 'not a customer') try: free_bytes = int(space_dict['free']) space_dict['free'] = free_bytes + int(space_dict[request.OwnerID]) except: lg.exc() return p2p_service.SendFail(request, 'broken space file') new_customers = list(contactsdb.customers()) new_customers.remove(request.OwnerID) contactsdb.update_customers(new_customers) contactsdb.save_customers() space_dict.pop(request.OwnerID) accounting.write_customers_quotas(space_dict) from supplier import local_tester reactor.callLater(0, local_tester.TestUpdateCustomers) return p2p_service.SendAck(request, 'accepted')
def _outbox_packet_sent(self, pkt_out): from p2p import commands from contacts import contactsdb from supplier import customer_assistant if pkt_out.outpacket.Command == commands.Identity(): if contactsdb.is_customer(pkt_out.outpacket.RemoteID): ca = customer_assistant.by_idurl(pkt_out.outpacket.RemoteID) if ca: ca.automat('propagate', pkt_out)
def cancel(self, json_payload, newpacket, info): from logs import lg from contacts import contactsdb from p2p import p2p_service customer_idurl = newpacket.OwnerID if not contactsdb.is_customer(customer_idurl): lg.warn("got packet from %s, but he is not a customer" % customer_idurl) from dht import dht_relations dht_relations.close_customer_supplier_relation(customer_idurl) return p2p_service.SendAck(newpacket, 'accepted')
def _on_inbox_packet_received(self, newpacket, info, status, error_message): from p2p import commands from contacts import contactsdb from supplier import customer_assistant if newpacket.Command in [commands.Ack(), commands.Fail()]: if contactsdb.is_customer(newpacket.OwnerID): ca = customer_assistant.by_idurl(newpacket.OwnerID) if ca: ca.automat(newpacket.Command.lower(), newpacket) return True return False
def _on_outbox_packet_sent(self, pkt_out): from twisted.internet import reactor # @UnresolvedImport from p2p import commands from contacts import contactsdb from supplier import customer_assistant if pkt_out.outpacket.Command == commands.Identity(): if contactsdb.is_customer(pkt_out.outpacket.RemoteID): ca = customer_assistant.by_idurl(pkt_out.outpacket.RemoteID) if ca: reactor.callLater(0, ca.automat, 'propagate', pkt_out) # @UndefinedVariable return False
def cancel(self, json_payload, newpacket, info): from twisted.internet import reactor # @UnresolvedImport from logs import lg from main import events from p2p import p2p_service from contacts import contactsdb from storage import accounting from crypt import my_keys customer_idurl = newpacket.OwnerID try: customer_public_key = json_payload['customer_public_key'] customer_public_key_id = customer_public_key['key_id'] except: customer_public_key = None customer_public_key_id = None customer_ecc_map = json_payload.get('ecc_map') if not contactsdb.is_customer(customer_idurl): lg.warn("got packet from %s, but he is not a customer" % customer_idurl) return p2p_service.SendFail(newpacket, 'not a customer') if accounting.check_create_customers_quotas(): lg.info('created a new space file') space_dict, free_space = accounting.read_customers_quotas() if customer_idurl.to_bin() not in list(space_dict.keys()): lg.warn( "got packet from %s, but not found him in space dictionary" % customer_idurl) return p2p_service.SendFail(newpacket, 'not a customer') try: free_bytes = int(free_space) free_space = free_bytes + int(space_dict[customer_idurl.to_bin()]) except: lg.exc() return p2p_service.SendFail(newpacket, 'broken space file') new_customers = list(contactsdb.customers()) new_customers.remove(customer_idurl) space_dict.pop(customer_idurl.to_bin()) accounting.write_customers_quotas(space_dict, free_space) contactsdb.remove_customer_meta_info(customer_idurl) contactsdb.update_customers(new_customers) contactsdb.save_customers() if customer_public_key_id: my_keys.erase_key(customer_public_key_id) # TODO: erase customer's groups keys also from supplier import local_tester reactor.callLater( 0, local_tester.TestUpdateCustomers) # @UndefinedVariable lg.info("OLD CUSTOMER TERMINATED %r" % customer_idurl) events.send('existing-customer-terminated', data=dict(idurl=customer_idurl, ecc_map=customer_ecc_map)) return p2p_service.SendAck(newpacket, 'accepted')
def _on_inbox_packet_received(self, newpacket, info, status, error_message): from twisted.internet import reactor # @UnresolvedImport from p2p import commands from contacts import contactsdb from supplier import customer_assistant if newpacket.Command in [commands.Ack(), commands.Fail()]: if contactsdb.is_customer(newpacket.OwnerID): ca = customer_assistant.by_idurl(newpacket.OwnerID) if ca: reactor.callLater(0, ca.automat, newpacket.Command.lower(), newpacket) # @UndefinedVariable return True return False
def Retrieve(request): """ Customer is asking us for data he previously stored with us. We send with ``outboxNoAck()`` method because he will ask again if he does not get it """ # TODO: rename to RetreiveData() if not driver.is_started("service_supplier"): return SendFail(request, "supplier service is off") if not contactsdb.is_customer(request.OwnerID): lg.warn("had unknown customer " + request.OwnerID) SendFail(request, "not a customer") return filename = makeFilename(request.OwnerID, request.PacketID) if filename == "": lg.warn("had empty filename") SendFail(request, "empty filename") return if not os.path.exists(filename): lg.warn("did not find requested file locally " + filename) SendFail(request, "did not find requested file locally") return if not os.access(filename, os.R_OK): lg.warn("no read access to requested packet " + filename) SendFail(request, "no read access to requested packet") return data = bpio.ReadBinaryFile(filename) if not data: lg.warn("empty data on disk " + filename) SendFail(request, "empty data on disk") return outpacket = signed.Unserialize(data) del data if outpacket is None: lg.warn("Unserialize fails, not Valid packet " + filename) SendFail(request, "unserialize fails") return if not outpacket.Valid(): lg.warn("unserialized packet is not Valid " + filename) SendFail(request, "unserialized packet is not Valid") return if _Debug: lg.out( _DebugLevel, "p2p_service.Retrieve sending %r back to %s" % (outpacket, nameurl.GetName(outpacket.CreatorID)), ) gateway.outbox(outpacket, target=outpacket.CreatorID)
def makeFilename(customerID, packetID): """ Must be a customer, and then we make full path filename for where this packet is stored locally. """ if not packetid.Valid(packetID): # SECURITY if packetID not in [ settings.BackupInfoFileName(), settings.BackupInfoFileNameOld(), settings.BackupInfoEncryptedFileName(), settings.BackupIndexFileName(), ]: # lg.out(1, "p2p_service.makeFilename ERROR failed packetID format: " + packetID ) return "" if not contactsdb.is_customer(customerID): # SECURITY lg.warn("%s is not a customer" % (customerID)) return "" return constructFilename(customerID, packetID)
def makeFilename(customerIDURL, packetID): """ Must be a customer, and then we make full path filename for where this packet is stored locally. """ customerGlobID, packetID = packetid.SplitPacketID(packetID) if not packetid.Valid(packetID): # SECURITY if packetID not in [ settings.BackupInfoFileName(), settings.BackupInfoFileNameOld(), settings.BackupInfoEncryptedFileName(), settings.BackupIndexFileName() ]: # lg.out(1, "p2p_service.makeFilename ERROR failed packetID format: " + packetID ) return '' if not contactsdb.is_customer(customerIDURL): # SECURITY lg.warn("%s is not a customer" % (customerIDURL)) return '' if customerGlobID: customerIDURL_packet = global_id.GlobalUserToIDURL(customerGlobID) if customerIDURL_packet != customerIDURL: lg.warn('making filename for another customer: %s != %s' % (customerIDURL_packet, customerIDURL)) return constructFilename(customerIDURL, packetID)
def Data(request): """ This is when we 1) save my requested data to restore the backup 2) or save the customer file on our local HDD. """ # 1. this is our Data! if request.OwnerID == my_id.getLocalID(): if _Debug: lg.out(_DebugLevel, "p2p_service.Data %r for us from %s" % (request, nameurl.GetName(request.RemoteID))) if driver.is_started("service_backups"): if request.PacketID in [settings.BackupIndexFileName()]: from storage import backup_control backup_control.IncomingSupplierBackupIndex(request) return True return False # 2. this Data is not belong to us if not driver.is_started("service_supplier"): return SendFail(request, "supplier service is off") if not contactsdb.is_customer(request.OwnerID): # SECURITY lg.warn("%s not a customer, packetID=%s" % (request.OwnerID, request.PacketID)) SendFail(request, "not a customer") return filename = makeFilename(request.OwnerID, request.PacketID) if filename == "": lg.warn("got empty filename, bad customer or wrong packetID? ") SendFail(request, "empty filename") return dirname = os.path.dirname(filename) if not os.path.exists(dirname): try: bpio._dirs_make(dirname) except: lg.warn("ERROR can not create sub dir " + dirname) SendFail(request, "write error") return data = request.Serialize() donated_bytes = settings.getDonatedBytes() if not os.path.isfile(settings.CustomersSpaceFile()): bpio._write_dict(settings.CustomersSpaceFile(), {"free": donated_bytes}) if _Debug: lg.out(_DebugLevel, "p2p_service.Data created a new space file") space_dict = bpio._read_dict(settings.CustomersSpaceFile()) if request.OwnerID not in space_dict.keys(): lg.warn("no info about donated space for %s" % request.OwnerID) SendFail(request, "no info about donated space") return used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) if request.OwnerID in used_space_dict.keys(): try: bytes_used_by_customer = int(used_space_dict[request.OwnerID]) bytes_donated_to_customer = int(space_dict[request.OwnerID]) if bytes_donated_to_customer - bytes_used_by_customer < len(data): lg.warn("no free space for %s" % request.OwnerID) SendFail(request, "no free space") return except: lg.exc() if not bpio.WriteFile(filename, data): lg.warn("ERROR can not write to " + str(filename)) SendFail(request, "write error") return SendAck(request, str(len(request.Payload))) from supplier import local_tester reactor.callLater(0, local_tester.TestSpaceTime) del data if _Debug: lg.out( _DebugLevel, "p2p_service.Data saved from [%s/%s] to %s" % (nameurl.GetName(request.OwnerID), nameurl.GetName(request.CreatorID), filename), )
def verify_packet_ownership(newpacket, raise_exception=False): """ At that point packet creator is already verified via signature, but creator could be not authorized to store data on that node. So based on owner ID decision must be made what to do with the packet. Returns IDURL of the user who should receive and Ack() or None if not authorized. """ # SECURITY owner_idurl = newpacket.OwnerID creator_idurl = newpacket.CreatorID owner_id = owner_idurl.to_id() creator_id = creator_idurl.to_id() packet_key_alias, packet_owner_id, _ = packetid.SplitKeyOwnerData( newpacket.PacketID) packet_key_id = my_keys.latest_key_id( my_keys.make_key_id(packet_key_alias, creator_idurl, creator_glob_id=packet_owner_id)) if _Debug: lg.args(_DebugLevel, owner_id=owner_id, creator_id=creator_id, packet_id=newpacket.PacketID, key_id_registered=my_keys.is_key_registered(packet_key_id)) if newpacket.Command == commands.Data(): if owner_idurl.to_bin() == creator_idurl.to_bin(): if contactsdb.is_customer(creator_idurl): if _Debug: lg.dbg( _DebugLevel, 'OK, scenario 1: customer is sending own data to own supplier' ) return owner_idurl lg.err( 'FAIL, scenario 6: user is not my customer but trying to store data' ) if raise_exception: raise Exception( 'non-authorized user is trying to store data on the supplier' ) return None if contactsdb.is_customer(creator_idurl): if _Debug: lg.dbg( _DebugLevel, 'OK, scenario 2: customer wants to store data for someone else on own supplier' ) # TODO: check that, why do we need that? return creator_idurl if packet_owner_id == owner_id: if contactsdb.is_customer(owner_idurl): if my_keys.is_key_registered(packet_key_id): if _Debug: lg.dbg( _DebugLevel, 'OK, scenario 3: another authorized user is sending data to customer to be stored on the supplier' ) return creator_idurl lg.err('non-authorized user is trying to store data on the supplier') return None if newpacket.Command in [ commands.DeleteFile(), commands.DeleteBackup(), ]: if owner_idurl == creator_idurl: if contactsdb.is_customer(creator_idurl): if _Debug: lg.dbg( _DebugLevel, 'OK, scenario 4: customer wants to remove already stored data on own supplier' ) return owner_idurl lg.err( 'FAIL, scenario 7: non-authorized user is trying to erase data owned by customer from the supplier' ) if raise_exception: raise Exception( 'non-authorized user is trying to erase data owned by customer from the supplier' ) return None if contactsdb.is_customer(creator_idurl): # TODO: check that, why do we need that? if _Debug: lg.dbg( _DebugLevel, 'OK, scenario 8: customer wants to erase existing data that belongs to someone else but stored on the supplier' ) return creator_idurl if packet_owner_id == owner_id: if contactsdb.is_customer(owner_idurl): if my_keys.is_key_registered(packet_key_id): if _Debug: lg.dbg( _DebugLevel, 'OK, scenario 5: another authorized user wants to remove already stored data from the supplier' ) return creator_idurl lg.err('non-authorized user is trying to erase data on the supplier') return None if driver.is_enabled('service_proxy_server'): if _Debug: lg.dbg( _DebugLevel, 'IGNORE, scenario 9: received Data() not authorized, but proxy router service was enabled' ) return None # TODO: # scenario 9: make possible to set "active" flag True/False for any key # this way customer can make virtual location available for other user but in read-only mode raise Exception('scenario not implemented yet, received %r' % newpacket)
def _on_data(self, newpacket): import os from twisted.internet import reactor # @UnresolvedImport from logs import lg from lib import jsn from system import bpio from main import settings from userid import my_id from userid import global_id from contacts import contactsdb from p2p import p2p_service from storage import accounting if newpacket.OwnerID == my_id.getLocalID(): # this Data belong to us, SKIP return False if not contactsdb.is_customer(newpacket.OwnerID): # SECURITY # TODO: process files from another customer : glob_path['idurl'] lg.warn("skip, %s not a customer, packetID=%s" % (newpacket.OwnerID, newpacket.PacketID)) # p2p_service.SendFail(newpacket, 'not a customer') return False glob_path = global_id.ParseGlobalID(newpacket.PacketID) if not glob_path['path']: # backward compatible check glob_path = global_id.ParseGlobalID( my_id.getGlobalID('master') + ':' + newpacket.PacketID) if not glob_path['path']: lg.err("got incorrect PacketID") p2p_service.SendFail(newpacket, 'incorrect path') return False filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path) if not filename: lg.warn("got empty filename, bad customer or wrong packetID?") p2p_service.SendFail(newpacket, 'empty filename') return False dirname = os.path.dirname(filename) if not os.path.exists(dirname): try: bpio._dirs_make(dirname) except: lg.err("can not create sub dir %s" % dirname) p2p_service.SendFail(newpacket, 'write error') return False data = newpacket.Serialize() donated_bytes = settings.getDonatedBytes() accounting.check_create_customers_quotas(donated_bytes) space_dict = accounting.read_customers_quotas() if newpacket.OwnerID not in list(space_dict.keys()): lg.err("no info about donated space for %s" % newpacket.OwnerID) p2p_service.SendFail(newpacket, 'no info about donated space') return False used_space_dict = accounting.read_customers_usage() if newpacket.OwnerID in list(used_space_dict.keys()): try: bytes_used_by_customer = int( used_space_dict[newpacket.OwnerID]) bytes_donated_to_customer = int(space_dict[newpacket.OwnerID]) if bytes_donated_to_customer - bytes_used_by_customer < len( data): lg.warn("no free space for %s" % newpacket.OwnerID) p2p_service.SendFail(newpacket, 'no free space') return False except: lg.exc() if not bpio.WriteBinaryFile(filename, data): lg.err("can not write to %s" % str(filename)) p2p_service.SendFail(newpacket, 'write error') return False # Here Data() packet was stored as it is on supplier node (current machine) sz = len(data) del data lg.out(self.debug_level, "service_supplier._on_data %r" % newpacket) lg.out( self.debug_level, " from [ %s | %s ]" % ( newpacket.OwnerID, newpacket.CreatorID, )) lg.out(self.debug_level, " saved with %d bytes to %s" % ( sz, filename, )) p2p_service.SendAck(newpacket, str(len(newpacket.Payload))) from supplier import local_tester reactor.callLater(0, local_tester.TestSpaceTime) # @UndefinedVariable if self.publish_event_supplier_file_modified: from main import events events.send('supplier-file-modified', data=dict( action='write', glob_path=glob_path['path'], owner_id=newpacket.OwnerID, )) return True
def request(self, json_payload, newpacket, info): from twisted.internet import reactor # @UnresolvedImport from logs import lg from main import events from crypt import my_keys from p2p import p2p_service from contacts import contactsdb from storage import accounting from userid import global_id customer_idurl = newpacket.OwnerID customer_id = global_id.UrlToGlobalID(customer_idurl) bytes_for_customer = 0 try: bytes_for_customer = int(json_payload['needed_bytes']) except: lg.warn("wrong payload" % newpacket.Payload) return p2p_service.SendFail(newpacket, 'wrong payload') try: customer_public_key = json_payload['customer_public_key'] customer_public_key_id = customer_public_key['key_id'] except: customer_public_key = None customer_public_key_id = None data_owner_idurl = None target_customer_idurl = None family_position = json_payload.get('position') ecc_map = json_payload.get('ecc_map') family_snapshot = json_payload.get('family_snapshot') key_id = json_payload.get('key_id') target_customer_id = json_payload.get('customer_id') if key_id: # this is a request from external user to access shared data stored by one of my customers # this is "second" customer requesting data from "first" customer if not key_id or not my_keys.is_valid_key_id(key_id): lg.warn('missed or invalid key id') return p2p_service.SendFail(newpacket, 'invalid key id') target_customer_idurl = global_id.GlobalUserToIDURL( target_customer_id) if not contactsdb.is_customer(target_customer_idurl): lg.warn("target user %s is not a customer" % target_customer_id) return p2p_service.SendFail(newpacket, 'not a customer') if target_customer_idurl == customer_idurl: lg.warn('customer %s requesting shared access to own files' % customer_idurl) return p2p_service.SendFail(newpacket, 'invalid case') if not my_keys.is_key_registered(key_id): lg.warn('key not registered: %s' % key_id) p2p_service.SendFail(newpacket, 'key not registered') return False data_owner_idurl = my_keys.split_key_id(key_id)[1] if data_owner_idurl != target_customer_idurl and data_owner_idurl != customer_idurl: # pretty complex scenario: # external customer requesting access to data which belongs not to that customer # this is "third" customer accessing data belongs to "second" customer # TODO: for now just stop it lg.warn( 'under construction, key_id=%s customer_idurl=%s target_customer_idurl=%s' % ( key_id, customer_idurl, target_customer_idurl, )) p2p_service.SendFail(newpacket, 'under construction') return False # do not create connection with that customer, only accept the request lg.info( 'external customer %s requested access to shared data at %s' % ( customer_id, key_id, )) return p2p_service.SendAck(newpacket, 'accepted') # key_id is not present in the request: # this is a request to connect new customer (or reconnect existing one) to that supplier if not bytes_for_customer or bytes_for_customer < 0: lg.warn("wrong payload : %s" % newpacket.Payload) return p2p_service.SendFail(newpacket, 'wrong storage value') current_customers = contactsdb.customers() if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.request created a new space file') space_dict = accounting.read_customers_quotas() try: free_bytes = int(space_dict[b'free']) except: lg.exc() return p2p_service.SendFail(newpacket, 'broken space file') if (customer_idurl not in current_customers and customer_idurl in list(space_dict.keys())): lg.warn("broken space file") return p2p_service.SendFail(newpacket, 'broken space file') if (customer_idurl in current_customers and customer_idurl not in list(space_dict.keys())): lg.warn("broken customers file") return p2p_service.SendFail(newpacket, 'broken customers file') if customer_idurl in current_customers: free_bytes += int(space_dict.get(customer_idurl, 0)) space_dict[b'free'] = free_bytes current_customers.remove(customer_idurl) space_dict.pop(customer_idurl) new_customer = False else: new_customer = True lg.out( 8, ' new_customer=%s current_allocated_bytes=%s' % ( new_customer, space_dict.get(customer_idurl), )) from supplier import local_tester if free_bytes <= bytes_for_customer: contactsdb.update_customers(current_customers) contactsdb.remove_customer_meta_info(customer_idurl) contactsdb.save_customers() accounting.write_customers_quotas(space_dict) if customer_public_key_id: my_keys.erase_key(customer_public_key_id) reactor.callLater( 0, local_tester.TestUpdateCustomers) # @UndefinedVariable if new_customer: lg.out( 8, " NEW CUSTOMER: DENIED !!!!!!!!!!! not enough space available" ) events.send('new-customer-denied', dict(idurl=customer_idurl)) else: lg.out( 8, " OLD CUSTOMER: DENIED !!!!!!!!!!! not enough space available" ) events.send('existing-customer-denied', dict(idurl=customer_idurl)) return p2p_service.SendAck(newpacket, 'deny') space_dict[b'free'] = free_bytes - bytes_for_customer current_customers.append(customer_idurl) space_dict[customer_idurl] = bytes_for_customer contactsdb.update_customers(current_customers) contactsdb.save_customers() contactsdb.add_customer_meta_info( customer_idurl, { 'ecc_map': ecc_map, 'position': family_position, 'family_snapshot': family_snapshot, }) accounting.write_customers_quotas(space_dict) if customer_public_key_id: my_keys.erase_key(customer_public_key_id) try: if not my_keys.is_key_registered(customer_public_key_id): key_id, key_object = my_keys.read_key_info( customer_public_key) if not my_keys.register_key(key_id, key_object): lg.err('failed to register customer public key') except: lg.exc() else: lg.warn('customer public key was not provided in the request') reactor.callLater( 0, local_tester.TestUpdateCustomers) # @UndefinedVariable if new_customer: lg.out( 8, " NEW CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s" % (customer_idurl, family_position, ecc_map, bytes_for_customer)) lg.out( 8, " family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot, ) events.send( 'new-customer-accepted', dict( idurl=customer_idurl, allocated_bytes=bytes_for_customer, ecc_map=ecc_map, position=family_position, family_snapshot=family_snapshot, key_id=customer_public_key_id, )) else: lg.out( 8, " OLD CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s" % (customer_idurl, family_position, ecc_map, bytes_for_customer)) lg.out( 8, " family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot) events.send( 'existing-customer-accepted', dict( idurl=customer_idurl, allocated_bytes=bytes_for_customer, ecc_map=ecc_map, position=family_position, key_id=customer_public_key_id, family_snapshot=family_snapshot, )) return p2p_service.SendAck(newpacket, 'accepted')
def _on_data(self, newpacket): import os from twisted.internet import reactor from logs import lg from system import bpio from main import settings from userid import my_id from userid import global_id from contacts import contactsdb from p2p import p2p_service if newpacket.OwnerID == my_id.getLocalID(): # this Data belong to us, SKIP return False if not contactsdb.is_customer(newpacket.OwnerID): # SECURITY lg.err("%s not a customer, packetID=%s" % (newpacket.OwnerID, newpacket.PacketID)) p2p_service.SendFail(newpacket, 'not a customer') return False glob_path = global_id.ParseGlobalID(newpacket.PacketID) if not glob_path['path']: # backward compatible check glob_path = global_id.ParseGlobalID( my_id.getGlobalID('master') + ':' + newpacket.PacketID) if not glob_path['path']: lg.err("got incorrect PacketID") p2p_service.SendFail(newpacket, 'incorrect path') return False # TODO: process files from another customer : glob_path['idurl'] filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path) if not filename: lg.warn("got empty filename, bad customer or wrong packetID?") p2p_service.SendFail(newpacket, 'empty filename') return False dirname = os.path.dirname(filename) if not os.path.exists(dirname): try: bpio._dirs_make(dirname) except: lg.err("can not create sub dir %s" % dirname) p2p_service.SendFail(newpacket, 'write error') return False data = newpacket.Serialize() donated_bytes = settings.getDonatedBytes() if not os.path.isfile(settings.CustomersSpaceFile()): bpio._write_dict(settings.CustomersSpaceFile(), { 'free': donated_bytes, }) lg.warn('created a new space file: %s' % settings.CustomersSpaceFile()) space_dict = bpio._read_dict(settings.CustomersSpaceFile()) if newpacket.OwnerID not in space_dict.keys(): lg.err("no info about donated space for %s" % newpacket.OwnerID) p2p_service.SendFail(newpacket, 'no info about donated space') return False used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) if newpacket.OwnerID in used_space_dict.keys(): try: bytes_used_by_customer = int( used_space_dict[newpacket.OwnerID]) bytes_donated_to_customer = int(space_dict[newpacket.OwnerID]) if bytes_donated_to_customer - bytes_used_by_customer < len( data): lg.warn("no free space for %s" % newpacket.OwnerID) p2p_service.SendFail(newpacket, 'no free space') return False except: lg.exc() if not bpio.WriteFile(filename, data): lg.err("can not write to %s" % str(filename)) p2p_service.SendFail(newpacket, 'write error') return False # Here Data() packet was stored as it is on supplier node (current machine) sz = len(data) del data lg.out( self.debug_level, "service_supplier._on_data %r saved from [%s | %s] to %s with %d bytes" % ( newpacket, newpacket.OwnerID, newpacket.CreatorID, filename, sz, )) p2p_service.SendAck(newpacket, str(len(newpacket.Payload))) from supplier import local_tester reactor.callLater(0, local_tester.TestSpaceTime) # temporary disabled # from main import events # events.send('supplier-file-modified', data=dict( # action='write', # glob_path=glob_path['path'], # owner_id=newpacket.OwnerID, # )) return True
def _on_retreive(self, newpacket): import os from logs import lg from system import bpio from userid import my_id from userid import global_id from crypt import signed from contacts import contactsdb from transport import gateway from p2p import p2p_service from p2p import commands if not contactsdb.is_customer(newpacket.OwnerID): lg.err("had unknown customer %s" % newpacket.OwnerID) p2p_service.SendFail(newpacket, 'not a customer') return False glob_path = global_id.ParseGlobalID(newpacket.PacketID) if not glob_path['path']: # backward compatible check glob_path = global_id.ParseGlobalID( my_id.getGlobalID('master') + ':' + newpacket.PacketID) if not glob_path['path']: lg.err("got incorrect PacketID") p2p_service.SendFail(newpacket, 'incorrect path') return False if glob_path['idurl']: if newpacket.CreatorID == glob_path['idurl']: pass # same customer, based on CreatorID : OK! else: lg.warn( 'one of customers requesting a Data from another customer!' ) else: lg.warn('no customer global id found in PacketID: %s' % newpacket.PacketID) # TODO: process requests from another customer : glob_path['idurl'] filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path) if not filename: if True: # TODO: settings.getCustomersDataSharingEnabled() and # SECURITY filename = self._do_make_valid_filename( glob_path['idurl'], glob_path) if not filename: lg.warn("had empty filename") p2p_service.SendFail(newpacket, 'empty filename') return False if not os.path.exists(filename): lg.warn("did not find requested file locally : %s" % filename) p2p_service.SendFail(newpacket, 'did not find requested file locally') return False if not os.access(filename, os.R_OK): lg.warn("no read access to requested packet %s" % filename) p2p_service.SendFail(newpacket, 'no read access to requested packet') return False data = bpio.ReadBinaryFile(filename) if not data: lg.warn("empty data on disk %s" % filename) p2p_service.SendFail(newpacket, 'empty data on disk') return False outpacket = signed.Unserialize(data) del data if outpacket is None: lg.warn("Unserialize fails, not Valid packet %s" % filename) p2p_service.SendFail(newpacket, 'unserialize fails') return False if not outpacket.Valid(): lg.warn("unserialized packet is not Valid %s" % filename) p2p_service.SendFail(newpacket, 'unserialized packet is not Valid') return False if outpacket.Command != commands.Data(): lg.warn('sending back packet which is not a Data') # here Data() packet is sent back as it is... # that means outpacket.RemoteID=my_id.getLocalID() - it was addressed to that node and stored as it is # need to take that in account every time you receive Data() packet # it can be not a new Data(), but the old data returning back as a response to Retreive() packet lg.warn('from request %r : sending %r back to %s' % (newpacket, outpacket, outpacket.CreatorID)) gateway.outbox(outpacket, target=outpacket.CreatorID) return True
def Data(request): """ This is when we 1) save my requested data to restore the backup 2) or save the customer file on our local HDD. """ if _Debug: lg.out( _DebugLevel, 'p2p_service.Data %d bytes in [%s] by %s | %s' % (len(request.Payload), request.PacketID, request.OwnerID, request.CreatorID)) # 1. this is our Data! if request.OwnerID == my_id.getLocalID(): if _Debug: lg.out( _DebugLevel, "p2p_service.Data %r for us from %s" % (request, nameurl.GetName(request.RemoteID))) if driver.is_on('service_backups'): # TODO: move this into callback settings.BackupIndexFileName() indexPacketID = global_id.MakeGlobalID( idurl=my_id.getLocalID(), path=settings.BackupIndexFileName()) if request.PacketID == indexPacketID: from storage import backup_control backup_control.IncomingSupplierBackupIndex(request) return True return False # 2. this Data is not belong to us if not driver.is_on('service_supplier'): return SendFail(request, 'supplier service is off') if not contactsdb.is_customer(request.OwnerID): # SECURITY lg.warn("%s not a customer, packetID=%s" % (request.OwnerID, request.PacketID)) SendFail(request, 'not a customer') return glob_path = global_id.ParseGlobalID(request.PacketID) if not glob_path['path']: # backward compatible check glob_path = global_id.ParseGlobalID(my_id.getGlobalID() + ':' + request.PacketID) if not glob_path['path']: lg.warn("got incorrect PacketID") SendFail(request, 'incorrect PacketID') return # TODO: process files from another customer : glob_path['idurl'] filename = makeFilename(request.OwnerID, glob_path['path']) if not filename: lg.warn("got empty filename, bad customer or wrong packetID? ") SendFail(request, 'empty filename') return dirname = os.path.dirname(filename) if not os.path.exists(dirname): try: bpio._dirs_make(dirname) except: lg.warn("ERROR can not create sub dir " + dirname) SendFail(request, 'write error') return data = request.Serialize() donated_bytes = settings.getDonatedBytes() if not os.path.isfile(settings.CustomersSpaceFile()): bpio._write_dict(settings.CustomersSpaceFile(), {'free': donated_bytes}) if _Debug: lg.out(_DebugLevel, 'p2p_service.Data created a new space file') space_dict = bpio._read_dict(settings.CustomersSpaceFile()) if request.OwnerID not in space_dict.keys(): lg.warn("no info about donated space for %s" % request.OwnerID) SendFail(request, 'no info about donated space') return used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) if request.OwnerID in used_space_dict.keys(): try: bytes_used_by_customer = int(used_space_dict[request.OwnerID]) bytes_donated_to_customer = int(space_dict[request.OwnerID]) if bytes_donated_to_customer - bytes_used_by_customer < len(data): lg.warn("no free space for %s" % request.OwnerID) SendFail(request, 'no free space') return except: lg.exc() if not bpio.WriteFile(filename, data): lg.warn("ERROR can not write to " + str(filename)) SendFail(request, 'write error') return SendAck(request, str(len(request.Payload))) from supplier import local_tester reactor.callLater(0, local_tester.TestSpaceTime) del data if _Debug: lg.out( _DebugLevel, "p2p_service.Data saved from [%s | %s] to %s" % ( request.OwnerID, request.CreatorID, filename, ))
def Retrieve(request): """ Customer is asking us for data he previously stored with us. We send with ``outboxNoAck()`` method because he will ask again if he does not get it """ # TODO: move to storage folder # TODO: rename to RetrieveData() if _Debug: lg.out( _DebugLevel, 'p2p_service.Retrieve [%s] by %s | %s' % (request.PacketID, request.OwnerID, request.CreatorID)) if not driver.is_on('service_supplier'): return SendFail(request, 'supplier service is off') if not contactsdb.is_customer(request.OwnerID): lg.warn("had unknown customer " + request.OwnerID) SendFail(request, 'not a customer') return glob_path = global_id.ParseGlobalID(request.PacketID) if not glob_path['path']: # backward compatible check glob_path = global_id.ParseGlobalID(my_id.getGlobalID() + ':' + request.PacketID) if not glob_path['path']: lg.warn("got incorrect PacketID") SendFail(request, 'incorrect PacketID') return if glob_path['idurl']: if request.CreatorID == glob_path['idurl']: if _Debug: lg.out(_DebugLevel, ' same customer CreatorID') else: lg.warn( 'one of customers requesting a Data from another customer!') else: lg.warn('no customer global id found in PacketID: %s' % request.PacketID) # TODO: process requests from another customer : glob_path['idurl'] filename = makeFilename(request.OwnerID, glob_path['path']) if filename == '': if True: # TODO: settings.getCustomersDataSharingEnabled() and # driver.services()['service_supplier'].has_permissions(request.CreatorID, ) filename = makeFilename(glob_path['idurl'], glob_path['path']) if filename == '': lg.warn("had empty filename") SendFail(request, 'empty filename') return if not os.path.exists(filename): lg.warn("did not find requested file locally " + filename) SendFail(request, 'did not find requested file locally') return if not os.access(filename, os.R_OK): lg.warn("no read access to requested packet " + filename) SendFail(request, 'no read access to requested packet') return data = bpio.ReadBinaryFile(filename) if not data: lg.warn("empty data on disk " + filename) SendFail(request, 'empty data on disk') return outpacket = signed.Unserialize(data) del data if outpacket is None: lg.warn("Unserialize fails, not Valid packet " + filename) SendFail(request, 'unserialize fails') return if not outpacket.Valid(): lg.warn("unserialized packet is not Valid " + filename) SendFail(request, 'unserialized packet is not Valid') return if _Debug: lg.out( _DebugLevel, "p2p_service.Retrieve sending %r back to %s" % (outpacket, nameurl.GetName(outpacket.CreatorID))) gateway.outbox(outpacket, target=outpacket.CreatorID)