def isStillNeeded(self, *args, **kwargs): """ Condition method. """ supplier_idurl = args[0] current_suppliers = contactsdb.suppliers() if supplier_idurl in current_suppliers: # this guy is already a supplier, we still need more then return True desired_number = settings.getSuppliersNumberDesired() needed_suppliers = current_suppliers[:desired_number] empty_suppliers = needed_suppliers.count(id_url.field(b'')) # if '' in needed_suppliers: # lg.warn('found empty suppliers!!!') # return True s = set(id_url.to_bin_list(needed_suppliers)) s.add(id_url.to_bin(supplier_idurl)) s.difference_update(set(id_url.to_bin_list(self.dismiss_list))) result = len(s) - empty_suppliers < settings.getSuppliersNumberDesired( ) # if _Debug: # lg.out(_DebugLevel, 'fire_hire.isStillNeeded %d %d %d %d %d, result=%s' % ( # contactsdb.num_suppliers(), len(needed_suppliers), len(self.dismiss_list), # len(s), settings.getSuppliersNumberDesired(), result)) return result
def isMoreNeeded(self, *args, **kwargs): """ Condition method. """ # if _Debug: # lg.out(_DebugLevel, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % ( # contactsdb.num_suppliers(), len(self.dismiss_list), # settings.getSuppliersNumberDesired())) if id_url.is_some_empty(contactsdb.suppliers()): if _Debug: lg.out(_DebugLevel, 'fire_hire.isMoreNeeded found empty supplier!!!') return True if isinstance(args[0], list): dismissed = args[0] else: dismissed = self.dismiss_list s = set(id_url.to_bin_list(contactsdb.suppliers())) s.difference_update(set(id_url.to_bin_list(dismissed))) result = len(s) < settings.getSuppliersNumberDesired() if _Debug: lg.out( _DebugLevel, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' % (contactsdb.num_suppliers(), len(dismissed), len(s), settings.getSuppliersNumberDesired(), result)) return result
def is_correspondent(idurl): """ Return True if given ID is found in correspondents list. """ if id_url.is_empty(idurl): return False return id_url.field(idurl).to_bin() in id_url.to_bin_list(correspondents_ids())
def is_customer(idurl): """ Return True if given ID is found in customers list. """ if id_url.is_empty(idurl): return False return id_url.field(idurl).to_bin() in id_url.to_bin_list(customers())
def is_supplier(idurl, customer_idurl=None): """ Return True if given ID is found in suppliers list. """ if id_url.is_empty(idurl): return False return id_url.field(idurl).to_bin() in id_url.to_bin_list(suppliers(customer_idurl=customer_idurl))
def _on_existing_customer_accepted(self, evt): from twisted.internet import reactor # @UnresolvedImport from logs import lg from supplier import family_member from userid import id_url from userid import my_id customer_idurl = evt.data['idurl'] if customer_idurl == my_id.getLocalID(): lg.warn('skipping my own identity') return if evt.data.get('position') is None: lg.warn('position of supplier in the family is still unclear') return fm = family_member.by_customer_idurl(customer_idurl) if not fm: lg.err( 'family_member() instance was not found for existing customer %s' % customer_idurl) return reactor.callLater(0, fm.automat, 'family-join', { # @UndefinedVariable 'supplier_idurl': my_id.getLocalID().to_bin(), 'ecc_map': evt.data.get('ecc_map'), 'position': evt.data.get('position'), 'family_snapshot': id_url.to_bin_list(evt.data.get('family_snapshot')), })
def doConnectSuppliers(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector from p2p import online_status self.connect_list = [] my_current_family = contactsdb.suppliers() for pos, supplier_idurl in enumerate(my_current_family): if not supplier_idurl: continue if self.configs[0] and pos >= self.configs[0]: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None: sc = supplier_connector.create( supplier_idurl=supplier_idurl, customer_idurl=my_id.getIDURL(), ) else: sc.needed_bytes = None sc.do_calculate_needed_bytes() sc.set_callback('fire_hire', self._on_supplier_connector_state_changed) self.connect_list.append(supplier_idurl) sc.automat( 'connect', family_position=pos, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list(my_current_family), ) online_status.add_online_status_listener_callback( idurl=supplier_idurl, callback_method=self._on_supplier_online_status_state_changed, )
def total_suppliers(): """ """ global _SuppliersList result = set() for suppliers_list in _SuppliersList.values(): result.update(set(id_url.to_bin_list(suppliers_list))) return len(result)
def contacts_remote(include_all=False, include_enabled=True): """ Return ID's list of all known peers. """ allcontactslist = id_url.to_bin_list(contacts(include_all=include_all, include_enabled=include_enabled)) if my_id.getLocalID().to_bin() in allcontactslist: allcontactslist.remove(my_id.getLocalID().to_bin()) return id_url.fields_list(allcontactslist)
def contacts_remote(): """ Return ID's list of all known peers. """ allcontactslist = id_url.to_bin_list(contacts_full()) if my_id.getLocalID().to_bin() in allcontactslist: allcontactslist.remove(my_id.getLocalID().to_bin()) return id_url.fields_list(allcontactslist)
def getSources(self, as_fields=True, as_originals=False): """ Return identity sources. """ if as_originals: return id_url.to_original_list(self.sources) if not as_fields: return id_url.to_bin_list(self.sources) return self.sources
def add_customer_meta_info(customer_idurl, info): """ """ global _CustomersMetaInfo customer_idurl = id_url.field(customer_idurl) if not customer_idurl.is_latest(): if customer_idurl.original() in _CustomersMetaInfo: if customer_idurl.to_bin() not in _CustomersMetaInfo: _CustomersMetaInfo[ customer_idurl.to_bin()] = _CustomersMetaInfo.pop( customer_idurl.original()) lg.info( 'detected and processed idurl rotate for customer meta info : %r -> %r' % (customer_idurl.original(), customer_idurl.to_bin())) customer_idurl = id_url.to_bin(customer_idurl) if 'family_snapshot' in info: info['family_snapshot'] = id_url.to_bin_list(info['family_snapshot']) if 'ecc_map' in info: info['ecc_map'] = strng.to_text(info['ecc_map']) if customer_idurl not in _CustomersMetaInfo: if _Debug: lg.out( _DebugLevel, 'contactsdb.add_customer_meta_info store new meta info for customer %r: %r' % ( customer_idurl, info, )) _CustomersMetaInfo[customer_idurl] = {} else: if _Debug: lg.out( _DebugLevel, 'contactsdb.add_customer_meta_info update existing meta info for customer %r: %r' % ( customer_idurl, info, )) _CustomersMetaInfo[customer_idurl].update(info) json_info = { k: jsn.dict_keys_to_text(v) for k, v in id_url.to_bin_dict(_CustomersMetaInfo).items() } try: raw_data = jsn.dumps( json_info, indent=2, sort_keys=True, keys_to_text=True, values_to_text=True, ) except: lg.exc() return None local_fs.WriteTextFile(settings.CustomersMetaInfoFilename(), raw_data) return _CustomersMetaInfo
def doFindNewSupplier(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out( _DebugLevel, 'fire_hire.doFindNewSupplier desired_suppliers=%d current_suppliers=%r' % (settings.getSuppliersNumberDesired(), contactsdb.suppliers())) from p2p import network_connector if network_connector.A().state != 'CONNECTED': if _Debug: lg.out( _DebugLevel, ' network_connector is not CONNECTED at the moment, SKIP' ) self.automat('search-failed') return position_for_new_supplier = None for pos in range(settings.getSuppliersNumberDesired()): if pos in self.hire_list: continue supplier_idurl = contactsdb.supplier(pos) if not supplier_idurl: lg.info( 'found empty supplier at position %d and going to find new supplier on that position' % pos) position_for_new_supplier = pos break if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False): lg.info( 'going to find new supplier on existing position %d to replace supplier %s' % ( pos, supplier_idurl, )) position_for_new_supplier = pos break if position_for_new_supplier is None: lg.err('did not found position for new supplier') self.automat('search-failed') return from customer import supplier_finder for idurl_txt in strng.to_text(config.conf().getData( 'services/employer/candidates')).split(','): if idurl_txt.strip(): supplier_finder.AddSupplierToHire(idurl_txt) self.hire_list.append(position_for_new_supplier) supplier_finder.A( 'start', family_position=position_for_new_supplier, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list(contactsdb.suppliers()), )
def doInit(self, *args, **kwargs): """ Action method. """ self.lookup_method = kwargs.get('lookup_method', None) self.target_service = kwargs['target_service'] self.request_service_params = kwargs.get('request_service_params', None) self.result_callback = kwargs.get('result_callback', None) self.exclude_nodes = id_url.to_bin_list(kwargs.get( 'exclude_nodes', []))
def extract_discovered_idurls(count=1, layer_id=0): if not discovered_idurls(layer_id=layer_id): if _Debug: lg.out(_DebugLevel, 'lookup.extract_discovered_idurls returns empty list') return [] discovered = list(discovered_idurls(layer_id=layer_id)) # random.shuffle(discovered) results = id_url.to_bin_list(discovered[:count]) if _Debug: lg.out(_DebugLevel, 'lookup.extract_discovered_idurls : %s' % results) return results
def _do_verify(dht_value, customer_idurl_bin): if customer_idurl_bin in rotated_idurls: rotated_idurls.remove(customer_idurl_bin) ret = { 'suppliers': [], 'ecc_map': None, 'customer_idurl': customer_idurl, 'revision': 0, 'publisher_idurl': None, 'timestamp': None, } if not dht_value or not isinstance(dht_value, dict): if not rotated_idurls: result.callback(ret) return ret another_customer_idurl_bin = rotated_idurls.pop(0) lg.warn( 'found another rotated idurl %r and re-try reading customer suppliers' % another_customer_idurl_bin) d = dht_records.get_suppliers(another_customer_idurl_bin, return_details=True, use_cache=False) d.addCallback(_do_verify, another_customer_idurl_bin) d.addErrback(_on_error) return ret try: _ecc_map = strng.to_text(dht_value['ecc_map']) if as_fields: _customer_idurl = id_url.field(dht_value['customer_idurl']) _publisher_idurl = id_url.field( dht_value.get('publisher_idurl')) _suppliers_list = id_url.fields_list(dht_value['suppliers']) else: _customer_idurl = id_url.to_bin(dht_value['customer_idurl']) _publisher_idurl = id_url.to_bin( dht_value.get('publisher_idurl')) _suppliers_list = id_url.to_bin_list(dht_value['suppliers']) _revision = int(dht_value.get('revision')) _timestamp = int(dht_value.get('timestamp')) except: lg.exc() result.callback(ret) return ret ret.update({ 'suppliers': _suppliers_list, 'ecc_map': _ecc_map, 'customer_idurl': _customer_idurl, 'revision': _revision, 'publisher_idurl': _publisher_idurl, 'timestamp': _timestamp, }) return _do_identity_cache(ret)
def customer_position(idurl): """ Return position of supplier with given ID or -1. """ if not idurl: return -1 if not id_url.is_cached(idurl): return -1 idurl = id_url.field(idurl) try: index = id_url.to_bin_list(customers()).index(idurl.to_bin()) except: index = -1 return index
def _do_request_supplier_service(self, ecc_map, family_position, family_snapshot): if _Debug: lg.args(_DebugLevel, supplier_idurl=self.supplier_idurl, ecc_map=ecc_map, family_position=family_position, family_snapshot=family_snapshot) if not self.supplier_idurl: lg.warn( 'supplier idurl is empty, SKIP sending supplier_service request' ) return service_info = { 'needed_bytes': self.needed_bytes, 'customer_id': global_id.UrlToGlobalID(self.customer_idurl), } my_customer_key_id = my_id.getGlobalID(key_alias='customer') if my_keys.is_key_registered(my_customer_key_id): service_info['customer_public_key'] = my_keys.get_key_info( key_id=my_customer_key_id, include_private=False, include_signature=True, generate_signature=True, ) else: lg.warn('my own customer key is not registered: %r' % my_customer_key_id) if self.key_id: service_info['key_id'] = self.key_id self._last_known_ecc_map = ecc_map if self._last_known_ecc_map is not None: service_info['ecc_map'] = self._last_known_ecc_map self._last_known_family_position = family_position if self._last_known_family_position is not None: service_info['position'] = self._last_known_family_position self._last_known_family_snapshot = family_snapshot if self._last_known_family_snapshot is not None: service_info['family_snapshot'] = id_url.to_bin_list( self._last_known_family_snapshot) request = p2p_service.SendRequestService( remote_idurl=self.supplier_idurl, service_name='service_supplier', json_payload=service_info, callbacks={ commands.Ack(): self._supplier_service_acked, commands.Fail(): self._supplier_service_failed, }, ) self.request_packet_id = request.PacketID
def doInit(self, *args, **kwargs): """ Action method. """ self.lookup_method = kwargs.get('lookup_method', None) self.target_service = kwargs['target_service'] self.request_service_params = kwargs.get('request_service_params', None) self.request_service_timeout = kwargs.get('request_service_timeout', 120) self.ping_retries = kwargs.get('ping_retries', None) self.ack_timeout = kwargs.get('ack_timeout', None) self.force_handshake = kwargs.get('force_handshake', False) self.result_callback = kwargs.get('result_callback', None) self.exclude_nodes = id_url.to_bin_list(kwargs.get( 'exclude_nodes', [])) self.retries = kwargs.get('attempts', 5)
def contact_position(idurl): """ Return position for given contact ID in the total list combined from suppliers, customers. Suppliers should be numbered 0 to 63 with customers after that not sure we can count on numbers staying. """ if not idurl: return -1 if not id_url.is_cached(idurl): return -1 idurl = id_url.field(idurl) try: index = id_url.to_bin_list(contacts_list()).index(idurl.to_bin()) except: index = -1 return index
def _on_my_dht_relations_discovered(self, dht_result): from p2p import p2p_service from contacts import contactsdb from userid import my_id from userid import id_url from crypt import my_keys from logs import lg if not (dht_result and isinstance(dht_result, dict) and len(dht_result.get('suppliers', [])) > 0): lg.warn('no dht records found for my customer family') return if id_url.is_some_empty(contactsdb.suppliers()): lg.warn( 'some of my suppliers are not hired yet, skip doing any changes' ) return suppliers_to_be_dismissed = set() dht_suppliers = id_url.to_bin_list(dht_result['suppliers']) # clean up old suppliers for idurl in dht_suppliers: if not idurl: continue if not contactsdb.is_supplier(idurl): lg.warn('dht relation with %r is not valid anymore' % idurl) suppliers_to_be_dismissed.add(idurl) for supplier_idurl in suppliers_to_be_dismissed: service_info = {} my_customer_key_id = my_id.getGlobalID(key_alias='customer') if my_keys.is_key_registered(my_customer_key_id): service_info['customer_public_key'] = my_keys.get_key_info( key_id=my_customer_key_id, include_private=False, include_signature=False, generate_signature=False, ) p2p_service.SendCancelService( remote_idurl=supplier_idurl, service_name='service_supplier', json_payload=service_info, ) if suppliers_to_be_dismissed: lg.info( 'found %d suppliers to be cleaned and sent CancelService() packets' % len(suppliers_to_be_dismissed))
def _on_new_customer_accepted(self, evt): from twisted.internet import reactor # @UnresolvedImport from logs import lg from userid import my_id from userid import id_url from supplier import family_member customer_idurl = evt.data['idurl'] fm = family_member.by_customer_idurl(customer_idurl) if not fm: fm = family_member.create_family(customer_idurl) fm.automat('init') else: lg.warn( 'family_member() instance already exists, but new customer just accepted %s' % customer_idurl) reactor.callLater(0, fm.automat, 'family-join', { # @UndefinedVariable 'supplier_idurl': my_id.getLocalID().to_bin(), 'ecc_map': evt.data.get('ecc_map'), 'position': evt.data.get('position', -1), 'family_snapshot': id_url.to_bin_list(evt.data.get('family_snapshot')), })
def start(self): from twisted.internet import reactor # @UnresolvedImport from logs import lg from main import events from contacts import contactsdb from userid import id_url from supplier import family_member from transport import callback from userid import my_id callback.append_inbox_callback(self._on_inbox_packet_received) for customer_idurl in contactsdb.customers(): if not customer_idurl: continue if not id_url.is_cached(customer_idurl): continue if customer_idurl == my_id.getLocalID(): lg.warn('skipping my own identity') continue fm = family_member.by_customer_idurl(customer_idurl) if not fm: fm = family_member.create_family(customer_idurl) fm.automat('init') local_customer_meta_info = contactsdb.get_customer_meta_info( customer_idurl) reactor.callLater(0, fm.automat, 'family-join', { # @UndefinedVariable 'supplier_idurl': my_id.getLocalID().to_bin(), 'ecc_map': local_customer_meta_info.get('ecc_map'), 'position': local_customer_meta_info.get('position', -1), 'family_snapshot': id_url.to_bin_list(local_customer_meta_info.get('family_snapshot')), }) events.add_subscriber(self._on_identity_url_changed, 'identity-url-changed') events.add_subscriber(self._on_existing_customer_accepted, 'existing-customer-accepted') events.add_subscriber(self._on_new_customer_accepted, 'new-customer-accepted') events.add_subscriber(self._on_existing_customer_terminated, 'existing-customer-terminated') return True
def _do_verify(dht_value): ret = { 'suppliers': [], 'ecc_map': None, 'customer_idurl': customer_idurl, 'revision': 0, 'publisher_idurl': None, 'timestamp': None, } if not dht_value or not isinstance(dht_value, dict): result.callback(ret) return ret try: _ecc_map = strng.to_text(dht_value['ecc_map']) if as_fields: _customer_idurl = id_url.field(dht_value['customer_idurl']) _publisher_idurl = id_url.field( dht_value.get('publisher_idurl')) _suppliers_list = id_url.fields_list(dht_value['suppliers']) else: _customer_idurl = id_url.to_bin(dht_value['customer_idurl']) _publisher_idurl = id_url.to_bin( dht_value.get('publisher_idurl')) _suppliers_list = id_url.to_bin_list(dht_value['suppliers']) _revision = int(dht_value.get('revision')) _timestamp = int(dht_value.get('timestamp')) except: lg.exc() result.callback(ret) return ret ret.update({ 'suppliers': _suppliers_list, 'ecc_map': _ecc_map, 'customer_idurl': _customer_idurl, 'revision': _revision, 'publisher_idurl': _publisher_idurl, 'timestamp': _timestamp, }) return _do_identity_cache(ret)
def _on_my_dht_relations_discovered(self, dht_result): if not (dht_result and isinstance(dht_result, dict) and len(dht_result.get('suppliers', [])) > 0): lg.warn('no dht records found for my customer family') self.automat('suppliers-read-failed') return dht_suppliers = id_url.to_bin_list(dht_result['suppliers']) dht_ecc_map = dht_result.get('ecc_map', settings.DefaultEccMapName()) try: dht_desired_suppliers_number = eccmap.GetEccMapSuppliersNumber( dht_ecc_map) except: lg.exc() dht_desired_suppliers_number = eccmap.GetEccMapSuppliersNumber( settings.DefaultEccMapName()) settings.config.conf().setInt('services/customer/suppliers-number', dht_desired_suppliers_number) contactsdb.set_suppliers(dht_suppliers) contactsdb.save_suppliers() lg.info('found and restored list of %d suppliers from DHT' % dht_desired_suppliers_number) self.automat('suppliers-read-ok')
def validate_customers_quotas(space_dict=None, free_space=None): unknown_customers = set() unused_quotas = set() if space_dict is None or free_space is None: space_dict, free_space = read_customers_quotas() for idurl in list(space_dict.keys()): idurl = strng.to_bin(idurl) try: space_dict[idurl] = int(space_dict[idurl]) except: unknown_customers.add(idurl) continue if space_dict[idurl] <= 0: unknown_customers.add(idurl) continue for idurl in contactsdb.customers(): if idurl.to_bin() not in list(space_dict.keys()): unknown_customers.add(idurl) for idurl in space_dict.keys(): if idurl not in id_url.to_bin_list(contactsdb.customers()): unused_quotas.add(idurl) return unknown_customers, unused_quotas
def doSubstituteSupplier(self, *args, **kwargs): """ Action method. """ new_idurl = id_url.field(args[0]) family_position = kwargs.get('family_position') current_suppliers = list(contactsdb.suppliers()) desired_suppliers = settings.getSuppliersNumberDesired() old_idurl = None if family_position in self.hire_list: self.hire_list.remove(family_position) lg.info( 'found position on which new supplier suppose to be hired: %d' % family_position) else: lg.warn('did not found position for new supplier to be hired on') if new_idurl in current_suppliers: raise Exception('%s is already supplier' % new_idurl) if family_position is None or family_position == -1: lg.warn( 'unknown family_position from supplier results, will pick first empty spot' ) position = -1 old_idurl = None for i in range(len(current_suppliers)): if not current_suppliers[i].strip(): position = i break if id_url.is_in(current_suppliers[i], self.dismiss_list, as_field=False): position = i old_idurl = current_suppliers[i] break family_position = position if _Debug: lg.out( _DebugLevel, 'fire_hire.doSubstituteSupplier family_position=%d' % family_position) contactsdb.add_supplier(idurl=new_idurl, position=family_position) contactsdb.save_suppliers() misc.writeSupplierData( new_idurl, 'connected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getIDURL(), ) from main import control control.on_suppliers_changed(current_suppliers) if family_position < 0: lg.info( 'added new supplier, family position unknown: %s desired_suppliers=%d current_suppliers=%d' % (new_idurl, desired_suppliers, len(contactsdb.suppliers()))) events.send('supplier-modified', data=dict( new_idurl=new_idurl, old_idurl=None, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list( contactsdb.suppliers()), )) else: if old_idurl: lg.info( 'hired new supplier and substitute existing supplier on position %d : %s->%s desired_suppliers=%d current_suppliers=%d' % (family_position, old_idurl, new_idurl, desired_suppliers, len(contactsdb.suppliers()))) events.send('supplier-modified', data=dict( new_idurl=new_idurl, old_idurl=old_idurl, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list( contactsdb.suppliers()), )) else: lg.info( 'hired new supplier on empty position %d : %s desired_suppliers=%d current_suppliers=%d' % (family_position, new_idurl, desired_suppliers, len(contactsdb.suppliers()))) events.send('supplier-modified', data=dict( new_idurl=new_idurl, old_idurl=None, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list( contactsdb.suppliers()), )) self.restart_interval = 1.0 if _Debug: lg.out(_DebugLevel, ' my current suppliers: %r' % contactsdb.suppliers())
def request(self, json_payload, newpacket, info): from twisted.internet import reactor # @UnresolvedImport from logs import lg from main import events from crypt import my_keys from p2p import p2p_service from contacts import contactsdb from storage import accounting from supplier import customer_space from userid import id_url from userid import global_id customer_idurl = newpacket.OwnerID customer_id = global_id.UrlToGlobalID(customer_idurl) bytes_for_customer = 0 try: bytes_for_customer = int(json_payload['needed_bytes']) except: lg.exc() return p2p_service.SendFail(newpacket, 'invalid payload') try: customer_public_key = json_payload['customer_public_key'] customer_public_key_id = customer_public_key['key_id'] except: customer_public_key = None customer_public_key_id = None data_owner_idurl = None target_customer_idurl = None family_position = json_payload.get('position') ecc_map = json_payload.get('ecc_map') family_snapshot = json_payload.get('family_snapshot') if family_snapshot: family_snapshot = id_url.to_bin_list(family_snapshot) key_id = json_payload.get('key_id') key_id = my_keys.latest_key_id(key_id) target_customer_id = json_payload.get('customer_id') if key_id: # this is a request from external user to access shared data stored by one of my customers # this is "second" customer requesting data from "first" customer if not key_id or not my_keys.is_valid_key_id(key_id): lg.warn('missed or invalid key id') return p2p_service.SendFail(newpacket, 'invalid key id') target_customer_idurl = global_id.GlobalUserToIDURL( target_customer_id) if not contactsdb.is_customer(target_customer_idurl): lg.warn("target user %s is not a customer" % target_customer_id) return p2p_service.SendFail(newpacket, 'not a customer') if target_customer_idurl == customer_idurl: lg.warn('customer %s requesting shared access to own files' % customer_idurl) return p2p_service.SendFail(newpacket, 'invalid case') if not my_keys.is_key_registered(key_id): lg.warn('key not registered: %s' % key_id) p2p_service.SendFail(newpacket, 'key not registered') return False data_owner_idurl = my_keys.split_key_id(key_id)[1] if data_owner_idurl != target_customer_idurl and data_owner_idurl != customer_idurl: # pretty complex scenario: # external customer requesting access to data which belongs not to that customer # this is "third" customer accessing data belongs to "second" customer # TODO: for now just stop it lg.warn( 'under construction, key_id=%s customer_idurl=%s target_customer_idurl=%s' % ( key_id, customer_idurl, target_customer_idurl, )) p2p_service.SendFail(newpacket, 'under construction') return False customer_space.register_customer_key(customer_public_key_id, customer_public_key) # do not create connection with that customer, only accept the request lg.info( 'external customer %s requested access to shared data at %s' % ( customer_id, key_id, )) return p2p_service.SendAck(newpacket, 'accepted') # key_id is not present in the request: # this is a request to connect new customer (or reconnect existing one) to that supplier if not bytes_for_customer or bytes_for_customer < 0: lg.warn("wrong payload : %s" % newpacket.Payload) return p2p_service.SendFail(newpacket, 'wrong storage value') current_customers = contactsdb.customers() if accounting.check_create_customers_quotas(): lg.info('created new customers quotas file') space_dict, free_space = accounting.read_customers_quotas() try: free_bytes = int(free_space) except: lg.exc() return p2p_service.SendFail(newpacket, 'broken space file') if (customer_idurl not in current_customers and customer_idurl.to_bin() in list(space_dict.keys())): lg.warn("broken space file") return p2p_service.SendFail(newpacket, 'broken space file') if (customer_idurl in current_customers and customer_idurl.to_bin() not in list(space_dict.keys())): # seems like customer's idurl was rotated, but space file still have the old idurl # need to find that old idurl value and replace with the new one for other_customer_idurl in space_dict.keys(): if other_customer_idurl and other_customer_idurl != 'free' and id_url.field( other_customer_idurl) == customer_idurl: lg.info( 'found rotated customer identity in space file, switching: %r -> %r' % (other_customer_idurl, customer_idurl.to_bin())) space_dict[customer_idurl.to_bin()] = space_dict.pop( other_customer_idurl) break if customer_idurl.to_bin() not in list(space_dict.keys()): lg.warn("broken customers file") return p2p_service.SendFail(newpacket, 'broken customers file') if customer_idurl in current_customers: free_bytes += int(space_dict.get(customer_idurl.to_bin(), 0)) current_customers.remove(customer_idurl) space_dict.pop(customer_idurl.to_bin()) new_customer = False else: new_customer = True lg.args(8, new_customer=new_customer, current_allocated_bytes=space_dict.get( customer_idurl.to_bin())) from supplier import local_tester if free_bytes <= bytes_for_customer: contactsdb.remove_customer_meta_info(customer_idurl) accounting.write_customers_quotas(space_dict, free_bytes) contactsdb.update_customers(current_customers) contactsdb.save_customers() if customer_public_key_id: my_keys.erase_key(customer_public_key_id) reactor.callLater( 0, local_tester.TestUpdateCustomers) # @UndefinedVariable if new_customer: lg.info("NEW CUSTOMER: DENIED not enough space available") events.send('new-customer-denied', data=dict(idurl=customer_idurl)) else: lg.info("OLD CUSTOMER: DENIED not enough space available") events.send('existing-customer-denied', data=dict(idurl=customer_idurl)) return p2p_service.SendAck(newpacket, 'deny') free_bytes = free_bytes - bytes_for_customer current_customers.append(customer_idurl) space_dict[customer_idurl.to_bin()] = bytes_for_customer contactsdb.add_customer_meta_info( customer_idurl, { 'ecc_map': ecc_map, 'position': family_position, 'family_snapshot': family_snapshot, }) accounting.write_customers_quotas(space_dict, free_bytes) contactsdb.update_customers(current_customers) contactsdb.save_customers() customer_space.register_customer_key(customer_public_key_id, customer_public_key) reactor.callLater( 0, local_tester.TestUpdateCustomers) # @UndefinedVariable if new_customer: lg.info( "NEW CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s" % (customer_idurl, family_position, ecc_map, bytes_for_customer)) events.send('new-customer-accepted', data=dict( idurl=customer_idurl, allocated_bytes=bytes_for_customer, ecc_map=ecc_map, position=family_position, family_snapshot=family_snapshot, key_id=customer_public_key_id, )) else: lg.info( "OLD CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s" % (customer_idurl, family_position, ecc_map, bytes_for_customer)) events.send('existing-customer-accepted', data=dict( idurl=customer_idurl, allocated_bytes=bytes_for_customer, ecc_map=ecc_map, position=family_position, key_id=customer_public_key_id, family_snapshot=family_snapshot, )) return p2p_service.SendAck(newpacket, 'accepted')
def doHireNewBroker(self, event, *args, **kwargs): """ Action method. """ known_brokers = {} known_brokers.update(self.cooperated_brokers or {}) known_brokers[self.desired_position] = self.my_broker_idurl target_pos = self.desired_position - 1 exclude_brokers = list( id_url.to_bin_list(filter(None, self.dht_brokers.values()))) exclude_brokers.extend( list( id_url.to_bin_list( filter(None, self.requestor_known_brokers.values())))) preferred_brokers = [] preferred_brokers_raw = config.conf().getData( 'services/message-broker/preferred-brokers').strip() if preferred_brokers_raw: preferred_brokers_list = re.split('\n|,|;| ', preferred_brokers_raw) preferred_brokers.extend(preferred_brokers_list) preferred_brokers = id_url.to_bin_list(preferred_brokers) if preferred_brokers: preferred_brokers = [ x for x in preferred_brokers if x not in exclude_brokers ] if _Debug: lg.args(_DebugLevel, e=event, my=self.my_position, desired=self.desired_position, target=target_pos, exclude=exclude_brokers, preferred=preferred_brokers) if preferred_brokers: preferred_broker_idurl = id_url.field(preferred_brokers[0]) if preferred_broker_idurl and id_url.is_not_in( preferred_broker_idurl, exclude_brokers, as_field=False): result = p2p_service_seeker.connect_known_node( remote_idurl=preferred_broker_idurl, service_name='service_message_broker', service_params=lambda idurl: self ._do_prepare_service_request_params( idurl, target_pos, known_brokers, event), request_service_timeout=self.broker_negotiate_ack_timeout * (target_pos + 1), exclude_nodes=list(exclude_brokers), force_handshake=True, attempts=1, ) result.addCallback(self._on_new_broker_hired, target_pos, self.my_position, self.desired_position) if _Debug: result.addErrback( lg.errback, debug=_Debug, debug_level=_DebugLevel, method='broker_negotiator.doHirePrevBroker') result.addErrback(self._on_new_broker_lookup_failed, target_pos) return result = p2p_service_seeker.connect_random_node( lookup_method=lookup.random_message_broker, service_name='service_message_broker', service_params=lambda idurl: self._do_prepare_service_request_params( idurl, target_pos, known_brokers, event), request_service_timeout=self.broker_negotiate_ack_timeout * (target_pos + 1), exclude_nodes=list(exclude_brokers), attempts=1, force_handshake=True, ) result.addCallback(self._on_new_broker_hired, target_pos, self.my_position, self.desired_position) if _Debug: result.addErrback(lg.errback, debug=_Debug, debug_level=_DebugLevel, method='broker_negotiator.doHirePrevBroker') result.addErrback(self._on_new_broker_lookup_failed, target_pos)
def report_donated_storage(): space_dict, free_space = read_customers_quotas() used_space_dict = read_customers_usage() r = {} r['customers_num'] = contactsdb.num_customers() r['customers'] = [] r['old_customers'] = [] r['errors'] = [] r['consumed'] = 0 r['donated'] = settings.getDonatedBytes() # r['donated_str'] = diskspace.MakeStringFromBytes(r['donated']) r['real'] = bpio.getDirectorySize(settings.getCustomersFilesDir()) try: r['free'] = int(free_space) except: r['free'] = 0 used = 0 for idurl in id_url.to_bin_list(contactsdb.customers()): consumed_by_customer = 0 used_by_customer = 0 if idurl not in list(space_dict.keys()): r['errors'].append('space consumed by customer %r is unknown' % idurl) else: try: consumed_by_customer = int(space_dict.pop(idurl)) r['consumed'] += consumed_by_customer except: r['errors'].append( 'incorrect value of consumed space for customer %r' % idurl) continue if idurl in list(used_space_dict.keys()): try: used_by_customer = int(used_space_dict.pop(idurl)) used += used_by_customer except: r['errors'].append( 'incorrect value of used space for customer %r' % idurl) continue if consumed_by_customer < used_by_customer: r['errors'].append( 'customer %r currently using more space than requested' % idurl) c = {} c['idurl'] = strng.to_text(idurl) c['used'] = used_by_customer # c['used_str'] = diskspace.MakeStringFromBytes(c['used']) c['consumed'] = consumed_by_customer # c['consumed_str'] = diskspace.MakeStringFromBytes(c['consumed']) c['real'] = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl)) # c['real_str'] = diskspace.MakeStringFromBytes(c['real']) r['customers'].append(c) r['used'] = used # r['used_str'] = diskspace.MakeStringFromBytes(r['used']) # r['consumed_str'] = diskspace.MakeStringFromBytes(r['consumed']) if r['donated'] != r['free'] + r['consumed']: r['errors'].append( 'total consumed %d and known free %d (%d total) bytes not match with donated %d bytes' % (r['consumed'], r['free'], r['consumed'] + r['free'], r['donated'])) if r['used'] > r['donated']: r['errors'].append( 'total space used by customers exceed the donated limit') if len(space_dict) > 0: r['errors'].append('found %d incorrect records of consumed space' % len(space_dict)) if r['real'] != r['used']: r['errors'].append( 'current info needs update, known size is %d bytes but real is %d bytes' % (r['used'], r['real'])) old_customers_used = 0 old_customers_real = 0 for idurl in used_space_dict.keys(): real = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl)) try: used = int(used_space_dict[idurl]) except: r['errors'].append( 'incorrect value of used space for customer %r' % idurl) continue r['old_customers'].append({ 'idurl': strng.to_text(idurl), 'used': used, # 'used_str': diskspace.MakeStringFromBytes(used_space_dict[idurl]), 'real': real, # 'real_str': diskspace.MakeStringFromBytes(real), }) old_customers_used += used old_customers_real += real r['old_customers_used'] = old_customers_used r['old_customers_real'] = old_customers_real try: r['used_percent'] = misc.value2percent(float(r['used']), float(r['donated']), 5) except: r['used_percent'] = '' try: r['consumed_percent'] = misc.value2percent(float(r['consumed']), float(r['donated']), 5) except: r['consumed_percent'] = '' return r