def __init__(self, supplier_idurl, customer_idurl, needed_bytes, key_id=None, queue_subscribe=True): """ """ self.supplier_idurl = supplier_idurl self.customer_idurl = customer_idurl self.needed_bytes = needed_bytes self.key_id = key_id self.queue_subscribe = queue_subscribe if self.needed_bytes is None: total_bytes_needed = diskspace.GetBytesFromString( settings.getNeededString(), 0) num_suppliers = -1 if self.customer_idurl == my_id.getLocalIDURL(): num_suppliers = settings.getSuppliersNumberDesired() else: known_ecc_map = contactsdb.get_customer_meta_info( customer_idurl).get('ecc_map') if known_ecc_map: num_suppliers = eccmap.GetEccMapSuppliersNumber( known_ecc_map) if num_suppliers > 0: self.needed_bytes = int( math.ceil(2.0 * total_bytes_needed / float(num_suppliers))) else: raise Exception( 'not possible to determine needed_bytes value to be requested from that supplier' ) # self.needed_bytes = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers()))) name = 'supplier_%s_%s' % ( nameurl.GetName(self.supplier_idurl), diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''), ) self.request_packet_id = None self.callbacks = {} try: st = bpio.ReadTextFile( settings.SupplierServiceFilename( idurl=self.supplier_idurl, customer_idurl=self.customer_idurl, )).strip() except: st = 'DISCONNECTED' automat.Automat.__init__( self, name, state=st, debug_level=_DebugLevel, log_events=_Debug, log_transitions=_Debug, ) for cb in self.callbacks.values(): cb(self.supplier_idurl, self.state, self.state)
def _on_my_dht_relations_discovered(self, dht_result): if not (dht_result and isinstance(dht_result, dict) and len(dht_result.get('suppliers', [])) > 0): lg.warn('no dht records found for my customer family') self.automat('suppliers-read-failed') return dht_suppliers = id_url.to_bin_list(dht_result['suppliers']) dht_ecc_map = dht_result.get('ecc_map', settings.DefaultEccMapName()) try: dht_desired_suppliers_number = eccmap.GetEccMapSuppliersNumber( dht_ecc_map) except: lg.exc() dht_desired_suppliers_number = eccmap.GetEccMapSuppliersNumber( settings.DefaultEccMapName()) settings.config.conf().setInt('services/customer/suppliers-number', dht_desired_suppliers_number) contactsdb.set_suppliers(dht_suppliers) contactsdb.save_suppliers() lg.info('found and restored list of %d suppliers from DHT' % dht_desired_suppliers_number) self.automat('suppliers-read-ok')
def do_calculate_needed_bytes(self): if self.needed_bytes is None: total_bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0) num_suppliers = -1 if self.customer_idurl == my_id.getLocalIDURL(): num_suppliers = settings.getSuppliersNumberDesired() else: known_ecc_map = contactsdb.get_customer_meta_info(self.customer_idurl).get('ecc_map') if known_ecc_map: num_suppliers = eccmap.GetEccMapSuppliersNumber(known_ecc_map) if num_suppliers > 0: self.needed_bytes = int(math.ceil(2.0 * total_bytes_needed / float(num_suppliers))) else: raise Exception('not possible to determine needed_bytes value to be requested from that supplier')
def _do_merge_revisions(self, dht_info, my_info, latest_revision): if dht_info is None or not isinstance(dht_info, dict): merged_info = my_info else: if latest_revision == int(dht_info['revision']): if my_info is not None: if latest_revision == int(my_info['revision']): # I have same revision as info from DHT merged_info = dht_info else: if int(my_info['revision']) > int( dht_info['revision']): # here my revision is higher, so I have some changes that needs to be published already merged_info = my_info else: # here my revision is lower, I need to take info from DHT merged_info = dht_info else: merged_info = dht_info else: # here my revision is higher, so I have some changes that needs to be published already merged_info = my_info if not merged_info: return None # make sure list of suppliers have correct length according to ecc_map if not merged_info['ecc_map']: known_ecc_map = contactsdb.get_customer_meta_info( self.customer_idurl).get('ecc_map', None) lg.warn('unknown ecc_map, will populate known value: %s' % known_ecc_map) merged_info['ecc_map'] = known_ecc_map if merged_info['ecc_map']: expected_suppliers_count = eccmap.GetEccMapSuppliersNumber( merged_info['ecc_map']) if len(merged_info['suppliers']) < expected_suppliers_count: merged_info['suppliers'] += [ b'', ] * (expected_suppliers_count - len(merged_info['suppliers'])) elif len(merged_info['suppliers']) > expected_suppliers_count: merged_info['suppliers'] = merged_info[ 'suppliers'][:expected_suppliers_count] if merged_info['revision'] != latest_revision: lg.info('will switch known revision %d to the latest: %d' % ( merged_info['revision'], latest_revision, )) merged_info['revision'] = latest_revision return merged_info
def doCheckAllConnected(self, *args, **kwargs): """ Action method. """ connected_count = 0 for supplier_idurl in self.known_suppliers_list: if not id_url.is_cached(supplier_idurl): continue sc = supplier_connector.by_idurl( supplier_idurl, customer_idurl=self.customer_idurl) if sc is None or sc.state != 'CONNECTED': continue connected_count += 1 critical_suppliers_number = 1 if self.known_ecc_map: from raid import eccmap critical_suppliers_number = eccmap.GetCorrectableErrors( eccmap.GetEccMapSuppliersNumber(self.known_ecc_map)) if connected_count >= critical_suppliers_number: self.automat('all-suppliers-connected')
def _do_check_supplier_connectors(self): connected_count = 0 for supplier_idurl in self.known_suppliers_list: if not id_url.is_cached(supplier_idurl): continue sc = supplier_connector.by_idurl( supplier_idurl, customer_idurl=self.customer_idurl) if sc is None or sc.state != 'CONNECTED': continue connected_count += 1 critical_suppliers_number = 1 if self.known_ecc_map: from raid import eccmap critical_suppliers_number = eccmap.GetCorrectableErrors( eccmap.GetEccMapSuppliersNumber(self.known_ecc_map)) if _Debug: lg.args(_DebugLevel, connected_count=connected_count, critical_suppliers_number=critical_suppliers_number) if connected_count >= critical_suppliers_number: self.automat('all-suppliers-connected')
def doRebuildFamily(self, *args, **kwargs): """ Action method. """ dht_info_valid = self._do_validate_dht_info(args[0]) my_info_valid = self._do_validate_my_info(self.my_info) latest_revision = self._do_detect_latest_revision( dht_info_valid, my_info_valid) merged_info = None if latest_revision > 0: merged_info = self._do_merge_revisions(dht_info_valid, my_info_valid, latest_revision) if not merged_info: merged_info = self._do_create_first_revision(self.current_request) # if not merged_info: # lg.err('failed to merge customer family info after reading from DHT, skip transaction') # self.transaction = None # return possible_transaction = self._do_process_request( merged_info, self.current_request) if not possible_transaction: lg.err( 'failed to process customer family change request, skip transaction' ) return self.transaction = self._do_increment_revision(possible_transaction) if _Debug: lg.out( _DebugLevel, 'family_member._do_build_transaction : %r' % self.transaction) self.refresh_period = DHT_RECORD_REFRESH_INTERVAL * settings.DefaultDesiredSuppliers( ) if self.transaction: known_ecc_map = self.transaction.get('ecc_map') if known_ecc_map: expected_suppliers_count = eccmap.GetEccMapSuppliersNumber( known_ecc_map) self.refresh_period = DHT_RECORD_REFRESH_INTERVAL * expected_suppliers_count
def _do_process_family_refresh_request(self, merged_info): if not self.my_info: self.my_info = self._do_create_possible_revision( int(merged_info['revision'])) lg.warn( '"family-refresh" request will use "possible" customer meta info: %r' % self.my_info) if int(self.my_info['revision']) > int(merged_info['revision']): lg.info( '"family-refresh" request will overwrite DHT record with my info because my revision is higher than record in DHT' ) return self.my_info.copy() try: my_position = self.my_info['suppliers'].index( my_id.getLocalIDURL()) except: my_position = -1 if my_position < 0: lg.warn( '"family-refresh" request failed because my info not exist or not valid, my own position in the family is unknown' ) return None my_expected_suppliers_count = None if self.my_info['ecc_map']: my_expected_suppliers_count = eccmap.GetEccMapSuppliersNumber( self.my_info['ecc_map']) if my_expected_suppliers_count and my_position >= my_expected_suppliers_count: lg.warn( '"family-refresh" request failed because my info is not valid, supplier position greater than expected suppliers count' ) return None if len(merged_info['suppliers']) != my_expected_suppliers_count: lg.warn( 'number of suppliers not expected during processing of "family-refresh" request' ) if len(merged_info['suppliers']) < my_expected_suppliers_count: merged_info['suppliers'] += [ b'', ] * (my_expected_suppliers_count - len(merged_info['suppliers'])) else: merged_info['suppliers'] = merged_info[ 'suppliers'][:my_expected_suppliers_count] try: existing_position = merged_info['suppliers'].index( my_id.getLocalIDURL()) except ValueError: existing_position = -1 if existing_position < 0: if merged_info['suppliers'][my_position] not in [b'', '', None]: # TODO: SECURITY need to implement a signature verification and # also build solution to validate that change was approved by customer lg.warn( 'overwriting another supplier %s with my IDURL at position %d in family of customer %s' % ( merged_info['suppliers'][my_position], my_position, self.customer_idurl, )) merged_info['suppliers'][my_position] = my_id.getLocalIDURL() if _Debug: lg.out( _DebugLevel, ' placed supplier %s at known position %d in the family of customer %s' % (my_id.getLocalIDURL(), my_position, self.customer_idurl)) existing_position = my_position if existing_position != my_position: merged_info['suppliers'][existing_position] = b'' merged_info['suppliers'][my_position] = my_id.getLocalIDURL() if _Debug: lg.out( _DebugLevel, ' found my IDURL on %d position and will move it on %d position in the family of customer %s' % (existing_position, my_position, self.customer_idurl)) return merged_info
def _do_process_family_join_request(self, merged_info, current_request): current_request_expected_suppliers_count = None if current_request['ecc_map']: current_request_expected_suppliers_count = eccmap.GetEccMapSuppliersNumber( current_request['ecc_map']) if current_request_expected_suppliers_count and current_request[ 'position'] > current_request_expected_suppliers_count: lg.warn( '"family-join" request is not valid, supplier position %d greater than expected suppliers count %d for %s' % (current_request['position'], current_request_expected_suppliers_count, current_request['ecc_map'])) return None if merged_info['ecc_map'] and current_request[ 'ecc_map'] and current_request['ecc_map'] != merged_info[ 'ecc_map']: lg.info( 'from "family-join" request, detected ecc_map change %s -> %s for customer %s' % (merged_info['ecc_map'], current_request['ecc_map'], self.customer_idurl)) merged_info['ecc_map'] = current_request['ecc_map'] if not merged_info['ecc_map'] and current_request['ecc_map']: lg.info( 'from "family-join" request, detected ecc_map was set to %s for the first time for customer %s' % (current_request['ecc_map'], self.customer_idurl)) merged_info['ecc_map'] = current_request['ecc_map'] if not merged_info['ecc_map']: lg.warn( 'still did not found actual ecc_map from DHT or from the request' ) return None expected_suppliers_count = eccmap.GetEccMapSuppliersNumber( merged_info['ecc_map']) if not merged_info['suppliers']: merged_info['suppliers'] = [ b'', ] * expected_suppliers_count if len(merged_info['suppliers']) < expected_suppliers_count: merged_info['suppliers'] += [ b'', ] * (expected_suppliers_count - len(merged_info['suppliers'])) else: merged_info['suppliers'] = merged_info[ 'suppliers'][:expected_suppliers_count] try: existing_position = merged_info['suppliers'].index( current_request['supplier_idurl']) except: existing_position = -1 if current_request[ 'position'] is not None and current_request['position'] >= 0: if current_request['position'] >= expected_suppliers_count: lg.warn( '"family-join" request is not valid, supplier position greater than expected suppliers count' ) return None if existing_position >= 0 and existing_position != current_request[ 'position']: merged_info['suppliers'][existing_position] = b'' merged_info['suppliers'][current_request[ 'position']] = current_request['supplier_idurl'] if _Debug: lg.out( _DebugLevel, ' found my IDURL on %d position and will move it on %d position in the family of customer %s' % (existing_position, current_request['position'], self.customer_idurl)) if merged_info['suppliers'][current_request[ 'position']] != current_request['supplier_idurl']: if merged_info['suppliers'][ current_request['position']] not in [b'', '', None]: # TODO: SECURITY need to implement a signature verification and # also build solution to validate that change was approved by customer lg.warn( 'overwriting another supplier %s with my IDURL at position %d in family of customer %s' % ( merged_info['suppliers'][ current_request['position']], current_request['position'], self.customer_idurl, )) merged_info['suppliers'][current_request[ 'position']] = current_request['supplier_idurl'] if _Debug: lg.out( _DebugLevel, ' placed supplier %s at known position %d in the family of customer %s' % (current_request['supplier_idurl'], current_request['position'], self.customer_idurl)) if current_request['supplier_idurl'] not in merged_info['suppliers']: if b'' in merged_info['suppliers']: first_empty_position = merged_info['suppliers'].index(b'') merged_info['suppliers'][ first_empty_position] = current_request['supplier_idurl'] if _Debug: lg.out( _DebugLevel, ' placed supplier %s at first empty position %d in family of customer %s' % (current_request['supplier_idurl'], first_empty_position, self.customer_idurl)) else: merged_info['suppliers'].append( current_request['supplier_idurl']) if _Debug: lg.out( _DebugLevel, ' added supplier %s to family of customer %s' % (current_request['supplier_idurl'], self.customer_idurl)) return merged_info