def isMoreNeeded(self, *args, **kwargs): """ Condition method. """ # if _Debug: # lg.out(_DebugLevel, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % ( # contactsdb.num_suppliers(), len(self.dismiss_list), # settings.getSuppliersNumberDesired())) if id_url.is_some_empty(contactsdb.suppliers()): if _Debug: lg.out(_DebugLevel, 'fire_hire.isMoreNeeded found empty supplier!!!') return True if isinstance(args[0], list): dismissed = args[0] else: dismissed = self.dismiss_list s = set(id_url.to_bin_list(contactsdb.suppliers())) s.difference_update(set(id_url.to_bin_list(dismissed))) result = len(s) < settings.getSuppliersNumberDesired() if _Debug: lg.out( _DebugLevel, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' % (contactsdb.num_suppliers(), len(dismissed), len(s), settings.getSuppliersNumberDesired(), result)) return result
def doDecideToDismiss(self, arg): """ Action method. """ global _SuppliersToFire result = set(_SuppliersToFire) _SuppliersToFire = [] # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. if '' in contactsdb.suppliers(): lg.out(10, 'fire_hire.doDecideToDismiss found empty supplier, SKIP') self.automat('made-decision', []) return for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if not sc: continue if sc.state == 'NO_SERVICE': result.add(supplier_idurl) if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired(): for supplier_index in range(settings.getSuppliersNumberDesired(), contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: result.add(idurl) result = list(result) lg.out(10, 'fire_hire.doDecideToDismiss %s' % result) self.automat('made-decision', result)
def isMoreNeeded(self, arg): """ Condition method. """ # lg.out(10, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % ( # contactsdb.num_suppliers(), len(self.dismiss_list), # settings.getSuppliersNumberDesired())) if '' in contactsdb.suppliers(): if _Debug: lg.out(_DebugLevel, 'fire_hire.isMoreNeeded found empty suppliers!!!') return True if isinstance(arg, list): dismissed = arg else: dismissed = self.dismiss_list s = set(contactsdb.suppliers()) s.difference_update(set(dismissed)) result = len(s) < settings.getSuppliersNumberDesired() if _Debug: lg.out( _DebugLevel, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' % (contactsdb.num_suppliers(), len(dismissed), len(s), settings.getSuppliersNumberDesired(), result)) return result
def isStillNeeded(self, *args, **kwargs): """ Condition method. """ supplier_idurl = args[0] current_suppliers = contactsdb.suppliers() if supplier_idurl in current_suppliers: # this guy is already a supplier, we still need more then return True desired_number = settings.getSuppliersNumberDesired() needed_suppliers = current_suppliers[:desired_number] empty_suppliers = needed_suppliers.count(id_url.field(b'')) # if '' in needed_suppliers: # lg.warn('found empty suppliers!!!') # return True s = set(id_url.to_bin_list(needed_suppliers)) s.add(id_url.to_bin(supplier_idurl)) s.difference_update(set(id_url.to_bin_list(self.dismiss_list))) result = len(s) - empty_suppliers < settings.getSuppliersNumberDesired( ) # if _Debug: # lg.out(_DebugLevel, 'fire_hire.isStillNeeded %d %d %d %d %d, result=%s' % ( # contactsdb.num_suppliers(), len(needed_suppliers), len(self.dismiss_list), # len(s), settings.getSuppliersNumberDesired(), result)) return result
def doDecideToDismiss(self, arg): """ Action method. """ global _SuppliersToFire result = set(_SuppliersToFire) _SuppliersToFire = [] # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. if '' in contactsdb.suppliers(): lg.out(10, 'fire_hire.doDecideToDismiss found empty supplier, SKIP') self.automat('made-decision', []) return for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if not sc: continue if sc.state == 'NO_SERVICE': result.add(supplier_idurl) if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired(): for supplier_index in range( settings.getSuppliersNumberDesired(), contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: result.add(idurl) result = list(result) lg.out(10, 'fire_hire.doDecideToDismiss %s' % result) self.automat('made-decision', result)
def doFindNewSupplier(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out( _DebugLevel, 'fire_hire.doFindNewSupplier desired_suppliers=%d current_suppliers=%r' % (settings.getSuppliersNumberDesired(), contactsdb.suppliers())) from p2p import network_connector if network_connector.A().state != 'CONNECTED': if _Debug: lg.out( _DebugLevel, ' network_connector is not CONNECTED at the moment, SKIP' ) self.automat('search-failed') return position_for_new_supplier = None for pos in range(settings.getSuppliersNumberDesired()): if pos in self.hire_list: continue supplier_idurl = contactsdb.supplier(pos) if not supplier_idurl: lg.info( 'found empty supplier at position %d and going to find new supplier on that position' % pos) position_for_new_supplier = pos break if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False): lg.info( 'going to find new supplier on existing position %d to replace supplier %s' % ( pos, supplier_idurl, )) position_for_new_supplier = pos break if position_for_new_supplier is None: lg.err('did not found position for new supplier') self.automat('search-failed') return from customer import supplier_finder for idurl_txt in strng.to_text(config.conf().getData( 'services/employer/candidates')).split(','): if idurl_txt.strip(): supplier_finder.AddSupplierToHire(idurl_txt) self.hire_list.append(position_for_new_supplier) supplier_finder.A( 'start', family_position=position_for_new_supplier, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list(contactsdb.suppliers()), )
def doRemoveSuppliers(self, arg): """ Action method. """ current_suppliers = contactsdb.suppliers() desired_suppliers = settings.getSuppliersNumberDesired() if len(current_suppliers) < desired_suppliers: lg.warn('must have more suppliers %d<%d' % ( len(current_suppliers), desired_suppliers)) for supplier_idurl in self.dismiss_list: if supplier_idurl not in current_suppliers: lg.warn('%s not a supplier' % supplier_idurl) continue pos = current_suppliers.index(supplier_idurl) # current_suppliers.remove(supplier_idurl) current_suppliers[pos] = '' misc.writeSupplierData( supplier_idurl, 'disconnected', time.strftime('%d-%m-%Y %H:%M:%S')) current_suppliers = current_suppliers[:desired_suppliers] contactsdb.update_suppliers(current_suppliers) contactsdb.save_suppliers() if settings.NewWebGUI(): from web import control control.on_suppliers_changed(current_suppliers) else: from web import webcontrol webcontrol.OnListSuppliers() lg.out(2, '!!!!!!!!!!! REMOVE SUPPLIERS : %d' % len(self.dismiss_list))
def doFindNewSupplier(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'fire_hire.doFindNewSupplier') position_for_new_supplier = None for pos in range(settings.getSuppliersNumberDesired()): if pos in self.hire_list: continue supplier_idurl = contactsdb.supplier(pos) if not supplier_idurl: lg.info('found empty supplier at position %d and going to find new supplier on that position' % pos) position_for_new_supplier = pos break if supplier_idurl in self.dismiss_list: lg.info('going to find new supplier on existing position %d to replace supplier %s' % ( pos, supplier_idurl, )) position_for_new_supplier = pos break if position_for_new_supplier is None: lg.err('did not found position for new supplier') self.automat('search-failed') return self.hire_list.append(position_for_new_supplier) supplier_finder.A( 'start', family_position=position_for_new_supplier, ecc_map=eccmap.Current().name, family_snapshot=contactsdb.suppliers(), )
def doRemoveSuppliers(self, *args, **kwargs): """ Action method. """ current_suppliers = contactsdb.suppliers() desired_suppliers = settings.getSuppliersNumberDesired() if len(current_suppliers) < desired_suppliers: lg.warn('must have more suppliers %d<%d' % ( len(current_suppliers), desired_suppliers)) removed_suppliers = [] for supplier_idurl in self.dismiss_list: if supplier_idurl not in current_suppliers: lg.warn('%s not a supplier' % supplier_idurl) continue pos = current_suppliers.index(supplier_idurl) # current_suppliers.remove(supplier_idurl) current_suppliers[pos] = '' removed_suppliers.append((pos, supplier_idurl,)) misc.writeSupplierData( supplier_idurl, 'disconnected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getLocalID(), ) current_suppliers = current_suppliers[:desired_suppliers] contactsdb.update_suppliers(current_suppliers) contactsdb.save_suppliers() from main import control control.on_suppliers_changed(current_suppliers) for position, supplier_idurl in removed_suppliers: events.send('supplier-modified', dict( new_idurl=None, old_idurl=supplier_idurl, position=position, )) lg.out(2, '!!!!!!!!!!! REMOVE SUPPLIERS : %d' % len(self.dismiss_list))
def doSaveConfig(self, arg): """ Action method. """ self.configs = (settings.getSuppliersNumberDesired(), diskspace.GetBytesFromString( settings.getNeededString()))
def _stats(params): from contacts import contactsdb from p2p import contact_status from lib import diskspace result = {} result['suppliers'] = contactsdb.num_suppliers() result['max_suppliers'] = settings.getSuppliersNumberDesired() result['online_suppliers'] = contact_status.countOnlineAmong( contactsdb.suppliers()) result['customers'] = contactsdb.num_customers() result['bytes_donated'] = settings.getDonatedBytes() result['value_donated'] = diskspace.MakeStringFromBytes( settings.getDonatedBytes()) result['bytes_needed'] = settings.getNeededBytes() result['value_needed'] = diskspace.MakeStringFromBytes( settings.getNeededBytes()) result['bytes_used_total'] = backup_fs.sizebackups() result['value_used_total'] = diskspace.MakeStringFromBytes( backup_fs.sizebackups()) result['bytes_used_supplier'] = 0 if ( contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() / contactsdb.num_suppliers())) result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders() result['files_count'] = backup_fs.numberfiles() result['folders_count'] = backup_fs.numberfolders() result['items_count'] = backup_fs.counter() result['timestamp'] = time.time() return { 'result': result, }
def doSaveConfig(self, arg): """ Action method. """ self.configs = ( settings.getSuppliersNumberDesired(), diskspace.GetBytesFromString( settings.getNeededString()))
def CurrentName(): """ Should return a ecc map name from current suppliers number - taken from user settings. """ from main import settings snum = settings.getSuppliersNumberDesired() if snum < 0: return DefaultName() return GetEccMapName(snum)
def isConfigChanged(self, *args, **kwargs): """ Condition method. """ curconfigs = (settings.getSuppliersNumberDesired(), diskspace.GetBytesFromString(settings.getNeededString())) if None in self.configs: return True return self.configs[0] != curconfigs[0] or self.configs[1] != curconfigs[1]
def __init__(self, supplier_idurl, customer_idurl, needed_bytes, key_id=None, queue_subscribe=True): """ """ self.supplier_idurl = supplier_idurl self.customer_idurl = customer_idurl self.needed_bytes = needed_bytes self.key_id = key_id self.queue_subscribe = queue_subscribe if self.needed_bytes is None: total_bytes_needed = diskspace.GetBytesFromString( settings.getNeededString(), 0) num_suppliers = -1 if self.customer_idurl == my_id.getLocalIDURL(): num_suppliers = settings.getSuppliersNumberDesired() else: known_ecc_map = contactsdb.get_customer_meta_info( customer_idurl).get('ecc_map') if known_ecc_map: num_suppliers = eccmap.GetEccMapSuppliersNumber( known_ecc_map) if num_suppliers > 0: self.needed_bytes = int( math.ceil(2.0 * total_bytes_needed / float(num_suppliers))) else: raise Exception( 'not possible to determine needed_bytes value to be requested from that supplier' ) # self.needed_bytes = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers()))) name = 'supplier_%s_%s' % ( nameurl.GetName(self.supplier_idurl), diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''), ) self.request_packet_id = None self.callbacks = {} try: st = bpio.ReadTextFile( settings.SupplierServiceFilename( idurl=self.supplier_idurl, customer_idurl=self.customer_idurl, )).strip() except: st = 'DISCONNECTED' automat.Automat.__init__( self, name, state=st, debug_level=_DebugLevel, log_events=_Debug, log_transitions=_Debug, ) for cb in self.callbacks.values(): cb(self.supplier_idurl, self.state, self.state)
def IsAllHired(): """ """ if settings.getSuppliersNumberDesired() < 0: # I must know how many suppliers I want lg.warn('my desired number of suppliers not set') return False if contactsdb.num_suppliers() != settings.getSuppliersNumberDesired(): # I must have exactly that amount of suppliers already if _Debug: lg.args(_DebugLevel, desiried_suppliers=settings.getSuppliersNumberDesired(), current_suppliers=contactsdb.num_suppliers(), ) return False if id_url.is_some_empty(contactsdb.suppliers()): # I must know all of my suppliers if _Debug: lg.args(_DebugLevel, my_suppliers=contactsdb.suppliers()) return False return True
def isConfigChanged(self, arg): """ Condition method. """ curconfigs = (settings.getSuppliersNumberDesired(), diskspace.GetBytesFromString(settings.getNeededString())) if None in self.configs: return True return self.configs[0] != curconfigs[ 0] or self.configs[1] != curconfigs[1]
def isMoreNeeded(self, arg): """ Condition method. """ # lg.out(10, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % ( # contactsdb.num_suppliers(), len(self.dismiss_list), # settings.getSuppliersNumberDesired())) if '' in contactsdb.suppliers(): lg.out(4, 'fire_hire.isMoreNeeded found empty suppliers!!!') return True if isinstance(arg, list): dismissed = arg else: dismissed = self.dismiss_list s = set(contactsdb.suppliers()) s.difference_update(set(dismissed)) result = len(s) < settings.getSuppliersNumberDesired() lg.out(14, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' % ( contactsdb.num_suppliers(), len(dismissed), len(s), settings.getSuppliersNumberDesired(), result)) return result
def doDecideToDismiss(self, arg): """ Action method. """ global _SuppliersToFire result = set(_SuppliersToFire) _SuppliersToFire = [] disconnected_suppliers = set() # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. if '' in contactsdb.suppliers(): lg.out(6, 'fire_hire.doDecideToDismiss found empty supplier, SKIP') self.automat('made-decision', []) return for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if not sc: continue if sc.state == 'NO_SERVICE': result.add(supplier_idurl) elif sc.state == 'DISCONNECTED': disconnected_suppliers.add(supplier_idurl) if contact_status.isOffline(supplier_idurl): disconnected_suppliers.add(supplier_idurl) if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired(): for supplier_index in range( settings.getSuppliersNumberDesired(), contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: result.add(idurl) if disconnected_suppliers: from raid import eccmap if len(disconnected_suppliers) >= eccmap.GetFireHireErrors(settings.getSuppliersNumberDesired()): result.update(disconnected_suppliers) result = list(result) lg.out(6, 'fire_hire.doDecideToDismiss %s' % result) self.automat('made-decision', result)
def do_calculate_needed_bytes(self): if self.needed_bytes is None: total_bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0) num_suppliers = -1 if self.customer_idurl == my_id.getLocalIDURL(): num_suppliers = settings.getSuppliersNumberDesired() else: known_ecc_map = contactsdb.get_customer_meta_info(self.customer_idurl).get('ecc_map') if known_ecc_map: num_suppliers = eccmap.GetEccMapSuppliersNumber(known_ecc_map) if num_suppliers > 0: self.needed_bytes = int(math.ceil(2.0 * total_bytes_needed / float(num_suppliers))) else: raise Exception('not possible to determine needed_bytes value to be requested from that supplier')
def isStillNeeded(self, arg): """ Condition method. """ supplier_idurl = arg current_suppliers = contactsdb.suppliers() if supplier_idurl in current_suppliers: # this guy is already a supplier, we still need more then return True desired_number = settings.getSuppliersNumberDesired() needed_suppliers = current_suppliers[:desired_number] empty_suppliers = needed_suppliers.count('') # if '' in needed_suppliers: # lg.warn('found empty suppliers!!!') # return True s = set(needed_suppliers) s.add(supplier_idurl) s.difference_update(set(self.dismiss_list)) result = len(s) - \ empty_suppliers < settings.getSuppliersNumberDesired() # lg.out(14, 'fire_hire.isStillNeeded %d %d %d %d %d, result=%s' % ( # contactsdb.num_suppliers(), len(needed_suppliers), len(self.dismiss_list), # len(s), settings.getSuppliersNumberDesired(), result)) return result
def doRemoveSuppliers(self, *args, **kwargs): """ Action method. """ current_suppliers = contactsdb.suppliers() desired_suppliers = settings.getSuppliersNumberDesired() if len(current_suppliers) < desired_suppliers: lg.warn('must have more suppliers %d<%d' % (len(current_suppliers), desired_suppliers)) removed_suppliers = [] for supplier_idurl in self.dismiss_list: if id_url.is_not_in(supplier_idurl, current_suppliers, as_field=False): lg.warn('%s not a supplier' % supplier_idurl) continue pos = current_suppliers.index(id_url.field(supplier_idurl)) current_suppliers[pos] = '' removed_suppliers.append(( pos, supplier_idurl, )) misc.writeSupplierData( supplier_idurl, 'disconnected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getLocalID(), ) current_suppliers = current_suppliers[:desired_suppliers] contactsdb.update_suppliers(current_suppliers) contactsdb.save_suppliers() from main import control control.on_suppliers_changed(current_suppliers) for position, supplier_idurl in removed_suppliers: events.send( 'supplier-modified', dict( new_idurl=None, old_idurl=supplier_idurl, position=position, )) lg.info( 'removed some suppliers : %d desired_suppliers=%d current_suppliers=%d' % (len(self.dismiss_list), desired_suppliers, len(contactsdb.suppliers()))) if _Debug: lg.out(_DebugLevel, ' my current suppliers: %r' % contactsdb.suppliers())
def __init__(self, supplier_idurl, customer_idurl, needed_bytes, key_id=None, queue_subscribe=True): """ """ self.supplier_idurl = supplier_idurl self.customer_idurl = customer_idurl self.needed_bytes = needed_bytes self.key_id = key_id self.queue_subscribe = queue_subscribe if self.needed_bytes is None: total_bytes_needed = diskspace.GetBytesFromString( settings.getNeededString(), 0) num_suppliers = settings.getSuppliersNumberDesired() if num_suppliers > 0: self.needed_bytes = int( math.ceil(2.0 * total_bytes_needed / float(num_suppliers))) else: self.needed_bytes = int( math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers()))) name = 'supplier_%s_%s' % ( nameurl.GetName(self.supplier_idurl), diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''), ) self.request_packet_id = None self.callbacks = {} try: st = bpio.ReadTextFile( settings.SupplierServiceFilename( idurl=self.supplier_idurl, customer_idurl=self.customer_idurl, )).strip() except: st = 'DISCONNECTED' automat.Automat.__init__( self, name, state=st, debug_level=_DebugLevel, log_events=_Debug, log_transitions=_Debug, ) for cb in self.callbacks.values(): cb(self.supplier_idurl, self.state, self.state)
def doRequestService(self, arg): """ Action method. """ bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0) num_suppliers = settings.getSuppliersNumberDesired() if num_suppliers > 0: bytes_per_supplier = int(math.ceil(2.0 * bytes_needed / float(num_suppliers))) else: bytes_per_supplier = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers()))) service_info = 'service_supplier %d' % bytes_per_supplier request = p2p_service.SendRequestService( self.idurl, service_info, callbacks={ commands.Ack(): self._supplier_acked, commands.Fail(): self._supplier_failed}) # commands.Ack(): lambda response, info: self.automat('ack', response), # commands.Fail(): lambda response, info: self.automat('fail', response)}) self.request_packet_id = request.PacketID
def __init__(self, supplier_idurl, customer_idurl, needed_bytes): """ """ self.supplier_idurl = supplier_idurl self.customer_idurl = customer_idurl self.needed_bytes = needed_bytes if self.needed_bytes is None: total_bytes_needed = diskspace.GetBytesFromString( settings.getNeededString(), 0) num_suppliers = settings.getSuppliersNumberDesired() if num_suppliers > 0: self.needed_bytes = int( math.ceil(2.0 * total_bytes_needed / float(num_suppliers))) else: self.needed_bytes = int( math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers()))) name = 'supplier_%s_%s' % ( nameurl.GetName(self.supplier_idurl), diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''), ) self.request_packet_id = None self.callbacks = {} try: st = bpio.ReadTextFile( settings.SupplierServiceFilename( idurl=self.supplier_idurl, customer_idurl=self.customer_idurl, )).strip() except: st = 'DISCONNECTED' if st == 'CONNECTED': automat.Automat.__init__(self, name, 'CONNECTED', _DebugLevel, _Debug) elif st == 'NO_SERVICE': automat.Automat.__init__(self, name, 'NO_SERVICE', _DebugLevel, _Debug) else: automat.Automat.__init__(self, name, 'DISCONNECTED', _DebugLevel, _Debug) for cb in self.callbacks.values(): cb(self.supplier_idurl, self.state, self.state)
def _stats(params): from contacts import contactsdb from p2p import contact_status from lib import diskspace result = {} result['suppliers'] = contactsdb.num_suppliers() result['max_suppliers'] = settings.getSuppliersNumberDesired() result['online_suppliers'] = contact_status.countOnlineAmong(contactsdb.suppliers()) result['customers'] = contactsdb.num_customers() result['bytes_donated'] = settings.getDonatedBytes() result['value_donated'] = diskspace.MakeStringFromBytes(settings.getDonatedBytes()) result['bytes_needed'] = settings.getNeededBytes() result['value_needed'] = diskspace.MakeStringFromBytes(settings.getNeededBytes()) result['bytes_used_total'] = backup_fs.sizebackups() result['value_used_total'] = diskspace.MakeStringFromBytes(backup_fs.sizebackups()) result['bytes_used_supplier'] = 0 if (contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() / contactsdb.num_suppliers())) result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders() result['files_count'] = backup_fs.numberfiles() result['folders_count'] = backup_fs.numberfolders() result['items_count'] = backup_fs.counter() result['timestamp'] = time.time() return {'result': result, }
def doRequestService(self, arg): """ Action method. """ bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0) num_suppliers = settings.getSuppliersNumberDesired() if num_suppliers > 0: bytes_per_supplier = int( math.ceil(2.0 * bytes_needed / float(num_suppliers))) else: bytes_per_supplier = int( math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers()))) service_info = 'service_supplier %d' % bytes_per_supplier request = p2p_service.SendRequestService(self.idurl, service_info, callbacks={ commands.Ack(): self._supplier_acked, commands.Fail(): self._supplier_failed, }) self.request_packet_id = request.PacketID
def doDecideToDismiss(self, *args, **kwargs): """ Action method. """ global _SuppliersToFire from p2p import p2p_connector from p2p import network_connector from customer import supplier_connector from p2p import online_status # take any actions only if I am connected to the network if not p2p_connector.A() or not network_connector.A(): if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss p2p_connector() is not ready yet, SKIP' ) self.automat('made-decision', []) return if not network_connector.A(): if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss network_connector() is not ready yet, SKIP' ) self.automat('made-decision', []) return if p2p_connector.A().state != 'CONNECTED' or network_connector.A( ).state != 'CONNECTED': if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss p2p/network is not connected at the moment, SKIP' ) self.automat('made-decision', []) return # if certain suppliers needs to be removed by manual/external request just do that to_be_fired = id_url.to_list(set(_SuppliersToFire)) _SuppliersToFire = [] if to_be_fired: lg.info('going to fire %d suppliers from external request' % len(to_be_fired)) self.automat('made-decision', to_be_fired) return # make sure to not go too far when i just want to decrease number of my suppliers number_desired = settings.getSuppliersNumberDesired() redundant_suppliers = set() if contactsdb.num_suppliers() > number_desired: for supplier_index in range(number_desired, contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: lg.info('found REDUNDANT supplier %s at position %d' % ( idurl, supplier_index, )) redundant_suppliers.add(idurl) if redundant_suppliers: result = list(redundant_suppliers) lg.info('will replace redundant suppliers: %s' % result) self.automat('made-decision', result) return # now I need to look more careful at my suppliers potentialy_fired = set() connected_suppliers = set() disconnected_suppliers = set() requested_suppliers = set() online_suppliers = set() offline_suppliers = set() # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. my_suppliers = contactsdb.suppliers() if _Debug: lg.args(_DebugLevel, my_suppliers=my_suppliers) if id_url.is_some_empty(my_suppliers): lg.warn('SKIP, found empty supplier') self.automat('made-decision', []) return for supplier_idurl in my_suppliers: sc = supplier_connector.by_idurl(supplier_idurl) if not sc: lg.warn('SKIP, supplier connector for supplier %s not exist' % supplier_idurl) continue if sc.state == 'NO_SERVICE': lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl) disconnected_suppliers.add(supplier_idurl) potentialy_fired.add(supplier_idurl) elif sc.state == 'CONNECTED': connected_suppliers.add(supplier_idurl) elif sc.state in [ 'DISCONNECTED', 'REFUSE', ]: disconnected_suppliers.add(supplier_idurl) # elif sc.state in ['QUEUE?', 'REQUEST', ]: # requested_suppliers.add(supplier_idurl) if online_status.isOffline(supplier_idurl): offline_suppliers.add(supplier_idurl) elif online_status.isOnline(supplier_idurl): online_suppliers.add(supplier_idurl) elif online_status.isCheckingNow(supplier_idurl): requested_suppliers.add(supplier_idurl) if not connected_suppliers or not online_suppliers: lg.warn('SKIP, no ONLINE suppliers found at the moment') self.automat('made-decision', []) return if requested_suppliers: lg.warn('SKIP, still waiting response from some of suppliers') self.automat('made-decision', []) return if not disconnected_suppliers: if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss SKIP, no OFFLINE suppliers found at the moment' ) # TODO: add more conditions to fire "slow" suppliers - they are still connected but useless self.automat('made-decision', []) return if len(offline_suppliers) + len(online_suppliers) != number_desired: lg.warn('SKIP, offline + online != total count: %s %s %s' % (offline_suppliers, online_suppliers, number_desired)) self.automat('made-decision', []) return max_offline_suppliers_count = eccmap.GetCorrectableErrors( number_desired) if len(offline_suppliers) > max_offline_suppliers_count: lg.warn( 'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % ( len(offline_suppliers), max_offline_suppliers_count, )) self.automat('made-decision', []) return critical_offline_suppliers_count = eccmap.GetFireHireErrors( number_desired) if len(offline_suppliers) >= critical_offline_suppliers_count and len( offline_suppliers) > 0: if config.conf().getBool( 'services/employer/replace-critically-offline-enabled'): # TODO: check that issue # too aggressive replacing suppliers who still have the data is very dangerous !!! one_dead_supplier = offline_suppliers.pop() lg.warn( 'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d' % ( one_dead_supplier, critical_offline_suppliers_count, )) potentialy_fired.add(one_dead_supplier) if not potentialy_fired: if _Debug: lg.out( _DebugLevel, 'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!' ) self.automat('made-decision', []) return # only replace suppliers one by one at the moment result = list(potentialy_fired) lg.info('will replace supplier %s' % result[0]) self.automat('made-decision', [ result[0], ])
def doDecideToDismiss(self, arg): """ Action method. """ global _SuppliersToFire to_be_fired = list(set(_SuppliersToFire)) _SuppliersToFire = [] if to_be_fired: lg.warn('going to fire %d suppliers from external request' % len(to_be_fired)) self.automat('made-decision', to_be_fired) return potentialy_fired = set() connected_suppliers = set() disconnected_suppliers = set() requested_suppliers = set() online_suppliers = set() offline_suppliers = set() redundant_suppliers = set() # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. if '' in contactsdb.suppliers() or None in contactsdb.suppliers(): lg.warn('SKIP, found empty supplier') self.automat('made-decision', []) return number_desired = settings.getSuppliersNumberDesired() for supplier_idurl in contactsdb.suppliers(): sc = supplier_connector.by_idurl(supplier_idurl) if not sc: lg.warn('SKIP, supplier connector for supplier %s not exist' % supplier_idurl) continue if sc.state == 'NO_SERVICE': lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl) disconnected_suppliers.add(supplier_idurl) potentialy_fired.add(supplier_idurl) elif sc.state == 'CONNECTED': connected_suppliers.add(supplier_idurl) elif sc.state in [ 'DISCONNECTED', 'REFUSE', ]: disconnected_suppliers.add(supplier_idurl) # elif sc.state in ['QUEUE?', 'REQUEST', ]: # requested_suppliers.add(supplier_idurl) if contact_status.isOffline(supplier_idurl): offline_suppliers.add(supplier_idurl) elif contact_status.isOnline(supplier_idurl): online_suppliers.add(supplier_idurl) elif contact_status.isCheckingNow(supplier_idurl): requested_suppliers.add(supplier_idurl) if contactsdb.num_suppliers() > number_desired: for supplier_index in range(number_desired, contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: lg.warn('found "REDUNDANT" supplier %s at position %d' % ( idurl, supplier_index, )) potentialy_fired.add(idurl) redundant_suppliers.add(idurl) else: lg.warn('supplier at position %d not exist' % supplier_index) if not connected_suppliers or not online_suppliers: lg.warn('SKIP, no ONLINE suppliers found at the moment') self.automat('made-decision', []) return if requested_suppliers: lg.warn('SKIP, still waiting response from some of suppliers') self.automat('made-decision', []) return if redundant_suppliers: result = list(redundant_suppliers) lg.info('will replace redundant suppliers: %s' % result) self.automat('made-decision', result) return if not disconnected_suppliers: lg.warn('SKIP, no OFFLINE suppliers found at the moment') # TODO: add more conditions to fire "slow" suppliers self.automat('made-decision', []) return if len(offline_suppliers) + len(online_suppliers) != number_desired: lg.warn('SKIP, offline + online != total count: %s %s %s' % (offline_suppliers, online_suppliers, number_desired)) self.automat('made-decision', []) return from raid import eccmap max_offline_suppliers_count = eccmap.GetCorrectableErrors( number_desired) if len(offline_suppliers) > max_offline_suppliers_count: lg.warn( 'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % ( len(offline_suppliers), max_offline_suppliers_count, )) self.automat('made-decision', []) return critical_offline_suppliers_count = eccmap.GetFireHireErrors( number_desired) # TODO: temporary disabled because of an issue: too aggressive replacing suppliers who still have the data if False: # len(offline_suppliers) >= critical_offline_suppliers_count: one_dead_supplier = offline_suppliers.pop() lg.warn( 'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d' % ( one_dead_supplier, critical_offline_suppliers_count, )) potentialy_fired.add(one_dead_supplier) if not potentialy_fired: lg.out( 6, 'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!' ) self.automat('made-decision', []) return # only replace suppliers one by one at the moment result = list(potentialy_fired) lg.info('will replace supplier %s' % result[0]) self.automat('made-decision', [ result[0], ])
def doSubstituteSupplier(self, *args, **kwargs): """ Action method. """ new_idurl = id_url.field(args[0]) family_position = kwargs.get('family_position') current_suppliers = list(contactsdb.suppliers()) desired_suppliers = settings.getSuppliersNumberDesired() old_idurl = None if family_position in self.hire_list: self.hire_list.remove(family_position) lg.info( 'found position on which new supplier suppose to be hired: %d' % family_position) else: lg.warn('did not found position for new supplier to be hired on') if new_idurl in current_suppliers: raise Exception('%s is already supplier' % new_idurl) if family_position is None or family_position == -1: lg.warn( 'unknown family_position from supplier results, will pick first empty spot' ) position = -1 old_idurl = None for i in range(len(current_suppliers)): if not current_suppliers[i].strip(): position = i break if id_url.is_in(current_suppliers[i], self.dismiss_list, as_field=False): position = i old_idurl = current_suppliers[i] break family_position = position if _Debug: lg.out( _DebugLevel, 'fire_hire.doSubstituteSupplier family_position=%d' % family_position) contactsdb.add_supplier(idurl=new_idurl, position=family_position) contactsdb.save_suppliers() misc.writeSupplierData( new_idurl, 'connected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getIDURL(), ) from main import control control.on_suppliers_changed(current_suppliers) if family_position < 0: lg.info( 'added new supplier, family position unknown: %s desired_suppliers=%d current_suppliers=%d' % (new_idurl, desired_suppliers, len(contactsdb.suppliers()))) events.send('supplier-modified', data=dict( new_idurl=new_idurl, old_idurl=None, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list( contactsdb.suppliers()), )) else: if old_idurl: lg.info( 'hired new supplier and substitute existing supplier on position %d : %s->%s desired_suppliers=%d current_suppliers=%d' % (family_position, old_idurl, new_idurl, desired_suppliers, len(contactsdb.suppliers()))) events.send('supplier-modified', data=dict( new_idurl=new_idurl, old_idurl=old_idurl, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list( contactsdb.suppliers()), )) else: lg.info( 'hired new supplier on empty position %d : %s desired_suppliers=%d current_suppliers=%d' % (family_position, new_idurl, desired_suppliers, len(contactsdb.suppliers()))) events.send('supplier-modified', data=dict( new_idurl=new_idurl, old_idurl=None, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list( contactsdb.suppliers()), )) self.restart_interval = 1.0 if _Debug: lg.out(_DebugLevel, ' my current suppliers: %r' % contactsdb.suppliers())