Beispiel #1
0
 def isMoreNeeded(self, *args, **kwargs):
     """
     Condition method.
     """
     # if _Debug:
     #     lg.out(_DebugLevel, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % (
     #         contactsdb.num_suppliers(), len(self.dismiss_list),
     #         settings.getSuppliersNumberDesired()))
     if id_url.is_some_empty(contactsdb.suppliers()):
         if _Debug:
             lg.out(_DebugLevel,
                    'fire_hire.isMoreNeeded found empty supplier!!!')
         return True
     if isinstance(args[0], list):
         dismissed = args[0]
     else:
         dismissed = self.dismiss_list
     s = set(id_url.to_bin_list(contactsdb.suppliers()))
     s.difference_update(set(id_url.to_bin_list(dismissed)))
     result = len(s) < settings.getSuppliersNumberDesired()
     if _Debug:
         lg.out(
             _DebugLevel, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' %
             (contactsdb.num_suppliers(), len(dismissed), len(s),
              settings.getSuppliersNumberDesired(), result))
     return result
Beispiel #2
0
 def doDecideToDismiss(self, arg):
     """
     Action method.
     """
     global _SuppliersToFire
     result = set(_SuppliersToFire)
     _SuppliersToFire = []
     # if you have some empty suppliers need to get rid of them,
     # but no need to dismiss anyone at the moment.
     if '' in contactsdb.suppliers():
         lg.out(10,
                'fire_hire.doDecideToDismiss found empty supplier, SKIP')
         self.automat('made-decision', [])
         return
     for supplier_idurl in contactsdb.suppliers():
         if not supplier_idurl:
             continue
         sc = supplier_connector.by_idurl(supplier_idurl)
         if not sc:
             continue
         if sc.state == 'NO_SERVICE':
             result.add(supplier_idurl)
     if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired():
         for supplier_index in range(settings.getSuppliersNumberDesired(),
                                     contactsdb.num_suppliers()):
             idurl = contactsdb.supplier(supplier_index)
             if idurl:
                 result.add(idurl)
     result = list(result)
     lg.out(10, 'fire_hire.doDecideToDismiss %s' % result)
     self.automat('made-decision', result)
Beispiel #3
0
 def isMoreNeeded(self, arg):
     """
     Condition method.
     """
     # lg.out(10, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % (
     # contactsdb.num_suppliers(), len(self.dismiss_list),
     # settings.getSuppliersNumberDesired()))
     if '' in contactsdb.suppliers():
         if _Debug:
             lg.out(_DebugLevel,
                    'fire_hire.isMoreNeeded found empty suppliers!!!')
         return True
     if isinstance(arg, list):
         dismissed = arg
     else:
         dismissed = self.dismiss_list
     s = set(contactsdb.suppliers())
     s.difference_update(set(dismissed))
     result = len(s) < settings.getSuppliersNumberDesired()
     if _Debug:
         lg.out(
             _DebugLevel, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' %
             (contactsdb.num_suppliers(), len(dismissed), len(s),
              settings.getSuppliersNumberDesired(), result))
     return result
Beispiel #4
0
 def isStillNeeded(self, *args, **kwargs):
     """
     Condition method.
     """
     supplier_idurl = args[0]
     current_suppliers = contactsdb.suppliers()
     if supplier_idurl in current_suppliers:
         # this guy is already a supplier, we still need more then
         return True
     desired_number = settings.getSuppliersNumberDesired()
     needed_suppliers = current_suppliers[:desired_number]
     empty_suppliers = needed_suppliers.count(id_url.field(b''))
     # if '' in needed_suppliers:
     # lg.warn('found empty suppliers!!!')
     # return True
     s = set(id_url.to_bin_list(needed_suppliers))
     s.add(id_url.to_bin(supplier_idurl))
     s.difference_update(set(id_url.to_bin_list(self.dismiss_list)))
     result = len(s) - empty_suppliers < settings.getSuppliersNumberDesired(
     )
     # if _Debug:
     #     lg.out(_DebugLevel, 'fire_hire.isStillNeeded %d %d %d %d %d, result=%s' % (
     #     contactsdb.num_suppliers(), len(needed_suppliers), len(self.dismiss_list),
     #     len(s), settings.getSuppliersNumberDesired(), result))
     return result
Beispiel #5
0
 def doDecideToDismiss(self, arg):
     """
     Action method.
     """
     global _SuppliersToFire
     result = set(_SuppliersToFire)
     _SuppliersToFire = []
     # if you have some empty suppliers need to get rid of them,
     # but no need to dismiss anyone at the moment.
     if '' in contactsdb.suppliers():
         lg.out(10, 'fire_hire.doDecideToDismiss found empty supplier, SKIP')
         self.automat('made-decision', [])
         return
     for supplier_idurl in contactsdb.suppliers():
         if not supplier_idurl:
             continue
         sc = supplier_connector.by_idurl(supplier_idurl)
         if not sc:
             continue
         if sc.state == 'NO_SERVICE':
             result.add(supplier_idurl)
     if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired():
         for supplier_index in range(
                 settings.getSuppliersNumberDesired(),
                 contactsdb.num_suppliers()):
             idurl = contactsdb.supplier(supplier_index)
             if idurl:
                 result.add(idurl)
     result = list(result)
     lg.out(10, 'fire_hire.doDecideToDismiss %s' % result)
     self.automat('made-decision', result)
Beispiel #6
0
 def doFindNewSupplier(self, *args, **kwargs):
     """
     Action method.
     """
     if _Debug:
         lg.out(
             _DebugLevel,
             'fire_hire.doFindNewSupplier desired_suppliers=%d current_suppliers=%r'
             %
             (settings.getSuppliersNumberDesired(), contactsdb.suppliers()))
     from p2p import network_connector
     if network_connector.A().state != 'CONNECTED':
         if _Debug:
             lg.out(
                 _DebugLevel,
                 '        network_connector is not CONNECTED at the moment, SKIP'
             )
         self.automat('search-failed')
         return
     position_for_new_supplier = None
     for pos in range(settings.getSuppliersNumberDesired()):
         if pos in self.hire_list:
             continue
         supplier_idurl = contactsdb.supplier(pos)
         if not supplier_idurl:
             lg.info(
                 'found empty supplier at position %d and going to find new supplier on that position'
                 % pos)
             position_for_new_supplier = pos
             break
         if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False):
             lg.info(
                 'going to find new supplier on existing position %d to replace supplier %s'
                 % (
                     pos,
                     supplier_idurl,
                 ))
             position_for_new_supplier = pos
             break
     if position_for_new_supplier is None:
         lg.err('did not found position for new supplier')
         self.automat('search-failed')
         return
     from customer import supplier_finder
     for idurl_txt in strng.to_text(config.conf().getData(
             'services/employer/candidates')).split(','):
         if idurl_txt.strip():
             supplier_finder.AddSupplierToHire(idurl_txt)
     self.hire_list.append(position_for_new_supplier)
     supplier_finder.A(
         'start',
         family_position=position_for_new_supplier,
         ecc_map=eccmap.Current().name,
         family_snapshot=id_url.to_bin_list(contactsdb.suppliers()),
     )
Beispiel #7
0
 def doRemoveSuppliers(self, arg):
     """
     Action method.
     """
     current_suppliers = contactsdb.suppliers()
     desired_suppliers = settings.getSuppliersNumberDesired()
     if len(current_suppliers) < desired_suppliers:
         lg.warn('must have more suppliers %d<%d' % (
             len(current_suppliers), desired_suppliers))
     for supplier_idurl in self.dismiss_list:
         if supplier_idurl not in current_suppliers:
             lg.warn('%s not a supplier' % supplier_idurl)
             continue
         pos = current_suppliers.index(supplier_idurl)
         # current_suppliers.remove(supplier_idurl)
         current_suppliers[pos] = ''
         misc.writeSupplierData(
             supplier_idurl,
             'disconnected',
             time.strftime('%d-%m-%Y %H:%M:%S'))
     current_suppliers = current_suppliers[:desired_suppliers]
     contactsdb.update_suppliers(current_suppliers)
     contactsdb.save_suppliers()
     if settings.NewWebGUI():
         from web import control
         control.on_suppliers_changed(current_suppliers)
     else:
         from web import webcontrol
         webcontrol.OnListSuppliers()
     lg.out(2, '!!!!!!!!!!! REMOVE SUPPLIERS : %d' % len(self.dismiss_list))
Beispiel #8
0
 def doFindNewSupplier(self, *args, **kwargs):
     """
     Action method.
     """
     if _Debug:
         lg.out(_DebugLevel, 'fire_hire.doFindNewSupplier')
     position_for_new_supplier = None
     for pos in range(settings.getSuppliersNumberDesired()):
         if pos in self.hire_list:
             continue
         supplier_idurl = contactsdb.supplier(pos)
         if not supplier_idurl:
             lg.info('found empty supplier at position %d and going to find new supplier on that position' % pos)
             position_for_new_supplier = pos
             break
         if supplier_idurl in self.dismiss_list:
             lg.info('going to find new supplier on existing position %d to replace supplier %s' % (
                 pos, supplier_idurl, ))
             position_for_new_supplier = pos
             break
     if position_for_new_supplier is None:
         lg.err('did not found position for new supplier')
         self.automat('search-failed')
         return
     self.hire_list.append(position_for_new_supplier)
     supplier_finder.A(
         'start',
         family_position=position_for_new_supplier,
         ecc_map=eccmap.Current().name,
         family_snapshot=contactsdb.suppliers(),
     )
Beispiel #9
0
 def doRemoveSuppliers(self, *args, **kwargs):
     """
     Action method.
     """
     current_suppliers = contactsdb.suppliers()
     desired_suppliers = settings.getSuppliersNumberDesired()
     if len(current_suppliers) < desired_suppliers:
         lg.warn('must have more suppliers %d<%d' % (
             len(current_suppliers), desired_suppliers))
     removed_suppliers = []
     for supplier_idurl in self.dismiss_list:
         if supplier_idurl not in current_suppliers:
             lg.warn('%s not a supplier' % supplier_idurl)
             continue
         pos = current_suppliers.index(supplier_idurl)
         # current_suppliers.remove(supplier_idurl)
         current_suppliers[pos] = ''
         removed_suppliers.append((pos, supplier_idurl,))
         misc.writeSupplierData(
             supplier_idurl,
             'disconnected',
             time.strftime('%d-%m-%Y %H:%M:%S'),
             my_id.getLocalID(),
         )
     current_suppliers = current_suppliers[:desired_suppliers]
     contactsdb.update_suppliers(current_suppliers)
     contactsdb.save_suppliers()
     from main import control
     control.on_suppliers_changed(current_suppliers)
     for position, supplier_idurl in removed_suppliers:
         events.send('supplier-modified', dict(
             new_idurl=None, old_idurl=supplier_idurl, position=position,
         ))
     lg.out(2, '!!!!!!!!!!! REMOVE SUPPLIERS : %d' % len(self.dismiss_list))
Beispiel #10
0
 def doSaveConfig(self, arg):
     """
     Action method.
     """
     self.configs = (settings.getSuppliersNumberDesired(),
                     diskspace.GetBytesFromString(
                         settings.getNeededString()))
def _stats(params):
    from contacts import contactsdb
    from p2p import contact_status
    from lib import diskspace
    result = {}
    result['suppliers'] = contactsdb.num_suppliers()
    result['max_suppliers'] = settings.getSuppliersNumberDesired()
    result['online_suppliers'] = contact_status.countOnlineAmong(
        contactsdb.suppliers())
    result['customers'] = contactsdb.num_customers()
    result['bytes_donated'] = settings.getDonatedBytes()
    result['value_donated'] = diskspace.MakeStringFromBytes(
        settings.getDonatedBytes())
    result['bytes_needed'] = settings.getNeededBytes()
    result['value_needed'] = diskspace.MakeStringFromBytes(
        settings.getNeededBytes())
    result['bytes_used_total'] = backup_fs.sizebackups()
    result['value_used_total'] = diskspace.MakeStringFromBytes(
        backup_fs.sizebackups())
    result['bytes_used_supplier'] = 0 if (
        contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() /
                                                   contactsdb.num_suppliers()))
    result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders()
    result['files_count'] = backup_fs.numberfiles()
    result['folders_count'] = backup_fs.numberfolders()
    result['items_count'] = backup_fs.counter()
    result['timestamp'] = time.time()
    return {
        'result': result,
    }
Beispiel #12
0
 def doSaveConfig(self, arg):
     """
     Action method.
     """
     self.configs = (
         settings.getSuppliersNumberDesired(),
         diskspace.GetBytesFromString(
             settings.getNeededString()))
Beispiel #13
0
def CurrentName():
    """
    Should return a ecc map name from current suppliers number - taken from user settings.
    """
    from main import settings
    snum = settings.getSuppliersNumberDesired()
    if snum < 0:
        return DefaultName()
    return GetEccMapName(snum)
Beispiel #14
0
 def isConfigChanged(self, *args, **kwargs):
     """
     Condition method.
     """
     curconfigs = (settings.getSuppliersNumberDesired(),
                   diskspace.GetBytesFromString(settings.getNeededString()))
     if None in self.configs:
         return True
     return self.configs[0] != curconfigs[0] or self.configs[1] != curconfigs[1]
Beispiel #15
0
def CurrentName():
    """
    Should return a ecc map name from current suppliers number - taken from user settings.
    """
    from main import settings
    snum = settings.getSuppliersNumberDesired()
    if snum < 0:
        return DefaultName()
    return GetEccMapName(snum)
Beispiel #16
0
 def __init__(self,
              supplier_idurl,
              customer_idurl,
              needed_bytes,
              key_id=None,
              queue_subscribe=True):
     """
     """
     self.supplier_idurl = supplier_idurl
     self.customer_idurl = customer_idurl
     self.needed_bytes = needed_bytes
     self.key_id = key_id
     self.queue_subscribe = queue_subscribe
     if self.needed_bytes is None:
         total_bytes_needed = diskspace.GetBytesFromString(
             settings.getNeededString(), 0)
         num_suppliers = -1
         if self.customer_idurl == my_id.getLocalIDURL():
             num_suppliers = settings.getSuppliersNumberDesired()
         else:
             known_ecc_map = contactsdb.get_customer_meta_info(
                 customer_idurl).get('ecc_map')
             if known_ecc_map:
                 num_suppliers = eccmap.GetEccMapSuppliersNumber(
                     known_ecc_map)
         if num_suppliers > 0:
             self.needed_bytes = int(
                 math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
         else:
             raise Exception(
                 'not possible to determine needed_bytes value to be requested from that supplier'
             )
             # self.needed_bytes = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers())))
     name = 'supplier_%s_%s' % (
         nameurl.GetName(self.supplier_idurl),
         diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
     )
     self.request_packet_id = None
     self.callbacks = {}
     try:
         st = bpio.ReadTextFile(
             settings.SupplierServiceFilename(
                 idurl=self.supplier_idurl,
                 customer_idurl=self.customer_idurl,
             )).strip()
     except:
         st = 'DISCONNECTED'
     automat.Automat.__init__(
         self,
         name,
         state=st,
         debug_level=_DebugLevel,
         log_events=_Debug,
         log_transitions=_Debug,
     )
     for cb in self.callbacks.values():
         cb(self.supplier_idurl, self.state, self.state)
Beispiel #17
0
def IsAllHired():
    """
    """
    if settings.getSuppliersNumberDesired() < 0:
        # I must know how many suppliers I want
        lg.warn('my desired number of suppliers not set')
        return False
    if contactsdb.num_suppliers() != settings.getSuppliersNumberDesired():
        # I must have exactly that amount of suppliers already
        if _Debug:
            lg.args(_DebugLevel, desiried_suppliers=settings.getSuppliersNumberDesired(),
                current_suppliers=contactsdb.num_suppliers(), )
        return False
    if id_url.is_some_empty(contactsdb.suppliers()):
        # I must know all of my suppliers
        if _Debug:
            lg.args(_DebugLevel, my_suppliers=contactsdb.suppliers())
        return False
    return True 
Beispiel #18
0
 def isConfigChanged(self, arg):
     """
     Condition method.
     """
     curconfigs = (settings.getSuppliersNumberDesired(),
                   diskspace.GetBytesFromString(settings.getNeededString()))
     if None in self.configs:
         return True
     return self.configs[0] != curconfigs[
         0] or self.configs[1] != curconfigs[1]
Beispiel #19
0
 def isMoreNeeded(self, arg):
     """
     Condition method.
     """
     # lg.out(10, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % (
     # contactsdb.num_suppliers(), len(self.dismiss_list),
     # settings.getSuppliersNumberDesired()))
     if '' in contactsdb.suppliers():
         lg.out(4, 'fire_hire.isMoreNeeded found empty suppliers!!!')
         return True
     if isinstance(arg, list):
         dismissed = arg
     else:
         dismissed = self.dismiss_list
     s = set(contactsdb.suppliers())
     s.difference_update(set(dismissed))
     result = len(s) < settings.getSuppliersNumberDesired()
     lg.out(14, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' % (
         contactsdb.num_suppliers(), len(dismissed), len(s),
         settings.getSuppliersNumberDesired(), result))
     return result
Beispiel #20
0
 def doDecideToDismiss(self, arg):
     """
     Action method.
     """
     global _SuppliersToFire
     result = set(_SuppliersToFire)
     _SuppliersToFire = []
     disconnected_suppliers = set()
     # if you have some empty suppliers need to get rid of them,
     # but no need to dismiss anyone at the moment.
     if '' in contactsdb.suppliers():
         lg.out(6, 'fire_hire.doDecideToDismiss found empty supplier, SKIP')
         self.automat('made-decision', [])
         return
     for supplier_idurl in contactsdb.suppliers():
         if not supplier_idurl:
             continue
         sc = supplier_connector.by_idurl(supplier_idurl)
         if not sc:
             continue
         if sc.state == 'NO_SERVICE':
             result.add(supplier_idurl)
         elif sc.state == 'DISCONNECTED':
             disconnected_suppliers.add(supplier_idurl)
         if contact_status.isOffline(supplier_idurl):
             disconnected_suppliers.add(supplier_idurl)
     if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired():
         for supplier_index in range(
                 settings.getSuppliersNumberDesired(),
                 contactsdb.num_suppliers()):
             idurl = contactsdb.supplier(supplier_index)
             if idurl:
                 result.add(idurl)
     if disconnected_suppliers:
         from raid import eccmap
         if len(disconnected_suppliers) >= eccmap.GetFireHireErrors(settings.getSuppliersNumberDesired()):
             result.update(disconnected_suppliers)
     result = list(result)
     lg.out(6, 'fire_hire.doDecideToDismiss %s' % result)
     self.automat('made-decision', result)
Beispiel #21
0
 def do_calculate_needed_bytes(self):
     if self.needed_bytes is None:
         total_bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0)
         num_suppliers = -1
         if self.customer_idurl == my_id.getLocalIDURL():
             num_suppliers = settings.getSuppliersNumberDesired()
         else:
             known_ecc_map = contactsdb.get_customer_meta_info(self.customer_idurl).get('ecc_map')
             if known_ecc_map:
                 num_suppliers = eccmap.GetEccMapSuppliersNumber(known_ecc_map)
         if num_suppliers > 0:
             self.needed_bytes = int(math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
         else:
             raise Exception('not possible to determine needed_bytes value to be requested from that supplier')
Beispiel #22
0
 def isStillNeeded(self, arg):
     """
     Condition method.
     """
     supplier_idurl = arg
     current_suppliers = contactsdb.suppliers()
     if supplier_idurl in current_suppliers:
         # this guy is already a supplier, we still need more then
         return True
     desired_number = settings.getSuppliersNumberDesired()
     needed_suppliers = current_suppliers[:desired_number]
     empty_suppliers = needed_suppliers.count('')
     # if '' in needed_suppliers:
     # lg.warn('found empty suppliers!!!')
     # return True
     s = set(needed_suppliers)
     s.add(supplier_idurl)
     s.difference_update(set(self.dismiss_list))
     result = len(s) - \
         empty_suppliers < settings.getSuppliersNumberDesired()
     # lg.out(14, 'fire_hire.isStillNeeded %d %d %d %d %d, result=%s' % (
     #     contactsdb.num_suppliers(), len(needed_suppliers), len(self.dismiss_list),
     #     len(s), settings.getSuppliersNumberDesired(), result))
     return result
Beispiel #23
0
 def doRemoveSuppliers(self, *args, **kwargs):
     """
     Action method.
     """
     current_suppliers = contactsdb.suppliers()
     desired_suppliers = settings.getSuppliersNumberDesired()
     if len(current_suppliers) < desired_suppliers:
         lg.warn('must have more suppliers %d<%d' %
                 (len(current_suppliers), desired_suppliers))
     removed_suppliers = []
     for supplier_idurl in self.dismiss_list:
         if id_url.is_not_in(supplier_idurl,
                             current_suppliers,
                             as_field=False):
             lg.warn('%s not a supplier' % supplier_idurl)
             continue
         pos = current_suppliers.index(id_url.field(supplier_idurl))
         current_suppliers[pos] = ''
         removed_suppliers.append((
             pos,
             supplier_idurl,
         ))
         misc.writeSupplierData(
             supplier_idurl,
             'disconnected',
             time.strftime('%d-%m-%Y %H:%M:%S'),
             my_id.getLocalID(),
         )
     current_suppliers = current_suppliers[:desired_suppliers]
     contactsdb.update_suppliers(current_suppliers)
     contactsdb.save_suppliers()
     from main import control
     control.on_suppliers_changed(current_suppliers)
     for position, supplier_idurl in removed_suppliers:
         events.send(
             'supplier-modified',
             dict(
                 new_idurl=None,
                 old_idurl=supplier_idurl,
                 position=position,
             ))
     lg.info(
         'removed some suppliers : %d  desired_suppliers=%d current_suppliers=%d'
         % (len(self.dismiss_list), desired_suppliers,
            len(contactsdb.suppliers())))
     if _Debug:
         lg.out(_DebugLevel,
                '    my current suppliers: %r' % contactsdb.suppliers())
 def __init__(self,
              supplier_idurl,
              customer_idurl,
              needed_bytes,
              key_id=None,
              queue_subscribe=True):
     """
     """
     self.supplier_idurl = supplier_idurl
     self.customer_idurl = customer_idurl
     self.needed_bytes = needed_bytes
     self.key_id = key_id
     self.queue_subscribe = queue_subscribe
     if self.needed_bytes is None:
         total_bytes_needed = diskspace.GetBytesFromString(
             settings.getNeededString(), 0)
         num_suppliers = settings.getSuppliersNumberDesired()
         if num_suppliers > 0:
             self.needed_bytes = int(
                 math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
         else:
             self.needed_bytes = int(
                 math.ceil(2.0 * settings.MinimumNeededBytes() /
                           float(settings.DefaultDesiredSuppliers())))
     name = 'supplier_%s_%s' % (
         nameurl.GetName(self.supplier_idurl),
         diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
     )
     self.request_packet_id = None
     self.callbacks = {}
     try:
         st = bpio.ReadTextFile(
             settings.SupplierServiceFilename(
                 idurl=self.supplier_idurl,
                 customer_idurl=self.customer_idurl,
             )).strip()
     except:
         st = 'DISCONNECTED'
     automat.Automat.__init__(
         self,
         name,
         state=st,
         debug_level=_DebugLevel,
         log_events=_Debug,
         log_transitions=_Debug,
     )
     for cb in self.callbacks.values():
         cb(self.supplier_idurl, self.state, self.state)
 def doRequestService(self, arg):
     """
     Action method.
     """
     bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0)
     num_suppliers = settings.getSuppliersNumberDesired()
     if num_suppliers > 0:
         bytes_per_supplier = int(math.ceil(2.0 * bytes_needed / float(num_suppliers)))
     else:
         bytes_per_supplier = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers())))
     service_info = 'service_supplier %d' % bytes_per_supplier
     request = p2p_service.SendRequestService(
         self.idurl, service_info, callbacks={
             commands.Ack(): self._supplier_acked,
             commands.Fail(): self._supplier_failed})
     # commands.Ack(): lambda response, info: self.automat('ack', response),
     # commands.Fail(): lambda response, info: self.automat('fail', response)})
     self.request_packet_id = request.PacketID
 def __init__(self, supplier_idurl, customer_idurl, needed_bytes):
     """
     """
     self.supplier_idurl = supplier_idurl
     self.customer_idurl = customer_idurl
     self.needed_bytes = needed_bytes
     if self.needed_bytes is None:
         total_bytes_needed = diskspace.GetBytesFromString(
             settings.getNeededString(), 0)
         num_suppliers = settings.getSuppliersNumberDesired()
         if num_suppliers > 0:
             self.needed_bytes = int(
                 math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
         else:
             self.needed_bytes = int(
                 math.ceil(2.0 * settings.MinimumNeededBytes() /
                           float(settings.DefaultDesiredSuppliers())))
     name = 'supplier_%s_%s' % (
         nameurl.GetName(self.supplier_idurl),
         diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
     )
     self.request_packet_id = None
     self.callbacks = {}
     try:
         st = bpio.ReadTextFile(
             settings.SupplierServiceFilename(
                 idurl=self.supplier_idurl,
                 customer_idurl=self.customer_idurl,
             )).strip()
     except:
         st = 'DISCONNECTED'
     if st == 'CONNECTED':
         automat.Automat.__init__(self, name, 'CONNECTED', _DebugLevel,
                                  _Debug)
     elif st == 'NO_SERVICE':
         automat.Automat.__init__(self, name, 'NO_SERVICE', _DebugLevel,
                                  _Debug)
     else:
         automat.Automat.__init__(self, name, 'DISCONNECTED', _DebugLevel,
                                  _Debug)
     for cb in self.callbacks.values():
         cb(self.supplier_idurl, self.state, self.state)
def _stats(params):
    from contacts import contactsdb
    from p2p import contact_status
    from lib import diskspace
    result = {}
    result['suppliers'] = contactsdb.num_suppliers()
    result['max_suppliers'] = settings.getSuppliersNumberDesired()
    result['online_suppliers'] = contact_status.countOnlineAmong(contactsdb.suppliers())
    result['customers'] = contactsdb.num_customers()
    result['bytes_donated'] = settings.getDonatedBytes()
    result['value_donated'] = diskspace.MakeStringFromBytes(settings.getDonatedBytes())
    result['bytes_needed'] = settings.getNeededBytes()
    result['value_needed'] = diskspace.MakeStringFromBytes(settings.getNeededBytes())
    result['bytes_used_total'] = backup_fs.sizebackups()
    result['value_used_total'] = diskspace.MakeStringFromBytes(backup_fs.sizebackups())
    result['bytes_used_supplier'] = 0 if (contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() / contactsdb.num_suppliers()))
    result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders()
    result['files_count'] = backup_fs.numberfiles()
    result['folders_count'] = backup_fs.numberfolders()
    result['items_count'] = backup_fs.counter()
    result['timestamp'] = time.time()
    return {'result': result, }
Beispiel #28
0
 def doRequestService(self, arg):
     """
     Action method.
     """
     bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(),
                                                 0)
     num_suppliers = settings.getSuppliersNumberDesired()
     if num_suppliers > 0:
         bytes_per_supplier = int(
             math.ceil(2.0 * bytes_needed / float(num_suppliers)))
     else:
         bytes_per_supplier = int(
             math.ceil(2.0 * settings.MinimumNeededBytes() /
                       float(settings.DefaultDesiredSuppliers())))
     service_info = 'service_supplier %d' % bytes_per_supplier
     request = p2p_service.SendRequestService(self.idurl,
                                              service_info,
                                              callbacks={
                                                  commands.Ack():
                                                  self._supplier_acked,
                                                  commands.Fail():
                                                  self._supplier_failed,
                                              })
     self.request_packet_id = request.PacketID
Beispiel #29
0
    def doDecideToDismiss(self, *args, **kwargs):
        """
        Action method.
        """
        global _SuppliersToFire
        from p2p import p2p_connector
        from p2p import network_connector
        from customer import supplier_connector
        from p2p import online_status
        # take any actions only if I am connected to the network
        if not p2p_connector.A() or not network_connector.A():
            if _Debug:
                lg.out(
                    _DebugLevel,
                    'fire_hire.doDecideToDismiss    p2p_connector() is not ready yet, SKIP'
                )
            self.automat('made-decision', [])
            return
        if not network_connector.A():
            if _Debug:
                lg.out(
                    _DebugLevel,
                    'fire_hire.doDecideToDismiss    network_connector() is not ready yet, SKIP'
                )
            self.automat('made-decision', [])
            return
        if p2p_connector.A().state != 'CONNECTED' or network_connector.A(
        ).state != 'CONNECTED':
            if _Debug:
                lg.out(
                    _DebugLevel,
                    'fire_hire.doDecideToDismiss    p2p/network is not connected at the moment, SKIP'
                )
            self.automat('made-decision', [])
            return
        # if certain suppliers needs to be removed by manual/external request just do that
        to_be_fired = id_url.to_list(set(_SuppliersToFire))
        _SuppliersToFire = []
        if to_be_fired:
            lg.info('going to fire %d suppliers from external request' %
                    len(to_be_fired))
            self.automat('made-decision', to_be_fired)
            return
        # make sure to not go too far when i just want to decrease number of my suppliers
        number_desired = settings.getSuppliersNumberDesired()
        redundant_suppliers = set()
        if contactsdb.num_suppliers() > number_desired:
            for supplier_index in range(number_desired,
                                        contactsdb.num_suppliers()):
                idurl = contactsdb.supplier(supplier_index)
                if idurl:
                    lg.info('found REDUNDANT supplier %s at position %d' % (
                        idurl,
                        supplier_index,
                    ))
                    redundant_suppliers.add(idurl)
        if redundant_suppliers:
            result = list(redundant_suppliers)
            lg.info('will replace redundant suppliers: %s' % result)
            self.automat('made-decision', result)
            return
        # now I need to look more careful at my suppliers
        potentialy_fired = set()
        connected_suppliers = set()
        disconnected_suppliers = set()
        requested_suppliers = set()
        online_suppliers = set()
        offline_suppliers = set()
        # if you have some empty suppliers need to get rid of them,
        # but no need to dismiss anyone at the moment.
        my_suppliers = contactsdb.suppliers()
        if _Debug:
            lg.args(_DebugLevel, my_suppliers=my_suppliers)
        if id_url.is_some_empty(my_suppliers):
            lg.warn('SKIP, found empty supplier')
            self.automat('made-decision', [])
            return
        for supplier_idurl in my_suppliers:
            sc = supplier_connector.by_idurl(supplier_idurl)
            if not sc:
                lg.warn('SKIP, supplier connector for supplier %s not exist' %
                        supplier_idurl)
                continue
            if sc.state == 'NO_SERVICE':
                lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl)
                disconnected_suppliers.add(supplier_idurl)
                potentialy_fired.add(supplier_idurl)
            elif sc.state == 'CONNECTED':
                connected_suppliers.add(supplier_idurl)
            elif sc.state in [
                    'DISCONNECTED',
                    'REFUSE',
            ]:
                disconnected_suppliers.add(supplier_idurl)
#             elif sc.state in ['QUEUE?', 'REQUEST', ]:
#                 requested_suppliers.add(supplier_idurl)
            if online_status.isOffline(supplier_idurl):
                offline_suppliers.add(supplier_idurl)
            elif online_status.isOnline(supplier_idurl):
                online_suppliers.add(supplier_idurl)
            elif online_status.isCheckingNow(supplier_idurl):
                requested_suppliers.add(supplier_idurl)
        if not connected_suppliers or not online_suppliers:
            lg.warn('SKIP, no ONLINE suppliers found at the moment')
            self.automat('made-decision', [])
            return
        if requested_suppliers:
            lg.warn('SKIP, still waiting response from some of suppliers')
            self.automat('made-decision', [])
            return
        if not disconnected_suppliers:
            if _Debug:
                lg.out(
                    _DebugLevel,
                    'fire_hire.doDecideToDismiss    SKIP, no OFFLINE suppliers found at the moment'
                )
            # TODO: add more conditions to fire "slow" suppliers - they are still connected but useless
            self.automat('made-decision', [])
            return
        if len(offline_suppliers) + len(online_suppliers) != number_desired:
            lg.warn('SKIP, offline + online != total count: %s %s %s' %
                    (offline_suppliers, online_suppliers, number_desired))
            self.automat('made-decision', [])
            return
        max_offline_suppliers_count = eccmap.GetCorrectableErrors(
            number_desired)
        if len(offline_suppliers) > max_offline_suppliers_count:
            lg.warn(
                'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % (
                    len(offline_suppliers),
                    max_offline_suppliers_count,
                ))
            self.automat('made-decision', [])
            return
        critical_offline_suppliers_count = eccmap.GetFireHireErrors(
            number_desired)
        if len(offline_suppliers) >= critical_offline_suppliers_count and len(
                offline_suppliers) > 0:
            if config.conf().getBool(
                    'services/employer/replace-critically-offline-enabled'):
                # TODO: check that issue
                # too aggressive replacing suppliers who still have the data is very dangerous !!!
                one_dead_supplier = offline_suppliers.pop()
                lg.warn(
                    'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d'
                    % (
                        one_dead_supplier,
                        critical_offline_suppliers_count,
                    ))
                potentialy_fired.add(one_dead_supplier)
        if not potentialy_fired:
            if _Debug:
                lg.out(
                    _DebugLevel,
                    'fire_hire.doDecideToDismiss   found no "bad" suppliers, all is good !!!!!'
                )
            self.automat('made-decision', [])
            return
        # only replace suppliers one by one at the moment
        result = list(potentialy_fired)
        lg.info('will replace supplier %s' % result[0])
        self.automat('made-decision', [
            result[0],
        ])
Beispiel #30
0
    def doDecideToDismiss(self, arg):
        """
        Action method.
        """
        global _SuppliersToFire
        to_be_fired = list(set(_SuppliersToFire))
        _SuppliersToFire = []
        if to_be_fired:
            lg.warn('going to fire %d suppliers from external request' %
                    len(to_be_fired))
            self.automat('made-decision', to_be_fired)
            return
        potentialy_fired = set()
        connected_suppliers = set()
        disconnected_suppliers = set()
        requested_suppliers = set()
        online_suppliers = set()
        offline_suppliers = set()
        redundant_suppliers = set()
        # if you have some empty suppliers need to get rid of them,
        # but no need to dismiss anyone at the moment.
        if '' in contactsdb.suppliers() or None in contactsdb.suppliers():
            lg.warn('SKIP, found empty supplier')
            self.automat('made-decision', [])
            return
        number_desired = settings.getSuppliersNumberDesired()
        for supplier_idurl in contactsdb.suppliers():
            sc = supplier_connector.by_idurl(supplier_idurl)
            if not sc:
                lg.warn('SKIP, supplier connector for supplier %s not exist' %
                        supplier_idurl)
                continue
            if sc.state == 'NO_SERVICE':
                lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl)
                disconnected_suppliers.add(supplier_idurl)
                potentialy_fired.add(supplier_idurl)
            elif sc.state == 'CONNECTED':
                connected_suppliers.add(supplier_idurl)
            elif sc.state in [
                    'DISCONNECTED',
                    'REFUSE',
            ]:
                disconnected_suppliers.add(supplier_idurl)
#             elif sc.state in ['QUEUE?', 'REQUEST', ]:
#                 requested_suppliers.add(supplier_idurl)
            if contact_status.isOffline(supplier_idurl):
                offline_suppliers.add(supplier_idurl)
            elif contact_status.isOnline(supplier_idurl):
                online_suppliers.add(supplier_idurl)
            elif contact_status.isCheckingNow(supplier_idurl):
                requested_suppliers.add(supplier_idurl)
        if contactsdb.num_suppliers() > number_desired:
            for supplier_index in range(number_desired,
                                        contactsdb.num_suppliers()):
                idurl = contactsdb.supplier(supplier_index)
                if idurl:
                    lg.warn('found "REDUNDANT" supplier %s at position %d' % (
                        idurl,
                        supplier_index,
                    ))
                    potentialy_fired.add(idurl)
                    redundant_suppliers.add(idurl)
                else:
                    lg.warn('supplier at position %d not exist' %
                            supplier_index)
        if not connected_suppliers or not online_suppliers:
            lg.warn('SKIP, no ONLINE suppliers found at the moment')
            self.automat('made-decision', [])
            return
        if requested_suppliers:
            lg.warn('SKIP, still waiting response from some of suppliers')
            self.automat('made-decision', [])
            return
        if redundant_suppliers:
            result = list(redundant_suppliers)
            lg.info('will replace redundant suppliers: %s' % result)
            self.automat('made-decision', result)
            return
        if not disconnected_suppliers:
            lg.warn('SKIP, no OFFLINE suppliers found at the moment')
            # TODO: add more conditions to fire "slow" suppliers
            self.automat('made-decision', [])
            return
        if len(offline_suppliers) + len(online_suppliers) != number_desired:
            lg.warn('SKIP, offline + online != total count: %s %s %s' %
                    (offline_suppliers, online_suppliers, number_desired))
            self.automat('made-decision', [])
            return
        from raid import eccmap
        max_offline_suppliers_count = eccmap.GetCorrectableErrors(
            number_desired)
        if len(offline_suppliers) > max_offline_suppliers_count:
            lg.warn(
                'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % (
                    len(offline_suppliers),
                    max_offline_suppliers_count,
                ))
            self.automat('made-decision', [])
            return
        critical_offline_suppliers_count = eccmap.GetFireHireErrors(
            number_desired)
        # TODO:  temporary disabled because of an issue: too aggressive replacing suppliers who still have the data
        if False:  # len(offline_suppliers) >= critical_offline_suppliers_count:
            one_dead_supplier = offline_suppliers.pop()
            lg.warn(
                'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d'
                % (
                    one_dead_supplier,
                    critical_offline_suppliers_count,
                ))
            potentialy_fired.add(one_dead_supplier)
        if not potentialy_fired:
            lg.out(
                6,
                'fire_hire.doDecideToDismiss   found no "bad" suppliers, all is good !!!!!'
            )
            self.automat('made-decision', [])
            return
        # only replace suppliers one by one at the moment
        result = list(potentialy_fired)
        lg.info('will replace supplier %s' % result[0])
        self.automat('made-decision', [
            result[0],
        ])
Beispiel #31
0
 def doSubstituteSupplier(self, *args, **kwargs):
     """
     Action method.
     """
     new_idurl = id_url.field(args[0])
     family_position = kwargs.get('family_position')
     current_suppliers = list(contactsdb.suppliers())
     desired_suppliers = settings.getSuppliersNumberDesired()
     old_idurl = None
     if family_position in self.hire_list:
         self.hire_list.remove(family_position)
         lg.info(
             'found position on which new supplier suppose to be hired: %d'
             % family_position)
     else:
         lg.warn('did not found position for new supplier to be hired on')
     if new_idurl in current_suppliers:
         raise Exception('%s is already supplier' % new_idurl)
     if family_position is None or family_position == -1:
         lg.warn(
             'unknown family_position from supplier results, will pick first empty spot'
         )
         position = -1
         old_idurl = None
         for i in range(len(current_suppliers)):
             if not current_suppliers[i].strip():
                 position = i
                 break
             if id_url.is_in(current_suppliers[i],
                             self.dismiss_list,
                             as_field=False):
                 position = i
                 old_idurl = current_suppliers[i]
                 break
         family_position = position
     if _Debug:
         lg.out(
             _DebugLevel,
             'fire_hire.doSubstituteSupplier family_position=%d' %
             family_position)
     contactsdb.add_supplier(idurl=new_idurl, position=family_position)
     contactsdb.save_suppliers()
     misc.writeSupplierData(
         new_idurl,
         'connected',
         time.strftime('%d-%m-%Y %H:%M:%S'),
         my_id.getIDURL(),
     )
     from main import control
     control.on_suppliers_changed(current_suppliers)
     if family_position < 0:
         lg.info(
             'added new supplier, family position unknown: %s desired_suppliers=%d current_suppliers=%d'
             % (new_idurl, desired_suppliers, len(contactsdb.suppliers())))
         events.send('supplier-modified',
                     data=dict(
                         new_idurl=new_idurl,
                         old_idurl=None,
                         position=family_position,
                         ecc_map=eccmap.Current().name,
                         family_snapshot=id_url.to_bin_list(
                             contactsdb.suppliers()),
                     ))
     else:
         if old_idurl:
             lg.info(
                 'hired new supplier and substitute existing supplier on position %d : %s->%s desired_suppliers=%d current_suppliers=%d'
                 % (family_position, old_idurl, new_idurl,
                    desired_suppliers, len(contactsdb.suppliers())))
             events.send('supplier-modified',
                         data=dict(
                             new_idurl=new_idurl,
                             old_idurl=old_idurl,
                             position=family_position,
                             ecc_map=eccmap.Current().name,
                             family_snapshot=id_url.to_bin_list(
                                 contactsdb.suppliers()),
                         ))
         else:
             lg.info(
                 'hired new supplier on empty position %d : %s desired_suppliers=%d current_suppliers=%d'
                 % (family_position, new_idurl, desired_suppliers,
                    len(contactsdb.suppliers())))
             events.send('supplier-modified',
                         data=dict(
                             new_idurl=new_idurl,
                             old_idurl=None,
                             position=family_position,
                             ecc_map=eccmap.Current().name,
                             family_snapshot=id_url.to_bin_list(
                                 contactsdb.suppliers()),
                         ))
     self.restart_interval = 1.0
     if _Debug:
         lg.out(_DebugLevel,
                '    my current suppliers: %r' % contactsdb.suppliers())