def doDecideToDismiss(self, arg): """ Action method. """ global _SuppliersToFire result = set(_SuppliersToFire) _SuppliersToFire = [] # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. if '' in contactsdb.suppliers(): lg.out(10, 'fire_hire.doDecideToDismiss found empty supplier, SKIP') self.automat('made-decision', []) return for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if not sc: continue if sc.state == 'NO_SERVICE': result.add(supplier_idurl) if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired(): for supplier_index in range(settings.getSuppliersNumberDesired(), contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: result.add(idurl) result = list(result) lg.out(10, 'fire_hire.doDecideToDismiss %s' % result) self.automat('made-decision', result)
def doOverallCheckUp(self, *args, **kwargs): """ Action method. """ if '' in contactsdb.suppliers() or b'' in contactsdb.suppliers(): if _Debug: lg.out( _DebugLevel, 'backup_monitor.doOverallCheckUp found empty supplier, restart now' ) self.automat('restart') return if online_status.listOfflineSuppliers(): if time.time() - self.last_execution_time > 60: # re-sync every 1 min. if at least on supplier is dead if _Debug: lg.out( _DebugLevel, 'backup_monitor.doOverallCheckUp restart after 1 min, found offline suppliers' ) self.automat('restart') return if time.time() - self.last_execution_time > 60 * 10: # also re-sync every 10 min. if _Debug: lg.out( _DebugLevel, 'backup_monitor.doOverallCheckUp periodic 10 min. restart' ) self.automat('restart') return
def isMoreNeeded(self, *args, **kwargs): """ Condition method. """ # if _Debug: # lg.out(_DebugLevel, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % ( # contactsdb.num_suppliers(), len(self.dismiss_list), # settings.getSuppliersNumberDesired())) if id_url.is_some_empty(contactsdb.suppliers()): if _Debug: lg.out(_DebugLevel, 'fire_hire.isMoreNeeded found empty supplier!!!') return True if isinstance(args[0], list): dismissed = args[0] else: dismissed = self.dismiss_list s = set(id_url.to_bin_list(contactsdb.suppliers())) s.difference_update(set(id_url.to_bin_list(dismissed))) result = len(s) < settings.getSuppliersNumberDesired() if _Debug: lg.out( _DebugLevel, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' % (contactsdb.num_suppliers(), len(dismissed), len(s), settings.getSuppliersNumberDesired(), result)) return result
def isMoreNeeded(self, arg): """ Condition method. """ # lg.out(10, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % ( # contactsdb.num_suppliers(), len(self.dismiss_list), # settings.getSuppliersNumberDesired())) if '' in contactsdb.suppliers(): if _Debug: lg.out(_DebugLevel, 'fire_hire.isMoreNeeded found empty suppliers!!!') return True if isinstance(arg, list): dismissed = arg else: dismissed = self.dismiss_list s = set(contactsdb.suppliers()) s.difference_update(set(dismissed)) result = len(s) < settings.getSuppliersNumberDesired() if _Debug: lg.out( _DebugLevel, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' % (contactsdb.num_suppliers(), len(dismissed), len(s), settings.getSuppliersNumberDesired(), result)) return result
def doDecideToDismiss(self, arg): """ Action method. """ global _SuppliersToFire result = set(_SuppliersToFire) _SuppliersToFire = [] # if you have some empty suppliers need to get rid of them, # but no need to dismiss anyone at the moment. if '' in contactsdb.suppliers(): lg.out(10, 'fire_hire.doDecideToDismiss found empty supplier, SKIP') self.automat('made-decision', []) return for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if not sc: continue if sc.state == 'NO_SERVICE': result.add(supplier_idurl) if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired(): for supplier_index in range( settings.getSuppliersNumberDesired(), contactsdb.num_suppliers()): idurl = contactsdb.supplier(supplier_index) if idurl: result.add(idurl) result = list(result) lg.out(10, 'fire_hire.doDecideToDismiss %s' % result) self.automat('made-decision', result)
def RunTask(): """ Checks current jobs and run a one task if it is possible. Verifies if it is possible to start a new task, the maximum number of simultaneously running ``Jobs`` is limited. """ if len(tasks()) == 0: return False if len(jobs()) >= MAXIMUM_JOBS_STARTED: return False if b'' in contactsdb.suppliers() or '' in contactsdb.suppliers(): if _Debug: lg.out( _DebugLevel, 'backup_control.RunTask found empty supplier, retry after 5 sec' ) reactor.callLater(5, RunTask) # @UndefinedVariable return False T = tasks().pop(0) message = T.run() if message: events.send('backup-task-failed', data=dict( path_id=T.pathID, message=message, )) T.result_defer.errback((T.pathID, message)) else: # events.send('backup-task-executed', data=dict(path_id=T.pathID, backup_id=T.backupID, )) T.result_defer.callback((T.backupID, None)) T.destroy(message) return True
def doFindNewSupplier(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out( _DebugLevel, 'fire_hire.doFindNewSupplier desired_suppliers=%d current_suppliers=%r' % (settings.getSuppliersNumberDesired(), contactsdb.suppliers())) from p2p import network_connector if network_connector.A().state != 'CONNECTED': if _Debug: lg.out( _DebugLevel, ' network_connector is not CONNECTED at the moment, SKIP' ) self.automat('search-failed') return position_for_new_supplier = None for pos in range(settings.getSuppliersNumberDesired()): if pos in self.hire_list: continue supplier_idurl = contactsdb.supplier(pos) if not supplier_idurl: lg.info( 'found empty supplier at position %d and going to find new supplier on that position' % pos) position_for_new_supplier = pos break if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False): lg.info( 'going to find new supplier on existing position %d to replace supplier %s' % ( pos, supplier_idurl, )) position_for_new_supplier = pos break if position_for_new_supplier is None: lg.err('did not found position for new supplier') self.automat('search-failed') return from customer import supplier_finder for idurl_txt in strng.to_text(config.conf().getData( 'services/employer/candidates')).split(','): if idurl_txt.strip(): supplier_finder.AddSupplierToHire(idurl_txt) self.hire_list.append(position_for_new_supplier) supplier_finder.A( 'start', family_position=position_for_new_supplier, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list(contactsdb.suppliers()), )
def doRemoveSuppliers(self, *args, **kwargs): """ Action method. """ current_suppliers = contactsdb.suppliers() desired_suppliers = settings.getSuppliersNumberDesired() if len(current_suppliers) < desired_suppliers: lg.warn('must have more suppliers %d<%d' % (len(current_suppliers), desired_suppliers)) removed_suppliers = [] for supplier_idurl in self.dismiss_list: if id_url.is_not_in(supplier_idurl, current_suppliers, as_field=False): lg.warn('%s not a supplier' % supplier_idurl) continue pos = current_suppliers.index(id_url.field(supplier_idurl)) current_suppliers[pos] = '' removed_suppliers.append(( pos, supplier_idurl, )) misc.writeSupplierData( supplier_idurl, 'disconnected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getLocalID(), ) current_suppliers = current_suppliers[:desired_suppliers] contactsdb.update_suppliers(current_suppliers) contactsdb.save_suppliers() from main import control control.on_suppliers_changed(current_suppliers) for position, supplier_idurl in removed_suppliers: events.send( 'supplier-modified', dict( new_idurl=None, old_idurl=supplier_idurl, position=position, )) lg.info( 'removed some suppliers : %d desired_suppliers=%d current_suppliers=%d' % (len(self.dismiss_list), desired_suppliers, len(contactsdb.suppliers()))) if _Debug: lg.out(_DebugLevel, ' my current suppliers: %r' % contactsdb.suppliers())
def doInit(self, arg): """ Action method. """ if data_receiver.A(): data_receiver.A().addStateChangedCallback( self._on_data_receiver_state_changed) self.known_suppliers = filter( None, contactsdb.suppliers(customer_idurl=self.customer_idurl)) known_eccmap_dict = {} for supplier_idurl in self.known_suppliers: known_ecc_map = contactsdb.get_supplier_meta_info( supplier_idurl=supplier_idurl, customer_idurl=self.customer_idurl, ).get('ecc_map', None) if known_ecc_map: if known_ecc_map not in known_eccmap_dict: known_eccmap_dict[known_ecc_map] = 0 known_eccmap_dict[known_ecc_map] += 1 if known_eccmap_dict: all_known_eccmaps = known_eccmap_dict.items() all_known_eccmaps.sort(key=lambda i: i[1], reverse=True) self.EccMap = eccmap.eccmap(all_known_eccmaps[0][0]) lg.info('eccmap %s recognized from suppliers meta info' % self.EccMap) else: self.EccMap = eccmap.eccmap( eccmap.GetEccMapName(len(self.known_suppliers))) lg.warn( 'no meta info found, guessed eccmap %s from %d known suppliers' % (self.EccMap, len(self.known_suppliers)))
def health_check(self): from raid import eccmap from contacts import contactsdb from userid import id_url missed_suppliers = id_url.empty_count(contactsdb.suppliers()) # to have that service running minimum amount of suppliers must be already in the family return missed_suppliers <= eccmap.Current().CorrectableErrors
def _do_request(self, x=None): from raid import eccmap self.received_lf_counter = 0 self.requested_lf_packet_ids.clear() known_suppliers = contactsdb.suppliers(customer_idurl=self.target_customer_idurl) try: self.critical_suppliers_number = eccmap.GetCorrectableErrors(len(known_suppliers)) except: lg.warn('number of known suppliers for customer %r is not standard' % self.target_customer_idurl) self.critical_suppliers_number = int(float(len(known_suppliers)) * 0.75) for idurl in known_suppliers: if idurl: if online_status.isOnline(idurl): if _Debug: lg.out(_DebugLevel, 'list_files_orator._do_request ListFiles() from my supplier %s' % idurl) outpacket = p2p_service.SendListFiles( target_supplier=idurl, customer_idurl=self.target_customer_idurl, timeout=30, ) if outpacket: self.requested_lf_packet_ids.add(outpacket.PacketID) else: lg.err('failed sending ListFiles() to %r' % idurl) else: lg.warn('skip sending ListFiles() because %s is not online' % idurl)
def doInit(self, *args, **kwargs): """ Action method. """ self._do_block_rebuilding() self.known_suppliers = [ _f for _f in contactsdb.suppliers(customer_idurl=self.customer_idurl) if _f ] if not self.EccMap: if self.customer_idurl == my_id.getIDURL(): self.EccMap = eccmap.Current() lg.info('ECC map %r set from local for my own suppliers' % self.EccMap) if not self.EccMap: known_eccmap_dict = {} for supplier_idurl in self.known_suppliers: known_ecc_map = contactsdb.get_supplier_meta_info( supplier_idurl=supplier_idurl, customer_idurl=self.customer_idurl, ).get('ecc_map', None) if known_ecc_map: if known_ecc_map not in known_eccmap_dict: known_eccmap_dict[known_ecc_map] = 0 known_eccmap_dict[known_ecc_map] += 1 if known_eccmap_dict: all_known_eccmaps = list(known_eccmap_dict.items()) all_known_eccmaps.sort(key=lambda i: i[1], reverse=True) self.EccMap = eccmap.eccmap(all_known_eccmaps[0][0]) lg.info('ECC map %r recognized from suppliers meta info' % self.EccMap) else: known_ecc_map = None if driver.is_on('service_shared_data'): from access import shared_access_coordinator active_share = shared_access_coordinator.get_active_share( self.key_id) if active_share: known_ecc_map = active_share.known_ecc_map if known_ecc_map: self.EccMap = eccmap.eccmap(known_ecc_map) lg.info('ECC map %r recognized from active share %r' % ( self.EccMap, active_share, )) else: num_suppliers = len(self.known_suppliers) if num_suppliers not in eccmap.GetPossibleSuppliersCount(): num_suppliers = settings.DefaultDesiredSuppliers() self.EccMap = eccmap.eccmap( eccmap.GetEccMapName(num_suppliers)) lg.warn( 'no meta info found, guessed ECC map %r from %d known suppliers' % (self.EccMap, len(self.known_suppliers))) self.max_errors = eccmap.GetCorrectableErrors( self.EccMap.NumSuppliers()) if data_receiver.A(): data_receiver.A().addStateChangedCallback( self._on_data_receiver_state_changed)
def doSuppliersSendDBInfo(self, arg): from p2p import contact_status lg.out(4, 'backup_db_keeper.doSuppliersSendDBInfo') packetID = settings.BackupIndexFileName() self.sentSuppliers.clear() src = bpio.ReadBinaryFile(settings.BackupIndexFilePath()) localID = my_id.getLocalID() b = encrypted.Block( localID, packetID, 0, key.NewSessionKey(), key.SessionKeyType(), True, src) Payload = b.Serialize() for supplierId in contactsdb.suppliers(): if not supplierId: continue if not contact_status.isOnline(supplierId): continue newpacket = signed.Packet( commands.Data(), localID, localID, packetID, Payload, supplierId) pkt_out = gateway.outbox(newpacket, callbacks={ commands.Ack(): self._supplier_acked, commands.Fail(): self._supplier_acked, }) self.sentSuppliers.add(supplierId) lg.out( 4, ' %s sending to %s' % (pkt_out, nameurl.GetName(supplierId)))
def _do_retrieve(self, x=None): packetID = global_id.MakeGlobalID( customer=my_id.getGlobalID(key_alias='master'), path=settings.BackupIndexFileName(), ) localID = my_id.getIDURL() for supplier_idurl in contactsdb.suppliers(): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None or sc.state != 'CONNECTED': continue if online_status.isOffline(supplier_idurl): continue pkt_out = p2p_service.SendRetreive(ownerID=localID, creatorID=localID, packetID=packetID, remoteID=supplier_idurl, response_timeout=60 * 2, callbacks={ commands.Data(): self._on_supplier_response, commands.Fail(): self._on_supplier_fail, }) if pkt_out: self.requesting_suppliers.add(supplier_idurl) self.requested_suppliers_number += 1 self.requests_packets_sent.append((packetID, supplier_idurl)) if _Debug: lg.out( _DebugLevel, ' %s sending to %s' % (pkt_out, nameurl.GetName(supplier_idurl)))
def RequestListFilesAll(customer_idurl=None): r = [] for supplier_idurl in contactsdb.suppliers(customer_idurl=customer_idurl): r.append( SendRequestListFiles(supplier_idurl, customer_idurl=customer_idurl)) return r
def doSupplierConnect(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector from customer import fire_hire from raid import eccmap position = self.family_position if not position: lg.warn('position for new supplier is unknown, will "guess"') current_suppliers = list(contactsdb.suppliers()) for i in range(len(current_suppliers)): if not current_suppliers[i].strip(): position = i break if current_suppliers[i] in fire_hire.A().dismiss_list: position = i break sc = supplier_connector.by_idurl(self.target_idurl) if not sc: sc = supplier_connector.create( supplier_idurl=self.target_idurl, customer_idurl=my_id.getLocalID(), ) sc.automat( 'connect', family_position=position, ecc_map=(self.ecc_map or eccmap.Current().name), family_snapshot=self.family_snapshot, ) sc.set_callback('supplier_finder', self._supplier_connector_state)
def doConnectSuppliers(self, *args, **kwargs): """ Action method. """ self.connect_list = [] my_current_family = list(contactsdb.suppliers()) for pos, supplier_idurl in enumerate(my_current_family): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None: sc = supplier_connector.create( supplier_idurl=supplier_idurl, customer_idurl=my_id.getLocalID(), ) sc.set_callback('fire_hire', self._on_supplier_connector_state_changed) self.connect_list.append(supplier_idurl) sc.automat( 'connect', family_position=pos, ecc_map=eccmap.Current().name, family_snapshot=my_current_family, ) supplier_contact_status = contact_status.getInstance( supplier_idurl) if supplier_contact_status: supplier_contact_status.addStateChangedCallback( self._on_supplier_contact_status_state_changed, newstate='OFFLINE', )
def _stats(params): from contacts import contactsdb from p2p import contact_status from lib import diskspace result = {} result['suppliers'] = contactsdb.num_suppliers() result['max_suppliers'] = settings.getSuppliersNumberDesired() result['online_suppliers'] = contact_status.countOnlineAmong( contactsdb.suppliers()) result['customers'] = contactsdb.num_customers() result['bytes_donated'] = settings.getDonatedBytes() result['value_donated'] = diskspace.MakeStringFromBytes( settings.getDonatedBytes()) result['bytes_needed'] = settings.getNeededBytes() result['value_needed'] = diskspace.MakeStringFromBytes( settings.getNeededBytes()) result['bytes_used_total'] = backup_fs.sizebackups() result['value_used_total'] = diskspace.MakeStringFromBytes( backup_fs.sizebackups()) result['bytes_used_supplier'] = 0 if ( contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() / contactsdb.num_suppliers())) result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders() result['files_count'] = backup_fs.numberfiles() result['folders_count'] = backup_fs.numberfolders() result['items_count'] = backup_fs.counter() result['timestamp'] = time.time() return { 'result': result, }
def ping_suppliers(customer_idurl=None, timeout=20): from p2p import online_status l = [] for supplier_idurl in contactsdb.suppliers(customer_idurl=customer_idurl): if supplier_idurl: l.append(online_status.ping(idurl=supplier_idurl, ack_timeout=timeout, channel='ping_suppliers', keep_alive=True)) return DeferredList(l, consumeErrors=True)
def SendSuppliers(customer_idurl=None): """ Send my identity file to all my suppliers, calls to ``SendToIDs()`` method. """ lg.out(6, "propagate.SendSuppliers") SendToIDs(contactsdb.suppliers(customer_idurl=customer_idurl), HandleSuppliersAck)
def doRemoveSuppliers(self, *args, **kwargs): """ Action method. """ current_suppliers = contactsdb.suppliers() desired_suppliers = settings.getSuppliersNumberDesired() if len(current_suppliers) < desired_suppliers: lg.warn('must have more suppliers %d<%d' % ( len(current_suppliers), desired_suppliers)) removed_suppliers = [] for supplier_idurl in self.dismiss_list: if supplier_idurl not in current_suppliers: lg.warn('%s not a supplier' % supplier_idurl) continue pos = current_suppliers.index(supplier_idurl) # current_suppliers.remove(supplier_idurl) current_suppliers[pos] = '' removed_suppliers.append((pos, supplier_idurl,)) misc.writeSupplierData( supplier_idurl, 'disconnected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getLocalID(), ) current_suppliers = current_suppliers[:desired_suppliers] contactsdb.update_suppliers(current_suppliers) contactsdb.save_suppliers() from main import control control.on_suppliers_changed(current_suppliers) for position, supplier_idurl in removed_suppliers: events.send('supplier-modified', dict( new_idurl=None, old_idurl=supplier_idurl, position=position, )) lg.out(2, '!!!!!!!!!!! REMOVE SUPPLIERS : %d' % len(self.dismiss_list))
def SendSuppliers(customer_idurl=None): """ Send my identity file to all my suppliers, calls to ``SendToIDs()`` method. """ if _Debug: lg.out(_DebugLevel, "propagate.SendSuppliers") SendToIDs(contactsdb.suppliers(customer_idurl=customer_idurl), ack_handler=HandleSuppliersAck, wide=True)
def doRemoveSuppliers(self, arg): """ Action method. """ current_suppliers = contactsdb.suppliers() desired_suppliers = settings.getSuppliersNumberDesired() if len(current_suppliers) < desired_suppliers: lg.warn('must have more suppliers %d<%d' % ( len(current_suppliers), desired_suppliers)) for supplier_idurl in self.dismiss_list: if supplier_idurl not in current_suppliers: lg.warn('%s not a supplier' % supplier_idurl) continue pos = current_suppliers.index(supplier_idurl) # current_suppliers.remove(supplier_idurl) current_suppliers[pos] = '' misc.writeSupplierData( supplier_idurl, 'disconnected', time.strftime('%d-%m-%Y %H:%M:%S')) current_suppliers = current_suppliers[:desired_suppliers] contactsdb.update_suppliers(current_suppliers) contactsdb.save_suppliers() if settings.NewWebGUI(): from web import control control.on_suppliers_changed(current_suppliers) else: from web import webcontrol webcontrol.OnListSuppliers() lg.out(2, '!!!!!!!!!!! REMOVE SUPPLIERS : %d' % len(self.dismiss_list))
def doFindNewSupplier(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'fire_hire.doFindNewSupplier') position_for_new_supplier = None for pos in range(settings.getSuppliersNumberDesired()): if pos in self.hire_list: continue supplier_idurl = contactsdb.supplier(pos) if not supplier_idurl: lg.info('found empty supplier at position %d and going to find new supplier on that position' % pos) position_for_new_supplier = pos break if supplier_idurl in self.dismiss_list: lg.info('going to find new supplier on existing position %d to replace supplier %s' % ( pos, supplier_idurl, )) position_for_new_supplier = pos break if position_for_new_supplier is None: lg.err('did not found position for new supplier') self.automat('search-failed') return self.hire_list.append(position_for_new_supplier) supplier_finder.A( 'start', family_position=position_for_new_supplier, ecc_map=eccmap.Current().name, family_snapshot=contactsdb.suppliers(), )
def doRememberSuppliers(self, *args, **kwargs): """ Action method. """ self.current_suppliers = list(contactsdb.suppliers()) self.backups_progress_last_iteration = 0 self.last_execution_time = time.time()
def doSupplierConnect(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector from customer import fire_hire from raid import eccmap position = self.family_position if position is None or position == -1: lg.warn('position for new supplier is unknown, will "guess"') current_suppliers = list(contactsdb.suppliers()) for i in range(len(current_suppliers)): supplier_idurl = current_suppliers[i].to_bin() if not supplier_idurl: position = i break if id_url.is_in(supplier_idurl, fire_hire.A().dismiss_list, as_field=False): position = i break sc = supplier_connector.by_idurl(self.target_idurl) if not sc: sc = supplier_connector.create( supplier_idurl=self.target_idurl, customer_idurl=my_id.getIDURL(), ) sc.set_callback('supplier_finder', self._supplier_connector_state) sc.automat( 'connect', family_position=position, ecc_map=(self.ecc_map or eccmap.Current().name), family_snapshot=self.family_snapshot, )
def isStillNeeded(self, *args, **kwargs): """ Condition method. """ supplier_idurl = args[0] current_suppliers = contactsdb.suppliers() if supplier_idurl in current_suppliers: # this guy is already a supplier, we still need more then return True desired_number = settings.getSuppliersNumberDesired() needed_suppliers = current_suppliers[:desired_number] empty_suppliers = needed_suppliers.count(id_url.field(b'')) # if '' in needed_suppliers: # lg.warn('found empty suppliers!!!') # return True s = set(id_url.to_bin_list(needed_suppliers)) s.add(id_url.to_bin(supplier_idurl)) s.difference_update(set(id_url.to_bin_list(self.dismiss_list))) result = len(s) - empty_suppliers < settings.getSuppliersNumberDesired( ) # if _Debug: # lg.out(_DebugLevel, 'fire_hire.isStillNeeded %d %d %d %d %d, result=%s' % ( # contactsdb.num_suppliers(), len(needed_suppliers), len(self.dismiss_list), # len(s), settings.getSuppliersNumberDesired(), result)) return result
def doUpdateSuppliers(self, *args, **kwargs): """ Action method. """ from customer import io_throttle # supplierList = contactsdb.suppliers() # take a list of suppliers positions that was changed changedSupplierNums = backup_matrix.SuppliersChangedNumbers( self.current_suppliers) # notify io_throttle that we do not neeed already this suppliers for supplierNum in changedSupplierNums: lg.out( 2, "backup_monitor.doUpdateSuppliers supplier %d changed: [%s]->[%s]" % ( supplierNum, nameurl.GetName(self.current_suppliers[supplierNum]), nameurl.GetName(contactsdb.suppliers()[supplierNum]), )) suplier_idurl = self.current_suppliers[supplierNum] io_throttle.DeleteSuppliers([ suplier_idurl, ]) # erase (set to 0) remote info for this guys backup_matrix.ClearSupplierRemoteInfo(supplierNum)
def isExistSomeSuppliers(self, *args, **kwargs): """ Condition method. """ sup_list = contactsdb.suppliers() return contactsdb.num_suppliers() > 0 and sup_list.count( id_url.field(b'')) < contactsdb.num_suppliers()
def doConnectSuppliers(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector from p2p import online_status self.connect_list = [] my_current_family = contactsdb.suppliers() for pos, supplier_idurl in enumerate(my_current_family): if not supplier_idurl: continue if self.configs[0] and pos >= self.configs[0]: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None: sc = supplier_connector.create( supplier_idurl=supplier_idurl, customer_idurl=my_id.getIDURL(), ) else: sc.needed_bytes = None sc.do_calculate_needed_bytes() sc.set_callback('fire_hire', self._on_supplier_connector_state_changed) self.connect_list.append(supplier_idurl) sc.automat( 'connect', family_position=pos, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list(my_current_family), ) online_status.add_online_status_listener_callback( idurl=supplier_idurl, callback_method=self._on_supplier_online_status_state_changed, )
def suppliers(AckHandler=None, wide=False, customer_idurl=None): """ Call ``propagate()`` for all suppliers. """ lg.out(6, 'propagate.suppliers') return propagate(contactsdb.suppliers(customer_idurl=customer_idurl), AckHandler, wide)
def isAnyDataComming(self, arg): """ Condition method. """ from transport import packet_in related_packets = packet_in.search(sender_idurl=contactsdb.suppliers()) return len(related_packets) > 0
def start(self): from customer import supplier_connector from contacts import contactsdb for supplier_idurl in contactsdb.suppliers(): if supplier_idurl and not supplier_connector.by_idurl( supplier_idurl): supplier_connector.create(supplier_idurl) return True
def doOverallCheckUp(self, arg): """ Action method. """ if '' in contactsdb.suppliers(): lg.out(6, 'backup_monitor.doOverallCheckUp found empty supplier, restart now') self.automat('restart') return
def doDHTFindRandomUser(self, *args, **kwargs): """ Action method. """ tsk = lookup.random_supplier(ignore_idurls=contactsdb.suppliers()) tsk.result_defer.addCallback(self._nodes_lookup_finished) tsk.result_defer.addErrback( lambda err: self.automat('users-not-found'))
def isMoreNeeded(self, arg): """ Condition method. """ # lg.out(10, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % ( # contactsdb.num_suppliers(), len(self.dismiss_list), # settings.getSuppliersNumberDesired())) if '' in contactsdb.suppliers(): lg.out(4, 'fire_hire.isMoreNeeded found empty suppliers!!!') return True if isinstance(arg, list): dismissed = arg else: dismissed = self.dismiss_list s = set(contactsdb.suppliers()) s.difference_update(set(dismissed)) result = len(s) < settings.getSuppliersNumberDesired() lg.out(14, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' % ( contactsdb.num_suppliers(), len(dismissed), len(s), settings.getSuppliersNumberDesired(), result)) return result
def listOfflineSuppliers(): """ Loops all suppliers and check their state, return a list of those with state OFFLINE. """ result = [] for idurl in contactsdb.suppliers(): if not idurl: result.append(idurl) elif isOffline(idurl): result.append(idurl) return result
def doRequestRemoteFiles(self, arg): global _RequestedListFilesCounter global _RequestedListFilesPacketIDs _RequestedListFilesCounter = 0 _RequestedListFilesPacketIDs.clear() for idurl in contactsdb.suppliers(): if idurl: if contact_status.isOnline(idurl): p2p_service.SendRequestListFiles(idurl) _RequestedListFilesPacketIDs.add(idurl) else: lg.out(6, 'list_files_orator.doRequestRemoteFiles SKIP %s is not online' % idurl)
def ReadLatestRawListFiles(): """ Call ``ReadRawListFiles()`` for every local file we have on hands and build whole "remote" matrix. """ lg.out(4, "backup_matrix.ReadLatestRawListFiles") for idurl in contactsdb.suppliers(): if idurl: filename = os.path.join(settings.SupplierPath(idurl, "listfiles")) if os.path.isfile(filename): listFileText = bpio.ReadTextFile(filename) if listFileText.strip() != "": ReadRawListFiles(contactsdb.supplier_position(idurl), listFileText)
def RequestDeleteListPaths(pathIDs): if _Debug: lg.out(_DebugLevel, "p2p_service.RequestDeleteListPaths wish to delete %d paths" % len(pathIDs)) for supplier in contactsdb.suppliers(): if not supplier: continue # found = False # for workitem in transport_control.SendQueue(): # if workitem.command == commands.DeleteFile() and workitem.remoteid == supplier: # found = True # break # if found: # continue SendDeleteListPaths(supplier, pathIDs)
def doSubstituteSupplier(self, arg): """ Action method. """ new_idurl = arg current_suppliers = list(contactsdb.suppliers()) if new_idurl in current_suppliers: raise Exception('%s is already supplier' % new_idurl) position = -1 old_idurl = None for i in range(len(current_suppliers)): if current_suppliers[i].strip() == '': position = i break if current_suppliers[i] in self.dismiss_list: # self.dismiss_list.remove(current_suppliers[i]) position = i old_idurl = current_suppliers[i] break lg.out(10, 'fire_hire.doSubstituteSupplier position=%d' % position) if position < 0: current_suppliers.append(new_idurl) else: current_suppliers[position] = new_idurl contactsdb.update_suppliers(current_suppliers) contactsdb.save_suppliers() misc.writeSupplierData( new_idurl, 'connected', time.strftime('%d-%m-%Y %H:%M:%S')) if settings.NewWebGUI(): from web import control # control.on_suppliers_changed(current_suppliers) else: from web import webcontrol webcontrol.OnListSuppliers() if position < 0: lg.out(2, '!!!!!!!!!!! ADD SUPPLIER : %s' % (new_idurl)) else: if old_idurl: lg.out( 2, '!!!!!!!!!!! SUBSTITUTE SUPPLIER %d : %s->%s' % (position, old_idurl, new_idurl)) else: lg.out( 2, '!!!!!!!!!!! REPLACE EMPTY SUPPLIER %d : %s' % (position, new_idurl)) self.restart_interval = 1.0
def doConnectSuppliers(self, arg): """ Action method. """ self.connect_list = [] for supplier_idurl in contactsdb.suppliers(): if supplier_idurl == '': continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None: sc = supplier_connector.create(supplier_idurl) sc.set_callback( 'fire_hire', self._supplier_connector_state_changed) self.connect_list.append(supplier_idurl) sc.automat('connect')
def RequestDeleteBackup(BackupID): """ Need to send a "DeleteBackup" command to all suppliers. """ if _Debug: lg.out(_DebugLevel, "p2p_service.RequestDeleteBackup with BackupID=" + str(BackupID)) for supplier in contactsdb.suppliers(): if not supplier: continue # prevItems = [] # transport_control.SendQueueSearch(BackupID) # found = False # for workitem in prevItems: # if workitem.remoteid == supplier: # found = True # break # if found: # continue SendDeleteBackup(supplier, BackupID)
def doUpdateSuppliers(self, arg): """ Action method. """ from customer import io_throttle # supplierList = contactsdb.suppliers() # take a list of suppliers positions that was changed changedSupplierNums = backup_matrix.SuppliersChangedNumbers(self.current_suppliers) # notify io_throttle that we do not neeed already this suppliers for supplierNum in changedSupplierNums: lg.out(2, "backup_monitor.doUpdateSuppliers supplier %d changed: [%s]->[%s]" % ( supplierNum, nameurl.GetName(self.current_suppliers[supplierNum]), nameurl.GetName(contactsdb.suppliers()[supplierNum]),)) suplier_idurl = self.current_suppliers[supplierNum] io_throttle.DeleteSuppliers([suplier_idurl, ]) # erase (set to 0) remote info for this guys backup_matrix.ClearSupplierRemoteInfo(supplierNum)
def doSuppliersRequestDBInfo(self, arg): lg.out(4, 'backup_db_keeper.doSuppliersRequestDBInfo') packetID = settings.BackupIndexFileName() self.requestedSuppliers.clear() Payload = '' localID = my_id.getLocalID() for supplierId in contactsdb.suppliers(): if not supplierId: continue newpacket = signed.Packet( commands.Retrieve(), localID, localID, packetID, Payload, supplierId) gateway.outbox(newpacket, callbacks={ commands.Data(): self._supplier_response, commands.Fail(): self._supplier_response, }) self.requestedSuppliers.add(supplierId)
def _stats(params): from contacts import contactsdb from p2p import contact_status from lib import diskspace result = {} result['suppliers'] = contactsdb.num_suppliers() result['max_suppliers'] = settings.getSuppliersNumberDesired() result['online_suppliers'] = contact_status.countOnlineAmong(contactsdb.suppliers()) result['customers'] = contactsdb.num_customers() result['bytes_donated'] = settings.getDonatedBytes() result['value_donated'] = diskspace.MakeStringFromBytes(settings.getDonatedBytes()) result['bytes_needed'] = settings.getNeededBytes() result['value_needed'] = diskspace.MakeStringFromBytes(settings.getNeededBytes()) result['bytes_used_total'] = backup_fs.sizebackups() result['value_used_total'] = diskspace.MakeStringFromBytes(backup_fs.sizebackups()) result['bytes_used_supplier'] = 0 if (contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() / contactsdb.num_suppliers())) result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders() result['files_count'] = backup_fs.numberfiles() result['folders_count'] = backup_fs.numberfolders() result['items_count'] = backup_fs.counter() result['timestamp'] = time.time() return {'result': result, }
def doSuppliersRequestIndexFile(self, arg): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'index_synchronizer.doSuppliersRequestIndexFile') if driver.is_started('service_backups'): from storage import backup_control self.current_local_revision = backup_control.revision() else: self.current_local_revision = -1 self.latest_supplier_revision = -1 self.requesting_suppliers.clear() self.requested_suppliers_number = 0 packetID = settings.BackupIndexFileName() localID = my_id.getLocalID() for supplierId in contactsdb.suppliers(): if not supplierId: continue if not contact_status.isOnline(supplierId): continue newpacket = signed.Packet( commands.Retrieve(), localID, localID, packetID, '', supplierId) pkt_out = gateway.outbox(newpacket, callbacks={ commands.Data(): self._on_supplier_response, commands.Fail(): self._on_supplier_response, }) if pkt_out: self.requesting_suppliers.add(supplierId) self.requested_suppliers_number += 1 if _Debug: lg.out(_DebugLevel, ' %s sending to %s' % (pkt_out, nameurl.GetName(supplierId)))
def doSuppliersSendIndexFile(self, arg): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile') packetID = settings.BackupIndexFileName() self.sending_suppliers.clear() self.sent_suppliers_number = 0 src = bpio.ReadBinaryFile(settings.BackupIndexFilePath()) localID = my_id.getLocalID() b = encrypted.Block( localID, packetID, 0, key.NewSessionKey(), key.SessionKeyType(), True, src) Payload = b.Serialize() for supplierId in contactsdb.suppliers(): if not supplierId: continue if not contact_status.isOnline(supplierId): continue newpacket = signed.Packet( commands.Data(), localID, localID, packetID, Payload, supplierId) pkt_out = gateway.outbox(newpacket, callbacks={ commands.Ack(): self._on_supplier_acked, commands.Fail(): self._on_supplier_acked, }) if pkt_out: self.sending_suppliers.add(supplierId) self.sent_suppliers_number += 1 if _Debug: lg.out(_DebugLevel, ' %s sending to %s' % (pkt_out, nameurl.GetName(supplierId)))
def ScanBlocksToSend(backupID): """ Opposite method - search for pieces which is not yet delivered to remote suppliers. """ if "" in contactsdb.suppliers(): return {} localMaxBlockNum = local_max_block_numbers().get(backupID, -1) supplierActiveArray = GetActiveArray() bySupplier = {} for supplierNum in xrange(len(supplierActiveArray)): bySupplier[supplierNum] = set() if backupID not in remote_files(): for blockNum in xrange(localMaxBlockNum + 1): localData = GetLocalDataArray(backupID, blockNum) localParity = GetLocalParityArray(backupID, blockNum) for supplierNum in xrange(len(supplierActiveArray)): if supplierActiveArray[supplierNum] != 1: continue if localData[supplierNum] == 1: bySupplier[supplierNum].add(packetid.MakePacketID(backupID, blockNum, supplierNum, "Data")) if localParity[supplierNum] == 1: bySupplier[supplierNum].add(packetid.MakePacketID(backupID, blockNum, supplierNum, "Parity")) else: for blockNum in xrange(localMaxBlockNum + 1): remoteData = GetRemoteDataArray(backupID, blockNum) remoteParity = GetRemoteParityArray(backupID, blockNum) localData = GetLocalDataArray(backupID, blockNum) localParity = GetLocalParityArray(backupID, blockNum) for supplierNum in xrange(len(supplierActiveArray)): if supplierActiveArray[supplierNum] != 1: continue if remoteData[supplierNum] != 1 and localData[supplierNum] == 1: bySupplier[supplierNum].add(packetid.MakePacketID(backupID, blockNum, supplierNum, "Data")) if remoteParity[supplierNum] != 1 and localParity[supplierNum] == 1: bySupplier[supplierNum].add(packetid.MakePacketID(backupID, blockNum, supplierNum, "Parity")) return bySupplier
def isStillNeeded(self, arg): """ Condition method. """ supplier_idurl = arg current_suppliers = contactsdb.suppliers() if supplier_idurl in current_suppliers: # this guy is already a supplier, we still need more then return True desired_number = settings.getSuppliersNumberDesired() needed_suppliers = current_suppliers[:desired_number] empty_suppliers = needed_suppliers.count('') # if '' in needed_suppliers: # lg.warn('found empty suppliers!!!') # return True s = set(needed_suppliers) s.add(supplier_idurl) s.difference_update(set(self.dismiss_list)) result = len(s) - \ empty_suppliers < settings.getSuppliersNumberDesired() # lg.out(14, 'fire_hire.isStillNeeded %d %d %d %d %d, result=%s' % ( # contactsdb.num_suppliers(), len(needed_suppliers), len(self.dismiss_list), # len(s), settings.getSuppliersNumberDesired(), result)) return result
def isExistSomeSuppliers(self, arg): """ Condition method. """ return contactsdb.num_suppliers() > 0 and contactsdb.suppliers().count( '') < contactsdb.num_suppliers()
def doRememberSuppliers(self, arg): """ Action method. """ self.current_suppliers = list(contactsdb.suppliers())