def doRequestRemoteFiles(self, arg): global _RequestedListFilesCounter global _RequestedListFilesPacketIDs _RequestedListFilesCounter = 0 _RequestedListFilesPacketIDs.clear() for idurl in contacts.getSupplierIDs(): if idurl: if contact_status.isOnline(idurl): p2p_service.RequestListFiles(idurl) _RequestedListFilesPacketIDs.add(idurl)
def GetActiveArray(self): """ Loops all suppliers in the set and returns who is alive at the moment. Return a list with integers: 0 for offline suppler and 1 if he is available right now. Uses `p2p.contact_status.isOnline()` to see the current state of supplier. """ activeArray = [0] * self.supplierCount for i in xrange(self.supplierCount): if not self.suppliers[i]: continue if contact_status.isOnline(self.suppliers[i]): activeArray[i] = 1 else: activeArray[i] = 0 return activeArray
def doRequestPackets(self, arg): packetsToRequest = [] for SupplierNumber in range(self.EccMap.datasegments): SupplierID = contacts.getSupplierID(SupplierNumber) if not SupplierID: continue if not self.OnHandData[SupplierNumber] and contact_status.isOnline(SupplierID): packetsToRequest.append((SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Data'))) for SupplierNumber in range(self.EccMap.paritysegments): SupplierID = contacts.getSupplierID(SupplierNumber) if not SupplierID: continue if not self.OnHandParity[SupplierNumber] and contact_status.isOnline(SupplierID): packetsToRequest.append((SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Parity'))) for SupplierID, packetID in packetsToRequest: io_throttle.QueueRequestFile( self.PacketCameIn, self.CreatorID, packetID, self.CreatorID, SupplierID) dhnio.Dprint(6, "restore.doRequestPackets requested %d packets for block %d" % (len(packetsToRequest), self.BlockNumber)) del packetsToRequest self.automat('request-done')
def rate_all_users(): lg.out(4, 'ratings.rate_all_users') monthStr = time.strftime('%B') for idurl in contactsdb.contacts_full(): isalive = contact_status.isOnline(idurl) mall, malive, tall, talive = increase_rating(idurl, isalive) month_percent = 100.0 * float(malive) / float(mall) total_percent = 100.0 * float(talive) / float(tall) lg.out(4, '[%6.2f%%: %s/%s] in %s and [%6.2f%%: %s/%s] total - %s' % ( month_percent, malive, mall, monthStr, total_percent, talive, tall, nameurl.GetName(idurl),)) read_index()
def rate_all_users(): dhnio.Dprint(4, 'ratings.rate_all_users') monthStr = time.strftime('%B') for idurl in contacts.getContactsAndCorrespondents(): #isalive = transport_control.ContactIsAlive(idurl) isalive = contact_status.isOnline(idurl) mall, malive, tall, talive = increase_rating(idurl, isalive) month_percent = 100.0*float(malive)/float(mall) total_percent = 100.0*float(talive)/float(tall) dhnio.Dprint(4, '[%6.2f%%: %s/%s] in %s and [%6.2f%%: %s/%s] total - %s' % ( month_percent, malive, mall, monthStr, total_percent, talive, tall, nameurl.GetName(idurl),)) read_index()
def doSuppliersSendDBInfo(self, arg): # dhnio.Dprint(4, 'backup_db_keeper.doSuppliersSendDBInfo') # packetID = settings.BackupInfoEncryptedFileName() packetID = settings.BackupIndexFileName() for supplierId in contacts.getSupplierIDs(): if supplierId: transport_control.RemoveInterest(supplierId, packetID) self.sentSuppliers.clear() # src = dhnio.ReadBinaryFile(settings.BackupInfoFileFullPath()) src = dhnio.ReadBinaryFile(settings.BackupIndexFilePath()) localID = misc.getLocalID() block = dhnblock.dhnblock(localID, packetID, 0, dhncrypto.NewSessionKey(), dhncrypto.SessionKeyType(), True, src) Payload = block.Serialize() for supplierId in contacts.getSupplierIDs(): if not supplierId: continue if not contact_status.isOnline(supplierId): continue newpacket = dhnpacket.dhnpacket(commands.Data(), localID, localID, packetID, Payload, supplierId) transport_control.outboxAck(newpacket) transport_control.RegisterInterest(self.SupplierAcked, supplierId, packetID) self.sentSuppliers.add(supplierId)
def WhoIsLost(): # if we have more than 50% data packets lost to someone and it was a long story - fire this guy # we check this first, because this is more important than other things. # many things can be a reason: slow connection, old code, network errors, timeout during sending # so if we can not send him our data or retreive it back - how can we do a backups to him even if he is online? unreliable_supplier = None most_fails = 0.0 for supplierNum in range(contacts.numSuppliers()): idurl = contacts.getSupplierID(supplierNum) if not idurl: continue if not data_sender.statistic().has_key(idurl): continue stats = data_sender.statistic()[idurl] total = stats[0] + stats[1] failed = stats[1] if total > 10: failed_percent = failed / total if failed_percent > 0.5: if most_fails < failed_percent: most_fails = failed_percent unreliable_supplier = idurl if unreliable_supplier: return 'found-one-lost-supplier', unreliable_supplier # we only fire offline suppliers offline_suppliers = {} # ask backup_monitor about current situation # check every offline supplier and see how many files he keep at the moment for supplierNum in range(contacts.numSuppliers()): idurl = contacts.getSupplierID(supplierNum) if not idurl: continue if contact_status.isOnline(idurl): continue blocks, total, stats = backup_matrix.GetSupplierStats(supplierNum) rating = 0 if total == 0 else blocks / total offline_suppliers[idurl] = rating # if all suppliers are online - we are very happy - no need to fire anybody! if len(offline_suppliers) == 0: dhnio.Dprint(4, 'fire_hire.WhoIsLost no offline suppliers, Cool!') return 'not-found-lost-suppliers', '' # sort users - we always fire worst supplier rating = offline_suppliers.keys() rating.sort(key=lambda idurl: offline_suppliers[idurl]) lost_supplier_idurl = rating[0] # we do not want to fire this man if he store at least 50% of our files # the fact that he is offline is not enough to fire him! if offline_suppliers[lost_supplier_idurl] < 0.5 and backup_fs.sizebackups() > 0: dhnio.Dprint(4, 'fire_hire.WhoIsLost !!!!!!!! %s is offline and keeps only %d%% of our data' % ( nameurl.GetName(lost_supplier_idurl), int(offline_suppliers[lost_supplier_idurl] * 100.0))) return 'found-one-lost-supplier', lost_supplier_idurl # but if we did not saw him for a long time - we do not want him for sure if time.time() - ratings.connected_time(lost_supplier_idurl) > 60 * 60 * 24 * 2: dhnio.Dprint(2, 'fire_hire.WhoIsLost !!!!!!!! %s is offline and keeps %d%% of our data, but he was online %d hours ago' % ( nameurl.GetName(lost_supplier_idurl), int(offline_suppliers[lost_supplier_idurl] * 100.0), int((time.time() - ratings.connected_time(lost_supplier_idurl)) * 60 * 60),)) return 'found-one-lost-supplier', lost_supplier_idurl dhnio.Dprint(2, 'fire_hire.WhoIsLost some people is not here, but we did not found the bad guy at this time') return 'not-found-lost-suppliers', ''