def doRequestRemoteFiles(self, arg):
global _RequestedListFilesCounter
global _RequestedListFilesPacketIDs
_RequestedListFilesCounter = 0
_RequestedListFilesPacketIDs.clear()
for idurl in contacts.getSupplierIDs():
if idurl:
if contact_status.isOnline(idurl):
p2p_service.RequestListFiles(idurl)
_RequestedListFilesPacketIDs.add(idurl)
def GetActiveArray(self):
"""
Loops all suppliers in the set and returns who is alive at the moment.
Return a list with integers: 0 for offline suppler and 1 if he is available right now.
Uses `p2p.contact_status.isOnline()` to see the current state of supplier.
"""
activeArray = [0] * self.supplierCount
for i in xrange(self.supplierCount):
if not self.suppliers[i]:
continue
if contact_status.isOnline(self.suppliers[i]):
activeArray[i] = 1
else:
activeArray[i] = 0
return activeArray
def doRequestPackets(self, arg):
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contacts.getSupplierID(SupplierNumber)
if not SupplierID:
continue
if not self.OnHandData[SupplierNumber] and contact_status.isOnline(SupplierID):
packetsToRequest.append((SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contacts.getSupplierID(SupplierNumber)
if not SupplierID:
continue
if not self.OnHandParity[SupplierNumber] and contact_status.isOnline(SupplierID):
packetsToRequest.append((SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Parity')))
for SupplierID, packetID in packetsToRequest:
io_throttle.QueueRequestFile(
self.PacketCameIn,
self.CreatorID,
packetID,
self.CreatorID,
SupplierID)
dhnio.Dprint(6, "restore.doRequestPackets requested %d packets for block %d" % (len(packetsToRequest), self.BlockNumber))
del packetsToRequest
self.automat('request-done')
def rate_all_users():
lg.out(4, 'ratings.rate_all_users')
monthStr = time.strftime('%B')
for idurl in contactsdb.contacts_full():
isalive = contact_status.isOnline(idurl)
mall, malive, tall, talive = increase_rating(idurl, isalive)
month_percent = 100.0 * float(malive) / float(mall)
total_percent = 100.0 * float(talive) / float(tall)
lg.out(4, '[%6.2f%%: %s/%s] in %s and [%6.2f%%: %s/%s] total - %s' % (
month_percent,
malive,
mall,
monthStr,
total_percent,
talive,
tall,
nameurl.GetName(idurl),))
read_index()
def rate_all_users():
dhnio.Dprint(4, 'ratings.rate_all_users')
monthStr = time.strftime('%B')
for idurl in contacts.getContactsAndCorrespondents():
#isalive = transport_control.ContactIsAlive(idurl)
isalive = contact_status.isOnline(idurl)
mall, malive, tall, talive = increase_rating(idurl, isalive)
month_percent = 100.0*float(malive)/float(mall)
total_percent = 100.0*float(talive)/float(tall)
dhnio.Dprint(4, '[%6.2f%%: %s/%s] in %s and [%6.2f%%: %s/%s] total - %s' % (
month_percent,
malive,
mall,
monthStr,
total_percent,
talive,
tall,
nameurl.GetName(idurl),))
read_index()
def doSuppliersSendDBInfo(self, arg):
# dhnio.Dprint(4, 'backup_db_keeper.doSuppliersSendDBInfo')
# packetID = settings.BackupInfoEncryptedFileName()
packetID = settings.BackupIndexFileName()
for supplierId in contacts.getSupplierIDs():
if supplierId:
transport_control.RemoveInterest(supplierId, packetID)
self.sentSuppliers.clear()
# src = dhnio.ReadBinaryFile(settings.BackupInfoFileFullPath())
src = dhnio.ReadBinaryFile(settings.BackupIndexFilePath())
localID = misc.getLocalID()
block = dhnblock.dhnblock(localID, packetID, 0, dhncrypto.NewSessionKey(), dhncrypto.SessionKeyType(), True, src)
Payload = block.Serialize()
for supplierId in contacts.getSupplierIDs():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket = dhnpacket.dhnpacket(commands.Data(), localID, localID, packetID, Payload, supplierId)
transport_control.outboxAck(newpacket)
transport_control.RegisterInterest(self.SupplierAcked, supplierId, packetID)
self.sentSuppliers.add(supplierId)
def WhoIsLost():
# if we have more than 50% data packets lost to someone and it was a long story - fire this guy
# we check this first, because this is more important than other things.
# many things can be a reason: slow connection, old code, network errors, timeout during sending
# so if we can not send him our data or retreive it back - how can we do a backups to him even if he is online?
unreliable_supplier = None
most_fails = 0.0
for supplierNum in range(contacts.numSuppliers()):
idurl = contacts.getSupplierID(supplierNum)
if not idurl:
continue
if not data_sender.statistic().has_key(idurl):
continue
stats = data_sender.statistic()[idurl]
total = stats[0] + stats[1]
failed = stats[1]
if total > 10:
failed_percent = failed / total
if failed_percent > 0.5:
if most_fails < failed_percent:
most_fails = failed_percent
unreliable_supplier = idurl
if unreliable_supplier:
return 'found-one-lost-supplier', unreliable_supplier
# we only fire offline suppliers
offline_suppliers = {}
# ask backup_monitor about current situation
# check every offline supplier and see how many files he keep at the moment
for supplierNum in range(contacts.numSuppliers()):
idurl = contacts.getSupplierID(supplierNum)
if not idurl:
continue
if contact_status.isOnline(idurl):
continue
blocks, total, stats = backup_matrix.GetSupplierStats(supplierNum)
rating = 0 if total == 0 else blocks / total
offline_suppliers[idurl] = rating
# if all suppliers are online - we are very happy - no need to fire anybody!
if len(offline_suppliers) == 0:
dhnio.Dprint(4, 'fire_hire.WhoIsLost no offline suppliers, Cool!')
return 'not-found-lost-suppliers', ''
# sort users - we always fire worst supplier
rating = offline_suppliers.keys()
rating.sort(key=lambda idurl: offline_suppliers[idurl])
lost_supplier_idurl = rating[0]
# we do not want to fire this man if he store at least 50% of our files
# the fact that he is offline is not enough to fire him!
if offline_suppliers[lost_supplier_idurl] < 0.5 and backup_fs.sizebackups() > 0:
dhnio.Dprint(4, 'fire_hire.WhoIsLost !!!!!!!! %s is offline and keeps only %d%% of our data' % (
nameurl.GetName(lost_supplier_idurl),
int(offline_suppliers[lost_supplier_idurl] * 100.0)))
return 'found-one-lost-supplier', lost_supplier_idurl
# but if we did not saw him for a long time - we do not want him for sure
if time.time() - ratings.connected_time(lost_supplier_idurl) > 60 * 60 * 24 * 2:
dhnio.Dprint(2, 'fire_hire.WhoIsLost !!!!!!!! %s is offline and keeps %d%% of our data, but he was online %d hours ago' % (
nameurl.GetName(lost_supplier_idurl),
int(offline_suppliers[lost_supplier_idurl] * 100.0),
int((time.time() - ratings.connected_time(lost_supplier_idurl)) * 60 * 60),))
return 'found-one-lost-supplier', lost_supplier_idurl
dhnio.Dprint(2, 'fire_hire.WhoIsLost some people is not here, but we did not found the bad guy at this time')
return 'not-found-lost-suppliers', ''
