def _send(index, payload, delay):
global _SlowSendIsWorking
idurl = contacts.getSupplierID(index)
if not idurl:
_SlowSendIsWorking = False
return
transport_control.ClearAliveTime(idurl)
SendToID(idurl, Payload=payload, wide=True)
reactor.callLater(delay, _send, index+1, payload, delay)
def doScanAndQueue(self, arg):
global _ShutdownFlag
dhnio.Dprint(10, 'data_sender.doScanAndQueue')
log = open(os.path.join(settings.LogsDir(), 'data_sender.log'), 'w')
log.write('doScanAndQueue %s\n' % time.asctime())
if _ShutdownFlag:
log.write('doScanAndQueue _ShutdownFlag is True\n')
self.automat('scan-done')
log.flush()
log.close()
return
if '' not in contacts.getSupplierIDs():
for backupID in misc.sorted_backup_ids(backup_matrix.local_files().keys(), True):
packetsBySupplier = backup_matrix.ScanBlocksToSend(backupID)
log.write('%s\n' % packetsBySupplier)
for supplierNum in packetsBySupplier.keys():
supplier_idurl = contacts.getSupplierID(supplierNum)
if not supplier_idurl:
dhnio.Dprint(2, 'data_sender.doScanAndQueue WARNING ?supplierNum? %s for %s' % (supplierNum, backupID))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, blockNum, supplierNum_, dataORparity = packetid.BidBnSnDp(packetID)
if backupID_ != backupID:
dhnio.Dprint(2, 'data_sender.doScanAndQueue WARNING ?backupID? %s for %s' % (packetID, backupID))
continue
if supplierNum_ != supplierNum:
dhnio.Dprint(2, 'data_sender.doScanAndQueue WARNING ?supplierNum? %s for %s' % (packetID, backupID))
continue
if io_throttle.HasPacketInSendQueue(supplier_idurl, packetID):
log.write('%s in the send queue to %s\n' % (packetID, supplier_idurl))
continue
if not io_throttle.OkToSend(supplier_idurl):
log.write('ok to send %s ? - NO!\n' % supplier_idurl)
continue
tranByiID = transport_control.transfers_by_idurl(supplier_idurl)
if len(tranByiID) > 3:
log.write('transfers by %s: %d\n' % (supplier_idurl, len(tranByiID)))
continue
filename = os.path.join(settings.getLocalBackupsDir(), packetID)
if not os.path.isfile(filename):
log.write('%s is not file\n' % filename)
continue
io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
misc.getLocalID(),
self._packetAcked,
self._packetFailed)
log.write('io_throttle.QueueSendFile %s\n' % packetID)
# dhnio.Dprint(6, ' %s for %s' % (packetID, backupID))
self.automat('scan-done')
log.flush()
log.close()
def SendChangeSupplier(numORidurl, newidurl, doAck=False):
"""
Send "FireContact" packet to Central server,
this is to change one supplier with another, by your choice.
"""
if isinstance(numORidurl, str):
idurl = numORidurl
else:
idurl = contacts.getSupplierID(numORidurl)
if not idurl or not newidurl:
return None
dhnio.Dprint(4, "central_service.SendChangeSupplier [%s]->[%s]" % (nameurl.GetName(idurl), nameurl.GetName(newidurl)))
data = 'N\n'+idurl+'\n'+newidurl
ret = send2central(commands.FireContact(), data, doAck)
def RequestListFiles(supplierNumORidurl):
if isinstance(supplierNumORidurl, str):
RemoteID = supplierNumORidurl
else:
RemoteID = contacts.getSupplierID(supplierNumORidurl)
if not RemoteID:
dhnio.Dprint(4, "p2p_service.RequestListFiles WARNING RemoteID is empty supplierNumORidurl=%s" % str(supplierNumORidurl))
return
dhnio.Dprint(8, "p2p_service.RequestListFiles [%s]" % nameurl.GetName(RemoteID))
MyID = misc.getLocalID()
PacketID = packetid.UniqueID()
Payload = settings.ListFilesFormat()
result = dhnpacket.dhnpacket(commands.ListFiles(), MyID, MyID, PacketID, Payload, RemoteID)
transport_control.outboxNoAck(result)
return PacketID
def doRequestPackets(self, arg):
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contacts.getSupplierID(SupplierNumber)
if not SupplierID:
continue
if not self.OnHandData[SupplierNumber] and contact_status.isOnline(SupplierID):
packetsToRequest.append((SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contacts.getSupplierID(SupplierNumber)
if not SupplierID:
continue
if not self.OnHandParity[SupplierNumber] and contact_status.isOnline(SupplierID):
packetsToRequest.append((SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Parity')))
for SupplierID, packetID in packetsToRequest:
io_throttle.QueueRequestFile(
self.PacketCameIn,
self.CreatorID,
packetID,
self.CreatorID,
SupplierID)
dhnio.Dprint(6, "restore.doRequestPackets requested %d packets for block %d" % (len(packetsToRequest), self.BlockNumber))
del packetsToRequest
self.automat('request-done')
def SendReplaceSupplier(numORidurl, doAck=False):
"""
Send "FireContact" packet to Central server,
this will replace given supplier with a random guy.
"""
if isinstance(numORidurl, str):
idurl = numORidurl
else:
idurl = contacts.getSupplierID(numORidurl)
if not idurl:
dhnio.Dprint(2, "central_service.SendReplaceSupplier ERROR supplier not found")
return None
dhnio.Dprint(4, "central_service.SendReplaceSupplier [%s]" % nameurl.GetName(idurl))
data = 'S\n'+idurl+'\n'+str(contacts.numberForSupplier(idurl))
ret = send2central(commands.FireContact(), data, doAck)
events.notify('central_service', 'sent request to dismiss supplier %s' % nameurl.GetName(idurl))
return ret
def OneFromList(filename):
WholeFile=dhnio.ReadBinaryFile(filename)
FileList=WholeFile.split("\n")
num=len(FileList)
dhnio.Dprint(7, "supplierpatrol.OneFromList number of items is " + str(num))
rnd=random.randint(0,num-1)
item=FileList[rnd]
command=commands.Data()
OwnerID=misc.getLocalID()
CreatorID=misc.getLocalID()
PacketID=item
Payload=""
supnum=packetid.SupplierNumber(item)
RemoteID=contacts.getSupplierID(supnum)
request=dhnpacket.dhnpacket(command, OwnerID, CreatorID, PacketID, Payload, RemoteID)
global DataResultsOutstanding
DataResultsOutstanding.append(item)
transport_control.RegisterInterest(DataResult, RemoteID, PacketID)
transport_control.outboxAck(request)
def UpdateListFiles():
if (not os.path.exists(settings.FileListDir())):
os.mkdir(settings.FileListDir())
for supnum in range(0, contacts.numSuppliers()):
filename= os.path.join(settings.FileListDir(), str(supnum))
dhnio.Dprint(7, "supplierpatrol.UpdateListFiles looking at = " + filename)
if (not os.path.exists(filename) or (fileAgeInSeconds(filename) > 3600*24)):
dhnio.Dprint(7, "supplierpatrol.UpdateListFiles found one to update " + filename)
command=commands.ListFiles()
OwnerID=misc.getLocalID()
CreatorID=misc.getLocalID()
PacketID="ListFiles" + str(supnum)
Payload=""
RemoteID= contacts.getSupplierID(supnum)
request=dhnpacket.dhnpacket(command, OwnerID, CreatorID, PacketID, Payload, RemoteID)
transport_control.RegisterInterest(ListResult, RemoteID, PacketID)
transport_control.outboxAck(request)
global NumRequestsOutstanding
NumRequestsOutstanding += 1
dhnio.Dprint(7, "supplierpatrol.UpdateListFiles sent request - now outstanding=" + str(NumRequestsOutstanding))
def WhoIsLost():
# if we have more than 50% data packets lost to someone and it was a long story - fire this guy
# we check this first, because this is more important than other things.
# many things can be a reason: slow connection, old code, network errors, timeout during sending
# so if we can not send him our data or retreive it back - how can we do a backups to him even if he is online?
unreliable_supplier = None
most_fails = 0.0
for supplierNum in range(contacts.numSuppliers()):
idurl = contacts.getSupplierID(supplierNum)
if not idurl:
continue
if not data_sender.statistic().has_key(idurl):
continue
stats = data_sender.statistic()[idurl]
total = stats[0] + stats[1]
failed = stats[1]
if total > 10:
failed_percent = failed / total
if failed_percent > 0.5:
if most_fails < failed_percent:
most_fails = failed_percent
unreliable_supplier = idurl
if unreliable_supplier:
return 'found-one-lost-supplier', unreliable_supplier
# we only fire offline suppliers
offline_suppliers = {}
# ask backup_monitor about current situation
# check every offline supplier and see how many files he keep at the moment
for supplierNum in range(contacts.numSuppliers()):
idurl = contacts.getSupplierID(supplierNum)
if not idurl:
continue
if contact_status.isOnline(idurl):
continue
blocks, total, stats = backup_matrix.GetSupplierStats(supplierNum)
rating = 0 if total == 0 else blocks / total
offline_suppliers[idurl] = rating
# if all suppliers are online - we are very happy - no need to fire anybody!
if len(offline_suppliers) == 0:
dhnio.Dprint(4, 'fire_hire.WhoIsLost no offline suppliers, Cool!')
return 'not-found-lost-suppliers', ''
# sort users - we always fire worst supplier
rating = offline_suppliers.keys()
rating.sort(key=lambda idurl: offline_suppliers[idurl])
lost_supplier_idurl = rating[0]
# we do not want to fire this man if he store at least 50% of our files
# the fact that he is offline is not enough to fire him!
if offline_suppliers[lost_supplier_idurl] < 0.5 and backup_fs.sizebackups() > 0:
dhnio.Dprint(4, 'fire_hire.WhoIsLost !!!!!!!! %s is offline and keeps only %d%% of our data' % (
nameurl.GetName(lost_supplier_idurl),
int(offline_suppliers[lost_supplier_idurl] * 100.0)))
return 'found-one-lost-supplier', lost_supplier_idurl
# but if we did not saw him for a long time - we do not want him for sure
if time.time() - ratings.connected_time(lost_supplier_idurl) > 60 * 60 * 24 * 2:
dhnio.Dprint(2, 'fire_hire.WhoIsLost !!!!!!!! %s is offline and keeps %d%% of our data, but he was online %d hours ago' % (
nameurl.GetName(lost_supplier_idurl),
int(offline_suppliers[lost_supplier_idurl] * 100.0),
int((time.time() - ratings.connected_time(lost_supplier_idurl)) * 60 * 60),))
return 'found-one-lost-supplier', lost_supplier_idurl
dhnio.Dprint(2, 'fire_hire.WhoIsLost some people is not here, but we did not found the bad guy at this time')
return 'not-found-lost-suppliers', ''
def cmd_suppliers(opts, args, overDict):
def _wait_replace_supplier_and_stop(src, supplier_name, count=0):
suppliers = []
for s in find_comments(src):
if s.count('[online ]') or s.count('[offline]'):
suppliers.append(s[18:38].strip())
if supplier_name not in suppliers:
print ' supplier %s is fired !' % supplier_name
print_and_stop(src)
return
if count >= 20:
print ' time is out\n'
reactor.stop()
return
else:
def _check_again(supplier_name, count):
sys.stdout.write('.')
run_url_command(webcontrol._PAGE_SUPPLIERS).addCallback(_wait_replace_supplier_and_stop, supplier_name, count)
reactor.callLater(1, _check_again, supplier_name, count+1)
if len(args) < 2 or args[1] in [ 'list', 'ls' ]:
url = webcontrol._PAGE_SUPPLIERS
run_url_command(url).addCallback(print_and_stop)
reactor.run()
return 0
elif args[1] in [ 'call', 'cl' ]:
url = webcontrol._PAGE_SUPPLIERS + '?action=call'
run_url_command(url).addCallback(print_and_stop)
reactor.run()
return 0
elif args[1] in [ 'replace', 'rep', 'rp' ] and len(args) >= 3:
contacts.init()
idurl = args[2].strip()
if not idurl.startswith('http://'):
try:
idurl = contacts.getSupplierID(int(idurl))
except:
idurl = 'http://'+settings.IdentityServerName()+'/'+idurl+'.xml'
if not idurl:
print 'supplier IDURL is None\n'
return 0
name = nameurl.GetName(idurl)
url = webcontrol._PAGE_SUPPLIERS + '?action=replace&idurl=%s' % misc.pack_url_param(idurl)
run_url_command(url).addCallback(_wait_replace_supplier_and_stop, name, 0)
reactor.run()
return 0
elif args[1] in [ 'change', 'ch' ] and len(args) >= 4:
contacts.init()
idurl = args[2].strip()
if not idurl.startswith('http://'):
try:
idurl = contacts.getSupplierID(int(idurl))
except:
idurl = 'http://'+settings.IdentityServerName()+'/'+idurl+'.xml'
if not idurl:
print 'supplier IDURL is None\n'
return 0
newidurl = args[3].strip()
if not newidurl.startswith('http://'):
newidurl = 'http://'+settings.IdentityServerName()+'/'+newidurl+'.xml'
name = nameurl.GetName(idurl)
newname = nameurl.GetName(newidurl)
url = webcontrol._PAGE_SUPPLIERS + '?action=change&idurl=%s&newidurl=%s' % (misc.pack_url_param(idurl), misc.pack_url_param(newidurl))
run_url_command(url).addCallback(_wait_replace_supplier_and_stop, name, 0)
reactor.run()
return 0
return 2