def hasOfflineSuppliers():
"""
Loops all suppliers and check their state, return True if at least one is OFFLINE.
"""
for idurl in contacts.getSupplierIDs():
if isOffline(idurl):
return True
return False
def isAllSuppliersResponded(self, arg):
onlines = contact_status.countOnlineAmong(contacts.getSupplierIDs())
# dhnio.Dprint(6, 'backup_monitor.isAllSuppliersResponded ackCounter=%d onlines=%d' % (self.ackCounter, onlines))
if self.ackCounter == contacts.numSuppliers():
return True
if self.ackCounter >= onlines - 1:
return True
return False
def doSuppliersRequestDBInfo(self, arg):
# dhnio.Dprint(4, 'backup_db_keeper.doSuppliersRequestDBInfo')
# packetID_ = settings.BackupInfoFileName()
# packetID = settings.BackupInfoEncryptedFileName()
packetID = settings.BackupIndexFileName()
for supplierId in contacts.getSupplierIDs():
if supplierId:
transport_control.RemoveInterest(supplierId, packetID)
self.requestedSuppliers.clear()
Payload = ''
localID = misc.getLocalID()
for supplierId in contacts.getSupplierIDs():
if not supplierId:
continue
newpacket = dhnpacket.dhnpacket(commands.Retrieve(), localID, localID, packetID, Payload, supplierId)
transport_control.outboxAck(newpacket)
transport_control.RegisterInterest(self.SupplierResponse, supplierId, packetID)
self.requestedSuppliers.add(supplierId)
def doRequestRemoteFiles(self, arg):
global _RequestedListFilesCounter
global _RequestedListFilesPacketIDs
_RequestedListFilesCounter = 0
_RequestedListFilesPacketIDs.clear()
for idurl in contacts.getSupplierIDs():
if idurl:
if contact_status.isOnline(idurl):
p2p_service.RequestListFiles(idurl)
_RequestedListFilesPacketIDs.add(idurl)
def suppliers_set():
"""
Return current set of suppliers, see `SuppliersSet` class.
PREPRO: Not sure do we need to duplicate suppliers IDs.
In `lib.contactsdb` we already store this list.
However scrubbers need to keep track of other suppliers also ...
"""
global _SuppliersSet
if _SuppliersSet is None:
_SuppliersSet = SuppliersSet(contacts.getSupplierIDs())
return _SuppliersSet
def doScanAndQueue(self, arg):
global _ShutdownFlag
dhnio.Dprint(10, 'data_sender.doScanAndQueue')
log = open(os.path.join(settings.LogsDir(), 'data_sender.log'), 'w')
log.write('doScanAndQueue %s\n' % time.asctime())
if _ShutdownFlag:
log.write('doScanAndQueue _ShutdownFlag is True\n')
self.automat('scan-done')
log.flush()
log.close()
return
if '' not in contacts.getSupplierIDs():
for backupID in misc.sorted_backup_ids(backup_matrix.local_files().keys(), True):
packetsBySupplier = backup_matrix.ScanBlocksToSend(backupID)
log.write('%s\n' % packetsBySupplier)
for supplierNum in packetsBySupplier.keys():
supplier_idurl = contacts.getSupplierID(supplierNum)
if not supplier_idurl:
dhnio.Dprint(2, 'data_sender.doScanAndQueue WARNING ?supplierNum? %s for %s' % (supplierNum, backupID))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, blockNum, supplierNum_, dataORparity = packetid.BidBnSnDp(packetID)
if backupID_ != backupID:
dhnio.Dprint(2, 'data_sender.doScanAndQueue WARNING ?backupID? %s for %s' % (packetID, backupID))
continue
if supplierNum_ != supplierNum:
dhnio.Dprint(2, 'data_sender.doScanAndQueue WARNING ?supplierNum? %s for %s' % (packetID, backupID))
continue
if io_throttle.HasPacketInSendQueue(supplier_idurl, packetID):
log.write('%s in the send queue to %s\n' % (packetID, supplier_idurl))
continue
if not io_throttle.OkToSend(supplier_idurl):
log.write('ok to send %s ? - NO!\n' % supplier_idurl)
continue
tranByiID = transport_control.transfers_by_idurl(supplier_idurl)
if len(tranByiID) > 3:
log.write('transfers by %s: %d\n' % (supplier_idurl, len(tranByiID)))
continue
filename = os.path.join(settings.getLocalBackupsDir(), packetID)
if not os.path.isfile(filename):
log.write('%s is not file\n' % filename)
continue
io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
misc.getLocalID(),
self._packetAcked,
self._packetFailed)
log.write('io_throttle.QueueSendFile %s\n' % packetID)
# dhnio.Dprint(6, ' %s for %s' % (packetID, backupID))
self.automat('scan-done')
log.flush()
log.close()
def ReadLatestRawListFiles():
"""
Call `ReadRawListFiles()` for every local file we have on hands and build whole "remote" matrix.
"""
dhnio.Dprint(4, 'backup_matrix.ReadLatestRawListFiles')
for idurl in contacts.getSupplierIDs():
if idurl:
filename = os.path.join(settings.SupplierPath(idurl, 'listfiles'))
if os.path.isfile(filename):
listFileText = dhnio.ReadTextFile(filename)
if listFileText.strip() != '':
ReadRawListFiles(contacts.numberForSupplier(idurl), listFileText)
def RequestDeleteListPaths(pathIDs):
dhnio.Dprint(4, "p2p_service.RequestDeleteListPaths wish to delete %d paths" % len(pathIDs))
for supplier in contacts.getSupplierIDs():
if not supplier:
continue
found = False
for workitem in transport_control.SendQueue():
if workitem.command == commands.DeleteFile() and workitem.remoteid == supplier:
found = True
break
if found:
continue
SendDeleteListPaths(supplier, pathIDs)
def doSuppliersSendDBInfo(self, arg):
# dhnio.Dprint(4, 'backup_db_keeper.doSuppliersSendDBInfo')
# packetID = settings.BackupInfoEncryptedFileName()
packetID = settings.BackupIndexFileName()
for supplierId in contacts.getSupplierIDs():
if supplierId:
transport_control.RemoveInterest(supplierId, packetID)
self.sentSuppliers.clear()
# src = dhnio.ReadBinaryFile(settings.BackupInfoFileFullPath())
src = dhnio.ReadBinaryFile(settings.BackupIndexFilePath())
localID = misc.getLocalID()
block = dhnblock.dhnblock(localID, packetID, 0, dhncrypto.NewSessionKey(), dhncrypto.SessionKeyType(), True, src)
Payload = block.Serialize()
for supplierId in contacts.getSupplierIDs():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket = dhnpacket.dhnpacket(commands.Data(), localID, localID, packetID, Payload, supplierId)
transport_control.outboxAck(newpacket)
transport_control.RegisterInterest(self.SupplierAcked, supplierId, packetID)
self.sentSuppliers.add(supplierId)
def RequestDeleteBackup(BackupID):
dhnio.Dprint(4, "p2p_service.RequestDeleteBackup with BackupID=" + str(BackupID))
for supplier in contacts.getSupplierIDs():
if not supplier:
continue
prevItems = transport_control.SendQueueSearch(BackupID)
found = False
for workitem in prevItems:
if workitem.remoteid == supplier:
found = True
break
if found:
continue
SendDeleteBackup(supplier, BackupID)
def doPingAllSuppliers(self, arg):
# check our suppliers first, if we do not have enough yet - do request
if '' in contacts.getSupplierIDs():
dhnio.Dprint(4, 'backup_monitor.doPingAllSuppliers found empty suppliers !!!!!!!!!!!!!!')
self.ackCounter = contacts.numSuppliers()
if time.time() - self.lastRequestSuppliersTime > 10 * 60:
central_service.SendRequestSuppliers()
self.lastRequestSuppliersTime = time.time()
return
# do not want to ping very often
if time.time() - self.pingTime < 60 * 3:
self.ackCounter = contacts.numSuppliers()
return
self.pingTime = time.time()
self.ackCounter = 0
def increaseAckCounter(packet):
self.ackCounter += 1
dhnio.Dprint(6, 'backup_monitor.doPingAllSuppliers going to call suppliers')
identitypropagate.suppliers(increaseAckCounter, True)
def ListContacts(request):
"""
Called when "ListContacts" packet is received,
it keeps a list of suppliers OR customers and some extra info.
I think this is a most important method here, call different code from here.
"""
global _CentralStatusDict
global legal_codes
global OnListSuppliersFunc
global OnListCustomersFunc
data = request.Payload
dhnio.Dprint(6, 'central_service.ListContacts\n%s' % data)
words = data.split('\n', 1)
if len(words) < 2:
dhnio.Dprint(1, 'central_service.ListContacts ERROR wrong data packet [%s]' % str(request.Payload))
return
code = words[0]
if not code in legal_codes:
dhnio.Dprint(1, 'central_service.ListContacts ERROR wrong data in the packet [%s] ' % str(request.Payload))
return
current_contacts = []
clist, tail = dhnio._unpack_list(words[1])
fire_flag = code.startswith('f')
ban_flag = code.startswith('b')
contact_type = code.lower()[-1]
error_flag = code[-1].islower()
spaceDict = None
if tail is not None:
# extract info about who is alive at the moment
onlineArray = ''
spaceDict = dhnio._unpack_dict_from_list(tail)
if spaceDict.has_key('online'):
onlineArray = spaceDict.pop('online')
for i in range(len(onlineArray)):
if i < len(clist):
if clist[i]:
_CentralStatusDict[clist[i]] = onlineArray[i]
# extract info about contacts local ip
# if they are in same LAN we need to connect to local IP, not external
local_ips = {}
i = 0
while True:
idurl_and_local_ip = spaceDict.get('localip%03d' % i, None)
if idurl_and_local_ip is None:
break
try:
contact_idurl, contact_local_ip = idurl_and_local_ip.split('|')
except:
break
local_ips[contact_idurl] = contact_local_ip
spaceDict.pop('localip%03d' % i)
i += 1
dhnio.Dprint(6, 'central_service.ListContacts got local IP for %s: %s' % (nameurl.GetName(contact_idurl), contact_local_ip))
identitycache.SetLocalIPs(local_ips)
#---suppliers
if contact_type == 's':
current_contacts = contacts.getSupplierIDs()
contacts.setSupplierIDs(clist)
eccmap.init()
contacts.saveSupplierIDs()
for supid in contacts.getSupplierIDs():
if supid.strip() == '':
error_flag = True
break
dhnio.Dprint(4, "central_service.ListContacts (SUPPLIERS) code:%s error:%s length:%d" % (str(code), str(error_flag), len(clist)))
for oldidurl in current_contacts:
if oldidurl:
if oldidurl not in clist:
events.info('central_service', 'supplier %s were disconnected' % nameurl.GetName(oldidurl),)
misc.writeSupplierData(oldidurl, 'disconnected', time.strftime('%d%m%y %H:%M:%S'))
dhnio.Dprint(6, 'central_service.ListContacts supplier %s were disconnected' % nameurl.GetName(oldidurl))
for newidurl in clist:
if newidurl:
if newidurl not in current_contacts:
transport_control.ClearAliveTime(newidurl)
misc.writeSupplierData(newidurl, 'connected', time.strftime('%d%m%y %H:%M:%S'))
events.info('central_service', 'new supplier %s connected' % nameurl.GetName(newidurl), '',)
dhnio.Dprint(6, 'central_service.ListContacts new supplier %s connected' % nameurl.GetName(newidurl))
backup_control.SetSupplierList(clist)
if not fire_flag and current_contacts != clist:
dhnio.Dprint(6, 'central_service.ListContacts going to call suppliers')
identitypropagate.suppliers(wide=True)
if OnListSuppliersFunc is not None:
OnListSuppliersFunc()
#---customers
elif contact_type == 'c':
current_contacts = contacts.getCustomerIDs()
contacts.setCustomerIDs(clist)
contacts.saveCustomerIDs()
if spaceDict is not None:
dhnio._write_dict(settings.CustomersSpaceFile(), spaceDict)
reactor.callLater(3, local_tester.TestUpdateCustomers)
# if not fire_flag and current_contacts != clist:
identitypropagate.customers(wide=True)
dhnio.Dprint(4, "central_service.ListContacts (CUSTOMERS) code:%s error:%s length:%d" % (str(code), str(error_flag), len(clist)))
for oldidurl in current_contacts:
if oldidurl not in clist:
events.info('central_service', 'customer %s were disconnected' % nameurl.GetName(oldidurl),)
dhnio.Dprint(6, 'central_service.ListContacts customer %s were disconnected' % nameurl.GetName(oldidurl))
for newidurl in clist:
if newidurl not in current_contacts:
transport_control.ClearAliveTime(newidurl)
events.info('central_service', 'new customer %s connected' % nameurl.GetName(newidurl))
dhnio.Dprint(6, 'central_service.ListContacts new customer %s connected' % nameurl.GetName(newidurl))
if OnListCustomersFunc is not None:
OnListCustomersFunc()
#---fire_flag
if fire_flag:
if contact_type == 's':
index = -1
for index in range(len(clist)):
if clist[index] != current_contacts[index]:
break
if index >= 0:
# we want to send our Identity to new supplier
# and than ask a list of files he have
# so it should start rebuilding backups immediately
# right after we got ListFiles from him
identitypropagate.single(clist[index], wide=True)
#---ban_flag
if ban_flag:
events.notify('central_service', 'you have negative balance, all your suppliers was removed', '',)
dhnio.Dprint(2, 'central_service.ListContacts !!! you have negative balance, all your suppliers was removed !!!')
#---error_flag
if error_flag:
#reactor.callLater(settings.DefaultNeedSuppliersPacketTimeOut(), SendRequestSuppliers)
events.info('central_service', 'could not find available suppliers',
'Central server can not find available suppliers for you.\nCheck your central settings.\n',)
dhnio.Dprint(2, 'central_service.ListContacts !!! could not find available suppliers !!!')
#---send ack
transport_control.SendAck(request)
#---automats
if contact_type == 's':
central_connector.A('list-suppliers', clist)
fire_hire.A('list-suppliers', (current_contacts, clist))
data_sender.A('restart')
elif contact_type == 'c':
central_connector.A('list-customers', clist)
#---transport_udp
if transport_control._TransportUDPEnable:
import lib.transport_udp as transport_udp
new_contacts = contacts.getContactsAndCorrespondents()
transport_udp.ListContactsCallback(current_contacts, new_contacts)
def suppliers(AckHandler=None, wide=False):
dhnio.Dprint(6, 'identitypropagate.suppliers')
return propagate(contacts.getSupplierIDs(), AckHandler, wide)
def RealSendSuppliers():
dhnio.Dprint(8, "identitypropagate.RealSendSuppliers")
SendToIDs(contacts.getSupplierIDs(), HandleSuppliersAck)
def FetchSuppliers():
return fetch(contacts.getSupplierIDs())
def doRequestAvailableBlocks(self, arg):
self.missingPackets = 0
# self.missingSuppliers.clear()
# here we want to request some packets before we start working to rebuild the missed blocks
supplierSet = backup_matrix.suppliers_set()
availableSuppliers = supplierSet.GetActiveArray()
# remember how many requests we did on this iteration
total_requests_count = 0
# at the moment I do download everything I have available and needed
if '' in contacts.getSupplierIDs():
self.automat('requests-sent', total_requests_count)
return
for supplierNum in range(supplierSet.supplierCount):
supplierID = supplierSet.suppliers[supplierNum]
requests_count = 0
# we do requests in reverse order because we start rebuilding from the last block
# for blockNum in range(self.currentBlockNumber, -1, -1):
for blockIndex in range(len(self.workingBlocksQueue)-1, -1, -1):
blockNum = self.workingBlocksQueue[blockIndex]
# do not keep too many requests in the queue
if io_throttle.GetRequestQueueLength(supplierID) >= 16:
break
# also don't do too many requests at once
if requests_count > 16:
break
remoteData = backup_matrix.GetRemoteDataArray(self.currentBackupID, blockNum)
remoteParity = backup_matrix.GetRemoteParityArray(self.currentBackupID, blockNum)
localData = backup_matrix.GetLocalDataArray(self.currentBackupID, blockNum)
localParity = backup_matrix.GetLocalParityArray(self.currentBackupID, blockNum)
# if the remote Data exist and is available because supplier is on line,
# but we do not have it on hand - do request
if localData[supplierNum] == 0:
PacketID = packetid.MakePacketID(self.currentBackupID, blockNum, supplierNum, 'Data')
if remoteData[supplierNum] == 1:
if availableSuppliers[supplierNum]:
# if supplier is not alive - we can't request from him
if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID):
io_throttle.QueueRequestFile(
self.FileReceived,
misc.getLocalID(),
PacketID,
misc.getLocalID(),
supplierID)
requests_count += 1
else:
# count this packet as missing
self.missingPackets += 1
# also mark this guy as one who dont have any data - nor local nor remote
# self.missingSuppliers.add(supplierNum)
# same for Parity
if localParity[supplierNum] == 0:
PacketID = packetid.MakePacketID(self.currentBackupID, blockNum, supplierNum, 'Parity')
if remoteParity[supplierNum] == 1:
if availableSuppliers[supplierNum]:
if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID):
io_throttle.QueueRequestFile(
self.FileReceived,
misc.getLocalID(),
PacketID,
misc.getLocalID(),
supplierID)
requests_count += 1
else:
self.missingPackets += 1
# self.missingSuppliers.add(supplierNum)
total_requests_count += requests_count
self.automat('requests-sent', total_requests_count)
