def isRequestQueueEmpty(self, arg):
supplierSet = backup_matrix.suppliers_set()
for supplierNum in range(supplierSet.supplierCount):
supplierID = supplierSet.suppliers[supplierNum]
if io_throttle.HasBackupIDInSendQueue(supplierID, self.currentBackupID):
return False
return True
def doPrepareNextBackup(self, arg):
global _BackupIDsQueue
# clear block number from previous iteration
self.currentBlockNumber = -1
# check it, may be we already fixed all things
if len(_BackupIDsQueue) == 0:
self.workingBlocksQueue = []
self.automat('backup-ready')
return
# take a first backup from queue to work on it
backupID = _BackupIDsQueue.pop(0)
# if remote data structure is not exist for this backup - create it
# this mean this is only local backup!
if not backup_matrix.remote_files().has_key(backupID):
backup_matrix.remote_files()[backupID] = {}
# we create empty remote info for every local block
# range(0) should return []
for blockNum in range(backup_matrix.local_max_block_numbers().get(backupID, -1) + 1):
backup_matrix.remote_files()[backupID][blockNum] = {
'D': [0] * backup_matrix.suppliers_set().supplierCount,
'P': [0] * backup_matrix.suppliers_set().supplierCount }
# detect missing blocks from remote info
self.workingBlocksQueue = backup_matrix.ScanMissingBlocks(backupID)
dhnio.Dprint(8, 'backup_rebuilder.doPrepareNextBackup [%s] working blocks: %s' % (backupID, str(self.workingBlocksQueue)))
# find the correct max block number for this backup
# we can have remote and local files
# will take biggest block number from both
backupMaxBlock = max(backup_matrix.remote_max_block_numbers().get(backupID, -1),
backup_matrix.local_max_block_numbers().get(backupID, -1))
# now need to remember this biggest block number
# remote info may have less blocks - need to create empty info for missing blocks
for blockNum in range(backupMaxBlock + 1):
if backup_matrix.remote_files()[backupID].has_key(blockNum):
continue
backup_matrix.remote_files()[backupID][blockNum] = {
'D': [0] * backup_matrix.suppliers_set().supplierCount,
'P': [0] * backup_matrix.suppliers_set().supplierCount }
if self.currentBackupID:
# clear requesting queue from previous task
io_throttle.DeleteBackupRequests(self.currentBackupID)
# really take the next backup
self.currentBackupID = backupID
# clear requesting queue, remove old packets for this backup, we will send them again
io_throttle.DeleteBackupRequests(self.currentBackupID)
# dhnio.Dprint(6, 'backup_rebuilder.doTakeNextBackup currentBackupID=%s workingBlocksQueue=%d' % (self.currentBackupID, len(self.workingBlocksQueue)))
self.automat('backup-ready')
def isChanceToRebuild(self, arg):
# return len(self.missingSuppliers) <= eccmap.Current().CorrectableErrors
supplierSet = backup_matrix.suppliers_set()
# start checking in reverse order, see below for explanation
for blockIndex in range(len(self.workingBlocksQueue)-1, -1, -1):
blockNumber = self.workingBlocksQueue[blockIndex]
if eccmap.Current().CanMakeProgress(
backup_matrix.GetLocalDataArray(self.currentBackupID, blockNumber),
backup_matrix.GetLocalParityArray(self.currentBackupID, blockNumber)):
return True
return False
def _prepare_one_block():
if self.blockIndex < 0:
# dhnio.Dprint(8, ' _prepare_one_block finish all blocks')
reactor.callLater(0, _finish_all_blocks)
return
self.currentBlockNumber = self.workingBlocksQueue[self.blockIndex]
# dhnio.Dprint(8, ' _prepare_one_block %d to rebuild' % self.currentBlockNumber)
self.workBlock = block_rebuilder.BlockRebuilder(
eccmap.Current(), #self.eccMap,
self.currentBackupID,
self.currentBlockNumber,
backup_matrix.suppliers_set(),
backup_matrix.GetRemoteDataArray(self.currentBackupID, self.currentBlockNumber),
backup_matrix.GetRemoteParityArray(self.currentBackupID, self.currentBlockNumber),
backup_matrix.GetLocalDataArray(self.currentBackupID, self.currentBlockNumber),
backup_matrix.GetLocalParityArray(self.currentBackupID, self.currentBlockNumber),)
reactor.callLater(0, _identify_block_packets)
def SetSupplierList(supplierList):
"""
Set a list of suppliers IDs, this is called by `p2p.central_service`
when a list of my suppliers comes from Central server.
Going from 2 to 4 suppliers (or whatever) invalidates all backups,
all suppliers was changed because its number was changed.
So we lost everything! Definitely suppliers number should be a sort of constant number.
"""
if len(supplierList) != backup_matrix.suppliers_set().supplierCount:
dhnio.Dprint(2, "backup_control.SetSupplierList got list of %d suppliers, but we have %d now!" % (len(supplierList), backup_matrix.suppliers_set().supplierCount))
# cancel all tasks and jobs
DeleteAllTasks()
AbortAllRunningBackups()
# remove all local files and all backups
DeleteAllBackups()
# erase all remote info
backup_matrix.ClearRemoteInfo()
# also erase local info
backup_matrix.ClearLocalInfo()
# restart backup_monitor
backup_monitor.Restart()
# restart db keeper to save the index on new machines
backup_db_keeper.A('restart')
# only single suppliers changed
# need to erase info only for them
elif backup_matrix.suppliers_set().SuppliersChanged(supplierList):
# take a list of suppliers positions that was changed
changedSupplierNums = backup_matrix.suppliers_set().SuppliersChangedNumbers(supplierList)
# notify io_throttle that we do not neeed already this suppliers
for supplierNum in changedSupplierNums:
dhnio.Dprint(2, "backup_control.SetSupplierList supplier %d changed: [%s]->[%s]" % (
supplierNum, nameurl.GetName(backup_matrix.suppliers_set().suppliers[supplierNum]), nameurl.GetName(supplierList[supplierNum])))
io_throttle.DeleteSuppliers([backup_matrix.suppliers_set().suppliers[supplierNum],])
# erase (set to 0) remote info for this guys
backup_matrix.ClearSupplierRemoteInfo(supplierNum)
# restart backup_monitor
backup_monitor.Restart()
# restart db keeper to save the index on all machines including a new one
backup_db_keeper.A('restart')
# finally save the list of current suppliers and clear all stats
backup_matrix.suppliers_set().UpdateSuppliers(supplierList)
def doRequestAvailableBlocks(self, arg):
self.missingPackets = 0
# self.missingSuppliers.clear()
# here we want to request some packets before we start working to rebuild the missed blocks
supplierSet = backup_matrix.suppliers_set()
availableSuppliers = supplierSet.GetActiveArray()
# remember how many requests we did on this iteration
total_requests_count = 0
# at the moment I do download everything I have available and needed
if '' in contacts.getSupplierIDs():
self.automat('requests-sent', total_requests_count)
return
for supplierNum in range(supplierSet.supplierCount):
supplierID = supplierSet.suppliers[supplierNum]
requests_count = 0
# we do requests in reverse order because we start rebuilding from the last block
# for blockNum in range(self.currentBlockNumber, -1, -1):
for blockIndex in range(len(self.workingBlocksQueue)-1, -1, -1):
blockNum = self.workingBlocksQueue[blockIndex]
# do not keep too many requests in the queue
if io_throttle.GetRequestQueueLength(supplierID) >= 16:
break
# also don't do too many requests at once
if requests_count > 16:
break
remoteData = backup_matrix.GetRemoteDataArray(self.currentBackupID, blockNum)
remoteParity = backup_matrix.GetRemoteParityArray(self.currentBackupID, blockNum)
localData = backup_matrix.GetLocalDataArray(self.currentBackupID, blockNum)
localParity = backup_matrix.GetLocalParityArray(self.currentBackupID, blockNum)
# if the remote Data exist and is available because supplier is on line,
# but we do not have it on hand - do request
if localData[supplierNum] == 0:
PacketID = packetid.MakePacketID(self.currentBackupID, blockNum, supplierNum, 'Data')
if remoteData[supplierNum] == 1:
if availableSuppliers[supplierNum]:
# if supplier is not alive - we can't request from him
if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID):
io_throttle.QueueRequestFile(
self.FileReceived,
misc.getLocalID(),
PacketID,
misc.getLocalID(),
supplierID)
requests_count += 1
else:
# count this packet as missing
self.missingPackets += 1
# also mark this guy as one who dont have any data - nor local nor remote
# self.missingSuppliers.add(supplierNum)
# same for Parity
if localParity[supplierNum] == 0:
PacketID = packetid.MakePacketID(self.currentBackupID, blockNum, supplierNum, 'Parity')
if remoteParity[supplierNum] == 1:
if availableSuppliers[supplierNum]:
if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID):
io_throttle.QueueRequestFile(
self.FileReceived,
misc.getLocalID(),
PacketID,
misc.getLocalID(),
supplierID)
requests_count += 1
else:
self.missingPackets += 1
# self.missingSuppliers.add(supplierNum)
total_requests_count += requests_count
self.automat('requests-sent', total_requests_count)
