def isChanceToRebuild(self, arg):
"""
Condition method.
"""
from storage import backup_matrix
# supplierSet = backup_matrix.suppliers_set()
# start checking in reverse order, see below for explanation
for blockIndex in range(len(self.workingBlocksQueue) - 1, -1, -1):
blockNumber = self.workingBlocksQueue[blockIndex]
if eccmap.Current().CanMakeProgress(
backup_matrix.GetLocalDataArray(self.currentBackupID,
blockNumber),
backup_matrix.GetLocalParityArray(self.currentBackupID,
blockNumber)):
return True
return False
def _request_files(self):
from storage import backup_matrix
from customer import io_throttle
from customer import data_sender
self.missingPackets = 0
# here we want to request some packets before we start working to
# rebuild the missed blocks
availableSuppliers = backup_matrix.GetActiveArray(customer_idurl=self.currentCustomerIDURL)
# remember how many requests we did on this iteration
total_requests_count = 0
# at the moment I do download everything I have available and needed
if '' in contactsdb.suppliers(customer_idurl=self.currentCustomerIDURL):
lg.out(8, 'backup_rebuilder._request_files SKIP - empty supplier')
self.automat('no-requests')
return
for supplierNum in range(contactsdb.num_suppliers(customer_idurl=self.currentCustomerIDURL)):
supplierID = contactsdb.supplier(supplierNum, customer_idurl=self.currentCustomerIDURL)
if not supplierID:
continue
requests_count = 0
# we do requests in reverse order because we start rebuilding from
# the last block
for blockIndex in range(len(self.workingBlocksQueue) - 1, -1, -1):
blockNum = self.workingBlocksQueue[blockIndex]
# do not keep too many requests in the queue
if io_throttle.GetRequestQueueLength(supplierID) >= 16:
break
# also don't do too many requests at once
if requests_count > 16:
break
remoteData = backup_matrix.GetRemoteDataArray(
self.currentBackupID, blockNum)
remoteParity = backup_matrix.GetRemoteParityArray(
self.currentBackupID, blockNum)
localData = backup_matrix.GetLocalDataArray(
self.currentBackupID, blockNum)
localParity = backup_matrix.GetLocalParityArray(
self.currentBackupID, blockNum)
if supplierNum >= len(remoteData) or supplierNum >= len(remoteParity):
break
if supplierNum >= len(localData) or supplierNum >= len(localParity):
break
# if remote Data exist and is available because supplier is on-line,
# but we do not have it on hand - do request
if localData[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Data')
if remoteData[supplierNum] == 1:
if availableSuppliers[supplierNum]:
# if supplier is not alive - we can't request from him
if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID):
requests_count += 1
else:
# count this packet as missing
self.missingPackets += 1
# also mark this guy as one who dont have any data - nor local nor remote
else:
# but if local Data already exists, but was not sent - do it now
if remoteData[supplierNum] != 1:
data_sender.A('new-data')
# same for Parity
if localParity[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Parity')
if remoteParity[supplierNum] == 1:
if availableSuppliers[supplierNum]:
if not io_throttle.HasPacketInRequestQueue(
supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID,
):
requests_count += 1
else:
self.missingPackets += 1
else:
# but if local Parity already exists, but was not sent - do it now
if remoteParity[supplierNum] != 1:
data_sender.A('new-data')
total_requests_count += requests_count
if total_requests_count > 0:
lg.out(8, 'backup_rebuilder._request_files : %d chunks requested' % total_requests_count)
self.automat('requests-sent', total_requests_count)
else:
if self.missingPackets:
lg.out(8, 'backup_rebuilder._request_files : found %d missing packets' % self.missingPackets)
self.automat('found-missing')
else:
lg.out(8, 'backup_rebuilder._request_files : nothing was requested')
self.automat('no-requests')