def doRequestPackets(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRequestPackets for %s at block %d' % (
self.backup_id,
self.block_number,
))
from customer import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandData[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandData is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.backup_id, self.block_number,
SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandParity[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandParity is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.backup_id, self.block_number,
SupplierNumber, 'Parity')))
if _Debug:
lg.out(_DebugLevel,
' packets to request: %s' % packetsToRequest)
requests_made = 0
already_requested = 0
for SupplierID, packetID in packetsToRequest:
if io_throttle.HasPacketInRequestQueue(SupplierID, packetID):
already_requested += 1
if packetID not in self.AlreadyRequestedCounts:
self.AlreadyRequestedCounts[packetID] = 0
self.AlreadyRequestedCounts[packetID] += 1
if _Debug:
lg.out(
_DebugLevel,
' packet already in request queue: %s %s' % (
SupplierID,
packetID,
))
continue
io_throttle.QueueRequestFile(self._on_packet_request_result,
self.creator_id, packetID,
self.creator_id, SupplierID)
requests_made += 1
del packetsToRequest
if requests_made:
if _Debug:
lg.out(
_DebugLevel, " requested %d packets for block %d" %
(requests_made, self.block_number))
else:
if not already_requested:
lg.warn('no requests made for block %d' % self.block_number)
self.automat('request-failed', None)
else:
if _Debug:
lg.out(
_DebugLevel,
" found %d already requested packets for block %d"
% (already_requested, self.block_number))
if self.AlreadyRequestedCounts:
all_counts = sorted(self.AlreadyRequestedCounts.values())
if all_counts[0] > 100:
lg.warn('too much requests made for block %d' %
self.block_number)
self.automat('request-failed', None)
def _request_files(self):
from storage import backup_matrix
from customer import io_throttle
from customer import data_sender
self.missingPackets = 0
# here we want to request some packets before we start working to
# rebuild the missed blocks
availableSuppliers = backup_matrix.GetActiveArray(customer_idurl=self.currentCustomerIDURL)
# remember how many requests we did on this iteration
total_requests_count = 0
# at the moment I do download everything I have available and needed
if '' in contactsdb.suppliers(customer_idurl=self.currentCustomerIDURL):
lg.out(8, 'backup_rebuilder._request_files SKIP - empty supplier')
self.automat('no-requests')
return
for supplierNum in range(contactsdb.num_suppliers(customer_idurl=self.currentCustomerIDURL)):
supplierID = contactsdb.supplier(supplierNum, customer_idurl=self.currentCustomerIDURL)
if not supplierID:
continue
requests_count = 0
# we do requests in reverse order because we start rebuilding from
# the last block
for blockIndex in range(len(self.workingBlocksQueue) - 1, -1, -1):
blockNum = self.workingBlocksQueue[blockIndex]
# do not keep too many requests in the queue
if io_throttle.GetRequestQueueLength(supplierID) >= 16:
break
# also don't do too many requests at once
if requests_count > 16:
break
remoteData = backup_matrix.GetRemoteDataArray(
self.currentBackupID, blockNum)
remoteParity = backup_matrix.GetRemoteParityArray(
self.currentBackupID, blockNum)
localData = backup_matrix.GetLocalDataArray(
self.currentBackupID, blockNum)
localParity = backup_matrix.GetLocalParityArray(
self.currentBackupID, blockNum)
if supplierNum >= len(remoteData) or supplierNum >= len(remoteParity):
break
if supplierNum >= len(localData) or supplierNum >= len(localParity):
break
# if remote Data exist and is available because supplier is on-line,
# but we do not have it on hand - do request
if localData[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Data')
if remoteData[supplierNum] == 1:
if availableSuppliers[supplierNum]:
# if supplier is not alive - we can't request from him
if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID):
requests_count += 1
else:
# count this packet as missing
self.missingPackets += 1
# also mark this guy as one who dont have any data - nor local nor remote
else:
# but if local Data already exists, but was not sent - do it now
if remoteData[supplierNum] != 1:
data_sender.A('new-data')
# same for Parity
if localParity[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Parity')
if remoteParity[supplierNum] == 1:
if availableSuppliers[supplierNum]:
if not io_throttle.HasPacketInRequestQueue(
supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID,
):
requests_count += 1
else:
self.missingPackets += 1
else:
# but if local Parity already exists, but was not sent - do it now
if remoteParity[supplierNum] != 1:
data_sender.A('new-data')
total_requests_count += requests_count
if total_requests_count > 0:
lg.out(8, 'backup_rebuilder._request_files : %d chunks requested' % total_requests_count)
self.automat('requests-sent', total_requests_count)
else:
if self.missingPackets:
lg.out(8, 'backup_rebuilder._request_files : found %d missing packets' % self.missingPackets)
self.automat('found-missing')
else:
lg.out(8, 'backup_rebuilder._request_files : nothing was requested')
self.automat('no-requests')
def doRequestPackets(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRequestPackets for %s at block %d' % (
self.BackupID,
self.BlockNumber,
))
from customer import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
lg.warn('bad supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandData[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandData is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
lg.warn('bad supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandParity[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandParity is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Parity')))
if _Debug:
lg.out(_DebugLevel,
' packets to request: %s' % packetsToRequest)
requests_made = 0
already_requested = 0
for SupplierID, packetID in packetsToRequest:
if io_throttle.HasPacketInRequestQueue(SupplierID, packetID):
already_requested += 1
if _Debug:
lg.out(
_DebugLevel,
' packet already in request queue: %s %s' % (
SupplierID,
packetID,
))
continue
io_throttle.QueueRequestFile(self._on_packet_request_result,
self.CreatorID, packetID,
self.CreatorID, SupplierID)
requests_made += 1
del packetsToRequest
if requests_made:
if _Debug:
lg.out(
_DebugLevel, " requested %d packets for block %d" %
(requests_made, self.BlockNumber))
else:
if already_requested:
if _Debug:
lg.out(
_DebugLevel,
" found %d already requested packets for block %d"
% (already_requested, self.BlockNumber))
else:
lg.warn('no requests made for block %d' % self.BlockNumber)
self.automat('request-failed', None)