def doRequestPackets(self, arg):
from customer import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
continue
if not self.OnHandData[SupplierNumber] and contact_status.isOnline(
SupplierID):
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
continue
if not self.OnHandParity[
SupplierNumber] and contact_status.isOnline(SupplierID):
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Parity')))
for SupplierID, packetID in packetsToRequest:
io_throttle.QueueRequestFile(self._on_inbox_packet, self.CreatorID,
packetID, self.CreatorID, SupplierID)
lg.out(
6, "restore.doRequestPackets requested %d packets for block %d" %
(len(packetsToRequest), self.BlockNumber))
del packetsToRequest
self.automat('request-done')
def doScanExistingPackets(self, *args, **kwargs):
"""
Action method.
"""
for SupplierNumber in range(self.EccMap.datasegments):
PacketID = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Data')
customerID, remotePath = packetid.SplitPacketID(PacketID)
self.OnHandData[SupplierNumber] = bool(os.path.exists(os.path.join(
settings.getLocalBackupsDir(), customerID, remotePath)))
for SupplierNumber in range(self.EccMap.paritysegments):
PacketID = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Parity')
customerID, remotePath = packetid.SplitPacketID(PacketID)
self.OnHandParity[SupplierNumber] = bool(os.path.exists(os.path.join(
settings.getLocalBackupsDir(), customerID, remotePath)))
def doScanExistingPackets(self, arg):
for SupplierNumber in range(self.EccMap.datasegments):
PacketID = packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Data')
customer, remotePath = packetid.SplitPacketID(PacketID)
self.OnHandData[SupplierNumber] = os.path.exists(
os.path.join(settings.getLocalBackupsDir(), customer,
remotePath))
for SupplierNumber in range(self.EccMap.paritysegments):
PacketID = packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Parity')
customer, remotePath = packetid.SplitPacketID(PacketID)
self.OnHandParity[SupplierNumber] = os.path.exists(
os.path.join(settings.getLocalBackupsDir(), customer,
remotePath))
def doRemoveTempFile(self, *args, **kwargs):
"""
Action method.
"""
if not args or not len(args) > 1:
return
filename = args[1]
if filename:
tmpfile.throw_out(filename, 'block restored')
if settings.getBackupsKeepLocalCopies():
return
from storage import backup_matrix
from storage import backup_rebuilder
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == self.backup_id:
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRemoveTempFile SKIP because rebuilding in process'
)
return
count = 0
for supplierNum in range(
contactsdb.num_suppliers(customer_idurl=self.customer_idurl)):
supplierIDURL = contactsdb.supplier(
supplierNum, customer_idurl=self.customer_idurl)
if not supplierIDURL:
continue
for dataORparity in [
'Data',
'Parity',
]:
packetID = packetid.MakePacketID(self.backup_id,
self.block_number,
supplierNum, dataORparity)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, remotePath)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
backup_matrix.LocalBlockReport(self.backup_id, self.block_number,
*args, **kwargs)
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRemoveTempFile %d files were removed' %
count)
def doRemoveTempFile(self, arg):
try:
filename = arg[1]
except:
return
tmpfile.throw_out(filename, 'block restored')
if settings.getBackupsKeepLocalCopies():
return
import backup_rebuilder
import backup_matrix
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == self.BackupID:
lg.out(
6,
'restore.doRemoveTempFile SKIP because rebuilding in process'
)
return
count = 0
for supplierNum in xrange(
contactsdb.num_suppliers(customer_idurl=self.CustomerIDURL)):
supplierIDURL = contactsdb.supplier(
supplierNum, customer_idurl=self.CustomerIDURL)
if not supplierIDURL:
continue
for dataORparity in ['Data', 'Parity']:
packetID = packetid.MakePacketID(self.BackupID,
self.BlockNumber, supplierNum,
dataORparity)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, remotePath)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
backup_matrix.LocalBlockReport(self.BackupID, self.BlockNumber, arg)
lg.out(6, 'restore.doRemoveTempFile %d files were removed' % count)
def _request_files(self):
from storage import backup_matrix
from customer import io_throttle
from customer import data_sender
self.missingPackets = 0
# here we want to request some packets before we start working to
# rebuild the missed blocks
availableSuppliers = backup_matrix.GetActiveArray(customer_idurl=self.currentCustomerIDURL)
# remember how many requests we did on this iteration
total_requests_count = 0
# at the moment I do download everything I have available and needed
if '' in contactsdb.suppliers(customer_idurl=self.currentCustomerIDURL):
lg.out(8, 'backup_rebuilder._request_files SKIP - empty supplier')
self.automat('no-requests')
return
for supplierNum in range(contactsdb.num_suppliers(customer_idurl=self.currentCustomerIDURL)):
supplierID = contactsdb.supplier(supplierNum, customer_idurl=self.currentCustomerIDURL)
if not supplierID:
continue
requests_count = 0
# we do requests in reverse order because we start rebuilding from
# the last block
for blockIndex in range(len(self.workingBlocksQueue) - 1, -1, -1):
blockNum = self.workingBlocksQueue[blockIndex]
# do not keep too many requests in the queue
if io_throttle.GetRequestQueueLength(supplierID) >= 16:
break
# also don't do too many requests at once
if requests_count > 16:
break
remoteData = backup_matrix.GetRemoteDataArray(
self.currentBackupID, blockNum)
remoteParity = backup_matrix.GetRemoteParityArray(
self.currentBackupID, blockNum)
localData = backup_matrix.GetLocalDataArray(
self.currentBackupID, blockNum)
localParity = backup_matrix.GetLocalParityArray(
self.currentBackupID, blockNum)
if supplierNum >= len(remoteData) or supplierNum >= len(remoteParity):
break
if supplierNum >= len(localData) or supplierNum >= len(localParity):
break
# if remote Data exist and is available because supplier is on-line,
# but we do not have it on hand - do request
if localData[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Data')
if remoteData[supplierNum] == 1:
if availableSuppliers[supplierNum]:
# if supplier is not alive - we can't request from him
if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID):
requests_count += 1
else:
# count this packet as missing
self.missingPackets += 1
# also mark this guy as one who dont have any data - nor local nor remote
else:
# but if local Data already exists, but was not sent - do it now
if remoteData[supplierNum] != 1:
data_sender.A('new-data')
# same for Parity
if localParity[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Parity')
if remoteParity[supplierNum] == 1:
if availableSuppliers[supplierNum]:
if not io_throttle.HasPacketInRequestQueue(
supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID,
):
requests_count += 1
else:
self.missingPackets += 1
else:
# but if local Parity already exists, but was not sent - do it now
if remoteParity[supplierNum] != 1:
data_sender.A('new-data')
total_requests_count += requests_count
if total_requests_count > 0:
lg.out(8, 'backup_rebuilder._request_files : %d chunks requested' % total_requests_count)
self.automat('requests-sent', total_requests_count)
else:
if self.missingPackets:
lg.out(8, 'backup_rebuilder._request_files : found %d missing packets' % self.missingPackets)
self.automat('found-missing')
else:
lg.out(8, 'backup_rebuilder._request_files : nothing was requested')
self.automat('no-requests')
def _do_send_packets(self, backup_id, block_num):
customer_id, path_id, version_name = packetid.SplitBackupID(backup_id)
archive_snapshot_dir = os.path.join(settings.getLocalBackupsDir(),
customer_id, path_id, version_name)
if _Debug:
lg.args(_DebugLevel,
backup_id=backup_id,
block_num=block_num,
archive_snapshot_dir=archive_snapshot_dir)
if not os.path.isdir(archive_snapshot_dir):
self.block_failed = True
lg.err('archive snapshot folder was not found in %r' %
archive_snapshot_dir)
return None
failed_supliers = 0
for supplier_num in range(len(self.suppliers_list)):
supplier_idurl = self.suppliers_list[supplier_num]
if not supplier_idurl:
failed_supliers += 1
lg.warn('unknown supplier supplier_num=%d' % supplier_num)
continue
for dataORparity in (
'Data',
'Parity',
):
packet_id = packetid.MakePacketID(backup_id, block_num,
supplier_num, dataORparity)
packet_filename = os.path.join(
archive_snapshot_dir, '%d-%d-%s' % (
block_num,
supplier_num,
dataORparity,
))
if not os.path.isfile(packet_filename):
lg.err('%s is not a file' % packet_filename)
continue
packet_payload = bpio.ReadBinaryFile(packet_filename)
if not packet_payload:
lg.err('file %r reading error' % packet_filename)
continue
if block_num not in self.packets_out:
self.packets_out[block_num] = {}
self.packets_out[block_num][packet_id] = None
p2p_service.SendData(
raw_data=packet_payload,
ownerID=self.queue_owner_idurl,
creatorID=my_id.getIDURL(),
remoteID=supplier_idurl,
packetID=packet_id,
callbacks={
commands.Ack():
lambda newpacket, _: self.automat('ack',
newpacket=newpacket),
commands.Fail():
lambda newpacket, _: self.automat('fail',
newpacket=newpacket),
},
)
if failed_supliers > self.correctable_errors:
self.block_failed = True
lg.err('too many failed suppliers %d in block %d' % (
failed_supliers,
block_num,
))
def _do_check_run_requests(self):
if _Debug:
lg.out(
_DebugLevel,
'restore_worker._do_check_run_requests for %s at block %d' % (
self.backup_id,
self.block_number,
))
from stream import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
request_packet_id = packetid.MakePacketID(self.backup_id,
self.block_number,
SupplierNumber, 'Data')
if self.OnHandData[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' SKIP, OnHandData is True for supplier %d' %
SupplierNumber)
if request_packet_id not in self.block_requests:
self.block_requests[request_packet_id] = True
continue
if request_packet_id in self.block_requests:
if _Debug:
lg.out(
_DebugLevel,
' SKIP, request for packet %r already sent to IO queue for supplier %d'
% (
request_packet_id,
SupplierNumber,
))
continue
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if online_status.isOffline(SupplierID):
if _Debug:
lg.out(_DebugLevel,
' SKIP, offline supplier: %s' % SupplierID)
continue
packetsToRequest.append((
SupplierID,
request_packet_id,
))
for SupplierNumber in range(self.EccMap.paritysegments):
request_packet_id = packetid.MakePacketID(self.backup_id,
self.block_number,
SupplierNumber, 'Parity')
if self.OnHandParity[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' SKIP, OnHandParity is True for supplier %d' %
SupplierNumber)
if request_packet_id not in self.block_requests:
self.block_requests[request_packet_id] = True
continue
if request_packet_id in self.block_requests:
if _Debug:
lg.out(
_DebugLevel,
' SKIP, request for packet %r already sent to IO queue for supplier %d'
% (
request_packet_id,
SupplierNumber,
))
continue
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if online_status.isOffline(SupplierID):
if _Debug:
lg.out(_DebugLevel,
' SKIP, offline supplier: %s' % SupplierID)
continue
packetsToRequest.append((
SupplierID,
request_packet_id,
))
requests_made = 0
# already_requested = 0
for SupplierID, packetID in packetsToRequest:
if io_throttle.HasPacketInRequestQueue(SupplierID, packetID):
# already_requested += 1
# if packetID not in self.AlreadyRequestedCounts:
# self.AlreadyRequestedCounts[packetID] = 0
# self.AlreadyRequestedCounts[packetID] += 1
lg.warn('packet already in IO queue for supplier %s : %s' % (
SupplierID,
packetID,
))
continue
self.block_requests[packetID] = None
if io_throttle.QueueRequestFile(
callOnReceived=self._on_packet_request_result,
creatorID=self.creator_id,
packetID=packetID,
ownerID=self.creator_id, # self.customer_idurl,
remoteID=SupplierID,
):
requests_made += 1
else:
self.block_requests[packetID] = False
if _Debug:
lg.dbg(
_DebugLevel, 'sent request %r to %r, other requests: %r' %
(packetID, SupplierID, list(self.block_requests.values())))
del packetsToRequest
if requests_made:
if _Debug:
lg.out(
_DebugLevel,
" requested %d packets for block %d" % (
requests_made,
self.block_number,
))
return
current_block_requests_results = list(self.block_requests.values())
if _Debug:
lg.args(_DebugLevel,
current_results=current_block_requests_results)
pending_count = current_block_requests_results.count(None)
if pending_count > 0:
if _Debug:
lg.out(
_DebugLevel,
" nothing for request, currently %d pending packets for block %d"
% (
pending_count,
self.block_number,
))
return
failed_count = current_block_requests_results.count(False)
if failed_count > self.max_errors:
lg.err(
'all requests finished and %d packets failed, not possible to read data for block %d'
% (
failed_count,
self.block_number,
))
reactor.callLater(0, self.automat, 'request-failed',
None) # @UndefinedVariable
return
if _Debug:
lg.out(
_DebugLevel,
" all requests finished for block %d : %r" % (
self.block_number,
current_block_requests_results,
))
reactor.callLater(0, self.automat, 'request-finished',
None) # @UndefinedVariable
def doRequestPackets(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRequestPackets for %s at block %d' % (
self.backup_id,
self.block_number,
))
from customer import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandData[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandData is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.backup_id, self.block_number,
SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandParity[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandParity is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.backup_id, self.block_number,
SupplierNumber, 'Parity')))
if _Debug:
lg.out(_DebugLevel,
' packets to request: %s' % packetsToRequest)
requests_made = 0
already_requested = 0
for SupplierID, packetID in packetsToRequest:
if io_throttle.HasPacketInRequestQueue(SupplierID, packetID):
already_requested += 1
if packetID not in self.AlreadyRequestedCounts:
self.AlreadyRequestedCounts[packetID] = 0
self.AlreadyRequestedCounts[packetID] += 1
if _Debug:
lg.out(
_DebugLevel,
' packet already in request queue: %s %s' % (
SupplierID,
packetID,
))
continue
io_throttle.QueueRequestFile(self._on_packet_request_result,
self.creator_id, packetID,
self.creator_id, SupplierID)
requests_made += 1
del packetsToRequest
if requests_made:
if _Debug:
lg.out(
_DebugLevel, " requested %d packets for block %d" %
(requests_made, self.block_number))
else:
if not already_requested:
lg.warn('no requests made for block %d' % self.block_number)
self.automat('request-failed', None)
else:
if _Debug:
lg.out(
_DebugLevel,
" found %d already requested packets for block %d"
% (already_requested, self.block_number))
if self.AlreadyRequestedCounts:
all_counts = sorted(self.AlreadyRequestedCounts.values())
if all_counts[0] > 100:
lg.warn('too much requests made for block %d' %
self.block_number)
self.automat('request-failed', None)
def doRequestPackets(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRequestPackets for %s at block %d' % (
self.BackupID,
self.BlockNumber,
))
from customer import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
lg.warn('bad supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandData[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandData is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
lg.warn('bad supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandParity[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandParity is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Parity')))
if _Debug:
lg.out(_DebugLevel,
' packets to request: %s' % packetsToRequest)
requests_made = 0
already_requested = 0
for SupplierID, packetID in packetsToRequest:
if io_throttle.HasPacketInRequestQueue(SupplierID, packetID):
already_requested += 1
if _Debug:
lg.out(
_DebugLevel,
' packet already in request queue: %s %s' % (
SupplierID,
packetID,
))
continue
io_throttle.QueueRequestFile(self._on_packet_request_result,
self.CreatorID, packetID,
self.CreatorID, SupplierID)
requests_made += 1
del packetsToRequest
if requests_made:
if _Debug:
lg.out(
_DebugLevel, " requested %d packets for block %d" %
(requests_made, self.BlockNumber))
else:
if already_requested:
if _Debug:
lg.out(
_DebugLevel,
" found %d already requested packets for block %d"
% (already_requested, self.BlockNumber))
else:
lg.warn('no requests made for block %d' % self.BlockNumber)
self.automat('request-failed', None)
