def doRequestPackets(self, arg): from customer import io_throttle packetsToRequest = [] for SupplierNumber in range(self.EccMap.datasegments): SupplierID = contactsdb.supplier(SupplierNumber, customer_idurl=self.CustomerIDURL) if not SupplierID: continue if not self.OnHandData[SupplierNumber] and contact_status.isOnline( SupplierID): packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Data'))) for SupplierNumber in range(self.EccMap.paritysegments): SupplierID = contactsdb.supplier(SupplierNumber, customer_idurl=self.CustomerIDURL) if not SupplierID: continue if not self.OnHandParity[ SupplierNumber] and contact_status.isOnline(SupplierID): packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Parity'))) for SupplierID, packetID in packetsToRequest: io_throttle.QueueRequestFile(self._on_inbox_packet, self.CreatorID, packetID, self.CreatorID, SupplierID) lg.out( 6, "restore.doRequestPackets requested %d packets for block %d" % (len(packetsToRequest), self.BlockNumber)) del packetsToRequest self.automat('request-done')
def doScanExistingPackets(self, *args, **kwargs): """ Action method. """ for SupplierNumber in range(self.EccMap.datasegments): PacketID = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Data') customerID, remotePath = packetid.SplitPacketID(PacketID) self.OnHandData[SupplierNumber] = bool(os.path.exists(os.path.join( settings.getLocalBackupsDir(), customerID, remotePath))) for SupplierNumber in range(self.EccMap.paritysegments): PacketID = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Parity') customerID, remotePath = packetid.SplitPacketID(PacketID) self.OnHandParity[SupplierNumber] = bool(os.path.exists(os.path.join( settings.getLocalBackupsDir(), customerID, remotePath)))
def doScanExistingPackets(self, arg): for SupplierNumber in range(self.EccMap.datasegments): PacketID = packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Data') customer, remotePath = packetid.SplitPacketID(PacketID) self.OnHandData[SupplierNumber] = os.path.exists( os.path.join(settings.getLocalBackupsDir(), customer, remotePath)) for SupplierNumber in range(self.EccMap.paritysegments): PacketID = packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Parity') customer, remotePath = packetid.SplitPacketID(PacketID) self.OnHandParity[SupplierNumber] = os.path.exists( os.path.join(settings.getLocalBackupsDir(), customer, remotePath))
def doRemoveTempFile(self, *args, **kwargs): """ Action method. """ if not args or not len(args) > 1: return filename = args[1] if filename: tmpfile.throw_out(filename, 'block restored') if settings.getBackupsKeepLocalCopies(): return from storage import backup_matrix from storage import backup_rebuilder if not backup_rebuilder.ReadStoppedFlag(): if backup_rebuilder.A().currentBackupID is not None: if backup_rebuilder.A().currentBackupID == self.backup_id: if _Debug: lg.out( _DebugLevel, 'restore_worker.doRemoveTempFile SKIP because rebuilding in process' ) return count = 0 for supplierNum in range( contactsdb.num_suppliers(customer_idurl=self.customer_idurl)): supplierIDURL = contactsdb.supplier( supplierNum, customer_idurl=self.customer_idurl) if not supplierIDURL: continue for dataORparity in [ 'Data', 'Parity', ]: packetID = packetid.MakePacketID(self.backup_id, self.block_number, supplierNum, dataORparity) customer, remotePath = packetid.SplitPacketID(packetID) filename = os.path.join(settings.getLocalBackupsDir(), customer, remotePath) if os.path.isfile(filename): try: os.remove(filename) except: lg.exc() continue count += 1 backup_matrix.LocalBlockReport(self.backup_id, self.block_number, *args, **kwargs) if _Debug: lg.out( _DebugLevel, 'restore_worker.doRemoveTempFile %d files were removed' % count)
def doRemoveTempFile(self, arg): try: filename = arg[1] except: return tmpfile.throw_out(filename, 'block restored') if settings.getBackupsKeepLocalCopies(): return import backup_rebuilder import backup_matrix if not backup_rebuilder.ReadStoppedFlag(): if backup_rebuilder.A().currentBackupID is not None: if backup_rebuilder.A().currentBackupID == self.BackupID: lg.out( 6, 'restore.doRemoveTempFile SKIP because rebuilding in process' ) return count = 0 for supplierNum in xrange( contactsdb.num_suppliers(customer_idurl=self.CustomerIDURL)): supplierIDURL = contactsdb.supplier( supplierNum, customer_idurl=self.CustomerIDURL) if not supplierIDURL: continue for dataORparity in ['Data', 'Parity']: packetID = packetid.MakePacketID(self.BackupID, self.BlockNumber, supplierNum, dataORparity) customer, remotePath = packetid.SplitPacketID(packetID) filename = os.path.join(settings.getLocalBackupsDir(), customer, remotePath) if os.path.isfile(filename): try: os.remove(filename) except: lg.exc() continue count += 1 backup_matrix.LocalBlockReport(self.BackupID, self.BlockNumber, arg) lg.out(6, 'restore.doRemoveTempFile %d files were removed' % count)
def _request_files(self): from storage import backup_matrix from customer import io_throttle from customer import data_sender self.missingPackets = 0 # here we want to request some packets before we start working to # rebuild the missed blocks availableSuppliers = backup_matrix.GetActiveArray(customer_idurl=self.currentCustomerIDURL) # remember how many requests we did on this iteration total_requests_count = 0 # at the moment I do download everything I have available and needed if '' in contactsdb.suppliers(customer_idurl=self.currentCustomerIDURL): lg.out(8, 'backup_rebuilder._request_files SKIP - empty supplier') self.automat('no-requests') return for supplierNum in range(contactsdb.num_suppliers(customer_idurl=self.currentCustomerIDURL)): supplierID = contactsdb.supplier(supplierNum, customer_idurl=self.currentCustomerIDURL) if not supplierID: continue requests_count = 0 # we do requests in reverse order because we start rebuilding from # the last block for blockIndex in range(len(self.workingBlocksQueue) - 1, -1, -1): blockNum = self.workingBlocksQueue[blockIndex] # do not keep too many requests in the queue if io_throttle.GetRequestQueueLength(supplierID) >= 16: break # also don't do too many requests at once if requests_count > 16: break remoteData = backup_matrix.GetRemoteDataArray( self.currentBackupID, blockNum) remoteParity = backup_matrix.GetRemoteParityArray( self.currentBackupID, blockNum) localData = backup_matrix.GetLocalDataArray( self.currentBackupID, blockNum) localParity = backup_matrix.GetLocalParityArray( self.currentBackupID, blockNum) if supplierNum >= len(remoteData) or supplierNum >= len(remoteParity): break if supplierNum >= len(localData) or supplierNum >= len(localParity): break # if remote Data exist and is available because supplier is on-line, # but we do not have it on hand - do request if localData[supplierNum] == 0: PacketID = packetid.MakePacketID( self.currentBackupID, blockNum, supplierNum, 'Data') if remoteData[supplierNum] == 1: if availableSuppliers[supplierNum]: # if supplier is not alive - we can't request from him if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID): customer, remotePath = packetid.SplitPacketID(PacketID) filename = os.path.join( settings.getLocalBackupsDir(), customer, remotePath, ) if not os.path.exists(filename): if io_throttle.QueueRequestFile( self._file_received, my_id.getLocalID(), PacketID, my_id.getLocalID(), supplierID): requests_count += 1 else: # count this packet as missing self.missingPackets += 1 # also mark this guy as one who dont have any data - nor local nor remote else: # but if local Data already exists, but was not sent - do it now if remoteData[supplierNum] != 1: data_sender.A('new-data') # same for Parity if localParity[supplierNum] == 0: PacketID = packetid.MakePacketID( self.currentBackupID, blockNum, supplierNum, 'Parity') if remoteParity[supplierNum] == 1: if availableSuppliers[supplierNum]: if not io_throttle.HasPacketInRequestQueue( supplierID, PacketID): customer, remotePath = packetid.SplitPacketID(PacketID) filename = os.path.join( settings.getLocalBackupsDir(), customer, remotePath, ) if not os.path.exists(filename): if io_throttle.QueueRequestFile( self._file_received, my_id.getLocalID(), PacketID, my_id.getLocalID(), supplierID, ): requests_count += 1 else: self.missingPackets += 1 else: # but if local Parity already exists, but was not sent - do it now if remoteParity[supplierNum] != 1: data_sender.A('new-data') total_requests_count += requests_count if total_requests_count > 0: lg.out(8, 'backup_rebuilder._request_files : %d chunks requested' % total_requests_count) self.automat('requests-sent', total_requests_count) else: if self.missingPackets: lg.out(8, 'backup_rebuilder._request_files : found %d missing packets' % self.missingPackets) self.automat('found-missing') else: lg.out(8, 'backup_rebuilder._request_files : nothing was requested') self.automat('no-requests')
def _do_send_packets(self, backup_id, block_num): customer_id, path_id, version_name = packetid.SplitBackupID(backup_id) archive_snapshot_dir = os.path.join(settings.getLocalBackupsDir(), customer_id, path_id, version_name) if _Debug: lg.args(_DebugLevel, backup_id=backup_id, block_num=block_num, archive_snapshot_dir=archive_snapshot_dir) if not os.path.isdir(archive_snapshot_dir): self.block_failed = True lg.err('archive snapshot folder was not found in %r' % archive_snapshot_dir) return None failed_supliers = 0 for supplier_num in range(len(self.suppliers_list)): supplier_idurl = self.suppliers_list[supplier_num] if not supplier_idurl: failed_supliers += 1 lg.warn('unknown supplier supplier_num=%d' % supplier_num) continue for dataORparity in ( 'Data', 'Parity', ): packet_id = packetid.MakePacketID(backup_id, block_num, supplier_num, dataORparity) packet_filename = os.path.join( archive_snapshot_dir, '%d-%d-%s' % ( block_num, supplier_num, dataORparity, )) if not os.path.isfile(packet_filename): lg.err('%s is not a file' % packet_filename) continue packet_payload = bpio.ReadBinaryFile(packet_filename) if not packet_payload: lg.err('file %r reading error' % packet_filename) continue if block_num not in self.packets_out: self.packets_out[block_num] = {} self.packets_out[block_num][packet_id] = None p2p_service.SendData( raw_data=packet_payload, ownerID=self.queue_owner_idurl, creatorID=my_id.getIDURL(), remoteID=supplier_idurl, packetID=packet_id, callbacks={ commands.Ack(): lambda newpacket, _: self.automat('ack', newpacket=newpacket), commands.Fail(): lambda newpacket, _: self.automat('fail', newpacket=newpacket), }, ) if failed_supliers > self.correctable_errors: self.block_failed = True lg.err('too many failed suppliers %d in block %d' % ( failed_supliers, block_num, ))
def _do_check_run_requests(self): if _Debug: lg.out( _DebugLevel, 'restore_worker._do_check_run_requests for %s at block %d' % ( self.backup_id, self.block_number, )) from stream import io_throttle packetsToRequest = [] for SupplierNumber in range(self.EccMap.datasegments): request_packet_id = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Data') if self.OnHandData[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' SKIP, OnHandData is True for supplier %d' % SupplierNumber) if request_packet_id not in self.block_requests: self.block_requests[request_packet_id] = True continue if request_packet_id in self.block_requests: if _Debug: lg.out( _DebugLevel, ' SKIP, request for packet %r already sent to IO queue for supplier %d' % ( request_packet_id, SupplierNumber, )) continue SupplierID = contactsdb.supplier( SupplierNumber, customer_idurl=self.customer_idurl) if not SupplierID: lg.warn('unknown supplier at position %s' % SupplierNumber) continue if online_status.isOffline(SupplierID): if _Debug: lg.out(_DebugLevel, ' SKIP, offline supplier: %s' % SupplierID) continue packetsToRequest.append(( SupplierID, request_packet_id, )) for SupplierNumber in range(self.EccMap.paritysegments): request_packet_id = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Parity') if self.OnHandParity[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' SKIP, OnHandParity is True for supplier %d' % SupplierNumber) if request_packet_id not in self.block_requests: self.block_requests[request_packet_id] = True continue if request_packet_id in self.block_requests: if _Debug: lg.out( _DebugLevel, ' SKIP, request for packet %r already sent to IO queue for supplier %d' % ( request_packet_id, SupplierNumber, )) continue SupplierID = contactsdb.supplier( SupplierNumber, customer_idurl=self.customer_idurl) if not SupplierID: lg.warn('unknown supplier at position %s' % SupplierNumber) continue if online_status.isOffline(SupplierID): if _Debug: lg.out(_DebugLevel, ' SKIP, offline supplier: %s' % SupplierID) continue packetsToRequest.append(( SupplierID, request_packet_id, )) requests_made = 0 # already_requested = 0 for SupplierID, packetID in packetsToRequest: if io_throttle.HasPacketInRequestQueue(SupplierID, packetID): # already_requested += 1 # if packetID not in self.AlreadyRequestedCounts: # self.AlreadyRequestedCounts[packetID] = 0 # self.AlreadyRequestedCounts[packetID] += 1 lg.warn('packet already in IO queue for supplier %s : %s' % ( SupplierID, packetID, )) continue self.block_requests[packetID] = None if io_throttle.QueueRequestFile( callOnReceived=self._on_packet_request_result, creatorID=self.creator_id, packetID=packetID, ownerID=self.creator_id, # self.customer_idurl, remoteID=SupplierID, ): requests_made += 1 else: self.block_requests[packetID] = False if _Debug: lg.dbg( _DebugLevel, 'sent request %r to %r, other requests: %r' % (packetID, SupplierID, list(self.block_requests.values()))) del packetsToRequest if requests_made: if _Debug: lg.out( _DebugLevel, " requested %d packets for block %d" % ( requests_made, self.block_number, )) return current_block_requests_results = list(self.block_requests.values()) if _Debug: lg.args(_DebugLevel, current_results=current_block_requests_results) pending_count = current_block_requests_results.count(None) if pending_count > 0: if _Debug: lg.out( _DebugLevel, " nothing for request, currently %d pending packets for block %d" % ( pending_count, self.block_number, )) return failed_count = current_block_requests_results.count(False) if failed_count > self.max_errors: lg.err( 'all requests finished and %d packets failed, not possible to read data for block %d' % ( failed_count, self.block_number, )) reactor.callLater(0, self.automat, 'request-failed', None) # @UndefinedVariable return if _Debug: lg.out( _DebugLevel, " all requests finished for block %d : %r" % ( self.block_number, current_block_requests_results, )) reactor.callLater(0, self.automat, 'request-finished', None) # @UndefinedVariable
def doRequestPackets(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out( _DebugLevel, 'restore_worker.doRequestPackets for %s at block %d' % ( self.backup_id, self.block_number, )) from customer import io_throttle packetsToRequest = [] for SupplierNumber in range(self.EccMap.datasegments): SupplierID = contactsdb.supplier( SupplierNumber, customer_idurl=self.customer_idurl) if not SupplierID: lg.warn('unknown supplier at position %s' % SupplierNumber) continue if contact_status.isOffline(SupplierID): lg.warn('offline supplier: %s' % SupplierID) continue if self.OnHandData[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' OnHandData is True for supplier %d' % SupplierNumber) continue packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Data'))) for SupplierNumber in range(self.EccMap.paritysegments): SupplierID = contactsdb.supplier( SupplierNumber, customer_idurl=self.customer_idurl) if not SupplierID: lg.warn('unknown supplier at position %s' % SupplierNumber) continue if contact_status.isOffline(SupplierID): lg.warn('offline supplier: %s' % SupplierID) continue if self.OnHandParity[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' OnHandParity is True for supplier %d' % SupplierNumber) continue packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Parity'))) if _Debug: lg.out(_DebugLevel, ' packets to request: %s' % packetsToRequest) requests_made = 0 already_requested = 0 for SupplierID, packetID in packetsToRequest: if io_throttle.HasPacketInRequestQueue(SupplierID, packetID): already_requested += 1 if packetID not in self.AlreadyRequestedCounts: self.AlreadyRequestedCounts[packetID] = 0 self.AlreadyRequestedCounts[packetID] += 1 if _Debug: lg.out( _DebugLevel, ' packet already in request queue: %s %s' % ( SupplierID, packetID, )) continue io_throttle.QueueRequestFile(self._on_packet_request_result, self.creator_id, packetID, self.creator_id, SupplierID) requests_made += 1 del packetsToRequest if requests_made: if _Debug: lg.out( _DebugLevel, " requested %d packets for block %d" % (requests_made, self.block_number)) else: if not already_requested: lg.warn('no requests made for block %d' % self.block_number) self.automat('request-failed', None) else: if _Debug: lg.out( _DebugLevel, " found %d already requested packets for block %d" % (already_requested, self.block_number)) if self.AlreadyRequestedCounts: all_counts = sorted(self.AlreadyRequestedCounts.values()) if all_counts[0] > 100: lg.warn('too much requests made for block %d' % self.block_number) self.automat('request-failed', None)
def doRequestPackets(self, arg): """ Action method. """ if _Debug: lg.out( _DebugLevel, 'restore_worker.doRequestPackets for %s at block %d' % ( self.BackupID, self.BlockNumber, )) from customer import io_throttle packetsToRequest = [] for SupplierNumber in range(self.EccMap.datasegments): SupplierID = contactsdb.supplier(SupplierNumber, customer_idurl=self.CustomerIDURL) if not SupplierID: lg.warn('bad supplier at position %s' % SupplierNumber) continue if contact_status.isOffline(SupplierID): lg.warn('offline supplier: %s' % SupplierID) continue if self.OnHandData[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' OnHandData is True for supplier %d' % SupplierNumber) continue packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Data'))) for SupplierNumber in range(self.EccMap.paritysegments): SupplierID = contactsdb.supplier(SupplierNumber, customer_idurl=self.CustomerIDURL) if not SupplierID: lg.warn('bad supplier at position %s' % SupplierNumber) continue if contact_status.isOffline(SupplierID): lg.warn('offline supplier: %s' % SupplierID) continue if self.OnHandParity[SupplierNumber]: if _Debug: lg.out( _DebugLevel, ' OnHandParity is True for supplier %d' % SupplierNumber) continue packetsToRequest.append( (SupplierID, packetid.MakePacketID(self.BackupID, self.BlockNumber, SupplierNumber, 'Parity'))) if _Debug: lg.out(_DebugLevel, ' packets to request: %s' % packetsToRequest) requests_made = 0 already_requested = 0 for SupplierID, packetID in packetsToRequest: if io_throttle.HasPacketInRequestQueue(SupplierID, packetID): already_requested += 1 if _Debug: lg.out( _DebugLevel, ' packet already in request queue: %s %s' % ( SupplierID, packetID, )) continue io_throttle.QueueRequestFile(self._on_packet_request_result, self.CreatorID, packetID, self.CreatorID, SupplierID) requests_made += 1 del packetsToRequest if requests_made: if _Debug: lg.out( _DebugLevel, " requested %d packets for block %d" % (requests_made, self.BlockNumber)) else: if already_requested: if _Debug: lg.out( _DebugLevel, " found %d already requested packets for block %d" % (already_requested, self.BlockNumber)) else: lg.warn('no requests made for block %d' % self.BlockNumber) self.automat('request-failed', None)