def start(self): from customer import io_throttle from customer import data_sender from customer import data_receiver io_throttle.init() data_sender.A('init') data_receiver.A('init') return True
def stop(self): from customer import io_throttle from customer import data_sender from customer import data_receiver data_receiver.A('shutdown') data_sender.SetShutdownFlag() data_sender.A('shutdown') io_throttle.shutdown() return True
def _block_finished(self, result, params): if not result: lg.out( 10, 'backup_rebuilder._block_finished FAILED, blockIndex=%d' % self.blockIndex) reactor.callLater(0, self._finish_rebuilding) return try: newData, localData, localParity, reconstructedData, reconstructedParity = result _backupID = params[0] _blockNumber = params[1] except: lg.exc() reactor.callLater(0, self._finish_rebuilding) return if newData: from storage import backup_matrix from customer import data_sender count = 0 customer_idurl = packetid.CustomerIDURL(_backupID) for supplierNum in xrange( contactsdb.num_suppliers(customer_idurl=customer_idurl)): if localData[supplierNum] == 1 and reconstructedData[ supplierNum] == 1: backup_matrix.LocalFileReport(None, _backupID, _blockNumber, supplierNum, 'Data') count += 1 if localParity[supplierNum] == 1 and reconstructedParity[ supplierNum] == 1: backup_matrix.LocalFileReport(None, _backupID, _blockNumber, supplierNum, 'Parity') count += 1 self.blocksSucceed.append(_blockNumber) data_sender.A('new-data') lg.out( 10, 'backup_rebuilder._block_finished !!!!!! %d NEW DATA segments reconstructed, blockIndex=%d' % (count, self.blockIndex)) else: lg.out( 10, 'backup_rebuilder._block_finished NO CHANGES, blockIndex=%d' % self.blockIndex) self.blockIndex -= 1 reactor.callLater(0, self._start_one_block)
def _block_finished(self, result, params): if not result: lg.out(10, 'backup_rebuilder._block_finished FAILED, blockIndex=%d' % self.blockIndex) reactor.callLater(0, self._finish_rebuilding) # @UndefinedVariable return try: newData, localData, localParity, reconstructedData, reconstructedParity = result _backupID = params[0] _blockNumber = params[1] except: lg.exc() reactor.callLater(0, self._finish_rebuilding) # @UndefinedVariable return lg.out(10, 'backup_rebuilder._block_finished backupID=%r blockNumber=%r newData=%r' % ( _backupID, _blockNumber, newData)) lg.out(10, ' localData=%r localParity=%r' % (localData, localParity)) if newData: from storage import backup_matrix from customer import data_sender count = 0 customer_idurl = packetid.CustomerIDURL(_backupID) for supplierNum in range(contactsdb.num_suppliers(customer_idurl=customer_idurl)): try: localData[supplierNum] localParity[supplierNum] reconstructedData[supplierNum] reconstructedParity[supplierNum] except: lg.err('invalid result from the task: %s' % repr(params)) lg.out(10, 'result is %s' % repr(result)) lg.exc() continue if localData[supplierNum] == 1 and reconstructedData[supplierNum] == 1: backup_matrix.LocalFileReport(None, _backupID, _blockNumber, supplierNum, 'Data') count += 1 if localParity[supplierNum] == 1 and reconstructedParity[supplierNum] == 1: backup_matrix.LocalFileReport(None, _backupID, _blockNumber, supplierNum, 'Parity') count += 1 self.blocksSucceed.append(_blockNumber) data_sender.A('new-data') lg.out(10, ' !!!!!! %d NEW DATA segments reconstructed, blockIndex=%d' % ( count, self.blockIndex)) else: lg.out(10, ' NO CHANGES, blockIndex=%d' % self.blockIndex) self.blockIndex -= 1 reactor.callLater(0, self._start_one_block) # @UndefinedVariable
def _request_files(self): from storage import backup_matrix from customer import io_throttle from customer import data_sender self.missingPackets = 0 # here we want to request some packets before we start working to # rebuild the missed blocks availableSuppliers = backup_matrix.GetActiveArray(customer_idurl=self.currentCustomerIDURL) # remember how many requests we did on this iteration total_requests_count = 0 # at the moment I do download everything I have available and needed if '' in contactsdb.suppliers(customer_idurl=self.currentCustomerIDURL): lg.out(8, 'backup_rebuilder._request_files SKIP - empty supplier') self.automat('no-requests') return for supplierNum in range(contactsdb.num_suppliers(customer_idurl=self.currentCustomerIDURL)): supplierID = contactsdb.supplier(supplierNum, customer_idurl=self.currentCustomerIDURL) if not supplierID: continue requests_count = 0 # we do requests in reverse order because we start rebuilding from # the last block for blockIndex in range(len(self.workingBlocksQueue) - 1, -1, -1): blockNum = self.workingBlocksQueue[blockIndex] # do not keep too many requests in the queue if io_throttle.GetRequestQueueLength(supplierID) >= 16: break # also don't do too many requests at once if requests_count > 16: break remoteData = backup_matrix.GetRemoteDataArray( self.currentBackupID, blockNum) remoteParity = backup_matrix.GetRemoteParityArray( self.currentBackupID, blockNum) localData = backup_matrix.GetLocalDataArray( self.currentBackupID, blockNum) localParity = backup_matrix.GetLocalParityArray( self.currentBackupID, blockNum) if supplierNum >= len(remoteData) or supplierNum >= len(remoteParity): break if supplierNum >= len(localData) or supplierNum >= len(localParity): break # if remote Data exist and is available because supplier is on-line, # but we do not have it on hand - do request if localData[supplierNum] == 0: PacketID = packetid.MakePacketID( self.currentBackupID, blockNum, supplierNum, 'Data') if remoteData[supplierNum] == 1: if availableSuppliers[supplierNum]: # if supplier is not alive - we can't request from him if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID): customer, remotePath = packetid.SplitPacketID(PacketID) filename = os.path.join( settings.getLocalBackupsDir(), customer, remotePath, ) if not os.path.exists(filename): if io_throttle.QueueRequestFile( self._file_received, my_id.getLocalID(), PacketID, my_id.getLocalID(), supplierID): requests_count += 1 else: # count this packet as missing self.missingPackets += 1 # also mark this guy as one who dont have any data - nor local nor remote else: # but if local Data already exists, but was not sent - do it now if remoteData[supplierNum] != 1: data_sender.A('new-data') # same for Parity if localParity[supplierNum] == 0: PacketID = packetid.MakePacketID( self.currentBackupID, blockNum, supplierNum, 'Parity') if remoteParity[supplierNum] == 1: if availableSuppliers[supplierNum]: if not io_throttle.HasPacketInRequestQueue( supplierID, PacketID): customer, remotePath = packetid.SplitPacketID(PacketID) filename = os.path.join( settings.getLocalBackupsDir(), customer, remotePath, ) if not os.path.exists(filename): if io_throttle.QueueRequestFile( self._file_received, my_id.getLocalID(), PacketID, my_id.getLocalID(), supplierID, ): requests_count += 1 else: self.missingPackets += 1 else: # but if local Parity already exists, but was not sent - do it now if remoteParity[supplierNum] != 1: data_sender.A('new-data') total_requests_count += requests_count if total_requests_count > 0: lg.out(8, 'backup_rebuilder._request_files : %d chunks requested' % total_requests_count) self.automat('requests-sent', total_requests_count) else: if self.missingPackets: lg.out(8, 'backup_rebuilder._request_files : found %d missing packets' % self.missingPackets) self.automat('found-missing') else: lg.out(8, 'backup_rebuilder._request_files : nothing was requested') self.automat('no-requests')
def A(self, event, *args, **kwargs): from customer import fire_hire from customer import data_sender from customer import list_files_orator from storage import backup_rebuilder from storage import index_synchronizer #---READY--- if self.state == 'READY': if event == 'timer-5sec': self.doOverallCheckUp(*args, **kwargs) elif event == 'restart' or event == 'suppliers-changed' or ( event == 'instant' and self.RestartAgain): self.state = 'FIRE_HIRE' self.RestartAgain = False self.doRememberSuppliers(*args, **kwargs) fire_hire.A('restart') #---LIST_FILES--- elif self.state == 'LIST_FILES': if (event == 'list_files_orator.state' and args[0] == 'NO_FILES'): self.state = 'READY' elif (event == 'list_files_orator.state' and args[0] == 'SAW_FILES'): self.state = 'LIST_BACKUPS' index_synchronizer.A('pull') data_sender.A('restart') self.doPrepareListBackups(*args, **kwargs) elif event == 'restart': self.RestartAgain = True elif event == 'suppliers-changed': self.state = 'READY' self.RestartAgain = True #---LIST_BACKUPS--- elif self.state == 'LIST_BACKUPS': if event == 'list-backups-done': self.state = 'REBUILDING' backup_rebuilder.A('start') elif event == 'restart': self.RestartAgain = True elif event == 'suppliers-changed': self.state = 'READY' self.RestartAgain = True elif event == 'restart': self.state = 'FIRE_HIRE' fire_hire.A('restart') #---REBUILDING--- elif self.state == 'REBUILDING': if (event == 'backup_rebuilder.state' and args[0] in ['DONE', 'STOPPED']): self.state = 'READY' self.doCleanUpBackups(*args, **kwargs) data_sender.A('restart') elif event == 'restart' or event == 'suppliers-changed': self.state = 'FIRE_HIRE' backup_rebuilder.SetStoppedFlag() fire_hire.A('restart') #---FIRE_HIRE--- elif self.state == 'FIRE_HIRE': if event == 'suppliers-changed' and self.isSuppliersNumberChanged( *args, **kwargs): self.state = 'LIST_FILES' self.doDeleteAllBackups(*args, **kwargs) self.doRememberSuppliers(*args, **kwargs) list_files_orator.A('need-files') elif event == 'fire-hire-finished': self.state = 'LIST_FILES' list_files_orator.A('need-files') elif event == 'suppliers-changed' and not self.isSuppliersNumberChanged( *args, **kwargs): self.state = 'LIST_FILES' self.doUpdateSuppliers(*args, **kwargs) self.doRememberSuppliers(*args, **kwargs) list_files_orator.A('need-files') elif event == 'restart': self.RestartAgain = True #---AT_STARTUP--- elif self.state == 'AT_STARTUP': if event == 'init': self.state = 'READY' self.RestartAgain = False return None
def A(self, event, *args, **kwargs): from customer import data_sender #---AT_STARTUP--- if self.state == 'AT_STARTUP': if event == 'start': self.state = 'READ' self.doInit(*args, **kwargs) self.doFirstBlock(*args, **kwargs) #---READ--- elif self.state == 'READ': if event == 'read-success' and not self.isReadingNow( *args, **kwargs) and (self.isBlockReady(*args, **kwargs) or self.isEOF(*args, **kwargs)): self.state = 'ENCRYPT' self.doEncryptBlock(*args, **kwargs) elif event == 'fail' or ( (event == 'read-success' or event == 'timer-001sec') and self.isAborted(*args, **kwargs)): self.state = 'ABORTED' self.doClose(*args, **kwargs) self.doReport(*args, **kwargs) self.doDestroyMe(*args, **kwargs) elif (event == 'read-success' or event == 'timer-001sec') and not self.isAborted( *args, **kwargs) and self.isPipeReady( *args, **kwargs) and not self.isEOF( *args, **kwargs) and not self.isReadingNow( *args, **kwargs) and not self.isBlockReady( *args, **kwargs): self.doRead(*args, **kwargs) elif event == 'block-raid-done' and not self.isAborted( *args, **kwargs): self.doPopBlock(*args, **kwargs) self.doBlockReport(*args, **kwargs) data_sender.A('new-data') #---RAID--- elif self.state == 'RAID': if event == 'block-raid-done' and not self.isMoreBlocks( *args, **kwargs) and not self.isAborted(*args, **kwargs): self.state = 'DONE' self.doPopBlock(*args, **kwargs) self.doBlockReport(*args, **kwargs) data_sender.A('new-data') self.doClose(*args, **kwargs) self.doReport(*args, **kwargs) self.doDestroyMe(*args, **kwargs) elif event == 'block-raid-started' and not self.isEOF( *args, **kwargs) and not self.isAborted(*args, **kwargs): self.state = 'READ' self.doNextBlock(*args, **kwargs) self.doRead(*args, **kwargs) elif event == 'block-raid-done' and self.isMoreBlocks( *args, **kwargs) and not self.isAborted(*args, **kwargs): self.doPopBlock(*args, **kwargs) self.doBlockReport(*args, **kwargs) data_sender.A('new-data') elif event == 'fail' or ( (event == 'timer-01sec' or event == 'block-raid-done' or event == 'block-raid-started') and self.isAborted(*args, **kwargs)): self.state = 'ABORTED' self.doClose(*args, **kwargs) self.doReport(*args, **kwargs) self.doDestroyMe(*args, **kwargs) #---DONE--- elif self.state == 'DONE': pass #---ABORTED--- elif self.state == 'ABORTED': pass #---ENCRYPT--- elif self.state == 'ENCRYPT': if event == 'block-encrypted': self.state = 'RAID' self.doBlockPushAndRaid(*args, **kwargs) elif event == 'fail': self.state = 'ABORTED' self.doClose(*args, **kwargs) self.doReport(*args, **kwargs) self.doDestroyMe(*args, **kwargs) elif event == 'block-raid-done' and not self.isAborted( *args, **kwargs): self.doPopBlock(*args, **kwargs) self.doBlockReport(*args, **kwargs) data_sender.A('new-data') return None