def _start_one_block(self): from storage import backup_matrix if self.blockIndex < 0: lg.out( 10, 'backup_rebuilder._start_one_block finish all blocks blockIndex=%d' % self.blockIndex) reactor.callLater(0, self._finish_rebuilding) return BlockNumber = self.workingBlocksQueue[self.blockIndex] lg.out( 10, 'backup_rebuilder._start_one_block %d to rebuild, blockIndex=%d, other blocks: %s' % ((BlockNumber, self.blockIndex, str(self.workingBlocksQueue)))) task_params = ( self.currentBackupID, BlockNumber, eccmap.Current(), backup_matrix.GetActiveArray(), backup_matrix.GetRemoteMatrix(self.currentBackupID, BlockNumber), backup_matrix.GetLocalMatrix(self.currentBackupID, BlockNumber), ) raid_worker.add_task( 'rebuild', task_params, lambda cmd, params, result: self._block_finished(result, params))
def doBlockPushAndRaid(self, arg): """ Action method. """ newblock = arg if self.terminating: self.automat('block-raid-done', (newblock.BlockNumber, None)) lg.out(_DebugLevel, 'backup.doBlockPushAndRaid SKIP, terminating=True') return fileno, filename = tmpfile.make('raid') serializedblock = newblock.Serialize() blocklen = len(serializedblock) os.write(fileno, str(blocklen) + ":" + serializedblock) os.close(fileno) self.workBlocks[newblock.BlockNumber] = filename dt = time.time() outputpath = os.path.join(settings.getLocalBackupsDir(), self.backupID) task_params = (filename, self.eccmap.name, self.backupID, newblock.BlockNumber, outputpath) raid_worker.add_task('make', task_params, lambda cmd, params, result: self._raidmakeCallback(params, result, dt),) self.automat('block-raid-started', newblock) del serializedblock if _Debug: lg.out(_DebugLevel, 'backup.doBlockPushAndRaid %s : start process data from %s to %s, %d' % ( newblock.BlockNumber, filename, outputpath, id(self.terminating)))
def doReadRaid(self, *args, **kwargs): """ Action method. """ _, outfilename = tmpfile.make( 'restore', extension='.raid', prefix=self.backup_id.replace(':', '_').replace('@', '_').replace('/', '_') + '_' + str(self.block_number) + '_', close_fd=True, ) inputpath = os.path.join(settings.getLocalBackupsDir(), self.customer_id, self.path_id) task_params = (outfilename, self.EccMap.name, self.version, self.block_number, inputpath) raid_worker.add_task('read', task_params, lambda cmd, params, result: self._on_block_restored(result, outfilename))
def doReadRaid(self, arg): fd, outfilename = tmpfile.make( 'restore', prefix=self.BackupID.replace(':', '_').replace('@', '_').replace( '/', '_') + '_' + str(self.BlockNumber) + '_', ) os.close(fd) inputpath = os.path.join(settings.getLocalBackupsDir(), self.CustomerGlobalID, self.PathID) task_params = (outfilename, eccmap.CurrentName(), self.Version, self.BlockNumber, inputpath) raid_worker.add_task( 'read', task_params, lambda cmd, params, result: self._on_block_restored( result, outfilename))
def doBlockPushAndRaid(self, arg): """ Action method. """ newblock = arg if newblock is None: self.abort() self.automat('fail') lg.out( _DebugLevel, 'backup.doBlockPushAndRaid ERROR newblock is empty, terminating=%s' % self.terminating) lg.warn('failed to encrypt block, ABORTING') return if self.terminating: self.automat('block-raid-done', (newblock.BlockNumber, None)) lg.out(_DebugLevel, 'backup.doBlockPushAndRaid SKIP, terminating=True') return fileno, filename = tmpfile.make('raid') serializedblock = newblock.Serialize() blocklen = len(serializedblock) os.write(fileno, str(blocklen) + ":" + serializedblock) os.close(fileno) self.workBlocks[newblock.BlockNumber] = filename # key_alias = 'master' # if self.keyID: # key_alias = packetid.KeyAlias(self.keyID) dt = time.time() customer_dir = self.customerGlobalID # global_id.MakeGlobalID(customer=self.customerGlobalID, key_alias=key_alias) outputpath = os.path.join(settings.getLocalBackupsDir(), customer_dir, self.pathID, self.version) task_params = (filename, self.eccmap.name, self.version, newblock.BlockNumber, outputpath) raid_worker.add_task( 'make', task_params, lambda cmd, params, result: self._raidmakeCallback( params, result, dt), ) self.automat('block-raid-started', newblock) del serializedblock if _Debug: lg.out( _DebugLevel, 'backup.doBlockPushAndRaid %s : start process data from %s to %s, %d' % (newblock.BlockNumber, filename, outputpath, id(self.terminating)))
def doBlockPushAndRaid(self, *args, **kwargs): """ Action method. """ newblock = args[0] if newblock is None: self.abort() self.automat('fail') if _Debug: lg.out( _DebugLevel, 'backup.doBlockPushAndRaid ERROR newblock is empty, terminating=%s' % self.terminating) lg.warn('failed to encrypt block, ABORTING') return if self.terminating: self.automat('block-raid-done', (newblock.BlockNumber, None)) if _Debug: lg.out(_DebugLevel, 'backup.doBlockPushAndRaid SKIP, terminating=True') return fileno, filename = tmpfile.make('raid', extension='.raid') serializedblock = newblock.Serialize() blocklen = len(serializedblock) os.write(fileno, strng.to_bin(blocklen) + b":" + serializedblock) os.close(fileno) self.workBlocks[newblock.BlockNumber] = filename dt = time.time() outputpath = os.path.join(settings.getLocalBackupsDir(), self.customerGlobalID, self.pathID, self.version) task_params = (filename, self.eccmap.name, self.version, newblock.BlockNumber, outputpath) raid_worker.add_task( 'make', task_params, lambda cmd, params, result: self._raidmakeCallback( params, result, dt)) self.automat('block-raid-started', newblock) del serializedblock if _Debug: lg.out( _DebugLevel, 'backup.doBlockPushAndRaid %s : start process data from %s to %s, %d' % (newblock.BlockNumber, filename, outputpath, id(self.terminating)))