def _start_one_block(self):
from storage import backup_matrix
if self.blockIndex < 0:
lg.out(
10,
'backup_rebuilder._start_one_block finish all blocks blockIndex=%d'
% self.blockIndex)
reactor.callLater(0, self._finish_rebuilding)
return
BlockNumber = self.workingBlocksQueue[self.blockIndex]
lg.out(
10,
'backup_rebuilder._start_one_block %d to rebuild, blockIndex=%d, other blocks: %s'
% ((BlockNumber, self.blockIndex, str(self.workingBlocksQueue))))
task_params = (
self.currentBackupID,
BlockNumber,
eccmap.Current(),
backup_matrix.GetActiveArray(),
backup_matrix.GetRemoteMatrix(self.currentBackupID, BlockNumber),
backup_matrix.GetLocalMatrix(self.currentBackupID, BlockNumber),
)
raid_worker.add_task(
'rebuild', task_params,
lambda cmd, params, result: self._block_finished(result, params))
def doBlockPushAndRaid(self, arg):
"""
Action method.
"""
newblock = arg
if self.terminating:
self.automat('block-raid-done', (newblock.BlockNumber, None))
lg.out(_DebugLevel, 'backup.doBlockPushAndRaid SKIP, terminating=True')
return
fileno, filename = tmpfile.make('raid')
serializedblock = newblock.Serialize()
blocklen = len(serializedblock)
os.write(fileno, str(blocklen) + ":" + serializedblock)
os.close(fileno)
self.workBlocks[newblock.BlockNumber] = filename
dt = time.time()
outputpath = os.path.join(settings.getLocalBackupsDir(), self.backupID)
task_params = (filename, self.eccmap.name, self.backupID, newblock.BlockNumber, outputpath)
raid_worker.add_task('make', task_params,
lambda cmd, params, result: self._raidmakeCallback(params, result, dt),)
self.automat('block-raid-started', newblock)
del serializedblock
if _Debug:
lg.out(_DebugLevel, 'backup.doBlockPushAndRaid %s : start process data from %s to %s, %d' % (
newblock.BlockNumber, filename, outputpath, id(self.terminating)))
def doReadRaid(self, *args, **kwargs):
"""
Action method.
"""
_, outfilename = tmpfile.make(
'restore',
extension='.raid',
prefix=self.backup_id.replace(':', '_').replace('@', '_').replace('/', '_') + '_' + str(self.block_number) + '_',
close_fd=True,
)
inputpath = os.path.join(settings.getLocalBackupsDir(), self.customer_id, self.path_id)
task_params = (outfilename, self.EccMap.name, self.version, self.block_number, inputpath)
raid_worker.add_task('read', task_params, lambda cmd, params, result: self._on_block_restored(result, outfilename))
def doReadRaid(self, arg):
fd, outfilename = tmpfile.make(
'restore',
prefix=self.BackupID.replace(':', '_').replace('@', '_').replace(
'/', '_') + '_' + str(self.BlockNumber) + '_',
)
os.close(fd)
inputpath = os.path.join(settings.getLocalBackupsDir(),
self.CustomerGlobalID, self.PathID)
task_params = (outfilename, eccmap.CurrentName(), self.Version,
self.BlockNumber, inputpath)
raid_worker.add_task(
'read', task_params,
lambda cmd, params, result: self._on_block_restored(
result, outfilename))
def doBlockPushAndRaid(self, arg):
"""
Action method.
"""
newblock = arg
if newblock is None:
self.abort()
self.automat('fail')
lg.out(
_DebugLevel,
'backup.doBlockPushAndRaid ERROR newblock is empty, terminating=%s'
% self.terminating)
lg.warn('failed to encrypt block, ABORTING')
return
if self.terminating:
self.automat('block-raid-done', (newblock.BlockNumber, None))
lg.out(_DebugLevel,
'backup.doBlockPushAndRaid SKIP, terminating=True')
return
fileno, filename = tmpfile.make('raid')
serializedblock = newblock.Serialize()
blocklen = len(serializedblock)
os.write(fileno, str(blocklen) + ":" + serializedblock)
os.close(fileno)
self.workBlocks[newblock.BlockNumber] = filename
# key_alias = 'master'
# if self.keyID:
# key_alias = packetid.KeyAlias(self.keyID)
dt = time.time()
customer_dir = self.customerGlobalID # global_id.MakeGlobalID(customer=self.customerGlobalID, key_alias=key_alias)
outputpath = os.path.join(settings.getLocalBackupsDir(), customer_dir,
self.pathID, self.version)
task_params = (filename, self.eccmap.name, self.version,
newblock.BlockNumber, outputpath)
raid_worker.add_task(
'make',
task_params,
lambda cmd, params, result: self._raidmakeCallback(
params, result, dt),
)
self.automat('block-raid-started', newblock)
del serializedblock
if _Debug:
lg.out(
_DebugLevel,
'backup.doBlockPushAndRaid %s : start process data from %s to %s, %d'
% (newblock.BlockNumber, filename, outputpath,
id(self.terminating)))
def doBlockPushAndRaid(self, *args, **kwargs):
"""
Action method.
"""
newblock = args[0]
if newblock is None:
self.abort()
self.automat('fail')
if _Debug:
lg.out(
_DebugLevel,
'backup.doBlockPushAndRaid ERROR newblock is empty, terminating=%s'
% self.terminating)
lg.warn('failed to encrypt block, ABORTING')
return
if self.terminating:
self.automat('block-raid-done', (newblock.BlockNumber, None))
if _Debug:
lg.out(_DebugLevel,
'backup.doBlockPushAndRaid SKIP, terminating=True')
return
fileno, filename = tmpfile.make('raid', extension='.raid')
serializedblock = newblock.Serialize()
blocklen = len(serializedblock)
os.write(fileno, strng.to_bin(blocklen) + b":" + serializedblock)
os.close(fileno)
self.workBlocks[newblock.BlockNumber] = filename
dt = time.time()
outputpath = os.path.join(settings.getLocalBackupsDir(),
self.customerGlobalID, self.pathID,
self.version)
task_params = (filename, self.eccmap.name, self.version,
newblock.BlockNumber, outputpath)
raid_worker.add_task(
'make', task_params,
lambda cmd, params, result: self._raidmakeCallback(
params, result, dt))
self.automat('block-raid-started', newblock)
del serializedblock
if _Debug:
lg.out(
_DebugLevel,
'backup.doBlockPushAndRaid %s : start process data from %s to %s, %d'
% (newblock.BlockNumber, filename, outputpath,
id(self.terminating)))
