def _do_select_archive_snapshots(self):
iterID_and_path = backup_fs.WalkByID(self.archive_folder_path,
iterID=backup_fs.fsID(
self.queue_owner_idurl,
self.queue_alias))
if iterID_and_path is None:
lg.err('did not found archive folder in the catalog: %r' %
self.archive_folder_path)
self.automat('restore-failed')
return False
iterID, _ = iterID_and_path
known_archive_snapshots_list = backup_fs.ListAllBackupIDsFull(
iterID=iterID)
if not known_archive_snapshots_list:
lg.err(
'failed to restore data from archive, no snapshots found in folder: %r'
% self.archive_folder_path)
self.automat('restore-failed')
return False
snapshots_list = []
for archive_item in known_archive_snapshots_list:
snapshots_list.append(archive_item[1])
if _Debug:
lg.args(_DebugLevel, snapshots_list=snapshots_list)
if not snapshots_list:
lg.err('no available snapshots found in archive list: %r' %
known_archive_snapshots_list)
self.automat('restore-failed')
return False
snapshot_sequence_ids = []
for backup_id in snapshots_list:
_, path_id, _ = packetid.SplitBackupID(backup_id)
if not path_id:
continue
try:
snapshot_sequence_id = int(path_id.split('/')[-1])
except:
lg.exc()
continue
if self.start_sequence_id is not None and self.start_sequence_id > snapshot_sequence_id:
continue
if self.end_sequence_id is not None and self.end_sequence_id < snapshot_sequence_id:
continue
snapshot_sequence_ids.append((
snapshot_sequence_id,
backup_id,
))
snapshot_sequence_ids.sort(key=lambda item: int(item[0]))
if _Debug:
lg.args(_DebugLevel, snapshot_sequence_ids=snapshot_sequence_ids)
self.selected_backups = [item[1] for item in snapshot_sequence_ids]
if not self.selected_backups:
lg.err('no backups selected from snapshot list')
self.automat('restore-failed')
return False
if _Debug:
lg.args(_DebugLevel, selected_backups=self.selected_backups)
return True
def doStartRestoreWorker(self, *args, **kwargs):
"""
Action method.
"""
iterID_and_path = backup_fs.WalkByID(self.archive_folder_path,
iterID=backup_fs.fsID(
self.queue_owner_idurl))
if iterID_and_path is None:
lg.err('did not found archive folder in the catalog: %r' %
self.archive_folder_path)
self.automat('restore-failed')
return
iterID, path = iterID_and_path
known_archive_snapshots_list = backup_fs.ListAllBackupIDsFull(
iterID=iterID)
if not known_archive_snapshots_list:
lg.err(
'failed to restore data from archive, no snapshots found in folder: %r'
% self.archive_folder_path)
self.automat('restore-failed')
return
snapshots_list = []
for archive_item in known_archive_snapshots_list:
snapshots_list.append(archive_item[1])
if _Debug:
lg.args(_DebugLevel, snapshots_list=snapshots_list)
if not snapshots_list:
lg.err('no available snapshots found in archive list: %r' %
known_archive_snapshots_list)
self.automat('restore-failed')
return
backupID = snapshots_list[0]
outfd, outfilename = tmpfile.make(
'restore',
extension='.tar.gz',
prefix=backupID.replace('@', '_').replace('.', '_').replace(
'/', '_').replace(':', '_') + '_',
)
rw = restore_worker.RestoreWorker(backupID,
outfd,
KeyID=self.group_key_id)
rw.MyDeferred.addCallback(self._on_restore_done, backupID, outfd,
outfilename)
rw.MyDeferred.addErrback(self._on_restore_failed, backupID, outfd,
outfilename)
if _Debug:
rw.MyDeferred.addErrback(
lg.errback,
debug=_Debug,
debug_level=_DebugLevel,
method='archive_reader.doStartRestoreWorker')
rw.automat('init')
def run(self):
"""
Runs a new ``Job`` from that ``Task``.
"""
iter_and_path = backup_fs.WalkByID(self.remotePath, iterID=backup_fs.fsID(self.customerIDURL))
if iter_and_path is None:
lg.out(4, 'backup_control.Task.run ERROR %s not found in the index' % self.remotePath)
# self.defer.callback('error', self.pathID)
# self._on_job_failed(self.pathID)
err = 'remote path "%s" not found in the catalog' % self.remotePath
OnTaskFailed(self.pathID, err)
return err
itemInfo, sourcePath = iter_and_path
if isinstance(itemInfo, dict):
try:
itemInfo = itemInfo[backup_fs.INFO_KEY]
except:
lg.exc()
# self._on_job_failed(self.pathID)
err = 'catalog item related to "%s" is broken' % self.remotePath
OnTaskFailed(self.pathID, err)
return err
if not self.localPath:
self.localPath = sourcePath
lg.out('backup_control.Task.run local path was populated from catalog: %s' % self.localPath)
if self.localPath != sourcePath:
lg.warn('local path is differ from catalog: %s != %s' % (self.localPath, sourcePath))
if not bpio.pathExist(self.localPath):
lg.warn('path not exist: %s' % self.localPath)
# self._on_job_failed(self.pathID)
err = 'local path "%s" not exist' % self.localPath
OnTaskFailed(self.pathID, err)
return err
# if os.path.isfile(self.localPath) and self.localPath != sourcePath:
# tmpfile.make(name, extension, prefix)
dataID = misc.NewBackupID()
if itemInfo.has_version(dataID):
# ups - we already have same version
# let's add 1,2,3... to the end to make absolutely unique version ID
i = 1
while itemInfo.has_version(dataID + str(i)):
i += 1
dataID += str(i)
self.backupID = packetid.MakeBackupID(
customer=self.fullCustomerID,
path_id=self.remotePath,
version=dataID,
)
if self.backupID in jobs():
lg.warn('backup job %s already started' % self.backupID)
return 'backup job %s already started' % self.backupID
try:
backup_fs.MakeLocalDir(settings.getLocalBackupsDir(), self.backupID)
except:
lg.exc()
lg.out(4, 'backup_control.Task.run ERROR creating destination folder for %s' % self.pathID)
# self.defer.callback('error', self.pathID)
# self._on_job_failed(self.backupID)
err = 'failed creating destination folder for "%s"' % self.backupID
return OnTaskFailed(self.backupID, err)
compress_mode = 'bz2' # 'none' # 'gz'
arcname = os.path.basename(sourcePath)
if bpio.pathIsDir(self.localPath):
backupPipe = backup_tar.backuptardir(self.localPath, arcname=arcname, compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(self.localPath, arcname=arcname, compress=compress_mode)
backupPipe.make_nonblocking()
job = backup.backup(
self.backupID,
backupPipe,
finishCallback=OnJobDone,
blockResultCallback=OnBackupBlockReport,
blockSize=settings.getBackupBlockSize(),
sourcePath=self.localPath,
keyID=self.keyID or itemInfo.key_id,
)
jobs()[self.backupID] = job
itemInfo.add_version(dataID)
if itemInfo.type == backup_fs.DIR:
dirsize.ask(self.localPath, OnFoundFolderSize, (self.pathID, dataID))
else:
sz = os.path.getsize(self.localPath)
jobs()[self.backupID].totalSize = sz
itemInfo.set_size(sz)
backup_fs.Calculate()
Save()
jobs()[self.backupID].automat('start')
reactor.callLater(0, FireTaskStartedCallbacks, self.pathID, dataID)
lg.out(4, 'backup_control.Task-%d.run [%s/%s], size=%d, %s' % (
self.number, self.pathID, dataID, itemInfo.size, self.localPath))
return None