def _on_extract_done(self, retcode, backupID, source_filename,
output_location, backup_index):
tmpfile.throw_out(source_filename, 'file extracted')
for snapshot_filename in os.listdir(output_location):
snapshot_path = os.path.join(output_location, snapshot_filename)
snapshot_data = serialization.BytesToDict(
local_fs.ReadBinaryFile(snapshot_path), values_to_text=True)
for archive_message in snapshot_data.get('items', []):
if self.start_sequence_id is not None:
if self.start_sequence_id > archive_message['sequence_id']:
continue
if self.end_sequence_id is not None:
if self.end_sequence_id < archive_message['sequence_id']:
continue
self.extracted_messages.append(archive_message)
if _Debug:
lg.dbg(
_DebugLevel,
'archive snapshot %r extracted successfully to %r, extracted %d archive messages so far'
% (
source_filename,
output_location,
len(self.extracted_messages),
))
self._do_restore_next_backup(backup_index + 1)
return retcode
def doPopBlock(self, arg):
"""
Action method.
"""
blockNumber, result = arg
filename = self.workBlocks.pop(blockNumber)
tmpfile.throw_out(filename, 'block raid done')
def _on_restore_done(self, result, backupID, outfd, tarfilename):
try:
os.close(outfd)
except:
lg.exc()
if result == 'done':
lg.info('archive %r restore success from %r' % (
backupID,
tarfilename,
))
else:
lg.err('archive %r restore failed from %r with : %r' % (
backupID,
tarfilename,
result,
))
if result == 'done':
_, pathID, versionName = packetid.SplitBackupID(backupID)
service_dir = settings.ServiceDir('service_private_groups')
queues_dir = os.path.join(service_dir, 'queues')
queue_dir = os.path.join(queues_dir, self.group_key_id)
snapshot_dir = os.path.join(queue_dir, pathID, versionName)
if not os.path.isdir(snapshot_dir):
bpio._dirs_make(snapshot_dir)
d = backup_tar.extracttar_thread(tarfilename, snapshot_dir)
d.addCallback(self._on_extract_done, backupID, tarfilename,
snapshot_dir)
d.addErrback(self._on_extract_failed, backupID, tarfilename,
snapshot_dir)
return d
tmpfile.throw_out(tarfilename, 'restore ' + result)
return None
def doPopBlock(self, *args, **kwargs):
"""
Action method.
"""
blockNumber, _ = args[0]
filename = self.workBlocks.pop(blockNumber)
tmpfile.throw_out(filename, 'block raid done')
def restore_done(result, backupID, outfd, tarfilename, outputlocation,
callback_method):
global _WorkingBackupIDs
global _WorkingRestoreProgress
global OnRestoreDoneFunc
if result == 'done':
lg.info('restore success of %s with result=%s' % (backupID, result))
else:
lg.err('restore failed of %s with result=%s' % (backupID, result))
try:
os.close(outfd)
except:
lg.exc()
if result == 'done':
d = backup_tar.extracttar_thread(tarfilename, outputlocation)
d.addCallback(extract_done, backupID, tarfilename, outputlocation,
callback_method)
d.addErrback(extract_failed, backupID, tarfilename, outputlocation,
callback_method)
return d
_WorkingBackupIDs.pop(backupID, None)
_WorkingRestoreProgress.pop(backupID, None)
tmpfile.throw_out(tarfilename, 'restore ' + result)
if OnRestoreDoneFunc is not None:
OnRestoreDoneFunc(backupID, result)
if callback_method:
try:
callback_method(backupID, result)
except:
lg.exc()
return result
def doClose(self, *args, **kwargs):
"""
Action method.
"""
self.closed = True
for filename in self.workBlocks.values():
tmpfile.throw_out(filename, 'backup aborted')
def _on_restore_done(self, result, backup_id, outfd, tarfilename,
backup_index):
try:
os.close(outfd)
except:
lg.exc()
if result == 'done':
lg.info('archive %r restore success from %r' % (
backup_id,
tarfilename,
))
else:
lg.err('archive %r restore failed from %r with : %r' % (
backup_id,
tarfilename,
result,
))
if result != 'done':
tmpfile.throw_out(tarfilename, 'restore ' + result)
self.automat('restore-failed',
backup_id=backup_id,
tarfilename=tarfilename)
return None
self.automat('restore-done',
backup_id=backup_id,
tarfilename=tarfilename,
backup_index=backup_index)
return
def _on_extract_failed(self, err, backupID, source_filename,
output_location):
lg.err('archive %r extract failed from %r to %r with: %r' %
(backupID, source_filename, output_location, err))
tmpfile.throw_out(source_filename, 'file extract failed')
self.automat('extract-failed', err)
return None
def doPopBlock(self, arg):
"""
Action method.
"""
blockNumber, result = arg
filename = self.workBlocks.pop(blockNumber)
tmpfile.throw_out(filename, 'block raid done')
def _on_extract_done(self, retcode, backupID, source_filename,
output_location):
lg.info(
'archive %r snapshot from %r extracted successfully to %r : %r' % (
backupID,
source_filename,
output_location,
retcode,
))
tmpfile.throw_out(source_filename, 'file extracted')
return retcode
def doRemoveTempFile(self, *args, **kwargs):
"""
Action method.
"""
if not args or not len(args) > 1:
return
filename = args[1]
if filename:
tmpfile.throw_out(filename, 'block restored')
if settings.getBackupsKeepLocalCopies():
return
from storage import backup_matrix
from storage import backup_rebuilder
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == self.backup_id:
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRemoveTempFile SKIP because rebuilding in process'
)
return
count = 0
for supplierNum in range(
contactsdb.num_suppliers(customer_idurl=self.customer_idurl)):
supplierIDURL = contactsdb.supplier(
supplierNum, customer_idurl=self.customer_idurl)
if not supplierIDURL:
continue
for dataORparity in [
'Data',
'Parity',
]:
packetID = packetid.MakePacketID(self.backup_id,
self.block_number,
supplierNum, dataORparity)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, remotePath)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
backup_matrix.LocalBlockReport(self.backup_id, self.block_number,
*args, **kwargs)
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRemoveTempFile %d files were removed' %
count)
def extract_done(retcode, backupID, tarfilename, callback):
lg.out(4, 'restore_monitor.extract_done %s result: %s' % (backupID, str(retcode)))
global OnRestoreDoneFunc
_WorkingBackupIDs.pop(backupID, None)
_WorkingRestoreProgress.pop(backupID, None)
tmpfile.throw_out(tarfilename, 'file extracted')
if OnRestoreDoneFunc is not None:
OnRestoreDoneFunc(backupID, 'restore done')
if callback:
callback(backupID, 'restore done')
def doRemoveTempFile(self, arg):
try:
filename = arg[1]
except:
return
tmpfile.throw_out(filename, 'block restored')
if settings.getBackupsKeepLocalCopies():
return
import backup_rebuilder
import backup_matrix
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == self.BackupID:
lg.out(
6,
'restore.doRemoveTempFile SKIP because rebuilding in process'
)
return
count = 0
for supplierNum in xrange(
contactsdb.num_suppliers(customer_idurl=self.CustomerIDURL)):
supplierIDURL = contactsdb.supplier(
supplierNum, customer_idurl=self.CustomerIDURL)
if not supplierIDURL:
continue
for dataORparity in ['Data', 'Parity']:
packetID = packetid.MakePacketID(self.BackupID,
self.BlockNumber, supplierNum,
dataORparity)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, remotePath)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
backup_matrix.LocalBlockReport(self.BackupID, self.BlockNumber, arg)
lg.out(6, 'restore.doRemoveTempFile %d files were removed' % count)
def extract_done(retcode, backupID, source_filename, output_location,
callback_method):
lg.info('extract success of %r with result : %r' %
(backupID, str(retcode)))
global OnRestoreDoneFunc
_WorkingBackupIDs.pop(backupID, None)
_WorkingRestoreProgress.pop(backupID, None)
tmpfile.throw_out(source_filename, 'file extracted')
if OnRestoreDoneFunc is not None:
OnRestoreDoneFunc(backupID, 'restore done')
if callback_method:
try:
callback_method(backupID, 'restore done')
except:
lg.exc()
events.send('restore-done',
dict(
backup_id=backupID,
output_location=output_location,
))
return retcode
def restore_done(x, tarfilename, outputlocation, callback):
lg.out(4, '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
lg.out(4, 'restore_monitor.restore_done ' + str(x))
global _WorkingBackupIDs
global _WorkingRestoreProgress
global OnRestoreDoneFunc
backupID, result = x.split(' ')
if result == 'done':
p = backup_tar.extracttar(tarfilename, outputlocation)
if p:
d = threads.deferToThread(p.wait)
d.addBoth(extract_done, backupID, tarfilename, callback)
return
result = 'extract failed'
_WorkingBackupIDs.pop(backupID, None)
_WorkingRestoreProgress.pop(backupID, None)
tmpfile.throw_out(tarfilename, 'restore ' + result)
if OnRestoreDoneFunc is not None:
OnRestoreDoneFunc(backupID, result)
if callback:
callback(backupID, result)
def restore_failed(x, tarfilename, callback):
lg.out(4, 'restore_monitor.restore_failed ' + str(x))
global _WorkingBackupIDs
global _WorkingRestoreProgress
global OnRestoreDoneFunc
backupID = result = None
try:
if isinstance(x, Exception):
backupID, result = x.getErrorMessage().split(' ')
elif isinstance(x, str):
backupID, result = x.split(' ')
except:
lg.exc()
if not backupID:
lg.warn('Unknown backupID: %s' % str(x))
return
_WorkingBackupIDs.pop(backupID, None)
_WorkingRestoreProgress.pop(backupID, None)
tmpfile.throw_out(tarfilename, 'restore ' + result)
if OnRestoreDoneFunc is not None:
OnRestoreDoneFunc(backupID, result)
if callback:
callback(backupID, result)
def extract_failed(err, backupID, source_filename, output_location,
callback_method):
lg.err('extract failed of %s with: %s' % (backupID, str(err)))
global OnRestoreDoneFunc
_WorkingBackupIDs.pop(backupID, None)
_WorkingRestoreProgress.pop(backupID, None)
tmpfile.throw_out(source_filename, 'file extract failed')
if OnRestoreDoneFunc is not None:
OnRestoreDoneFunc(backupID, 'extract failed')
if callback_method:
try:
callback_method(backupID, 'extract failed')
except:
lg.exc()
events.send(
'restore-failed',
dict(
backup_id=backupID,
output_location=output_location,
reason='extracting file failed',
error=str(err),
))
return err
def doClose(self, arg):
"""
"""
self.closed = True
for filename in self.workBlocks.values():
tmpfile.throw_out(filename, 'backup aborted')
def doClose(self, arg):
self.closed = True
for filename in self.workBlocks.values():
tmpfile.throw_out(filename, 'backup aborted')
