class SCRestore(SCOperation):
def __init__(self, filepath = None):
self.storage = BaseStorage.storage_factory()
if filepath:
self.filepath = filepath
else:
self.filepath = self.storage.get_latest_backup()
self.servername = DB_SERVERNAME
def restore_git_sources(self):
sys.stdout.write("Restoring to %s from %s\n" % (settings.GIT_SOURCES_BASE_DIRECTORY, self.directory))
ts = tarfile.open(os.path.join(self.directory, 'sources.tar'), 'r')
shutil.rmtree(settings.GIT_SOURCES_BASE_DIRECTORY, ignore_errors = True)
if not os.path.exists(settings.GIT_SOURCES_BASE_DIRECTORY):
os.mkdir(settings.GIT_SOURCES_BASE_DIRECTORY)
os.chdir(settings.GIT_SOURCES_BASE_DIRECTORY)
ts.extractall()
def restore_db(self):
database = settings.DATABASES['default']
self.dbcommands = DBCommands(database)
filepath = os.path.join(self.directory, 'dbbackup')
with open(filepath, 'r') as inputfile:
self.dbcommands.run_restore_commands(inputfile)
def restore_ruleset_middleware(self):
try:
middleware_backup = __import__("%s.%s" % (settings.RULESET_MIDDLEWARE, 'backup'))
except ImportError:
return
Probe = __import__(settings.RULESET_MIDDLEWARE)
Probe.backup.restore(self.directory)
def test_migration_level(self):
miglevel = None
with open(os.path.join(self.directory, 'miglevel'), 'r') as migfile:
miglevel = json.load(migfile)
return self.is_migration_level_lower(miglevel)
def run(self):
# extract archive in tmp directory
inputfile = self.storage.read_file(self.filepath)
call_dir = os.getcwd()
ts = tarfile.open(self.filepath, 'r', fileobj=inputfile)
tmpdir = tempfile.mkdtemp()
os.chdir(tmpdir)
ts.extractall()
ts.close()
self.directory = tmpdir
if self.test_migration_level() == False:
raise SCBackupException("Backup is newer than local Scirius version, please update local instance and apply migrations.")
self.restore_git_sources()
self.restore_db()
# Apply upgrades
call_command('migrate', '--noinput')
self.restore_ruleset_middleware()
shutil.rmtree(tmpdir)
os.chdir(call_dir)
class Command(BaseDbBackupCommand):
help = "dbrestore [-d <dbname>] [-f <filename>] [-s <servername>]"
option_list = BaseDbBackupCommand.option_list + (
make_option("-d", "--database", help="Database to restore"),
make_option("-f", "--filepath", help="Specific file to backup from"),
make_option("-x", "--backup-extension", help="The extension to use when scanning for files to restore from."),
make_option("-s", "--servername", help="Use a different servername backup"),
make_option("-l", "--list", action='store_true', default=False, help="List backups in the backup directory"),
make_option("-c", "--decrypt", help="Decrypt data before restoring", default=False, action='store_true'),
make_option("-p", "--passphrase", help="Passphrase for decrypt file", default=None),
make_option("-z", "--uncompress", help="Uncompress gzip data before restoring", action='store_true'),
)
def handle(self, **options):
""" Django command handler. """
self.verbosity = int(options.get('verbosity'))
self.quiet = options.get('quiet')
try:
connection.close()
self.filepath = options.get('filepath')
self.backup_extension = options.get('backup_extension') or 'backup'
self.servername = options.get('servername')
self.decrypt = options.get('decrypt')
self.uncompress = options.get('uncompress')
self.passphrase = options.get('passphrase')
self.database = self._get_database(options)
self.storage = BaseStorage.storage_factory()
self.dbcommands = DBCommands(self.database)
if options.get('list'):
return self.list_backups()
self.restore_backup()
except StorageError as err:
raise CommandError(err)
def _get_database(self, options):
""" Get the database to restore. """
database_key = options.get('database')
if not database_key:
if len(settings.DATABASES) >= 2:
errmsg = "Because this project contains more than one database, you"
errmsg += " must specify the --database option."
raise CommandError(errmsg)
database_key = list(settings.DATABASES.keys())[0]
return settings.DATABASES[database_key]
def restore_backup(self):
""" Restore the specified database. """
self.log("Restoring backup for database: %s" % self.database['NAME'], 1)
# Fetch the latest backup if filepath not specified
if not self.filepath:
self.log(" Finding latest backup", 1)
filepaths = self.storage.list_directory()
filepaths = [f for f in filepaths if f.endswith('.' + self.backup_extension)]
if not filepaths:
raise CommandError("No backup files found in: /%s" % self.storage.backup_dir)
self.filepath = filepaths[-1]
# Restore the specified filepath backup
self.log(" Restoring: %s" % self.filepath, 1)
input_filename = self.filepath
inputfile = self.storage.read_file(input_filename)
if self.decrypt:
unencrypted_file, input_filename = self.unencrypt_file(inputfile, input_filename)
inputfile.close()
inputfile = unencrypted_file
if self.uncompress:
uncompressed_file = self.uncompress_file(inputfile)
inputfile.close()
inputfile = uncompressed_file
self.log(" Restore tempfile created: %s" % utils.handle_size(inputfile), 1)
answer = input("Are you sure you want to continue? [Y/n]")
if answer.lower() not in ('y', 'yes', ''):
self.log("Quitting", 1)
sys.exit(0)
inputfile.seek(0)
self.dbcommands.run_restore_commands(inputfile)
def get_extension(self, filename):
_, extension = os.path.splitext(filename)
return extension
def uncompress_file(self, inputfile):
""" Uncompress this file using gzip. The input and the output are filelike objects. """
outputfile = tempfile.SpooledTemporaryFile(
max_size=500 * 1024 * 1024,
dir=dbbackup_settings.TMP_DIR)
zipfile = gzip.GzipFile(fileobj=inputfile, mode="rb")
try:
inputfile.seek(0)
outputfile.write(zipfile.read())
finally:
zipfile.close()
return outputfile
def unencrypt_file(self, inputfile, inputfilename):
""" Unencrypt this file using gpg. The input and the output are filelike objects. """
import gnupg
def get_passphrase():
return self.passphrase or getpass('Input Passphrase: ') or None
temp_dir = tempfile.mkdtemp(dir=dbbackup_settings.TMP_DIR)
try:
new_basename = os.path.basename(inputfilename).replace('.gpg', '')
temp_filename = os.path.join(temp_dir, new_basename)
try:
inputfile.seek(0)
g = gnupg.GPG()
result = g.decrypt_file(file=inputfile, passphrase=get_passphrase(), output=temp_filename)
if not result:
raise Exception('Decryption failed; status: %s' % result.status)
outputfile = tempfile.SpooledTemporaryFile(
max_size=10 * 1024 * 1024,
dir=dbbackup_settings.TMP_DIR)
outputfile.name = new_basename
f = open(temp_filename)
try:
outputfile.write(f.read())
finally:
f.close()
finally:
if os.path.exists(temp_filename):
os.remove(temp_filename)
finally:
os.rmdir(temp_dir)
return outputfile, new_basename
def list_backups(self):
""" List backups in the backup directory. """
self.log("Listing backups on %s in /%s:" % (self.storage.name, self.storage.backup_dir), 1)
for filepath in self.storage.list_directory():
self.log(" %s" % os.path.basename(filepath), 1)
class SCRestore(SCOperation):
def __init__(self, filepath=None):
self.storage = BaseStorage.storage_factory()
if filepath:
self.filepath = filepath
else:
self.filepath = self.storage.get_latest_backup()
self.servername = DB_SERVERNAME
def restore_git_sources(self):
sys.stdout.write("Restoring to %s from %s\n" %
(settings.GIT_SOURCES_BASE_DIRECTORY, self.directory))
ts = tarfile.open(os.path.join(self.directory, 'sources.tar'), 'r')
shutil.rmtree(settings.GIT_SOURCES_BASE_DIRECTORY, ignore_errors=True)
if not os.path.exists(settings.GIT_SOURCES_BASE_DIRECTORY):
os.mkdir(settings.GIT_SOURCES_BASE_DIRECTORY)
os.chdir(settings.GIT_SOURCES_BASE_DIRECTORY)
ts.extractall()
def restore_db(self):
database = settings.DATABASES['default']
self.dbcommands = DBCommands(database)
filepath = os.path.join(self.directory, 'dbbackup')
with open(filepath, 'r') as inputfile:
self.dbcommands.run_restore_commands(inputfile)
def restore_ruleset_middleware(self):
try:
__import__("%s.%s" % (settings.RULESET_MIDDLEWARE, 'backup'))
except ImportError:
return
probe_class = __import__(settings.RULESET_MIDDLEWARE)
probe_class.backup.restore(self.directory)
def test_migration_level(self):
miglevel = None
with open(os.path.join(self.directory, 'miglevel'), 'r') as migfile:
miglevel = json.load(migfile)
return self.is_migration_level_lower(miglevel)
def run(self):
# extract archive in tmp directory
inputfile = self.storage.read_file(self.filepath)
call_dir = os.getcwd()
ts = tarfile.open(self.filepath, 'r', fileobj=inputfile)
tmpdir = tempfile.mkdtemp()
os.chdir(tmpdir)
ts.extractall()
ts.close()
self.directory = tmpdir
if self.test_migration_level() is False:
raise SCBackupException(
"Backup is newer than local Scirius version, please update local instance and apply migrations."
)
self.restore_git_sources()
self.restore_db()
# Apply upgrades
call_command('migrate', '--noinput')
self.restore_ruleset_middleware()
shutil.rmtree(tmpdir)
os.chdir(call_dir)
class Command(LabelCommand):
help = "dbrestore [-d <dbname>] [-f <filename>] [-s <servername>]"
option_list = BaseCommand.option_list + (
make_option("-d", "--database", help="Database to restore"),
make_option("-f", "--filepath", help="Specific file to backup from"),
make_option(
"-x",
"--backup-extension",
help="The extension to use when scanning for files to restore from."
),
make_option(
"-s", "--servername", help="Use a different servername backup"),
make_option("-l",
"--list",
action='store_true',
default=False,
help="List backups in the backup directory"),
make_option("-c",
"--decrypt",
help="Decrypt data before restoring",
default=False,
action='store_true'),
make_option("-p",
"--passphrase",
help="Passphrase for decrypt file",
default=None),
make_option("-z",
"--uncompress",
help="Uncompress gzip data before restoring",
action='store_true'),
)
def handle(self, **options):
""" Django command handler. """
try:
connection.close()
self.filepath = options.get('filepath')
self.backup_extension = options.get('backup_extension') or 'backup'
self.servername = options.get('servername')
self.decrypt = options.get('decrypt')
self.uncompress = options.get('uncompress')
self.passphrase = options.get('passphrase')
self.database = self._get_database(options)
self.storage = BaseStorage.storage_factory()
self.dbcommands = DBCommands(self.database)
if options.get('list'):
return self.list_backups()
self.restore_backup()
except StorageError as err:
raise CommandError(err)
def _get_database(self, options):
""" Get the database to restore. """
database_key = options.get('database')
if not database_key:
if len(settings.DATABASES) >= 2:
errmsg = "Because this project contains more than one database, you"
errmsg += " must specify the --database option."
raise CommandError(errmsg)
database_key = list(settings.DATABASES.keys())[0]
return settings.DATABASES[database_key]
def restore_backup(self):
""" Restore the specified database. """
self.stdout.write("Restoring backup for database: %s" %
self.database['NAME'])
# Fetch the latest backup if filepath not specified
if not self.filepath:
self.stdout.write(" Finding latest backup")
filepaths = self.storage.list_directory()
filepaths = [
f for f in filepaths if f.endswith('.' + self.backup_extension)
]
if not filepaths:
raise CommandError("No backup files found in: /%s" %
self.storage.backup_dir)
self.filepath = filepaths[-1]
# Restore the specified filepath backup
self.stdout.write(" Restoring: %s" % self.filepath)
input_filename = self.filepath
inputfile = self.storage.read_file(input_filename)
if self.decrypt:
unencrypted_file = self.unencrypt_file(inputfile)
inputfile.close()
inputfile = unencrypted_file
input_filename = inputfile.name
if self.uncompress:
uncompressed_file = self.uncompress_file(inputfile)
inputfile.close()
inputfile = uncompressed_file
self.stdout.write(" Restore tempfile created: %s" %
utils.handle_size(inputfile))
answer = input("Are you sure you want to continue? [Y/n]")
if answer.lower() not in ('y', 'yes', ''):
self.stdout.write("Quitting")
sys.exit(0)
inputfile.seek(0)
self.dbcommands.run_restore_commands(inputfile)
def get_extension(self, filename):
_, extension = os.path.splitext(filename)
return extension
def uncompress_file(self, inputfile):
""" Uncompress this file using gzip. The input and the output are filelike objects. """
outputfile = tempfile.SpooledTemporaryFile(
max_size=500 * 1024 * 1024, dir=dbbackup_settings.TMP_DIR)
zipfile = gzip.GzipFile(fileobj=inputfile, mode="rb")
try:
inputfile.seek(0)
outputfile.write(zipfile.read())
finally:
zipfile.close()
return outputfile
def unencrypt_file(self, inputfile):
""" Unencrypt this file using gpg. The input and the output are filelike objects. """
import gnupg
def get_passphrase():
return self.passphrase or getpass('Input Passphrase: ') or None
temp_dir = tempfile.mkdtemp(dir=dbbackup_settings.TMP_DIR)
try:
inputfile.fileno(
) # Convert inputfile from SpooledTemporaryFile to regular file (Fixes Issue #21)
new_basename = os.path.basename(inputfile.name).replace('.gpg', '')
temp_filename = os.path.join(temp_dir, new_basename)
try:
inputfile.seek(0)
g = gnupg.GPG()
result = g.decrypt_file(file=inputfile,
passphrase=get_passphrase(),
output=temp_filename)
if not result:
raise Exception('Decryption failed; status: %s' %
result.status)
outputfile = tempfile.SpooledTemporaryFile(
max_size=10 * 1024 * 1024, dir=dbbackup_settings.TMP_DIR)
outputfile.name = new_basename
f = open(temp_filename)
try:
outputfile.write(f.read())
finally:
f.close()
finally:
if os.path.exists(temp_filename):
os.remove(temp_filename)
finally:
os.rmdir(temp_dir)
return outputfile
def list_backups(self):
""" List backups in the backup directory. """
self.stdout.write("Listing backups on %s in /%s:" %
(self.storage.name, self.storage.backup_dir))
for filepath in self.storage.list_directory():
self.stdout.write(" %s" % os.path.basename(filepath))
