def _backup_stream(config, src, dst, callbacks=None):
stream = src.get_stream()
# Gzip modifier
stream = Gzip(stream).get_stream()
src.suffix += '.gz'
# KeepLocal modifier
try:
keep_local_path = config.get('destination', 'keep_local_path')
kl_modifier = KeepLocal(stream,
os.path.join(keep_local_path, src.get_name()))
stream = kl_modifier.get_stream()
if callbacks is not None:
callbacks.append((kl_modifier, {
'keep_local_path': keep_local_path,
'dst': dst
}))
except ConfigParser.NoOptionError:
LOG.debug('keep_local_path is not present in the config file')
# GPG modifier
try:
stream = Gpg(stream, config.get('gpg', 'recipient'),
config.get('gpg', 'keyring')).get_stream()
src.suffix += '.gpg'
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
pass
except ModifierException as err:
LOG.warning(err)
LOG.warning('Will skip encryption')
if not dst.save(stream, src.get_name()):
LOG.error('Failed to save backup copy %s', src.get_name())
exit(1)
def restore_from_mysql_full(stream, dst_dir, config,
redo_only=False):
"""
Restore MySQL datadir from a backup copy
:param stream: Generator that provides backup copy
:param dst_dir: Path to destination directory. Must exist and be empty.
:type dst_dir: str
:param config: Tool configuration.
:type config: ConfigParser.ConfigParser
:param redo_only: True if the function has to do final apply of
the redo log. For example, if you restore backup from a full copy
it should be False. If you restore from incremental copy and
you restore base full copy redo_only should be True.
:type redo_only: bool
:return: If success, return True
:rtype: bool
"""
# GPG modifier
try:
gpg = Gpg(stream,
config.get('gpg', 'recipient'),
config.get('gpg', 'keyring'),
secret_keyring=config.get('gpg', 'secret_keyring'))
LOG.debug('Decrypting stream')
stream = gpg.revert_stream()
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
LOG.debug('Not decrypting the stream')
stream = Gzip(stream).revert_stream()
with stream as handler:
if not _extract_xbstream(handler, dst_dir):
return False
mem_usage = psutil.virtual_memory()
try:
xtrabackup_cmd = ['innobackupex',
'--use-memory=%d' % (mem_usage.available/2),
'--apply-log']
if redo_only:
xtrabackup_cmd += ['--redo-only']
xtrabackup_cmd += [dst_dir]
LOG.debug('Running %s', ' '.join(xtrabackup_cmd))
xtrabackup_proc = Popen(xtrabackup_cmd,
stdout=None,
stderr=None)
xtrabackup_proc.communicate()
ret = xtrabackup_proc.returncode
if ret:
LOG.error('%s exited with code %d', " ".join(xtrabackup_cmd), ret)
return ret == 0
except OSError as err:
LOG.error('Failed to prepare backup in %s: %s', dst_dir, err)
return False
def backup_files(run_type, config):
"""Backup local directories
:param run_type: Run type
:type run_type: str
:param config: Configuration
:type config: ConfigParser.ConfigParser
"""
for directory in get_directories_to_backup(config):
LOG.debug('copying %s', directory)
src = FileSource(directory, run_type)
dst = get_destination(config)
stream = src.get_stream()
# Gzip modifier
stream = Gzip(stream).get_stream()
src.suffix += '.gz'
# KeepLocal modifier
try:
keep_local_path = config.get('destination', 'keep_local_path')
# src.suffix = 'tar.gz.aaa'
dst_name = src.get_name()
kl_modifier = KeepLocal(stream,
os.path.join(keep_local_path, dst_name))
stream = kl_modifier.get_stream()
except ConfigParser.NoOptionError:
pass
# GPG modifier
try:
keyring = config.get('gpg', 'keyring')
recipient = config.get('gpg', 'recipient')
gpg = Gpg(stream, recipient, keyring)
stream = gpg.get_stream()
src.suffix += '.gpg'
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
pass
except ModifierException as err:
LOG.warning(err)
LOG.warning('Will skip encryption')
dst.save(stream, src.get_name())
src.apply_retention_policy(dst, config, run_type)
def restore_from_mysql_incremental(stream,
dst_dir,
config,
tmp_dir=None,
xtrabackup_binary=XTRABACKUP_BINARY,
xbstream_binary=XBSTREAM_BINARY):
"""
Restore MySQL datadir from an incremental copy.
:param stream: Generator that provides backup copy
:param dst_dir: Path to destination directory. Must exist and be empty.
:type dst_dir: str
:param config: Tool configuration.
:type config: ConfigParser.ConfigParser
:param tmp_dir: Path to temp dir
:type tmp_dir: str
:param xtrabackup_binary: Path to xtrabackup binary.
:param xbstream_binary: Path to xbstream binary
:return: If success, return True
:rtype: bool
"""
if tmp_dir is None:
try:
inc_dir = tempfile.mkdtemp()
except (IOError, OSError):
try:
empty_dir(dst_dir)
except (IOError, OSError):
raise
raise
else:
inc_dir = tmp_dir
# GPG modifier
try:
gpg = Gpg(stream,
config.get('gpg', 'recipient'),
config.get('gpg', 'keyring'),
secret_keyring=config.get('gpg', 'secret_keyring'))
LOG.debug('Decrypting stream')
stream = gpg.revert_stream()
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
LOG.debug('Not decrypting the stream')
stream = Gzip(stream).revert_stream()
with stream as handler:
if not _extract_xbstream(handler, inc_dir, xbstream_binary):
return False
try:
mem_usage = psutil.virtual_memory()
try:
xtrabackup_cmd = [
xtrabackup_binary,
'--use-memory=%d' % (mem_usage.available / 2), '--prepare',
'--apply-log-only',
'--target-dir=%s' % dst_dir
]
LOG.debug('Running %s', ' '.join(xtrabackup_cmd))
xtrabackup_proc = Popen(xtrabackup_cmd, stdout=None, stderr=None)
xtrabackup_proc.communicate()
ret = xtrabackup_proc.returncode
if ret:
LOG.error('%s exited with code %d', " ".join(xtrabackup_cmd),
ret)
return False
xtrabackup_cmd = [
xtrabackup_binary,
'--use-memory=%d' % (mem_usage.available / 2), '--prepare',
"--target-dir=%s" % dst_dir,
"--incremental-dir=%s" % inc_dir
]
LOG.debug('Running %s', ' '.join(xtrabackup_cmd))
xtrabackup_proc = Popen(xtrabackup_cmd, stdout=None, stderr=None)
xtrabackup_proc.communicate()
ret = xtrabackup_proc.returncode
if ret:
LOG.error('%s exited with code %d', " ".join(xtrabackup_cmd),
ret)
return ret == 0
except OSError as err:
LOG.error('Failed to prepare backup in %s: %s', dst_dir, err)
return False
finally:
try:
pass
except OSError as exc:
if exc.errno != errno.ENOENT: # ENOENT - no such file or directory
raise # re-raise exception
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: ConfigParser.ConfigParser
:return: None
"""
try:
if not config.getboolean('source', 'backup_mysql'):
raise TwinDBBackupError('MySQL backups are not enabled in config')
except (ConfigParser.NoOptionError, TwinDBBackupError) as err:
LOG.debug(err)
LOG.debug('Not backing up MySQL')
return
dst = get_destination(config)
try:
full_backup = config.get('mysql', 'full_backup')
except ConfigParser.NoOptionError:
full_backup = 'daily'
backup_start = time.time()
src = MySQLSource(
MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type,
full_backup, dst)
callbacks = []
stream = src.get_stream()
src_name = src.get_name()
# Gzip modifier
stream = Gzip(stream).get_stream()
src_name += '.gz'
# KeepLocal modifier
try:
keep_local_path = config.get('destination', 'keep_local_path')
kl_modifier = KeepLocal(stream, os.path.join(keep_local_path,
src_name))
stream = kl_modifier.get_stream()
callbacks.append((kl_modifier, {
'keep_local_path': keep_local_path,
'dst': dst
}))
except ConfigParser.NoOptionError:
LOG.debug('keep_local_path is not present in the config file')
# GPG modifier
try:
stream = Gpg(stream, config.get('gpg', 'recipient'),
config.get('gpg', 'keyring')).get_stream()
src_name += '.gpg'
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
pass
except ModifierException as err:
LOG.warning(err)
LOG.warning('Will skip encryption')
if not dst.save(stream, src_name):
LOG.error('Failed to save backup copy %s', src_name)
exit(1)
status = prepare_status(dst, src, run_type, src_name, backup_start)
src.apply_retention_policy(dst, config, run_type, status)
dst.status(status)
LOG.debug('Callbacks are %r', callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])