def test__get_destination_ssh_invalid_port(config_content):
s_config = config_content.format(destination="ssh", port='foo')
buf = StringIO.StringIO(s_config)
config = ConfigParser.ConfigParser()
config.readfp(buf)
with pytest.raises(ValueError):
get_destination(config)
def restore_mysql(cfg, dst, backup_copy, cache):
"""Restore from mysql backup"""
LOG.debug('mysql: %r', cfg)
if not backup_copy:
LOG.info('No backup copy specified. Choose one from below:')
list_available_backups(cfg)
exit(1)
try:
ensure_empty(dst)
dst_storage = get_destination(
cfg,
get_hostname_from_backup_copy(backup_copy)
)
key = dst_storage.basename(backup_copy)
copy = dst_storage.status()[key]
if cache:
restore_from_mysql(cfg, copy, dst, cache=Cache(cache))
else:
restore_from_mysql(cfg, copy, dst)
except (TwinDBBackupError, CacheException) as err:
LOG.error(err)
exit(1)
except (OSError, IOError) as err:
LOG.error(err)
exit(1)
def verify_mysql_backup(config, dst_path, backup_copy, hostname=None):
"""Restore mysql backup and measure time"""
if backup_copy == "latest":
dst = get_destination(config, hostname)
url = dst.get_latest_backup()
else:
url = backup_copy
if url is None:
return json.dumps({
'backup_copy': url,
'restore_time': 0,
'success': False
}, indent=4, sort_keys=True)
start_restore_time = time.time()
success = True
try:
tmp_dir = tempfile.mkdtemp()
restore_from_mysql(config, url, dst_path, tmp_dir)
edit_backup_my_cnf(dst_path)
shutil.rmtree(tmp_dir)
except (TwinDBBackupError, OSError, IOError) as err:
LOG.error(err)
success = False
end_restore_time = time.time()
restore_time = end_restore_time - start_restore_time
return json.dumps({
'backup_copy': url,
'restore_time': restore_time,
'success': success
}, indent=4, sort_keys=True)
def status(cfg, copy_type, hostname):
"""Print backups status"""
dst = get_destination(cfg, hostname)
print(
dst.status(
cls=MEDIA_STATUS_MAP[copy_type]
)
)
def test__get_destination_ssh_valid_port(config_content):
s_config = config_content.format(destination="ssh", port=4321)
buf = StringIO.StringIO(s_config)
config = ConfigParser.ConfigParser()
config.readfp(buf)
dst = get_destination(config)
assert isinstance(dst, Ssh)
assert dst.client.port == 4321
def test__get_destination_ssh_valid_port_as_str(config_content):
s_config = config_content.format(destination="ssh", port='1234')
buf = StringIO.StringIO(s_config)
config = ConfigParser.ConfigParser()
config.readfp(buf)
dst = get_destination(config)
assert isinstance(dst, Ssh)
assert dst._ssh_client.ssh_connect_info.port == 1234
def restore_from_file(config, backup_copy, dst_dir):
"""
Restore a directory from a backup copy in the directory
:param config: Tool configuration.
:type config: ConfigParser.ConfigParser
:param backup_copy: Backup name.
:type backup_copy: str
:param dst_dir: Path to destination directory. Must exist and be empty.
:type dst_dir: str
"""
LOG.info('Restoring %s in %s', backup_copy, dst_dir)
mkdir_p(dst_dir)
restore_start = time.time()
if os.path.exists(backup_copy):
dst = Local(backup_copy)
stream = dst.get_stream(backup_copy)
else:
dst = get_destination(config)
stream = dst.get_stream(backup_copy)
# GPG modifier
try:
gpg = Gpg(stream,
config.get('gpg', 'recipient'),
config.get('gpg', 'keyring'),
secret_keyring=config.get('gpg', 'secret_keyring'))
LOG.debug('Decrypting stream')
stream = gpg.revert_stream()
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
LOG.debug('Not decrypting the stream')
with stream as handler:
try:
LOG.debug('handler type: %s', type(handler))
LOG.debug('stream type: %s', type(stream))
cmd = ["tar", "zvxf", "-"]
LOG.debug('Running %s', ' '.join(cmd))
proc = Popen(cmd, stdin=handler, cwd=dst_dir)
cout, cerr = proc.communicate()
ret = proc.returncode
if ret:
LOG.error('%s exited with code %d', cmd, ret)
if cout:
LOG.error('STDOUT: %s', cout)
if cerr:
LOG.error('STDERR: %s', cerr)
return
LOG.info('Successfully restored %s in %s', backup_copy, dst_dir)
except (OSError, DestinationError) as err:
LOG.error('Failed to decompress %s: %s', backup_copy, err)
exit(1)
export_info(config,
data=time.time() - restore_start,
category=ExportCategory.files,
measure_type=ExportMeasureType.restore)
def backup_binlogs(run_type, config): # pylint: disable=too-many-locals
"""Copy MySQL binlog files to the backup destination.
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: ConfigParser.ConfigParser
"""
dst = get_destination(config)
status = dst.status(cls=BinlogStatus)
try:
mysql_client = MySQLClient(
defaults_file=config.get('mysql', 'mysql_defaults_file'))
except ConfigParser.NoSectionError:
LOG.debug('No mysql section in the config. Not backing up binlogs')
return
last_copy = status.get_latest_backup()
LOG.debug('Latest copied binlog %s', last_copy)
with mysql_client.cursor() as cur:
cur.execute("FLUSH BINARY LOGS")
backup_set = binlogs_to_backup(
cur, last_binlog=last_copy.name if last_copy else None)
cur.execute("SELECT @@log_bin_basename")
row = cur.fetchone()
binlog_dir = osp.dirname(row['@@log_bin_basename'])
for binlog_name in backup_set:
src = BinlogSource(run_type, mysql_client, binlog_name)
binlog_copy = BinlogCopy(
src.host, binlog_name,
BinlogParser(osp.join(binlog_dir, binlog_name)).created_at)
_backup_stream(config, src, dst)
status.add(binlog_copy)
try:
expire_log_days = config.get('mysql', 'expire_log_days')
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
expire_log_days = 7
for copy in status:
now = int(time.time())
LOG.debug('Reviewing copy %s. Now: %d', copy, now)
if copy.created_at < now - expire_log_days * 24 * 3600:
LOG.debug('Deleting copy that was taken %d seconds ago',
now - copy.created_at)
dst.delete(copy.key + ".gz")
status.remove(copy.key)
dst.status(status, cls=BinlogStatus)
def share(config, s3_url):
"""
Function for generate make public file and get public url
:param config: Config file
:param s3_url: S3 url to file
:type s3_url: str
:raise: TwinDBBackupError
"""
dst = get_destination(config)
try:
print(dst.share(s3_url))
except NotImplementedError as err:
raise TwinDBBackupError(err)
def verify_mysql_backup(config, dst_path, backup_copy, hostname=None):
"""Restore mysql backup and measure time"""
dst = get_destination(config, hostname)
status = dst.status()
if backup_copy == "latest":
copy = status.get_latest_backup()
else:
key = dst.basename(backup_copy)
copy = status[key]
if copy is None:
return json.dumps(
{
'backup_copy': backup_copy,
'restore_time': 0,
'success': False
},
indent=4,
sort_keys=True)
start_restore_time = time.time()
success = True
tmp_dir = tempfile.mkdtemp()
try:
LOG.debug('Verifying backup copy in %s', tmp_dir)
restore_from_mysql(config, copy, dst_path, tmp_dir)
edit_backup_my_cnf(dst_path)
except (TwinDBBackupError, OSError, IOError) as err:
LOG.error(err)
LOG.debug(traceback.format_exc())
success = False
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
end_restore_time = time.time()
restore_time = end_restore_time - start_restore_time
return json.dumps(
{
'backup_copy': copy.key,
'restore_time': restore_time,
'success': success
},
indent=4,
sort_keys=True)
def backup_files(run_type, config):
"""Backup local directories
:param run_type: Run type
:type run_type: str
:param config: Configuration
:type config: ConfigParser.ConfigParser
"""
for directory in get_directories_to_backup(config):
LOG.debug('copying %s', directory)
src = FileSource(directory, run_type)
dst = get_destination(config)
stream = src.get_stream()
# Gzip modifier
stream = Gzip(stream).get_stream()
src.suffix += '.gz'
# KeepLocal modifier
try:
keep_local_path = config.get('destination', 'keep_local_path')
# src.suffix = 'tar.gz.aaa'
dst_name = src.get_name()
kl_modifier = KeepLocal(stream,
os.path.join(keep_local_path, dst_name))
stream = kl_modifier.get_stream()
except ConfigParser.NoOptionError:
pass
# GPG modifier
try:
keyring = config.get('gpg', 'keyring')
recipient = config.get('gpg', 'recipient')
gpg = Gpg(stream, recipient, keyring)
stream = gpg.get_stream()
src.suffix += '.gpg'
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
pass
except ModifierException as err:
LOG.warning(err)
LOG.warning('Will skip encryption')
dst.save(stream, src.get_name())
src.apply_retention_policy(dst, config, run_type)
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: ConfigParser.ConfigParser
:return: None
"""
try:
if not config.getboolean('source', 'backup_mysql'):
raise TwinDBBackupError('MySQL backups are not enabled in config')
except (ConfigParser.NoOptionError, TwinDBBackupError) as err:
LOG.debug(err)
LOG.debug('Not backing up MySQL')
return
dst = get_destination(config)
try:
full_backup = config.get('mysql', 'full_backup')
except ConfigParser.NoOptionError:
full_backup = 'daily'
backup_start = time.time()
src = MySQLSource(
MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type,
full_backup, dst)
callbacks = []
src_name = _backup_stream(config, src, dst, callbacks)
status = prepare_status(dst, src, run_type, src_name, backup_start)
status = src.apply_retention_policy(dst, config, run_type, status)
backup_duration = \
status[run_type][src_name]['backup_finished'] - \
status[run_type][src_name]['backup_started']
export_info(config,
data=backup_duration,
category=ExportCategory.mysql,
measure_type=ExportMeasureType.backup)
dst.status(status)
LOG.debug('Callbacks are %r', callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])
def backup_files(run_type, config):
"""Backup local directories
:param run_type: Run type
:type run_type: str
:param config: Configuration
:type config: ConfigParser.ConfigParser
"""
backup_start = time.time()
for directory in get_directories_to_backup(config):
LOG.debug('copying %s', directory)
src = FileSource(directory, run_type)
dst = get_destination(config)
_backup_stream(config, src, dst)
src.apply_retention_policy(dst, config, run_type)
export_info(config,
data=time.time() - backup_start,
category=ExportCategory.files,
measure_type=ExportMeasureType.backup)
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: ConfigParser.ConfigParser
:return: None
"""
try:
if not config.getboolean('source', 'backup_mysql'):
raise TwinDBBackupError('MySQL backups are not enabled in config')
except (ConfigParser.NoOptionError, TwinDBBackupError) as err:
LOG.debug(err)
LOG.debug('Not backing up MySQL')
return
dst = get_destination(config)
try:
full_backup = config.get('mysql', 'full_backup')
except ConfigParser.NoOptionError:
full_backup = 'daily'
backup_start = time.time()
try:
xtrabackup_binary = config.get('mysql', 'xtrabackup_binary')
except ConfigParser.NoOptionError:
xtrabackup_binary = XTRABACKUP_BINARY
status = dst.status()
kwargs = {
'backup_type': status.next_backup_type(full_backup, run_type),
'dst': dst,
'xtrabackup_binary': xtrabackup_binary
}
parent = status.eligble_parent(run_type)
if kwargs['backup_type'] == 'incremental':
kwargs['parent_lsn'] = parent.lsn
LOG.debug('Creating source %r', kwargs)
src = MySQLSource(
MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type,
**kwargs)
callbacks = []
_backup_stream(config, src, dst, callbacks=callbacks)
LOG.debug('Backup copy name: %s', src.get_name())
kwargs = {
'type': src.type,
'binlog': src.binlog_coordinate[0],
'position': src.binlog_coordinate[1],
'lsn': src.lsn,
'backup_started': backup_start,
'backup_finished': time.time(),
'config_files': my_cnfs(MY_CNF_COMMON_PATHS)
}
if src.incremental:
kwargs['parent'] = parent.key
backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs)
status.add(backup_copy)
status = src.apply_retention_policy(dst, config, run_type, status)
LOG.debug('status after apply_retention_policy():\n%s', status)
backup_duration = status.backup_duration(run_type, src.get_name())
export_info(config,
data=backup_duration,
category=ExportCategory.mysql,
measure_type=ExportMeasureType.backup)
dst.status(status)
LOG.debug('Callbacks are %r', callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])
def status(cfg):
"""Print backups status"""
dst = get_destination(cfg)
print(json.dumps(dst.status(), indent=4, sort_keys=True))
def restore_from_mysql(config,
copy,
dst_dir,
tmp_dir=None,
cache=None,
hostname=None):
"""
Restore MySQL datadir in a given directory
:param config: Tool configuration.
:type config: ConfigParser.ConfigParser
:param copy: Backup copy instance.
:type copy: MySQLCopy
:param dst_dir: Destination directory. Must exist and be empty.
:type dst_dir: str
:param tmp_dir: Path to temp directory
:type tmp_dir: str
:param cache: Local cache object.
:type cache: Cache
:param hostname: Hostname
:type hostname: str
"""
LOG.info('Restoring %s in %s', copy, dst_dir)
mkdir_p(dst_dir)
dst = None
restore_start = time.time()
try:
xtrabackup_binary = config.get('mysql', 'xtrabackup_binary')
except ConfigParser.NoOptionError:
xtrabackup_binary = XTRABACKUP_BINARY
try:
xbstream_binary = config.get('mysql', 'xbstream_binary')
except ConfigParser.NoOptionError:
xbstream_binary = XBSTREAM_BINARY
try:
keep_local_path = config.get('destination', 'keep_local_path')
if osp.exists(osp.join(keep_local_path, copy.key)):
dst = Local(keep_local_path)
except ConfigParser.NoOptionError:
pass
if not dst:
if not hostname:
hostname = copy.host
if not hostname:
raise DestinationError('Failed to get hostname from %s' % copy)
dst = get_destination(config, hostname=hostname)
key = copy.key
status = dst.status()
stream = dst.get_stream(copy)
if status[key].type == "full":
cache_key = os.path.basename(key)
if cache:
if cache_key in cache:
# restore from cache
cache.restore_in(cache_key, dst_dir)
else:
restore_from_mysql_full(stream,
dst_dir,
config,
redo_only=False,
xtrabackup_binary=xtrabackup_binary,
xbstream_binary=xbstream_binary)
cache.add(dst_dir, cache_key)
else:
restore_from_mysql_full(stream,
dst_dir,
config,
redo_only=False,
xtrabackup_binary=xtrabackup_binary,
xbstream_binary=xbstream_binary)
else:
full_copy = status.candidate_parent(copy.run_type)
full_stream = dst.get_stream(full_copy)
LOG.debug("Full parent copy is %s", full_copy.key)
cache_key = os.path.basename(full_copy.key)
if cache:
if cache_key in cache:
# restore from cache
cache.restore_in(cache_key, dst_dir)
else:
restore_from_mysql_full(full_stream,
dst_dir,
config,
redo_only=True,
xtrabackup_binary=xtrabackup_binary,
xbstream_binary=xbstream_binary)
cache.add(dst_dir, cache_key)
else:
restore_from_mysql_full(full_stream,
dst_dir,
config,
redo_only=True,
xtrabackup_binary=xtrabackup_binary,
xbstream_binary=xbstream_binary)
restore_from_mysql_incremental(stream,
dst_dir,
config,
tmp_dir,
xtrabackup_binary=xtrabackup_binary,
xbstream_binary=xbstream_binary)
config_dir = os.path.join(dst_dir, "_config")
for path, content in get_my_cnf(status, key):
config_sub_dir = os.path.join(config_dir,
os.path.dirname(path).lstrip('/'))
mkdir_p(config_sub_dir, mode=0755)
with open(os.path.join(config_sub_dir, os.path.basename(path)),
'w') as mysql_config:
mysql_config.write(content)
update_grastate(dst_dir, status, key)
export_info(config,
data=time.time() - restore_start,
category=ExportCategory.mysql,
measure_type=ExportMeasureType.restore)
LOG.info('Successfully restored %s in %s.', copy.key, dst_dir)
LOG.info(
'Now copy content of %s to MySQL datadir: '
'cp -R %s /var/lib/mysql/', dst_dir, osp.join(dst_dir, '*'))
LOG.info('Fix permissions: chown -R mysql:mysql /var/lib/mysql/')
LOG.info(
'Make sure innodb_log_file_size and innodb_log_files_in_group '
'in %s/backup-my.cnf and in /etc/my.cnf are same.', dst_dir)
if osp.exists(config_dir):
LOG.info('Original my.cnf is restored in %s.', config_dir)
LOG.info('Then you can start MySQL normally.')
def restore_from_mysql(config, backup_copy, dst_dir, cache=None):
"""
Restore MySQL datadir in a given directory
:param config: Tool configuration.
:type config: ConfigParser.ConfigParser
:param backup_copy: Backup copy name.
:type backup_copy: str
:param dst_dir: Destination directory. Must exist and be empty.
:type dst_dir: str
:param cache: Local cache object.
:type cache: Cache
"""
LOG.info('Restoring %s in %s', backup_copy, dst_dir)
mkdir_p(dst_dir)
dst = None
try:
keep_local_path = config.get('destination', 'keep_local_path')
if os.path.exists(backup_copy) \
and backup_copy.startswith(keep_local_path):
dst = Local(keep_local_path)
except ConfigParser.NoOptionError:
pass
if not dst:
hostname = get_hostname_from_backup_copy(backup_copy)
if not hostname:
raise DestinationError('Failed to get hostname from %s'
% backup_copy)
dst = get_destination(config, hostname=hostname)
key = dst.basename(backup_copy)
status = dst.status()
stream = dst.get_stream(backup_copy)
if get_backup_type(status, key) == "full":
cache_key = os.path.basename(key)
if cache:
if cache_key in cache:
# restore from cache
cache.restore_in(cache_key, dst_dir)
else:
restore_from_mysql_full(stream, dst_dir, config)
cache.add(dst_dir, cache_key)
else:
restore_from_mysql_full(stream, dst_dir, config)
else:
full_copy = dst.get_full_copy_name(backup_copy)
full_stream = dst.get_stream(full_copy)
cache_key = os.path.basename(full_copy)
if cache:
if cache_key in cache:
# restore from cache
cache.restore_in(cache_key, dst_dir)
else:
restore_from_mysql_full(full_stream, dst_dir,
config, redo_only=True)
cache.add(dst_dir, cache_key)
else:
restore_from_mysql_full(full_stream, dst_dir,
config, redo_only=True)
restore_from_mysql_incremental(stream, dst_dir, config)
config_dir = os.path.join(dst_dir, "_config")
for path, content in get_my_cnf(status, key):
config_sub_dir = os.path.join(config_dir,
os.path.dirname(path).lstrip('/'))
os.makedirs(config_sub_dir)
with open(os.path.join(config_sub_dir,
os.path.basename(path)), 'w') as mysql_config:
mysql_config.write(content)
update_grastate(dst_dir, status, key)
LOG.info('Successfully restored %s in %s.', backup_copy, dst_dir)
LOG.info('Now copy content of %s to MySQL datadir: '
'cp -R %s/* /var/lib/mysql/', dst_dir, dst_dir)
LOG.info('Fix permissions: chown -R mysql:mysql /var/lib/mysql/')
LOG.info('Make sure innodb_log_file_size and innodb_log_files_in_group '
'in %s/backup-my.cnf and in /etc/my.cnf are same.', dst_dir)
if os.path.exists(config_dir):
LOG.info('Original my.cnf is restored in %s.', config_dir)
LOG.info('Then you can start MySQL normally.')
def status(cfg):
"""Print backups status"""
dst = get_destination(cfg)
print(dst.status())
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: ConfigParser.ConfigParser
:return: None
"""
try:
if not config.getboolean('source', 'backup_mysql'):
raise TwinDBBackupError('MySQL backups are not enabled in config')
except (ConfigParser.NoOptionError, TwinDBBackupError) as err:
LOG.debug(err)
LOG.debug('Not backing up MySQL')
return
dst = get_destination(config)
try:
full_backup = config.get('mysql', 'full_backup')
except ConfigParser.NoOptionError:
full_backup = 'daily'
backup_start = time.time()
src = MySQLSource(
MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type,
full_backup, dst)
callbacks = []
stream = src.get_stream()
src_name = src.get_name()
# Gzip modifier
stream = Gzip(stream).get_stream()
src_name += '.gz'
# KeepLocal modifier
try:
keep_local_path = config.get('destination', 'keep_local_path')
kl_modifier = KeepLocal(stream, os.path.join(keep_local_path,
src_name))
stream = kl_modifier.get_stream()
callbacks.append((kl_modifier, {
'keep_local_path': keep_local_path,
'dst': dst
}))
except ConfigParser.NoOptionError:
LOG.debug('keep_local_path is not present in the config file')
# GPG modifier
try:
stream = Gpg(stream, config.get('gpg', 'recipient'),
config.get('gpg', 'keyring')).get_stream()
src_name += '.gpg'
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
pass
except ModifierException as err:
LOG.warning(err)
LOG.warning('Will skip encryption')
if not dst.save(stream, src_name):
LOG.error('Failed to save backup copy %s', src_name)
exit(1)
status = prepare_status(dst, src, run_type, src_name, backup_start)
src.apply_retention_policy(dst, config, run_type, status)
dst.status(status)
LOG.debug('Callbacks are %r', callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])
