def test_apply_retention_policy(mock_get_files_to_delete,
mock_delete_local_files,
mock_get_prefix,
tmpdir):
mock_get_files_to_delete.return_value = []
mock_get_prefix.return_value = 'master.box/hourly'
my_cnf = tmpdir.join('my.cnf')
mock_config = mock.Mock()
src = MySQLSource(
MySQLConnectInfo(str(my_cnf)),
'hourly',
'full',
dst=mock.Mock()
)
mock_dst = mock.Mock()
mock_dst.remote_path = '/foo/bar'
# noinspection PyTypeChecker
src.apply_retention_policy(mock_dst, mock_config, 'hourly', mock.Mock())
mock_delete_local_files.assert_called_once_with('mysql', mock_config)
mock_dst.list_files.assert_called_once_with(
'/foo/bar/master.box/hourly/mysql',
files_only=True
)
def test__get_connection_raises_mysql_source_error(mock_connect):
mock_connect.side_effect = OperationalError
source = MySQLSource(MySQLConnectInfo(None, hostname=None), 'daily',
'full')
with pytest.raises(MySQLSourceError):
with source.get_connection():
pass
def test_full_copy_exists(run_type, full_backup, status, expected):
mock_dst = mock.Mock()
mock_dst.status.return_value = status
src = MySQLSource(MySQLConnectInfo('/foo/bar'), run_type, full_backup,
mock_dst)
assert src._full_copy_exists() == expected
def test_get_backup_type(full_backup, run_type, backup_type, status, tmpdir):
mock_dst = mock.Mock()
mock_dst.status.return_value = status
src = MySQLSource(MySQLConnectInfo('/foo/bar'), run_type, full_backup,
mock_dst)
assert src._get_backup_type() == backup_type
def test_get_name(mock_time, mock_socket):
src = MySQLSource(MySQLConnectInfo('/foo/bar'), 'daily', 'hourly',
mock.Mock())
host = 'some-host'
mock_socket.gethostname.return_value = host
timestamp = '2017-02-13_15_40_29'
mock_time.strftime.return_value = timestamp
assert src.get_name(
) == "some-host/daily/mysql/mysql-2017-02-13_15_40_29.xbstream"
def test__is_galera_returns_true_on_str_higher_wsrep_on(mock_connect):
logging.basicConfig()
mock_cursor = mock.MagicMock()
mock_cursor.fetchone.return_value = {'wsrep_on': 'ON'}
mock_connect.return_value.__enter__.return_value. \
cursor.return_value.__enter__.return_value = mock_cursor
source = MySQLSource(MySQLConnectInfo(None), 'daily', 'daily', None)
assert source.is_galera() is True
def test__is_galera_returns_false_on_int_wsrep_on(mock_connect):
logging.basicConfig()
mock_cursor = mock.MagicMock()
mock_cursor.fetchone.return_value = {'wsrep_on': 0}
mock_connect.return_value.__enter__.return_value. \
cursor.return_value.__enter__.return_value = mock_cursor
source = MySQLSource(MySQLConnectInfo(None), 'daily', 'full')
assert source.is_galera() is False
def test__is_galera_returns_false_on_int_wsrep_on(mock_connect):
logging.basicConfig()
mock_cursor = mock.MagicMock()
mock_cursor.fetchone.return_value = {'wsrep_on': 0}
mock_connect.return_value.__enter__.return_value. \
cursor.return_value.__enter__.return_value = mock_cursor
source = MySQLSource(MySQLConnectInfo(None), 'daily', 'full')
assert source.is_galera() is False
def test__enable_wsrep_desync_sets_wsrep_desync_to_on(mock_connect):
logging.basicConfig()
mock_cursor = mock.MagicMock()
mock_connect.return_value.__enter__.return_value. \
cursor.return_value.__enter__.return_value = mock_cursor
source = MySQLSource(MySQLConnectInfo(None), 'daily', 'full')
source.enable_wsrep_desync()
mock_cursor.execute.assert_called_with("SET GLOBAL wsrep_desync=ON")
def test__is_galera_returns_true_on_galera_node(mock_connect):
logging.basicConfig()
mock_cursor = mock.MagicMock()
mock_cursor.execute.side_effect = InternalError(
1193, "Unknown system variable "
"'wsrep_on'")
mock_connect.return_value.__enter__.return_value. \
cursor.return_value.__enter__.return_value = mock_cursor
source = MySQLSource(MySQLConnectInfo(None), 'daily', 'full')
assert source.is_galera() is False
def test__is_galera_returns_true_on_galera_node(mock_connect):
logging.basicConfig()
mock_cursor = mock.MagicMock()
mock_cursor.execute.side_effect = InternalError(1193,
"Unknown system variable "
"'wsrep_on'")
mock_connect.return_value.__enter__.return_value. \
cursor.return_value.__enter__.return_value = mock_cursor
source = MySQLSource(MySQLConnectInfo(None), 'daily', 'full')
assert source.is_galera() is False
def test_get_name(mock_time, mock_socket):
host = 'some-host'
mock_socket.gethostname.return_value = host
timestamp = '2017-02-13_15_40_29'
mock_time.strftime.return_value = timestamp
src = MySQLSource(
MySQLConnectInfo('/foo/bar'),
'daily', 'full',
dst=mock.Mock()
)
assert src.get_name() == "some-host/daily/mysql/mysql-2017-02-13_15_40_29.xbstream"
def test_last_full_lsn(full_backup, run_type, status, tmpdir):
mock_dst = mock.Mock()
mock_dst.status.return_value = status
src = MySQLSource(MySQLConnectInfo('/foo/bar'), run_type, full_backup,
mock_dst)
assert src.parent_lsn == 19629412
def test_mysql_raises_on_wrong_run_type(run_type):
with pytest.raises(MySQLSourceError):
MySQLSource(
MySQLConnectInfo('/foo/bar'),
'foo', 'full',
dst=mock.Mock()
)
def test_mysql_source_raises_on_wrong_connect_info():
with pytest.raises(MySQLSourceError):
MySQLSource(
'/foo/bar',
'hourly', 'full',
dst=mock.Mock()
)
def test_apply_retention_policy(mock_get_files_to_delete,
mock_delete_local_files, mock_get_prefix,
tmpdir):
mock_get_files_to_delete.return_value = []
mock_get_prefix.return_value = 'master.box/hourly'
my_cnf = tmpdir.join('my.cnf')
mock_config = mock.Mock()
src = MySQLSource(MySQLConnectInfo(str(my_cnf)), 'hourly', 'daily',
mock.Mock())
mock_dst = mock.Mock()
mock_dst.remote_path = '/foo/bar'
src.apply_retention_policy(mock_dst, mock_config, 'hourly', mock.Mock())
mock_delete_local_files.assert_called_once_with('mysql', mock_config)
mock_dst.list_files.assert_called_once_with(
'/foo/bar/master.box/hourly/mysql/mysql-')
def test__disable_wsrep_desync_sets_wsrep_desync_to_off(mock_connect):
logging.basicConfig()
mock_cursor = mock.MagicMock()
mock_cursor.fetchall.return_value = [
{'Variable_name': 'wsrep_local_recv_queue', 'Value': '0'},
]
mock_connect.return_value.__enter__.return_value. \
cursor.return_value.__enter__.return_value = mock_cursor
source = MySQLSource(MySQLConnectInfo(None), 'daily', 'full')
source.disable_wsrep_desync()
mock_cursor.execute.assert_any_call("SHOW GLOBAL STATUS LIKE "
"'wsrep_local_recv_queue'")
mock_cursor.execute.assert_called_with("SET GLOBAL wsrep_desync=OFF")
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: ConfigParser.ConfigParser
:return: None
"""
try:
if not config.getboolean('source', 'backup_mysql'):
raise TwinDBBackupError('MySQL backups are not enabled in config')
except (ConfigParser.NoOptionError, TwinDBBackupError) as err:
LOG.debug(err)
LOG.debug('Not backing up MySQL')
return
dst = get_destination(config)
try:
full_backup = config.get('mysql', 'full_backup')
except ConfigParser.NoOptionError:
full_backup = 'daily'
backup_start = time.time()
src = MySQLSource(
MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type,
full_backup, dst)
callbacks = []
src_name = _backup_stream(config, src, dst, callbacks)
status = prepare_status(dst, src, run_type, src_name, backup_start)
status = src.apply_retention_policy(dst, config, run_type, status)
backup_duration = \
status[run_type][src_name]['backup_finished'] - \
status[run_type][src_name]['backup_started']
export_info(config,
data=backup_duration,
category=ExportCategory.mysql,
measure_type=ExportMeasureType.backup)
dst.status(status)
LOG.debug('Callbacks are %r', callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])
def test_mysql_source_has_methods():
src = MySQLSource(MySQLConnectInfo('/foo/bar'),
'hourly',
'full',
dst=mock.Mock())
assert src._connect_info.defaults_file == '/foo/bar'
assert src.run_type == 'hourly'
assert src.suffix == 'xbstream'
assert src._media_type == 'mysql'
def test_apply_retention_policy_remove(mock_get_files_to_delete,
mock_delete_local_files, tmpdir):
mock_get_files_to_delete.return_value = ['key-foo']
my_cnf = tmpdir.join('my.cnf')
mock_config = mock.Mock()
src = MySQLSource(MySQLConnectInfo(str(my_cnf)),
'hourly',
'full',
dst=mock.Mock())
mock_dst = mock.Mock()
mock_dst.remote_path = '/foo/bar'
mock_dst.basename.return_value = 'key-foo'
mock_status = mock.Mock()
# noinspection PyTypeChecker
src.apply_retention_policy(mock_dst, mock_config, 'hourly', mock_status)
mock_status.remove.assert_called_once_with('key-foo')
def test_apply_retention_policy_remove(
mock_get_files_to_delete,
mock_delete_local_files,
tmpdir):
mock_get_files_to_delete.return_value = ['key-foo']
my_cnf = tmpdir.join('my.cnf')
mock_config = mock.Mock()
src = MySQLSource(
MySQLConnectInfo(str(my_cnf)),
'hourly',
'full',
dst=mock.Mock()
)
mock_dst = mock.Mock()
mock_dst.remote_path = '/foo/bar'
mock_dst.basename.return_value = 'key-foo'
mock_status = mock.Mock()
# noinspection PyTypeChecker
src.apply_retention_policy(mock_dst, mock_config, 'hourly', mock_status)
mock_status.remove.assert_called_once_with('key-foo')
def test__wsrep_provider_version_returns_correct_version(mock_connect):
logging.basicConfig()
mock_cursor = mock.MagicMock()
mock_cursor.fetchall.return_value = [
{
'Variable_name': 'wsrep_provider_version',
'Value': '3.19(rb98f92f)'
},
]
mock_connect.return_value.__enter__.return_value. \
cursor.return_value.__enter__.return_value = mock_cursor
source = MySQLSource(MySQLConnectInfo(None), 'daily', 'full')
assert source.wsrep_provider_version == '3.19'
def test_get_binlog_coordinates(error_log, binlog_coordinate, tmpdir):
err_log = tmpdir.join('err.log')
err_log.write(error_log)
assert MySQLSource.get_binlog_coordinates(str(err_log)) == \
binlog_coordinate
def test_get_lsn(error_log, lsn, tmpdir):
err_log = tmpdir.join('err.log')
err_log.write(error_log)
# noinspection PyProtectedMember
assert MySQLSource._get_lsn(str(err_log)) == lsn
def test_suffix():
fs = MySQLSource(MySQLConnectInfo('/foo/bar'), INTERVALS[0], 'full')
assert fs.suffix == 'xbstream'
fs.suffix += '.gz'
assert fs.suffix == 'xbstream.gz'
def test__get_connection_raises_mysql_source_error(mock_connect):
mock_connect.side_effect = OperationalError
source = MySQLSource(MySQLConnectInfo(None,hostname=None), 'daily', 'full')
with pytest.raises(MySQLSourceError):
with source.get_connection():
pass
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: TwinDBBackupConfig
"""
if config.backup_mysql is False:
LOG.debug("Not backing up MySQL")
return
dst = config.destination()
try:
full_backup = config.mysql.full_backup
except configparser.NoOptionError:
full_backup = "daily"
backup_start = time.time()
status = MySQLStatus(dst=dst)
kwargs = {
"backup_type": status.next_backup_type(full_backup, run_type),
"dst": dst,
"xtrabackup_binary": config.mysql.xtrabackup_binary,
}
parent = status.candidate_parent(run_type)
if kwargs["backup_type"] == "incremental":
kwargs["parent_lsn"] = parent.lsn
LOG.debug("Creating source %r", kwargs)
src = MySQLSource(MySQLConnectInfo(config.mysql.defaults_file), run_type,
**kwargs)
callbacks = []
try:
_backup_stream(config, src, dst, callbacks=callbacks)
except (DestinationError, SourceError, SshClientException) as err:
raise OperationError(err)
LOG.debug("Backup copy name: %s", src.get_name())
kwargs = {
"type": src.type,
"binlog": src.binlog_coordinate[0],
"position": src.binlog_coordinate[1],
"lsn": src.lsn,
"backup_started": backup_start,
"backup_finished": time.time(),
"config_files": my_cnfs(MY_CNF_COMMON_PATHS),
}
if src.incremental:
kwargs["parent"] = parent.key
backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs)
status.add(backup_copy)
status = src.apply_retention_policy(dst, config, run_type, status)
LOG.debug("status after apply_retention_policy():\n%s", status)
backup_duration = backup_copy.duration
export_info(
config,
data=backup_duration,
category=ExportCategory.mysql,
measure_type=ExportMeasureType.backup,
)
status.save(dst)
LOG.debug("Callbacks are %r", callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: ConfigParser.ConfigParser
:return: None
"""
try:
if not config.getboolean('source', 'backup_mysql'):
raise TwinDBBackupError('MySQL backups are not enabled in config')
except (ConfigParser.NoOptionError, TwinDBBackupError) as err:
LOG.debug(err)
LOG.debug('Not backing up MySQL')
return
dst = get_destination(config)
try:
full_backup = config.get('mysql', 'full_backup')
except ConfigParser.NoOptionError:
full_backup = 'daily'
backup_start = time.time()
src = MySQLSource(
MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type,
full_backup, dst)
callbacks = []
stream = src.get_stream()
src_name = src.get_name()
# Gzip modifier
stream = Gzip(stream).get_stream()
src_name += '.gz'
# KeepLocal modifier
try:
keep_local_path = config.get('destination', 'keep_local_path')
kl_modifier = KeepLocal(stream, os.path.join(keep_local_path,
src_name))
stream = kl_modifier.get_stream()
callbacks.append((kl_modifier, {
'keep_local_path': keep_local_path,
'dst': dst
}))
except ConfigParser.NoOptionError:
LOG.debug('keep_local_path is not present in the config file')
# GPG modifier
try:
stream = Gpg(stream, config.get('gpg', 'recipient'),
config.get('gpg', 'keyring')).get_stream()
src_name += '.gpg'
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
pass
except ModifierException as err:
LOG.warning(err)
LOG.warning('Will skip encryption')
if not dst.save(stream, src_name):
LOG.error('Failed to save backup copy %s', src_name)
exit(1)
status = prepare_status(dst, src, run_type, src_name, backup_start)
src.apply_retention_policy(dst, config, run_type, status)
dst.status(status)
LOG.debug('Callbacks are %r', callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])
def test_mysql_source_raises_on_wrong_connect_info():
with pytest.raises(MySQLSourceError):
MySQLSource('/foo/bar',
run_type='hourly',
full_backup='daily',
dst=mock.Mock())
def test_mysql_raises_on_wrong_full_backup(full_backup):
with pytest.raises(MySQLSourceError):
MySQLSource(MySQLConnectInfo('/foo/bar'),
run_type='daily',
full_backup=full_backup,
dst=mock.Mock())
def test_get_binlog_coordinates(error_log, binlog_coordinate, tmpdir):
err_log = tmpdir.join('err.log')
err_log.write(error_log)
assert MySQLSource.get_binlog_coordinates(str(err_log)) \
== binlog_coordinate
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: TwinDBBackupConfig
"""
if config.backup_mysql is False:
LOG.debug('Not backing up MySQL')
return
dst = config.destination()
try:
full_backup = config.mysql.full_backup
except ConfigParser.NoOptionError:
full_backup = 'daily'
backup_start = time.time()
status = MySQLStatus(dst=dst)
kwargs = {
'backup_type': status.next_backup_type(full_backup, run_type),
'dst': dst,
'xtrabackup_binary': config.mysql.xtrabackup_binary
}
parent = status.candidate_parent(run_type)
if kwargs['backup_type'] == 'incremental':
kwargs['parent_lsn'] = parent.lsn
LOG.debug('Creating source %r', kwargs)
src = MySQLSource(MySQLConnectInfo(config.mysql.defaults_file), run_type,
**kwargs)
callbacks = []
try:
_backup_stream(config, src, dst, callbacks=callbacks)
except (DestinationError, SourceError, SshClientException) as err:
raise OperationError(err)
LOG.debug('Backup copy name: %s', src.get_name())
kwargs = {
'type': src.type,
'binlog': src.binlog_coordinate[0],
'position': src.binlog_coordinate[1],
'lsn': src.lsn,
'backup_started': backup_start,
'backup_finished': time.time(),
'config_files': my_cnfs(MY_CNF_COMMON_PATHS)
}
if src.incremental:
kwargs['parent'] = parent.key
backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs)
status.add(backup_copy)
status = src.apply_retention_policy(dst, config, run_type, status)
LOG.debug('status after apply_retention_policy():\n%s', status)
backup_duration = backup_copy.duration
export_info(config,
data=backup_duration,
category=ExportCategory.mysql,
measure_type=ExportMeasureType.backup)
status.save(dst)
LOG.debug('Callbacks are %r', callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])
def test_delete_from_status(status, run_type, remote, fl, expected_status):
src = MySQLSource(MySQLConnectInfo('/foo/bar'), run_type, 'daily',
mock.Mock())
assert src._delete_from_status(status, remote, fl) == expected_status
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: ConfigParser.ConfigParser
:return: None
"""
try:
if not config.getboolean('source', 'backup_mysql'):
raise TwinDBBackupError('MySQL backups are not enabled in config')
except (ConfigParser.NoOptionError, TwinDBBackupError) as err:
LOG.debug(err)
LOG.debug('Not backing up MySQL')
return
dst = get_destination(config)
try:
full_backup = config.get('mysql', 'full_backup')
except ConfigParser.NoOptionError:
full_backup = 'daily'
backup_start = time.time()
try:
xtrabackup_binary = config.get('mysql', 'xtrabackup_binary')
except ConfigParser.NoOptionError:
xtrabackup_binary = XTRABACKUP_BINARY
status = dst.status()
kwargs = {
'backup_type': status.next_backup_type(full_backup, run_type),
'dst': dst,
'xtrabackup_binary': xtrabackup_binary
}
parent = status.eligble_parent(run_type)
if kwargs['backup_type'] == 'incremental':
kwargs['parent_lsn'] = parent.lsn
LOG.debug('Creating source %r', kwargs)
src = MySQLSource(
MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type,
**kwargs)
callbacks = []
_backup_stream(config, src, dst, callbacks=callbacks)
LOG.debug('Backup copy name: %s', src.get_name())
kwargs = {
'type': src.type,
'binlog': src.binlog_coordinate[0],
'position': src.binlog_coordinate[1],
'lsn': src.lsn,
'backup_started': backup_start,
'backup_finished': time.time(),
'config_files': my_cnfs(MY_CNF_COMMON_PATHS)
}
if src.incremental:
kwargs['parent'] = parent.key
backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs)
status.add(backup_copy)
status = src.apply_retention_policy(dst, config, run_type, status)
LOG.debug('status after apply_retention_policy():\n%s', status)
backup_duration = status.backup_duration(run_type, src.get_name())
export_info(config,
data=backup_duration,
category=ExportCategory.mysql,
measure_type=ExportMeasureType.backup)
dst.status(status)
LOG.debug('Callbacks are %r', callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: TwinDBBackupConfig
"""
if config.backup_mysql is False:
LOG.debug('Not backing up MySQL')
return
dst = config.destination()
try:
full_backup = config.mysql.full_backup
except ConfigParser.NoOptionError:
full_backup = 'daily'
backup_start = time.time()
status = MySQLStatus(dst=dst)
kwargs = {
'backup_type': status.next_backup_type(full_backup, run_type),
'dst': dst,
'xtrabackup_binary': config.mysql.xtrabackup_binary
}
parent = status.candidate_parent(run_type)
if kwargs['backup_type'] == 'incremental':
kwargs['parent_lsn'] = parent.lsn
LOG.debug('Creating source %r', kwargs)
src = MySQLSource(
MySQLConnectInfo(config.mysql.defaults_file),
run_type,
**kwargs
)
callbacks = []
try:
_backup_stream(config, src, dst, callbacks=callbacks)
except (DestinationError, SourceError, SshClientException) as err:
raise OperationError(err)
LOG.debug('Backup copy name: %s', src.get_name())
kwargs = {
'type': src.type,
'binlog': src.binlog_coordinate[0],
'position': src.binlog_coordinate[1],
'lsn': src.lsn,
'backup_started': backup_start,
'backup_finished': time.time(),
'config_files': my_cnfs(MY_CNF_COMMON_PATHS)
}
if src.incremental:
kwargs['parent'] = parent.key
backup_copy = MySQLCopy(
src.host,
run_type,
src.basename,
**kwargs
)
status.add(backup_copy)
status = src.apply_retention_policy(dst, config, run_type, status)
LOG.debug('status after apply_retention_policy():\n%s', status)
backup_duration = backup_copy.duration
export_info(
config,
data=backup_duration,
category=ExportCategory.mysql,
measure_type=ExportMeasureType.backup
)
status.save(dst)
LOG.debug('Callbacks are %r', callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])