def test_remove(status_raw_empty):
status = MySQLStatus(status_raw_empty)
copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full')
status.add(copy)
assert len(status.daily) == 1
status.remove(copy.key)
assert len(status.daily) == 0
def test_add(status_raw_empty, tmpdir):
status = MySQLStatus(status_raw_empty)
assert status.valid
mycnf_1 = tmpdir.join('my-1.cnf')
mycnf_1.write('some_content_1')
mycnf_2 = tmpdir.join('my-2.cnf')
mycnf_2.write('some_content_2')
backup_copy = MySQLCopy('master1',
'daily',
'foo.txt',
binlog='binlog1',
position=101,
type='full',
lsn=1230,
backup_started=123,
backup_finished=456,
config_files=[str(mycnf_1),
str(mycnf_2)])
status.add(backup_copy)
assert len(status.daily) == 1
assert status.daily[backup_copy.key].binlog == 'binlog1'
assert status.daily[backup_copy.key].position == 101
assert status.daily[backup_copy.key].type == 'full'
assert status.daily[backup_copy.key].lsn == 1230
assert status.daily[backup_copy.key].backup_started == 123
assert status.daily[backup_copy.key].backup_finished == 456
assert status.daily[backup_copy.key].duration == 333
assert {
str(mycnf_1): 'some_content_1'
} in status.daily[backup_copy.key].config
assert {
str(mycnf_2): 'some_content_2'
} in status.daily[backup_copy.key].config
def test_get_my_cnf_2_cnf(tmpdir):
status = MySQLStatus()
mycnf_1 = tmpdir.join('my-1.cnf')
mycnf_1.write('some_content_1')
mycnf_2 = tmpdir.join('my-2.cnf')
mycnf_2.write('some_content_2')
backup_copy = MySQLCopy('master1',
'daily',
'foo.txt',
binlog='binlog1',
position=101,
type='full',
lsn=1230,
backup_started=123,
backup_finished=456,
config_files=[str(mycnf_1),
str(mycnf_2)])
status.add(backup_copy)
expected = {str(mycnf_1): 'some_content_1', str(mycnf_2): 'some_content_2'}
for path, content in get_my_cnf(status, backup_copy.key):
assert path in expected
expected_value = expected.pop(path)
assert content == expected_value
assert expected == {}
def test_init_set_galera():
copy = MySQLCopy(
'foo', 'daily', 'some_file.txt',
type='full',
wsrep_provider_version='123'
)
assert copy.galera is True
assert copy.wsrep_provider_version == '123'
def test_init_created_at():
copy = MySQLCopy('foo',
'daily',
'some_file.txt',
type='full',
backup_started=123)
assert copy.backup_started == 123
assert copy.created_at == 123
def test_as_dict():
copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full')
assert copy.as_dict() == {
"type": "full",
"backup_finished": None,
"backup_started": None,
"binlog": None,
"config": {},
"host": "foo",
"lsn": None,
"name": "some_file.txt",
"parent": None,
"position": None,
"run_type": "daily",
"galera": False,
"wsrep_provider_version": None
}
def test_init_set_defaults():
copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full')
assert copy.backup_started is None
assert copy.backup_finished is None
assert copy.binlog is None
assert copy.config == {}
assert copy.host == 'foo'
assert copy.lsn is None
assert copy.name == 'some_file.txt'
assert copy.parent is None
assert copy.position is None
assert copy.run_type == 'daily'
assert copy.galera is False
assert copy.wsrep_provider_version is None
def restore_mysql(ctx, dst, backup_copy, cache):
"""Restore from mysql backup"""
LOG.debug('mysql: %r', ctx.obj['twindb_config'])
if not backup_copy:
LOG.info('No backup copy specified. Choose one from below:')
list_available_backups(ctx.obj['twindb_config'])
exit(1)
try:
ensure_empty(dst)
incomplete_copy = MySQLCopy(path=backup_copy)
dst_storage = ctx.obj['twindb_config'].destination(
backup_source=incomplete_copy.host)
mysql_status = MySQLStatus(dst=dst_storage,
status_directory=incomplete_copy.host)
copies = [cp for cp in mysql_status if backup_copy.endswith(cp.name)]
try:
copy = copies.pop(0)
except IndexError:
raise TwinDBBackupError(
'Can not find copy %s in MySQL status. '
'Inspect output of `twindb-backup status` and verify '
'that correct copy is specified.' % backup_copy)
if copies:
raise TwinDBBackupError(
'Multiple copies match pattern %s. Make sure you give unique '
'copy name for restore.')
if cache:
restore_from_mysql(ctx.obj['twindb_config'],
copy,
dst,
cache=Cache(cache))
else:
restore_from_mysql(ctx.obj['twindb_config'], copy, dst)
except (TwinDBBackupError, CacheException) as err:
LOG.error(err)
LOG.debug(traceback.format_exc())
exit(1)
except (OSError, IOError) as err:
LOG.error(err)
LOG.debug(traceback.format_exc())
exit(1)
def _load(self, status_as_json):
status = []
try:
status_as_obj = json.loads(status_as_json)
except ValueError:
raise CorruptedStatus(
'Could not load status from a bad JSON string %s'
% (status_as_json, )
)
for run_type in INTERVALS:
for key, value in status_as_obj[run_type].iteritems():
try:
host = key.split('/')[0]
file_name = key.split('/')[3]
kwargs = {
'type': value['type'],
'config': self.__serialize_config(value)
}
keys = [
'backup_started',
'backup_finished',
'binlog',
'parent',
'lsn',
'position',
'wsrep_provider_version',
]
for copy_key in keys:
if copy_key in value:
kwargs[copy_key] = value[copy_key]
copy = MySQLCopy(
host,
run_type,
file_name,
**kwargs
)
status.append(copy)
except IndexError as err:
LOG.error(err)
raise CorruptedStatus('Unexpected key %s' % key)
return status
def test_str():
copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full')
expected = """MySQLCopy(foo/daily/mysql/some_file.txt) = {
"backup_finished": null,
"backup_started": null,
"binlog": null,
"config": {},
"galera": false,
"host": "foo",
"lsn": null,
"name": "some_file.txt",
"parent": null,
"position": null,
"run_type": "daily",
"type": "full",
"wsrep_provider_version": null
}"""
assert str(copy) == expected
def _load(self, status_as_json):
status = []
try:
status_as_obj = json.loads(status_as_json)
except ValueError:
raise CorruptedStatus(
"Could not load status from a bad JSON string %s"
% (status_as_json,)
)
for run_type in INTERVALS:
for key, value in status_as_obj[run_type].items():
try:
host = key.split("/")[0]
file_name = key.split("/")[3]
kwargs = {
"type": value["type"],
"config": self.__serialize_config(value),
}
keys = [
"backup_started",
"backup_finished",
"binlog",
"parent",
"lsn",
"position",
"wsrep_provider_version",
]
for copy_key in keys:
if copy_key in value:
kwargs[copy_key] = value[copy_key]
copy = MySQLCopy(host, run_type, file_name, **kwargs)
status.append(copy)
except IndexError as err:
LOG.error(err)
raise CorruptedStatus("Unexpected key %s" % key)
return status
def test_init_creates_instance_from_new(status_raw_content):
status = MySQLStatus(status_raw_content)
assert status.version == STATUS_FORMAT_VERSION
key = 'master1/hourly/mysql/mysql-2018-03-28_04_11_16.xbstream.gz'
copy = MySQLCopy(
'master1',
'hourly',
'mysql-2018-03-28_04_11_16.xbstream.gz',
backup_started=1522210276,
backup_finished=1522210295,
binlog='mysql-bin.000001',
parent='master1/daily/mysql/mysql-2018-03-28_04_09_53.xbstream.gz',
lsn=19903207,
config={
'/etc/my.cnf':
"""[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
user=mysql
# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0
server_id=100
gtid_mode=ON
log-bin=mysql-bin
log-slave-updates
enforce-gtid-consistency
[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid
"""
},
position=46855,
type='incremental')
assert key in status.hourly
LOG.debug("Copy %s: %r", copy.key, copy)
LOG.debug("Copy from status %s: %r", key, status[key])
assert status[key] == copy
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: TwinDBBackupConfig
"""
if config.backup_mysql is False:
LOG.debug("Not backing up MySQL")
return
dst = config.destination()
try:
full_backup = config.mysql.full_backup
except configparser.NoOptionError:
full_backup = "daily"
backup_start = time.time()
status = MySQLStatus(dst=dst)
kwargs = {
"backup_type": status.next_backup_type(full_backup, run_type),
"dst": dst,
"xtrabackup_binary": config.mysql.xtrabackup_binary,
}
parent = status.candidate_parent(run_type)
if kwargs["backup_type"] == "incremental":
kwargs["parent_lsn"] = parent.lsn
LOG.debug("Creating source %r", kwargs)
src = MySQLSource(MySQLConnectInfo(config.mysql.defaults_file), run_type,
**kwargs)
callbacks = []
try:
_backup_stream(config, src, dst, callbacks=callbacks)
except (DestinationError, SourceError, SshClientException) as err:
raise OperationError(err)
LOG.debug("Backup copy name: %s", src.get_name())
kwargs = {
"type": src.type,
"binlog": src.binlog_coordinate[0],
"position": src.binlog_coordinate[1],
"lsn": src.lsn,
"backup_started": backup_start,
"backup_finished": time.time(),
"config_files": my_cnfs(MY_CNF_COMMON_PATHS),
}
if src.incremental:
kwargs["parent"] = parent.key
backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs)
status.add(backup_copy)
status = src.apply_retention_policy(dst, config, run_type, status)
LOG.debug("status after apply_retention_policy():\n%s", status)
backup_duration = backup_copy.duration
export_info(
config,
data=backup_duration,
category=ExportCategory.mysql,
measure_type=ExportMeasureType.backup,
)
status.save(dst)
LOG.debug("Callbacks are %r", callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])
def test_eq():
copy_1 = MySQLCopy('foo', 'daily', 'some_file.txt', type='full')
copy_2 = MySQLCopy('foo', 'daily', 'some_file.txt', type='full')
copy_3 = MySQLCopy('bar', 'daily', 'some_file.txt', type='full')
assert copy_1 == copy_2
assert copy_1 != copy_3
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: ConfigParser.ConfigParser
:return: None
"""
try:
if not config.getboolean('source', 'backup_mysql'):
raise TwinDBBackupError('MySQL backups are not enabled in config')
except (ConfigParser.NoOptionError, TwinDBBackupError) as err:
LOG.debug(err)
LOG.debug('Not backing up MySQL')
return
dst = get_destination(config)
try:
full_backup = config.get('mysql', 'full_backup')
except ConfigParser.NoOptionError:
full_backup = 'daily'
backup_start = time.time()
try:
xtrabackup_binary = config.get('mysql', 'xtrabackup_binary')
except ConfigParser.NoOptionError:
xtrabackup_binary = XTRABACKUP_BINARY
status = dst.status()
kwargs = {
'backup_type': status.next_backup_type(full_backup, run_type),
'dst': dst,
'xtrabackup_binary': xtrabackup_binary
}
parent = status.eligble_parent(run_type)
if kwargs['backup_type'] == 'incremental':
kwargs['parent_lsn'] = parent.lsn
LOG.debug('Creating source %r', kwargs)
src = MySQLSource(
MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type,
**kwargs)
callbacks = []
_backup_stream(config, src, dst, callbacks=callbacks)
LOG.debug('Backup copy name: %s', src.get_name())
kwargs = {
'type': src.type,
'binlog': src.binlog_coordinate[0],
'position': src.binlog_coordinate[1],
'lsn': src.lsn,
'backup_started': backup_start,
'backup_finished': time.time(),
'config_files': my_cnfs(MY_CNF_COMMON_PATHS)
}
if src.incremental:
kwargs['parent'] = parent.key
backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs)
status.add(backup_copy)
status = src.apply_retention_policy(dst, config, run_type, status)
LOG.debug('status after apply_retention_policy():\n%s', status)
backup_duration = status.backup_duration(run_type, src.get_name())
export_info(config,
data=backup_duration,
category=ExportCategory.mysql,
measure_type=ExportMeasureType.backup)
dst.status(status)
LOG.debug('Callbacks are %r', callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])
def test_init_raises_if_name_is_not_relative():
with pytest.raises(WrongInputData):
MySQLCopy('foo', 'daily', 'some/non/relative/path', type='full')
def test_copy_from_path(path, host, run_type, name):
copy = MySQLCopy(path=path)
assert copy.host == host
assert copy.run_type == run_type
assert copy.name == name
def backup_mysql(run_type, config):
"""Take backup of local MySQL instance
:param run_type: Run type
:type run_type: str
:param config: Tool configuration
:type config: TwinDBBackupConfig
"""
if config.backup_mysql is False:
LOG.debug('Not backing up MySQL')
return
dst = config.destination()
try:
full_backup = config.mysql.full_backup
except ConfigParser.NoOptionError:
full_backup = 'daily'
backup_start = time.time()
status = MySQLStatus(dst=dst)
kwargs = {
'backup_type': status.next_backup_type(full_backup, run_type),
'dst': dst,
'xtrabackup_binary': config.mysql.xtrabackup_binary
}
parent = status.candidate_parent(run_type)
if kwargs['backup_type'] == 'incremental':
kwargs['parent_lsn'] = parent.lsn
LOG.debug('Creating source %r', kwargs)
src = MySQLSource(MySQLConnectInfo(config.mysql.defaults_file), run_type,
**kwargs)
callbacks = []
try:
_backup_stream(config, src, dst, callbacks=callbacks)
except (DestinationError, SourceError, SshClientException) as err:
raise OperationError(err)
LOG.debug('Backup copy name: %s', src.get_name())
kwargs = {
'type': src.type,
'binlog': src.binlog_coordinate[0],
'position': src.binlog_coordinate[1],
'lsn': src.lsn,
'backup_started': backup_start,
'backup_finished': time.time(),
'config_files': my_cnfs(MY_CNF_COMMON_PATHS)
}
if src.incremental:
kwargs['parent'] = parent.key
backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs)
status.add(backup_copy)
status = src.apply_retention_policy(dst, config, run_type, status)
LOG.debug('status after apply_retention_policy():\n%s', status)
backup_duration = backup_copy.duration
export_info(config,
data=backup_duration,
category=ExportCategory.mysql,
measure_type=ExportMeasureType.backup)
status.save(dst)
LOG.debug('Callbacks are %r', callbacks)
for callback in callbacks:
callback[0].callback(**callback[1])
def test_init_has_config():
copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full')
assert copy.config == {}
def test_key():
backup_copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full')
assert backup_copy.key == 'foo/daily/mysql/some_file.txt'
def test_repr():
copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full')
assert repr(copy) == 'MySQLCopy(foo/daily/mysql/some_file.txt)'