def test_remove(status_raw_empty): status = MySQLStatus(status_raw_empty) copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full') status.add(copy) assert len(status.daily) == 1 status.remove(copy.key) assert len(status.daily) == 0
def test_add(status_raw_empty, tmpdir): status = MySQLStatus(status_raw_empty) assert status.valid mycnf_1 = tmpdir.join('my-1.cnf') mycnf_1.write('some_content_1') mycnf_2 = tmpdir.join('my-2.cnf') mycnf_2.write('some_content_2') backup_copy = MySQLCopy('master1', 'daily', 'foo.txt', binlog='binlog1', position=101, type='full', lsn=1230, backup_started=123, backup_finished=456, config_files=[str(mycnf_1), str(mycnf_2)]) status.add(backup_copy) assert len(status.daily) == 1 assert status.daily[backup_copy.key].binlog == 'binlog1' assert status.daily[backup_copy.key].position == 101 assert status.daily[backup_copy.key].type == 'full' assert status.daily[backup_copy.key].lsn == 1230 assert status.daily[backup_copy.key].backup_started == 123 assert status.daily[backup_copy.key].backup_finished == 456 assert status.daily[backup_copy.key].duration == 333 assert { str(mycnf_1): 'some_content_1' } in status.daily[backup_copy.key].config assert { str(mycnf_2): 'some_content_2' } in status.daily[backup_copy.key].config
def test_get_my_cnf_2_cnf(tmpdir): status = MySQLStatus() mycnf_1 = tmpdir.join('my-1.cnf') mycnf_1.write('some_content_1') mycnf_2 = tmpdir.join('my-2.cnf') mycnf_2.write('some_content_2') backup_copy = MySQLCopy('master1', 'daily', 'foo.txt', binlog='binlog1', position=101, type='full', lsn=1230, backup_started=123, backup_finished=456, config_files=[str(mycnf_1), str(mycnf_2)]) status.add(backup_copy) expected = {str(mycnf_1): 'some_content_1', str(mycnf_2): 'some_content_2'} for path, content in get_my_cnf(status, backup_copy.key): assert path in expected expected_value = expected.pop(path) assert content == expected_value assert expected == {}
def test_init_set_galera(): copy = MySQLCopy( 'foo', 'daily', 'some_file.txt', type='full', wsrep_provider_version='123' ) assert copy.galera is True assert copy.wsrep_provider_version == '123'
def test_init_created_at(): copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full', backup_started=123) assert copy.backup_started == 123 assert copy.created_at == 123
def test_as_dict(): copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full') assert copy.as_dict() == { "type": "full", "backup_finished": None, "backup_started": None, "binlog": None, "config": {}, "host": "foo", "lsn": None, "name": "some_file.txt", "parent": None, "position": None, "run_type": "daily", "galera": False, "wsrep_provider_version": None }
def test_init_set_defaults(): copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full') assert copy.backup_started is None assert copy.backup_finished is None assert copy.binlog is None assert copy.config == {} assert copy.host == 'foo' assert copy.lsn is None assert copy.name == 'some_file.txt' assert copy.parent is None assert copy.position is None assert copy.run_type == 'daily' assert copy.galera is False assert copy.wsrep_provider_version is None
def restore_mysql(ctx, dst, backup_copy, cache): """Restore from mysql backup""" LOG.debug('mysql: %r', ctx.obj['twindb_config']) if not backup_copy: LOG.info('No backup copy specified. Choose one from below:') list_available_backups(ctx.obj['twindb_config']) exit(1) try: ensure_empty(dst) incomplete_copy = MySQLCopy(path=backup_copy) dst_storage = ctx.obj['twindb_config'].destination( backup_source=incomplete_copy.host) mysql_status = MySQLStatus(dst=dst_storage, status_directory=incomplete_copy.host) copies = [cp for cp in mysql_status if backup_copy.endswith(cp.name)] try: copy = copies.pop(0) except IndexError: raise TwinDBBackupError( 'Can not find copy %s in MySQL status. ' 'Inspect output of `twindb-backup status` and verify ' 'that correct copy is specified.' % backup_copy) if copies: raise TwinDBBackupError( 'Multiple copies match pattern %s. Make sure you give unique ' 'copy name for restore.') if cache: restore_from_mysql(ctx.obj['twindb_config'], copy, dst, cache=Cache(cache)) else: restore_from_mysql(ctx.obj['twindb_config'], copy, dst) except (TwinDBBackupError, CacheException) as err: LOG.error(err) LOG.debug(traceback.format_exc()) exit(1) except (OSError, IOError) as err: LOG.error(err) LOG.debug(traceback.format_exc()) exit(1)
def _load(self, status_as_json): status = [] try: status_as_obj = json.loads(status_as_json) except ValueError: raise CorruptedStatus( 'Could not load status from a bad JSON string %s' % (status_as_json, ) ) for run_type in INTERVALS: for key, value in status_as_obj[run_type].iteritems(): try: host = key.split('/')[0] file_name = key.split('/')[3] kwargs = { 'type': value['type'], 'config': self.__serialize_config(value) } keys = [ 'backup_started', 'backup_finished', 'binlog', 'parent', 'lsn', 'position', 'wsrep_provider_version', ] for copy_key in keys: if copy_key in value: kwargs[copy_key] = value[copy_key] copy = MySQLCopy( host, run_type, file_name, **kwargs ) status.append(copy) except IndexError as err: LOG.error(err) raise CorruptedStatus('Unexpected key %s' % key) return status
def test_str(): copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full') expected = """MySQLCopy(foo/daily/mysql/some_file.txt) = { "backup_finished": null, "backup_started": null, "binlog": null, "config": {}, "galera": false, "host": "foo", "lsn": null, "name": "some_file.txt", "parent": null, "position": null, "run_type": "daily", "type": "full", "wsrep_provider_version": null }""" assert str(copy) == expected
def _load(self, status_as_json): status = [] try: status_as_obj = json.loads(status_as_json) except ValueError: raise CorruptedStatus( "Could not load status from a bad JSON string %s" % (status_as_json,) ) for run_type in INTERVALS: for key, value in status_as_obj[run_type].items(): try: host = key.split("/")[0] file_name = key.split("/")[3] kwargs = { "type": value["type"], "config": self.__serialize_config(value), } keys = [ "backup_started", "backup_finished", "binlog", "parent", "lsn", "position", "wsrep_provider_version", ] for copy_key in keys: if copy_key in value: kwargs[copy_key] = value[copy_key] copy = MySQLCopy(host, run_type, file_name, **kwargs) status.append(copy) except IndexError as err: LOG.error(err) raise CorruptedStatus("Unexpected key %s" % key) return status
def test_init_creates_instance_from_new(status_raw_content): status = MySQLStatus(status_raw_content) assert status.version == STATUS_FORMAT_VERSION key = 'master1/hourly/mysql/mysql-2018-03-28_04_11_16.xbstream.gz' copy = MySQLCopy( 'master1', 'hourly', 'mysql-2018-03-28_04_11_16.xbstream.gz', backup_started=1522210276, backup_finished=1522210295, binlog='mysql-bin.000001', parent='master1/daily/mysql/mysql-2018-03-28_04_09_53.xbstream.gz', lsn=19903207, config={ '/etc/my.cnf': """[mysqld] datadir=/var/lib/mysql socket=/var/lib/mysql/mysql.sock user=mysql # Disabling symbolic-links is recommended to prevent assorted security risks symbolic-links=0 server_id=100 gtid_mode=ON log-bin=mysql-bin log-slave-updates enforce-gtid-consistency [mysqld_safe] log-error=/var/log/mysqld.log pid-file=/var/run/mysqld/mysqld.pid """ }, position=46855, type='incremental') assert key in status.hourly LOG.debug("Copy %s: %r", copy.key, copy) LOG.debug("Copy from status %s: %r", key, status[key]) assert status[key] == copy
def backup_mysql(run_type, config): """Take backup of local MySQL instance :param run_type: Run type :type run_type: str :param config: Tool configuration :type config: TwinDBBackupConfig """ if config.backup_mysql is False: LOG.debug("Not backing up MySQL") return dst = config.destination() try: full_backup = config.mysql.full_backup except configparser.NoOptionError: full_backup = "daily" backup_start = time.time() status = MySQLStatus(dst=dst) kwargs = { "backup_type": status.next_backup_type(full_backup, run_type), "dst": dst, "xtrabackup_binary": config.mysql.xtrabackup_binary, } parent = status.candidate_parent(run_type) if kwargs["backup_type"] == "incremental": kwargs["parent_lsn"] = parent.lsn LOG.debug("Creating source %r", kwargs) src = MySQLSource(MySQLConnectInfo(config.mysql.defaults_file), run_type, **kwargs) callbacks = [] try: _backup_stream(config, src, dst, callbacks=callbacks) except (DestinationError, SourceError, SshClientException) as err: raise OperationError(err) LOG.debug("Backup copy name: %s", src.get_name()) kwargs = { "type": src.type, "binlog": src.binlog_coordinate[0], "position": src.binlog_coordinate[1], "lsn": src.lsn, "backup_started": backup_start, "backup_finished": time.time(), "config_files": my_cnfs(MY_CNF_COMMON_PATHS), } if src.incremental: kwargs["parent"] = parent.key backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs) status.add(backup_copy) status = src.apply_retention_policy(dst, config, run_type, status) LOG.debug("status after apply_retention_policy():\n%s", status) backup_duration = backup_copy.duration export_info( config, data=backup_duration, category=ExportCategory.mysql, measure_type=ExportMeasureType.backup, ) status.save(dst) LOG.debug("Callbacks are %r", callbacks) for callback in callbacks: callback[0].callback(**callback[1])
def test_eq(): copy_1 = MySQLCopy('foo', 'daily', 'some_file.txt', type='full') copy_2 = MySQLCopy('foo', 'daily', 'some_file.txt', type='full') copy_3 = MySQLCopy('bar', 'daily', 'some_file.txt', type='full') assert copy_1 == copy_2 assert copy_1 != copy_3
def backup_mysql(run_type, config): """Take backup of local MySQL instance :param run_type: Run type :type run_type: str :param config: Tool configuration :type config: ConfigParser.ConfigParser :return: None """ try: if not config.getboolean('source', 'backup_mysql'): raise TwinDBBackupError('MySQL backups are not enabled in config') except (ConfigParser.NoOptionError, TwinDBBackupError) as err: LOG.debug(err) LOG.debug('Not backing up MySQL') return dst = get_destination(config) try: full_backup = config.get('mysql', 'full_backup') except ConfigParser.NoOptionError: full_backup = 'daily' backup_start = time.time() try: xtrabackup_binary = config.get('mysql', 'xtrabackup_binary') except ConfigParser.NoOptionError: xtrabackup_binary = XTRABACKUP_BINARY status = dst.status() kwargs = { 'backup_type': status.next_backup_type(full_backup, run_type), 'dst': dst, 'xtrabackup_binary': xtrabackup_binary } parent = status.eligble_parent(run_type) if kwargs['backup_type'] == 'incremental': kwargs['parent_lsn'] = parent.lsn LOG.debug('Creating source %r', kwargs) src = MySQLSource( MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type, **kwargs) callbacks = [] _backup_stream(config, src, dst, callbacks=callbacks) LOG.debug('Backup copy name: %s', src.get_name()) kwargs = { 'type': src.type, 'binlog': src.binlog_coordinate[0], 'position': src.binlog_coordinate[1], 'lsn': src.lsn, 'backup_started': backup_start, 'backup_finished': time.time(), 'config_files': my_cnfs(MY_CNF_COMMON_PATHS) } if src.incremental: kwargs['parent'] = parent.key backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs) status.add(backup_copy) status = src.apply_retention_policy(dst, config, run_type, status) LOG.debug('status after apply_retention_policy():\n%s', status) backup_duration = status.backup_duration(run_type, src.get_name()) export_info(config, data=backup_duration, category=ExportCategory.mysql, measure_type=ExportMeasureType.backup) dst.status(status) LOG.debug('Callbacks are %r', callbacks) for callback in callbacks: callback[0].callback(**callback[1])
def test_init_raises_if_name_is_not_relative(): with pytest.raises(WrongInputData): MySQLCopy('foo', 'daily', 'some/non/relative/path', type='full')
def test_copy_from_path(path, host, run_type, name): copy = MySQLCopy(path=path) assert copy.host == host assert copy.run_type == run_type assert copy.name == name
def backup_mysql(run_type, config): """Take backup of local MySQL instance :param run_type: Run type :type run_type: str :param config: Tool configuration :type config: TwinDBBackupConfig """ if config.backup_mysql is False: LOG.debug('Not backing up MySQL') return dst = config.destination() try: full_backup = config.mysql.full_backup except ConfigParser.NoOptionError: full_backup = 'daily' backup_start = time.time() status = MySQLStatus(dst=dst) kwargs = { 'backup_type': status.next_backup_type(full_backup, run_type), 'dst': dst, 'xtrabackup_binary': config.mysql.xtrabackup_binary } parent = status.candidate_parent(run_type) if kwargs['backup_type'] == 'incremental': kwargs['parent_lsn'] = parent.lsn LOG.debug('Creating source %r', kwargs) src = MySQLSource(MySQLConnectInfo(config.mysql.defaults_file), run_type, **kwargs) callbacks = [] try: _backup_stream(config, src, dst, callbacks=callbacks) except (DestinationError, SourceError, SshClientException) as err: raise OperationError(err) LOG.debug('Backup copy name: %s', src.get_name()) kwargs = { 'type': src.type, 'binlog': src.binlog_coordinate[0], 'position': src.binlog_coordinate[1], 'lsn': src.lsn, 'backup_started': backup_start, 'backup_finished': time.time(), 'config_files': my_cnfs(MY_CNF_COMMON_PATHS) } if src.incremental: kwargs['parent'] = parent.key backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs) status.add(backup_copy) status = src.apply_retention_policy(dst, config, run_type, status) LOG.debug('status after apply_retention_policy():\n%s', status) backup_duration = backup_copy.duration export_info(config, data=backup_duration, category=ExportCategory.mysql, measure_type=ExportMeasureType.backup) status.save(dst) LOG.debug('Callbacks are %r', callbacks) for callback in callbacks: callback[0].callback(**callback[1])
def test_init_has_config(): copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full') assert copy.config == {}
def test_key(): backup_copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full') assert backup_copy.key == 'foo/daily/mysql/some_file.txt'
def test_repr(): copy = MySQLCopy('foo', 'daily', 'some_file.txt', type='full') assert repr(copy) == 'MySQLCopy(foo/daily/mysql/some_file.txt)'