def test_last_full_lsn(full_backup, run_type, status, tmpdir): mock_dst = mock.Mock() mock_dst.status.return_value = status src = MySQLSource(MySQLConnectInfo('/foo/bar'), run_type, full_backup, mock_dst) assert src.parent_lsn == 19629412
def test_mysql_source_raises_on_wrong_connect_info(): with pytest.raises(MySQLSourceError): MySQLSource( '/foo/bar', 'hourly', 'full', dst=mock.Mock() )
def test__get_connection_raises_mysql_source_error(mock_connect): mock_connect.side_effect = OperationalError source = MySQLSource(MySQLConnectInfo(None, hostname=None), 'daily', 'full') with pytest.raises(MySQLSourceError): with source.get_connection(): pass
def test_mysql_raises_on_wrong_run_type(run_type): with pytest.raises(MySQLSourceError): MySQLSource( MySQLConnectInfo('/foo/bar'), 'foo', 'full', dst=mock.Mock() )
def test_full_copy_exists(run_type, full_backup, status, expected): mock_dst = mock.Mock() mock_dst.status.return_value = status src = MySQLSource(MySQLConnectInfo('/foo/bar'), run_type, full_backup, mock_dst) assert src._full_copy_exists() == expected
def test_mysql_source_has_methods(): src = MySQLSource(MySQLConnectInfo('/foo/bar'), 'hourly', 'full', dst=mock.Mock()) assert src._connect_info.defaults_file == '/foo/bar' assert src.run_type == 'hourly' assert src.suffix == 'xbstream' assert src._media_type == 'mysql'
def test_get_backup_type(full_backup, run_type, backup_type, status, tmpdir): mock_dst = mock.Mock() mock_dst.status.return_value = status src = MySQLSource(MySQLConnectInfo('/foo/bar'), run_type, full_backup, mock_dst) assert src._get_backup_type() == backup_type
def test_get_name(mock_time, mock_socket): src = MySQLSource(MySQLConnectInfo('/foo/bar'), 'daily', 'hourly', mock.Mock()) host = 'some-host' mock_socket.gethostname.return_value = host timestamp = '2017-02-13_15_40_29' mock_time.strftime.return_value = timestamp assert src.get_name( ) == "some-host/daily/mysql/mysql-2017-02-13_15_40_29.xbstream"
def test__is_galera_returns_true_on_str_higher_wsrep_on(mock_connect): logging.basicConfig() mock_cursor = mock.MagicMock() mock_cursor.fetchone.return_value = {'wsrep_on': 'ON'} mock_connect.return_value.__enter__.return_value. \ cursor.return_value.__enter__.return_value = mock_cursor source = MySQLSource(MySQLConnectInfo(None), 'daily', 'daily', None) assert source.is_galera() is True
def test__is_galera_returns_false_on_int_wsrep_on(mock_connect): logging.basicConfig() mock_cursor = mock.MagicMock() mock_cursor.fetchone.return_value = {'wsrep_on': 0} mock_connect.return_value.__enter__.return_value. \ cursor.return_value.__enter__.return_value = mock_cursor source = MySQLSource(MySQLConnectInfo(None), 'daily', 'full') assert source.is_galera() is False
def test__enable_wsrep_desync_sets_wsrep_desync_to_on(mock_connect): logging.basicConfig() mock_cursor = mock.MagicMock() mock_connect.return_value.__enter__.return_value. \ cursor.return_value.__enter__.return_value = mock_cursor source = MySQLSource(MySQLConnectInfo(None), 'daily', 'full') source.enable_wsrep_desync() mock_cursor.execute.assert_called_with("SET GLOBAL wsrep_desync=ON")
def test__is_galera_returns_true_on_galera_node(mock_connect): logging.basicConfig() mock_cursor = mock.MagicMock() mock_cursor.execute.side_effect = InternalError( 1193, "Unknown system variable " "'wsrep_on'") mock_connect.return_value.__enter__.return_value. \ cursor.return_value.__enter__.return_value = mock_cursor source = MySQLSource(MySQLConnectInfo(None), 'daily', 'full') assert source.is_galera() is False
def test__wsrep_provider_version_returns_correct_version(mock_connect): logging.basicConfig() mock_cursor = mock.MagicMock() mock_cursor.fetchall.return_value = [ { 'Variable_name': 'wsrep_provider_version', 'Value': '3.19(rb98f92f)' }, ] mock_connect.return_value.__enter__.return_value. \ cursor.return_value.__enter__.return_value = mock_cursor source = MySQLSource(MySQLConnectInfo(None), 'daily', 'full') assert source.wsrep_provider_version == '3.19'
def test_apply_retention_policy(mock_get_files_to_delete, mock_delete_local_files, mock_get_prefix, tmpdir): mock_get_files_to_delete.return_value = [] mock_get_prefix.return_value = 'master.box/hourly' my_cnf = tmpdir.join('my.cnf') mock_config = mock.Mock() src = MySQLSource(MySQLConnectInfo(str(my_cnf)), 'hourly', 'daily', mock.Mock()) mock_dst = mock.Mock() mock_dst.remote_path = '/foo/bar' src.apply_retention_policy(mock_dst, mock_config, 'hourly', mock.Mock()) mock_delete_local_files.assert_called_once_with('mysql', mock_config) mock_dst.list_files.assert_called_once_with( '/foo/bar/master.box/hourly/mysql/mysql-')
def test__disable_wsrep_desync_sets_wsrep_desync_to_off(mock_connect): logging.basicConfig() mock_cursor = mock.MagicMock() mock_cursor.fetchall.return_value = [ {'Variable_name': 'wsrep_local_recv_queue', 'Value': '0'}, ] mock_connect.return_value.__enter__.return_value. \ cursor.return_value.__enter__.return_value = mock_cursor source = MySQLSource(MySQLConnectInfo(None), 'daily', 'full') source.disable_wsrep_desync() mock_cursor.execute.assert_any_call("SHOW GLOBAL STATUS LIKE " "'wsrep_local_recv_queue'") mock_cursor.execute.assert_called_with("SET GLOBAL wsrep_desync=OFF")
def backup_mysql(run_type, config): """Take backup of local MySQL instance :param run_type: Run type :type run_type: str :param config: Tool configuration :type config: ConfigParser.ConfigParser :return: None """ try: if not config.getboolean('source', 'backup_mysql'): raise TwinDBBackupError('MySQL backups are not enabled in config') except (ConfigParser.NoOptionError, TwinDBBackupError) as err: LOG.debug(err) LOG.debug('Not backing up MySQL') return dst = get_destination(config) try: full_backup = config.get('mysql', 'full_backup') except ConfigParser.NoOptionError: full_backup = 'daily' backup_start = time.time() src = MySQLSource( MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type, full_backup, dst) callbacks = [] src_name = _backup_stream(config, src, dst, callbacks) status = prepare_status(dst, src, run_type, src_name, backup_start) status = src.apply_retention_policy(dst, config, run_type, status) backup_duration = \ status[run_type][src_name]['backup_finished'] - \ status[run_type][src_name]['backup_started'] export_info(config, data=backup_duration, category=ExportCategory.mysql, measure_type=ExportMeasureType.backup) dst.status(status) LOG.debug('Callbacks are %r', callbacks) for callback in callbacks: callback[0].callback(**callback[1])
def test_apply_retention_policy_remove(mock_get_files_to_delete, mock_delete_local_files, tmpdir): mock_get_files_to_delete.return_value = ['key-foo'] my_cnf = tmpdir.join('my.cnf') mock_config = mock.Mock() src = MySQLSource(MySQLConnectInfo(str(my_cnf)), 'hourly', 'full', dst=mock.Mock()) mock_dst = mock.Mock() mock_dst.remote_path = '/foo/bar' mock_dst.basename.return_value = 'key-foo' mock_status = mock.Mock() # noinspection PyTypeChecker src.apply_retention_policy(mock_dst, mock_config, 'hourly', mock_status) mock_status.remove.assert_called_once_with('key-foo')
def test_suffix(): fs = MySQLSource(MySQLConnectInfo('/foo/bar'), INTERVALS[0], 'full') assert fs.suffix == 'xbstream' fs.suffix += '.gz' assert fs.suffix == 'xbstream.gz'
def backup_mysql(run_type, config): """Take backup of local MySQL instance :param run_type: Run type :type run_type: str :param config: Tool configuration :type config: TwinDBBackupConfig """ if config.backup_mysql is False: LOG.debug("Not backing up MySQL") return dst = config.destination() try: full_backup = config.mysql.full_backup except configparser.NoOptionError: full_backup = "daily" backup_start = time.time() status = MySQLStatus(dst=dst) kwargs = { "backup_type": status.next_backup_type(full_backup, run_type), "dst": dst, "xtrabackup_binary": config.mysql.xtrabackup_binary, } parent = status.candidate_parent(run_type) if kwargs["backup_type"] == "incremental": kwargs["parent_lsn"] = parent.lsn LOG.debug("Creating source %r", kwargs) src = MySQLSource(MySQLConnectInfo(config.mysql.defaults_file), run_type, **kwargs) callbacks = [] try: _backup_stream(config, src, dst, callbacks=callbacks) except (DestinationError, SourceError, SshClientException) as err: raise OperationError(err) LOG.debug("Backup copy name: %s", src.get_name()) kwargs = { "type": src.type, "binlog": src.binlog_coordinate[0], "position": src.binlog_coordinate[1], "lsn": src.lsn, "backup_started": backup_start, "backup_finished": time.time(), "config_files": my_cnfs(MY_CNF_COMMON_PATHS), } if src.incremental: kwargs["parent"] = parent.key backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs) status.add(backup_copy) status = src.apply_retention_policy(dst, config, run_type, status) LOG.debug("status after apply_retention_policy():\n%s", status) backup_duration = backup_copy.duration export_info( config, data=backup_duration, category=ExportCategory.mysql, measure_type=ExportMeasureType.backup, ) status.save(dst) LOG.debug("Callbacks are %r", callbacks) for callback in callbacks: callback[0].callback(**callback[1])
def backup_mysql(run_type, config): """Take backup of local MySQL instance :param run_type: Run type :type run_type: str :param config: Tool configuration :type config: ConfigParser.ConfigParser :return: None """ try: if not config.getboolean('source', 'backup_mysql'): raise TwinDBBackupError('MySQL backups are not enabled in config') except (ConfigParser.NoOptionError, TwinDBBackupError) as err: LOG.debug(err) LOG.debug('Not backing up MySQL') return dst = get_destination(config) try: full_backup = config.get('mysql', 'full_backup') except ConfigParser.NoOptionError: full_backup = 'daily' backup_start = time.time() src = MySQLSource( MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type, full_backup, dst) callbacks = [] stream = src.get_stream() src_name = src.get_name() # Gzip modifier stream = Gzip(stream).get_stream() src_name += '.gz' # KeepLocal modifier try: keep_local_path = config.get('destination', 'keep_local_path') kl_modifier = KeepLocal(stream, os.path.join(keep_local_path, src_name)) stream = kl_modifier.get_stream() callbacks.append((kl_modifier, { 'keep_local_path': keep_local_path, 'dst': dst })) except ConfigParser.NoOptionError: LOG.debug('keep_local_path is not present in the config file') # GPG modifier try: stream = Gpg(stream, config.get('gpg', 'recipient'), config.get('gpg', 'keyring')).get_stream() src_name += '.gpg' except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): pass except ModifierException as err: LOG.warning(err) LOG.warning('Will skip encryption') if not dst.save(stream, src_name): LOG.error('Failed to save backup copy %s', src_name) exit(1) status = prepare_status(dst, src, run_type, src_name, backup_start) src.apply_retention_policy(dst, config, run_type, status) dst.status(status) LOG.debug('Callbacks are %r', callbacks) for callback in callbacks: callback[0].callback(**callback[1])
def test_mysql_source_raises_on_wrong_connect_info(): with pytest.raises(MySQLSourceError): MySQLSource('/foo/bar', run_type='hourly', full_backup='daily', dst=mock.Mock())
def test_mysql_raises_on_wrong_full_backup(full_backup): with pytest.raises(MySQLSourceError): MySQLSource(MySQLConnectInfo('/foo/bar'), run_type='daily', full_backup=full_backup, dst=mock.Mock())
def backup_mysql(run_type, config): """Take backup of local MySQL instance :param run_type: Run type :type run_type: str :param config: Tool configuration :type config: TwinDBBackupConfig """ if config.backup_mysql is False: LOG.debug('Not backing up MySQL') return dst = config.destination() try: full_backup = config.mysql.full_backup except ConfigParser.NoOptionError: full_backup = 'daily' backup_start = time.time() status = MySQLStatus(dst=dst) kwargs = { 'backup_type': status.next_backup_type(full_backup, run_type), 'dst': dst, 'xtrabackup_binary': config.mysql.xtrabackup_binary } parent = status.candidate_parent(run_type) if kwargs['backup_type'] == 'incremental': kwargs['parent_lsn'] = parent.lsn LOG.debug('Creating source %r', kwargs) src = MySQLSource(MySQLConnectInfo(config.mysql.defaults_file), run_type, **kwargs) callbacks = [] try: _backup_stream(config, src, dst, callbacks=callbacks) except (DestinationError, SourceError, SshClientException) as err: raise OperationError(err) LOG.debug('Backup copy name: %s', src.get_name()) kwargs = { 'type': src.type, 'binlog': src.binlog_coordinate[0], 'position': src.binlog_coordinate[1], 'lsn': src.lsn, 'backup_started': backup_start, 'backup_finished': time.time(), 'config_files': my_cnfs(MY_CNF_COMMON_PATHS) } if src.incremental: kwargs['parent'] = parent.key backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs) status.add(backup_copy) status = src.apply_retention_policy(dst, config, run_type, status) LOG.debug('status after apply_retention_policy():\n%s', status) backup_duration = backup_copy.duration export_info(config, data=backup_duration, category=ExportCategory.mysql, measure_type=ExportMeasureType.backup) status.save(dst) LOG.debug('Callbacks are %r', callbacks) for callback in callbacks: callback[0].callback(**callback[1])
def backup_mysql(run_type, config): """Take backup of local MySQL instance :param run_type: Run type :type run_type: str :param config: Tool configuration :type config: ConfigParser.ConfigParser :return: None """ try: if not config.getboolean('source', 'backup_mysql'): raise TwinDBBackupError('MySQL backups are not enabled in config') except (ConfigParser.NoOptionError, TwinDBBackupError) as err: LOG.debug(err) LOG.debug('Not backing up MySQL') return dst = get_destination(config) try: full_backup = config.get('mysql', 'full_backup') except ConfigParser.NoOptionError: full_backup = 'daily' backup_start = time.time() try: xtrabackup_binary = config.get('mysql', 'xtrabackup_binary') except ConfigParser.NoOptionError: xtrabackup_binary = XTRABACKUP_BINARY status = dst.status() kwargs = { 'backup_type': status.next_backup_type(full_backup, run_type), 'dst': dst, 'xtrabackup_binary': xtrabackup_binary } parent = status.eligble_parent(run_type) if kwargs['backup_type'] == 'incremental': kwargs['parent_lsn'] = parent.lsn LOG.debug('Creating source %r', kwargs) src = MySQLSource( MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type, **kwargs) callbacks = [] _backup_stream(config, src, dst, callbacks=callbacks) LOG.debug('Backup copy name: %s', src.get_name()) kwargs = { 'type': src.type, 'binlog': src.binlog_coordinate[0], 'position': src.binlog_coordinate[1], 'lsn': src.lsn, 'backup_started': backup_start, 'backup_finished': time.time(), 'config_files': my_cnfs(MY_CNF_COMMON_PATHS) } if src.incremental: kwargs['parent'] = parent.key backup_copy = MySQLCopy(src.host, run_type, src.basename, **kwargs) status.add(backup_copy) status = src.apply_retention_policy(dst, config, run_type, status) LOG.debug('status after apply_retention_policy():\n%s', status) backup_duration = status.backup_duration(run_type, src.get_name()) export_info(config, data=backup_duration, category=ExportCategory.mysql, measure_type=ExportMeasureType.backup) dst.status(status) LOG.debug('Callbacks are %r', callbacks) for callback in callbacks: callback[0].callback(**callback[1])
def test_delete_from_status(status, run_type, remote, fl, expected_status): src = MySQLSource(MySQLConnectInfo('/foo/bar'), run_type, 'daily', mock.Mock()) assert src._delete_from_status(status, remote, fl) == expected_status