def test_setup(self, rsync_mock): """ Test the method that set up a recovery """ backup_info = testing_helpers.build_test_backup_info() backup_manager = testing_helpers.build_backup_manager() executor = RecoveryExecutor.factory(backup_manager) backup_info.version = 90300 # setup should create a temporary directory # and teardown should delete it ret = executor._setup(backup_info, None, "/tmp") assert os.path.exists(ret['tempdir']) executor._teardown(ret) assert not os.path.exists(ret['tempdir']) # no postgresql.auto.conf on version 9.3 ret = executor._setup(backup_info, None, "/tmp") executor._teardown(ret) assert "postgresql.auto.conf" not in ret['configuration_files'] # Check the present for postgresql.auto.conf on version 9.4 backup_info.version = 90400 ret = executor._setup(backup_info, None, "/tmp") executor._teardown(ret) assert "postgresql.auto.conf" in ret['configuration_files'] # Receive a error if the remote command is invalid with pytest.raises(SystemExit): executor.server.path = None executor._setup(backup_info, "invalid", "/tmp")
def test_rsync_backup_executor_init(self): """ Test the construction of a RecoveryExecutor """ # Test backup_manager = testing_helpers.build_backup_manager() assert RecoveryExecutor.factory(backup_manager)
def test_set_pitr_targets(self, tmpdir): """ Evaluate targets for point in time recovery """ # Build basic folder/files structure tempdir = tmpdir.mkdir('temp_dir') dest = tmpdir.mkdir('dest') wal_dest = tmpdir.mkdir('wal_dest') recovery_info = { 'configuration_files': ['postgresql.conf', 'postgresql.auto.conf'], 'tempdir': tempdir.strpath, 'results': { 'changes': [], 'warnings': [] }, 'is_pitr': False, 'wal_dest': wal_dest.strpath, 'get_wal': False, } backup_info = testing_helpers.build_test_backup_info() backup_manager = testing_helpers.build_backup_manager() # Build a recovery executor executor = RecoveryExecutor.factory(backup_manager) executor._set_pitr_targets(recovery_info, backup_info, dest.strpath, '', '', '', '') # Test with empty values (no PITR) assert recovery_info['target_epoch'] is None assert recovery_info['target_datetime'] is None assert recovery_info['wal_dest'] == wal_dest.strpath # Test for PITR targets executor._set_pitr_targets( recovery_info, backup_info, dest.strpath, 'target_name', '2015-06-03 16:11:03.71038+02', '2', None, ) target_datetime = dateutil.parser.parse( '2015-06-03 16:11:03.710380+02:00') target_epoch = (time.mktime(target_datetime.timetuple()) + (target_datetime.microsecond / 1000000.)) assert recovery_info['target_datetime'] == target_datetime assert recovery_info['target_epoch'] == target_epoch assert recovery_info['wal_dest'] == dest.join('barman_xlog').strpath
def test_map_temporary_config_files(self, tmpdir): """ Test the method that prepares configuration files for the final steps of a recovery """ # Build directory/files structure for testing tempdir = tmpdir.mkdir('tempdir') recovery_info = { 'configuration_files': ['postgresql.conf', 'postgresql.auto.conf'], 'tempdir': tempdir.strpath, 'temporary_configuration_files': [], 'results': { 'changes': [], 'warnings': [], 'missing_files': [] }, } backup_info = testing_helpers.build_test_backup_info() backup_info.config.basebackups_directory = tmpdir.strpath datadir = tmpdir.mkdir(backup_info.backup_id).mkdir('data') postgresql_conf_local = datadir.join('postgresql.conf') postgresql_auto_local = datadir.join('postgresql.auto.conf') postgresql_conf_local.write('archive_command = something\n' 'data_directory = something') postgresql_auto_local.write('archive_command = something\n' 'data_directory = something') # Build a RecoveryExecutor object (using a mock as server and backup # manager. backup_manager = testing_helpers.build_backup_manager() executor = RecoveryExecutor.factory(backup_manager) executor._map_temporary_config_files(recovery_info, backup_info, 'ssh@something') # check that configuration files have been moved by the method assert tempdir.join('postgresql.conf').check() assert tempdir.join('postgresql.conf').computehash() == \ postgresql_conf_local.computehash() assert tempdir.join('postgresql.auto.conf').check() assert tempdir.join('postgresql.auto.conf').computehash() == \ postgresql_auto_local.computehash() assert recovery_info['results']['missing_files'] == [ 'pg_hba.conf', 'pg_ident.conf' ]
def test_analyse_temporary_config_files(self, tmpdir): """ Test the method that identifies dangerous options into the configuration files """ # Build directory/files structure for testing tempdir = tmpdir.mkdir('tempdir') recovery_info = { 'configuration_files': ['postgresql.conf', 'postgresql.auto.conf'], 'tempdir': tempdir.strpath, 'temporary_configuration_files': [], 'results': { 'changes': [], 'warnings': [] } } postgresql_conf = tempdir.join('postgresql.conf') postgresql_auto = tempdir.join('postgresql.auto.conf') postgresql_conf.write('archive_command = something\n' 'data_directory = something\n' 'include = something\n' 'include "without braces"') postgresql_auto.write('archive_command = something\n' 'data_directory = something') recovery_info['temporary_configuration_files'].append( postgresql_conf.strpath) recovery_info['temporary_configuration_files'].append( postgresql_auto.strpath) # Build a RecoveryExecutor object (using a mock as server and backup # manager. backup_manager = testing_helpers.build_backup_manager() executor = RecoveryExecutor.factory(backup_manager) # Identify dangerous options into config files for remote recovery executor._analyse_temporary_config_files(recovery_info) assert len(recovery_info['results']['changes']) == 2 assert len(recovery_info['results']['warnings']) == 4 # Clean for a local recovery test recovery_info['results']['changes'] = [] recovery_info['results']['warnings'] = [] # Identify dangerous options for local recovery executor._analyse_temporary_config_files(recovery_info) assert len(recovery_info['results']['changes']) == 2 assert len(recovery_info['results']['warnings']) == 4
def test_recover_backup_copy(self, copy_controller_mock, tmpdir): """ Test the copy of a content of a backup during a recovery """ # Build basic folder/files structure dest = tmpdir.mkdir('destination') server = testing_helpers.build_real_server() backup_info = testing_helpers.build_test_backup_info( server=server, tablespaces=[('tbs1', 16387, '/fake/location')]) # Build a executor executor = RecoveryExecutor.factory(server.backup_manager) executor.config.tablespace_bandwidth_limit = {'tbs1': ''} executor.config.bandwidth_limit = 10 executor._backup_copy(backup_info, dest.strpath, tablespaces=None) # Check the calls assert copy_controller_mock.mock_calls == [ mock.call(network_compression=False, path=None, safe_horizon=None, ssh_command=None, retry_sleep=30, retry_times=0), mock.call().add_directory( bwlimit='', dst='/fake/location', item_class=copy_controller_mock.return_value.TABLESPACE_CLASS, label='tbs1', src=backup_info.get_data_directory(16387) + '/'), mock.call().add_directory( bwlimit=10, dst=dest.strpath, exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid' ], exclude_and_protect=['/pg_tblspc/16387'], item_class=copy_controller_mock.return_value.PGDATA_CLASS, label='pgdata', src=backup_info.get_data_directory() + '/'), mock.call().copy(), ]
def test_generate_recovery_conf(self, rsync_pg_mock, tmpdir): """ Test the generation of recovery.conf file """ # Build basic folder/files structure recovery_info = { 'configuration_files': ['postgresql.conf', 'postgresql.auto.conf'], 'tempdir': tmpdir.strpath, 'results': { 'changes': [], 'warnings': [] }, 'get_wal': False, } backup_info = testing_helpers.build_test_backup_info() dest = tmpdir.mkdir('destination') # Build a recovery executor using a real server server = testing_helpers.build_real_server() executor = RecoveryExecutor.factory(server.backup_manager) executor._generate_recovery_conf(recovery_info, backup_info, dest.strpath, True, 'remote@command', 'target_name', '2015-06-03 16:11:03.71038+02', '2', '') # Check that the recovery.conf file exists recovery_conf_file = tmpdir.join("recovery.conf") assert recovery_conf_file.check() # Parse the generated recovery.conf recovery_conf = {} for line in recovery_conf_file.readlines(): key, value = (s.strip() for s in line.strip().split('=', 1)) recovery_conf[key] = value # check for contents assert 'recovery_end_command' in recovery_conf assert 'recovery_target_time' in recovery_conf assert 'recovery_target_timeline' in recovery_conf assert 'recovery_target_xid' not in recovery_conf assert 'recovery_target_name' in recovery_conf assert recovery_conf['recovery_end_command'] == "'rm -fr barman_xlog'" assert recovery_conf['recovery_target_time'] == \ "'2015-06-03 16:11:03.71038+02'" assert recovery_conf['recovery_target_timeline'] == '2' assert recovery_conf['recovery_target_name'] == "'target_name'"
def recover(self, backup_info, dest, tablespaces=None, target_tli=None, target_time=None, target_xid=None, target_name=None, exclusive=False, remote_command=None): """ Performs a recovery of a backup :param barman.infofile.BackupInfo backup_info: the backup to recover :param str dest: the destination directory :param dict[str,str]|None tablespaces: a tablespace name -> location map (for relocation) :param str|None target_tli: the target timeline :param str|None target_time: the target time :param str|None target_xid: the target xid :param str|None target_name: the target name created previously with pg_create_restore_point() function call :param bool exclusive: whether the recovery is exclusive or not :param str|None remote_command: default None. The remote command to recover the base backup, in case of remote backup. """ # Archive every WAL files in the incoming directory of the server self.server.archive_wal(verbose=False) # Delegate the recovery operation to a RecoveryExecutor object executor = RecoveryExecutor.factory(self) recovery_info = executor.recover(backup_info, dest, tablespaces, target_tli, target_time, target_xid, target_name, exclusive, remote_command) # Output recovery results output.result('recovery', recovery_info['results'])
def test_prepare_tablespaces(self, tmpdir): """ Test tablespaces preparation for recovery """ # Prepare basic directory/files structure dest = tmpdir.mkdir('destination') wals = tmpdir.mkdir('wals') backup_info = testing_helpers.build_test_backup_info( tablespaces=[('tbs1', 16387, '/fake/location')]) # build an executor server = testing_helpers.build_real_server( main_conf={'wals_directory': wals.strpath}) executor = RecoveryExecutor.factory(server.backup_manager) # use a mock as cmd obj cmd_mock = mock.Mock() executor._prepare_tablespaces(backup_info, cmd_mock, dest.strpath, {}) cmd_mock.create_dir_if_not_exists.assert_any_call( dest.join('pg_tblspc').strpath) cmd_mock.create_dir_if_not_exists.assert_any_call('/fake/location') cmd_mock.delete_if_exists.assert_called_once_with( dest.join('pg_tblspc').join('16387').strpath) cmd_mock.create_symbolic_link.assert_called_once_with( '/fake/location', dest.join('pg_tblspc').join('16387').strpath)
def test_recovery(self, remote_cmd_mock, rsync_pg_mock, copy_controller_mock, tmpdir): """ Test the execution of a recovery """ # Prepare basic directory/files structure dest = tmpdir.mkdir('destination') base = tmpdir.mkdir('base') wals = tmpdir.mkdir('wals') backup_info = testing_helpers.build_test_backup_info(tablespaces=[]) backup_info.config.basebackups_directory = base.strpath backup_info.config.wals_directory = wals.strpath backup_info.version = 90400 datadir = base.mkdir(backup_info.backup_id).mkdir('data') backup_info.pgdata = datadir.strpath postgresql_conf_local = datadir.join('postgresql.conf') postgresql_auto_local = datadir.join('postgresql.auto.conf') postgresql_conf_local.write('archive_command = something\n' 'data_directory = something') postgresql_auto_local.write('archive_command = something\n' 'data_directory = something') shutil.copy2(postgresql_conf_local.strpath, dest.strpath) shutil.copy2(postgresql_auto_local.strpath, dest.strpath) # Avoid triggering warning for missing config files datadir.ensure('pg_hba.conf') datadir.ensure('pg_ident.conf') # Build an executor server = testing_helpers.build_real_server( global_conf={ "barman_lock_directory": tmpdir.mkdir('lock').strpath }, main_conf={"wals_directory": wals.strpath}) executor = RecoveryExecutor.factory(server.backup_manager) # test local recovery rec_info = executor.recover(backup_info, dest.strpath, None, None, None, None, None, True, None) # remove not usefull keys from the result del rec_info['cmd'] sys_tempdir = rec_info['tempdir'] assert rec_info == { 'rsync': None, 'tempdir': sys_tempdir, 'wal_dest': dest.join('pg_xlog').strpath, 'recovery_dest': 'local', 'destination_path': dest.strpath, 'temporary_configuration_files': [ dest.join('postgresql.conf').strpath, dest.join('postgresql.auto.conf').strpath ], 'results': { 'delete_barman_xlog': False, 'get_wal': False, 'changes': [ Assertion._make( ['postgresql.conf', 0, 'archive_command', 'false']), Assertion._make([ 'postgresql.auto.conf', 0, 'archive_command', 'false' ]) ], 'missing_files': [], 'warnings': [ Assertion._make( ['postgresql.conf', 2, 'data_directory', 'something']), Assertion._make([ 'postgresql.auto.conf', 2, 'data_directory', 'something' ]) ] }, 'target_epoch': None, 'configuration_files': ['postgresql.conf', 'postgresql.auto.conf'], 'target_datetime': None, 'safe_horizon': None, 'is_pitr': False, 'get_wal': False, } # test remote recovery rec_info = executor.recover(backup_info, dest.strpath, {}, None, None, None, None, True, "remote@command") # remove not useful keys from the result del rec_info['cmd'] del rec_info['rsync'] sys_tempdir = rec_info['tempdir'] assert rec_info == { 'tempdir': sys_tempdir, 'wal_dest': dest.join('pg_xlog').strpath, 'recovery_dest': 'remote', 'destination_path': dest.strpath, 'temporary_configuration_files': [ os.path.join(sys_tempdir, 'postgresql.conf'), os.path.join(sys_tempdir, 'postgresql.auto.conf') ], 'results': { 'delete_barman_xlog': False, 'get_wal': False, 'changes': [ Assertion._make( ['postgresql.conf', 0, 'archive_command', 'false']), Assertion._make([ 'postgresql.auto.conf', 0, 'archive_command', 'false' ]) ], 'missing_files': [], 'warnings': [ Assertion._make( ['postgresql.conf', 2, 'data_directory', 'something']), Assertion._make([ 'postgresql.auto.conf', 2, 'data_directory', 'something' ]) ] }, 'target_epoch': None, 'configuration_files': ['postgresql.conf', 'postgresql.auto.conf'], 'target_datetime': None, 'safe_horizon': None, 'is_pitr': False, 'get_wal': False, } # test failed rsync rsync_pg_mock.side_effect = CommandFailedException() with pytest.raises(CommandFailedException): executor.recover(backup_info, dest.strpath, {}, None, None, None, None, True, "remote@command")
def test_recover_xlog(self, rsync_pg_mock, cm_mock, tmpdir): """ Test the recovery of the xlogs of a backup :param rsync_pg_mock: Mock rsync object for the purpose if this test """ # Build basic folders/files structure dest = tmpdir.mkdir('destination') wals = tmpdir.mkdir('wals') # Create 3 WAL files with different compressions xlog_dir = wals.mkdir(xlog.hash_dir('000000000000000000000002')) xlog_plain = xlog_dir.join('000000000000000000000001') xlog_gz = xlog_dir.join('000000000000000000000002') xlog_bz2 = xlog_dir.join('000000000000000000000003') xlog_plain.write('dummy content') xlog_gz.write('dummy content gz') xlog_bz2.write('dummy content bz2') server = testing_helpers.build_real_server( main_conf={'wals_directory': wals.strpath}) # Prepare compressors mock c = { 'gzip': mock.Mock(name='gzip'), 'bzip2': mock.Mock(name='bzip2'), } cm_mock.return_value.get_compressor = \ lambda compression=None, path=None: c[compression] # touch destination files to avoid errors on cleanup c['gzip'].decompress.side_effect = lambda src, dst: open(dst, 'w') c['bzip2'].decompress.side_effect = lambda src, dst: open(dst, 'w') # Build executor executor = RecoveryExecutor.factory(server.backup_manager) # Test: local copy required_wals = ( WalFileInfo.from_xlogdb_line( '000000000000000000000001\t42\t43\tNone\n'), WalFileInfo.from_xlogdb_line( '000000000000000000000002\t42\t43\tgzip\n'), WalFileInfo.from_xlogdb_line( '000000000000000000000003\t42\t43\tbzip2\n'), ) executor._xlog_copy(required_wals, dest.strpath, None) # Check for a correct invocation of rsync using local paths rsync_pg_mock.assert_called_once_with(network_compression=False, bwlimit=None, path=None, ssh=None) assert not rsync_pg_mock.return_value.from_file_list.called c['gzip'].decompress.assert_called_once_with(xlog_gz.strpath, mock.ANY) c['bzip2'].decompress.assert_called_once_with(xlog_bz2.strpath, mock.ANY) # Reset mock calls rsync_pg_mock.reset_mock() c['gzip'].reset_mock() c['bzip2'].reset_mock() # Test: remote copy executor._xlog_copy(required_wals, dest.strpath, 'remote_command') # Check for the invocation of rsync on a remote call rsync_pg_mock.assert_called_once_with(network_compression=False, bwlimit=None, path=mock.ANY, ssh='remote_command') rsync_pg_mock.return_value.from_file_list.assert_called_once_with([ '000000000000000000000001', '000000000000000000000002', '000000000000000000000003' ], mock.ANY, mock.ANY) c['gzip'].decompress.assert_called_once_with(xlog_gz.strpath, mock.ANY) c['bzip2'].decompress.assert_called_once_with(xlog_bz2.strpath, mock.ANY)