def test_delete_running_backup(self, delete_mock, get_first_backup_mock, tmpdir, capsys):
"""
Simple test for the deletion of a running backup.
We want to test the behaviour of the server.delete_backup method
when invoked on a running backup
"""
# Test the removal of a running backup. status STARTED
server = build_real_server({"barman_home": tmpdir.strpath})
backup_info_started = build_test_backup_info(status=BackupInfo.STARTED, server_name=server.config.name)
get_first_backup_mock.return_value = backup_info_started.backup_id
with ServerBackupLock(tmpdir.strpath, server.config.name):
server.delete_backup(backup_info_started)
out, err = capsys.readouterr()
assert "Cannot delete a running backup (%s %s)" % (server.config.name, backup_info_started.backup_id) in err
# Test the removal of a running backup. status EMPTY
backup_info_empty = build_test_backup_info(status=BackupInfo.EMPTY, server_name=server.config.name)
get_first_backup_mock.return_value = backup_info_empty.backup_id
with ServerBackupLock(tmpdir.strpath, server.config.name):
server.delete_backup(backup_info_empty)
out, err = capsys.readouterr()
assert "Cannot delete a running backup (%s %s)" % (server.config.name, backup_info_started.backup_id) in err
# Test the removal of a running backup. status DONE
backup_info_done = build_test_backup_info(status=BackupInfo.DONE, server_name=server.config.name)
with ServerBackupLock(tmpdir.strpath, server.config.name):
server.delete_backup(backup_info_done)
delete_mock.assert_called_with(backup_info_done)
# Test the removal of a backup not running. status STARTED
server.delete_backup(backup_info_started)
delete_mock.assert_called_with(backup_info_started)
def test_get_backup(self, tmpdir):
"""
Check the get_backup method that uses the backups cache to retrieve
a backup using the id
"""
# Setup temp dir and server
# build a backup_manager and setup a basic configuration
backup_manager = build_backup_manager(
name='TestServer',
global_conf={
'barman_home': tmpdir.strpath
})
# Create a BackupInfo object with status DONE
b_info = build_test_backup_info(
backup_id='fake_backup_id',
server=backup_manager.server,
)
b_info.save()
assert backup_manager._backup_cache is None
# Check that the backup returned is the same
assert backup_manager.get_backup(b_info.backup_id).to_dict() == \
b_info.to_dict()
# Empty the backup manager cache
backup_manager._backup_cache = {}
# Check that the backup returned is None
assert backup_manager.get_backup(b_info.backup_id) is None
def test_recover_basebackup_copy(self, rsync_pg_mock, tmpdir):
"""
Test the copy of a content of a backup during a recovery
:param rsync_pg_mock: Mock rsync object for the purpose if this test
"""
# Build basic folder/files structure
dest = tmpdir.mkdir('destination')
server = testing_helpers.build_real_server()
backup_info = testing_helpers.build_test_backup_info(
server=server,
tablespaces=[('tbs1', 16387, '/fake/location')])
# Build a executor
executor = RecoveryExecutor(server.backup_manager)
executor.config.tablespace_bandwidth_limit = {'tbs1': ''}
executor.config.bandwidth_limit = 10
executor.basebackup_copy(
backup_info, dest.strpath, tablespaces=None)
rsync_pg_mock.assert_called_with(
network_compression=False, bwlimit=10, ssh=None, path=None,
exclude_and_protect=['/pg_tblspc/16387'])
rsync_pg_mock.assert_any_call(
network_compression=False, bwlimit='', ssh=None, path=None,
check=True)
rsync_pg_mock.return_value.smart_copy.assert_any_call(
'/some/barman/home/main/base/1234567890/16387/',
'/fake/location', None)
rsync_pg_mock.return_value.smart_copy.assert_called_with(
'/some/barman/home/main/base/1234567890/data/',
dest.strpath, None)
def test_prepare_tablespaces(self, tmpdir):
"""
Test tablespaces preparation for recovery
"""
# Prepare basic directory/files structure
dest = tmpdir.mkdir('destination')
wals = tmpdir.mkdir('wals')
backup_info = testing_helpers.build_test_backup_info(
tablespaces=[('tbs1', 16387, '/fake/location')])
# build an executor
server = testing_helpers.build_real_server(
main_conf={'wals_directory': wals.strpath})
executor = RecoveryExecutor(server.backup_manager)
# use a mock as cmd obj
cmd_mock = Mock()
executor.prepare_tablespaces(backup_info, cmd_mock, dest.strpath, {})
cmd_mock.create_dir_if_not_exists.assert_any_call(
dest.join('pg_tblspc').strpath)
cmd_mock.create_dir_if_not_exists.assert_any_call(
'/fake/location')
cmd_mock.delete_if_exists.assert_called_once_with(
dest.join('pg_tblspc').join('16387').strpath)
cmd_mock.create_symbolic_link.assert_called_once_with(
'/fake/location',
dest.join('pg_tblspc').join('16387').strpath)
def test_backup_cache_remove(self, tmpdir):
"""
Check the method responsible for the removal of a BackupInfo object from
the backups cache
"""
# build a backup_manager and setup a basic configuration
backup_manager = build_backup_manager(
name='TestServer',
global_conf={
'barman_home': tmpdir.strpath
})
assert backup_manager._backup_cache is None
# Create a BackupInfo object with status DONE
b_info = build_test_backup_info(
backup_id='fake_backup_id',
server=backup_manager.server,
)
# Remove the backup from the uninitialized cache
backup_manager.backup_cache_remove(b_info)
# Check that the test backup is still not initialized
assert backup_manager._backup_cache is None
# Initialize the cache
backup_manager._backup_cache = {b_info.backup_id: b_info}
# Remove the backup from the cache
backup_manager.backup_cache_remove(b_info)
assert b_info.backup_id not in backup_manager._backup_cache
def test_set_pitr_targets(self, tmpdir):
"""
Evaluate targets for point in time recovery
"""
# Build basic folder/files structure
tempdir = tmpdir.mkdir("temp_dir")
dest = tmpdir.mkdir("dest")
wal_dest = tmpdir.mkdir("wal_dest")
recovery_info = {
"configuration_files": ["postgresql.conf", "postgresql.auto.conf"],
"tempdir": tempdir.strpath,
"results": {"changes": [], "warnings": []},
"is_pitr": False,
"wal_dest": wal_dest.strpath,
"get_wal": False,
}
backup_info = testing_helpers.build_test_backup_info()
backup_manager = testing_helpers.build_backup_manager()
# Build a recovery executor
executor = RecoveryExecutor(backup_manager)
executor._set_pitr_targets(recovery_info, backup_info, dest.strpath, "", "", "", "")
# Test with empty values (no PITR)
assert recovery_info["target_epoch"] is None
assert recovery_info["target_datetime"] is None
assert recovery_info["wal_dest"] == wal_dest.strpath
# Test for PITR targets
executor._set_pitr_targets(
recovery_info, backup_info, dest.strpath, "target_name", "2015-06-03 16:11:03.71038+02", "2", None
)
target_datetime = dateutil.parser.parse("2015-06-03 16:11:03.710380+02:00")
target_epoch = time.mktime(target_datetime.timetuple()) + (target_datetime.microsecond / 1000000.0)
assert recovery_info["target_datetime"] == target_datetime
assert recovery_info["target_epoch"] == target_epoch
assert recovery_info["wal_dest"] == dest.join("barman_xlog").strpath
def test_get_wal_info(self, get_wal_mock, tmpdir):
"""
Basic test for get_wal_info method
Test the wals per second and total time in seconds values.
:return:
"""
# Build a test server with a test path
server = build_real_server(global_conf={
'barman_home': tmpdir.strpath
})
# Mock method get_wal_until_next_backup for returning a list of
# 3 fake WAL. the first one is the start and stop WAL of the backup
wal_list = [
WalFileInfo.from_xlogdb_line(
"000000010000000000000002\t16777216\t1434450086.53\tNone\n"),
WalFileInfo.from_xlogdb_line(
"000000010000000000000003\t16777216\t1434450087.54\tNone\n"),
WalFileInfo.from_xlogdb_line(
"000000010000000000000004\t16777216\t1434450088.55\tNone\n")]
get_wal_mock.return_value = wal_list
backup_info = build_test_backup_info(
server=server,
begin_wal=wal_list[0].name,
end_wal=wal_list[0].name)
backup_info.save()
# Evaluate total time in seconds:
# last_wal_timestamp - first_wal_timestamp
wal_total_seconds = wal_list[-1].time - wal_list[0].time
# Evaluate the wals_per_second value:
# wals_in_backup + wals_until_next_backup / total_time_in_seconds
wals_per_second = len(wal_list) / wal_total_seconds
wal_info = server.get_wal_info(backup_info)
assert wal_info
assert wal_info['wal_total_seconds'] == wal_total_seconds
assert wal_info['wals_per_second'] == wals_per_second
def test_backup_copy(self, rsync_mock, tmpdir):
"""
Test the execution of a rsync copy
:param rsync_mock: mock for the rsync command
:param tmpdir: temporary dir
"""
backup_manager = build_backup_manager(global_conf={
'barman_home': tmpdir.mkdir('home').strpath
})
backup_info = build_test_backup_info(
server=backup_manager.server,
pgdata="/pg/data",
config_file="/etc/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/data/pg_ident.conf",
begin_xlog="0/2000028",
begin_wal="000000010000000000000002",
begin_offset=28)
backup_info.save()
# This is to check that all the preparation is done correctly
assert os.path.exists(backup_info.filename)
backup_manager.executor.backup_copy(backup_info)
assert rsync_mock.mock_calls == [
mock.call(check=True, network_compression=False, args=[],
bwlimit=None, ssh='ssh',
ssh_options=['-c', '"arcfour"', '-p', '22',
'*****@*****.**', '-o',
'BatchMode=yes', '-o',
'StrictHostKeyChecking=no']),
mock.call().smart_copy(':/fake/location/',
backup_info.get_data_directory(16387),
None, None),
mock.call(check=True, network_compression=False, args=[],
bwlimit=None, ssh='ssh',
ssh_options=['-c', '"arcfour"', '-p', '22',
'*****@*****.**', '-o',
'BatchMode=yes', '-o',
'StrictHostKeyChecking=no']),
mock.call().smart_copy(':/another/location/',
backup_info.get_data_directory(16405),
None, None),
mock.call(network_compression=False,
exclude_and_protect=['/pg_tblspc/16387',
'/pg_tblspc/16405'],
args=[], bwlimit=None, ssh='ssh',
ssh_options=['-c', '"arcfour"', '-p', '22',
'*****@*****.**',
'-o', 'BatchMode=yes',
'-o', 'StrictHostKeyChecking=no']),
mock.call().smart_copy(':/pg/data/',
backup_info.get_data_directory(),
None, None),
mock.call()(
':/pg/data/global/pg_control',
'%s/global/pg_control' % backup_info.get_data_directory()),
mock.call()(':/etc/postgresql.conf',
backup_info.get_data_directory())]
def test_map_temporary_config_files(self, tmpdir):
"""
Test the method that prepares configuration files
for the final steps of a recovery
"""
# Build directory/files structure for testing
tempdir = tmpdir.mkdir("tempdir")
recovery_info = {
"configuration_files": ["postgresql.conf", "postgresql.auto.conf"],
"tempdir": tempdir.strpath,
"temporary_configuration_files": [],
"results": {"changes": [], "warnings": [], "missing_files": []},
}
backup_info = testing_helpers.build_test_backup_info()
backup_info.config.basebackups_directory = tmpdir.strpath
datadir = tmpdir.mkdir(backup_info.backup_id).mkdir("data")
postgresql_conf_local = datadir.join("postgresql.conf")
postgresql_auto_local = datadir.join("postgresql.auto.conf")
postgresql_conf_local.write("archive_command = something\n" "data_directory = something")
postgresql_auto_local.write("archive_command = something\n" "data_directory = something")
# Build a RecoveryExecutor object (using a mock as server and backup
# manager.
backup_manager = testing_helpers.build_backup_manager()
executor = RecoveryExecutor(backup_manager)
executor._map_temporary_config_files(recovery_info, backup_info, "ssh@something")
# check that configuration files have been moved by the method
assert tempdir.join("postgresql.conf").check()
assert tempdir.join("postgresql.conf").computehash() == postgresql_conf_local.computehash()
assert tempdir.join("postgresql.auto.conf").check()
assert tempdir.join("postgresql.auto.conf").computehash() == postgresql_auto_local.computehash()
assert recovery_info["results"]["missing_files"] == ["pg_hba.conf", "pg_ident.conf"]
def test_pgespresso_stop_backup(self, tbs_map_mock, label_mock):
"""
Basic test for the pgespresso_stop_backup method
"""
# Build a backup info and configure the mocks
server = build_mocked_server(main_conf={
'backup_options':
BackupOptions.CONCURRENT_BACKUP
})
backup_manager = build_backup_manager(server=server)
# Mock executor._pgespresso_stop_backup(backup_info) call
stop_time = datetime.datetime.now()
server.postgres.server_version = 90500
server.postgres.pgespresso_stop_backup.return_value = {
'end_wal': "000000060000A25700000044",
'timestamp': stop_time
}
backup_info = build_test_backup_info(timeline=6)
backup_manager.executor.strategy.stop_backup(backup_info)
assert backup_info.end_xlog == 'A257/44FFFFFF'
assert backup_info.end_wal == '000000060000A25700000044'
assert backup_info.end_offset == 0xFFFFFF
assert backup_info.end_time == stop_time
def test_exclusive_stop_backup(self, stop_mock):
"""
Basic test for the start_backup method
:param stop_mock: mimic the response od _pg_stop_backup
"""
# Build a backup info and configure the mocks
server = build_mocked_server(main_conf={
'backup_options':
BackupOptions.EXCLUSIVE_BACKUP
})
backup_manager = build_backup_manager(server=server)
# Mock executor._pg_stop_backup(backup_info) call
stop_time = datetime.datetime.now()
stop_mock.return_value = ("266/4A9C1EF8",
"00000010000002660000004A",
10231544,
stop_time)
backup_info = build_test_backup_info()
backup_manager.executor.strategy.stop_backup(backup_info)
# check that the submitted values are stored inside the BackupInfo obj
assert backup_info.end_xlog == '266/4A9C1EF8'
assert backup_info.end_wal == '00000010000002660000004A'
assert backup_info.end_offset == 10231544
assert backup_info.end_time == stop_time
def test_backup_copy_with_included_files(self, rsync_moc, tmpdir, capsys):
backup_manager = build_backup_manager(global_conf={
'barman_home': tmpdir.mkdir('home').strpath
})
# Create a backup info with additional configuration files
backup_info = build_test_backup_info(
server=backup_manager.server,
pgdata="/pg/data",
config_file="/etc/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/data/pg_ident.conf",
begin_xlog="0/2000028",
begin_wal="000000010000000000000002",
included_files=["/tmp/config/file.conf"],
begin_offset=28)
backup_info.save()
# This is to check that all the preparation is done correctly
assert os.path.exists(backup_info.filename)
# Execute a backup
backup_manager.executor.backup_copy(backup_info)
out, err = capsys.readouterr()
# check for the presence of the warning in the stderr
assert "WARNING: The usage of include directives is not supported" in err
# check that the additional configuration file is present in the output
assert backup_info.included_files[0] in err
def test_concurrent_stop_backup(self, label_mock, stop_mock,):
"""
Basic test for the start_backup method
:param label_mock: mimic the response of _write_backup_label
:param stop_mock: mimic the response of _pgespresso_stop_backup
"""
# Build a backup info and configure the mocks
server = build_mocked_server(main_conf={
'backup_options':
BackupOptions.CONCURRENT_BACKUP
})
backup_manager = build_backup_manager(server=server)
# Mock executor._pgespresso_stop_backup(backup_info) call
stop_time = datetime.datetime.now()
stop_mock.return_value = ("000000060000A25700000044", stop_time)
backup_info = build_test_backup_info()
backup_manager.executor.strategy.stop_backup(backup_info)
assert backup_info.end_xlog == 'A257/45000000'
assert backup_info.end_wal == '000000060000A25700000044'
assert backup_info.end_offset == 0
assert backup_info.end_time == stop_time
def test_load_backup_cache(self, tmpdir):
"""
Check the loading of backups inside the backup_cache
"""
# build a backup_manager and setup a basic configuration
backup_manager = build_backup_manager(
name='TestServer',
global_conf={
'barman_home': tmpdir.strpath
})
# Make sure the cache is uninitialized
assert backup_manager._backup_cache is None
# Create a BackupInfo object with status DONE
b_info = build_test_backup_info(
backup_id='fake_backup_id',
server=backup_manager.server,
)
b_info.save()
# Load backups inside the cache
backup_manager._load_backup_cache()
# Check that the test backup is inside the backups_cache
assert backup_manager._backup_cache[b_info.backup_id].to_dict() == \
b_info.to_dict()
def test_setup(self, rsync_mock):
"""
Test the method that set up a recovery
"""
backup_info = testing_helpers.build_test_backup_info()
backup_manager = testing_helpers.build_backup_manager()
executor = RecoveryExecutor(backup_manager)
backup_info.version = 90300
# setup should create a temporary directory
# and teardown should delete it
ret = executor._setup(backup_info, None, "/tmp")
assert os.path.exists(ret["tempdir"])
executor._teardown(ret)
assert not os.path.exists(ret["tempdir"])
# no postgresql.auto.conf on version 9.3
ret = executor._setup(backup_info, None, "/tmp")
executor._teardown(ret)
assert "postgresql.auto.conf" not in ret["configuration_files"]
# Check the present for postgresql.auto.conf on version 9.4
backup_info.version = 90400
ret = executor._setup(backup_info, None, "/tmp")
executor._teardown(ret)
assert "postgresql.auto.conf" in ret["configuration_files"]
# Receive a error if the remote command is invalid
with pytest.raises(SystemExit):
executor.server.path = None
executor._setup(backup_info, "invalid", "/tmp")
def test_backup_cache_add(self, tmpdir):
"""
Check the method responsible for the registration of a BackupInfo obj
into the backups cache
"""
# build a backup_manager and setup a basic configuration
backup_manager = build_backup_manager(
name='TestServer',
global_conf={
'barman_home': tmpdir.strpath
})
# Create a BackupInfo object with status DONE
b_info = build_test_backup_info(
backup_id='fake_backup_id',
server=backup_manager.server,
)
b_info.save()
assert backup_manager._backup_cache is None
# Register the object to cache. The cache is not initialized, so it
# must load the cache from disk.
backup_manager.backup_cache_add(b_info)
# Check that the test backup is in the cache
assert backup_manager.get_backup(b_info.backup_id) is b_info
# Initialize an empty cache
backup_manager._backup_cache = {}
# Add the backup again
backup_manager.backup_cache_add(b_info)
assert backup_manager.get_backup(b_info.backup_id) is b_info
def test_get_wal_until_next_backup(self, get_backup_mock, tmpdir):
"""
Simple test for the management of .history files
"""
# build a WalFileInfo object
wfile_info = WalFileInfo()
wfile_info.name = '000000010000000000000003'
wfile_info.size = 42
wfile_info.time = 43
wfile_info.compression = None
# build a WalFileInfo history object
history_info = WalFileInfo()
history_info.name = '00000001.history'
history_info.size = 42
history_info.time = 43
history_info.compression = None
# create a xlog.db and add the 2 entries
wals_dir = tmpdir.mkdir("wals")
xlog = wals_dir.join("xlog.db")
xlog.write(wfile_info.to_xlogdb_line() + history_info.to_xlogdb_line())
# fake backup
backup = build_test_backup_info(
begin_wal='000000010000000000000001',
end_wal='000000010000000000000004')
# mock a server object and mock a return call to get_next_backup method
server = build_real_server(
global_conf={
"barman_lock_directory": tmpdir.mkdir('lock').strpath
},
main_conf={
"wals_directory": wals_dir.strpath
})
get_backup_mock.return_value = build_test_backup_info(
backup_id="1234567899",
begin_wal='000000010000000000000005',
end_wal='000000010000000000000009')
wals = []
for wal_file in server.get_wal_until_next_backup(backup,
include_history=True):
# get the result of the xlogdb read
wals.append(wal_file.name)
# check for the presence of the .history file
assert history_info.name in wals
def test_first_backup(self):
server = build_mocked_server()
rp = RetentionPolicyFactory.create(
server,
'retention_policy',
'RECOVERY WINDOW OF 4 WEEKS')
assert isinstance(rp, RecoveryWindowRetentionPolicy)
# Build a BackupInfo object with status to DONE
backup_info = build_test_backup_info(
server=rp.server,
backup_id='test1',
end_time=datetime.now(tzlocal()))
# instruct the get_available_backups method to return a map with
# our mock as result and minimum_redundancy = 1
rp.server.get_available_backups.return_value = {
"test_backup": backup_info
}
rp.server.config.minimum_redundancy = 1
# execute retention policy report
report = rp.first_backup()
assert report == 'test_backup'
rp = RetentionPolicyFactory.create(
server,
'retention_policy',
'REDUNDANCY 2')
assert isinstance(rp, RedundancyRetentionPolicy)
# Build a BackupInfo object with status to DONE
backup_info = build_test_backup_info(
server=rp.server,
backup_id='test1',
end_time=datetime.now(tzlocal()))
# instruct the get_available_backups method to return a map with
# our mock as result and minimum_redundancy = 1
rp.server.get_available_backups.return_value = {
"test_backup": backup_info
}
rp.server.config.minimum_redundancy = 1
# execute retention policy report
report = rp.first_backup()
assert report == 'test_backup'
def test_recovery_window_report(self, caplog):
"""
Basic unit test of RecoveryWindowRetentionPolicy
Given a mock simulating a Backup with status DONE and
the end_date not over the point of recoverability,
the report method of the RecoveryWindowRetentionPolicy class must mark
it as valid
"""
server = build_mocked_server()
rp = RetentionPolicyFactory.create(
server,
'retention_policy',
'RECOVERY WINDOW OF 4 WEEKS')
assert isinstance(rp, RecoveryWindowRetentionPolicy)
# Build a BackupInfo object with status to DONE
backup_info = build_test_backup_info(
server=rp.server,
backup_id='test1',
end_time=datetime.now(tzlocal()))
backup_source = {'test_backup3': backup_info}
# Add a obsolete backup
backup_info.end_time = datetime.now(tzlocal()) - timedelta(weeks=5)
backup_source['test_backup2'] = backup_info
# Add a second obsolete backup
backup_info.end_time = datetime.now(tzlocal()) - timedelta(weeks=6)
backup_source['test_backup'] = backup_info
rp.server.get_available_backups.return_value = backup_source
# instruct the get_available_backups method to return a map with
# our mock as result and minimum_redundancy = 1
rp.server.config.minimum_redundancy = 1
rp.server.config.name = "test"
# execute retention policy report
report = rp.report()
# check that our mock is valid for the retention policy
assert report == {'test_backup3': 'VALID',
'test_backup2': 'OBSOLETE',
'test_backup': 'OBSOLETE'}
# Expect a ValueError if passed context is invalid
with pytest.raises(ValueError):
rp.report(context='invalid')
# Set a new minimum_redundancy parameter, enforcing the usage of the
# configuration parameter instead of the retention policy default
rp.server.config.minimum_redundancy = 4
# execute retention policy report
rp.report()
# Check for the warning inside the log
caplog.set_level(logging.WARNING)
log = caplog.text
warn = "WARNING Keeping obsolete backup test_backup2 for " \
"server test (older than %s) due to minimum redundancy " \
"requirements (4)\n" % rp._point_of_recoverability()
assert log.find(warn)
def test_stop_backup(self, espressostop_mock, stop_mock):
"""
Basic test for the start_backup method
:param espressostop_mock: mimic the response of pg_espresso_stop_backup
:param stop_mock: mimic the response od pg_stop_backup
"""
# Build a backup info and configure the mocks
backup_manager = build_backup_manager()
# Test 1: stop exclusive backup
# Mock executor.pg_stop_backup(backup_info) call
stop_time = datetime.datetime.now()
stop_mock.return_value = ("266/4A9C1EF8",
"00000010000002660000004A",
10231544,
stop_time)
backup_info = build_test_backup_info()
backup_manager.executor.stop_backup(backup_info)
# check that the submitted values are stored inside the BackupInfo obj
assert backup_info.end_xlog == '266/4A9C1EF8'
assert backup_info.end_wal == '00000010000002660000004A'
assert backup_info.end_offset == 10231544
assert backup_info.end_time == stop_time
# Test 2: stop concurrent backup
# change the configuration to concurrent backup
backup_manager.executor.config.backup_options = [
BackupOptions.CONCURRENT_BACKUP]
# Mock executor.pgespresso_stop_backup(backup_info) call
stop_time = datetime.datetime.now()
espressostop_mock.return_value = ("000000060000A25700000044", stop_time)
backup_info = build_test_backup_info()
backup_manager.executor.stop_backup(backup_info)
assert backup_info.end_xlog == 'A257/45000000'
assert backup_info.end_wal == '000000060000A25700000044'
assert backup_info.end_offset == 0
assert backup_info.end_time == stop_time
def test_result_list_backup(self, capsys):
# mock the backup info
bi = build_test_backup_info()
backup_size = 12345
wal_size = 54321
retention_status = 'test status'
writer = output.ConsoleOutputWriter()
# test minimal
writer.init_list_backup(bi.server_name, True)
writer.result_list_backup(bi, backup_size, wal_size, retention_status)
writer.close()
(out, err) = capsys.readouterr()
assert writer.minimal
assert bi.backup_id in out
assert err == ''
# test status=DONE output
writer.init_list_backup(bi.server_name, False)
writer.result_list_backup(bi, backup_size, wal_size, retention_status)
writer.close()
(out, err) = capsys.readouterr()
assert not writer.minimal
assert bi.server_name in out
assert bi.backup_id in out
assert str(bi.end_time.ctime()) in out
for name, _, location in bi.tablespaces:
assert '%s:%s' % (name, location)
assert 'Size: ' + pretty_size(backup_size) in out
assert 'WAL Size: ' + pretty_size(wal_size) in out
assert err == ''
# test status = FAILED output
bi = build_test_backup_info(status=BackupInfo.FAILED)
writer.init_list_backup(bi.server_name, False)
writer.result_list_backup(bi, backup_size, wal_size, retention_status)
writer.close()
(out, err) = capsys.readouterr()
assert not writer.minimal
assert bi.server_name in out
assert bi.backup_id in out
assert bi.status in out
def test_archive_wal_timeline_lower_than_backup(self, tmpdir, capsys):
"""
Test archive-wal command behaviour when the WAL files are older than
the first backup of a server.
Expect it to archive the files anyway
"""
# Build a real backup manager and a fake backup
backup_manager = build_backup_manager(
name='TestServer',
global_conf={
'barman_home': tmpdir.strpath
})
b_info = build_test_backup_info(
backup_id='fake_backup_id',
server=backup_manager.server,
begin_wal='000000020000000000000002',
timeline=2
)
b_info.save()
# Build the basic folder structure and files
backup_manager.compression_manager.get_compressor.return_value = None
backup_manager.server.get_backup.return_value = b_info
basedir = tmpdir.join('main')
incoming_dir = basedir.join('incoming')
basedir.mkdir('errors')
archive_dir = basedir.join('wals')
xlog_db = archive_dir.join('xlog.db')
wal_name = '000000010000000000000001'
wal_file = incoming_dir.join(wal_name)
wal_file.ensure()
archive_dir.ensure(dir=True)
xlog_db.ensure()
backup_manager.server.xlogdb.return_value.__enter__.return_value = \
xlog_db.open(mode='a')
backup_manager.server.archivers = [FileWalArchiver(backup_manager)]
backup_manager.archive_wal()
# Check that the WAL file is present inside the wal catalog
with xlog_db.open() as f:
line = str(f.readline())
assert wal_name in line
wal_path = os.path.join(archive_dir.strpath,
barman.xlog.hash_dir(wal_name),
wal_name)
# Check that the wal file have been archived
assert os.path.exists(wal_path)
# Check the output for the archival of the wal file
out, err = capsys.readouterr()
assert ("\t%s\n" % wal_name) in out
def test_available_backups(self, tmpdir):
"""
Test the get_available_backups that retrieves all the
backups from the backups_cache using a set of backup status as filter
"""
# build a backup_manager and setup a basic configuration
backup_manager = build_backup_manager(
name='TestServer',
global_conf={
'barman_home': tmpdir.strpath
})
# BackupInfo object with status DONE
b_info = build_test_backup_info(
backup_id='fake_backup_id',
server=backup_manager.server,
status=BackupInfo.DONE
)
b_info.save()
# Create a BackupInfo object with status FAILED
failed_b_info = build_test_backup_info(
backup_id='failed_backup_id',
server=backup_manager.server,
status=BackupInfo.FAILED
)
failed_b_info.save()
assert backup_manager._backup_cache is None
available_backups = backup_manager.get_available_backups(
(BackupInfo.DONE,))
assert available_backups[b_info.backup_id].to_dict() == (
b_info.to_dict())
# Check that the failed backup have been filtered from the result
assert failed_b_info.backup_id not in available_backups
assert len(available_backups) == 1
def test_redundancy_report(self, caplog):
"""
Test of the management of the minimum_redundancy parameter
into the backup_report method of the RedundancyRetentionPolicy class
"""
server = build_mocked_server()
rp = RetentionPolicyFactory.create(
server,
'retention_policy',
'REDUNDANCY 2')
assert isinstance(rp, RedundancyRetentionPolicy)
# Build a BackupInfo object with status to DONE
backup_info = build_test_backup_info(
server=rp.server,
backup_id='test1',
end_time=datetime.now(tzlocal()))
# instruct the get_available_backups method to return a map with
# our mock as result and minimum_redundancy = 1
rp.server.get_available_backups.return_value = {
"test_backup": backup_info,
"test_backup2": backup_info,
"test_backup3": backup_info,
}
rp.server.config.minimum_redundancy = 1
# execute retention policy report
report = rp.report()
# check that our mock is valid for the retention policy because
# the total number of valid backups is lower than the retention policy
# redundancy.
assert report == {'test_backup': BackupInfo.OBSOLETE,
'test_backup2': BackupInfo.VALID,
'test_backup3': BackupInfo.VALID}
# Expect a ValueError if passed context is invalid
with pytest.raises(ValueError):
rp.report(context='invalid')
# Set a new minimum_redundancy parameter, enforcing the usage of the
# configuration parameter instead of the retention policy default
rp.server.config.minimum_redundancy = 3
# execute retention policy report
rp.report()
# Check for the warning inside the log
caplog.set_level(logging.WARNING)
log = caplog.text
assert log.find("WARNING Retention policy redundancy (2) "
"is lower than the required minimum redundancy (3). "
"Enforce 3.")
def test_archive_wal_timeline_lower_than_backup(self, tmpdir, capsys):
"""
Test archive-wal command behaviour when the WAL files are older than
the first backup of a server.
Expect it to trash WAL files
"""
# Build a real backup manager and a fake backup
backup_manager = build_backup_manager(name="TestServer", global_conf={"barman_home": tmpdir.strpath})
b_info = build_test_backup_info(
backup_id="fake_backup_id", server=backup_manager.server, begin_wal="000000020000000000000002", timeline=2
)
b_info.save()
# Build the basic folder structure and files
backup_manager.compression_manager.get_compressor.return_value = None
backup_manager.server.get_backup.return_value = b_info
basedir = tmpdir.join("main")
incoming_dir = basedir.join("incoming")
basedir.mkdir("errors")
archive_dir = basedir.join("wals")
xlog_db = archive_dir.join("xlog.db")
wal_name = "000000010000000000000001"
wal_timeline = int(wal_name[0:8], 16)
wal_file = incoming_dir.join(wal_name)
wal_file.ensure()
archive_dir.ensure(dir=True)
xlog_db.ensure()
backup_manager.server.xlogdb.return_value.__enter__.return_value = xlog_db.open(mode="a")
backup_manager.server.archivers = [FileWalArchiver(backup_manager)]
backup_manager.archive_wal()
with xlog_db.open() as f:
line = str(f.readline())
assert wal_name not in line
# Check that the WAL file is not present inside the wal catalog
wal_path = os.path.join(archive_dir.strpath, barman.xlog.hash_dir(wal_name), wal_name)
# Check that the wal file have not been archived
assert not os.path.exists(wal_path)
# Check the output for the removal of the wal file
out, err = capsys.readouterr()
assert (
"The timeline of the WAL file %s (%s), is lower "
"than the one of the oldest backup of "
"server %s (%s). Moving the WAL in "
"the error directory" % (wal_name, wal_timeline, backup_manager.config.name, b_info.timeline)
) in out
def test_base_archive_wal(self, tmpdir):
"""
Basic archiving test
Provide a WAL file and check for the correct location of the file at
the end of the process
"""
# Build a real backup manager
backup_manager = build_backup_manager(
name='TestServer',
global_conf={
'barman_home': tmpdir.strpath
})
b_info = build_test_backup_info(
backup_id='fake_backup_id',
server=backup_manager.server,
begin_wal='000000010000000000000001'
)
b_info.save()
backup_manager.server.get_backup.return_value = b_info
backup_manager.compression_manager.get_compressor.return_value = None
# Build the basic folder structure and files
basedir = tmpdir.join('main')
incoming_dir = basedir.join('incoming')
archive_dir = basedir.join('wals')
xlog_db = archive_dir.join('xlog.db')
wal_name = '000000010000000000000001'
wal_file = incoming_dir.join(wal_name)
wal_file.ensure()
archive_dir.ensure(dir=True)
xlog_db.ensure()
backup_manager.server.xlogdb.return_value.__enter__.return_value = \
xlog_db.open(mode='a')
backup_manager.server.archivers = [FileWalArchiver(backup_manager)]
backup_manager.archive_wal()
wal_path = os.path.join(archive_dir.strpath,
barman.xlog.hash_dir(wal_name),
wal_name)
# Check for the presence of the wal file in the wal catalog
with xlog_db.open() as f:
line = str(f.readline())
assert wal_name in line
# Check that the wal file have been moved from the incoming dir
assert not os.path.exists(wal_file.strpath)
# Check that the wal file have been archived to the expected location
assert os.path.exists(wal_path)
def test_generate_recovery_conf(self, rsync_pg_mock, tmpdir):
"""
Test the generation of recovery.conf file
"""
# Build basic folder/files structure
recovery_info = {
"configuration_files": ["postgresql.conf", "postgresql.auto.conf"],
"tempdir": tmpdir.strpath,
"results": {"changes": [], "warnings": []},
"get_wal": False,
}
backup_info = testing_helpers.build_test_backup_info()
dest = tmpdir.mkdir("destination")
# Build a recovery executor using a real server
server = testing_helpers.build_real_server()
executor = RecoveryExecutor(server.backup_manager)
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
)
# Check that the recovery.conf file exists
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
# Parse the generated recovery.conf
recovery_conf = {}
for line in recovery_conf_file.readlines():
key, value = (s.strip() for s in line.strip().split("=", 1))
recovery_conf[key] = value
# check for contents
assert "recovery_end_command" in recovery_conf
assert "recovery_target_time" in recovery_conf
assert "recovery_target_timeline" in recovery_conf
assert "recovery_target_xid" not in recovery_conf
assert "recovery_target_name" in recovery_conf
assert recovery_conf["recovery_end_command"] == "'rm -fr barman_xlog'"
assert recovery_conf["recovery_target_time"] == "'2015-06-03 16:11:03.71038+02'"
assert recovery_conf["recovery_target_timeline"] == "2"
assert recovery_conf["recovery_target_name"] == "'target_name'"
def test_set_pitr_targets(self, tmpdir):
"""
Evaluate targets for point in time recovery
"""
# Build basic folder/files structure
tempdir = tmpdir.mkdir('temp_dir')
dest = tmpdir.mkdir('dest')
wal_dest = tmpdir.mkdir('wal_dest')
recovery_info = {
'configuration_files': ['postgresql.conf', 'postgresql.auto.conf'],
'tempdir': tempdir.strpath,
'results': {'changes': [], 'warnings': []},
'is_pitr': False,
'wal_dest': wal_dest.strpath,
'get_wal': False,
}
backup_info = testing_helpers.build_test_backup_info()
server = testing_helpers.build_mocked_server()
backup_manager = Mock(server=server, config=server.config)
# Build a recovery executor
executor = RecoveryExecutor(backup_manager)
executor.set_pitr_targets(recovery_info, backup_info,
dest.strpath,
'', '', '', '')
# Test with empty values (no PITR)
assert recovery_info['target_epoch'] is None
assert recovery_info['target_datetime'] is None
assert recovery_info['wal_dest'] == wal_dest.strpath
# Test for PITR targets
executor.set_pitr_targets(recovery_info, backup_info,
dest.strpath,
'target_name',
'2015-06-03 16:11:03.71038+02',
'2',
None,)
target_datetime = dateutil.parser.parse(
'2015-06-03 16:11:03.710380+02:00')
target_epoch = (time.mktime(target_datetime.timetuple()) +
(target_datetime.microsecond / 1000000.))
assert recovery_info['target_datetime'] == target_datetime
assert recovery_info['target_epoch'] == target_epoch
assert recovery_info['wal_dest'] == dest.join('barman_xlog').strpath
def test_setup(self):
"""
Test the method that set up a recovery
"""
backup_info = testing_helpers.build_test_backup_info()
server = testing_helpers.build_mocked_server()
backup_manager = Mock(server=server, config=server.config)
executor = RecoveryExecutor(backup_manager)
backup_info.version = 90300
# no postgresql.auto.conf on version 9.3
ret = executor.setup(backup_info, None, "/tmp")
assert "postgresql.auto.conf" not in ret['configuration_files']
# Check the present for postgresql.auto.conf on version 9.4
backup_info.version = 90400
ret = executor.setup(backup_info, None, "/tmp")
assert "postgresql.auto.conf" in ret['configuration_files']
# Receive a error if the remote command is invalid
with pytest.raises(SystemExit):
executor.setup(backup_info, "invalid", "/tmp")
def test_concurrent_stop_backup(self, tbs_map_mock, label_mock,):
"""
Basic test for the stop_backup method for 9.6 concurrent api
:param label_mock: mimic the response of _write_backup_label
"""
# Build a backup info and configure the mocks
server = build_mocked_server(main_conf={
'backup_options':
BackupOptions.CONCURRENT_BACKUP
})
backup_manager = build_backup_manager(server=server)
stop_time = datetime.datetime.now()
# This is a pg 9.6
server.postgres.server_version = 90600
# Mock stop backup call for the new api method
start_time = datetime.datetime.now(tz.tzlocal()).replace(microsecond=0)
server.postgres.stop_concurrent_backup.return_value = {
'location': "A266/4A9C1EF8",
'timeline': 6,
'timestamp': stop_time,
'backup_label':
'START WAL LOCATION: A257/44B4C0D8 '
# Timeline 0 simulates a bug in PostgreSQL 9.6 beta2
'(file 000000000000A25700000044)\n'
'START TIME: %s\n' %
start_time.strftime('%Y-%m-%d %H:%M:%S %Z')
}
backup_info = build_test_backup_info()
backup_manager.executor.strategy.stop_backup(backup_info)
assert backup_info.end_xlog == 'A266/4A9C1EF8'
assert backup_info.end_wal == '000000060000A2660000004A'
assert backup_info.end_offset == 0x9C1EF8
assert backup_info.end_time == stop_time
assert backup_info.backup_label == (
'START WAL LOCATION: A257/44B4C0D8 '
'(file 000000000000A25700000044)\n'
'START TIME: %s\n' %
start_time.strftime('%Y-%m-%d %H:%M:%S %Z')
)
def test_set_pitr_targets(self, tmpdir):
"""
Evaluate targets for point in time recovery
"""
# Build basic folder/files structure
tempdir = tmpdir.mkdir("temp_dir")
dest = tmpdir.mkdir("dest")
wal_dest = tmpdir.mkdir("wal_dest")
recovery_info = {
"configuration_files": ["postgresql.conf", "postgresql.auto.conf"],
"tempdir": tempdir.strpath,
"results": {"changes": [], "warnings": []},
"is_pitr": False,
"wal_dest": wal_dest.strpath,
"get_wal": False,
}
backup_info = testing_helpers.build_test_backup_info(
end_time=dateutil.parser.parse("2015-06-03 16:11:01.71038+02")
)
backup_manager = testing_helpers.build_backup_manager()
# Build a recovery executor
executor = RecoveryExecutor(backup_manager)
executor._set_pitr_targets(
recovery_info, backup_info, dest.strpath, "", "", "", "", "", False, None
)
# Test with empty values (no PITR)
assert recovery_info["target_epoch"] is None
assert recovery_info["target_datetime"] is None
assert recovery_info["wal_dest"] == wal_dest.strpath
# Test for PITR targets
executor._set_pitr_targets(
recovery_info,
backup_info,
dest.strpath,
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
None,
"",
False,
None,
)
target_datetime = dateutil.parser.parse("2015-06-03 16:11:03.710380+02:00")
target_epoch = time.mktime(target_datetime.timetuple()) + (
target_datetime.microsecond / 1000000.0
)
assert recovery_info["target_datetime"] == target_datetime
assert recovery_info["target_epoch"] == target_epoch
assert recovery_info["wal_dest"] == dest.join("barman_wal").strpath
# Test for too early PITR target
with pytest.raises(RecoveryInvalidTargetException) as exc_info:
executor._set_pitr_targets(
recovery_info,
backup_info,
dest.strpath,
None,
"2015-06-03 16:11:00.71038+02",
None,
None,
None,
False,
None,
)
assert (
str(exc_info.value) == "The requested target time "
"2015-06-03 16:11:00.710380+02:00 "
"is before the backup end time "
"2015-06-03 16:11:01.710380+02:00"
)
# Tests for PostgreSQL < 9.1
backup_info.version = 90000
with pytest.raises(RecoveryTargetActionException) as exc_info:
executor._set_pitr_targets(
recovery_info,
backup_info,
dest.strpath,
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
None,
None,
False,
"pause",
)
assert (
str(exc_info.value) == "Illegal target action 'pause' "
"for this version of PostgreSQL"
)
# Tests for PostgreSQL between 9.1 and 9.4 included
backup_info.version = 90100
executor._set_pitr_targets(
recovery_info,
backup_info,
dest.strpath,
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
None,
None,
False,
None,
)
assert "pause_at_recovery_target" not in recovery_info
executor._set_pitr_targets(
recovery_info,
backup_info,
dest.strpath,
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
None,
None,
False,
"pause",
)
assert recovery_info["pause_at_recovery_target"] == "on"
del recovery_info["pause_at_recovery_target"]
with pytest.raises(RecoveryTargetActionException) as exc_info:
executor._set_pitr_targets(
recovery_info,
backup_info,
dest.strpath,
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
None,
None,
False,
"promote",
)
assert (
str(exc_info.value) == "Illegal target action 'promote' "
"for this version of PostgreSQL"
)
# Tests for PostgreSQL >= 9.5
backup_info.version = 90500
executor._set_pitr_targets(
recovery_info,
backup_info,
dest.strpath,
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
None,
None,
False,
"pause",
)
assert recovery_info["recovery_target_action"] == "pause"
executor._set_pitr_targets(
recovery_info,
backup_info,
dest.strpath,
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
None,
None,
False,
"promote",
)
assert recovery_info["recovery_target_action"] == "promote"
with pytest.raises(RecoveryTargetActionException) as exc_info:
executor._set_pitr_targets(
recovery_info,
backup_info,
dest.strpath,
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
None,
None,
False,
"unavailable",
)
assert (
str(exc_info.value) == "Illegal target action 'unavailable' "
"for this version of PostgreSQL"
)
# Recovery target action should not be available is PITR is not
# enabled
backup_info.version = 90500
with pytest.raises(RecoveryTargetActionException) as exc_info:
executor._set_pitr_targets(
recovery_info,
backup_info,
dest.strpath,
None,
None,
None,
None,
None,
False,
"pause",
)
assert (
str(exc_info.value) == "Can't enable recovery target action "
"when PITR is not required"
)
# Test that we are not using target_lsn with a version < 10
backup_info.version = 90500
with pytest.raises(RecoveryInvalidTargetException) as exc_info:
executor._set_pitr_targets(
recovery_info,
backup_info,
dest.strpath,
None,
None,
None,
None,
10000,
False,
"pause",
)
assert (
str(exc_info.value) == "Illegal use of recovery_target_lsn "
"'10000' for this version "
"of PostgreSQL "
"(version 10 minimum required)"
)
# Test that we are not using target_immediate with a version < 9.4
backup_info.version = 90300
with pytest.raises(RecoveryInvalidTargetException) as exc_info:
executor._set_pitr_targets(
recovery_info,
backup_info,
dest.strpath,
None,
None,
None,
None,
None,
True,
"pause",
)
assert (
str(exc_info.value) == "Illegal use of "
"recovery_target_immediate "
"for this version "
"of PostgreSQL "
"(version 9.4 minimum required)"
)
def test_generate_recovery_conf_pre12(self, rsync_pg_mock, tmpdir):
"""
Test the generation of recovery.conf file
"""
# Build basic folder/files structure
recovery_info = {
"configuration_files": ["postgresql.conf", "postgresql.auto.conf"],
"tempdir": tmpdir.strpath,
"results": {"changes": [], "warnings": []},
"get_wal": False,
}
backup_info = testing_helpers.build_test_backup_info()
dest = tmpdir.mkdir("destination")
# Build a recovery executor using a real server
server = testing_helpers.build_real_server()
executor = RecoveryExecutor(server.backup_manager)
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
"",
None,
)
# Check that the recovery.conf file exists
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
# Parse the generated recovery.conf
recovery_conf = testing_helpers.parse_recovery_conf(recovery_conf_file)
# check for contents
assert "recovery_end_command" in recovery_conf
assert "recovery_target_time" in recovery_conf
assert "recovery_target_timeline" in recovery_conf
assert "recovery_target_xid" not in recovery_conf
assert "recovery_target_lsn" not in recovery_conf
assert "recovery_target_name" in recovery_conf
assert "recovery_target" not in recovery_conf
assert recovery_conf["recovery_end_command"] == "'rm -fr barman_wal'"
assert recovery_conf["recovery_target_time"] == "'2015-06-03 16:11:03.71038+02'"
assert recovery_conf["recovery_target_timeline"] == "2"
assert recovery_conf["recovery_target_name"] == "'target_name'"
# Test 'pause_at_recovery_target' recovery_info entry
recovery_info["pause_at_recovery_target"] = "on"
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
"",
None,
)
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
recovery_conf = testing_helpers.parse_recovery_conf(recovery_conf_file)
assert recovery_conf["pause_at_recovery_target"] == "'on'"
# Test 'recovery_target_action'
del recovery_info["pause_at_recovery_target"]
recovery_info["recovery_target_action"] = "pause"
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
"",
None,
)
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
recovery_conf = testing_helpers.parse_recovery_conf(recovery_conf_file)
assert recovery_conf["recovery_target_action"] == "'pause'"
# Test 'standby_mode'
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
"",
True,
)
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
recovery_conf = testing_helpers.parse_recovery_conf(recovery_conf_file)
assert recovery_conf["standby_mode"] == "'on'"
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
"",
False,
)
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
recovery_conf = testing_helpers.parse_recovery_conf(recovery_conf_file)
assert "standby_mode" not in recovery_conf
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
"",
None,
)
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
recovery_conf = testing_helpers.parse_recovery_conf(recovery_conf_file)
assert "standby_mode" not in recovery_conf
def test_generate_recovery_conf(self, rsync_pg_mock, tmpdir):
"""
Test the generation of recovery configuration
:type tmpdir: py.path.local
"""
# Build basic folder/files structure
recovery_info = {
"configuration_files": ["postgresql.conf", "postgresql.auto.conf"],
"tempdir": tmpdir.strpath,
"results": {"changes": [], "warnings": []},
"get_wal": False,
}
backup_info = testing_helpers.build_test_backup_info(
version=120000,
)
dest = tmpdir.mkdir("destination")
# Build a recovery executor using a real server
server = testing_helpers.build_real_server()
executor = RecoveryExecutor(server.backup_manager)
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
"",
None,
)
# Check that the recovery.conf file doesn't exist
recovery_conf_file = tmpdir.join("recovery.conf")
assert not recovery_conf_file.check()
# Check that the recovery.signal file exists
signal_file = tmpdir.join("recovery.signal")
assert signal_file.check()
# Parse the generated recovery configuration
pg_auto_conf = self.parse_auto_conf_lines(recovery_info)
# check for contents
assert "recovery_end_command" in pg_auto_conf
assert "recovery_target_time" in pg_auto_conf
assert "recovery_target_timeline" in pg_auto_conf
assert "recovery_target_xid" not in pg_auto_conf
assert "recovery_target_lsn" not in pg_auto_conf
assert "recovery_target_name" in pg_auto_conf
assert "recovery_target" in pg_auto_conf
assert pg_auto_conf["recovery_end_command"] == "'rm -fr barman_wal'"
assert pg_auto_conf["recovery_target_time"] == "'2015-06-03 16:11:03.71038+02'"
assert pg_auto_conf["recovery_target_timeline"] == "2"
assert pg_auto_conf["recovery_target_name"] == "'target_name'"
# Test 'pause_at_recovery_target' recovery_info entry
signal_file.remove()
recovery_info["pause_at_recovery_target"] = "on"
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
"",
None,
)
# Check that the recovery.conf file doesn't exist
recovery_conf_file = tmpdir.join("recovery.conf")
assert not recovery_conf_file.check()
# Check that the recovery.signal file exists
signal_file = tmpdir.join("recovery.signal")
assert signal_file.check()
# Parse the generated recovery configuration
pg_auto_conf = self.parse_auto_conf_lines(recovery_info)
# Finally check pause_at_recovery_target value
assert pg_auto_conf["pause_at_recovery_target"] == "'on'"
# Test 'recovery_target_action'
signal_file.remove()
del recovery_info["pause_at_recovery_target"]
recovery_info["recovery_target_action"] = "pause"
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
"",
None,
)
# Check that the recovery.conf file doesn't exist
recovery_conf_file = tmpdir.join("recovery.conf")
assert not recovery_conf_file.check()
# Check that the recovery.signal file exists
signal_file = tmpdir.join("recovery.signal")
assert signal_file.check()
# Parse the generated recovery configuration
pg_auto_conf = self.parse_auto_conf_lines(recovery_info)
# Finally check recovery_target_action value
assert pg_auto_conf["recovery_target_action"] == "'pause'"
# Test 'standby_mode'
signal_file.remove()
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
"",
True,
)
# Check that the recovery.conf file doesn't exist
recovery_conf_file = tmpdir.join("recovery.conf")
assert not recovery_conf_file.check()
# Check that the recovery.signal file doesn't exist
wrong_signal_file = tmpdir.join("recovery.signal")
assert not wrong_signal_file.check()
# Check that the standby.signal file exists
signal_file = tmpdir.join("standby.signal")
assert signal_file.check()
# Parse the generated recovery configuration
pg_auto_conf = self.parse_auto_conf_lines(recovery_info)
# standby_mode is not a valid configuration in PostgreSQL 12
assert "standby_mode" not in pg_auto_conf
signal_file.remove()
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
"",
False,
)
# Check that the recovery.conf file doesn't exist
recovery_conf_file = tmpdir.join("recovery.conf")
assert not recovery_conf_file.check()
# Check that the standby.signal file doesn't exist
wrong_signal_file = tmpdir.join("standby.signal")
assert not wrong_signal_file.check()
# Check that the recovery.signal file exists
signal_file = tmpdir.join("recovery.signal")
assert signal_file.check()
# Parse the generated recovery configuration
pg_auto_conf = self.parse_auto_conf_lines(recovery_info)
# standby_mode is not a valid configuration in PostgreSQL 12
assert "standby_mode" not in pg_auto_conf
signal_file.remove()
executor._generate_recovery_conf(
recovery_info,
backup_info,
dest.strpath,
True,
True,
"remote@command",
"target_name",
"2015-06-03 16:11:03.71038+02",
"2",
"",
"",
None,
)
# Check that the recovery.conf file doesn't exist
recovery_conf_file = tmpdir.join("recovery.conf")
assert not recovery_conf_file.check()
# Check that the standby.signal file doesn't exist
wrong_signal_file = tmpdir.join("standby.signal")
assert not wrong_signal_file.check()
# Check that the recovery.signal file exists
signal_file = tmpdir.join("recovery.signal")
assert signal_file.check()
# Parse the generated recovery configuration
pg_auto_conf = self.parse_auto_conf_lines(recovery_info)
# standby_mode is not a valid configuration in PostgreSQL 12
assert "standby_mode" not in pg_auto_conf
def test_copy(self, rsync_ignore_mock, tmpdir):
"""
Unit test for RsyncCopyController._copy's code
"""
# Build the prerequisites
server = build_real_server(
global_conf={'barman_home': tmpdir.mkdir('home').strpath})
config = server.config
executor = server.backup_manager.executor
# Create the RsyncCopyController putting the safe_horizon between
# the tmp/safe and tmp2/check timestamps
rcc = RsyncCopyController(
path=server.path,
ssh_command=executor.ssh_command,
ssh_options=executor.ssh_options,
network_compression=config.network_compression,
reuse_backup=None,
safe_horizon=datetime(year=2015,
month=2,
day=20,
hour=19,
minute=0,
second=0,
tzinfo=dateutil.tz.tzlocal()))
backup_info = build_test_backup_info(
server=server,
pgdata="/pg/data",
config_file="/etc/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/data/pg_ident.conf",
begin_xlog="0/2000028",
begin_wal="000000010000000000000002",
begin_offset=28)
backup_info.save()
# This is to check that all the preparation is done correctly
assert os.path.exists(backup_info.filename)
# Create an rsync mock
rsync_mock = mock.Mock(name='Rsync()')
# Then run the _copy method
rcc._copy(rsync_mock,
':/pg/data/',
backup_info.get_data_directory(),
'/path/to/file.list',
checksum=True)
# Verify that _rsync_ignore_vanished_files has been called correctly
assert rsync_ignore_mock.mock_calls == [
mock.call(rsync_mock,
':/pg/data/',
backup_info.get_data_directory(),
'--files-from=/path/to/file.list',
'--checksum',
check=True),
]
# Try again without checksum
rsync_ignore_mock.reset_mock()
rcc._copy(rsync_mock,
':/pg/data/',
backup_info.get_data_directory(),
'/path/to/file.list',
checksum=False)
# Verify that _rsync_ignore_vanished_files has been called correctly
assert rsync_ignore_mock.mock_calls == [
mock.call(rsync_mock,
':/pg/data/',
backup_info.get_data_directory(),
'--files-from=/path/to/file.list',
check=True),
]
def test_check_sync_required(self):
"""
Test the behaviour of the check_sync_required method,
testing all the possible error conditions.
"""
backup_name = "test_backup_name"
backups = {"backups": {"test_backup_name": {}}}
server = build_real_server()
# Test 1 pass no exception
server.check_sync_required(backup_name, backups, None)
# Test 2 backup_name not in backups and no local backup. SyncError
backup_name = "wrong_test_backup_name"
with pytest.raises(SyncError):
server.check_sync_required(backup_name, backups, None)
# Test 3 backup_name not in backups, and incomplete local
# copy. Remove partial sync and raise SyncError
backup_name = "wrong_test_backup_name"
local_backup_info_mock = build_test_backup_info(
server=server, status=BackupInfo.FAILED)
with pytest.raises(SyncToBeDeleted):
server.check_sync_required(backup_name, backups,
local_backup_info_mock)
# Test 4 Local only copy, nothing to do.
backup_name = "wrong_test_backup_name"
local_backup_info_mock = build_test_backup_info(server=server,
status=BackupInfo.DONE)
with pytest.raises(SyncNothingToDo):
server.check_sync_required(backup_name, backups,
local_backup_info_mock)
# Test 5 already synced backup. Nothing to do.
backup_name = "test_backup_name"
local_backup_info_mock = build_test_backup_info(server=server,
status=BackupInfo.DONE)
with pytest.raises(SyncNothingToDo):
server.check_sync_required(backup_name, backups,
local_backup_info_mock)
# Test 6 check backup with local retention policies.
# Case one: Redundancy retention 1
# Expect "nothing to do"
backup_name = "test_backup6"
# build a new server with new configuration that uses retention
# policies
server = build_real_server(global_conf={
"retention_policy": "redundancy 1",
"wal_retention_policy": "main",
})
backups = {
"backups": {
"test_backup6":
build_test_backup_info(server=server,
backup_id="test_backup6").to_json()
},
"config": {
"name": "test_server"
},
}
with mock.patch("barman.server.Server.get_available_backups") as bk:
local_backup_info_mock = None
bk.return_value = {
"test_backup5":
build_test_backup_info(server=server,
backup_id="test_backup5"),
"test_backup7":
build_test_backup_info(server=server,
backup_id="test_backup7"),
}
with pytest.raises(SyncNothingToDo):
server.check_sync_required(backup_name, backups,
local_backup_info_mock)
# Test 7 check backup with local retention policies.
# Case two: Recovery window of 1 day
# Expect "nothing to do"
backup_name = "test_backup6"
# build a new server with new configuration that uses retention
# policies
server = build_real_server(
global_conf={
"retention_policy": "RECOVERY WINDOW OF 1 day",
"wal_retention_policy": "main",
})
backups = {
"backups": {
"test_backup6":
build_test_backup_info(
server=server,
backup_id="test_backup6",
begin_time=(datetime.now(tz.tzlocal()) +
timedelta(days=4)),
end_time=(datetime.now(tz.tzlocal()) - timedelta(days=3)),
).to_json()
},
"config": {
"name": "test_server"
},
}
with mock.patch("barman.server.Server.get_available_backups") as bk:
local_backup_info_mock = None
bk.return_value = {
"test_backup7":
build_test_backup_info(
server=server,
backup_id="test_backup7",
begin_time=(datetime.now(tz.tzlocal()) +
timedelta(days=4)),
end_time=(datetime.now(tz.tzlocal()) - timedelta(days=3)),
)
}
with pytest.raises(SyncNothingToDo):
server.check_sync_required(backup_name, backups,
local_backup_info_mock)
def test_backup_status(self):
"""
Basic unit test of method backup_status
Given a mock simulating a Backup with status DONE and
requesting the status through the backup_status method, the
RetentionPolicy class must mark it as valid
This method tests the validity of a backup using both
RedundancyRetentionPolicy and RecoveryWindowRetentionPolicy
"""
server = build_mocked_server()
rp = RetentionPolicyFactory.create(server, 'retention_policy',
'REDUNDANCY 2')
assert isinstance(rp, RedundancyRetentionPolicy)
# Build a BackupInfo object with status to DONE
backup_info = build_test_backup_info(server=rp.server,
backup_id='test1',
end_time=datetime.now(tzlocal()))
# instruct the get_available_backups method to return a map with
# our mock as result and minimum_redundancy = 1
rp.server.get_available_backups.return_value = {
"test_backup": backup_info
}
rp.server.config.minimum_redundancy = 1
# execute retention policy report
report = rp.backup_status('test_backup')
assert report == 'VALID'
# Force context of retention policy for testing purposes.
# Expect the method to return a BackupInfo.NONE value
rp.context = 'invalid'
empty_report = rp.backup_status('test_backup')
assert empty_report == BackupInfo.NONE
rp = RetentionPolicyFactory.create(server, 'retention_policy',
'RECOVERY WINDOW OF 4 WEEKS')
assert isinstance(rp, RecoveryWindowRetentionPolicy)
# Build a BackupInfo object with status to DONE
backup_info = build_test_backup_info(server=rp.server,
backup_id='test1',
end_time=datetime.now(tzlocal()))
# instruct the get_available_backups method to return a map with
# our mock as result and minimum_redundancy = 1
rp.server.get_available_backups.return_value = {
"test_backup": backup_info
}
rp.server.config.minimum_redundancy = 1
# execute retention policy report
report = rp.backup_status("test_backup")
assert report == 'VALID'
# Force context of retention policy for testing purposes.
# Expect the method to return a BackupInfo.NONE value
rp.context = 'invalid'
empty_report = rp.backup_status('test_backup')
assert empty_report == BackupInfo.NONE
def test_backup(self, gpb_mock, pbc_mock, capsys, tmpdir):
"""
Test backup
:param gpb_mock: mock for the get_previous_backup method
:param pbc_mock: mock for the backup_copy method
:param capsys: stdout capture module
:param tmpdir: pytest temp directory
"""
tmp_home = tmpdir.mkdir('home')
backup_manager = build_backup_manager(global_conf={
'barman_home': tmp_home.strpath,
'backup_method': 'postgres'
})
backup_info = build_test_backup_info(
backup_id='fake_backup_id',
server=backup_manager.server,
pgdata="/pg/data",
config_file="/pg/data/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/pg_ident.conf",
begin_offset=28)
timestamp = datetime.datetime(2015, 10, 26, 14, 38)
backup_manager.server.postgres.current_xlog_info = dict(
location='0/12000090',
file_name='000000010000000000000012',
file_offset=144,
timestamp=timestamp,
)
backup_manager.server.postgres.get_setting.return_value = '/pg/data'
tmp_backup_label = tmp_home.mkdir('main')\
.mkdir('base').mkdir('fake_backup_id')\
.mkdir('data').join('backup_label')
start_time = datetime.datetime.now(tz.tzlocal()).replace(microsecond=0)
tmp_backup_label.write(
'START WAL LOCATION: 0/40000028 (file 000000010000000000000040)\n'
'CHECKPOINT LOCATION: 0/40000028\n'
'BACKUP METHOD: streamed\n'
'BACKUP FROM: master\n'
'START TIME: %s\n'
'LABEL: pg_basebackup base backup' %
start_time.strftime('%Y-%m-%d %H:%M:%S %Z'))
backup_manager.executor.backup(backup_info)
out, err = capsys.readouterr()
gpb_mock.assert_called_once_with(backup_info.backup_id)
assert err.strip() == 'WARNING: pg_basebackup does not copy ' \
'the PostgreSQL configuration files that '\
'reside outside PGDATA. ' \
'Please manually backup the following files:' \
'\n\t/pg/pg_ident.conf'
assert 'Copying files.' in out
assert 'Copy done.' in out
assert 'Finalising the backup.' in out
assert backup_info.end_xlog == '0/12000090'
assert backup_info.end_offset == 144
assert backup_info.begin_time == start_time
assert backup_info.begin_wal == '000000010000000000000040'
# Check the CommandFailedException re raising
with pytest.raises(CommandFailedException):
pbc_mock.side_effect = CommandFailedException('test')
backup_manager.executor.backup(backup_info)
def test_statistics(
self,
signal_mock,
tempfile_mock,
copy_mock,
create_and_purge_mock,
analyse_mock,
rsync_mock,
tmpdir,
workers,
):
"""
Unit test for RsyncCopyController.statistics's code
"""
# Do a fake copy run to populate the start/stop timestamps.
# The steps are the same of the full run test
tempdir = tmpdir.mkdir("tmp")
tempfile_mock.return_value = tempdir.strpath
server = build_real_server(
global_conf={"barman_home": tmpdir.mkdir("home").strpath})
config = server.config
executor = server.backup_manager.executor
rcc = RsyncCopyController(
path=server.path,
ssh_command=executor.ssh_command,
ssh_options=executor.ssh_options,
network_compression=config.network_compression,
reuse_backup=None,
safe_horizon=None,
workers=workers,
)
backup_info = build_test_backup_info(
server=server,
pgdata="/pg/data",
config_file="/etc/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/data/pg_ident.conf",
begin_xlog="0/2000028",
begin_wal="000000010000000000000002",
begin_offset=28,
)
backup_info.save()
# This is to check that all the preparation is done correctly
assert os.path.exists(backup_info.filename)
# Silence the access to result properties
rsync_mock.return_value.out = ""
rsync_mock.return_value.err = ""
rsync_mock.return_value.ret = 0
# Mock analyze directory
def analyse_func(item):
label = item.label
item.dir_file = label + "_dir_file"
item.exclude_and_protect_file = label + "_exclude_and_protect_file"
item.safe_list = [_FileItem("mode", 1, "date", "path")]
item.check_list = [_FileItem("mode", 1, "date", "path")]
analyse_mock.side_effect = analyse_func
rcc.add_directory(
label="tbs1",
src=":/fake/location/",
dst=backup_info.get_data_directory(16387),
reuse=None,
bwlimit=None,
item_class=rcc.TABLESPACE_CLASS,
)
rcc.add_directory(
label="tbs2",
src=":/another/location/",
dst=backup_info.get_data_directory(16405),
reuse=None,
bwlimit=None,
item_class=rcc.TABLESPACE_CLASS,
)
rcc.add_directory(
label="pgdata",
src=":/pg/data/",
dst=backup_info.get_data_directory(),
reuse=None,
bwlimit=None,
item_class=rcc.PGDATA_CLASS,
exclude=[
"/pg_xlog/*",
"/pg_log/*",
"/log/*",
"/recovery.conf",
"/postmaster.pid",
],
exclude_and_protect=["pg_tblspc/16387", "pg_tblspc/16405"],
)
rcc.add_file(
label="pg_control",
src=":/pg/data/global/pg_control",
dst="%s/global/pg_control" % backup_info.get_data_directory(),
item_class=rcc.PGCONTROL_CLASS,
)
rcc.add_file(
label="config_file",
src=":/etc/postgresql.conf",
dst=backup_info.get_data_directory(),
item_class=rcc.CONFIG_CLASS,
optional=False,
)
# Do the fake run
rcc.copy()
# Calculate statistics
result = rcc.statistics()
# We cannot check the actual result because it is not predictable,
# so we check that every value is present and is a number and it is
# greater than 0
assert result.get("analysis_time") > 0
assert "analysis_time_per_item" in result
for tbs in ("pgdata", "tbs1", "tbs2"):
assert result["analysis_time_per_item"][tbs] > 0
assert result.get("copy_time") > 0
assert "copy_time_per_item" in result
assert "serialized_copy_time_per_item" in result
for tbs in ("pgdata", "tbs1", "tbs2", "config_file", "pg_control"):
assert result["copy_time_per_item"][tbs] > 0
assert result["serialized_copy_time_per_item"][tbs] > 0
assert result.get("number_of_workers") == rcc.workers
assert result.get("total_time") > 0
def test_backup_copy(self, remote_mock, pg_basebackup_mock,
tmpdir, capsys):
"""
Test backup folder structure
:param remote_mock: mock for the fetch_remote_status method
:param pg_basebackup_mock: mock for the PgBaseBackup object
:param tmpdir: pytest temp directory
"""
backup_manager = build_backup_manager(global_conf={
'barman_home': tmpdir.mkdir('home').strpath,
'backup_method': 'postgres'
})
# simulate a old version of pg_basebackup
# not supporting bandwidth_limit
remote_mock.return_value = {
'pg_basebackup_version': '9.2',
'pg_basebackup_path': '/fake/path',
'pg_basebackup_bwlimit': False,
}
server_mock = backup_manager.server
streaming_mock = server_mock.streaming
server_mock.config.bandwidth_limit = 1
streaming_mock.get_connection_string.return_value = 'fake=connstring'
streaming_mock.conn_parameters = {
'host': 'fakeHost',
'port': 'fakePort',
'user': '******'
}
backup_info = build_test_backup_info(server=backup_manager.server,
backup_id='fake_backup_id')
backup_manager.executor.backup_copy(backup_info)
out, err = capsys.readouterr()
assert out == ''
assert err == ''
# check that the bwlimit option have been ignored
assert pg_basebackup_mock.mock_calls == [
mock.call(
connection=mock.ANY,
version='9.2',
app_name='barman_streaming_backup',
destination=mock.ANY,
command='/fake/path',
tbs_mapping=mock.ANY,
bwlimit=None,
immediate=False,
retry_times=0,
retry_sleep=30,
retry_handler=mock.ANY,
path=mock.ANY),
mock.call()(),
]
# Check with newer version
remote_mock.reset_mock()
pg_basebackup_mock.reset_mock()
backup_manager.executor._remote_status = None
remote_mock.return_value = {
'pg_basebackup_version': '9.5',
'pg_basebackup_path': '/fake/path',
'pg_basebackup_bwlimit': True,
}
backup_manager.executor.config.immediate_checkpoint = True
backup_manager.executor.config.streaming_conninfo = 'fake=connstring'
backup_manager.executor.backup_copy(backup_info)
out, err = capsys.readouterr()
assert out == ''
assert err == ''
# check that the bwlimit option have been passed to the test call
assert pg_basebackup_mock.mock_calls == [
mock.call(
connection=mock.ANY,
version='9.5',
app_name='barman_streaming_backup',
destination=mock.ANY,
command='/fake/path',
tbs_mapping=mock.ANY,
bwlimit=1,
immediate=True,
retry_times=0,
retry_sleep=30,
retry_handler=mock.ANY,
path=mock.ANY),
mock.call()(),
]
# Check with a config file outside the data directory
remote_mock.reset_mock()
pg_basebackup_mock.reset_mock()
backup_info.ident_file = '/pg/pg_ident.conf'
backup_manager.executor.backup_copy(backup_info)
out, err = capsys.readouterr()
assert out == ''
assert err.strip() == 'WARNING: pg_basebackup does not copy ' \
'the PostgreSQL configuration files that '\
'reside outside PGDATA. ' \
'Please manually backup the following files:' \
'\n\t/pg/pg_ident.conf'
# check that the bwlimit option have been passed to the test call
assert pg_basebackup_mock.mock_calls == [
mock.call(
connection=mock.ANY,
version='9.5',
app_name='barman_streaming_backup',
destination=mock.ANY,
command='/fake/path',
tbs_mapping=mock.ANY,
bwlimit=1,
immediate=True,
retry_times=0,
retry_sleep=30,
retry_handler=mock.ANY,
path=mock.ANY),
mock.call()(),
]
# Check with a config file outside the data directory and
# external_configurations backup option
remote_mock.reset_mock()
pg_basebackup_mock.reset_mock()
backup_manager.config.backup_options.add(
BackupOptions.EXTERNAL_CONFIGURATION)
backup_manager.executor.backup_copy(backup_info)
out, err = capsys.readouterr()
assert out == ''
assert err == ''
# check that the bwlimit option have been passed to the test call
assert pg_basebackup_mock.mock_calls == [
mock.call(
connection=mock.ANY,
version='9.5',
app_name='barman_streaming_backup',
destination=mock.ANY,
command='/fake/path',
tbs_mapping=mock.ANY,
bwlimit=1,
immediate=True,
retry_times=0,
retry_sleep=30,
retry_handler=mock.ANY,
path=mock.ANY),
mock.call()(),
]
# Raise a test CommandFailedException and expect it to be wrapped
# inside a DataTransferFailure exception
remote_mock.reset_mock()
pg_basebackup_mock.reset_mock()
pg_basebackup_mock.return_value.side_effect = \
CommandFailedException(dict(ret='ret', out='out', err='err'))
with pytest.raises(DataTransferFailure):
backup_manager.executor.backup_copy(backup_info)
def test_full_copy(self, tempfile_mock, copy_mock, create_and_purge_mock,
analyse_mock, rsync_mock, tmpdir):
"""
Test the execution of a full copy
"""
# Build the prerequisites
tempdir = tmpdir.mkdir('tmp')
tempfile_mock.return_value = tempdir.strpath
server = build_real_server(global_conf={
'barman_home': tmpdir.mkdir('home').strpath
})
config = server.config
executor = server.backup_manager.executor
rcc = RsyncCopyController(
path=server.path,
ssh_command=executor.ssh_command,
ssh_options=executor.ssh_options,
network_compression=config.network_compression,
reuse_backup=None,
safe_horizon=None)
backup_info = build_test_backup_info(
server=server,
pgdata="/pg/data",
config_file="/etc/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/data/pg_ident.conf",
begin_xlog="0/2000028",
begin_wal="000000010000000000000002",
begin_offset=28)
backup_info.save()
# This is to check that all the preparation is done correctly
assert os.path.exists(backup_info.filename)
# Silence the access to result properties
rsync_mock.return_value.out = ''
rsync_mock.return_value.err = ''
rsync_mock.return_value.ret = 0
rcc.add_directory(
label='tbs1',
src=':/fake/location/',
dst=backup_info.get_data_directory(16387),
reuse=None,
bwlimit=None,
item_class=rcc.TABLESPACE_CLASS),
rcc.add_directory(
label='tbs2',
src=':/another/location/',
dst=backup_info.get_data_directory(16405),
reuse=None,
bwlimit=None,
item_class=rcc.TABLESPACE_CLASS),
rcc.add_directory(
label='pgdata',
src=':/pg/data/',
dst=backup_info.get_data_directory(),
reuse=None,
bwlimit=None,
item_class=rcc.PGDATA_CLASS,
exclude=['/pg_xlog/*',
'/pg_log/*',
'/recovery.conf',
'/postmaster.pid'],
exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405']),
rcc.add_file(
label='pg_control',
src=':/pg/data/global/pg_control',
dst='%s/global/pg_control' % backup_info.get_data_directory(),
item_class=rcc.PGCONTROL_CLASS),
rcc.add_file(
label='config_file',
src=':/etc/postgresql.conf',
dst=backup_info.get_data_directory(),
item_class=rcc.CONFIG_CLASS,
optional=False),
rcc.copy(),
# Check the order of calls to the Rsync mock
assert rsync_mock.mock_calls == [
mock.call(network_compression=False,
args=['--itemize-changes',
'--itemize-changes'],
bwlimit=None, ssh='ssh', path=None,
ssh_options=['-c', '"arcfour"', '-p', '22',
'*****@*****.**', '-o',
'BatchMode=yes', '-o',
'StrictHostKeyChecking=no'],
exclude=None, exclude_and_protect=None,
retry_sleep=0, retry_times=0, retry_handler=mock.ANY),
mock.call(network_compression=False,
args=['--itemize-changes',
'--itemize-changes'],
bwlimit=None, ssh='ssh', path=None,
ssh_options=['-c', '"arcfour"', '-p', '22',
'*****@*****.**', '-o',
'BatchMode=yes', '-o',
'StrictHostKeyChecking=no'],
exclude=None, exclude_and_protect=None,
retry_sleep=0, retry_times=0, retry_handler=mock.ANY),
mock.call(network_compression=False,
args=['--itemize-changes',
'--itemize-changes'],
bwlimit=None, ssh='ssh', path=None,
ssh_options=['-c', '"arcfour"', '-p', '22',
'*****@*****.**', '-o',
'BatchMode=yes', '-o',
'StrictHostKeyChecking=no'],
exclude=[
'/pg_xlog/*',
'/pg_log/*',
'/recovery.conf',
'/postmaster.pid'],
exclude_and_protect=[
'pg_tblspc/16387',
'pg_tblspc/16405'],
retry_sleep=0, retry_times=0, retry_handler=mock.ANY),
mock.call(network_compression=False,
args=['--itemize-changes',
'--itemize-changes'],
bwlimit=None, ssh='ssh', path=None,
ssh_options=['-c', '"arcfour"', '-p', '22',
'*****@*****.**', '-o',
'BatchMode=yes', '-o',
'StrictHostKeyChecking=no'],
exclude=None, exclude_and_protect=None,
retry_sleep=0, retry_times=0, retry_handler=mock.ANY),
mock.call()(
':/pg/data/global/pg_control',
'%s/global/pg_control' % backup_info.get_data_directory(),
allowed_retval=(0, 23, 24)),
mock.call(network_compression=False,
args=['--itemize-changes',
'--itemize-changes'],
bwlimit=None, ssh='ssh', path=None,
ssh_options=['-c', '"arcfour"', '-p', '22',
'*****@*****.**', '-o',
'BatchMode=yes', '-o',
'StrictHostKeyChecking=no'],
exclude=None, exclude_and_protect=None,
retry_sleep=0, retry_times=0, retry_handler=mock.ANY),
mock.call()(
':/etc/postgresql.conf',
backup_info.get_data_directory(),
allowed_retval=(0, 23, 24)),
]
# Check calls to _analyse_directory method
assert analyse_mock.mock_calls == [
mock.call(item, tempdir.strpath) for item in rcc.item_list
if item.is_directory
]
# Check calls to _create_dir_and_purge method
assert create_and_purge_mock.mock_calls == [
mock.call(item) for item in rcc.item_list
if item.is_directory
]
# Check the order of calls to the copy method
# All the file_list arguments are None because the analyze part
# has not really been executed
assert copy_mock.mock_calls == [
mock.call(
mock.ANY, ':/fake/location/',
backup_info.get_data_directory(16387), checksum=False,
file_list=None),
mock.call(
mock.ANY, ':/fake/location/',
backup_info.get_data_directory(16387), checksum=True,
file_list=None),
mock.call(
mock.ANY, ':/another/location/',
backup_info.get_data_directory(16405), checksum=False,
file_list=None),
mock.call(
mock.ANY, ':/another/location/',
backup_info.get_data_directory(16405), checksum=True,
file_list=None),
mock.call(mock.ANY, ':/pg/data/',
backup_info.get_data_directory(), checksum=False,
file_list=None),
mock.call(mock.ANY, ':/pg/data/',
backup_info.get_data_directory(), checksum=True,
file_list=None),
]
def test_check_redundancy(self, tmpdir):
"""
Test the check method
"""
# Setup temp dir and server
# build a backup_manager and setup a basic configuration
backup_manager = build_backup_manager(
name='TestServer',
global_conf={
'barman_home': tmpdir.strpath,
'minimum_redundancy': "1"
})
backup_manager.executor = mock.MagicMock()
# Test the unsatisfied minimum_redundancy option
strategy_mock = mock.MagicMock()
backup_manager.check(strategy_mock)
# Expect a failure from the method
strategy_mock.result.assert_called_with(
'TestServer',
'minimum redundancy requirements',
False,
'have 0 backups, expected at least 1'
)
# Test the satisfied minimum_redundancy option
b_info = build_test_backup_info(
backup_id='fake_backup_id',
server=backup_manager.server,
)
b_info.save()
strategy_mock.reset_mock()
backup_manager._load_backup_cache()
backup_manager.check(strategy_mock)
# Expect a success from the method
strategy_mock.result.assert_called_with(
'TestServer',
'minimum redundancy requirements',
True,
'have 1 backups, expected at least 1'
)
# Test for no failed backups
strategy_mock.reset_mock()
backup_manager._load_backup_cache()
backup_manager.check(strategy_mock)
# Expect a failure from the method
strategy_mock.result.assert_any_call(
'TestServer',
'failed backups',
True,
'there are 0 failed backups'
)
# Test for failed backups in catalog
b_info = build_test_backup_info(
backup_id='failed_backup_id',
server=backup_manager.server,
status=BackupInfo.FAILED,
)
b_info.save()
strategy_mock.reset_mock()
backup_manager._load_backup_cache()
backup_manager.check(strategy_mock)
# Expect a failure from the method
strategy_mock.result.assert_any_call(
'TestServer',
'failed backups',
False,
'there are 1 failed backups'
)
# Test unknown compression
backup_manager.config.compression = 'test_compression'
backup_manager.compression_manager.check.return_value = False
strategy_mock.reset_mock()
backup_manager.check(strategy_mock)
# Expect a failure from the method
strategy_mock.result.assert_any_call(
'TestServer',
'compression settings',
False
)
# Test valid compression
backup_manager.config.compression = 'test_compression'
backup_manager.compression_manager.check.return_value = True
strategy_mock.reset_mock()
backup_manager.check(strategy_mock)
# Expect a success from the method
strategy_mock.result.assert_any_call(
'TestServer',
'compression settings',
True
)
# Test failure retrieving a compressor
backup_manager.config.compression = 'test_compression'
backup_manager.compression_manager.check.return_value = True
backup_manager.compression_manager.get_compressor.side_effect = \
CompressionIncompatibility()
strategy_mock.reset_mock()
backup_manager.check(strategy_mock)
# Expect a failure from the method
strategy_mock.result.assert_any_call(
'TestServer',
'compression settings',
False
)
def test_recovery_window_report(self, server, caplog):
"""
Basic unit test of RecoveryWindowRetentionPolicy
Given a mock simulating a Backup with status DONE and
the end_date not over the point of recoverability,
the report method of the RecoveryWindowRetentionPolicy class must mark
it as valid
"""
rp = RetentionPolicyFactory.create(
"retention_policy", "RECOVERY WINDOW OF 4 WEEKS", server=server
)
assert isinstance(rp, RecoveryWindowRetentionPolicy)
# Build a BackupInfo object with status to DONE
backup_source = {
"test_backup3": build_test_backup_info(
server=server,
backup_id="test_backup3",
end_time=datetime.now(tzlocal()),
)
}
# Add a obsolete backup
backup_source["test_backup2"] = build_test_backup_info(
server=server,
backup_id="test_backup2",
end_time=datetime.now(tzlocal()) - timedelta(weeks=5),
)
# Add a second obsolete backup
backup_source["test_backup"] = build_test_backup_info(
server=server,
backup_id="test_backup",
end_time=datetime.now(tzlocal()) - timedelta(weeks=6),
)
server.get_available_backups.return_value = backup_source
# instruct the get_available_backups method to return a map with
# our mock as result and minimum_redundancy = 1
server.config.minimum_redundancy = 1
server.config.name = "test"
# execute retention policy report
report = rp.report()
# check that our mock is valid for the retention policy
assert report == {
"test_backup3": "VALID",
"test_backup2": "VALID",
"test_backup": "OBSOLETE",
}
# Expect a ValueError if passed context is invalid
with pytest.raises(ValueError):
rp.report(context="invalid")
# Set a new minimum_redundancy parameter, enforcing the usage of the
# configuration parameter instead of the retention policy default
server.config.minimum_redundancy = 4
# execute retention policy report
rp.report()
# Check for the warning inside the log
caplog.set_level(logging.WARNING)
log = caplog.text
warn = (
r"WARNING .*Keeping obsolete backup test_backup for "
r"server test \(older than .*\) due to minimum redundancy "
r"requirements \(4\)\n"
)
assert re.search(warn, log)
def test_create_dir_and_purge(self, rsync_ignore_mock, rsync_factory_mock,
tmpdir):
"""
Unit test for RsyncCopyController._create_dir_and_purge's code
"""
# Build the prerequisites
server = build_real_server(
global_conf={'barman_home': tmpdir.mkdir('home').strpath})
config = server.config
executor = server.backup_manager.executor
# Create the RsyncCopyController putting the safe_horizon between
# the tmp/safe and tmp2/check timestamps
rcc = RsyncCopyController(
path=server.path,
ssh_command=executor.ssh_command,
ssh_options=executor.ssh_options,
network_compression=config.network_compression,
reuse_backup=None,
safe_horizon=datetime(year=2015,
month=2,
day=20,
hour=19,
minute=0,
second=0,
tzinfo=dateutil.tz.tzlocal()))
backup_info = build_test_backup_info(
server=server,
pgdata="/pg/data",
config_file="/etc/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/data/pg_ident.conf",
begin_xlog="0/2000028",
begin_wal="000000010000000000000002",
begin_offset=28)
backup_info.save()
# This is to check that all the preparation is done correctly
assert os.path.exists(backup_info.filename)
# Create an item to inspect
item = _RsyncCopyItem(label='pgdata',
src=':/pg/data/',
dst=backup_info.get_data_directory(),
is_directory=True,
item_class=rcc.PGDATA_CLASS,
optional=False)
# Then run the _create_dir_and_purge method
rcc._create_dir_and_purge(item)
# Verify that _rsync_factory has been called correctly
assert rsync_factory_mock.mock_calls == [
mock.call(item),
]
# Verify that _rsync_ignore_vanished_files has been called correctly
assert rsync_ignore_mock.mock_calls == [
mock.call(rsync_factory_mock.return_value,
'--recursive',
'--delete',
'--files-from=None',
'--filter',
'merge None',
':/pg/data/',
backup_info.get_data_directory(),
check=True),
]
def test_sync_wals(self, rsync_mock, tmpdir, capsys):
"""
Test the WAL synchronisation method, testing all
the possible error conditions.
:param MagicMock rsync_mock: MagicMock replacing Rsync class
:param py.local.path tmpdir: py.test temporary directory
:param capsys: fixture that allow to access stdout/stderr output
"""
server_name = "main"
# Prepare paths
barman_home = tmpdir.mkdir("barman_home")
backup_dir = barman_home.mkdir(server_name)
wals_dir = backup_dir.mkdir("wals")
primary_info_file = backup_dir.join(barman.server.PRIMARY_INFO_FILE)
# prepare the primary_info file
remote_basebackup_dir = tmpdir.mkdir("primary")
primary_info_content = dict(EXPECTED_MINIMAL)
primary_info_content["config"].update(
compression=None,
basebackups_directory=str(remote_basebackup_dir),
wals_directory=str(wals_dir),
)
primary_info_file.write(json.dumps(primary_info_content))
# Test 1: Not a passive node.
# Expect SyncError
server = build_real_server(global_conf=dict(
barman_home=str(barman_home)))
with pytest.raises(SyncError):
server.sync_wals()
# Test 2: different compression between Master and Passive node.
# Expect a SyncError
server = build_real_server(
global_conf=dict(barman_home=str(barman_home)),
main_conf=dict(compression="gzip",
primary_ssh_command="ssh fakeuser@fakehost"),
)
server.sync_wals()
(out, err) = capsys.readouterr()
assert "Compression method on server %s " % server_name in err
# Test 3: No base backup for server, exit with warning
server = build_real_server(
global_conf=dict(barman_home=str(barman_home)),
main_conf=dict(
compression=None,
wals_directory=str(wals_dir),
primary_ssh_command="ssh fakeuser@fakehost",
),
)
server.sync_wals()
(out, err) = capsys.readouterr()
assert "WARNING: No base backup for server %s" % server.config.name in err
# Test 4: No wal synchronisation required, expect a warning
# set return for get_first_backup and get_backup methods
server.get_first_backup_id = lambda: "too_new"
server.get_backup = lambda x: build_test_backup_info(
server=server,
begin_wal="000000010000000000000005",
begin_time=dateutil.parser.parse("Wed Jul 23 11:00:43 2014"),
end_time=dateutil.parser.parse("Wed Jul 23 12:00:43 2014"),
)
server.sync_wals()
(out, err) = capsys.readouterr()
assert ("WARNING: Skipping WAL synchronisation for "
"server %s: no available local backup for %s" %
(server.config.name, primary_info_content["wals"][0]["name"])
in err)
# Test 6: simulate rsync failure.
# Expect a custom error message
server.get_backup = lambda x: build_test_backup_info(
server=server,
begin_wal="000000010000000000000002",
begin_time=dateutil.parser.parse("Wed Jul 23 11:00:43 2014"),
end_time=dateutil.parser.parse("Wed Jul 23 12:00:43 2014"),
)
rsync_mock.side_effect = CommandFailedException("TestFailure")
server.sync_wals()
(out, err) = capsys.readouterr()
# check stdout for the Custom error message
assert "TestFailure" in err
# Test 7: simulate keyboard interruption
rsync_mock.side_effect = KeyboardInterrupt()
server.sync_wals()
# control the error message for KeyboardInterrupt
(out, err) = capsys.readouterr()
assert "KeyboardInterrupt" in err
# Test 8: normal execution, expect no output. xlog.db
# must contain information about the primary info wals
# reset the rsync_moc, and remove the side_effect
rsync_mock.reset_mock()
rsync_mock.side_effect = mock.Mock(name="rsync")
server.sync_wals()
# check for no output on stdout and sterr
(out, err) = capsys.readouterr()
assert out == ""
assert err == ""
# check the xlog content for primary.info wals
exp_xlog = [
"000000010000000000000002\t16777216\t1406019026.0\tNone\n",
"000000010000000000000003\t16777216\t1406019026.0\tNone\n",
"000000010000000000000004\t16777216\t1406019329.93\tNone\n",
"000000010000000000000005\t16777216\t1406019330.84\tNone\n",
]
with server.xlogdb() as fxlogdb:
xlog = fxlogdb.readlines()
assert xlog == exp_xlog
def test_full_copy(
self,
signal_mock,
tempfile_mock,
copy_mock,
create_and_purge_mock,
analyse_mock,
rsync_mock,
tmpdir,
):
"""
Test the execution of a full copy
"""
# Build the prerequisites
tempdir = tmpdir.mkdir("tmp")
tempfile_mock.return_value = tempdir.strpath
server = build_real_server(
global_conf={"barman_home": tmpdir.mkdir("home").strpath})
config = server.config
executor = server.backup_manager.executor
rcc = RsyncCopyController(
path=server.path,
ssh_command=executor.ssh_command,
ssh_options=executor.ssh_options,
network_compression=config.network_compression,
reuse_backup=None,
safe_horizon=None,
)
backup_info = build_test_backup_info(
server=server,
pgdata="/pg/data",
config_file="/etc/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/data/pg_ident.conf",
begin_xlog="0/2000028",
begin_wal="000000010000000000000002",
begin_offset=28,
)
backup_info.save()
# This is to check that all the preparation is done correctly
assert os.path.exists(backup_info.filename)
# Silence the access to result properties
rsync_mock.return_value.out = ""
rsync_mock.return_value.err = ""
rsync_mock.return_value.ret = 0
# Mock analyze directory
def analyse_func(item):
label = item.label
item.dir_file = label + "_dir_file"
item.exclude_and_protect_file = label + "_exclude_and_protect_file"
item.safe_list = [_FileItem("mode", 1, "date", "path")]
item.check_list = [_FileItem("mode", 1, "date", "path")]
analyse_mock.side_effect = analyse_func
rcc.add_directory(
label="tbs1",
src=":/fake/location/",
dst=backup_info.get_data_directory(16387),
reuse=None,
bwlimit=None,
item_class=rcc.TABLESPACE_CLASS,
)
rcc.add_directory(
label="tbs2",
src=":/another/location/",
dst=backup_info.get_data_directory(16405),
reuse=None,
bwlimit=None,
item_class=rcc.TABLESPACE_CLASS,
)
rcc.add_directory(
label="pgdata",
src=":/pg/data/",
dst=backup_info.get_data_directory(),
reuse=None,
bwlimit=None,
item_class=rcc.PGDATA_CLASS,
exclude=[
"/pg_xlog/*",
"/pg_log/*",
"/log/*",
"/recovery.conf",
"/postmaster.pid",
],
exclude_and_protect=["pg_tblspc/16387", "pg_tblspc/16405"],
)
rcc.add_file(
label="pg_control",
src=":/pg/data/global/pg_control",
dst="%s/global/pg_control" % backup_info.get_data_directory(),
item_class=rcc.PGCONTROL_CLASS,
)
rcc.add_file(
label="config_file",
src=":/etc/postgresql.conf",
dst=backup_info.get_data_directory(),
item_class=rcc.CONFIG_CLASS,
optional=False,
)
rcc.copy()
# Check the order of calls to the Rsync mock
assert rsync_mock.mock_calls == [
mock.call(
network_compression=False,
args=[
"--ignore-missing-args",
"--itemize-changes",
"--itemize-changes",
],
bwlimit=None,
ssh="ssh",
path=None,
ssh_options=[
"-c",
'"arcfour"',
"-p",
"22",
"*****@*****.**",
"-o",
"BatchMode=yes",
"-o",
"StrictHostKeyChecking=no",
],
exclude=None,
exclude_and_protect=None,
include=None,
retry_sleep=0,
retry_times=0,
retry_handler=mock.ANY,
),
mock.call(
network_compression=False,
args=[
"--ignore-missing-args",
"--itemize-changes",
"--itemize-changes",
],
bwlimit=None,
ssh="ssh",
path=None,
ssh_options=[
"-c",
'"arcfour"',
"-p",
"22",
"*****@*****.**",
"-o",
"BatchMode=yes",
"-o",
"StrictHostKeyChecking=no",
],
exclude=None,
exclude_and_protect=None,
include=None,
retry_sleep=0,
retry_times=0,
retry_handler=mock.ANY,
),
mock.call(
network_compression=False,
args=[
"--ignore-missing-args",
"--itemize-changes",
"--itemize-changes",
],
bwlimit=None,
ssh="ssh",
path=None,
ssh_options=[
"-c",
'"arcfour"',
"-p",
"22",
"*****@*****.**",
"-o",
"BatchMode=yes",
"-o",
"StrictHostKeyChecking=no",
],
exclude=[
"/pg_xlog/*",
"/pg_log/*",
"/log/*",
"/recovery.conf",
"/postmaster.pid",
],
exclude_and_protect=["pg_tblspc/16387", "pg_tblspc/16405"],
include=None,
retry_sleep=0,
retry_times=0,
retry_handler=mock.ANY,
),
mock.call(
network_compression=False,
args=[
"--ignore-missing-args",
"--itemize-changes",
"--itemize-changes",
],
bwlimit=None,
ssh="ssh",
path=None,
ssh_options=[
"-c",
'"arcfour"',
"-p",
"22",
"*****@*****.**",
"-o",
"BatchMode=yes",
"-o",
"StrictHostKeyChecking=no",
],
exclude=None,
exclude_and_protect=None,
include=None,
retry_sleep=0,
retry_times=0,
retry_handler=mock.ANY,
),
mock.call()(
":/etc/postgresql.conf",
backup_info.get_data_directory(),
allowed_retval=(0, 23, 24),
),
mock.call(
network_compression=False,
args=[
"--ignore-missing-args",
"--itemize-changes",
"--itemize-changes",
],
bwlimit=None,
ssh="ssh",
path=None,
ssh_options=[
"-c",
'"arcfour"',
"-p",
"22",
"*****@*****.**",
"-o",
"BatchMode=yes",
"-o",
"StrictHostKeyChecking=no",
],
exclude=None,
exclude_and_protect=None,
include=None,
retry_sleep=0,
retry_times=0,
retry_handler=mock.ANY,
),
mock.call()(
":/pg/data/global/pg_control",
"%s/global/pg_control" % backup_info.get_data_directory(),
allowed_retval=(0, 23, 24),
),
]
# Check calls to _analyse_directory method
assert analyse_mock.mock_calls == [
mock.call(item) for item in rcc.item_list if item.is_directory
]
# Check calls to _create_dir_and_purge method
assert create_and_purge_mock.mock_calls == [
mock.call(item) for item in rcc.item_list if item.is_directory
]
# Utility function to build the file_list name
def file_list_name(label, kind):
return "%s/%s_%s_%s.list" % (tempdir.strpath, label, kind,
os.getpid())
# Check the order of calls to the copy method
# All the file_list arguments are None because the analyze part
# has not really been executed
assert copy_mock.mock_calls == [
mock.call(
mock.ANY,
":/fake/location/",
backup_info.get_data_directory(16387),
checksum=False,
file_list=file_list_name("tbs1", "safe"),
),
mock.call(
mock.ANY,
":/fake/location/",
backup_info.get_data_directory(16387),
checksum=True,
file_list=file_list_name("tbs1", "check"),
),
mock.call(
mock.ANY,
":/another/location/",
backup_info.get_data_directory(16405),
checksum=False,
file_list=file_list_name("tbs2", "safe"),
),
mock.call(
mock.ANY,
":/another/location/",
backup_info.get_data_directory(16405),
checksum=True,
file_list=file_list_name("tbs2", "check"),
),
mock.call(
mock.ANY,
":/pg/data/",
backup_info.get_data_directory(),
checksum=False,
file_list=file_list_name("pgdata", "safe"),
),
mock.call(
mock.ANY,
":/pg/data/",
backup_info.get_data_directory(),
checksum=True,
file_list=file_list_name("pgdata", "check"),
),
]
def test_recover_waiting_for_wals(
self,
backup_info_mock,
rsync_copy_controller_mock,
output_mock,
rsync_pgdata_mock,
unix_remote_command_mock,
tmpdir,
):
# This backup is waiting for WALs and it remains in that status
# even after having copied the data files
backup_info_mock.WAITING_FOR_WALS = "WAITING_FOR_WALS"
backup_info_mock.return_value.status = BackupInfo.WAITING_FOR_WALS
backup_info = testing_helpers.build_test_backup_info()
backup_manager = testing_helpers.build_backup_manager()
executor = RecoveryExecutor(backup_manager)
backup_info.status = BackupInfo.WAITING_FOR_WALS
destination = tmpdir.mkdir("destination").strpath
with closing(executor):
executor.recover(backup_info, destination, standby_mode=None)
# The backup info has been read again
backup_info_mock.assert_called()
# The following two warning messages have been emitted
output_mock.warning.assert_has_calls(
[
mock.call(
"IMPORTANT: You have requested a recovery operation for "
"a backup that does not have yet all the WAL files that "
"are required for consistency."
),
mock.call(
"IMPORTANT: The backup we have recovered IS NOT "
"VALID. Required WAL files for consistency are "
"missing. Please verify that WAL archiving is "
"working correctly or evaluate using the 'get-wal' "
"option for recovery"
),
]
)
# In the following test case, the backup will be validated during
# the copy of the data files, so there is no need for the warning
# message at the end of the recovery process to be emitted again
output_mock.warning.reset_mock()
backup_info_mock.return_value.status = BackupInfo.DONE
with closing(executor):
executor.recover(backup_info, destination, standby_mode=None)
# The backup info has been read again
backup_info_mock.assert_called()
# The following two warning messages have been emitted
output_mock.warning.assert_has_calls(
[
mock.call(
"IMPORTANT: You have requested a recovery operation for "
"a backup that does not have yet all the WAL files that "
"are required for consistency."
)
]
)
def test_status(self, capsys, tmpdir):
"""
Test the status method.
Given a test xlog.db expect the method to produce a json output.
Compare the produced json with the EXPECTED_MINIMAL map
:param path tmpdir: py.test temporary directory unique to the test
:param capsys: fixture that allow to access stdout/stderr output
"""
# Create a test xlog.db
tmp_path = tmpdir.join("xlog.db")
tmp_path.write(
"000000010000000000000001\t16777216\t1406019022.4\tNone\n"
"000000010000000000000002\t16777216\t1406019026.0\tNone\n"
"000000010000000000000003\t16777216\t1406019026.0\tNone\n"
"000000010000000000000004\t16777216\t1406019329.93\tNone\n"
"000000010000000000000005\t16777216\t1406019330.84\tNone\n"
)
# Build a server, replacing some function to use the tmpdir objects
server = build_real_server()
server.xlogdb = lambda: tmp_path.open()
server.get_available_backups = lambda: {
"1234567890": build_test_backup_info(
server=server,
begin_time=dateutil.parser.parse("Wed Jul 23 11:00:43 2014"),
end_time=dateutil.parser.parse("Wed Jul 23 12:00:43 2014"),
)
}
# Call the status method capturing the output using capsys
server.sync_status(None, None)
(out, err) = capsys.readouterr()
# prepare the expected results
# (complex values have to be converted to json)
expected = dict(EXPECTED_MINIMAL)
expected["config"] = dict(
[
(k, v.to_json() if hasattr(v, "to_json") else v)
for k, v in server.config.to_json().items()
]
)
assert json.loads(out) == expected
# Test that status method raises a SyncError
# if last_wal is older than the first entry of the xlog.db
with pytest.raises(SyncError):
server.sync_status("000000010000000000000000")
# Test that status method raises a SyncError
# if last_wal is newer than the last entry of the xlog.db
with pytest.raises(SyncError):
server.sync_status("000000010000000000000007")
# test with an empty file
tmp_path.write("")
server.sync_status("000000010000000000000001")
(out, err) = capsys.readouterr()
result = json.loads(out)
assert result["last_position"] == 0
assert result["last_name"] == ""
def _run_analyze_directory(self, list_files_mock, tmpdir, ref_list,
src_list):
# Apply it to _list_files calls
list_files_mock.side_effect = [ref_list, src_list]
# Build the prerequisites
server = build_real_server(
global_conf={"barman_home": tmpdir.mkdir("home").strpath})
config = server.config
executor = server.backup_manager.executor
# Create the RsyncCopyController putting the safe_horizon between
# the tmp/safe and tmp2/check timestamps
rcc = RsyncCopyController(
path=server.path,
ssh_command=executor.ssh_command,
ssh_options=executor.ssh_options,
network_compression=config.network_compression,
reuse_backup=None,
safe_horizon=datetime(
year=2015,
month=2,
day=20,
hour=19,
minute=0,
second=0,
tzinfo=dateutil.tz.tzlocal(),
),
)
backup_info = build_test_backup_info(
server=server,
pgdata="/pg/data",
config_file="/etc/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/data/pg_ident.conf",
begin_xlog="0/2000028",
begin_wal="000000010000000000000002",
begin_offset=28,
)
backup_info.save()
# This is to check that all the preparation is done correctly
assert os.path.exists(backup_info.filename)
# Add a temp dir (usually created by copy method
rcc.temp_dir = tmpdir.mkdir("tmp").strpath
# Create an item to inspect
item = _RsyncCopyItem(
label="pgdata",
src=":/pg/data/",
dst=backup_info.get_data_directory(),
is_directory=True,
item_class=rcc.PGDATA_CLASS,
optional=False,
)
# Then run the _analyze_directory method
rcc._analyze_directory(item)
return item, backup_info
def test_analyze_directory(self, list_files_mock, rsync_factory_mock,
tmpdir):
"""
Unit test for RsyncCopyController._analyze_directory's code
"""
# Build file list for ref
ref_list = [
_FileItem(
'drwxrwxrwt', 69632,
datetime(year=2015,
month=2,
day=9,
hour=15,
minute=1,
second=0,
tzinfo=dateutil.tz.tzlocal()), '.'),
_FileItem(
'drwxrwxrwt', 69612,
datetime(year=2015,
month=2,
day=19,
hour=15,
minute=1,
second=22,
tzinfo=dateutil.tz.tzlocal()), 'tmp'),
_FileItem(
'-rw-r--r--', 69632,
datetime(year=2015,
month=2,
day=20,
hour=18,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/safe'),
_FileItem(
'-rw-r--r--', 69612,
datetime(year=2015,
month=2,
day=20,
hour=19,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/check'),
_FileItem(
'-rw-r--r--', 69612,
datetime(year=2015,
month=2,
day=20,
hour=19,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_time'),
_FileItem(
'-rw-r--r--', 69612,
datetime(year=2015,
month=2,
day=20,
hour=19,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_size'),
]
# Build the list for source adding a new file, ...
src_list = ref_list + [
_FileItem(
'-rw-r--r--', 69612,
datetime(year=2015,
month=2,
day=20,
hour=22,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/new'),
]
# ... changing the timestamp one old file ...
src_list[4] = _FileItem(
'-rw-r--r--', 69612,
datetime(year=2015,
month=2,
day=20,
hour=20,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_time')
# ... and changing the size of another
src_list[5] = _FileItem(
'-rw-r--r--', 77777,
datetime(year=2015,
month=2,
day=20,
hour=19,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_size')
# Apply it to _list_files calls
list_files_mock.side_effect = [ref_list, src_list]
# Build the prerequisites
server = build_real_server(
global_conf={'barman_home': tmpdir.mkdir('home').strpath})
config = server.config
executor = server.backup_manager.executor
# Create the RsyncCopyController putting the safe_horizon between
# the tmp/safe and tmp2/check timestamps
rcc = RsyncCopyController(
path=server.path,
ssh_command=executor.ssh_command,
ssh_options=executor.ssh_options,
network_compression=config.network_compression,
reuse_backup=None,
safe_horizon=datetime(year=2015,
month=2,
day=20,
hour=19,
minute=0,
second=0,
tzinfo=dateutil.tz.tzlocal()))
backup_info = build_test_backup_info(
server=server,
pgdata="/pg/data",
config_file="/etc/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/data/pg_ident.conf",
begin_xlog="0/2000028",
begin_wal="000000010000000000000002",
begin_offset=28)
backup_info.save()
# This is to check that all the preparation is done correctly
assert os.path.exists(backup_info.filename)
# Add a temp dir (usually created by copy method
rcc.temp_dir = tmpdir.mkdir('tmp').strpath
# Create an item to inspect
item = _RsyncCopyItem(label='pgdata',
src=':/pg/data/',
dst=backup_info.get_data_directory(),
is_directory=True,
item_class=rcc.PGDATA_CLASS,
optional=False)
# Then run the _analyze_directory method
rcc._analyze_directory(item)
# Verify that _rsync_factory has been called correctly
assert rsync_factory_mock.mock_calls == [
mock.call(item),
]
# Verify that _list_files has been called correctly
assert list_files_mock.mock_calls == [
mock.call(rsync_factory_mock.return_value,
backup_info.get_data_directory() + '/'),
mock.call(rsync_factory_mock.return_value, ':/pg/data/')
]
# Check the result
# 1) The list of directories should be there and should contain all
# the directories
assert item.dir_file
assert open(item.dir_file).read() == ('.\n' 'tmp\n')
# The exclude_and_protect file should be populated correctly with all
# the files in the source
assert item.exclude_and_protect_file
assert open(
item.exclude_and_protect_file).read() == ('P tmp/safe\n'
'- tmp/safe\n'
'P tmp/check\n'
'- tmp/check\n'
'P tmp/diff_time\n'
'- tmp/diff_time\n'
'P tmp/diff_size\n'
'- tmp/diff_size\n'
'P tmp/new\n'
'- tmp/new\n')
# The check list must contain identical files after the safe_horizon
assert len(item.check_list) == 1
assert item.check_list[0].path == 'tmp/check'
# The safe list must contain every file that is not in check and is
# present in the source
assert len(item.safe_list) == 4
assert item.safe_list[0].path == 'tmp/safe'
assert item.safe_list[1].path == 'tmp/diff_time'
assert item.safe_list[2].path == 'tmp/diff_size'
assert item.safe_list[3].path == 'tmp/new'
def test_backup_copy(self, rsync_mock, tmpdir):
"""
Test the execution of a rsync copy
:param rsync_mock: mock for the RsyncCopyController object
:param tmpdir: temporary dir
"""
backup_manager = build_backup_manager(
global_conf={'barman_home': tmpdir.mkdir('home').strpath})
backup_manager.server.path = None
backup_manager.server.postgres.server_major_version = '9.6'
backup_info = build_test_backup_info(
server=backup_manager.server,
pgdata="/pg/data",
config_file="/etc/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/data/pg_ident.conf",
begin_xlog="0/2000028",
begin_wal="000000010000000000000002",
begin_offset=28)
backup_info.save()
# This is to check that all the preparation is done correctly
assert os.path.exists(backup_info.filename)
backup_manager.executor.backup_copy(backup_info)
assert rsync_mock.mock_calls == [
mock.call(reuse_backup=None,
safe_horizon=None,
network_compression=False,
ssh_command='ssh',
path=None,
ssh_options=[
'-c', '"arcfour"', '-p', '22',
'*****@*****.**', '-o', 'BatchMode=yes', '-o',
'StrictHostKeyChecking=no'
],
retry_sleep=30,
retry_times=0),
mock.call().add_directory(
label='tbs1',
src=':/fake/location/',
dst=backup_info.get_data_directory(16387),
reuse=None,
bwlimit=None,
item_class=rsync_mock.return_value.TABLESPACE_CLASS,
exclude=["/*"] + RsyncBackupExecutor.EXCLUDE_LIST,
include=["/PG_9.6_*"]),
mock.call().add_directory(
label='tbs2',
src=':/another/location/',
dst=backup_info.get_data_directory(16405),
reuse=None,
bwlimit=None,
item_class=rsync_mock.return_value.TABLESPACE_CLASS,
exclude=["/*"] + RsyncBackupExecutor.EXCLUDE_LIST,
include=["/PG_9.6_*"]),
mock.call().add_directory(
label='pgdata',
src=':/pg/data/',
dst=backup_info.get_data_directory(),
reuse=None,
bwlimit=None,
item_class=rsync_mock.return_value.PGDATA_CLASS,
exclude=RsyncBackupExecutor.PGDATA_EXCLUDE_LIST +
RsyncBackupExecutor.EXCLUDE_LIST,
exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405']),
mock.call().add_file(
label='pg_control',
src=':/pg/data/global/pg_control',
dst='%s/global/pg_control' % backup_info.get_data_directory(),
item_class=rsync_mock.return_value.PGCONTROL_CLASS),
mock.call().add_file(
label='config_file',
src=':/etc/postgresql.conf',
dst=backup_info.get_data_directory(),
item_class=rsync_mock.return_value.CONFIG_CLASS,
optional=False),
mock.call().copy(),
]
def test_set_pitr_targets(self, tmpdir):
"""
Evaluate targets for point in time recovery
"""
# Build basic folder/files structure
tempdir = tmpdir.mkdir('temp_dir')
dest = tmpdir.mkdir('dest')
wal_dest = tmpdir.mkdir('wal_dest')
recovery_info = {
'configuration_files': ['postgresql.conf', 'postgresql.auto.conf'],
'tempdir': tempdir.strpath,
'results': {'changes': [], 'warnings': []},
'is_pitr': False,
'wal_dest': wal_dest.strpath,
'get_wal': False,
}
backup_info = testing_helpers.build_test_backup_info(
end_time=dateutil.parser.parse('2015-06-03 16:11:01.71038+02'))
backup_manager = testing_helpers.build_backup_manager()
# Build a recovery executor
executor = RecoveryExecutor(backup_manager)
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
'', '', '', '', '', False, None)
# Test with empty values (no PITR)
assert recovery_info['target_epoch'] is None
assert recovery_info['target_datetime'] is None
assert recovery_info['wal_dest'] == wal_dest.strpath
# Test for PITR targets
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
'target_name',
'2015-06-03 16:11:03.71038+02',
'2',
None, '', False, None)
target_datetime = dateutil.parser.parse(
'2015-06-03 16:11:03.710380+02:00')
target_epoch = (
time.mktime(target_datetime.timetuple()) + (
target_datetime.microsecond / 1000000.))
assert recovery_info['target_datetime'] == target_datetime
assert recovery_info['target_epoch'] == target_epoch
assert recovery_info['wal_dest'] == dest.join('barman_wal').strpath
# Test for too early PITR target
with pytest.raises(RecoveryInvalidTargetException) as exc_info:
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
None,
'2015-06-03 16:11:00.71038+02',
None, None, None, False, None)
assert str(exc_info.value) == \
"The requested target time " \
"2015-06-03 16:11:00.710380+02:00 " \
"is before the backup end time " \
"2015-06-03 16:11:01.710380+02:00"
# Tests for PostgreSQL < 9.1
backup_info.version = 90000
with pytest.raises(RecoveryTargetActionException) as exc_info:
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
'target_name',
'2015-06-03 16:11:03.71038+02',
'2',
None, None, False, 'pause')
assert str(exc_info.value) == "Illegal target action 'pause' " \
"for this version of PostgreSQL"
# Tests for PostgreSQL between 9.1 and 9.4 included
backup_info.version = 90100
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
'target_name',
'2015-06-03 16:11:03.71038+02',
'2',
None, None, False, None)
assert 'pause_at_recovery_target' not in recovery_info
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
'target_name',
'2015-06-03 16:11:03.71038+02',
'2',
None, None, False, 'pause')
assert recovery_info['pause_at_recovery_target'] == "on"
del recovery_info['pause_at_recovery_target']
with pytest.raises(RecoveryTargetActionException) as exc_info:
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
'target_name',
'2015-06-03 16:11:03.71038+02',
'2',
None, None, False, 'promote')
assert str(exc_info.value) == "Illegal target action 'promote' " \
"for this version of PostgreSQL"
# Tests for PostgreSQL >= 9.5
backup_info.version = 90500
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
'target_name',
'2015-06-03 16:11:03.71038+02',
'2',
None, None, False, 'pause')
assert recovery_info['recovery_target_action'] == "pause"
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
'target_name',
'2015-06-03 16:11:03.71038+02',
'2',
None, None, False, 'promote')
assert recovery_info['recovery_target_action'] == "promote"
with pytest.raises(RecoveryTargetActionException) as exc_info:
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
'target_name',
'2015-06-03 16:11:03.71038+02',
'2',
None, None, False, 'unavailable')
assert str(exc_info.value) == "Illegal target action 'unavailable' " \
"for this version of PostgreSQL"
# Recovery target action should not be available is PITR is not
# enabled
backup_info.version = 90500
with pytest.raises(RecoveryTargetActionException) as exc_info:
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
None,
None,
None,
None,
None, False, 'pause')
assert str(exc_info.value) == "Can't enable recovery target action " \
"when PITR is not required"
# Test that we are not using target_lsn with a version < 10
backup_info.version = 90500
with pytest.raises(RecoveryInvalidTargetException) as exc_info:
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
None,
None,
None,
None,
10000, False, 'pause')
assert str(exc_info.value) == "Illegal use of recovery_target_lsn " \
"'10000' for this version " \
"of PostgreSQL " \
"(version 10 minimum required)"
# Test that we are not using target_immediate with a version < 9.4
backup_info.version = 90300
with pytest.raises(RecoveryInvalidTargetException) as exc_info:
executor._set_pitr_targets(recovery_info, backup_info,
dest.strpath,
None,
None,
None,
None,
None, True, 'pause')
assert str(exc_info.value) == "Illegal use of " \
"recovery_target_immediate " \
"for this version " \
"of PostgreSQL " \
"(version 9.4 minimum required)"
def test_backup_copy(self, remote_mock, pg_basebackup_mock, tmpdir):
"""
Test backup folder structure
:param remote_mock: mock for the fetch_remote_status method
:param pg_basebackup_mock: mock for the PgBaseBackup object
:param tmpdir: pytest temp directory
"""
backup_manager = build_backup_manager(
global_conf={
'barman_home': tmpdir.mkdir('home').strpath,
'backup_method': 'postgres'
})
# simulate a old version of pg_basebackup
# not supporting bandwidth_limit
remote_mock.return_value = {
'pg_basebackup_version': '9.2',
'pg_basebackup_path': '/fake/path',
'pg_basebackup_bwlimit': False,
}
server_mock = backup_manager.server
streaming_mock = server_mock.streaming
server_mock.config.bandwidth_limit = 1
streaming_mock.get_connection_string.return_value = 'fake=connstring'
streaming_mock.conn_parameters = {
'host': 'fakeHost',
'port': 'fakePort',
'user': '******'
}
backup_info = build_test_backup_info(server=backup_manager.server,
backup_id='fake_backup_id')
backup_manager.executor.backup_copy(backup_info)
# check that the bwlimit option have been ignored
assert pg_basebackup_mock.mock_calls == [
mock.call(connection=mock.ANY,
version='9.2',
app_name='barman_streaming_backup',
destination=mock.ANY,
command='/fake/path',
tbs_mapping=mock.ANY,
bwlimit=None,
immediate=False,
retry_times=0,
retry_sleep=30,
retry_handler=mock.ANY,
path=mock.ANY),
mock.call()(),
]
# Check with newer version
remote_mock.reset_mock()
pg_basebackup_mock.reset_mock()
backup_manager.executor._remote_status = None
remote_mock.return_value = {
'pg_basebackup_version': '9.5',
'pg_basebackup_path': '/fake/path',
'pg_basebackup_bwlimit': True,
}
backup_manager.executor.config.immediate_checkpoint = True
backup_manager.executor.config.streaming_conninfo = 'fake=connstring'
backup_manager.executor.backup_copy(backup_info)
# check that the bwlimit option have been passed to the test call
assert pg_basebackup_mock.mock_calls == [
mock.call(connection=mock.ANY,
version='9.5',
app_name='barman_streaming_backup',
destination=mock.ANY,
command='/fake/path',
tbs_mapping=mock.ANY,
bwlimit=1,
immediate=True,
retry_times=0,
retry_sleep=30,
retry_handler=mock.ANY,
path=mock.ANY),
mock.call()(),
]
# Raise a test CommandFailedException and expect it to be wrapped
# inside a DataTransferFailure exception
remote_mock.reset_mock()
pg_basebackup_mock.reset_mock()
pg_basebackup_mock.return_value.side_effect = \
CommandFailedException(dict(ret='ret', out='out', err='err'))
with pytest.raises(DataTransferFailure):
backup_manager.executor.backup_copy(backup_info)
def test_generate_recovery_conf_pre12(self, rsync_pg_mock, tmpdir):
"""
Test the generation of recovery.conf file
"""
# Build basic folder/files structure
recovery_info = {
'configuration_files': ['postgresql.conf', 'postgresql.auto.conf'],
'tempdir': tmpdir.strpath,
'results': {'changes': [], 'warnings': []},
'get_wal': False,
}
backup_info = testing_helpers.build_test_backup_info()
dest = tmpdir.mkdir('destination')
# Build a recovery executor using a real server
server = testing_helpers.build_real_server()
executor = RecoveryExecutor(server.backup_manager)
executor._generate_recovery_conf(recovery_info, backup_info,
dest.strpath,
True, True, 'remote@command',
'target_name',
'2015-06-03 16:11:03.71038+02', '2',
'', '', None)
# Check that the recovery.conf file exists
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
# Parse the generated recovery.conf
recovery_conf = testing_helpers.parse_recovery_conf(recovery_conf_file)
# check for contents
assert 'recovery_end_command' in recovery_conf
assert 'recovery_target_time' in recovery_conf
assert 'recovery_target_timeline' in recovery_conf
assert 'recovery_target_xid' not in recovery_conf
assert 'recovery_target_lsn' not in recovery_conf
assert 'recovery_target_name' in recovery_conf
assert 'recovery_target' not in recovery_conf
assert recovery_conf['recovery_end_command'] == "'rm -fr barman_wal'"
assert recovery_conf['recovery_target_time'] == \
"'2015-06-03 16:11:03.71038+02'"
assert recovery_conf['recovery_target_timeline'] == '2'
assert recovery_conf['recovery_target_name'] == "'target_name'"
# Test 'pause_at_recovery_target' recovery_info entry
recovery_info['pause_at_recovery_target'] = 'on'
executor._generate_recovery_conf(recovery_info, backup_info,
dest.strpath,
True, True, 'remote@command',
'target_name',
'2015-06-03 16:11:03.71038+02', '2',
'', '', None)
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
recovery_conf = testing_helpers.parse_recovery_conf(recovery_conf_file)
assert recovery_conf['pause_at_recovery_target'] == "'on'"
# Test 'recovery_target_action'
del recovery_info['pause_at_recovery_target']
recovery_info['recovery_target_action'] = 'pause'
executor._generate_recovery_conf(recovery_info, backup_info,
dest.strpath,
True, True, 'remote@command',
'target_name',
'2015-06-03 16:11:03.71038+02', '2',
'', '', None)
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
recovery_conf = testing_helpers.parse_recovery_conf(recovery_conf_file)
assert recovery_conf['recovery_target_action'] == "'pause'"
# Test 'standby_mode'
executor._generate_recovery_conf(recovery_info, backup_info,
dest.strpath,
True, True, 'remote@command',
'target_name',
'2015-06-03 16:11:03.71038+02', '2',
'', '', True)
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
recovery_conf = testing_helpers.parse_recovery_conf(recovery_conf_file)
assert recovery_conf['standby_mode'] == "'on'"
executor._generate_recovery_conf(recovery_info, backup_info,
dest.strpath,
True, True, 'remote@command',
'target_name',
'2015-06-03 16:11:03.71038+02', '2',
'', '', False)
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
recovery_conf = testing_helpers.parse_recovery_conf(recovery_conf_file)
assert 'standby_mode' not in recovery_conf
executor._generate_recovery_conf(recovery_info, backup_info,
dest.strpath,
True, True, 'remote@command',
'target_name',
'2015-06-03 16:11:03.71038+02', '2',
'', '', None)
recovery_conf_file = tmpdir.join("recovery.conf")
assert recovery_conf_file.check()
recovery_conf = testing_helpers.parse_recovery_conf(recovery_conf_file)
assert 'standby_mode' not in recovery_conf
def test_generate_recovery_conf(self, rsync_pg_mock, tmpdir):
"""
Test the generation of recovery configuration
:type tmpdir: py.path.local
"""
# Build basic folder/files structure
recovery_info = {
'configuration_files': ['postgresql.conf', 'postgresql.auto.conf'],
'tempdir': tmpdir.strpath,
'results': {'changes': [], 'warnings': []},
'get_wal': False,
}
backup_info = testing_helpers.build_test_backup_info(
version=120000,
)
dest = tmpdir.mkdir('destination')
# Build a recovery executor using a real server
server = testing_helpers.build_real_server()
executor = RecoveryExecutor(server.backup_manager)
executor._generate_recovery_conf(recovery_info, backup_info,
dest.strpath,
True, True, 'remote@command',
'target_name',
'2015-06-03 16:11:03.71038+02', '2',
'', '', None)
# Check that the recovery.conf file doesn't exist
recovery_conf_file = tmpdir.join("recovery.conf")
assert not recovery_conf_file.check()
# Check that the recovery.signal file exists
signal_file = tmpdir.join("recovery.signal")
assert signal_file.check()
# Parse the generated recovery configuration
pg_auto_conf = self.parse_auto_conf_lines(recovery_info)
# check for contents
assert 'recovery_end_command' in pg_auto_conf
assert 'recovery_target_time' in pg_auto_conf
assert 'recovery_target_timeline' in pg_auto_conf
assert 'recovery_target_xid' not in pg_auto_conf
assert 'recovery_target_lsn' not in pg_auto_conf
assert 'recovery_target_name' in pg_auto_conf
assert 'recovery_target' in pg_auto_conf
assert pg_auto_conf['recovery_end_command'] == "'rm -fr barman_wal'"
assert pg_auto_conf['recovery_target_time'] == \
"'2015-06-03 16:11:03.71038+02'"
assert pg_auto_conf['recovery_target_timeline'] == '2'
assert pg_auto_conf['recovery_target_name'] == "'target_name'"
# Test 'pause_at_recovery_target' recovery_info entry
signal_file.remove()
recovery_info['pause_at_recovery_target'] = 'on'
executor._generate_recovery_conf(recovery_info, backup_info,
dest.strpath,
True, True, 'remote@command',
'target_name',
'2015-06-03 16:11:03.71038+02', '2',
'', '', None)
# Check that the recovery.conf file doesn't exist
recovery_conf_file = tmpdir.join("recovery.conf")
assert not recovery_conf_file.check()
# Check that the recovery.signal file exists
signal_file = tmpdir.join("recovery.signal")
assert signal_file.check()
# Parse the generated recovery configuration
pg_auto_conf = self.parse_auto_conf_lines(recovery_info)
# Finally check pause_at_recovery_target value
assert pg_auto_conf['pause_at_recovery_target'] == "'on'"
# Test 'recovery_target_action'
signal_file.remove()
del recovery_info['pause_at_recovery_target']
recovery_info['recovery_target_action'] = 'pause'
executor._generate_recovery_conf(recovery_info, backup_info,
dest.strpath,
True, True, 'remote@command',
'target_name',
'2015-06-03 16:11:03.71038+02', '2',
'', '', None)
# Check that the recovery.conf file doesn't exist
recovery_conf_file = tmpdir.join("recovery.conf")
assert not recovery_conf_file.check()
# Check that the recovery.signal file exists
signal_file = tmpdir.join("recovery.signal")
assert signal_file.check()
# Parse the generated recovery configuration
pg_auto_conf = self.parse_auto_conf_lines(recovery_info)
# Finally check recovery_target_action value
assert pg_auto_conf['recovery_target_action'] == "'pause'"
# Test 'standby_mode'
signal_file.remove()
executor._generate_recovery_conf(recovery_info, backup_info,
dest.strpath,
True, True, 'remote@command',
'target_name',
'2015-06-03 16:11:03.71038+02', '2',
'', '', True)
# Check that the recovery.conf file doesn't exist
recovery_conf_file = tmpdir.join("recovery.conf")
assert not recovery_conf_file.check()
# Check that the recovery.signal file doesn't exist
wrong_signal_file = tmpdir.join("recovery.signal")
assert not wrong_signal_file.check()
# Check that the standby.signal file exists
signal_file = tmpdir.join("standby.signal")
assert signal_file.check()
# Parse the generated recovery configuration
pg_auto_conf = self.parse_auto_conf_lines(recovery_info)
# standby_mode is not a valid configuration in PostgreSQL 12
assert 'standby_mode' not in pg_auto_conf
signal_file.remove()
executor._generate_recovery_conf(recovery_info, backup_info,
dest.strpath,
True, True, 'remote@command',
'target_name',
'2015-06-03 16:11:03.71038+02', '2',
'', '', False)
# Check that the recovery.conf file doesn't exist
recovery_conf_file = tmpdir.join("recovery.conf")
assert not recovery_conf_file.check()
# Check that the standby.signal file doesn't exist
wrong_signal_file = tmpdir.join("standby.signal")
assert not wrong_signal_file.check()
# Check that the recovery.signal file exists
signal_file = tmpdir.join("recovery.signal")
assert signal_file.check()
# Parse the generated recovery configuration
pg_auto_conf = self.parse_auto_conf_lines(recovery_info)
# standby_mode is not a valid configuration in PostgreSQL 12
assert 'standby_mode' not in pg_auto_conf
signal_file.remove()
executor._generate_recovery_conf(recovery_info, backup_info,
dest.strpath,
True, True, 'remote@command',
'target_name',
'2015-06-03 16:11:03.71038+02', '2',
'', '', None)
# Check that the recovery.conf file doesn't exist
recovery_conf_file = tmpdir.join("recovery.conf")
assert not recovery_conf_file.check()
# Check that the standby.signal file doesn't exist
wrong_signal_file = tmpdir.join("standby.signal")
assert not wrong_signal_file.check()
# Check that the recovery.signal file exists
signal_file = tmpdir.join("recovery.signal")
assert signal_file.check()
# Parse the generated recovery configuration
pg_auto_conf = self.parse_auto_conf_lines(recovery_info)
# standby_mode is not a valid configuration in PostgreSQL 12
assert 'standby_mode' not in pg_auto_conf
def test_recovery(self, remote_cmd_mock, rsync_pg_mock,
copy_controller_mock, tmpdir):
"""
Test the execution of a recovery
"""
# Prepare basic directory/files structure
dest = tmpdir.mkdir('destination')
base = tmpdir.mkdir('base')
wals = tmpdir.mkdir('wals')
backup_info = testing_helpers.build_test_backup_info(tablespaces=[])
backup_info.config.basebackups_directory = base.strpath
backup_info.config.wals_directory = wals.strpath
backup_info.version = 90400
datadir = base.mkdir(backup_info.backup_id).mkdir('data')
backup_info.pgdata = datadir.strpath
postgresql_conf_local = datadir.join('postgresql.conf')
postgresql_auto_local = datadir.join('postgresql.auto.conf')
postgresql_conf_local.write('archive_command = something\n'
'data_directory = something')
postgresql_auto_local.write('archive_command = something\n'
'data_directory = something')
shutil.copy2(postgresql_conf_local.strpath, dest.strpath)
shutil.copy2(postgresql_auto_local.strpath, dest.strpath)
# Avoid triggering warning for missing config files
datadir.ensure('pg_hba.conf')
datadir.ensure('pg_ident.conf')
# Build an executor
server = testing_helpers.build_real_server(
global_conf={
"barman_lock_directory": tmpdir.mkdir('lock').strpath
},
main_conf={
"wals_directory": wals.strpath
})
executor = RecoveryExecutor(server.backup_manager)
# test local recovery
with closing(executor):
rec_info = executor.recover(backup_info, dest.strpath,
exclusive=True)
# remove not useful keys from the result
del rec_info['cmd']
sys_tempdir = rec_info['tempdir']
assert rec_info == {
'rsync': None,
'tempdir': sys_tempdir,
'wal_dest': dest.join('pg_xlog').strpath,
'recovery_dest': 'local',
'destination_path': dest.strpath,
'temporary_configuration_files': [
dest.join('postgresql.conf').strpath,
dest.join('postgresql.auto.conf').strpath],
'results': {
'delete_barman_wal': False,
'recovery_start_time': rec_info['results'][
'recovery_start_time'
],
'get_wal': False,
'changes': [
Assertion._make([
'postgresql.conf',
0,
'archive_command',
'false']),
Assertion._make([
'postgresql.auto.conf',
0,
'archive_command',
'false'])],
'missing_files': [],
'recovery_configuration_file': 'recovery.conf',
'warnings': [
Assertion._make([
'postgresql.conf',
2,
'data_directory',
'something']),
Assertion._make([
'postgresql.auto.conf',
2,
'data_directory',
'something'])]},
'target_epoch': None,
'configuration_files': [
'postgresql.conf',
'postgresql.auto.conf'],
'target_datetime': None,
'safe_horizon': None,
'is_pitr': False,
'get_wal': False,
}
# test remote recovery
with closing(executor):
rec_info = executor.recover(backup_info, dest.strpath,
remote_command="remote@command",
exclusive=True)
# remove not useful keys from the result
del rec_info['cmd']
del rec_info['rsync']
sys_tempdir = rec_info['tempdir']
assert rec_info == {
'tempdir': sys_tempdir,
'wal_dest': dest.join('pg_xlog').strpath,
'recovery_dest': 'remote',
'destination_path': dest.strpath,
'temporary_configuration_files': [
os.path.join(sys_tempdir, 'postgresql.conf'),
os.path.join(sys_tempdir, 'postgresql.auto.conf')],
'results': {
'delete_barman_wal': False,
'get_wal': False,
'recovery_start_time': rec_info['results'][
'recovery_start_time'
],
'changes': [
Assertion._make([
'postgresql.conf',
0,
'archive_command',
'false']),
Assertion._make([
'postgresql.auto.conf',
0,
'archive_command',
'false'])],
'missing_files': [],
'recovery_configuration_file': 'recovery.conf',
'warnings': [
Assertion._make([
'postgresql.conf',
2,
'data_directory',
'something']),
Assertion._make([
'postgresql.auto.conf',
2,
'data_directory',
'something'])]},
'target_epoch': None,
'configuration_files': [
'postgresql.conf',
'postgresql.auto.conf'],
'target_datetime': None,
'safe_horizon': None,
'is_pitr': False,
'get_wal': False,
}
# test failed rsync
rsync_pg_mock.side_effect = CommandFailedException()
with pytest.raises(CommandFailedException):
with closing(executor):
executor.recover(backup_info, dest.strpath, exclusive=True,
remote_command="remote@command")
def test_check_redundancy(self, tmpdir):
"""
Test the check method
"""
# Setup temp dir and server
# build a backup_manager and setup a basic configuration
backup_manager = build_backup_manager(
name="TestServer",
global_conf={"barman_home": tmpdir.strpath, "minimum_redundancy": "1"},
)
backup_manager.executor = mock.MagicMock()
# Test the unsatisfied minimum_redundancy option
strategy_mock = mock.MagicMock()
backup_manager.check(strategy_mock)
# Expect a failure from the method
strategy_mock.result.assert_called_with(
"TestServer", False, hint="have 0 backups, expected at least 1"
)
# Test the satisfied minimum_redundancy option
b_info = build_test_backup_info(
backup_id="fake_backup_id",
server=backup_manager.server,
)
b_info.save()
strategy_mock.reset_mock()
backup_manager._load_backup_cache()
backup_manager.check(strategy_mock)
# Expect a success from the method
strategy_mock.result.assert_called_with(
"TestServer", True, hint="have 1 backups, expected at least 1"
)
# Test for no failed backups
strategy_mock.reset_mock()
backup_manager._load_backup_cache()
backup_manager.check(strategy_mock)
# Expect a failure from the method
strategy_mock.result.assert_any_call(
"TestServer", True, hint="there are 0 failed backups"
)
# Test for failed backups in catalog
b_info = build_test_backup_info(
backup_id="failed_backup_id",
server=backup_manager.server,
status=BackupInfo.FAILED,
)
b_info.save()
strategy_mock.reset_mock()
backup_manager._load_backup_cache()
backup_manager.check(strategy_mock)
# Expect a failure from the method
strategy_mock.result.assert_any_call(
"TestServer", False, hint="there are 1 failed backups"
)
# Test unknown compression
backup_manager.config.compression = "test_compression"
backup_manager.compression_manager.check.return_value = False
strategy_mock.reset_mock()
backup_manager.check(strategy_mock)
# Expect a failure from the method
strategy_mock.result.assert_any_call("TestServer", False)
# Test valid compression
backup_manager.config.compression = "test_compression"
backup_manager.compression_manager.check.return_value = True
strategy_mock.reset_mock()
backup_manager.check(strategy_mock)
# Expect a success from the method
strategy_mock.result.assert_any_call("TestServer", True)
# Test failure retrieving a compressor
backup_manager.config.compression = "test_compression"
backup_manager.compression_manager.check.return_value = True
backup_manager.compression_manager.get_default_compressor.side_effect = (
CompressionIncompatibility()
)
strategy_mock.reset_mock()
backup_manager.check(strategy_mock)
# Expect a failure from the method
strategy_mock.result.assert_any_call("TestServer", False)
def test_recovery(
self, remote_cmd_mock, rsync_pg_mock, copy_controller_mock, tmpdir
):
"""
Test the execution of a recovery
"""
# Prepare basic directory/files structure
dest = tmpdir.mkdir("destination")
base = tmpdir.mkdir("base")
wals = tmpdir.mkdir("wals")
backup_info = testing_helpers.build_test_backup_info(tablespaces=[])
backup_info.config.basebackups_directory = base.strpath
backup_info.config.wals_directory = wals.strpath
backup_info.version = 90400
datadir = base.mkdir(backup_info.backup_id).mkdir("data")
backup_info.pgdata = datadir.strpath
postgresql_conf_local = datadir.join("postgresql.conf")
postgresql_auto_local = datadir.join("postgresql.auto.conf")
postgresql_conf_local.write(
"archive_command = something\n" "data_directory = something"
)
postgresql_auto_local.write(
"archive_command = something\n" "data_directory = something"
)
shutil.copy2(postgresql_conf_local.strpath, dest.strpath)
shutil.copy2(postgresql_auto_local.strpath, dest.strpath)
# Avoid triggering warning for missing config files
datadir.ensure("pg_hba.conf")
datadir.ensure("pg_ident.conf")
# Build an executor
server = testing_helpers.build_real_server(
global_conf={"barman_lock_directory": tmpdir.mkdir("lock").strpath},
main_conf={"wals_directory": wals.strpath},
)
executor = RecoveryExecutor(server.backup_manager)
# test local recovery
with closing(executor):
rec_info = executor.recover(backup_info, dest.strpath, exclusive=True)
# remove not useful keys from the result
del rec_info["cmd"]
sys_tempdir = rec_info["tempdir"]
assert rec_info == {
"rsync": None,
"tempdir": sys_tempdir,
"wal_dest": dest.join("pg_xlog").strpath,
"recovery_dest": "local",
"destination_path": dest.strpath,
"temporary_configuration_files": [
dest.join("postgresql.conf").strpath,
dest.join("postgresql.auto.conf").strpath,
],
"results": {
"delete_barman_wal": False,
"recovery_start_time": rec_info["results"]["recovery_start_time"],
"get_wal": False,
"changes": [
Assertion._make(["postgresql.conf", 0, "archive_command", "false"]),
Assertion._make(
["postgresql.auto.conf", 0, "archive_command", "false"]
),
],
"missing_files": [],
"recovery_configuration_file": "recovery.conf",
"warnings": [
Assertion._make(
["postgresql.conf", 2, "data_directory", "something"]
),
Assertion._make(
["postgresql.auto.conf", 2, "data_directory", "something"]
),
],
},
"target_epoch": None,
"configuration_files": ["postgresql.conf", "postgresql.auto.conf"],
"target_datetime": None,
"safe_horizon": None,
"is_pitr": False,
"get_wal": False,
}
# test remote recovery
with closing(executor):
rec_info = executor.recover(
backup_info,
dest.strpath,
remote_command="remote@command",
exclusive=True,
)
# remove not useful keys from the result
del rec_info["cmd"]
del rec_info["rsync"]
sys_tempdir = rec_info["tempdir"]
assert rec_info == {
"tempdir": sys_tempdir,
"wal_dest": dest.join("pg_xlog").strpath,
"recovery_dest": "remote",
"destination_path": dest.strpath,
"temporary_configuration_files": [
os.path.join(sys_tempdir, "postgresql.conf"),
os.path.join(sys_tempdir, "postgresql.auto.conf"),
],
"results": {
"delete_barman_wal": False,
"get_wal": False,
"recovery_start_time": rec_info["results"]["recovery_start_time"],
"changes": [
Assertion._make(["postgresql.conf", 0, "archive_command", "false"]),
Assertion._make(
["postgresql.auto.conf", 0, "archive_command", "false"]
),
],
"missing_files": [],
"recovery_configuration_file": "recovery.conf",
"warnings": [
Assertion._make(
["postgresql.conf", 2, "data_directory", "something"]
),
Assertion._make(
["postgresql.auto.conf", 2, "data_directory", "something"]
),
],
},
"target_epoch": None,
"configuration_files": ["postgresql.conf", "postgresql.auto.conf"],
"target_datetime": None,
"safe_horizon": None,
"is_pitr": False,
"get_wal": False,
}
# test failed rsync
rsync_pg_mock.side_effect = CommandFailedException()
with pytest.raises(CommandFailedException):
with closing(executor):
executor.recover(
backup_info,
dest.strpath,
exclusive=True,
remote_command="remote@command",
)
def test_delete_backup(self, mock_available_backups, tmpdir, caplog):
"""
Simple test for the deletion of a backup.
We want to test the behaviour of the delete_backup method
"""
# Setup of the test backup_manager
backup_manager = build_backup_manager()
backup_manager.server.config.name = 'TestServer'
backup_manager.server.config.barman_lock_directory = tmpdir.strpath
backup_manager.server.config.backup_options = []
# Create a fake backup directory inside tmpdir (old format)
base_dir = tmpdir.mkdir('base')
backup_dir = base_dir.mkdir('fake_backup_id')
pg_data = backup_dir.mkdir('pgdata')
pg_data_v2 = backup_dir.mkdir('data')
wal_dir = tmpdir.mkdir('wals')
wal_history_file02 = wal_dir.join('00000002.history')
wal_history_file03 = wal_dir.join('00000003.history')
wal_history_file04 = wal_dir.join('00000004.history')
wal_history_file02.write('1\t0/2000028\tat restore point "myrp"\n')
wal_history_file03.write('1\t0/2000028\tat restore point "myrp"\n')
wal_history_file04.write('1\t0/2000028\tat restore point "myrp"\n')
wal_history_file04.write('2\t0/3000028\tunknown\n')
wal_file = wal_dir.join('0000000100000000/000000010000000000000001')
wal_file.ensure()
xlog_db = wal_dir.join('xlog.db')
xlog_db.write(
'000000010000000000000001\t42\t43\tNone\n'
'00000002.history\t42\t43\tNone\n'
'00000003.history\t42\t43\tNone\n'
'00000004.history\t42\t43\tNone\n')
backup_manager.server.xlogdb.return_value.__enter__.return_value = (
xlog_db.open())
backup_manager.server.config.basebackups_directory = base_dir.strpath
backup_manager.server.config.wals_directory = wal_dir.strpath
# The following tablespaces are defined in the default backup info
# generated by build_test_backup_info
b_info = build_test_backup_info(
backup_id='fake_backup_id',
server=backup_manager.server,
)
# Make sure we are not trying to delete any WAL file,
# just by having a previous backup
b_pre_info = build_test_backup_info(
backup_id='fake_backup',
server=backup_manager.server,
)
mock_available_backups.return_value = {
"fake_backup": b_pre_info,
"fake_backup_id": b_info,
}
# Test 1: minimum redundancy not satisfied
caplog_reset(caplog)
backup_manager.server.config.minimum_redundancy = 2
b_info.set_attribute('backup_version', 1)
build_backup_directories(b_info)
backup_manager.delete_backup(b_info)
assert 'WARNING Skipping delete of backup ' in caplog.text
assert 'ERROR' not in caplog.text
assert os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 2: normal delete expecting no errors (old format)
caplog_reset(caplog)
backup_manager.server.config.minimum_redundancy = 1
b_info.set_attribute('backup_version', 1)
build_backup_directories(b_info)
backup_manager.delete_backup(b_info)
# the backup must not exists on disk anymore
assert 'WARNING' not in caplog.text
assert 'ERROR' not in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 3: delete the backup again, expect a failure in log
caplog_reset(caplog)
backup_manager.delete_backup(b_info)
assert 'ERROR Failure deleting backup fake_backup_id' in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 4: normal delete expecting no errors (new format)
caplog_reset(caplog)
b_info.set_attribute('backup_version', 2)
build_backup_directories(b_info)
backup_manager.delete_backup(b_info)
assert 'WARNING' not in caplog.text
assert 'ERROR' not in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 5: normal delete of first backup no errors and no skip
# removing one of the two backups present (new format)
# and all the previous wal
caplog_reset(caplog)
b_pre_info.set_attribute('backup_version', 2)
build_backup_directories(b_pre_info)
backup_manager.delete_backup(b_pre_info)
assert 'WARNING' not in caplog.text
assert 'ERROR' not in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert not os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 6: normal delete of first backup no errors and no skip
# removing one of the two backups present (new format)
# the previous wal is retained as on a different timeline
caplog_reset(caplog)
wal_file.ensure()
b_pre_info.set_attribute('timeline', 2)
b_pre_info.set_attribute('backup_version', 2)
build_backup_directories(b_pre_info)
backup_manager.delete_backup(b_pre_info)
assert 'WARNING' not in caplog.text
assert 'ERROR' not in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 7: simulate an error deleting the the backup.
with patch('barman.backup.BackupManager.delete_backup_data')\
as mock_delete_data:
caplog_reset(caplog)
# We force delete_pgdata method to raise an exception.
mock_delete_data.side_effect = OSError('TestError')
wal_file.ensure()
b_pre_info.set_attribute('backup_version', 2)
build_backup_directories(b_pre_info)
backup_manager.delete_backup(b_info)
assert 'TestError' in caplog.text
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
def test_statistics(self, signal_mock, tempfile_mock, copy_mock,
create_and_purge_mock, analyse_mock, rsync_mock,
tmpdir, workers):
"""
Unit test for RsyncCopyController.statistics's code
"""
# Do a fake copy run to populate the start/stop timestamps.
# The steps are the same of the full run test
tempdir = tmpdir.mkdir('tmp')
tempfile_mock.return_value = tempdir.strpath
server = build_real_server(
global_conf={'barman_home': tmpdir.mkdir('home').strpath})
config = server.config
executor = server.backup_manager.executor
rcc = RsyncCopyController(
path=server.path,
ssh_command=executor.ssh_command,
ssh_options=executor.ssh_options,
network_compression=config.network_compression,
reuse_backup=None,
safe_horizon=None,
workers=workers)
backup_info = build_test_backup_info(
server=server,
pgdata="/pg/data",
config_file="/etc/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/data/pg_ident.conf",
begin_xlog="0/2000028",
begin_wal="000000010000000000000002",
begin_offset=28)
backup_info.save()
# This is to check that all the preparation is done correctly
assert os.path.exists(backup_info.filename)
# Silence the access to result properties
rsync_mock.return_value.out = ''
rsync_mock.return_value.err = ''
rsync_mock.return_value.ret = 0
# Mock analyze directory
def analyse_func(item):
l = item.label
item.dir_file = l + '_dir_file'
item.exclude_and_protect_file = l + '_exclude_and_protect_file'
item.safe_list = [_FileItem('mode', 1, 'date', 'path')]
item.check_list = [_FileItem('mode', 1, 'date', 'path')]
analyse_mock.side_effect = analyse_func
rcc.add_directory(label='tbs1',
src=':/fake/location/',
dst=backup_info.get_data_directory(16387),
reuse=None,
bwlimit=None,
item_class=rcc.TABLESPACE_CLASS)
rcc.add_directory(label='tbs2',
src=':/another/location/',
dst=backup_info.get_data_directory(16405),
reuse=None,
bwlimit=None,
item_class=rcc.TABLESPACE_CLASS)
rcc.add_directory(
label='pgdata',
src=':/pg/data/',
dst=backup_info.get_data_directory(),
reuse=None,
bwlimit=None,
item_class=rcc.PGDATA_CLASS,
exclude=[
'/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid'
],
exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405'])
rcc.add_file(label='pg_control',
src=':/pg/data/global/pg_control',
dst='%s/global/pg_control' %
backup_info.get_data_directory(),
item_class=rcc.PGCONTROL_CLASS)
rcc.add_file(label='config_file',
src=':/etc/postgresql.conf',
dst=backup_info.get_data_directory(),
item_class=rcc.CONFIG_CLASS,
optional=False)
# Do the fake run
rcc.copy()
# Calculate statistics
result = rcc.statistics()
# We cannot check the actual result because it is not predictable,
# so we check that every value is present and is a number and it is
# greather than 0
assert result.get('analysis_time') > 0
assert 'analysis_time_per_item' in result
for tbs in ('pgdata', 'tbs1', 'tbs2'):
assert result['analysis_time_per_item'][tbs] > 0
assert result.get('copy_time') > 0
assert 'copy_time_per_item' in result
assert 'serialized_copy_time_per_item' in result
for tbs in ('pgdata', 'tbs1', 'tbs2', 'config_file', 'pg_control'):
assert result['copy_time_per_item'][tbs] > 0
assert result['serialized_copy_time_per_item'][tbs] > 0
assert result.get('number_of_workers') == rcc.workers
assert result.get('total_time') > 0
