def test_get_next_batch(
self, from_file_mock, isfile_mock, exists_mock, glob_mock, caplog
):
"""
Test the FileWalArchiver.get_next_batch method
"""
# See all logs
caplog.set_level(0)
# WAL batch, with 000000010000000000000001 that is currently being
# written
glob_mock.return_value = ["000000010000000000000001"]
isfile_mock.return_value = True
# This is an hack, instead of a WalFileInfo we use a simple string to
# ease all the comparisons. The resulting string is the name enclosed
# in colons. e.g. ":000000010000000000000001:"
from_file_mock.side_effect = lambda wal_name, compression: (":%s:" % wal_name)
backup_manager = build_backup_manager(name="TestServer")
archiver = StreamingWalArchiver(backup_manager)
backup_manager.server.archivers = [archiver]
caplog_reset(caplog)
batch = archiver.get_next_batch()
assert ["000000010000000000000001"] == batch.skip
assert "" == caplog.text
# WAL batch, with 000000010000000000000002 that is currently being
# written and 000000010000000000000001 can be archived
caplog_reset(caplog)
glob_mock.return_value = [
"000000010000000000000001",
"000000010000000000000002",
]
batch = archiver.get_next_batch()
assert [":000000010000000000000001:"] == batch
assert ["000000010000000000000002"] == batch.skip
assert "" == caplog.text
# WAL batch, with two partial files.
caplog_reset(caplog)
glob_mock.return_value = [
"000000010000000000000001.partial",
"000000010000000000000002.partial",
]
batch = archiver.get_next_batch()
assert [":000000010000000000000001.partial:"] == batch
assert ["000000010000000000000002.partial"] == batch.skip
assert (
"Archiving partial files for server %s: "
"000000010000000000000001.partial" % archiver.config.name
) in caplog.text
# WAL batch, with history files.
caplog_reset(caplog)
glob_mock.return_value = [
"00000001.history",
"000000010000000000000002.partial",
]
batch = archiver.get_next_batch()
assert [":00000001.history:"] == batch
assert ["000000010000000000000002.partial"] == batch.skip
assert "" == caplog.text
# WAL batch with errors
wrong_file_name = "test_wrong_wal_file.2"
glob_mock.return_value = ["test_wrong_wal_file.2"]
batch = archiver.get_next_batch()
assert [wrong_file_name] == batch.errors
# WAL batch, with two partial files, but one has been just renamed.
caplog_reset(caplog)
exists_mock.side_effect = [False, True]
glob_mock.return_value = [
"000000010000000000000001.partial",
"000000010000000000000002.partial",
]
batch = archiver.get_next_batch()
assert len(batch) == 0
assert ["000000010000000000000002.partial"] == batch.skip
assert "" in caplog.text
def test_archive(
self,
datetime_mock,
move_mock,
archive_wal_mock,
get_next_batch_mock,
unlink_mock,
capsys,
caplog,
):
"""
Test FileWalArchiver.archive method
"""
# See all logs
caplog.set_level(0)
fxlogdb_mock = MagicMock()
backup_manager = MagicMock()
archiver = FileWalArchiver(backup_manager)
archiver.config.name = "test_server"
archiver.config.errors_directory = "/server/errors"
wal_info = WalFileInfo(name="test_wal_file")
wal_info.orig_filename = "test_wal_file"
batch = WalArchiverQueue([wal_info])
assert batch.size == 1
assert batch.run_size == 1
get_next_batch_mock.return_value = batch
archive_wal_mock.side_effect = DuplicateWalFile
datetime_mock.utcnow.return_value.strftime.return_value = "test_time"
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert (
"\tError: %s is already present in server %s. "
"File moved to errors directory." % (wal_info.name, archiver.config.name)
) in out
assert (
"\tError: %s is already present in server %s. "
"File moved to errors directory." % (wal_info.name, archiver.config.name)
) in caplog.text
archive_wal_mock.side_effect = MatchingDuplicateWalFile
archiver.archive(fxlogdb_mock)
unlink_mock.assert_called_with(wal_info.orig_filename)
# Test batch errors
caplog_reset(caplog)
batch.errors = ["testfile_1", "testfile_2"]
archive_wal_mock.side_effect = DuplicateWalFile
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert (
"Some unknown objects have been found while "
"processing xlog segments for %s. "
"Objects moved to errors directory:" % archiver.config.name
) in out
assert (
"Archiver is about to move %s unexpected file(s) to errors "
"directory for %s from %s"
% (len(batch.errors), archiver.config.name, archiver.name)
) in caplog.text
assert (
"Moving unexpected file for %s from %s: %s"
% (archiver.config.name, archiver.name, "testfile_1")
) in caplog.text
assert (
"Moving unexpected file for %s from %s: %s"
% (archiver.config.name, archiver.name, "testfile_2")
) in caplog.text
move_mock.assert_any_call(
"testfile_1",
os.path.join(
archiver.config.errors_directory,
"%s.%s.unknown" % ("testfile_1", "test_time"),
),
)
move_mock.assert_any_call(
"testfile_2",
os.path.join(
archiver.config.errors_directory,
"%s.%s.unknown" % ("testfile_2", "test_time"),
),
)
def test_delete_backup(self, mock_available_backups, tmpdir, caplog):
"""
Simple test for the deletion of a backup.
We want to test the behaviour of the delete_backup method
"""
# Setup of the test backup_manager
backup_manager = build_backup_manager()
backup_manager.server.config.name = 'TestServer'
backup_manager.server.config.barman_lock_directory = tmpdir.strpath
backup_manager.server.config.backup_options = []
# Create a fake backup directory inside tmpdir (old format)
base_dir = tmpdir.mkdir('base')
backup_dir = base_dir.mkdir('fake_backup_id')
pg_data = backup_dir.mkdir('pgdata')
pg_data_v2 = backup_dir.mkdir('data')
wal_dir = tmpdir.mkdir('wals')
wal_history_file02 = wal_dir.join('00000002.history')
wal_history_file03 = wal_dir.join('00000003.history')
wal_history_file04 = wal_dir.join('00000004.history')
wal_history_file02.write('1\t0/2000028\tat restore point "myrp"\n')
wal_history_file03.write('1\t0/2000028\tat restore point "myrp"\n')
wal_history_file04.write('1\t0/2000028\tat restore point "myrp"\n')
wal_history_file04.write('2\t0/3000028\tunknown\n')
wal_file = wal_dir.join('0000000100000000/000000010000000000000001')
wal_file.ensure()
xlog_db = wal_dir.join('xlog.db')
xlog_db.write(
'000000010000000000000001\t42\t43\tNone\n'
'00000002.history\t42\t43\tNone\n'
'00000003.history\t42\t43\tNone\n'
'00000004.history\t42\t43\tNone\n')
backup_manager.server.xlogdb.return_value.__enter__.return_value = (
xlog_db.open())
backup_manager.server.config.basebackups_directory = base_dir.strpath
backup_manager.server.config.wals_directory = wal_dir.strpath
# The following tablespaces are defined in the default backup info
# generated by build_test_backup_info
b_info = build_test_backup_info(
backup_id='fake_backup_id',
server=backup_manager.server,
)
# Make sure we are not trying to delete any WAL file,
# just by having a previous backup
b_pre_info = build_test_backup_info(
backup_id='fake_backup',
server=backup_manager.server,
)
mock_available_backups.return_value = {
"fake_backup": b_pre_info,
"fake_backup_id": b_info,
}
# Test 1: minimum redundancy not satisfied
caplog_reset(caplog)
backup_manager.server.config.minimum_redundancy = 2
b_info.set_attribute('backup_version', 1)
build_backup_directories(b_info)
backup_manager.delete_backup(b_info)
assert 'WARNING Skipping delete of backup ' in caplog.text
assert 'ERROR' not in caplog.text
assert os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 2: normal delete expecting no errors (old format)
caplog_reset(caplog)
backup_manager.server.config.minimum_redundancy = 1
b_info.set_attribute('backup_version', 1)
build_backup_directories(b_info)
backup_manager.delete_backup(b_info)
# the backup must not exists on disk anymore
assert 'WARNING' not in caplog.text
assert 'ERROR' not in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 3: delete the backup again, expect a failure in log
caplog_reset(caplog)
backup_manager.delete_backup(b_info)
assert 'ERROR Failure deleting backup fake_backup_id' in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 4: normal delete expecting no errors (new format)
caplog_reset(caplog)
b_info.set_attribute('backup_version', 2)
build_backup_directories(b_info)
backup_manager.delete_backup(b_info)
assert 'WARNING' not in caplog.text
assert 'ERROR' not in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 5: normal delete of first backup no errors and no skip
# removing one of the two backups present (new format)
# and all the previous wal
caplog_reset(caplog)
b_pre_info.set_attribute('backup_version', 2)
build_backup_directories(b_pre_info)
backup_manager.delete_backup(b_pre_info)
assert 'WARNING' not in caplog.text
assert 'ERROR' not in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert not os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 6: normal delete of first backup no errors and no skip
# removing one of the two backups present (new format)
# the previous wal is retained as on a different timeline
caplog_reset(caplog)
wal_file.ensure()
b_pre_info.set_attribute('timeline', 2)
b_pre_info.set_attribute('backup_version', 2)
build_backup_directories(b_pre_info)
backup_manager.delete_backup(b_pre_info)
assert 'WARNING' not in caplog.text
assert 'ERROR' not in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 7: simulate an error deleting the the backup.
with patch('barman.backup.BackupManager.delete_backup_data')\
as mock_delete_data:
caplog_reset(caplog)
# We force delete_pgdata method to raise an exception.
mock_delete_data.side_effect = OSError('TestError')
wal_file.ensure()
b_pre_info.set_attribute('backup_version', 2)
build_backup_directories(b_pre_info)
backup_manager.delete_backup(b_info)
assert 'TestError' in caplog.text
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
def test_delete_backup(self, mock_available_backups, tmpdir, caplog):
"""
Simple test for the deletion of a backup.
We want to test the behaviour of the delete_backup method
"""
# Setup of the test backup_manager
backup_manager = build_backup_manager()
backup_manager.server.config.name = 'TestServer'
backup_manager.server.config.barman_lock_directory = tmpdir.strpath
backup_manager.server.config.backup_options = []
# Create a fake backup directory inside tmpdir (old format)
base_dir = tmpdir.mkdir('base')
backup_dir = base_dir.mkdir('fake_backup_id')
pg_data = backup_dir.mkdir('pgdata')
pg_data_v2 = backup_dir.mkdir('data')
wal_dir = tmpdir.mkdir('wals')
wal_history_file02 = wal_dir.join('00000002.history')
wal_history_file03 = wal_dir.join('00000003.history')
wal_history_file04 = wal_dir.join('00000004.history')
wal_history_file02.write('1\t0/2000028\tat restore point "myrp"\n')
wal_history_file03.write('1\t0/2000028\tat restore point "myrp"\n')
wal_history_file04.write('1\t0/2000028\tat restore point "myrp"\n')
wal_history_file04.write('2\t0/3000028\tunknown\n')
wal_file = wal_dir.join('0000000100000000/000000010000000000000001')
wal_file.ensure()
xlog_db = wal_dir.join('xlog.db')
xlog_db.write(
'000000010000000000000001\t42\t43\tNone\n'
'00000002.history\t42\t43\tNone\n'
'00000003.history\t42\t43\tNone\n'
'00000004.history\t42\t43\tNone\n')
backup_manager.server.xlogdb.return_value.__enter__.return_value = (
xlog_db.open())
backup_manager.server.config.basebackups_directory = base_dir.strpath
backup_manager.server.config.wals_directory = wal_dir.strpath
# The following tablespaces are defined in the default backup info
# generated by build_test_backup_info
b_info = build_test_backup_info(
backup_id='fake_backup_id',
server=backup_manager.server,
)
# Make sure we are not trying to delete any WAL file,
# just by having a previous backup
b_pre_info = build_test_backup_info(
backup_id='fake_backup',
server=backup_manager.server,
)
mock_available_backups.return_value = {
"fake_backup": b_pre_info,
"fake_backup_id": b_info,
}
# Test 1: minimum redundancy not satisfied
caplog_reset(caplog)
backup_manager.server.config.minimum_redundancy = 2
b_info.set_attribute('backup_version', 1)
build_backup_directories(b_info)
backup_manager.delete_backup(b_info)
assert 'WARNING Skipping delete of backup ' in caplog.text
assert 'ERROR' not in caplog.text
assert os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 2: normal delete expecting no errors (old format)
caplog_reset(caplog)
backup_manager.server.config.minimum_redundancy = 1
b_info.set_attribute('backup_version', 1)
build_backup_directories(b_info)
backup_manager.delete_backup(b_info)
# the backup must not exists on disk anymore
assert 'WARNING' not in caplog.text
assert 'ERROR' not in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 3: delete the backup again, expect a failure in log
caplog_reset(caplog)
backup_manager.delete_backup(b_info)
assert 'ERROR Failure deleting backup fake_backup_id' in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 4: normal delete expecting no errors (new format)
caplog_reset(caplog)
b_info.set_attribute('backup_version', 2)
build_backup_directories(b_info)
backup_manager.delete_backup(b_info)
assert 'WARNING' not in caplog.text
assert 'ERROR' not in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 5: normal delete of first backup no errors and no skip
# removing one of the two backups present (new format)
# and all the previous wal
caplog_reset(caplog)
b_pre_info.set_attribute('backup_version', 2)
build_backup_directories(b_pre_info)
backup_manager.delete_backup(b_pre_info)
assert 'WARNING' not in caplog.text
assert 'ERROR' not in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert not os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 6: normal delete of first backup no errors and no skip
# removing one of the two backups present (new format)
# the previous wal is retained as on a different timeline
caplog_reset(caplog)
wal_file.ensure()
b_pre_info.set_attribute('timeline', 2)
b_pre_info.set_attribute('backup_version', 2)
build_backup_directories(b_pre_info)
backup_manager.delete_backup(b_pre_info)
assert 'WARNING' not in caplog.text
assert 'ERROR' not in caplog.text
assert not os.path.exists(pg_data.strpath)
assert not os.path.exists(pg_data_v2.strpath)
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
# Test 7: simulate an error deleting the the backup.
with patch('barman.backup.BackupManager.delete_backup_data')\
as mock_delete_data:
caplog_reset(caplog)
# We force delete_pgdata method to raise an exception.
mock_delete_data.side_effect = OSError('TestError')
wal_file.ensure()
b_pre_info.set_attribute('backup_version', 2)
build_backup_directories(b_pre_info)
backup_manager.delete_backup(b_info)
assert 'TestError' in caplog.text
assert os.path.exists(wal_file.strpath)
assert os.path.exists(wal_history_file02.strpath)
assert os.path.exists(wal_history_file03.strpath)
assert os.path.exists(wal_history_file04.strpath)
def test_archive(self, datetime_mock, move_mock, archive_wal_mock,
get_next_batch_mock, unlink_mock, capsys, caplog):
"""
Test FileWalArchiver.archive method
"""
fxlogdb_mock = MagicMock()
backup_manager = MagicMock()
archiver = FileWalArchiver(backup_manager)
archiver.config.name = "test_server"
wal_info = WalFileInfo(name="test_wal_file")
wal_info.orig_filename = "test_wal_file"
batch = WalArchiverQueue([wal_info])
assert batch.size == 1
assert batch.run_size == 1
get_next_batch_mock.return_value = batch
archive_wal_mock.side_effect = DuplicateWalFile
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert ("\tError: %s is already present in server %s. "
"File moved to errors directory." %
(wal_info.name, archiver.config.name)) in out
assert ("\tError: %s is already present in server %s. "
"File moved to errors directory." %
(wal_info.name, archiver.config.name)) in caplog.text
archive_wal_mock.side_effect = MatchingDuplicateWalFile
archiver.archive(fxlogdb_mock)
unlink_mock.assert_called_with(wal_info.orig_filename)
# Test batch errors
caplog_reset(caplog)
datetime_mock.utcnow.strftime.return_value = 'test_time'
batch.errors = ['testfile_1', 'testfile_2']
archive_wal_mock.side_effect = DuplicateWalFile
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert ("Some unknown objects have been found while "
"processing xlog segments for %s. "
"Objects moved to errors directory:" %
archiver.config.name) in out
assert ("Archiver is about to move %s unexpected file(s) to errors "
"directory for %s from %s" %
(len(batch.errors),
archiver.config.name,
archiver.name)) in caplog.text
assert ("Moving unexpected file for %s from %s: %s" %
(archiver.config.name,
archiver.name, 'testfile_1')) in caplog.text
assert ("Moving unexpected file for %s from %s: %s" %
(archiver.config.name,
archiver.name, 'testfile_2')) in caplog.text
move_mock.assert_any_call(
'testfile_1',
os.path.join(archiver.config.errors_directory,
"%s.%s.unknown" % ('testfile_1', 'test_time')))
move_mock.assert_any_call(
'testfile_2',
os.path.join(archiver.config.errors_directory,
"%s.%s.unknown" % ('testfile_2', 'test_time')))
