def test_archive_batch(self, archive_wal_mock, get_next_batch_mock,
fsync_mock, caplog):
"""
Test archive using batch limit
"""
# See all logs
caplog.set_level(0)
# Setup the test
fxlogdb_mock = MagicMock()
backup_manager = MagicMock()
archiver = FileWalArchiver(backup_manager)
archiver.config.name = "test_server"
wal_info = WalFileInfo(name="test_wal_file")
wal_info.orig_filename = "test_wal_file"
wal_info2 = WalFileInfo(name="test_wal_file2")
wal_info2.orig_filename = "test_wal_file2"
# Test queue with batch limit 1 with a list of 2 files
batch = WalArchiverQueue([wal_info, wal_info2], batch_size=1)
assert batch.size == 2
assert batch.run_size == 1
get_next_batch_mock.return_value = batch
archiver.archive(fxlogdb_mock)
# check the log for messages
assert ("Found %s xlog segments from %s for %s."
" Archive a batch of %s segments in this run." %
(batch.size, archiver.name, archiver.config.name,
batch.run_size)) in caplog.text
assert ("Batch size reached (%s) - "
"Exit %s process for %s" %
(batch.batch_size, archiver.name,
archiver.config.name)) in caplog.text
def test_get_wal_until_next_backup(self, get_backup_mock, tmpdir):
"""
Simple test for the management of .history files
"""
# build a WalFileInfo object
wfile_info = WalFileInfo()
wfile_info.name = '000000010000000000000003'
wfile_info.size = 42
wfile_info.time = 43
wfile_info.compression = None
# build a WalFileInfo history object
history_info = WalFileInfo()
history_info.name = '00000001.history'
history_info.size = 42
history_info.time = 43
history_info.compression = None
# create a xlog.db and add the 2 entries
wals_dir = tmpdir.mkdir("wals")
xlog = wals_dir.join("xlog.db")
xlog.write(wfile_info.to_xlogdb_line() + history_info.to_xlogdb_line())
# fake backup
backup = build_test_backup_info(
begin_wal='000000010000000000000001',
end_wal='000000010000000000000004')
# mock a server object and mock a return call to get_next_backup method
server = build_real_server(
global_conf={
"barman_lock_directory": tmpdir.mkdir('lock').strpath
},
main_conf={
"wals_directory": wals_dir.strpath
})
get_backup_mock.return_value = build_test_backup_info(
backup_id="1234567899",
begin_wal='000000010000000000000005',
end_wal='000000010000000000000009')
wals = []
for wal_file in server.get_wal_until_next_backup(backup,
include_history=True):
# get the result of the xlogdb read
wals.append(wal_file.name)
# check for the presence of the .history file
assert history_info.name in wals
def test_to_xlogdb_line(self):
wfile_info = WalFileInfo()
wfile_info.name = '000000000000000000000002'
wfile_info.size = 42
wfile_info.time = 43
wfile_info.compression = None
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000002')
assert wfile_info.to_xlogdb_line() == (
'000000000000000000000002\t42\t43\tNone\n')
def test_archive(self, datetime_mock, move_mock, archive_wal_mock,
get_next_batch_mock, unlink_mock, capsys):
"""
Test FileWalArchiver.archive method
"""
fxlogdb_mock = MagicMock()
backup_manager = MagicMock()
archiver = FileWalArchiver(backup_manager)
archiver.config.name = "test_server"
wal_info = WalFileInfo(name="test_wal_file")
wal_info.orig_filename = "test_wal_file"
batch = WalArchiverBatch([wal_info])
get_next_batch_mock.return_value = batch
archive_wal_mock.side_effect = DuplicateWalFile
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert ("\tError: %s is already present in server %s. "
"File moved to errors directory." %
(wal_info.name, archiver.config.name)) in out
archive_wal_mock.side_effect = MatchingDuplicateWalFile
archiver.archive(fxlogdb_mock)
unlink_mock.assert_called_with(wal_info.orig_filename)
# Test batch errors
datetime_mock.utcnow.strftime.return_value = 'test_time'
batch.errors = ['testfile_1', 'testfile_2']
archive_wal_mock.side_effect = DuplicateWalFile
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert ("Some unknown objects have been found while "
"processing xlog segments for %s. "
"Objects moved to errors directory:" %
archiver.config.name) in out
move_mock.assert_any_call(
'testfile_1',
os.path.join(archiver.config.errors_directory,
"%s.%s.unknown" % ('testfile_1', 'test_time')))
move_mock.assert_any_call(
'testfile_2',
os.path.join(archiver.config.errors_directory,
"%s.%s.unknown" % ('testfile_2', 'test_time')))
def test_from_xlogdb_line(self):
"""
Test the conversion from a string to a WalFileInfo file
"""
# build a WalFileInfo object
wfile_info = WalFileInfo()
wfile_info.name = '000000000000000000000001'
wfile_info.size = 42
wfile_info.time = 43
wfile_info.compression = None
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000001')
# mock a server object
server = mock.Mock(name='server')
server.config.wals_directory = '/tmp/wals'
# parse the string
info_file = wfile_info.from_xlogdb_line(
'000000000000000000000001\t42\t43\tNone\n')
assert list(wfile_info.items()) == list(info_file.items())
def test_archive(
self,
datetime_mock,
move_mock,
archive_wal_mock,
get_next_batch_mock,
unlink_mock,
capsys,
caplog,
):
"""
Test FileWalArchiver.archive method
"""
# See all logs
caplog.set_level(0)
fxlogdb_mock = MagicMock()
backup_manager = MagicMock()
archiver = FileWalArchiver(backup_manager)
archiver.config.name = "test_server"
archiver.config.errors_directory = "/server/errors"
wal_info = WalFileInfo(name="test_wal_file")
wal_info.orig_filename = "test_wal_file"
batch = WalArchiverQueue([wal_info])
assert batch.size == 1
assert batch.run_size == 1
get_next_batch_mock.return_value = batch
archive_wal_mock.side_effect = DuplicateWalFile
datetime_mock.utcnow.return_value.strftime.return_value = "test_time"
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert (
"\tError: %s is already present in server %s. "
"File moved to errors directory." % (wal_info.name, archiver.config.name)
) in out
assert (
"\tError: %s is already present in server %s. "
"File moved to errors directory." % (wal_info.name, archiver.config.name)
) in caplog.text
archive_wal_mock.side_effect = MatchingDuplicateWalFile
archiver.archive(fxlogdb_mock)
unlink_mock.assert_called_with(wal_info.orig_filename)
# Test batch errors
caplog_reset(caplog)
batch.errors = ["testfile_1", "testfile_2"]
archive_wal_mock.side_effect = DuplicateWalFile
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert (
"Some unknown objects have been found while "
"processing xlog segments for %s. "
"Objects moved to errors directory:" % archiver.config.name
) in out
assert (
"Archiver is about to move %s unexpected file(s) to errors "
"directory for %s from %s"
% (len(batch.errors), archiver.config.name, archiver.name)
) in caplog.text
assert (
"Moving unexpected file for %s from %s: %s"
% (archiver.config.name, archiver.name, "testfile_1")
) in caplog.text
assert (
"Moving unexpected file for %s from %s: %s"
% (archiver.config.name, archiver.name, "testfile_2")
) in caplog.text
move_mock.assert_any_call(
"testfile_1",
os.path.join(
archiver.config.errors_directory,
"%s.%s.unknown" % ("testfile_1", "test_time"),
),
)
move_mock.assert_any_call(
"testfile_2",
os.path.join(
archiver.config.errors_directory,
"%s.%s.unknown" % ("testfile_2", "test_time"),
),
)
