def test_recover_xlog(self, rsync_pg_mock, tmpdir):
"""
Test the recovery of the xlogs of a backup
:param rsync_pg_mock: Mock rsync object for the purpose if this test
"""
# Build basic folders/files structure
dest = tmpdir.mkdir('destination')
wals = tmpdir.mkdir('wals')
xlog_dir = wals.mkdir(xlog.hash_dir('000000000000000000000002'))
xlog_file = xlog_dir.join('000000000000000000000002')
xlog_file.write('dummy content')
server = testing_helpers.build_real_server(
main_conf={'wals_directory': wals.strpath})
# build executor
executor = RecoveryExecutor(server.backup_manager)
required_wals = (WalFileInfo.from_xlogdb_line(
'000000000000000000000002\t42\t43\tNone\n'),)
executor.xlog_copy(required_wals, dest.strpath, None)
# check for a correct invocation of rsync using local paths
rsync_pg_mock.from_file_list.assert_called_once(
['000000000000000000000002'],
xlog_dir.strpath,
dest.strpath)
# reset mock calls
rsync_pg_mock.reset_mock()
required_wals = (WalFileInfo.from_xlogdb_line(
'000000000000000000000002\t42\t43\tNone\n'),)
executor.backup_manager.compression_manager = Mock()
executor.xlog_copy(required_wals, dest.strpath, 'remote_command')
# check for the invocation of rsync on a remote call
rsync_pg_mock.assert_called_once(network_compression=False,
bwlimit=None,
ssh='remote_command')
def test_archive_batch(self, archive_wal_mock, get_next_batch_mock,
fsync_mock, caplog):
"""
Test archive using batch limit
"""
# Setup the test
fxlogdb_mock = MagicMock()
backup_manager = MagicMock()
archiver = FileWalArchiver(backup_manager)
archiver.config.name = "test_server"
wal_info = WalFileInfo(name="test_wal_file")
wal_info.orig_filename = "test_wal_file"
wal_info2 = WalFileInfo(name="test_wal_file2")
wal_info2.orig_filename = "test_wal_file2"
# Test queue with batch limit 1 with a list of 2 files
batch = WalArchiverQueue([wal_info, wal_info2], batch_size=1)
assert batch.size == 2
assert batch.run_size == 1
get_next_batch_mock.return_value = batch
archiver.archive(fxlogdb_mock)
# check the log for messages
assert ("Found %s xlog segments from %s for %s."
" Archive a batch of %s segments in this run." %
(batch.size,
archiver.name,
archiver.config.name,
batch.run_size)) in caplog.text
assert ("Batch size reached (%s) - "
"Exit %s process for %s" %
(batch.batch_size,
archiver.name,
archiver.config.name)) in caplog.text
def test_get_wal_info(self, get_wal_mock, tmpdir):
"""
Basic test for get_wal_info method
Test the wals per second and total time in seconds values.
:return:
"""
# Build a test server with a test path
server = build_real_server(global_conf={
'barman_home': tmpdir.strpath
})
# Mock method get_wal_until_next_backup for returning a list of
# 3 fake WAL. the first one is the start and stop WAL of the backup
wal_list = [
WalFileInfo.from_xlogdb_line(
"000000010000000000000002\t16777216\t1434450086.53\tNone\n"),
WalFileInfo.from_xlogdb_line(
"000000010000000000000003\t16777216\t1434450087.54\tNone\n"),
WalFileInfo.from_xlogdb_line(
"000000010000000000000004\t16777216\t1434450088.55\tNone\n")]
get_wal_mock.return_value = wal_list
backup_info = build_test_backup_info(
server=server,
begin_wal=wal_list[0].name,
end_wal=wal_list[0].name)
backup_info.save()
# Evaluate total time in seconds:
# last_wal_timestamp - first_wal_timestamp
wal_total_seconds = wal_list[-1].time - wal_list[0].time
# Evaluate the wals_per_second value:
# wals_in_backup + wals_until_next_backup / total_time_in_seconds
wals_per_second = len(wal_list) / wal_total_seconds
wal_info = server.get_wal_info(backup_info)
assert wal_info
assert wal_info['wal_total_seconds'] == wal_total_seconds
assert wal_info['wals_per_second'] == wals_per_second
def get_required_xlog_files(self, backup, target_tli=None, target_time=None,
target_xid=None):
"""
Get the xlog files required for a recovery
"""
begin = backup.begin_wal
end = backup.end_wal
# If timeline isn't specified, assume it is the same timeline
# of the backup
if not target_tli:
target_tli, _, _ = xlog.decode_segment_name(end)
with self.xlogdb() as fxlogdb:
for line in fxlogdb:
wal_info = WalFileInfo.from_xlogdb_line(line)
# Handle .history files: add all of them to the output,
# regardless of their age
if xlog.is_history_file(wal_info.name):
yield wal_info
continue
if wal_info.name < begin:
continue
tli, _, _ = xlog.decode_segment_name(wal_info.name)
if tli > target_tli:
continue
yield wal_info
if wal_info.name > end:
end = wal_info.name
if target_time and target_time < wal_info.time:
break
# return all the remaining history files
for line in fxlogdb:
wal_info = WalFileInfo.from_xlogdb_line(line)
if xlog.is_history_file(wal_info.name):
yield wal_info
def test_from_file_override(self, id_compression, tmpdir):
# prepare
id_compression.return_value = None
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write('dummy_content\n')
wfile_info = WalFileInfo.from_file(tmp_file.strpath,
name="000000000000000000000002")
assert wfile_info.name == '000000000000000000000002'
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000002')
wfile_info = WalFileInfo.from_file(tmp_file.strpath, size=42)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == 42
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000001')
wfile_info = WalFileInfo.from_file(tmp_file.strpath, time=43)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == 43
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000001')
def test_from_file_override(self, id_compression, tmpdir):
# prepare
id_compression.return_value = None
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write("dummy_content\n")
wfile_info = WalFileInfo.from_file(tmp_file.strpath, name="000000000000000000000002")
assert wfile_info.name == "000000000000000000000002"
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == ("0000000000000000/000000000000000000000002")
wfile_info = WalFileInfo.from_file(tmp_file.strpath, size=42)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == 42
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == ("0000000000000000/000000000000000000000001")
wfile_info = WalFileInfo.from_file(tmp_file.strpath, time=43)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == 43
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == ("0000000000000000/000000000000000000000001")
def test_recover_xlog(self, rsync_pg_mock, cm_mock, tmpdir):
"""
Test the recovery of the xlogs of a backup
:param rsync_pg_mock: Mock rsync object for the purpose if this test
"""
# Build basic folders/files structure
dest = tmpdir.mkdir("destination")
wals = tmpdir.mkdir("wals")
# Create 3 WAL files with different compressions
xlog_dir = wals.mkdir(xlog.hash_dir("000000000000000000000002"))
xlog_plain = xlog_dir.join("000000000000000000000001")
xlog_gz = xlog_dir.join("000000000000000000000002")
xlog_bz2 = xlog_dir.join("000000000000000000000003")
xlog_plain.write("dummy content")
xlog_gz.write("dummy content gz")
xlog_bz2.write("dummy content bz2")
server = testing_helpers.build_real_server(main_conf={"wals_directory": wals.strpath})
# Prepare compressors mock
c = {"gzip": mock.Mock(name="gzip"), "bzip2": mock.Mock(name="bzip2")}
cm_mock.return_value.get_compressor = lambda compression=None, path=None: c[compression]
# touch destination files to avoid errors on cleanup
c["gzip"].decompress.side_effect = lambda src, dst: open(dst, "w")
c["bzip2"].decompress.side_effect = lambda src, dst: open(dst, "w")
# Build executor
executor = RecoveryExecutor(server.backup_manager)
# Test: local copy
required_wals = (
WalFileInfo.from_xlogdb_line("000000000000000000000001\t42\t43\tNone\n"),
WalFileInfo.from_xlogdb_line("000000000000000000000002\t42\t43\tgzip\n"),
WalFileInfo.from_xlogdb_line("000000000000000000000003\t42\t43\tbzip2\n"),
)
executor._xlog_copy(required_wals, dest.strpath, None)
# Check for a correct invocation of rsync using local paths
rsync_pg_mock.assert_called_once_with(network_compression=False, bwlimit=None, path=None, ssh=None)
assert not rsync_pg_mock.return_value.from_file_list.called
c["gzip"].decompress.assert_called_once_with(xlog_gz.strpath, mock.ANY)
c["bzip2"].decompress.assert_called_once_with(xlog_bz2.strpath, mock.ANY)
# Reset mock calls
rsync_pg_mock.reset_mock()
c["gzip"].reset_mock()
c["bzip2"].reset_mock()
# Test: remote copy
executor._xlog_copy(required_wals, dest.strpath, "remote_command")
# Check for the invocation of rsync on a remote call
rsync_pg_mock.assert_called_once_with(
network_compression=False, bwlimit=None, path=mock.ANY, ssh="remote_command"
)
rsync_pg_mock.return_value.from_file_list.assert_called_once_with(
["000000000000000000000001", "000000000000000000000002", "000000000000000000000003"], mock.ANY, mock.ANY
)
c["gzip"].decompress.assert_called_once_with(xlog_gz.strpath, mock.ANY)
c["bzip2"].decompress.assert_called_once_with(xlog_bz2.strpath, mock.ANY)
def test_archive(self, datetime_mock, move_mock, archive_wal_mock,
get_next_batch_mock, unlink_mock, capsys):
"""
Test FileWalArchiver.archive method
"""
fxlogdb_mock = MagicMock()
backup_manager = MagicMock()
archiver = FileWalArchiver(backup_manager)
archiver.config.name = "test_server"
wal_info = WalFileInfo(name="test_wal_file")
wal_info.orig_filename = "test_wal_file"
batch = WalArchiverBatch([wal_info])
get_next_batch_mock.return_value = batch
archive_wal_mock.side_effect = DuplicateWalFile
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert ("\tError: %s is already present in server %s. "
"File moved to errors directory." %
(wal_info.name, archiver.config.name)) in out
archive_wal_mock.side_effect = MatchingDuplicateWalFile
archiver.archive(fxlogdb_mock)
unlink_mock.assert_called_with(wal_info.orig_filename)
# Test batch errors
datetime_mock.utcnow.strftime.return_value = 'test_time'
batch.errors = ['testfile_1', 'testfile_2']
archive_wal_mock.side_effect = DuplicateWalFile
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert ("Some unknown objects have been found while "
"processing xlog segments for %s. "
"Objects moved to errors directory:" %
archiver.config.name) in out
move_mock.assert_any_call(
'testfile_1',
os.path.join(archiver.config.errors_directory,
"%s.%s.unknown" % ('testfile_1', 'test_time')))
move_mock.assert_any_call(
'testfile_2',
os.path.join(archiver.config.errors_directory,
"%s.%s.unknown" % ('testfile_2', 'test_time')))
def test_archive(self, datetime_mock, move_mock, archive_wal_mock,
get_next_batch_mock, unlink_mock, capsys):
"""
Test FileWalArchiver.archive method
"""
fxlogdb_mock = MagicMock()
backup_manager = MagicMock()
archiver = FileWalArchiver(backup_manager)
archiver.config.name = "test_server"
wal_info = WalFileInfo(name="test_wal_file")
wal_info.orig_filename = "test_wal_file"
batch = WalArchiverBatch([wal_info])
get_next_batch_mock.return_value = batch
archive_wal_mock.side_effect = DuplicateWalFile
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert ("\tError: %s is already present in server %s. "
"File moved to errors directory." %
(wal_info.name, archiver.config.name)) in out
archive_wal_mock.side_effect = MatchingDuplicateWalFile
archiver.archive(fxlogdb_mock)
unlink_mock.assert_called_with(wal_info.orig_filename)
# Test batch errors
datetime_mock.utcnow.strftime.return_value = 'test_time'
batch.errors = ['testfile_1', 'testfile_2']
archive_wal_mock.side_effect = DuplicateWalFile
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert ("Some unknown objects have been found while "
"processing xlog segments for %s. "
"Objects moved to errors directory:" %
archiver.config.name) in out
move_mock.assert_any_call(
'testfile_1',
os.path.join(archiver.config.errors_directory,
"%s.%s.unknown" % ('testfile_1', 'test_time')))
move_mock.assert_any_call(
'testfile_2',
os.path.join(archiver.config.errors_directory,
"%s.%s.unknown" % ('testfile_2', 'test_time')))
def remove_wal_before_backup(self, backup_info):
"""
Remove WAL files which have been archived before the start of
the provided backup.
If no backup_info is provided delete all available WAL files
:param BackupInfo|None backup_info: the backup information structure
:return list: a list of removed WAL files
"""
removed = []
with self.server.xlogdb() as fxlogdb:
xlogdb_new = fxlogdb.name + ".new"
with open(xlogdb_new, 'w') as fxlogdb_new:
for line in fxlogdb:
wal_info = WalFileInfo.from_xlogdb_line(line)
# Keeps the WAL segment if it is a history file or later
# than the given backup (the first available)
if (xlog.is_history_file(wal_info.name) or
(backup_info and wal_info.name >= backup_info.begin_wal)):
fxlogdb_new.write(wal_info.to_xlogdb_line())
continue
else:
self.delete_wal(wal_info)
removed.append(wal_info.name)
fxlogdb_new.flush()
os.fsync(fxlogdb_new.fileno())
shutil.move(xlogdb_new, fxlogdb.name)
fsync_dir(os.path.dirname(fxlogdb.name))
return removed
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived via
streaming replication (in the 'streaming' directory)
This method always leaves one file in the "streaming" directory,
because the 'pg_receivexlog' process needs at least one file to
detect the current streaming position after a restart.
:return: WalArchiverQueue: list of WAL files
"""
# Get the batch size from configuration (0 = unlimited)
batch_size = self.config.streaming_archiver_batch_size
# List and sort all files in the incoming directory
file_names = glob(
os.path.join(self.config.streaming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file,
# including partial ones and history files.
# Anything else is treated like an error/anomaly
files = []
skip = []
errors = []
for file_name in file_names:
# Ignore temporary files
if file_name.endswith('.tmp'):
continue
# If the file doesn't exist, it has been renamed/removed while
# we were reading the directory. Ignore it.
if not os.path.exists(file_name):
continue
if not os.path.isfile(file_name):
errors.append(file_name)
elif xlog.is_partial_file(file_name):
skip.append(file_name)
elif xlog.is_any_xlog_file(file_name):
files.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as normal files
if len(skip) > 1:
partials = skip[:-1]
_logger.info('Archiving partial files for server %s: %s' %
(self.config.name, ", ".join(
[os.path.basename(f) for f in partials])))
files.extend(partials)
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
return WalArchiverQueue(wal_files,
batch_size=batch_size,
errors=errors,
skip=skip)
def get_wal_until_next_backup(self, backup, include_history=False):
"""
Get the xlog files between backup and the next
:param BackupInfo backup: a backup object, the starting point
to retrieve WALs
:param bool include_history: option for the inclusion of
include_history files into the output
"""
begin = backup.begin_wal
next_end = None
if self.get_next_backup(backup.backup_id):
next_end = self.get_next_backup(backup.backup_id).end_wal
backup_tli, _, _ = xlog.decode_segment_name(begin)
with self.xlogdb() as fxlogdb:
for line in fxlogdb:
wal_info = WalFileInfo.from_xlogdb_line(line)
# Handle .history files: add all of them to the output,
# regardless of their age, if requested (the 'include_history'
# parameter is True)
if xlog.is_history_file(wal_info.name):
if include_history:
yield wal_info
continue
if wal_info.name < begin:
continue
tli, _, _ = xlog.decode_segment_name(wal_info.name)
if tli > backup_tli:
continue
if not xlog.is_wal_file(wal_info.name):
continue
if next_end and wal_info.name > next_end:
break
yield wal_info
def get_latest_archived_wal(self):
"""
Return the WalFileInfo of the last WAL file in the archive,
or None if the archive doesn't contain any WAL file.
:rtype: WalFileInfo|None
"""
# TODO: consider timeline?
from os.path import isdir, join
root = self.config.wals_directory
# If the WAL archive directory doesn't exists the archive is empty
if not isdir(root):
return None
# Traverse all the directory in the archive in reverse order,
# returning the first WAL file found
for name in sorted(os.listdir(root), reverse=True):
fullname = join(root, name)
# All relevant files are in subdirectories, so
# we skip any non-directory entry
if isdir(fullname):
hash_dir = fullname
# Inspect contained files in reverse order
for wal_name in sorted(os.listdir(hash_dir), reverse=True):
fullname = join(hash_dir, wal_name)
# Return the first file that has the correct name
if not isdir(fullname) and xlog.is_wal_file(fullname):
return WalFileInfo.from_file(fullname)
# If we get here, no WAL files have been found
return None
def missing_wals(server, args):
warn = args.warning
crit = args.critical
from barman.xlog import is_wal_file
from barman.infofile import WalFileInfo
wals_directory = server.config.wals_directory
missing_wals = 0
with server.xlogdb() as fxlogdb:
for line in fxlogdb:
#name, size, time, compression = server.xlogdb_parse_line(line)
wal_info = WalFileInfo.from_xlogdb_line(line)
name = wal_info.name
directory = name[0:16]
if is_wal_file(name):
file_path = os.path.join(wals_directory, directory, name)
if not os.path.exists(file_path):
missing_wals = missing_wals + 1
exit_check(missing_wals, warn, crit,
"There are %d missing wals for the last backup." % missing_wals,
perfdata_key="missing", perfdata_min=0)
def last_wal_age(server, args):
warn = args.warning
crit = args.critical
from barman.infofile import WalFileInfo
with server.xlogdb() as fxlogdb:
line = None
for line in fxlogdb:
pass
if line is None:
critical("No WAL received yet.")
#name, size, time, compression = server.xlogdb_parse_line(line)
wal_info = WalFileInfo.from_xlogdb_line(line)
time = datetime.fromtimestamp(wal_info.time)
now = datetime.now()
age = now - time
minutes = age.seconds / 60
minutes = minutes + age.days * 60 * 24
exit_check(minutes, warn, crit, "Last WAL is %s minutes old." % minutes,
perfdata_key="minutes", perfdata_min=0)
def last_wal_age(server, args):
warn = args.warning
crit = args.critical
from barman.infofile import WalFileInfo
with server.xlogdb() as fxlogdb:
line = None
for line in fxlogdb:
pass
if line is None:
critical("No WAL received yet.")
#name, size, time, compression = server.xlogdb_parse_line(line)
wal_info = WalFileInfo.from_xlogdb_line(line)
time = datetime.fromtimestamp(wal_info.time)
now = datetime.now()
age = now - time
minutes = age.seconds / 60
minutes = minutes + age.days * 60 * 24
exit_check(minutes,
warn,
crit,
"Last WAL is %s minutes old." % minutes,
perfdata_key="minutes",
perfdata_min=0)
def missing_wals(server, args):
warn = args.warning
crit = args.critical
from barman.xlog import is_wal_file
from barman.infofile import WalFileInfo
wals_directory = server.config.wals_directory
missing_wals = 0
with server.xlogdb() as fxlogdb:
for line in fxlogdb:
#name, size, time, compression = server.xlogdb_parse_line(line)
wal_info = WalFileInfo.from_xlogdb_line(line)
name = wal_info.name
directory = name[0:16]
if is_wal_file(name):
file_path = os.path.join(wals_directory, directory, name)
if not os.path.exists(file_path):
missing_wals = missing_wals + 1
exit_check(missing_wals,
warn,
crit,
"There are %d missing wals for the last backup." % missing_wals,
perfdata_key="missing",
perfdata_min=0)
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived through
a PostgreSQL's 'archive_command' (in the 'incoming' directory)
:return: WalArchiverBatch: list of WAL files
"""
# List and sort all files in the incoming directory
file_names = glob(os.path.join(
self.config.incoming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file. Anything
# else is treated like an error/anomaly
files = []
errors = []
for file_name in file_names:
if xlog.is_any_xlog_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
else:
errors.append(file_name)
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f) for f in files]
return WalArchiverBatch(wal_files, errors=errors)
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived through
a PostgreSQL's 'archive_command' (in the 'incoming' directory)
:return: WalArchiverQueue: list of WAL files
"""
# Get the batch size from configuration (0 = unlimited)
batch_size = self.config.archiver_batch_size
# List and sort all files in the incoming directory
# IMPORTANT: the list is sorted, and this allows us to know that the
# WAL stream we have is monotonically increasing. That allows us to
# verify that a backup has all the WALs required for the restore.
file_names = glob(
os.path.join(self.config.incoming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file. Anything
# else is treated like an error/anomaly
files = []
errors = []
for file_name in file_names:
# Ignore temporary files
if file_name.endswith('.tmp'):
continue
if xlog.is_any_xlog_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
else:
errors.append(file_name)
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f) for f in files]
return WalArchiverQueue(wal_files,
batch_size=batch_size,
errors=errors)
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived through
a PostgreSQL's 'archive_command' (in the 'incoming' directory)
:return: WalArchiverQueue: list of WAL files
"""
# Get the batch size from configuration (0 = unlimited)
batch_size = self.config.archiver_batch_size
# List and sort all files in the incoming directory
file_names = glob(
os.path.join(self.config.incoming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file. Anything
# else is treated like an error/anomaly
files = []
errors = []
for file_name in file_names:
if xlog.is_any_xlog_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
else:
errors.append(file_name)
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f) for f in files]
return WalArchiverQueue(wal_files,
batch_size=batch_size,
errors=errors)
def remove_wal_before_backup(self, backup_info, timelines_to_protect=None):
"""
Remove WAL files which have been archived before the start of
the provided backup.
If no backup_info is provided delete all available WAL files
If timelines_to_protect list is passed, never remove a wal in one of
these timelines.
:param BackupInfo|None backup_info: the backup information structure
:param set timelines_to_protect: optional list of timelines
to protect
:return list: a list of removed WAL files
"""
removed = []
with self.server.xlogdb("r+") as fxlogdb:
xlogdb_dir = os.path.dirname(fxlogdb.name)
with tempfile.TemporaryFile(mode="w+",
dir=xlogdb_dir) as fxlogdb_new:
for line in fxlogdb:
wal_info = WalFileInfo.from_xlogdb_line(line)
if not xlog.is_any_xlog_file(wal_info.name):
output.error(
"invalid WAL segment name %r\n"
'HINT: Please run "barman rebuild-xlogdb %s" '
"to solve this issue",
wal_info.name,
self.config.name,
)
continue
# Keeps the WAL segment if it is a history file
keep = xlog.is_history_file(wal_info.name)
# Keeps the WAL segment if its timeline is in
# `timelines_to_protect`
if timelines_to_protect:
tli, _, _ = xlog.decode_segment_name(wal_info.name)
keep |= tli in timelines_to_protect
# Keeps the WAL segment if it is a newer
# than the given backup (the first available)
if backup_info and backup_info.begin_wal is not None:
keep |= wal_info.name >= backup_info.begin_wal
# If the file has to be kept write it in the new xlogdb
# otherwise delete it and record it in the removed list
if keep:
fxlogdb_new.write(wal_info.to_xlogdb_line())
else:
self.delete_wal(wal_info)
removed.append(wal_info.name)
fxlogdb_new.flush()
fxlogdb_new.seek(0)
fxlogdb.seek(0)
shutil.copyfileobj(fxlogdb_new, fxlogdb)
fxlogdb.truncate()
return removed
def test_to_xlogdb_line(self):
wfile_info = WalFileInfo()
wfile_info.name = '000000000000000000000002'
wfile_info.size = 42
wfile_info.time = 43
wfile_info.compression = None
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000002')
assert wfile_info.to_xlogdb_line() == (
'000000000000000000000002\t42\t43\tNone\n')
def test_from_file_no_compression(self, tmpdir):
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write('dummy_content\n')
stat = os.stat(tmp_file.strpath)
wfile_info = WalFileInfo.from_file(tmp_file.strpath)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == stat.st_size
assert wfile_info.time == stat.st_mtime
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.relpath() == '0000000000000000/000000000000000000000001'
def remove_wal_before_backup(self, backup_info, timelines_to_protect=None):
"""
Remove WAL files which have been archived before the start of
the provided backup.
If no backup_info is provided delete all available WAL files
If timelines_to_protect list is passed, never remove a wal in one of
these timelines.
:param BackupInfo|None backup_info: the backup information structure
:param set timelines_to_protect: optional list of timelines
to protect
:return list: a list of removed WAL files
"""
removed = []
with self.server.xlogdb() as fxlogdb:
xlogdb_new = fxlogdb.name + ".new"
with open(xlogdb_new, 'w') as fxlogdb_new:
for line in fxlogdb:
wal_info = WalFileInfo.from_xlogdb_line(line)
if not xlog.is_any_xlog_file(wal_info.name):
output.error(
"invalid xlog segment name %r\n"
"HINT: Please run \"barman rebuild-xlogdb %s\" "
"to solve this issue",
wal_info.name, self.config.name)
continue
# Keeps the WAL segment if it is a history file
keep = xlog.is_history_file(wal_info.name)
# Keeps the WAL segment if its timeline is in
# `timelines_to_protect`
if timelines_to_protect:
tli, _, _ = xlog.decode_segment_name(wal_info.name)
keep |= tli in timelines_to_protect
# Keeps the WAL segment if it is a newer
# than the given backup (the first available)
if backup_info:
keep |= wal_info.name >= backup_info.begin_wal
# If the file has to be kept write it in the new xlogdb
# otherwise delete it and record it in the removed list
if keep:
fxlogdb_new.write(wal_info.to_xlogdb_line())
else:
self.delete_wal(wal_info)
removed.append(wal_info.name)
fxlogdb_new.flush()
os.fsync(fxlogdb_new.fileno())
shutil.move(xlogdb_new, fxlogdb.name)
fsync_dir(os.path.dirname(fxlogdb.name))
return removed
def test_to_xlogdb_line(self):
wfile_info = WalFileInfo()
wfile_info.name = '000000000000000000000002'
wfile_info.size = 42
wfile_info.time = 43
wfile_info.compression = None
assert wfile_info.relpath() == '0000000000000000/000000000000000000000002'
assert wfile_info.to_xlogdb_line() == '000000000000000000000002\t42\t43\tNone\n'
def test_from_file_no_compression(self, tmpdir):
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write('dummy_content\n')
stat = os.stat(tmp_file.strpath)
wfile_info = WalFileInfo.from_file(tmp_file.strpath)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == stat.st_size
assert wfile_info.time == stat.st_mtime
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000001')
def test_from_file_no_compression(self, mock_compression_manager, tmpdir):
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write("dummy_content\n")
stat = os.stat(tmp_file.strpath)
wfile_info = WalFileInfo.from_file(tmp_file.strpath,
mock_compression_manager)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == stat.st_size
assert wfile_info.time == stat.st_mtime
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.relpath() == (
"0000000000000000/000000000000000000000001")
def test_from_file_override(self, compression_manager, tmpdir):
# prepare
compression_manager.identify_compression.return_value = None
compression_manager.unidentified_compression = None
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write("dummy_content\n")
wfile_info = WalFileInfo.from_file(tmp_file.strpath,
compression_manager,
name="000000000000000000000002")
assert wfile_info.name == "000000000000000000000002"
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == (
"0000000000000000/000000000000000000000002")
wfile_info = WalFileInfo.from_file(tmp_file.strpath,
compression_manager,
size=42)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == 42
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == (
"0000000000000000/000000000000000000000001")
wfile_info = WalFileInfo.from_file(tmp_file.strpath,
compression_manager,
time=43)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == 43
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == (
"0000000000000000/000000000000000000000001")
def test_archive_batch(self, archive_wal_mock, get_next_batch_mock,
fsync_mock, caplog):
"""
Test archive using batch limit
"""
# See all logs
caplog.set_level(0)
# Setup the test
fxlogdb_mock = MagicMock()
backup_manager = MagicMock()
archiver = FileWalArchiver(backup_manager)
archiver.config.name = "test_server"
wal_info = WalFileInfo(name="test_wal_file")
wal_info.orig_filename = "test_wal_file"
wal_info2 = WalFileInfo(name="test_wal_file2")
wal_info2.orig_filename = "test_wal_file2"
# Test queue with batch limit 1 with a list of 2 files
batch = WalArchiverQueue([wal_info, wal_info2], batch_size=1)
assert batch.size == 2
assert batch.run_size == 1
get_next_batch_mock.return_value = batch
archiver.archive(fxlogdb_mock)
# check the log for messages
assert ("Found %s xlog segments from %s for %s."
" Archive a batch of %s segments in this run." %
(batch.size, archiver.name, archiver.config.name,
batch.run_size)) in caplog.text
assert ("Batch size reached (%s) - "
"Exit %s process for %s" %
(batch.batch_size, archiver.name,
archiver.config.name)) in caplog.text
def test_from_file_default_compression(self, id_compression, tmpdir):
# prepare
id_compression.return_value = None
tmp_file = tmpdir.join("00000001000000E500000064")
tmp_file.write("dummy_content\n")
wfile_info = WalFileInfo.from_file(tmp_file.strpath, default_compression="test_default_compression")
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression == "test_default_compression"
assert wfile_info.relpath() == ("00000001000000E5/00000001000000E500000064")
def test_from_file_compression(self, id_compression, tmpdir):
# prepare
id_compression.return_value = 'test_compression'
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write('dummy_content\n')
wfile_info = WalFileInfo.from_file(tmp_file.strpath)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.compression == 'test_compression'
assert wfile_info.relpath() == '0000000000000000/000000000000000000000001'
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived via
streaming replication (in the 'streaming' directory)
This method always leaves one file in the "streaming" directory,
because the 'pg_receivexlog' process needs at least one file to
detect the current streaming position after a restart.
:return: WalArchiverQueue: list of WAL files
"""
# Get the batch size from configuration (0 = unlimited)
batch_size = self.config.streaming_archiver_batch_size
# List and sort all files in the incoming directory
file_names = glob(os.path.join(
self.config.streaming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file,
# including partial ones and history files.
# Anything else is treated like an error/anomaly
files = []
skip = []
errors = []
for file_name in file_names:
if not os.path.isfile(file_name):
errors.append(file_name)
elif xlog.is_partial_file(file_name):
skip.append(file_name)
elif xlog.is_any_xlog_file(file_name):
files.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as errors
if len(skip) > 1:
errors.extend(skip[:-1])
_logger.warning('Multiple partial files found for server %s: %s' %
(self.config.name,
", ".join([os.path.basename(f) for f in errors])))
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
return WalArchiverQueue(wal_files,
batch_size=batch_size,
errors=errors, skip=skip)
def test_from_file_compression(self, id_compression, tmpdir):
# prepare
id_compression.return_value = 'test_compression'
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write('dummy_content\n')
wfile_info = WalFileInfo.from_file(tmp_file.strpath)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.compression == 'test_compression'
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000001')
def test_from_file_compression(self, mock_compression_manager, tmpdir):
# prepare
mock_compression_manager.identify_compression.return_value = "test_compression"
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write("dummy_content\n")
wfile_info = WalFileInfo.from_file(tmp_file.strpath,
mock_compression_manager)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression == "test_compression"
assert wfile_info.relpath() == (
"0000000000000000/000000000000000000000001")
def test_from_file_unidentified_compression(self, id_compression, tmpdir):
# prepare
id_compression.return_value = None
tmp_file = tmpdir.join("00000001000000E500000064")
tmp_file.write("dummy_content\n")
wfile_info = WalFileInfo.from_file(
tmp_file.strpath,
unidentified_compression="test_unidentified_compression")
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression == "test_unidentified_compression"
assert wfile_info.relpath() == (
"00000001000000E5/00000001000000E500000064")
def test_from_xlogdb_line(self):
"""
Test the conversion from a string to a WalFileInfo file
"""
# build a WalFileInfo object
wfile_info = WalFileInfo()
wfile_info.name = '000000000000000000000001'
wfile_info.size = 42
wfile_info.time = 43
wfile_info.compression = None
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000001')
# mock a server object
server = mock.Mock(name='server')
server.config.wals_directory = '/tmp/wals'
# parse the string
info_file = wfile_info.from_xlogdb_line(
'000000000000000000000001\t42\t43\tNone\n')
assert list(wfile_info.items()) == list(info_file.items())
def get_latest_archived_wals_info(self):
"""
Return a dictionary of timelines associated with the
WalFileInfo of the last WAL file in the archive,
or None if the archive doesn't contain any WAL file.
:rtype: dict[str, WalFileInfo]|None
"""
from os.path import isdir, join
root = self.config.wals_directory
# If the WAL archive directory doesn't exists the archive is empty
if not isdir(root):
return None
# Traverse all the directory in the archive in reverse order,
# returning the first WAL file found
timelines = {}
for name in sorted(os.listdir(root), reverse=True):
fullname = join(root, name)
# All relevant files are in subdirectories, so
# we skip any non-directory entry
if isdir(fullname):
# Extract the timeline. If it is not valid, skip this directory
try:
timeline = name[0:8]
int(timeline, 16)
except ValueError:
continue
# If this timeline already has a file, skip this directory
if timeline in timelines:
continue
hash_dir = fullname
# Inspect contained files in reverse order
for wal_name in sorted(os.listdir(hash_dir), reverse=True):
fullname = join(hash_dir, wal_name)
# Return the first file that has the correct name
if not isdir(fullname) and xlog.is_wal_file(fullname):
timelines[timeline] = WalFileInfo.from_file(fullname)
break
# Return the timeline map or None if it is empty
return timelines or None
def test_from_xlogdb_line(self):
"""
Test the conversion from a string to a WalFileInfo file
"""
# build a WalFileInfo object
wfile_info = WalFileInfo()
wfile_info.name = "000000000000000000000001"
wfile_info.size = 42
wfile_info.time = 43
wfile_info.compression = None
assert wfile_info.relpath() == ("0000000000000000/000000000000000000000001")
# mock a server object
server = mock.Mock(name="server")
server.config.wals_directory = "/tmp/wals"
# parse the string
info_file = wfile_info.from_xlogdb_line("000000000000000000000001\t42\t43\tNone\n")
assert list(wfile_info.items()) == list(info_file.items())
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived via
streaming replication (in the 'streaming' directory)
This method always leaves one file in the "streaming" directory,
because the 'pg_receivexlog' process needs at least one file to
detect the current streaming position after a restart.
:return: WalArchiverBatch: list of WAL files
"""
# List and sort all files in the incoming directory
file_names = glob(os.path.join(
self.config.streaming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file,
# including partial ones.
# Anything else is treated like an error/anomaly
files = []
skip = []
errors = []
for file_name in file_names:
if xlog.is_wal_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
elif xlog.is_partial_file(file_name) and os.path.isfile(file_name):
skip.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as errors
if len(skip) > 1:
errors.extend(skip[:-1])
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
return WalArchiverBatch(wal_files, errors=errors, skip=skip)
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived via
streaming replication (in the 'streaming' directory)
This method always leaves one file in the "streaming" directory,
because the 'pg_receivexlog' process needs at least one file to
detect the current streaming position after a restart.
:return: WalArchiverBatch: list of WAL files
"""
# List and sort all files in the incoming directory
file_names = glob(
os.path.join(self.config.streaming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file,
# including partial ones.
# Anything else is treated like an error/anomaly
files = []
skip = []
errors = []
for file_name in file_names:
if xlog.is_wal_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
elif xlog.is_partial_file(file_name) and os.path.isfile(file_name):
skip.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as errors
if len(skip) > 1:
errors.extend(skip[:-1])
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
return WalArchiverBatch(wal_files, errors=errors, skip=skip)
def build_backup_manager(
server=None, name=None, config=None, global_conf=None, main_conf=None
):
"""
Instantiate a BackupManager object using mocked parameters
The compression_manager member is mocked
:param barman.server.Server|None server: Optional Server object
:rtype: barman.backup.BackupManager
"""
if server is None:
server = build_mocked_server(name, config, global_conf, main_conf)
with mock.patch("barman.backup.CompressionManager"):
manager = BackupManager(server=server)
manager.compression_manager.unidentified_compression = None
manager.compression_manager.get_wal_file_info.side_effect = (
lambda filename: WalFileInfo.from_file(filename, manager.compression_manager)
)
server.backup_manager = manager
return manager
def remove_wal_before_backup(self, backup_info):
"""
Remove WAL files which have been archived before the start of
the provided backup.
If no backup_info is provided delete all available WAL files
:param BackupInfo|None backup_info: the backup information structure
:return list: a list of removed WAL files
"""
removed = []
with self.server.xlogdb() as fxlogdb:
xlogdb_new = fxlogdb.name + ".new"
with open(xlogdb_new, 'w') as fxlogdb_new:
for line in fxlogdb:
wal_info = WalFileInfo.from_xlogdb_line(line)
if not xlog.is_any_xlog_file(wal_info.name):
output.error(
"invalid xlog segment name %r\n"
"HINT: Please run \"barman rebuild-xlogdb %s\" "
"to solve this issue", wal_info.name,
self.config.name)
continue
# Keeps the WAL segment if it is a history file or later
# than the given backup (the first available)
if (xlog.is_history_file(wal_info.name)
or (backup_info
and wal_info.name >= backup_info.begin_wal)):
fxlogdb_new.write(wal_info.to_xlogdb_line())
continue
else:
self.delete_wal(wal_info)
removed.append(wal_info.name)
fxlogdb_new.flush()
os.fsync(fxlogdb_new.fileno())
shutil.move(xlogdb_new, fxlogdb.name)
fsync_dir(os.path.dirname(fxlogdb.name))
return removed
def test_archive_wal(self, tmpdir, capsys):
"""
Test WalArchiver.archive_wal behaviour when the WAL file already
exists in the archive
"""
# Setup the test environment
backup_manager = build_backup_manager(
name="TestServer", global_conf={"barman_home": tmpdir.strpath}
)
backup_manager.compression_manager.get_compressor.return_value = None
backup_manager.server.get_backup.return_value = None
basedir = tmpdir.join("main")
incoming_dir = basedir.join("incoming")
archive_dir = basedir.join("wals")
xlog_db = archive_dir.join("xlog.db")
wal_name = "000000010000000000000001"
wal_file = incoming_dir.join(wal_name)
wal_file.ensure()
archive_dir.ensure(dir=True)
xlog_db.ensure()
backup_manager.server.xlogdb.return_value.__enter__.return_value = xlog_db.open(
mode="a"
)
archiver = FileWalArchiver(backup_manager)
backup_manager.server.archivers = [archiver]
# Tests a basic archival process
wal_info = WalFileInfo.from_file(wal_file.strpath)
archiver.archive_wal(None, wal_info)
assert not os.path.exists(wal_file.strpath)
assert os.path.exists(wal_info.fullpath(backup_manager.server))
# Tests the archiver behaviour for duplicate WAL files, as the
# wal file named '000000010000000000000001' was already archived
# in the previous test
wal_file.ensure()
wal_info = WalFileInfo.from_file(wal_file.strpath)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Tests the archiver behaviour for duplicated WAL files with
# different contents
wal_file.write("test")
wal_info = WalFileInfo.from_file(wal_file.strpath)
with pytest.raises(DuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Tests the archiver behaviour for duplicate WAL files, as the
# wal file named '000000010000000000000001' was already archived
# in the previous test and the input file uses compression
compressor = PyGZipCompressor(backup_manager.config, "pygzip")
compressor.compress(wal_file.strpath, wal_file.strpath)
wal_info = WalFileInfo.from_file(wal_file.strpath)
assert os.path.exists(wal_file.strpath)
backup_manager.compression_manager.get_compressor.return_value = compressor
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Test the archiver behaviour when the incoming file is compressed
# and it has been already archived and compressed.
compressor.compress(
wal_info.fullpath(backup_manager.server),
wal_info.fullpath(backup_manager.server),
)
wal_info = WalFileInfo.from_file(wal_file.strpath)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Reset the status of the incoming and WALs directory
# removing the files archived during the preceding tests.
os.unlink(wal_info.fullpath(backup_manager.server))
os.unlink(wal_file.strpath)
# Test the archival of a WAL file using compression.
wal_file.write("test")
wal_info = WalFileInfo.from_file(wal_file.strpath)
archiver.archive_wal(compressor, wal_info)
assert os.path.exists(wal_info.fullpath(backup_manager.server))
assert not os.path.exists(wal_file.strpath)
assert "gzip" == identify_compression(wal_info.fullpath(backup_manager.server))
def test_archive(
self,
datetime_mock,
move_mock,
archive_wal_mock,
get_next_batch_mock,
unlink_mock,
capsys,
caplog,
):
"""
Test FileWalArchiver.archive method
"""
# See all logs
caplog.set_level(0)
fxlogdb_mock = MagicMock()
backup_manager = MagicMock()
archiver = FileWalArchiver(backup_manager)
archiver.config.name = "test_server"
archiver.config.errors_directory = "/server/errors"
wal_info = WalFileInfo(name="test_wal_file")
wal_info.orig_filename = "test_wal_file"
batch = WalArchiverQueue([wal_info])
assert batch.size == 1
assert batch.run_size == 1
get_next_batch_mock.return_value = batch
archive_wal_mock.side_effect = DuplicateWalFile
datetime_mock.utcnow.return_value.strftime.return_value = "test_time"
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert (
"\tError: %s is already present in server %s. "
"File moved to errors directory." % (wal_info.name, archiver.config.name)
) in out
assert (
"\tError: %s is already present in server %s. "
"File moved to errors directory." % (wal_info.name, archiver.config.name)
) in caplog.text
archive_wal_mock.side_effect = MatchingDuplicateWalFile
archiver.archive(fxlogdb_mock)
unlink_mock.assert_called_with(wal_info.orig_filename)
# Test batch errors
caplog_reset(caplog)
batch.errors = ["testfile_1", "testfile_2"]
archive_wal_mock.side_effect = DuplicateWalFile
archiver.archive(fxlogdb_mock)
out, err = capsys.readouterr()
assert (
"Some unknown objects have been found while "
"processing xlog segments for %s. "
"Objects moved to errors directory:" % archiver.config.name
) in out
assert (
"Archiver is about to move %s unexpected file(s) to errors "
"directory for %s from %s"
% (len(batch.errors), archiver.config.name, archiver.name)
) in caplog.text
assert (
"Moving unexpected file for %s from %s: %s"
% (archiver.config.name, archiver.name, "testfile_1")
) in caplog.text
assert (
"Moving unexpected file for %s from %s: %s"
% (archiver.config.name, archiver.name, "testfile_2")
) in caplog.text
move_mock.assert_any_call(
"testfile_1",
os.path.join(
archiver.config.errors_directory,
"%s.%s.unknown" % ("testfile_1", "test_time"),
),
)
move_mock.assert_any_call(
"testfile_2",
os.path.join(
archiver.config.errors_directory,
"%s.%s.unknown" % ("testfile_2", "test_time"),
),
)
def archive_wal(self, compressor, wal_info):
"""
Archive a WAL segment and update the wal_info object
:param compressor: the compressor for the file (if any)
:param WalFileInfo wal_info: the WAL file is being processed
"""
src_file = wal_info.orig_filename
src_dir = os.path.dirname(src_file)
dst_file = wal_info.fullpath(self.server)
tmp_file = dst_file + '.tmp'
dst_dir = os.path.dirname(dst_file)
error = None
try:
# Run the pre_archive_script if present.
script = HookScriptRunner(self.backup_manager,
'archive_script', 'pre')
script.env_from_wal_info(wal_info, src_file)
script.run()
# Run the pre_archive_retry_script if present.
retry_script = RetryHookScriptRunner(self.backup_manager,
'archive_retry_script',
'pre')
retry_script.env_from_wal_info(wal_info, src_file)
retry_script.run()
# Check if destination already exists
if os.path.exists(dst_file):
src_uncompressed = src_file
dst_uncompressed = dst_file
dst_info = WalFileInfo.from_file(dst_file)
try:
comp_manager = self.backup_manager.compression_manager
if dst_info.compression is not None:
dst_uncompressed = dst_file + '.uncompressed'
comp_manager.get_compressor(
compression=dst_info.compression).decompress(
dst_file, dst_uncompressed)
if wal_info.compression:
src_uncompressed = src_file + '.uncompressed'
comp_manager.get_compressor(
compression=wal_info.compression).decompress(
src_file, src_uncompressed)
# Directly compare files.
# When the files are identical
# raise a MatchingDuplicateWalFile exception,
# otherwise raise a DuplicateWalFile exception.
if filecmp.cmp(dst_uncompressed, src_uncompressed):
raise MatchingDuplicateWalFile(wal_info)
else:
raise DuplicateWalFile(wal_info)
finally:
if src_uncompressed != src_file:
os.unlink(src_uncompressed)
if dst_uncompressed != dst_file:
os.unlink(dst_uncompressed)
mkpath(dst_dir)
# Compress the file only if not already compressed
if compressor and not wal_info.compression:
compressor.compress(src_file, tmp_file)
shutil.copystat(src_file, tmp_file)
os.rename(tmp_file, dst_file)
os.unlink(src_file)
# Update wal_info
stat = os.stat(dst_file)
wal_info.size = stat.st_size
wal_info.compression = compressor.compression
else:
# Try to atomically rename the file. If successful,
# the renaming will be an atomic operation
# (this is a POSIX requirement).
try:
os.rename(src_file, dst_file)
except OSError:
# Source and destination are probably on different
# filesystems
shutil.copy2(src_file, tmp_file)
os.rename(tmp_file, dst_file)
os.unlink(src_file)
# At this point the original file has been removed
wal_info.orig_filename = None
# Execute fsync() on the archived WAL file
file_fd = os.open(dst_file, os.O_RDONLY)
os.fsync(file_fd)
os.close(file_fd)
# Execute fsync() on the archived WAL containing directory
fsync_dir(dst_dir)
# Execute fsync() also on the incoming directory
fsync_dir(src_dir)
except Exception as e:
# In case of failure save the exception for the post scripts
error = e
raise
# Ensure the execution of the post_archive_retry_script and
# the post_archive_script
finally:
# Run the post_archive_retry_script if present.
try:
retry_script = RetryHookScriptRunner(self,
'archive_retry_script',
'post')
retry_script.env_from_wal_info(wal_info, dst_file, error)
retry_script.run()
except AbortedRetryHookScript as e:
# Ignore the ABORT_STOP as it is a post-hook operation
_logger.warning("Ignoring stop request after receiving "
"abort (exit code %d) from post-archive "
"retry hook script: %s",
e.hook.exit_status, e.hook.script)
# Run the post_archive_script if present.
script = HookScriptRunner(self, 'archive_script', 'post', error)
script.env_from_wal_info(wal_info, dst_file)
script.run()
def test_decode_history_file(self, tmpdir):
compressor = mock.Mock()
# Regular history file
p = tmpdir.join('00000002.history')
p.write('1\t2/83000168\tat restore point "myrp"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=2,
parent_tli=1,
reason='at restore point "myrp"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# Comments must be skipped
p = tmpdir.join('00000003.history')
p.write('# Comment\n1\t2/83000168\tat restore point "testcomment"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=3,
parent_tli=1,
reason='at restore point "testcomment"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# History file with comments and empty lines
p = tmpdir.join('00000004.history')
p.write('# Comment\n\n1\t2/83000168\ttesting "testemptyline"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=4,
parent_tli=1,
reason='testing "testemptyline"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# Test compression handling Fix for bug #66 on github
config_mock = mock.Mock()
config_mock.compression = "gzip"
# check custom compression method creation
comp_manager = CompressionManager(config_mock, None)
u = tmpdir.join('00000005.uncompressed')
p = tmpdir.join('00000005.history')
u.write('1\t2/83000168\tat restore point "myrp"\n')
result = xlog.HistoryFileData(
tli=5,
parent_tli=1,
reason='at restore point "myrp"',
switchpoint=0x283000168)
comp_manager.get_compressor('gzip').compress(u.strpath,
p.strpath)
wal_info = WalFileInfo.from_file(p.strpath)
assert xlog.decode_history_file(wal_info, comp_manager) == [result]
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Empty file
p.write('')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Missing field
p.write('1\t2/83000168')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Unattended field
p.write('1\t2/83000168\tat restore point "myrp"\ttest')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
def test_get_wal_until_next_backup(self, get_backup_mock, tmpdir):
"""
Simple test for the management of .history files
"""
# build a WalFileInfo object
wfile_info = WalFileInfo()
wfile_info.name = '000000010000000000000003'
wfile_info.size = 42
wfile_info.time = 43
wfile_info.compression = None
# build a WalFileInfo history object
history_info = WalFileInfo()
history_info.name = '00000001.history'
history_info.size = 42
history_info.time = 43
history_info.compression = None
# create a xlog.db and add the 2 entries
wals_dir = tmpdir.mkdir("wals")
xlog = wals_dir.join("xlog.db")
xlog.write(wfile_info.to_xlogdb_line() + history_info.to_xlogdb_line())
# fake backup
backup = build_test_backup_info(
begin_wal='000000010000000000000001',
end_wal='000000010000000000000004')
# mock a server object and mock a return call to get_next_backup method
server = build_real_server(
global_conf={
"barman_lock_directory": tmpdir.mkdir('lock').strpath
},
main_conf={
"wals_directory": wals_dir.strpath
})
get_backup_mock.return_value = build_test_backup_info(
backup_id="1234567899",
begin_wal='000000010000000000000005',
end_wal='000000010000000000000009')
wals = []
for wal_file in server.get_wal_until_next_backup(backup,
include_history=True):
# get the result of the xlogdb read
wals.append(wal_file.name)
# check for the presence of the .history file
assert history_info.name in wals
def archive_wal(self, compressor, wal_info):
"""
Archive a WAL segment and update the wal_info object
:param compressor: the compressor for the file (if any)
:param WalFileInfo wal_info: the WAL file is being processed
"""
src_file = wal_info.orig_filename
src_dir = os.path.dirname(src_file)
dst_file = wal_info.fullpath(self.server)
tmp_file = dst_file + '.tmp'
dst_dir = os.path.dirname(dst_file)
error = None
try:
# Run the pre_archive_script if present.
script = HookScriptRunner(self.backup_manager, 'archive_script',
'pre')
script.env_from_wal_info(wal_info, src_file)
script.run()
# Run the pre_archive_retry_script if present.
retry_script = RetryHookScriptRunner(self.backup_manager,
'archive_retry_script', 'pre')
retry_script.env_from_wal_info(wal_info, src_file)
retry_script.run()
# Check if destination already exists
if os.path.exists(dst_file):
src_uncompressed = src_file
dst_uncompressed = dst_file
dst_info = WalFileInfo.from_file(dst_file)
try:
comp_manager = self.backup_manager.compression_manager
if dst_info.compression is not None:
dst_uncompressed = dst_file + '.uncompressed'
comp_manager.get_compressor(
compression=dst_info.compression).decompress(
dst_file, dst_uncompressed)
if wal_info.compression:
src_uncompressed = src_file + '.uncompressed'
comp_manager.get_compressor(
compression=wal_info.compression).decompress(
src_file, src_uncompressed)
# Directly compare files.
# When the files are identical
# raise a MatchingDuplicateWalFile exception,
# otherwise raise a DuplicateWalFile exception.
if filecmp.cmp(dst_uncompressed, src_uncompressed):
raise MatchingDuplicateWalFile(wal_info)
else:
raise DuplicateWalFile(wal_info)
finally:
if src_uncompressed != src_file:
os.unlink(src_uncompressed)
if dst_uncompressed != dst_file:
os.unlink(dst_uncompressed)
mkpath(dst_dir)
# Compress the file only if not already compressed
if compressor and not wal_info.compression:
compressor.compress(src_file, tmp_file)
shutil.copystat(src_file, tmp_file)
os.rename(tmp_file, dst_file)
os.unlink(src_file)
# Update wal_info
stat = os.stat(dst_file)
wal_info.size = stat.st_size
wal_info.compression = compressor.compression
else:
# Try to atomically rename the file. If successful,
# the renaming will be an atomic operation
# (this is a POSIX requirement).
try:
os.rename(src_file, dst_file)
except OSError:
# Source and destination are probably on different
# filesystems
shutil.copy2(src_file, tmp_file)
os.rename(tmp_file, dst_file)
os.unlink(src_file)
# At this point the original file has been removed
wal_info.orig_filename = None
# Execute fsync() on the archived WAL file
file_fd = os.open(dst_file, os.O_RDONLY)
os.fsync(file_fd)
os.close(file_fd)
# Execute fsync() on the archived WAL containing directory
fsync_dir(dst_dir)
# Execute fsync() also on the incoming directory
fsync_dir(src_dir)
except Exception as e:
# In case of failure save the exception for the post scripts
error = e
raise
# Ensure the execution of the post_archive_retry_script and
# the post_archive_script
finally:
# Run the post_archive_retry_script if present.
try:
retry_script = RetryHookScriptRunner(self,
'archive_retry_script',
'post')
retry_script.env_from_wal_info(wal_info, dst_file, error)
retry_script.run()
except AbortedRetryHookScript as e:
# Ignore the ABORT_STOP as it is a post-hook operation
_logger.warning(
"Ignoring stop request after receiving "
"abort (exit code %d) from post-archive "
"retry hook script: %s", e.hook.exit_status, e.hook.script)
# Run the post_archive_script if present.
script = HookScriptRunner(self, 'archive_script', 'post', error)
script.env_from_wal_info(wal_info, dst_file)
script.run()
def rebuild_xlogdb(self):
"""
Rebuild the whole xlog database guessing it from the archive content.
"""
from os.path import isdir, join
output.info("Rebuilding xlogdb for server %s", self.config.name)
root = self.config.wals_directory
default_compression = self.config.compression
wal_count = label_count = history_count = 0
# lock the xlogdb as we are about replacing it completely
with self.server.xlogdb('w') as fxlogdb:
xlogdb_new = fxlogdb.name + ".new"
with open(xlogdb_new, 'w') as fxlogdb_new:
for name in sorted(os.listdir(root)):
# ignore the xlogdb and its lockfile
if name.startswith(self.server.XLOG_DB):
continue
fullname = join(root, name)
if isdir(fullname):
# all relevant files are in subdirectories
hash_dir = fullname
for wal_name in sorted(os.listdir(hash_dir)):
fullname = join(hash_dir, wal_name)
if isdir(fullname):
_logger.warning(
'unexpected directory '
'rebuilding the wal database: %s',
fullname)
else:
if xlog.is_wal_file(fullname):
wal_count += 1
elif xlog.is_backup_file(fullname):
label_count += 1
elif fullname.endswith('.tmp'):
_logger.warning(
'temporary file found '
'rebuilding the wal database: %s',
fullname)
continue
else:
_logger.warning(
'unexpected file '
'rebuilding the wal database: %s',
fullname)
continue
wal_info = WalFileInfo.from_file(
fullname,
default_compression=default_compression)
fxlogdb_new.write(wal_info.to_xlogdb_line())
else:
# only history files are here
if xlog.is_history_file(fullname):
history_count += 1
wal_info = WalFileInfo.from_file(
fullname,
default_compression=default_compression)
fxlogdb_new.write(wal_info.to_xlogdb_line())
else:
_logger.warning(
'unexpected file '
'rebuilding the wal database: %s', fullname)
os.fsync(fxlogdb_new.fileno())
shutil.move(xlogdb_new, fxlogdb.name)
fsync_dir(os.path.dirname(fxlogdb.name))
output.info(
'Done rebuilding xlogdb for server %s '
'(history: %s, backup_labels: %s, wal_file: %s)', self.config.name,
history_count, label_count, wal_count)
def test_recover_xlog(self, rsync_pg_mock, cm_mock, tmpdir):
"""
Test the recovery of the xlogs of a backup
:param rsync_pg_mock: Mock rsync object for the purpose if this test
"""
# Build basic folders/files structure
dest = tmpdir.mkdir('destination')
wals = tmpdir.mkdir('wals')
# Create 3 WAL files with different compressions
xlog_dir = wals.mkdir(xlog.hash_dir('000000000000000000000002'))
xlog_plain = xlog_dir.join('000000000000000000000001')
xlog_gz = xlog_dir.join('000000000000000000000002')
xlog_bz2 = xlog_dir.join('000000000000000000000003')
xlog_plain.write('dummy content')
xlog_gz.write('dummy content gz')
xlog_bz2.write('dummy content bz2')
server = testing_helpers.build_real_server(
main_conf={'wals_directory': wals.strpath})
# Prepare compressors mock
c = {
'gzip': Mock(name='gzip'),
'bzip2': Mock(name='bzip2'),
}
cm_mock.return_value.get_compressor = \
lambda compression=None, path=None: c[compression]
# touch destination files to avoid errors on cleanup
c['gzip'].decompress.side_effect = lambda src, dst: open(dst, 'w')
c['bzip2'].decompress.side_effect = lambda src, dst: open(dst, 'w')
# Build executor
executor = RecoveryExecutor(server.backup_manager)
# Test: local copy
required_wals = (
WalFileInfo.from_xlogdb_line(
'000000000000000000000001\t42\t43\tNone\n'),
WalFileInfo.from_xlogdb_line(
'000000000000000000000002\t42\t43\tgzip\n'),
WalFileInfo.from_xlogdb_line(
'000000000000000000000003\t42\t43\tbzip2\n'),
)
executor.xlog_copy(required_wals, dest.strpath, None)
# Check for a correct invocation of rsync using local paths
rsync_pg_mock.assert_called_once_with(
network_compression=False,
bwlimit=None, path=None,
ssh=None)
assert not rsync_pg_mock.return_value.from_file_list.called
c['gzip'].decompress.assert_called_once_with(xlog_gz.strpath, ANY)
c['bzip2'].decompress.assert_called_once_with(xlog_bz2.strpath, ANY)
# Reset mock calls
rsync_pg_mock.reset_mock()
c['gzip'].reset_mock()
c['bzip2'].reset_mock()
# Test: remote copy
executor.xlog_copy(required_wals, dest.strpath, 'remote_command')
# Check for the invocation of rsync on a remote call
rsync_pg_mock.assert_called_once_with(
network_compression=False,
bwlimit=None, path=ANY,
ssh='remote_command')
rsync_pg_mock.return_value.from_file_list.assert_called_once_with(
[
'000000000000000000000001',
'000000000000000000000002',
'000000000000000000000003'],
ANY,
ANY)
c['gzip'].decompress.assert_called_once_with(xlog_gz.strpath, ANY)
c['bzip2'].decompress.assert_called_once_with(xlog_bz2.strpath, ANY)
def test_archive_wal(self, tmpdir, capsys):
"""
Test WalArchiver.archive_wal behaviour when the WAL file already
exists in the archive
"""
# Setup the test environment
backup_manager = build_backup_manager(
name='TestServer',
global_conf={
'barman_home': tmpdir.strpath
})
backup_manager.compression_manager.get_compressor.return_value = None
backup_manager.server.get_backup.return_value = None
basedir = tmpdir.join('main')
incoming_dir = basedir.join('incoming')
archive_dir = basedir.join('wals')
xlog_db = archive_dir.join('xlog.db')
wal_name = '000000010000000000000001'
wal_file = incoming_dir.join(wal_name)
wal_file.ensure()
archive_dir.ensure(dir=True)
xlog_db.ensure()
backup_manager.server.xlogdb.return_value.__enter__.return_value = (
xlog_db.open(mode='a'))
archiver = FileWalArchiver(backup_manager)
backup_manager.server.archivers = [archiver]
# Tests a basic archival process
wal_info = WalFileInfo.from_file(wal_file.strpath)
archiver.archive_wal(None, wal_info)
assert not os.path.exists(wal_file.strpath)
assert os.path.exists(wal_info.fullpath(backup_manager.server))
# Tests the archiver behaviour for duplicate WAL files, as the
# wal file named '000000010000000000000001' was already archived
# in the previous test
wal_file.ensure()
wal_info = WalFileInfo.from_file(wal_file.strpath)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Tests the archiver behaviour for duplicated WAL files with
# different contents
wal_file.write('test')
wal_info = WalFileInfo.from_file(wal_file.strpath)
with pytest.raises(DuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Tests the archiver behaviour for duplicate WAL files, as the
# wal file named '000000010000000000000001' was already archived
# in the previous test and the input file uses compression
compressor = PyGZipCompressor(backup_manager.config, 'pygzip')
compressor.compress(wal_file.strpath, wal_file.strpath)
wal_info = WalFileInfo.from_file(wal_file.strpath)
assert os.path.exists(wal_file.strpath)
backup_manager.compression_manager.get_compressor.return_value = (
compressor)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Test the archiver behaviour when the incoming file is compressed
# and it has been already archived and compressed.
compressor.compress(wal_info.fullpath(backup_manager.server),
wal_info.fullpath(backup_manager.server))
wal_info = WalFileInfo.from_file(wal_file.strpath)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Reset the status of the incoming and WALs directory
# removing the files archived during the preceding tests.
os.unlink(wal_info.fullpath(backup_manager.server))
os.unlink(wal_file.strpath)
# Test the archival of a WAL file using compression.
wal_file.write('test')
wal_info = WalFileInfo.from_file(wal_file.strpath)
archiver.archive_wal(compressor, wal_info)
assert os.path.exists(wal_info.fullpath(backup_manager.server))
assert not os.path.exists(wal_file.strpath)
assert 'gzip' == identify_compression(
wal_info.fullpath(backup_manager.server)
)
def test_decode_history_file(self, tmpdir):
compressor = mock.Mock()
# Regular history file
p = tmpdir.join('00000002.history')
p.write('1\t2/83000168\tat restore point "myrp"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=2,
parent_tli=1,
reason='at restore point "myrp"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# Comments must be skipped
p = tmpdir.join('00000003.history')
p.write('# Comment\n1\t2/83000168\tat restore point "testcomment"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=3,
parent_tli=1,
reason='at restore point "testcomment"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# History file with comments and empty lines
p = tmpdir.join('00000004.history')
p.write('# Comment\n\n1\t2/83000168\ttesting "testemptyline"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=4,
parent_tli=1,
reason='testing "testemptyline"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# Test compression handling Fix for bug #66 on github
config_mock = mock.Mock()
config_mock.compression = "gzip"
# check custom compression method creation
comp_manager = CompressionManager(config_mock, None)
u = tmpdir.join('00000005.uncompressed')
p = tmpdir.join('00000005.history')
u.write('1\t2/83000168\tat restore point "myrp"\n')
result = xlog.HistoryFileData(
tli=5,
parent_tli=1,
reason='at restore point "myrp"',
switchpoint=0x283000168)
comp_manager.get_compressor('gzip').compress(u.strpath,
p.strpath)
wal_info = WalFileInfo.from_file(p.strpath)
assert xlog.decode_history_file(wal_info, comp_manager) == [result]
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Empty file
p.write('')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Missing field
p.write('1\t2/83000168')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Unattended field
p.write('1\t2/83000168\tat restore point "myrp"\ttest')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
def archive_wal(self, verbose=True):
"""
Executes WAL maintenance operations, such as archiving and compression
If verbose is set to False, outputs something only if there is
at least one file
:param bool verbose: report even if no actions
"""
found = False
compressor = self.compression_manager.get_compressor()
with self.server.xlogdb('a') as fxlogdb:
if verbose:
output.info("Processing xlog segments for %s",
self.config.name,
log=False)
# Get the first available backup
first_backup_id = self.get_first_backup(BackupInfo.STATUS_NOT_EMPTY)
first_backup = self.server.get_backup(first_backup_id)
for filename in sorted(glob(
os.path.join(self.config.incoming_wals_directory, '*'))):
if not found and not verbose:
output.info("Processing xlog segments for %s",
self.config.name,
log=False)
found = True
# Create WAL Info object
wal_info = WalFileInfo.from_file(filename, compression=None)
# If there are no available backups ...
if first_backup is None:
# ... delete xlog segments only for exclusive backups
if BackupOptions.CONCURRENT_BACKUP \
not in self.config.backup_options:
# Skipping history files
if not xlog.is_history_file(filename):
output.info("\tNo base backup available."
" Trashing file %s"
" from server %s",
wal_info.name, self.config.name)
os.unlink(filename)
continue
# ... otherwise
else:
# ... delete xlog segments older than the first backup
if wal_info.name < first_backup.begin_wal:
# Skipping history files
if not xlog.is_history_file(filename):
output.info("\tOlder than first backup."
" Trashing file %s"
" from server %s",
wal_info.name, self.config.name)
os.unlink(filename)
continue
# Report to the user the WAL file we are archiving
output.info("\t%s", os.path.basename(filename), log=False)
_logger.info("Archiving %s/%s",
self.config.name,
os.path.basename(filename))
# Archive the WAL file
try:
self.cron_wal_archival(compressor, wal_info)
except AbortedRetryHookScript as e:
_logger.warning("Archiving of %s/%s aborted by "
"pre_archive_retry_script."
"Reason: %s" % (self.config.name,
os.path.basename(),
e))
return
# Updates the information of the WAL archive with
# the latest segments
fxlogdb.write(wal_info.to_xlogdb_line())
# flush and fsync for every line
fxlogdb.flush()
os.fsync(fxlogdb.fileno())
if not found and verbose:
output.info("\tno file found", log=False)
def rebuild_xlogdb(self):
"""
Rebuild the whole xlog database guessing it from the archive content.
"""
from os.path import isdir, join
output.info("Rebuilding xlogdb for server %s", self.config.name)
root = self.config.wals_directory
default_compression = self.config.compression
wal_count = label_count = history_count = 0
# lock the xlogdb as we are about replacing it completely
with self.server.xlogdb('w') as fxlogdb:
xlogdb_new = fxlogdb.name + ".new"
with open(xlogdb_new, 'w') as fxlogdb_new:
for name in sorted(os.listdir(root)):
# ignore the xlogdb and its lockfile
if name.startswith(self.server.XLOG_DB):
continue
fullname = join(root, name)
if isdir(fullname):
# all relevant files are in subdirectories
hash_dir = fullname
for wal_name in sorted(os.listdir(hash_dir)):
fullname = join(hash_dir, wal_name)
if isdir(fullname):
_logger.warning(
'unexpected directory '
'rebuilding the wal database: %s',
fullname)
else:
if xlog.is_wal_file(fullname):
wal_count += 1
elif xlog.is_backup_file(fullname):
label_count += 1
else:
_logger.warning(
'unexpected file '
'rebuilding the wal database: %s',
fullname)
continue
wal_info = WalFileInfo.from_file(
fullname,
default_compression=default_compression)
fxlogdb_new.write(wal_info.to_xlogdb_line())
else:
# only history files are here
if xlog.is_history_file(fullname):
history_count += 1
wal_info = WalFileInfo.from_file(
fullname,
default_compression=default_compression)
fxlogdb_new.write(wal_info.to_xlogdb_line())
else:
_logger.warning(
'unexpected file '
'rebuilding the wal database: %s',
fullname)
os.fsync(fxlogdb_new.fileno())
shutil.move(xlogdb_new, fxlogdb.name)
fsync_dir(os.path.dirname(fxlogdb.name))
output.info('Done rebuilding xlogdb for server %s '
'(history: %s, backup_labels: %s, wal_file: %s)',
self.config.name, history_count, label_count, wal_count)
def test_recover_xlog(self, rsync_pg_mock, cm_mock, tmpdir):
"""
Test the recovery of the xlogs of a backup
:param rsync_pg_mock: Mock rsync object for the purpose if this test
"""
# Build basic folders/files structure
dest = tmpdir.mkdir('destination')
wals = tmpdir.mkdir('wals')
# Create 3 WAL files with different compressions
xlog_dir = wals.mkdir(xlog.hash_dir('000000000000000000000002'))
xlog_plain = xlog_dir.join('000000000000000000000001')
xlog_gz = xlog_dir.join('000000000000000000000002')
xlog_bz2 = xlog_dir.join('000000000000000000000003')
xlog_plain.write('dummy content')
xlog_gz.write('dummy content gz')
xlog_bz2.write('dummy content bz2')
server = testing_helpers.build_real_server(
main_conf={'wals_directory': wals.strpath})
# Prepare compressors mock
c = {
'gzip': mock.Mock(name='gzip'),
'bzip2': mock.Mock(name='bzip2'),
}
cm_mock.return_value.get_compressor = \
lambda compression=None, path=None: c[compression]
# touch destination files to avoid errors on cleanup
c['gzip'].decompress.side_effect = lambda src, dst: open(dst, 'w')
c['bzip2'].decompress.side_effect = lambda src, dst: open(dst, 'w')
# Build executor
executor = RecoveryExecutor(server.backup_manager)
# Test: local copy
required_wals = (
WalFileInfo.from_xlogdb_line(
'000000000000000000000001\t42\t43\tNone\n'),
WalFileInfo.from_xlogdb_line(
'000000000000000000000002\t42\t43\tgzip\n'),
WalFileInfo.from_xlogdb_line(
'000000000000000000000003\t42\t43\tbzip2\n'),
)
executor._xlog_copy(required_wals, dest.strpath, None)
# Check for a correct invocation of rsync using local paths
rsync_pg_mock.assert_called_once_with(network_compression=False,
bwlimit=None,
path=None,
ssh=None)
assert not rsync_pg_mock.return_value.from_file_list.called
c['gzip'].decompress.assert_called_once_with(xlog_gz.strpath, mock.ANY)
c['bzip2'].decompress.assert_called_once_with(xlog_bz2.strpath,
mock.ANY)
# Reset mock calls
rsync_pg_mock.reset_mock()
c['gzip'].reset_mock()
c['bzip2'].reset_mock()
# Test: remote copy
executor._xlog_copy(required_wals, dest.strpath, 'remote_command')
# Check for the invocation of rsync on a remote call
rsync_pg_mock.assert_called_once_with(network_compression=False,
bwlimit=None,
path=mock.ANY,
ssh='remote_command')
rsync_pg_mock.return_value.from_file_list.assert_called_once_with([
'000000000000000000000001', '000000000000000000000002',
'000000000000000000000003'
], mock.ANY, mock.ANY)
c['gzip'].decompress.assert_called_once_with(xlog_gz.strpath, mock.ANY)
c['bzip2'].decompress.assert_called_once_with(xlog_bz2.strpath,
mock.ANY)
