def test_from_file_override(self, id_compression, tmpdir):
# prepare
id_compression.return_value = None
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write("dummy_content\n")
wfile_info = WalFileInfo.from_file(tmp_file.strpath, name="000000000000000000000002")
assert wfile_info.name == "000000000000000000000002"
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == ("0000000000000000/000000000000000000000002")
wfile_info = WalFileInfo.from_file(tmp_file.strpath, size=42)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == 42
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == ("0000000000000000/000000000000000000000001")
wfile_info = WalFileInfo.from_file(tmp_file.strpath, time=43)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == 43
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == ("0000000000000000/000000000000000000000001")
def test_from_file_override(self, id_compression, tmpdir):
# prepare
id_compression.return_value = None
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write('dummy_content\n')
wfile_info = WalFileInfo.from_file(tmp_file.strpath,
name="000000000000000000000002")
assert wfile_info.name == '000000000000000000000002'
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000002')
wfile_info = WalFileInfo.from_file(tmp_file.strpath, size=42)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == 42
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000001')
wfile_info = WalFileInfo.from_file(tmp_file.strpath, time=43)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == 43
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000001')
def get_latest_archived_wal(self):
"""
Return the WalFileInfo of the last WAL file in the archive,
or None if the archive doesn't contain any WAL file.
:rtype: WalFileInfo|None
"""
# TODO: consider timeline?
from os.path import isdir, join
root = self.config.wals_directory
# If the WAL archive directory doesn't exists the archive is empty
if not isdir(root):
return None
# Traverse all the directory in the archive in reverse order,
# returning the first WAL file found
for name in sorted(os.listdir(root), reverse=True):
fullname = join(root, name)
# All relevant files are in subdirectories, so
# we skip any non-directory entry
if isdir(fullname):
hash_dir = fullname
# Inspect contained files in reverse order
for wal_name in sorted(os.listdir(hash_dir), reverse=True):
fullname = join(hash_dir, wal_name)
# Return the first file that has the correct name
if not isdir(fullname) and xlog.is_wal_file(fullname):
return WalFileInfo.from_file(fullname)
# If we get here, no WAL files have been found
return None
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived through
a PostgreSQL's 'archive_command' (in the 'incoming' directory)
:return: WalArchiverQueue: list of WAL files
"""
# Get the batch size from configuration (0 = unlimited)
batch_size = self.config.archiver_batch_size
# List and sort all files in the incoming directory
file_names = glob(
os.path.join(self.config.incoming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file. Anything
# else is treated like an error/anomaly
files = []
errors = []
for file_name in file_names:
if xlog.is_any_xlog_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
else:
errors.append(file_name)
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f) for f in files]
return WalArchiverQueue(wal_files,
batch_size=batch_size,
errors=errors)
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived through
a PostgreSQL's 'archive_command' (in the 'incoming' directory)
:return: WalArchiverBatch: list of WAL files
"""
# List and sort all files in the incoming directory
file_names = glob(os.path.join(
self.config.incoming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file. Anything
# else is treated like an error/anomaly
files = []
errors = []
for file_name in file_names:
if xlog.is_any_xlog_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
else:
errors.append(file_name)
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f) for f in files]
return WalArchiverBatch(wal_files, errors=errors)
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived via
streaming replication (in the 'streaming' directory)
This method always leaves one file in the "streaming" directory,
because the 'pg_receivexlog' process needs at least one file to
detect the current streaming position after a restart.
:return: WalArchiverQueue: list of WAL files
"""
# Get the batch size from configuration (0 = unlimited)
batch_size = self.config.streaming_archiver_batch_size
# List and sort all files in the incoming directory
file_names = glob(
os.path.join(self.config.streaming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file,
# including partial ones and history files.
# Anything else is treated like an error/anomaly
files = []
skip = []
errors = []
for file_name in file_names:
# Ignore temporary files
if file_name.endswith('.tmp'):
continue
# If the file doesn't exist, it has been renamed/removed while
# we were reading the directory. Ignore it.
if not os.path.exists(file_name):
continue
if not os.path.isfile(file_name):
errors.append(file_name)
elif xlog.is_partial_file(file_name):
skip.append(file_name)
elif xlog.is_any_xlog_file(file_name):
files.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as normal files
if len(skip) > 1:
partials = skip[:-1]
_logger.info('Archiving partial files for server %s: %s' %
(self.config.name, ", ".join(
[os.path.basename(f) for f in partials])))
files.extend(partials)
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
return WalArchiverQueue(wal_files,
batch_size=batch_size,
errors=errors,
skip=skip)
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived through
a PostgreSQL's 'archive_command' (in the 'incoming' directory)
:return: WalArchiverQueue: list of WAL files
"""
# Get the batch size from configuration (0 = unlimited)
batch_size = self.config.archiver_batch_size
# List and sort all files in the incoming directory
# IMPORTANT: the list is sorted, and this allows us to know that the
# WAL stream we have is monotonically increasing. That allows us to
# verify that a backup has all the WALs required for the restore.
file_names = glob(
os.path.join(self.config.incoming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file. Anything
# else is treated like an error/anomaly
files = []
errors = []
for file_name in file_names:
# Ignore temporary files
if file_name.endswith('.tmp'):
continue
if xlog.is_any_xlog_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
else:
errors.append(file_name)
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f) for f in files]
return WalArchiverQueue(wal_files,
batch_size=batch_size,
errors=errors)
def test_from_file_no_compression(self, tmpdir):
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write('dummy_content\n')
stat = os.stat(tmp_file.strpath)
wfile_info = WalFileInfo.from_file(tmp_file.strpath)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == stat.st_size
assert wfile_info.time == stat.st_mtime
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.relpath() == '0000000000000000/000000000000000000000001'
def test_from_file_no_compression(self, tmpdir):
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write('dummy_content\n')
stat = os.stat(tmp_file.strpath)
wfile_info = WalFileInfo.from_file(tmp_file.strpath)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == stat.st_size
assert wfile_info.time == stat.st_mtime
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000001')
def test_from_file_no_compression(self, mock_compression_manager, tmpdir):
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write("dummy_content\n")
stat = os.stat(tmp_file.strpath)
wfile_info = WalFileInfo.from_file(tmp_file.strpath,
mock_compression_manager)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == stat.st_size
assert wfile_info.time == stat.st_mtime
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.relpath() == (
"0000000000000000/000000000000000000000001")
def test_from_file_override(self, compression_manager, tmpdir):
# prepare
compression_manager.identify_compression.return_value = None
compression_manager.unidentified_compression = None
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write("dummy_content\n")
wfile_info = WalFileInfo.from_file(tmp_file.strpath,
compression_manager,
name="000000000000000000000002")
assert wfile_info.name == "000000000000000000000002"
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == (
"0000000000000000/000000000000000000000002")
wfile_info = WalFileInfo.from_file(tmp_file.strpath,
compression_manager,
size=42)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == 42
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == (
"0000000000000000/000000000000000000000001")
wfile_info = WalFileInfo.from_file(tmp_file.strpath,
compression_manager,
time=43)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == 43
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression is None
assert wfile_info.relpath() == (
"0000000000000000/000000000000000000000001")
def test_from_file_compression(self, id_compression, tmpdir):
# prepare
id_compression.return_value = 'test_compression'
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write('dummy_content\n')
wfile_info = WalFileInfo.from_file(tmp_file.strpath)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.compression == 'test_compression'
assert wfile_info.relpath() == '0000000000000000/000000000000000000000001'
def test_from_file_default_compression(self, id_compression, tmpdir):
# prepare
id_compression.return_value = None
tmp_file = tmpdir.join("00000001000000E500000064")
tmp_file.write("dummy_content\n")
wfile_info = WalFileInfo.from_file(tmp_file.strpath, default_compression="test_default_compression")
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression == "test_default_compression"
assert wfile_info.relpath() == ("00000001000000E5/00000001000000E500000064")
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived via
streaming replication (in the 'streaming' directory)
This method always leaves one file in the "streaming" directory,
because the 'pg_receivexlog' process needs at least one file to
detect the current streaming position after a restart.
:return: WalArchiverQueue: list of WAL files
"""
# Get the batch size from configuration (0 = unlimited)
batch_size = self.config.streaming_archiver_batch_size
# List and sort all files in the incoming directory
file_names = glob(os.path.join(
self.config.streaming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file,
# including partial ones and history files.
# Anything else is treated like an error/anomaly
files = []
skip = []
errors = []
for file_name in file_names:
if not os.path.isfile(file_name):
errors.append(file_name)
elif xlog.is_partial_file(file_name):
skip.append(file_name)
elif xlog.is_any_xlog_file(file_name):
files.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as errors
if len(skip) > 1:
errors.extend(skip[:-1])
_logger.warning('Multiple partial files found for server %s: %s' %
(self.config.name,
", ".join([os.path.basename(f) for f in errors])))
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
return WalArchiverQueue(wal_files,
batch_size=batch_size,
errors=errors, skip=skip)
def test_from_file_compression(self, id_compression, tmpdir):
# prepare
id_compression.return_value = 'test_compression'
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write('dummy_content\n')
wfile_info = WalFileInfo.from_file(tmp_file.strpath)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == '%s.meta' % tmp_file.strpath
assert wfile_info.compression == 'test_compression'
assert wfile_info.relpath() == (
'0000000000000000/000000000000000000000001')
def test_from_file_compression(self, mock_compression_manager, tmpdir):
# prepare
mock_compression_manager.identify_compression.return_value = "test_compression"
tmp_file = tmpdir.join("000000000000000000000001")
tmp_file.write("dummy_content\n")
wfile_info = WalFileInfo.from_file(tmp_file.strpath,
mock_compression_manager)
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression == "test_compression"
assert wfile_info.relpath() == (
"0000000000000000/000000000000000000000001")
def test_from_file_unidentified_compression(self, id_compression, tmpdir):
# prepare
id_compression.return_value = None
tmp_file = tmpdir.join("00000001000000E500000064")
tmp_file.write("dummy_content\n")
wfile_info = WalFileInfo.from_file(
tmp_file.strpath,
unidentified_compression="test_unidentified_compression")
assert wfile_info.name == tmp_file.basename
assert wfile_info.size == tmp_file.size()
assert wfile_info.time == tmp_file.mtime()
assert wfile_info.filename == "%s.meta" % tmp_file.strpath
assert wfile_info.compression == "test_unidentified_compression"
assert wfile_info.relpath() == (
"00000001000000E5/00000001000000E500000064")
def get_latest_archived_wals_info(self):
"""
Return a dictionary of timelines associated with the
WalFileInfo of the last WAL file in the archive,
or None if the archive doesn't contain any WAL file.
:rtype: dict[str, WalFileInfo]|None
"""
from os.path import isdir, join
root = self.config.wals_directory
# If the WAL archive directory doesn't exists the archive is empty
if not isdir(root):
return None
# Traverse all the directory in the archive in reverse order,
# returning the first WAL file found
timelines = {}
for name in sorted(os.listdir(root), reverse=True):
fullname = join(root, name)
# All relevant files are in subdirectories, so
# we skip any non-directory entry
if isdir(fullname):
# Extract the timeline. If it is not valid, skip this directory
try:
timeline = name[0:8]
int(timeline, 16)
except ValueError:
continue
# If this timeline already has a file, skip this directory
if timeline in timelines:
continue
hash_dir = fullname
# Inspect contained files in reverse order
for wal_name in sorted(os.listdir(hash_dir), reverse=True):
fullname = join(hash_dir, wal_name)
# Return the first file that has the correct name
if not isdir(fullname) and xlog.is_wal_file(fullname):
timelines[timeline] = WalFileInfo.from_file(fullname)
break
# Return the timeline map or None if it is empty
return timelines or None
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived via
streaming replication (in the 'streaming' directory)
This method always leaves one file in the "streaming" directory,
because the 'pg_receivexlog' process needs at least one file to
detect the current streaming position after a restart.
:return: WalArchiverBatch: list of WAL files
"""
# List and sort all files in the incoming directory
file_names = glob(os.path.join(
self.config.streaming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file,
# including partial ones.
# Anything else is treated like an error/anomaly
files = []
skip = []
errors = []
for file_name in file_names:
if xlog.is_wal_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
elif xlog.is_partial_file(file_name) and os.path.isfile(file_name):
skip.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as errors
if len(skip) > 1:
errors.extend(skip[:-1])
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
return WalArchiverBatch(wal_files, errors=errors, skip=skip)
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived via
streaming replication (in the 'streaming' directory)
This method always leaves one file in the "streaming" directory,
because the 'pg_receivexlog' process needs at least one file to
detect the current streaming position after a restart.
:return: WalArchiverBatch: list of WAL files
"""
# List and sort all files in the incoming directory
file_names = glob(
os.path.join(self.config.streaming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file,
# including partial ones.
# Anything else is treated like an error/anomaly
files = []
skip = []
errors = []
for file_name in file_names:
if xlog.is_wal_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
elif xlog.is_partial_file(file_name) and os.path.isfile(file_name):
skip.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as errors
if len(skip) > 1:
errors.extend(skip[:-1])
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
return WalArchiverBatch(wal_files, errors=errors, skip=skip)
def build_backup_manager(
server=None, name=None, config=None, global_conf=None, main_conf=None
):
"""
Instantiate a BackupManager object using mocked parameters
The compression_manager member is mocked
:param barman.server.Server|None server: Optional Server object
:rtype: barman.backup.BackupManager
"""
if server is None:
server = build_mocked_server(name, config, global_conf, main_conf)
with mock.patch("barman.backup.CompressionManager"):
manager = BackupManager(server=server)
manager.compression_manager.unidentified_compression = None
manager.compression_manager.get_wal_file_info.side_effect = (
lambda filename: WalFileInfo.from_file(filename, manager.compression_manager)
)
server.backup_manager = manager
return manager
def archive_wal(self, compressor, wal_info):
"""
Archive a WAL segment and update the wal_info object
:param compressor: the compressor for the file (if any)
:param WalFileInfo wal_info: the WAL file is being processed
"""
src_file = wal_info.orig_filename
src_dir = os.path.dirname(src_file)
dst_file = wal_info.fullpath(self.server)
tmp_file = dst_file + '.tmp'
dst_dir = os.path.dirname(dst_file)
error = None
try:
# Run the pre_archive_script if present.
script = HookScriptRunner(self.backup_manager,
'archive_script', 'pre')
script.env_from_wal_info(wal_info, src_file)
script.run()
# Run the pre_archive_retry_script if present.
retry_script = RetryHookScriptRunner(self.backup_manager,
'archive_retry_script',
'pre')
retry_script.env_from_wal_info(wal_info, src_file)
retry_script.run()
# Check if destination already exists
if os.path.exists(dst_file):
src_uncompressed = src_file
dst_uncompressed = dst_file
dst_info = WalFileInfo.from_file(dst_file)
try:
comp_manager = self.backup_manager.compression_manager
if dst_info.compression is not None:
dst_uncompressed = dst_file + '.uncompressed'
comp_manager.get_compressor(
compression=dst_info.compression).decompress(
dst_file, dst_uncompressed)
if wal_info.compression:
src_uncompressed = src_file + '.uncompressed'
comp_manager.get_compressor(
compression=wal_info.compression).decompress(
src_file, src_uncompressed)
# Directly compare files.
# When the files are identical
# raise a MatchingDuplicateWalFile exception,
# otherwise raise a DuplicateWalFile exception.
if filecmp.cmp(dst_uncompressed, src_uncompressed):
raise MatchingDuplicateWalFile(wal_info)
else:
raise DuplicateWalFile(wal_info)
finally:
if src_uncompressed != src_file:
os.unlink(src_uncompressed)
if dst_uncompressed != dst_file:
os.unlink(dst_uncompressed)
mkpath(dst_dir)
# Compress the file only if not already compressed
if compressor and not wal_info.compression:
compressor.compress(src_file, tmp_file)
shutil.copystat(src_file, tmp_file)
os.rename(tmp_file, dst_file)
os.unlink(src_file)
# Update wal_info
stat = os.stat(dst_file)
wal_info.size = stat.st_size
wal_info.compression = compressor.compression
else:
# Try to atomically rename the file. If successful,
# the renaming will be an atomic operation
# (this is a POSIX requirement).
try:
os.rename(src_file, dst_file)
except OSError:
# Source and destination are probably on different
# filesystems
shutil.copy2(src_file, tmp_file)
os.rename(tmp_file, dst_file)
os.unlink(src_file)
# At this point the original file has been removed
wal_info.orig_filename = None
# Execute fsync() on the archived WAL file
file_fd = os.open(dst_file, os.O_RDONLY)
os.fsync(file_fd)
os.close(file_fd)
# Execute fsync() on the archived WAL containing directory
fsync_dir(dst_dir)
# Execute fsync() also on the incoming directory
fsync_dir(src_dir)
except Exception as e:
# In case of failure save the exception for the post scripts
error = e
raise
# Ensure the execution of the post_archive_retry_script and
# the post_archive_script
finally:
# Run the post_archive_retry_script if present.
try:
retry_script = RetryHookScriptRunner(self,
'archive_retry_script',
'post')
retry_script.env_from_wal_info(wal_info, dst_file, error)
retry_script.run()
except AbortedRetryHookScript as e:
# Ignore the ABORT_STOP as it is a post-hook operation
_logger.warning("Ignoring stop request after receiving "
"abort (exit code %d) from post-archive "
"retry hook script: %s",
e.hook.exit_status, e.hook.script)
# Run the post_archive_script if present.
script = HookScriptRunner(self, 'archive_script', 'post', error)
script.env_from_wal_info(wal_info, dst_file)
script.run()
def test_decode_history_file(self, tmpdir):
compressor = mock.Mock()
# Regular history file
p = tmpdir.join('00000002.history')
p.write('1\t2/83000168\tat restore point "myrp"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=2,
parent_tli=1,
reason='at restore point "myrp"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# Comments must be skipped
p = tmpdir.join('00000003.history')
p.write('# Comment\n1\t2/83000168\tat restore point "testcomment"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=3,
parent_tli=1,
reason='at restore point "testcomment"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# History file with comments and empty lines
p = tmpdir.join('00000004.history')
p.write('# Comment\n\n1\t2/83000168\ttesting "testemptyline"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=4,
parent_tli=1,
reason='testing "testemptyline"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# Test compression handling Fix for bug #66 on github
config_mock = mock.Mock()
config_mock.compression = "gzip"
# check custom compression method creation
comp_manager = CompressionManager(config_mock, None)
u = tmpdir.join('00000005.uncompressed')
p = tmpdir.join('00000005.history')
u.write('1\t2/83000168\tat restore point "myrp"\n')
result = xlog.HistoryFileData(
tli=5,
parent_tli=1,
reason='at restore point "myrp"',
switchpoint=0x283000168)
comp_manager.get_compressor('gzip').compress(u.strpath,
p.strpath)
wal_info = WalFileInfo.from_file(p.strpath)
assert xlog.decode_history_file(wal_info, comp_manager) == [result]
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Empty file
p.write('')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Missing field
p.write('1\t2/83000168')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Unattended field
p.write('1\t2/83000168\tat restore point "myrp"\ttest')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
def archive_wal(self, compressor, wal_info):
"""
Archive a WAL segment and update the wal_info object
:param compressor: the compressor for the file (if any)
:param WalFileInfo wal_info: the WAL file is being processed
"""
src_file = wal_info.orig_filename
src_dir = os.path.dirname(src_file)
dst_file = wal_info.fullpath(self.server)
tmp_file = dst_file + '.tmp'
dst_dir = os.path.dirname(dst_file)
error = None
try:
# Run the pre_archive_script if present.
script = HookScriptRunner(self.backup_manager, 'archive_script',
'pre')
script.env_from_wal_info(wal_info, src_file)
script.run()
# Run the pre_archive_retry_script if present.
retry_script = RetryHookScriptRunner(self.backup_manager,
'archive_retry_script', 'pre')
retry_script.env_from_wal_info(wal_info, src_file)
retry_script.run()
# Check if destination already exists
if os.path.exists(dst_file):
src_uncompressed = src_file
dst_uncompressed = dst_file
dst_info = WalFileInfo.from_file(dst_file)
try:
comp_manager = self.backup_manager.compression_manager
if dst_info.compression is not None:
dst_uncompressed = dst_file + '.uncompressed'
comp_manager.get_compressor(
compression=dst_info.compression).decompress(
dst_file, dst_uncompressed)
if wal_info.compression:
src_uncompressed = src_file + '.uncompressed'
comp_manager.get_compressor(
compression=wal_info.compression).decompress(
src_file, src_uncompressed)
# Directly compare files.
# When the files are identical
# raise a MatchingDuplicateWalFile exception,
# otherwise raise a DuplicateWalFile exception.
if filecmp.cmp(dst_uncompressed, src_uncompressed):
raise MatchingDuplicateWalFile(wal_info)
else:
raise DuplicateWalFile(wal_info)
finally:
if src_uncompressed != src_file:
os.unlink(src_uncompressed)
if dst_uncompressed != dst_file:
os.unlink(dst_uncompressed)
mkpath(dst_dir)
# Compress the file only if not already compressed
if compressor and not wal_info.compression:
compressor.compress(src_file, tmp_file)
shutil.copystat(src_file, tmp_file)
os.rename(tmp_file, dst_file)
os.unlink(src_file)
# Update wal_info
stat = os.stat(dst_file)
wal_info.size = stat.st_size
wal_info.compression = compressor.compression
else:
# Try to atomically rename the file. If successful,
# the renaming will be an atomic operation
# (this is a POSIX requirement).
try:
os.rename(src_file, dst_file)
except OSError:
# Source and destination are probably on different
# filesystems
shutil.copy2(src_file, tmp_file)
os.rename(tmp_file, dst_file)
os.unlink(src_file)
# At this point the original file has been removed
wal_info.orig_filename = None
# Execute fsync() on the archived WAL file
file_fd = os.open(dst_file, os.O_RDONLY)
os.fsync(file_fd)
os.close(file_fd)
# Execute fsync() on the archived WAL containing directory
fsync_dir(dst_dir)
# Execute fsync() also on the incoming directory
fsync_dir(src_dir)
except Exception as e:
# In case of failure save the exception for the post scripts
error = e
raise
# Ensure the execution of the post_archive_retry_script and
# the post_archive_script
finally:
# Run the post_archive_retry_script if present.
try:
retry_script = RetryHookScriptRunner(self,
'archive_retry_script',
'post')
retry_script.env_from_wal_info(wal_info, dst_file, error)
retry_script.run()
except AbortedRetryHookScript as e:
# Ignore the ABORT_STOP as it is a post-hook operation
_logger.warning(
"Ignoring stop request after receiving "
"abort (exit code %d) from post-archive "
"retry hook script: %s", e.hook.exit_status, e.hook.script)
# Run the post_archive_script if present.
script = HookScriptRunner(self, 'archive_script', 'post', error)
script.env_from_wal_info(wal_info, dst_file)
script.run()
def rebuild_xlogdb(self):
"""
Rebuild the whole xlog database guessing it from the archive content.
"""
from os.path import isdir, join
output.info("Rebuilding xlogdb for server %s", self.config.name)
root = self.config.wals_directory
default_compression = self.config.compression
wal_count = label_count = history_count = 0
# lock the xlogdb as we are about replacing it completely
with self.server.xlogdb('w') as fxlogdb:
xlogdb_new = fxlogdb.name + ".new"
with open(xlogdb_new, 'w') as fxlogdb_new:
for name in sorted(os.listdir(root)):
# ignore the xlogdb and its lockfile
if name.startswith(self.server.XLOG_DB):
continue
fullname = join(root, name)
if isdir(fullname):
# all relevant files are in subdirectories
hash_dir = fullname
for wal_name in sorted(os.listdir(hash_dir)):
fullname = join(hash_dir, wal_name)
if isdir(fullname):
_logger.warning(
'unexpected directory '
'rebuilding the wal database: %s',
fullname)
else:
if xlog.is_wal_file(fullname):
wal_count += 1
elif xlog.is_backup_file(fullname):
label_count += 1
else:
_logger.warning(
'unexpected file '
'rebuilding the wal database: %s',
fullname)
continue
wal_info = WalFileInfo.from_file(
fullname,
default_compression=default_compression)
fxlogdb_new.write(wal_info.to_xlogdb_line())
else:
# only history files are here
if xlog.is_history_file(fullname):
history_count += 1
wal_info = WalFileInfo.from_file(
fullname,
default_compression=default_compression)
fxlogdb_new.write(wal_info.to_xlogdb_line())
else:
_logger.warning(
'unexpected file '
'rebuilding the wal database: %s',
fullname)
os.fsync(fxlogdb_new.fileno())
shutil.move(xlogdb_new, fxlogdb.name)
fsync_dir(os.path.dirname(fxlogdb.name))
output.info('Done rebuilding xlogdb for server %s '
'(history: %s, backup_labels: %s, wal_file: %s)',
self.config.name, history_count, label_count, wal_count)
def archive_wal(self, verbose=True):
"""
Executes WAL maintenance operations, such as archiving and compression
If verbose is set to False, outputs something only if there is
at least one file
:param bool verbose: report even if no actions
"""
found = False
compressor = self.compression_manager.get_compressor()
with self.server.xlogdb('a') as fxlogdb:
if verbose:
output.info("Processing xlog segments for %s",
self.config.name,
log=False)
# Get the first available backup
first_backup_id = self.get_first_backup(BackupInfo.STATUS_NOT_EMPTY)
first_backup = self.server.get_backup(first_backup_id)
for filename in sorted(glob(
os.path.join(self.config.incoming_wals_directory, '*'))):
if not found and not verbose:
output.info("Processing xlog segments for %s",
self.config.name,
log=False)
found = True
# Create WAL Info object
wal_info = WalFileInfo.from_file(filename, compression=None)
# If there are no available backups ...
if first_backup is None:
# ... delete xlog segments only for exclusive backups
if BackupOptions.CONCURRENT_BACKUP \
not in self.config.backup_options:
# Skipping history files
if not xlog.is_history_file(filename):
output.info("\tNo base backup available."
" Trashing file %s"
" from server %s",
wal_info.name, self.config.name)
os.unlink(filename)
continue
# ... otherwise
else:
# ... delete xlog segments older than the first backup
if wal_info.name < first_backup.begin_wal:
# Skipping history files
if not xlog.is_history_file(filename):
output.info("\tOlder than first backup."
" Trashing file %s"
" from server %s",
wal_info.name, self.config.name)
os.unlink(filename)
continue
# Report to the user the WAL file we are archiving
output.info("\t%s", os.path.basename(filename), log=False)
_logger.info("Archiving %s/%s",
self.config.name,
os.path.basename(filename))
# Archive the WAL file
try:
self.cron_wal_archival(compressor, wal_info)
except AbortedRetryHookScript as e:
_logger.warning("Archiving of %s/%s aborted by "
"pre_archive_retry_script."
"Reason: %s" % (self.config.name,
os.path.basename(),
e))
return
# Updates the information of the WAL archive with
# the latest segments
fxlogdb.write(wal_info.to_xlogdb_line())
# flush and fsync for every line
fxlogdb.flush()
os.fsync(fxlogdb.fileno())
if not found and verbose:
output.info("\tno file found", log=False)
def test_archive_wal(self, tmpdir, capsys):
"""
Test WalArchiver.archive_wal behaviour when the WAL file already
exists in the archive
"""
# Setup the test environment
backup_manager = build_backup_manager(
name='TestServer',
global_conf={
'barman_home': tmpdir.strpath
})
backup_manager.compression_manager.get_compressor.return_value = None
backup_manager.server.get_backup.return_value = None
basedir = tmpdir.join('main')
incoming_dir = basedir.join('incoming')
archive_dir = basedir.join('wals')
xlog_db = archive_dir.join('xlog.db')
wal_name = '000000010000000000000001'
wal_file = incoming_dir.join(wal_name)
wal_file.ensure()
archive_dir.ensure(dir=True)
xlog_db.ensure()
backup_manager.server.xlogdb.return_value.__enter__.return_value = (
xlog_db.open(mode='a'))
archiver = FileWalArchiver(backup_manager)
backup_manager.server.archivers = [archiver]
# Tests a basic archival process
wal_info = WalFileInfo.from_file(wal_file.strpath)
archiver.archive_wal(None, wal_info)
assert not os.path.exists(wal_file.strpath)
assert os.path.exists(wal_info.fullpath(backup_manager.server))
# Tests the archiver behaviour for duplicate WAL files, as the
# wal file named '000000010000000000000001' was already archived
# in the previous test
wal_file.ensure()
wal_info = WalFileInfo.from_file(wal_file.strpath)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Tests the archiver behaviour for duplicated WAL files with
# different contents
wal_file.write('test')
wal_info = WalFileInfo.from_file(wal_file.strpath)
with pytest.raises(DuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Tests the archiver behaviour for duplicate WAL files, as the
# wal file named '000000010000000000000001' was already archived
# in the previous test and the input file uses compression
compressor = PyGZipCompressor(backup_manager.config, 'pygzip')
compressor.compress(wal_file.strpath, wal_file.strpath)
wal_info = WalFileInfo.from_file(wal_file.strpath)
assert os.path.exists(wal_file.strpath)
backup_manager.compression_manager.get_compressor.return_value = (
compressor)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Test the archiver behaviour when the incoming file is compressed
# and it has been already archived and compressed.
compressor.compress(wal_info.fullpath(backup_manager.server),
wal_info.fullpath(backup_manager.server))
wal_info = WalFileInfo.from_file(wal_file.strpath)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Reset the status of the incoming and WALs directory
# removing the files archived during the preceding tests.
os.unlink(wal_info.fullpath(backup_manager.server))
os.unlink(wal_file.strpath)
# Test the archival of a WAL file using compression.
wal_file.write('test')
wal_info = WalFileInfo.from_file(wal_file.strpath)
archiver.archive_wal(compressor, wal_info)
assert os.path.exists(wal_info.fullpath(backup_manager.server))
assert not os.path.exists(wal_file.strpath)
assert 'gzip' == identify_compression(
wal_info.fullpath(backup_manager.server)
)
def test_decode_history_file(self, tmpdir):
compressor = mock.Mock()
# Regular history file
p = tmpdir.join('00000002.history')
p.write('1\t2/83000168\tat restore point "myrp"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=2,
parent_tli=1,
reason='at restore point "myrp"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# Comments must be skipped
p = tmpdir.join('00000003.history')
p.write('# Comment\n1\t2/83000168\tat restore point "testcomment"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=3,
parent_tli=1,
reason='at restore point "testcomment"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# History file with comments and empty lines
p = tmpdir.join('00000004.history')
p.write('# Comment\n\n1\t2/83000168\ttesting "testemptyline"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=4,
parent_tli=1,
reason='testing "testemptyline"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# Test compression handling Fix for bug #66 on github
config_mock = mock.Mock()
config_mock.compression = "gzip"
# check custom compression method creation
comp_manager = CompressionManager(config_mock, None)
u = tmpdir.join('00000005.uncompressed')
p = tmpdir.join('00000005.history')
u.write('1\t2/83000168\tat restore point "myrp"\n')
result = xlog.HistoryFileData(
tli=5,
parent_tli=1,
reason='at restore point "myrp"',
switchpoint=0x283000168)
comp_manager.get_compressor('gzip').compress(u.strpath,
p.strpath)
wal_info = WalFileInfo.from_file(p.strpath)
assert xlog.decode_history_file(wal_info, comp_manager) == [result]
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Empty file
p.write('')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Missing field
p.write('1\t2/83000168')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Unattended field
p.write('1\t2/83000168\tat restore point "myrp"\ttest')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
def test_archive_wal(self, tmpdir, capsys):
"""
Test WalArchiver.archive_wal behaviour when the WAL file already
exists in the archive
"""
# Setup the test environment
backup_manager = build_backup_manager(
name="TestServer", global_conf={"barman_home": tmpdir.strpath}
)
backup_manager.compression_manager.get_compressor.return_value = None
backup_manager.server.get_backup.return_value = None
basedir = tmpdir.join("main")
incoming_dir = basedir.join("incoming")
archive_dir = basedir.join("wals")
xlog_db = archive_dir.join("xlog.db")
wal_name = "000000010000000000000001"
wal_file = incoming_dir.join(wal_name)
wal_file.ensure()
archive_dir.ensure(dir=True)
xlog_db.ensure()
backup_manager.server.xlogdb.return_value.__enter__.return_value = xlog_db.open(
mode="a"
)
archiver = FileWalArchiver(backup_manager)
backup_manager.server.archivers = [archiver]
# Tests a basic archival process
wal_info = WalFileInfo.from_file(wal_file.strpath)
archiver.archive_wal(None, wal_info)
assert not os.path.exists(wal_file.strpath)
assert os.path.exists(wal_info.fullpath(backup_manager.server))
# Tests the archiver behaviour for duplicate WAL files, as the
# wal file named '000000010000000000000001' was already archived
# in the previous test
wal_file.ensure()
wal_info = WalFileInfo.from_file(wal_file.strpath)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Tests the archiver behaviour for duplicated WAL files with
# different contents
wal_file.write("test")
wal_info = WalFileInfo.from_file(wal_file.strpath)
with pytest.raises(DuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Tests the archiver behaviour for duplicate WAL files, as the
# wal file named '000000010000000000000001' was already archived
# in the previous test and the input file uses compression
compressor = PyGZipCompressor(backup_manager.config, "pygzip")
compressor.compress(wal_file.strpath, wal_file.strpath)
wal_info = WalFileInfo.from_file(wal_file.strpath)
assert os.path.exists(wal_file.strpath)
backup_manager.compression_manager.get_compressor.return_value = compressor
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Test the archiver behaviour when the incoming file is compressed
# and it has been already archived and compressed.
compressor.compress(
wal_info.fullpath(backup_manager.server),
wal_info.fullpath(backup_manager.server),
)
wal_info = WalFileInfo.from_file(wal_file.strpath)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Reset the status of the incoming and WALs directory
# removing the files archived during the preceding tests.
os.unlink(wal_info.fullpath(backup_manager.server))
os.unlink(wal_file.strpath)
# Test the archival of a WAL file using compression.
wal_file.write("test")
wal_info = WalFileInfo.from_file(wal_file.strpath)
archiver.archive_wal(compressor, wal_info)
assert os.path.exists(wal_info.fullpath(backup_manager.server))
assert not os.path.exists(wal_file.strpath)
assert "gzip" == identify_compression(wal_info.fullpath(backup_manager.server))
def rebuild_xlogdb(self):
"""
Rebuild the whole xlog database guessing it from the archive content.
"""
from os.path import isdir, join
output.info("Rebuilding xlogdb for server %s", self.config.name)
root = self.config.wals_directory
default_compression = self.config.compression
wal_count = label_count = history_count = 0
# lock the xlogdb as we are about replacing it completely
with self.server.xlogdb('w') as fxlogdb:
xlogdb_new = fxlogdb.name + ".new"
with open(xlogdb_new, 'w') as fxlogdb_new:
for name in sorted(os.listdir(root)):
# ignore the xlogdb and its lockfile
if name.startswith(self.server.XLOG_DB):
continue
fullname = join(root, name)
if isdir(fullname):
# all relevant files are in subdirectories
hash_dir = fullname
for wal_name in sorted(os.listdir(hash_dir)):
fullname = join(hash_dir, wal_name)
if isdir(fullname):
_logger.warning(
'unexpected directory '
'rebuilding the wal database: %s',
fullname)
else:
if xlog.is_wal_file(fullname):
wal_count += 1
elif xlog.is_backup_file(fullname):
label_count += 1
elif fullname.endswith('.tmp'):
_logger.warning(
'temporary file found '
'rebuilding the wal database: %s',
fullname)
continue
else:
_logger.warning(
'unexpected file '
'rebuilding the wal database: %s',
fullname)
continue
wal_info = WalFileInfo.from_file(
fullname,
default_compression=default_compression)
fxlogdb_new.write(wal_info.to_xlogdb_line())
else:
# only history files are here
if xlog.is_history_file(fullname):
history_count += 1
wal_info = WalFileInfo.from_file(
fullname,
default_compression=default_compression)
fxlogdb_new.write(wal_info.to_xlogdb_line())
else:
_logger.warning(
'unexpected file '
'rebuilding the wal database: %s', fullname)
os.fsync(fxlogdb_new.fileno())
shutil.move(xlogdb_new, fxlogdb.name)
fsync_dir(os.path.dirname(fxlogdb.name))
output.info(
'Done rebuilding xlogdb for server %s '
'(history: %s, backup_labels: %s, wal_file: %s)', self.config.name,
history_count, label_count, wal_count)
