def test_generate_segment_names_xlog_file_size_unknown(self):
assert tuple(
xlog.generate_segment_names('0000000100000001000000FD',
'000000010000000100000102',
90200)) == ('0000000100000001000000FD',
'0000000100000001000000FE',
'0000000100000001000000FF',
'000000010000000100000100',
'000000010000000100000101',
'000000010000000100000102')
assert tuple(
xlog.generate_segment_names('00000001000000010007FFFE',
'000000010000000200000002',
90300)) == ('00000001000000010007FFFE',
'00000001000000010007FFFF',
'000000010000000200000000',
'000000010000000200000001',
'000000010000000200000002')
# The last segment of a file is skipped in
# PostgreSQL < 9.3
assert tuple(
xlog.generate_segment_names('00000001000000010007FFFE',
'000000010000000200000002',
90200)) == ('00000001000000010007FFFE',
'000000010000000200000000',
'000000010000000200000001',
'000000010000000200000002')
def test_generate_segment_names_xlog_file_size_unknown(self):
assert tuple(
xlog.generate_segment_names("0000000100000001000000FD",
"000000010000000100000102",
90200)) == (
"0000000100000001000000FD",
"0000000100000001000000FE",
"0000000100000001000000FF",
"000000010000000100000100",
"000000010000000100000101",
"000000010000000100000102",
)
assert tuple(
xlog.generate_segment_names("00000001000000010007FFFE",
"000000010000000200000002",
90300)) == (
"00000001000000010007FFFE",
"00000001000000010007FFFF",
"000000010000000200000000",
"000000010000000200000001",
"000000010000000200000002",
)
# The last segment of a file is skipped in
# PostgreSQL < 9.3
assert tuple(
xlog.generate_segment_names("00000001000000010007FFFE",
"000000010000000200000002",
90200)) == (
"00000001000000010007FFFE",
"000000010000000200000000",
"000000010000000200000001",
"000000010000000200000002",
)
def test_generate_segment_names(self):
assert tuple(
xlog.generate_segment_names('0000000100000001000000FD',
'000000010000000200000002',
90200)) == ('0000000100000001000000FD',
'0000000100000001000000FE',
'000000010000000200000000',
'000000010000000200000001',
'000000010000000200000002')
assert tuple(
xlog.generate_segment_names('0000000100000001000000FD',
'0000000100000001000000FF',
90200)) == ('0000000100000001000000FD',
'0000000100000001000000FE')
assert tuple(
xlog.generate_segment_names('0000000100000001000000FD',
'000000010000000200000002',
90300)) == ('0000000100000001000000FD',
'0000000100000001000000FE',
'0000000100000001000000FF',
'000000010000000200000000',
'000000010000000200000001',
'000000010000000200000002')
assert tuple(
xlog.generate_segment_names('0000000100000001000000FD',
'0000000100000001000000FF',
90300)) == ('0000000100000001000000FD',
'0000000100000001000000FE',
'0000000100000001000000FF')
# Test the behaviour of generate_segment_names at log boundaries
# for recent versions
assert tuple(
itertools.islice(
xlog.generate_segment_names('0000000300000004000000FD'),
6)) == ('0000000300000004000000FD', '0000000300000004000000FE',
'0000000300000004000000FF', '000000030000000500000000',
'000000030000000500000001', '000000030000000500000002')
# Test the behaviour of generate_segment_names at log boundaries
# for versions < 9.3
assert tuple(
itertools.islice(
xlog.generate_segment_names('0000000300000004000000FD',
version=90201),
6)) == ('0000000300000004000000FD', '0000000300000004000000FE',
'000000030000000500000000', '000000030000000500000001',
'000000030000000500000002', '000000030000000500000003')
# Test the number of items produced between two segments
assert sum(1 for _ in xlog.generate_segment_names(
'000000040000000500000067', '000000040000000700000067')) == 513
# The number of items produced between the same two segments is lower
# with version < 9.3
assert sum(
1 for _ in xlog.generate_segment_names('000000040000000500000067',
'000000040000000700000067',
version=90201)) == 511
def get_required_wal_segments(self):
"""
Get the list of required WAL segments for the current backup
"""
return xlog.generate_segment_names(self.begin_wal, self.end_wal,
self.version,
self.xlog_segment_size)
def test_generate_segment_names_xlog_file_size_known(self):
assert tuple(
xlog.generate_segment_names(
'0000000100000001000000FD',
'000000010000000200000002',
90200,
xlog.DEFAULT_XLOG_SEG_SIZE
)) == (
'0000000100000001000000FD',
'0000000100000001000000FE',
'000000010000000200000000',
'000000010000000200000001',
'000000010000000200000002')
assert tuple(
xlog.generate_segment_names(
'0000000100000001000000FD',
'0000000100000001000000FF',
90200,
xlog.DEFAULT_XLOG_SEG_SIZE
)) == (
'0000000100000001000000FD',
'0000000100000001000000FE')
assert tuple(
xlog.generate_segment_names(
'0000000100000001000000FD',
'000000010000000200000002',
90300,
xlog.DEFAULT_XLOG_SEG_SIZE
)) == (
'0000000100000001000000FD',
'0000000100000001000000FE',
'0000000100000001000000FF',
'000000010000000200000000',
'000000010000000200000001',
'000000010000000200000002')
assert tuple(
xlog.generate_segment_names(
'0000000100000001000000FD',
'0000000100000001000000FF',
90300,
xlog.DEFAULT_XLOG_SEG_SIZE
)) == (
'0000000100000001000000FD',
'0000000100000001000000FE',
'0000000100000001000000FF')
# Test the behaviour of generate_segment_names at log boundaries
# for recent versions
assert tuple(
itertools.islice(
xlog.generate_segment_names(
'0000000300000004000000FD',
xlog_segment_size=xlog.DEFAULT_XLOG_SEG_SIZE),
6)
) == (
'0000000300000004000000FD',
'0000000300000004000000FE',
'0000000300000004000000FF',
'000000030000000500000000',
'000000030000000500000001',
'000000030000000500000002',
)
# Test the behaviour of generate_segment_names at log boundaries
# for versions < 9.3
assert tuple(
itertools.islice(
xlog.generate_segment_names(
'0000000300000004000000FD',
version=90201,
xlog_segment_size=xlog.DEFAULT_XLOG_SEG_SIZE),
6)
) == (
'0000000300000004000000FD',
'0000000300000004000000FE',
'000000030000000500000000',
'000000030000000500000001',
'000000030000000500000002',
'000000030000000500000003',
)
# Test the number of items produced between two segments
assert sum(
1 for _ in
xlog.generate_segment_names(
'000000040000000500000067',
'000000040000000700000067',
xlog_segment_size=xlog.DEFAULT_XLOG_SEG_SIZE
)) == 513
# The number of items produced between the same two segments is lower
# with version < 9.3
assert sum(
1 for _ in
xlog.generate_segment_names(
'000000040000000500000067',
'000000040000000700000067',
version=90201,
xlog_segment_size=xlog.DEFAULT_XLOG_SEG_SIZE
)) == 511
def test_generate_segment_names_xlog_file_size_known(self):
assert tuple(
xlog.generate_segment_names(
"0000000100000001000000FD",
"000000010000000200000002",
90200,
xlog.DEFAULT_XLOG_SEG_SIZE,
)
) == (
"0000000100000001000000FD",
"0000000100000001000000FE",
"000000010000000200000000",
"000000010000000200000001",
"000000010000000200000002",
)
assert (
tuple(
xlog.generate_segment_names(
"0000000100000001000000FD",
"0000000100000001000000FF",
90200,
xlog.DEFAULT_XLOG_SEG_SIZE,
)
)
== ("0000000100000001000000FD", "0000000100000001000000FE")
)
assert tuple(
xlog.generate_segment_names(
"0000000100000001000000FD",
"000000010000000200000002",
90300,
xlog.DEFAULT_XLOG_SEG_SIZE,
)
) == (
"0000000100000001000000FD",
"0000000100000001000000FE",
"0000000100000001000000FF",
"000000010000000200000000",
"000000010000000200000001",
"000000010000000200000002",
)
assert tuple(
xlog.generate_segment_names(
"0000000100000001000000FD",
"0000000100000001000000FF",
90300,
xlog.DEFAULT_XLOG_SEG_SIZE,
)
) == (
"0000000100000001000000FD",
"0000000100000001000000FE",
"0000000100000001000000FF",
)
# Test the behaviour of generate_segment_names at log boundaries
# for recent versions
assert tuple(
itertools.islice(
xlog.generate_segment_names(
"0000000300000004000000FD",
xlog_segment_size=xlog.DEFAULT_XLOG_SEG_SIZE,
),
6,
)
) == (
"0000000300000004000000FD",
"0000000300000004000000FE",
"0000000300000004000000FF",
"000000030000000500000000",
"000000030000000500000001",
"000000030000000500000002",
)
# Test the behaviour of generate_segment_names at log boundaries
# for versions < 9.3
assert tuple(
itertools.islice(
xlog.generate_segment_names(
"0000000300000004000000FD",
version=90201,
xlog_segment_size=xlog.DEFAULT_XLOG_SEG_SIZE,
),
6,
)
) == (
"0000000300000004000000FD",
"0000000300000004000000FE",
"000000030000000500000000",
"000000030000000500000001",
"000000030000000500000002",
"000000030000000500000003",
)
# Test the number of items produced between two segments
assert (
sum(
1
for _ in xlog.generate_segment_names(
"000000040000000500000067",
"000000040000000700000067",
xlog_segment_size=xlog.DEFAULT_XLOG_SEG_SIZE,
)
)
== 513
)
# The number of items produced between the same two segments is lower
# with version < 9.3
assert (
sum(
1
for _ in xlog.generate_segment_names(
"000000040000000500000067",
"000000040000000700000067",
version=90201,
xlog_segment_size=xlog.DEFAULT_XLOG_SEG_SIZE,
)
)
== 511
)
def check_backup(self, backup_info):
"""
Make sure that all the required WAL files to check
the consistency of a physical backup (that is, from the
beginning to the end of the full backup) are correctly
archived. This command is automatically invoked by the
cron command and at the end of every backup operation.
:param backup_info: the target backup
"""
# Gather the list of the latest archived wals
timelines = self.get_latest_archived_wals_info()
# Get the basic info for the backup
begin_wal = backup_info.begin_wal
end_wal = backup_info.end_wal
timeline = begin_wal[:8]
# Case 1: Barman still doesn't know about the timeline the backup
# started with. We still haven't archived any WAL corresponding
# to the backup, so we can't proceed with checking the existence
# of the required WAL files
if not timelines or timeline not in timelines:
backup_info.status = BackupInfo.WAITING_FOR_WALS
backup_info.save()
return
# Find the most recent archived WAL for this server in the timeline
# where the backup was taken
last_archived_wal = timelines[timeline].name
# Case 2: the most recent WAL file archived is older than the
# start of the backup. We must wait for the archiver to receive
# and/or process the WAL files.
if last_archived_wal < begin_wal:
backup_info.status = BackupInfo.WAITING_FOR_WALS
backup_info.save()
return
# Check the intersection between the required WALs and the archived
# ones. They should all exist
bound_end = min(last_archived_wal, end_wal)
segments = xlog.generate_segment_names(
begin_wal,
bound_end,
xlog_segment_size=backup_info.xlog_segment_size)
missing_wal = None
for wal in segments:
wal_full_path = self.server.get_wal_full_path(wal)
if not os.path.exists(wal_full_path):
missing_wal = wal
break
if missing_wal:
# Case 3: the most recent WAL file archived is more recent than
# the one corresponding to the start of a backup. If WAL
# file is missing, then we can't recover from the backup so we
# must mark the backup as FAILED.
# TODO: Verify if the error field is the right place
# to store the error message
backup_info.error = ("At least one WAL file is missing. "
"The first missing WAL file is %s" %
missing_wal)
backup_info.status = BackupInfo.FAILED
backup_info.save()
return
if end_wal <= last_archived_wal:
# Case 4: if the most recent WAL file archived is more recent or
# equal than the one corresponding to the end of the backup and
# every WAL that will be required by the recovery is available,
# we can mark the backup as DONE.
backup_info.status = BackupInfo.DONE
else:
# Case 5: if the most recent WAL file archived is older than
# the one corresponding to the end of the backup but
# all the WAL files until that point are present.
backup_info.status = BackupInfo.WAITING_FOR_WALS
backup_info.save()
def test_generate_segment_names(self):
assert tuple(
xlog.generate_segment_names(
'0000000100000001000000FD',
'000000010000000200000002',
90200
)) == (
'0000000100000001000000FD',
'0000000100000001000000FE',
'000000010000000200000000',
'000000010000000200000001',
'000000010000000200000002')
assert tuple(
xlog.generate_segment_names(
'0000000100000001000000FD',
'0000000100000001000000FF',
90200
)) == (
'0000000100000001000000FD',
'0000000100000001000000FE')
assert tuple(
xlog.generate_segment_names(
'0000000100000001000000FD',
'000000010000000200000002',
90300
)) == (
'0000000100000001000000FD',
'0000000100000001000000FE',
'0000000100000001000000FF',
'000000010000000200000000',
'000000010000000200000001',
'000000010000000200000002')
assert tuple(
xlog.generate_segment_names(
'0000000100000001000000FD',
'0000000100000001000000FF',
90300
)) == (
'0000000100000001000000FD',
'0000000100000001000000FE',
'0000000100000001000000FF')
# Test the behaviour of generate_segment_names at log boundaries
# for recent versions
assert tuple(itertools.islice(
xlog.generate_segment_names(
'0000000300000004000000FD'), 6
)) == (
'0000000300000004000000FD',
'0000000300000004000000FE',
'0000000300000004000000FF',
'000000030000000500000000',
'000000030000000500000001',
'000000030000000500000002')
# Test the behaviour of generate_segment_names at log boundaries
# for versions < 9.3
assert tuple(itertools.islice(
xlog.generate_segment_names(
'0000000300000004000000FD',
version=90201), 6
)) == (
'0000000300000004000000FD',
'0000000300000004000000FE',
'000000030000000500000000',
'000000030000000500000001',
'000000030000000500000002',
'000000030000000500000003')
# Test the number of items produced between two segments
assert sum(
1 for _ in
xlog.generate_segment_names(
'000000040000000500000067',
'000000040000000700000067'
)) == 513
# The number of items produced between the same two segments is lower
# with version < 9.3
assert sum(
1 for _ in
xlog.generate_segment_names(
'000000040000000500000067',
'000000040000000700000067',
version=90201
)) == 511
def get_required_wal_segments(self):
"""
Get the list of required WAL segments for the current backup
"""
return xlog.generate_segment_names(self.begin_wal, self.end_wal,
self.version)