def __init__(self, server):
'''Constructor'''
self.name = "default"
self.server = server
self.config = server.config
self.available_backups = {}
self.compression_manager = CompressionManager(self.config)
def test_get_compressor_invalid(self):
# prepare mock obj
config_mock = mock.Mock()
# check custom compression method creation
comp_manager = CompressionManager(config_mock, None)
assert comp_manager.get_compressor("test_compression") is None
def test_get_compressor_custom(self, _reset_custom_compressor):
# GIVEN a Barman config which specifies custom compression
config_mock = mock.Mock()
config_mock.compression = "custom"
config_mock.custom_compression_filter = "test_custom_compression_filter"
config_mock.custom_decompression_filter = "test_custom_decompression_filter"
# AND the custom compression magic bytes are set
config_mock.custom_compression_magic = "0x28b52ffd"
# WHEN the compression manager is created
comp_manager = CompressionManager(config_mock, None)
# THEN a default compressor can be obtained
assert comp_manager.get_default_compressor() is not None
# AND the magic bytes of the compressor match those in the config
assert comp_manager.get_default_compressor(
).MAGIC == b"\x28\xb5\x2f\xfd"
# AND unidentified_compression is set to None as there is no need
# to make the legacy assumption that unidentified compression means
# custom compression
assert comp_manager.unidentified_compression is None
# AND the value of MAGIC_MAX_LENGTH equals the length of the magic bytes
assert comp_manager.MAGIC_MAX_LENGTH == 4
def test_bzip2(self, tmpdir):
config_mock = mock.Mock()
compression_manager = CompressionManager(config_mock, tmpdir.strpath)
compressor = PyBZip2Compressor(config=config_mock,
compression="pybzip2")
src = tmpdir.join("sourcefile")
src.write("content")
compressor.compress(src.strpath, BZIP2_FILE % tmpdir.strpath)
assert os.path.exists(BZIP2_FILE % tmpdir.strpath)
compression_zip = compression_manager.identify_compression(
BZIP2_FILE % tmpdir.strpath, )
assert compression_zip == "bzip2"
compressor.decompress(
BZIP2_FILE % tmpdir.strpath,
BZIP2_FILE_UNCOMPRESSED % tmpdir.strpath,
)
f = open(BZIP2_FILE_UNCOMPRESSED % tmpdir.strpath).read()
assert f == "content"
def test_identify_compression(self, tmpdir):
# prepare mock obj
config_mock = mock.Mock()
config_mock.compression = "bzip2"
# check custom compression method creation
comp_manager = CompressionManager(config_mock, None)
assert comp_manager.get_default_compressor() is not None
bz2_tmp_file = tmpdir.join("test_file")
# "test" in bz2 compression
bz2_tmp_file.write(
base64.b64decode(
b"QlpoOTFBWSZTWczDcdQAAAJBgAAQAgAMACAAIZpoM00Zl4u5IpwoSGZhuOoA"
),
mode="wb",
)
compression_bz2 = comp_manager.identify_compression(
bz2_tmp_file.strpath)
assert compression_bz2 == "bzip2"
zip_tmp_file = tmpdir.join("test_file")
# "test" in bz2 compression
zip_tmp_file.write(
base64.b64decode(b"H4sIAF0ssFIAAytJLS7hAgDGNbk7BQAAAA=="),
mode="wb")
# check custom compression method creation
compression_zip = comp_manager.identify_compression(
zip_tmp_file.strpath)
assert compression_zip == "gzip"
def test_get_compressor_custom_nomagic(self, _reset_custom_compressor):
# GIVEN a Barman config which specifies custom compression
config_mock = mock.Mock()
config_mock.compression = "custom"
config_mock.custom_compression_filter = "test_custom_compression_filter"
config_mock.custom_decompression_filter = "test_custom_decompression_filter"
# AND no magic bytes are set
config_mock.custom_compression_magic = None
# WHEN the compression manager is created
comp_manager = CompressionManager(config_mock, None)
# THEN a default compressor can be obtained
assert comp_manager.get_default_compressor() is not None
# AND the magic bytes of the compressor are None
assert comp_manager.get_default_compressor().MAGIC is None
# AND unidentified_compression is set to "custom" as this assumption
# is the legacy way of identifying custom compression, used when magic
# bytes is not set
assert comp_manager.unidentified_compression == "custom"
# AND the value of MAGIC_MAX_LENGTH equals the max length of the default
# compressions
assert comp_manager.MAGIC_MAX_LENGTH == 3
def test_get_compressor_bzip2(self):
# prepare mock obj
config_mock = mock.Mock()
config_mock.compression = "bzip2"
# check custom compression method creation
comp_manager = CompressionManager(config_mock, None)
assert comp_manager.get_default_compressor() is not None
def test_get_compressor_custom(self):
# prepare mock obj
config_mock = mock.Mock()
config_mock.compression = "custom"
config_mock.custom_compression_filter = "test_custom_compression_filter"
config_mock.custom_decompression_filter = "test_custom_decompression_filter"
# check custom compression method creation
comp_manager = CompressionManager(config_mock, None)
assert comp_manager.get_default_compressor() is not None
def __init__(self, server):
'''Constructor'''
self.name = "default"
self.server = server
self.config = server.config
self.available_backups = {}
self.compression_manager = CompressionManager(self.config)
# used for error messages
self.current_action = None
def __init__(self, server):
"""
Constructor
"""
super(BackupManager, self).__init__()
self.server = server
self.config = server.config
self._backup_cache = None
self.compression_manager = CompressionManager(self.config, server.path)
self.executor = None
try:
self.executor = BackupExecutor.factory(self)
except SshCommandException as e:
self.config.disabled = True
self.config.msg_list.append(str(e).strip())
def __init__(self, server):
"""
Constructor
"""
super(BackupManager, self).__init__()
self.name = "default"
self.server = server
self.config = server.config
self._backup_cache = None
self.compression_manager = CompressionManager(self.config, server.path)
self.executor = None
try:
if self.config.backup_method == "postgres":
self.executor = PostgresBackupExecutor(self)
else:
self.executor = RsyncBackupExecutor(self)
except SshCommandException as e:
self.config.disabled = True
self.config.msg_list.append(str(e).strip())
def test_decode_history_file(self, tmpdir):
compressor = mock.Mock()
# Regular history file
p = tmpdir.join('00000002.history')
p.write('1\t2/83000168\tat restore point "myrp"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=2,
parent_tli=1,
reason='at restore point "myrp"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# Comments must be skipped
p = tmpdir.join('00000003.history')
p.write('# Comment\n1\t2/83000168\tat restore point "testcomment"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=3,
parent_tli=1,
reason='at restore point "testcomment"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# History file with comments and empty lines
p = tmpdir.join('00000004.history')
p.write('# Comment\n\n1\t2/83000168\ttesting "testemptyline"\n')
wal_info = WalFileInfo.from_file(p.strpath)
result = xlog.HistoryFileData(
tli=4,
parent_tli=1,
reason='testing "testemptyline"',
switchpoint=0x283000168)
assert xlog.decode_history_file(wal_info, compressor) == [result]
assert len(compressor.mock_calls) == 0
# Test compression handling Fix for bug #66 on github
config_mock = mock.Mock()
config_mock.compression = "gzip"
# check custom compression method creation
comp_manager = CompressionManager(config_mock, None)
u = tmpdir.join('00000005.uncompressed')
p = tmpdir.join('00000005.history')
u.write('1\t2/83000168\tat restore point "myrp"\n')
result = xlog.HistoryFileData(
tli=5,
parent_tli=1,
reason='at restore point "myrp"',
switchpoint=0x283000168)
comp_manager.get_compressor('gzip').compress(u.strpath,
p.strpath)
wal_info = WalFileInfo.from_file(p.strpath)
assert xlog.decode_history_file(wal_info, comp_manager) == [result]
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Empty file
p.write('')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Missing field
p.write('1\t2/83000168')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
with pytest.raises(barman.exceptions.BadHistoryFileContents):
# Unattended field
p.write('1\t2/83000168\tat restore point "myrp"\ttest')
assert xlog.decode_history_file(wal_info, compressor)
assert len(compressor.mock_calls) == 0
def test_archive_wal(self, mock_compression_registry, tmpdir, capsys):
"""
Test WalArchiver.archive_wal behaviour when the WAL file already
exists in the archive
"""
# Hack the compression registry so we do not attempt to use native gzip
with patch.dict(
"barman.compression.compression_registry",
mock_compression_registry,
clear=True,
):
# Setup the test environment
backup_manager = build_backup_manager(
name="TestServer", global_conf={"barman_home": tmpdir.strpath})
# Replace mock compression manager with a real compression manager
backup_manager.compression_manager = CompressionManager(
backup_manager.config, tmpdir.strpath)
backup_manager.server.get_backup.return_value = None
basedir = tmpdir.join("main")
incoming_dir = basedir.join("incoming")
archive_dir = basedir.join("wals")
xlog_db = archive_dir.join("xlog.db")
wal_name = "000000010000000000000001"
wal_file = incoming_dir.join(wal_name)
wal_file.ensure()
archive_dir.ensure(dir=True)
xlog_db.ensure()
backup_manager.server.xlogdb.return_value.__enter__.return_value = (
xlog_db.open(mode="a"))
archiver = FileWalArchiver(backup_manager)
backup_manager.server.archivers = [archiver]
# Tests a basic archival process
wal_info = WalFileInfo.from_file(
wal_file.strpath, backup_manager.compression_manager)
archiver.archive_wal(None, wal_info)
assert not os.path.exists(wal_file.strpath)
assert os.path.exists(wal_info.fullpath(backup_manager.server))
# Tests the archiver behaviour for duplicate WAL files, as the
# wal file named '000000010000000000000001' was already archived
# in the previous test
wal_file.ensure()
wal_info = WalFileInfo.from_file(
wal_file.strpath, backup_manager.compression_manager)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Tests the archiver behaviour for duplicated WAL files with
# different contents
wal_file.write("test")
wal_info = WalFileInfo.from_file(
wal_file.strpath, backup_manager.compression_manager)
with pytest.raises(DuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Tests the archiver behaviour for duplicate WAL files, as the
# wal file named '000000010000000000000001' was already archived
# in the previous test and the input file uses compression
compressor = PyGZipCompressor(backup_manager.config, "pygzip")
compressor.compress(wal_file.strpath, wal_file.strpath)
wal_info = WalFileInfo.from_file(
wal_file.strpath, backup_manager.compression_manager)
assert os.path.exists(wal_file.strpath)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Test the archiver behaviour when the incoming file is compressed
# and it has been already archived and compressed.
compressor.compress(
wal_info.fullpath(backup_manager.server),
wal_info.fullpath(backup_manager.server),
)
wal_info = WalFileInfo.from_file(
wal_file.strpath, backup_manager.compression_manager)
with pytest.raises(MatchingDuplicateWalFile):
archiver.archive_wal(None, wal_info)
# Reset the status of the incoming and WALs directory
# removing the files archived during the preceding tests.
os.unlink(wal_info.fullpath(backup_manager.server))
os.unlink(wal_file.strpath)
# Test the archival of a WAL file using compression.
wal_file.write("test")
wal_info = WalFileInfo.from_file(
wal_file.strpath, backup_manager.compression_manager)
archiver.archive_wal(compressor, wal_info)
assert os.path.exists(wal_info.fullpath(backup_manager.server))
assert not os.path.exists(wal_file.strpath)
assert "pygzip" == CompressionManager(
MagicMock(), "").identify_compression(
wal_info.fullpath(backup_manager.server))
def test_check_with_compression(self):
# prepare mock obj
config_mock = mock.Mock()
comp_manager = CompressionManager(config_mock, None)
assert comp_manager.check("test_compression") is False
def test_check_compression_none(self):
# prepare mock obj
config_mock = mock.Mock()
config_mock.compression = "custom"
comp_manager = CompressionManager(config_mock, None)
assert comp_manager.check() is True
def test_compression_manager_creation(self):
# prepare mock obj
config_mock = mock.Mock()
comp_manager = CompressionManager(config_mock, None)
assert comp_manager
