def test_get_next_batch( self, from_file_mock, isfile_mock, exists_mock, glob_mock, caplog ): """ Test the FileWalArchiver.get_next_batch method """ # See all logs caplog.set_level(0) # WAL batch, with 000000010000000000000001 that is currently being # written glob_mock.return_value = ["000000010000000000000001"] isfile_mock.return_value = True # This is an hack, instead of a WalFileInfo we use a simple string to # ease all the comparisons. The resulting string is the name enclosed # in colons. e.g. ":000000010000000000000001:" from_file_mock.side_effect = lambda wal_name, compression: (":%s:" % wal_name) backup_manager = build_backup_manager(name="TestServer") archiver = StreamingWalArchiver(backup_manager) backup_manager.server.archivers = [archiver] caplog_reset(caplog) batch = archiver.get_next_batch() assert ["000000010000000000000001"] == batch.skip assert "" == caplog.text # WAL batch, with 000000010000000000000002 that is currently being # written and 000000010000000000000001 can be archived caplog_reset(caplog) glob_mock.return_value = [ "000000010000000000000001", "000000010000000000000002", ] batch = archiver.get_next_batch() assert [":000000010000000000000001:"] == batch assert ["000000010000000000000002"] == batch.skip assert "" == caplog.text # WAL batch, with two partial files. caplog_reset(caplog) glob_mock.return_value = [ "000000010000000000000001.partial", "000000010000000000000002.partial", ] batch = archiver.get_next_batch() assert [":000000010000000000000001.partial:"] == batch assert ["000000010000000000000002.partial"] == batch.skip assert ( "Archiving partial files for server %s: " "000000010000000000000001.partial" % archiver.config.name ) in caplog.text # WAL batch, with history files. caplog_reset(caplog) glob_mock.return_value = [ "00000001.history", "000000010000000000000002.partial", ] batch = archiver.get_next_batch() assert [":00000001.history:"] == batch assert ["000000010000000000000002.partial"] == batch.skip assert "" == caplog.text # WAL batch with errors wrong_file_name = "test_wrong_wal_file.2" glob_mock.return_value = ["test_wrong_wal_file.2"] batch = archiver.get_next_batch() assert [wrong_file_name] == batch.errors # WAL batch, with two partial files, but one has been just renamed. caplog_reset(caplog) exists_mock.side_effect = [False, True] glob_mock.return_value = [ "000000010000000000000001.partial", "000000010000000000000002.partial", ] batch = archiver.get_next_batch() assert len(batch) == 0 assert ["000000010000000000000002.partial"] == batch.skip assert "" in caplog.text
def test_archive( self, datetime_mock, move_mock, archive_wal_mock, get_next_batch_mock, unlink_mock, capsys, caplog, ): """ Test FileWalArchiver.archive method """ # See all logs caplog.set_level(0) fxlogdb_mock = MagicMock() backup_manager = MagicMock() archiver = FileWalArchiver(backup_manager) archiver.config.name = "test_server" archiver.config.errors_directory = "/server/errors" wal_info = WalFileInfo(name="test_wal_file") wal_info.orig_filename = "test_wal_file" batch = WalArchiverQueue([wal_info]) assert batch.size == 1 assert batch.run_size == 1 get_next_batch_mock.return_value = batch archive_wal_mock.side_effect = DuplicateWalFile datetime_mock.utcnow.return_value.strftime.return_value = "test_time" archiver.archive(fxlogdb_mock) out, err = capsys.readouterr() assert ( "\tError: %s is already present in server %s. " "File moved to errors directory." % (wal_info.name, archiver.config.name) ) in out assert ( "\tError: %s is already present in server %s. " "File moved to errors directory." % (wal_info.name, archiver.config.name) ) in caplog.text archive_wal_mock.side_effect = MatchingDuplicateWalFile archiver.archive(fxlogdb_mock) unlink_mock.assert_called_with(wal_info.orig_filename) # Test batch errors caplog_reset(caplog) batch.errors = ["testfile_1", "testfile_2"] archive_wal_mock.side_effect = DuplicateWalFile archiver.archive(fxlogdb_mock) out, err = capsys.readouterr() assert ( "Some unknown objects have been found while " "processing xlog segments for %s. " "Objects moved to errors directory:" % archiver.config.name ) in out assert ( "Archiver is about to move %s unexpected file(s) to errors " "directory for %s from %s" % (len(batch.errors), archiver.config.name, archiver.name) ) in caplog.text assert ( "Moving unexpected file for %s from %s: %s" % (archiver.config.name, archiver.name, "testfile_1") ) in caplog.text assert ( "Moving unexpected file for %s from %s: %s" % (archiver.config.name, archiver.name, "testfile_2") ) in caplog.text move_mock.assert_any_call( "testfile_1", os.path.join( archiver.config.errors_directory, "%s.%s.unknown" % ("testfile_1", "test_time"), ), ) move_mock.assert_any_call( "testfile_2", os.path.join( archiver.config.errors_directory, "%s.%s.unknown" % ("testfile_2", "test_time"), ), )
def test_delete_backup(self, mock_available_backups, tmpdir, caplog): """ Simple test for the deletion of a backup. We want to test the behaviour of the delete_backup method """ # Setup of the test backup_manager backup_manager = build_backup_manager() backup_manager.server.config.name = 'TestServer' backup_manager.server.config.barman_lock_directory = tmpdir.strpath backup_manager.server.config.backup_options = [] # Create a fake backup directory inside tmpdir (old format) base_dir = tmpdir.mkdir('base') backup_dir = base_dir.mkdir('fake_backup_id') pg_data = backup_dir.mkdir('pgdata') pg_data_v2 = backup_dir.mkdir('data') wal_dir = tmpdir.mkdir('wals') wal_history_file02 = wal_dir.join('00000002.history') wal_history_file03 = wal_dir.join('00000003.history') wal_history_file04 = wal_dir.join('00000004.history') wal_history_file02.write('1\t0/2000028\tat restore point "myrp"\n') wal_history_file03.write('1\t0/2000028\tat restore point "myrp"\n') wal_history_file04.write('1\t0/2000028\tat restore point "myrp"\n') wal_history_file04.write('2\t0/3000028\tunknown\n') wal_file = wal_dir.join('0000000100000000/000000010000000000000001') wal_file.ensure() xlog_db = wal_dir.join('xlog.db') xlog_db.write( '000000010000000000000001\t42\t43\tNone\n' '00000002.history\t42\t43\tNone\n' '00000003.history\t42\t43\tNone\n' '00000004.history\t42\t43\tNone\n') backup_manager.server.xlogdb.return_value.__enter__.return_value = ( xlog_db.open()) backup_manager.server.config.basebackups_directory = base_dir.strpath backup_manager.server.config.wals_directory = wal_dir.strpath # The following tablespaces are defined in the default backup info # generated by build_test_backup_info b_info = build_test_backup_info( backup_id='fake_backup_id', server=backup_manager.server, ) # Make sure we are not trying to delete any WAL file, # just by having a previous backup b_pre_info = build_test_backup_info( backup_id='fake_backup', server=backup_manager.server, ) mock_available_backups.return_value = { "fake_backup": b_pre_info, "fake_backup_id": b_info, } # Test 1: minimum redundancy not satisfied caplog_reset(caplog) backup_manager.server.config.minimum_redundancy = 2 b_info.set_attribute('backup_version', 1) build_backup_directories(b_info) backup_manager.delete_backup(b_info) assert 'WARNING Skipping delete of backup ' in caplog.text assert 'ERROR' not in caplog.text assert os.path.exists(pg_data.strpath) assert not os.path.exists(pg_data_v2.strpath) assert os.path.exists(wal_file.strpath) assert os.path.exists(wal_history_file02.strpath) assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath) # Test 2: normal delete expecting no errors (old format) caplog_reset(caplog) backup_manager.server.config.minimum_redundancy = 1 b_info.set_attribute('backup_version', 1) build_backup_directories(b_info) backup_manager.delete_backup(b_info) # the backup must not exists on disk anymore assert 'WARNING' not in caplog.text assert 'ERROR' not in caplog.text assert not os.path.exists(pg_data.strpath) assert not os.path.exists(pg_data_v2.strpath) assert os.path.exists(wal_file.strpath) assert os.path.exists(wal_history_file02.strpath) assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath) # Test 3: delete the backup again, expect a failure in log caplog_reset(caplog) backup_manager.delete_backup(b_info) assert 'ERROR Failure deleting backup fake_backup_id' in caplog.text assert not os.path.exists(pg_data.strpath) assert not os.path.exists(pg_data_v2.strpath) assert os.path.exists(wal_file.strpath) assert os.path.exists(wal_history_file02.strpath) assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath) # Test 4: normal delete expecting no errors (new format) caplog_reset(caplog) b_info.set_attribute('backup_version', 2) build_backup_directories(b_info) backup_manager.delete_backup(b_info) assert 'WARNING' not in caplog.text assert 'ERROR' not in caplog.text assert not os.path.exists(pg_data.strpath) assert not os.path.exists(pg_data_v2.strpath) assert os.path.exists(wal_file.strpath) assert os.path.exists(wal_history_file02.strpath) assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath) # Test 5: normal delete of first backup no errors and no skip # removing one of the two backups present (new format) # and all the previous wal caplog_reset(caplog) b_pre_info.set_attribute('backup_version', 2) build_backup_directories(b_pre_info) backup_manager.delete_backup(b_pre_info) assert 'WARNING' not in caplog.text assert 'ERROR' not in caplog.text assert not os.path.exists(pg_data.strpath) assert not os.path.exists(pg_data_v2.strpath) assert not os.path.exists(wal_file.strpath) assert os.path.exists(wal_history_file02.strpath) assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath) # Test 6: normal delete of first backup no errors and no skip # removing one of the two backups present (new format) # the previous wal is retained as on a different timeline caplog_reset(caplog) wal_file.ensure() b_pre_info.set_attribute('timeline', 2) b_pre_info.set_attribute('backup_version', 2) build_backup_directories(b_pre_info) backup_manager.delete_backup(b_pre_info) assert 'WARNING' not in caplog.text assert 'ERROR' not in caplog.text assert not os.path.exists(pg_data.strpath) assert not os.path.exists(pg_data_v2.strpath) assert os.path.exists(wal_file.strpath) assert os.path.exists(wal_history_file02.strpath) assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath) # Test 7: simulate an error deleting the the backup. with patch('barman.backup.BackupManager.delete_backup_data')\ as mock_delete_data: caplog_reset(caplog) # We force delete_pgdata method to raise an exception. mock_delete_data.side_effect = OSError('TestError') wal_file.ensure() b_pre_info.set_attribute('backup_version', 2) build_backup_directories(b_pre_info) backup_manager.delete_backup(b_info) assert 'TestError' in caplog.text assert os.path.exists(wal_file.strpath) assert os.path.exists(wal_history_file02.strpath) assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath)
def test_archive(self, datetime_mock, move_mock, archive_wal_mock, get_next_batch_mock, unlink_mock, capsys, caplog): """ Test FileWalArchiver.archive method """ fxlogdb_mock = MagicMock() backup_manager = MagicMock() archiver = FileWalArchiver(backup_manager) archiver.config.name = "test_server" wal_info = WalFileInfo(name="test_wal_file") wal_info.orig_filename = "test_wal_file" batch = WalArchiverQueue([wal_info]) assert batch.size == 1 assert batch.run_size == 1 get_next_batch_mock.return_value = batch archive_wal_mock.side_effect = DuplicateWalFile archiver.archive(fxlogdb_mock) out, err = capsys.readouterr() assert ("\tError: %s is already present in server %s. " "File moved to errors directory." % (wal_info.name, archiver.config.name)) in out assert ("\tError: %s is already present in server %s. " "File moved to errors directory." % (wal_info.name, archiver.config.name)) in caplog.text archive_wal_mock.side_effect = MatchingDuplicateWalFile archiver.archive(fxlogdb_mock) unlink_mock.assert_called_with(wal_info.orig_filename) # Test batch errors caplog_reset(caplog) datetime_mock.utcnow.strftime.return_value = 'test_time' batch.errors = ['testfile_1', 'testfile_2'] archive_wal_mock.side_effect = DuplicateWalFile archiver.archive(fxlogdb_mock) out, err = capsys.readouterr() assert ("Some unknown objects have been found while " "processing xlog segments for %s. " "Objects moved to errors directory:" % archiver.config.name) in out assert ("Archiver is about to move %s unexpected file(s) to errors " "directory for %s from %s" % (len(batch.errors), archiver.config.name, archiver.name)) in caplog.text assert ("Moving unexpected file for %s from %s: %s" % (archiver.config.name, archiver.name, 'testfile_1')) in caplog.text assert ("Moving unexpected file for %s from %s: %s" % (archiver.config.name, archiver.name, 'testfile_2')) in caplog.text move_mock.assert_any_call( 'testfile_1', os.path.join(archiver.config.errors_directory, "%s.%s.unknown" % ('testfile_1', 'test_time'))) move_mock.assert_any_call( 'testfile_2', os.path.join(archiver.config.errors_directory, "%s.%s.unknown" % ('testfile_2', 'test_time')))