Ejemplo n.º 1
0
    def get_wal_until_next_backup(self, backup):
        """Get the xlog files between backup and the next

        :param backup: a backup object, the starting point to retrieve wals
        """
        begin = backup.begin_wal
        next_end = None
        if self.get_next_backup(backup.backup_id):
            next_end = self.get_next_backup(backup.backup_id).end_wal
        backup_tli, _, _ = xlog.decode_segment_name(begin)

        with self.xlogdb() as fxlogdb:
            for line in fxlogdb:
                name, size, _, _ = self.xlogdb_parse_line(line)
                if name < begin:
                    continue
                tli, _, _ = xlog.decode_segment_name(name)
                if tli > backup_tli:
                    continue
                if not xlog.is_wal_file(name):
                    continue
                if next_end and name > next_end:
                    break
                # count
                yield (name, size)
Ejemplo n.º 2
0
 def get_required_xlog_files(self, backup, target_tli=None, target_time=None, target_xid=None):
     """Get the xlog files required for a backup"""
     begin = backup.begin_wal
     end = backup.end_wal
     # If timeline isn't specified, assume it is the same timeline of the backup
     if not target_tli:
         target_tli, _, _ = xlog.decode_segment_name(end)
     with self.xlogdb() as fxlogdb:
         for line in fxlogdb:
             name, _, stamp, _ = self.xlogdb_parse_line(line)
             if name < begin:
                 continue
             tli, _, _ = xlog.decode_segment_name(name)
             if tli > target_tli:
                 continue
             yield name
             if name > end:
                 end = name
                 if target_time and target_time < stamp:
                     break
         # return all the remaining history files
         for line in fxlogdb:
             name, _, stamp, _ = self.xlogdb_parse_line(line)
             if xlog.is_history_file(name):
                 yield name
Ejemplo n.º 3
0
    def get_wal_until_next_backup(self, backup, include_history=False):
        """
        Get the xlog files between backup and the next

        :param BackupInfo backup: a backup object, the starting point
            to retrieve WALs
        :param bool include_history: option for the inclusion of
            include_history files into the output
        """
        begin = backup.begin_wal
        next_end = None
        if self.get_next_backup(backup.backup_id):
            next_end = self.get_next_backup(backup.backup_id).end_wal
        backup_tli, _, _ = xlog.decode_segment_name(begin)

        with self.xlogdb() as fxlogdb:
            for line in fxlogdb:
                wal_info = WalFileInfo.from_xlogdb_line(line)
                # Handle .history files: add all of them to the output,
                # regardless of their age, if requested (the 'include_history'
                # parameter is True)
                if xlog.is_history_file(wal_info.name):
                    if include_history:
                        yield wal_info
                    continue
                if wal_info.name < begin:
                    continue
                tli, _, _ = xlog.decode_segment_name(wal_info.name)
                if tli > backup_tli:
                    continue
                if not xlog.is_wal_file(wal_info.name):
                    continue
                if next_end and wal_info.name > next_end:
                    break
                yield wal_info
Ejemplo n.º 4
0
 def get_required_xlog_files(self, backup, target_tli=None, target_time=None,
                             target_xid=None):
     """
     Get the xlog files required for a recovery
     """
     begin = backup.begin_wal
     end = backup.end_wal
     # If timeline isn't specified, assume it is the same timeline
     # of the backup
     if not target_tli:
         target_tli, _, _ = xlog.decode_segment_name(end)
     with self.xlogdb() as fxlogdb:
         for line in fxlogdb:
             wal_info = WalFileInfo.from_xlogdb_line(line)
             # Handle .history files: add all of them to the output,
             # regardless of their age
             if xlog.is_history_file(wal_info.name):
                 yield wal_info
                 continue
             if wal_info.name < begin:
                 continue
             tli, _, _ = xlog.decode_segment_name(wal_info.name)
             if tli > target_tli:
                 continue
             yield wal_info
             if wal_info.name > end:
                 end = wal_info.name
                 if target_time and target_time < wal_info.time:
                     break
         # return all the remaining history files
         for line in fxlogdb:
             wal_info = WalFileInfo.from_xlogdb_line(line)
             if xlog.is_history_file(wal_info.name):
                 yield wal_info
Ejemplo n.º 5
0
    def stop_backup(self, backup_info):
        """
        Stop backup wrapper

        :param barman.infofile.BackupInfo backup_info: backup information
        """
        postgres = self.executor.server.postgres
        stop_row = postgres.pgespresso_stop_backup(backup_info.backup_label)
        if stop_row:
            end_wal, stop_time = stop_row
            decoded_segment = xlog.decode_segment_name(end_wal)
            backup_info.set_attribute('end_time', stop_time)
            backup_info.set_attribute('end_xlog',
                                      "%X/%X" % (decoded_segment[1],
                                                 (decoded_segment[
                                                  2] + 1) << 24))
            backup_info.set_attribute('end_wal', end_wal)
            backup_info.set_attribute('end_offset', 0)
        else:
            raise Exception('Cannot terminate exclusive backup. You might '
                            'have to  manually execute '
                            'pgespresso_abort_backup() on your PostgreSQL '
                            'server')
        self.current_action = "writing backup label"
        self._write_backup_label(backup_info)
Ejemplo n.º 6
0
    def is_wal_relevant(self, wal_info, first_backup):
        """
        Check the relevance of a WAL file according to a provided BackupInfo
        (usually the oldest on the server) to ensure that the WAL is newer than
        the start_wal of the backup.

        :param WalFileInfo wal_info: the WAL file we are checking
        :param BackupInfo first_backup: the backup used for the checks
            (usually the oldest available on the server)
        """

        # Skip history files
        if xlog.is_history_file(wal_info.name):
            return True

        # If the WAL file has a timeline smaller than the one of
        # the oldest backup it cannot be used in any way.
        wal_timeline = xlog.decode_segment_name(wal_info.name)[0]
        if wal_timeline < first_backup.timeline:
            output.info("\tThe timeline of the WAL file %s (%s), is lower "
                        "than the one of the oldest backup of "
                        "server %s (%s). Moving the WAL in "
                        "the error directory",
                        wal_info.name, wal_timeline, self.config.name,
                        first_backup.timeline)
            return False
        # Manage xlog segments older than the first backup
        if wal_info.name < first_backup.begin_wal:
            output.info("\tOlder than first backup of server %s. "
                        "Moving the WAL file %s in the error directory",
                        self.config.name, wal_info.name)
            return False
        return True
Ejemplo n.º 7
0
 def testDecodeSegmentName(self):
     self.assertEqual(xlog.decode_segment_name('000000000000000000000000'), [0, 0, 0])
     self.assertEqual(xlog.decode_segment_name('000000010000000100000001'), [1, 1, 1])
     self.assertEqual(xlog.decode_segment_name('0000000A0000000A0000000A'), [10, 10, 10])
     self.assertEqual(xlog.decode_segment_name('000000110000001100000011'), [17, 17, 17])
     self.assertEqual(xlog.decode_segment_name('000000000000000200000001'), [0, 2, 1])
     self.assertEqual(xlog.decode_segment_name('000000010000000000000002'), [1, 0, 2])
     self.assertEqual(xlog.decode_segment_name('000000020000000100000000'), [2, 1, 0])
     self.assertRaises(xlog.BadXlogSegmentName, xlog.decode_segment_name, '00000000000000000000000')
     self.assertRaises(xlog.BadXlogSegmentName, xlog.decode_segment_name, '0000000000000000000000000')
     self.assertRaises(xlog.BadXlogSegmentName, xlog.decode_segment_name, '000000000000X00000000000')
     self.assertEqual(xlog.decode_segment_name('00000001000000000000000A.00000020.backup'), [1, 0, 10])
     self.assertEqual(xlog.decode_segment_name('00000001.history'), [1, None, None])
Ejemplo n.º 8
0
    def remove_wal_before_backup(self, backup_info, timelines_to_protect=None):
        """
        Remove WAL files which have been archived before the start of
        the provided backup.

        If no backup_info is provided delete all available WAL files

        If timelines_to_protect list is passed, never remove a wal in one of
        these timelines.

        :param BackupInfo|None backup_info: the backup information structure
        :param set timelines_to_protect: optional list of timelines
            to protect
        :return list: a list of removed WAL files
        """
        removed = []
        with self.server.xlogdb() as fxlogdb:
            xlogdb_new = fxlogdb.name + ".new"
            with open(xlogdb_new, 'w') as fxlogdb_new:
                for line in fxlogdb:
                    wal_info = WalFileInfo.from_xlogdb_line(line)
                    if not xlog.is_any_xlog_file(wal_info.name):
                        output.error(
                            "invalid xlog segment name %r\n"
                            "HINT: Please run \"barman rebuild-xlogdb %s\" "
                            "to solve this issue",
                            wal_info.name, self.config.name)
                        continue

                    # Keeps the WAL segment if it is a history file
                    keep = xlog.is_history_file(wal_info.name)

                    # Keeps the WAL segment if its timeline is in
                    # `timelines_to_protect`
                    if timelines_to_protect:
                        tli, _, _ = xlog.decode_segment_name(wal_info.name)
                        keep |= tli in timelines_to_protect

                    # Keeps the WAL segment if it is a newer
                    # than the given backup (the first available)
                    if backup_info:
                        keep |= wal_info.name >= backup_info.begin_wal

                    # If the file has to be kept write it in the new xlogdb
                    # otherwise delete it  and record it in the removed list
                    if keep:
                        fxlogdb_new.write(wal_info.to_xlogdb_line())
                    else:
                        self.delete_wal(wal_info)
                        removed.append(wal_info.name)
                fxlogdb_new.flush()
                os.fsync(fxlogdb_new.fileno())
            shutil.move(xlogdb_new, fxlogdb.name)
            fsync_dir(os.path.dirname(fxlogdb.name))
        return removed
Ejemplo n.º 9
0
    def stop_backup(self, backup_info):
        """
        Stop backup wrapper

        :param barman.infofile.BackupInfo backup_info: backup_info object
        """
        if BackupOptions.CONCURRENT_BACKUP not in self.config.backup_options:
            stop_row = self.pg_stop_backup()
            if stop_row:
                stop_xlog, stop_file_name, stop_file_offset, stop_time = \
                    stop_row
                backup_info.set_attribute('end_time', stop_time)
                backup_info.set_attribute('end_xlog', stop_xlog)
                backup_info.set_attribute('end_wal', stop_file_name)
                backup_info.set_attribute('end_offset', stop_file_offset)
            else:
                raise Exception('Cannot terminate exclusive backup. You might '
                                'have to manually execute pg_stop_backup() on '
                                'your PostgreSQL server')
        else:
            stop_row = self.pgespresso_stop_backup(backup_info.backup_label)
            if stop_row:
                end_wal, stop_time = stop_row
                decoded_segment = xlog.decode_segment_name(end_wal)
                backup_info.set_attribute('end_time', stop_time)
                backup_info.set_attribute('end_xlog',
                                          "%X/%X" % (decoded_segment[1],
                                                     (decoded_segment[
                                                      2] + 1) << 24))
                backup_info.set_attribute('end_wal', end_wal)
                backup_info.set_attribute('end_offset', 0)
            else:
                raise Exception('Cannot terminate exclusive backup. You might '
                                'have to  manually execute '
                                'pg_espresso_abort_backup() on your PostgreSQL '
                                'server')
Ejemplo n.º 10
0
 def test_decode_segment_name(self):
     assert xlog.decode_segment_name(
         '000000000000000000000000') == [0, 0, 0]
     assert xlog.decode_segment_name(
         '000000010000000100000001') == [1, 1, 1]
     assert xlog.decode_segment_name(
         '0000000A0000000A0000000A') == [10, 10, 10]
     assert xlog.decode_segment_name(
         '000000110000001100000011') == [17, 17, 17]
     assert xlog.decode_segment_name(
         '000000000000000200000001') == [0, 2, 1]
     assert xlog.decode_segment_name(
         '000000010000000000000002') == [1, 0, 2]
     assert xlog.decode_segment_name(
         '000000020000000100000000') == [2, 1, 0]
     assert xlog.decode_segment_name(
         '00000001000000000000000A.00000020.backup') == [1, 0, 10]
     assert xlog.decode_segment_name(
         '00000001.history') == [1, None, None]
     with pytest.raises(xlog.BadXlogSegmentName):
         xlog.decode_segment_name('00000000000000000000000')
     with pytest.raises(xlog.BadXlogSegmentName):
         xlog.decode_segment_name('0000000000000000000000000')
     with pytest.raises(xlog.BadXlogSegmentName):
         xlog.decode_segment_name('000000000000X00000000000')