예제 #1
0
    def _reset_streaming_status(self, postgres_status, streaming_status):
        """
        Reset the status of receive-wal by removing the .partial file that
        is marking the current position and creating one that is current with
        the PostgreSQL insert location
        """
        current_wal = xlog.location_to_xlogfile_name_offset(
            postgres_status["current_lsn"],
            streaming_status["timeline"],
            postgres_status["xlog_segment_size"],
        )["file_name"]
        restart_wal = current_wal
        if (postgres_status["replication_slot"]
                and postgres_status["replication_slot"].restart_lsn):
            restart_wal = xlog.location_to_xlogfile_name_offset(
                postgres_status["replication_slot"].restart_lsn,
                streaming_status["timeline"],
                postgres_status["xlog_segment_size"],
            )["file_name"]
        restart_path = os.path.join(self.config.streaming_wals_directory,
                                    restart_wal)
        restart_partial_path = restart_path + ".partial"
        wal_files = sorted(glob(
            os.path.join(self.config.streaming_wals_directory, "*")),
                           reverse=True)

        # Pick the newer file
        last = None
        for last in wal_files:
            if xlog.is_wal_file(last) or xlog.is_partial_file(last):
                break

        # Check if the status is already up-to-date
        if not last or last == restart_partial_path or last == restart_path:
            output.info("Nothing to do. Position of receive-wal is aligned.")
            return

        if os.path.basename(last) > current_wal:
            output.error(
                "The receive-wal position is ahead of PostgreSQL "
                "current WAL lsn (%s > %s)",
                os.path.basename(last),
                postgres_status["current_xlog"],
            )
            return

        output.info("Resetting receive-wal directory status")
        if xlog.is_partial_file(last):
            output.info("Removing status file %s" % last)
            os.unlink(last)
        output.info("Creating status file %s" % restart_partial_path)
        open(restart_partial_path, "w").close()
예제 #2
0
 def test_partial_file(self):
     assert not xlog.is_partial_file("000000000000000200000001")
     assert xlog.is_partial_file("00000001000000000000000A.partial")
     assert xlog.is_partial_file("test/00000001000000000000000A.partial")
     assert not xlog.is_partial_file("00000002.history")
     assert not xlog.is_partial_file("00000000000000000000000.partial")
     assert not xlog.is_partial_file("0000000000000000000000000.partial")
     assert not xlog.is_partial_file("000000000000X00000000000.partial")
     assert not xlog.is_partial_file("00000001000000000000000A.00000020.partial")
     assert not xlog.is_any_xlog_file("test.00000001000000000000000A.partial")
     assert not xlog.is_partial_file("00000001.partial")
예제 #3
0
    def _truncate_partial_file_if_needed(self, xlog_segment_size):
        """
        Truncate .partial WAL file if size is not 0 or xlog_segment_size

        :param int xlog_segment_size:
        """
        # Retrieve the partial list (only one is expected)
        partial_files = glob(
            os.path.join(self.config.streaming_wals_directory, "*.partial"))

        # Take the last partial file, ignoring wrongly formatted file names
        last_partial = None
        for partial in partial_files:
            if not is_partial_file(partial):
                continue
            if not last_partial or partial > last_partial:
                last_partial = partial

        # Skip further work if there is no good partial file
        if not last_partial:
            return

        # If size is either 0 or wal_segment_size everything is fine...
        partial_size = os.path.getsize(last_partial)
        if partial_size == 0 or partial_size == xlog_segment_size:
            return

        # otherwise truncate the file to be empty. This is safe because
        # pg_receivewal pads the file to the full size before start writing.
        output.info("Truncating partial file %s that has wrong size %s "
                    "while %s was expected." %
                    (last_partial, partial_size, xlog_segment_size))
        open(last_partial, "wb").close()
예제 #4
0
    def get_next_batch(self):
        """
        Returns the next batch of WAL files that have been archived via
        streaming replication (in the 'streaming' directory)

        This method always leaves one file in the "streaming" directory,
        because the 'pg_receivexlog' process needs at least one file to
        detect the current streaming position after a restart.

        :return: WalArchiverQueue: list of WAL files
        """
        # Get the batch size from configuration (0 = unlimited)
        batch_size = self.config.streaming_archiver_batch_size
        # List and sort all files in the incoming directory
        file_names = glob(
            os.path.join(self.config.streaming_wals_directory, '*'))
        file_names.sort()

        # Process anything that looks like a valid WAL file,
        # including partial ones and history files.
        # Anything else is treated like an error/anomaly
        files = []
        skip = []
        errors = []
        for file_name in file_names:
            # Ignore temporary files
            if file_name.endswith('.tmp'):
                continue
            # If the file doesn't exist, it has been renamed/removed while
            # we were reading the directory. Ignore it.
            if not os.path.exists(file_name):
                continue
            if not os.path.isfile(file_name):
                errors.append(file_name)
            elif xlog.is_partial_file(file_name):
                skip.append(file_name)
            elif xlog.is_any_xlog_file(file_name):
                files.append(file_name)
            else:
                errors.append(file_name)
        # In case of more than a partial file, keep the last
        # and treat the rest as normal files
        if len(skip) > 1:
            partials = skip[:-1]
            _logger.info('Archiving partial files for server %s: %s' %
                         (self.config.name, ", ".join(
                             [os.path.basename(f) for f in partials])))
            files.extend(partials)
            skip = skip[-1:]

        # Keep the last full WAL file in case no partial file is present
        elif len(skip) == 0 and files:
            skip.append(files.pop())

        # Build the list of WalFileInfo
        wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
        return WalArchiverQueue(wal_files,
                                batch_size=batch_size,
                                errors=errors,
                                skip=skip)
예제 #5
0
 def test_partial_file(self):
     assert not xlog.is_partial_file('000000000000000200000001')
     assert xlog.is_partial_file('00000001000000000000000A.partial')
     assert xlog.is_partial_file('test/00000001000000000000000A.partial')
     assert not xlog.is_partial_file('00000002.history')
     assert not xlog.is_partial_file('00000000000000000000000.partial')
     assert not xlog.is_partial_file('0000000000000000000000000.partial')
     assert not xlog.is_partial_file('000000000000X00000000000.partial')
     assert not xlog.is_partial_file(
         '00000001000000000000000A.00000020.partial')
     assert not xlog.is_any_xlog_file(
         'test.00000001000000000000000A.partial')
     assert not xlog.is_partial_file('00000001.partial')
예제 #6
0
 def test_partial_file(self):
     assert not xlog.is_partial_file('000000000000000200000001')
     assert xlog.is_partial_file('00000001000000000000000A.partial')
     assert xlog.is_partial_file('test/00000001000000000000000A.partial')
     assert not xlog.is_partial_file('00000002.history')
     assert not xlog.is_partial_file('00000000000000000000000.partial')
     assert not xlog.is_partial_file('0000000000000000000000000.partial')
     assert not xlog.is_partial_file('000000000000X00000000000.partial')
     assert not xlog.is_partial_file(
         '00000001000000000000000A.00000020.partial')
     assert not xlog.is_any_xlog_file(
         'test.00000001000000000000000A.partial')
     assert not xlog.is_partial_file('00000001.partial')
예제 #7
0
    def get_next_batch(self):
        """
        Returns the next batch of WAL files that have been archived via
        streaming replication (in the 'streaming' directory)

        This method always leaves one file in the "streaming" directory,
        because the 'pg_receivexlog' process needs at least one file to
        detect the current streaming position after a restart.

        :return: WalArchiverQueue: list of WAL files
        """
        # Get the batch size from configuration (0 = unlimited)
        batch_size = self.config.streaming_archiver_batch_size
        # List and sort all files in the incoming directory
        file_names = glob(os.path.join(
            self.config.streaming_wals_directory, '*'))
        file_names.sort()

        # Process anything that looks like a valid WAL file,
        # including partial ones and history files.
        # Anything else is treated like an error/anomaly
        files = []
        skip = []
        errors = []
        for file_name in file_names:
            if not os.path.isfile(file_name):
                errors.append(file_name)
            elif xlog.is_partial_file(file_name):
                skip.append(file_name)
            elif xlog.is_any_xlog_file(file_name):
                files.append(file_name)
            else:
                errors.append(file_name)
        # In case of more than a partial file, keep the last
        # and treat the rest as errors
        if len(skip) > 1:
            errors.extend(skip[:-1])
            _logger.warning('Multiple partial files found for server %s: %s' %
                            (self.config.name,
                             ", ".join([os.path.basename(f) for f in errors])))
            skip = skip[-1:]

        # Keep the last full WAL file in case no partial file is present
        elif len(skip) == 0 and files:
            skip.append(files.pop())

        # Build the list of WalFileInfo
        wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
        return WalArchiverQueue(wal_files,
                                batch_size=batch_size,
                                errors=errors, skip=skip)
예제 #8
0
    def get_next_batch(self):
        """
        Returns the next batch of WAL files that have been archived via
        streaming replication (in the 'streaming' directory)

        This method always leaves one file in the "streaming" directory,
        because the 'pg_receivexlog' process needs at least one file to
        detect the current streaming position after a restart.

        :return: WalArchiverBatch: list of WAL files
        """
        # List and sort all files in the incoming directory
        file_names = glob(os.path.join(
            self.config.streaming_wals_directory, '*'))
        file_names.sort()

        # Process anything that looks like a valid WAL file,
        # including partial ones.
        # Anything else is treated like an error/anomaly
        files = []
        skip = []
        errors = []
        for file_name in file_names:
            if xlog.is_wal_file(file_name) and os.path.isfile(file_name):
                files.append(file_name)
            elif xlog.is_partial_file(file_name) and os.path.isfile(file_name):
                skip.append(file_name)
            else:
                errors.append(file_name)
        # In case of more than a partial file, keep the last
        # and treat the rest as errors
        if len(skip) > 1:
            errors.extend(skip[:-1])
            skip = skip[-1:]

        # Keep the last full WAL file in case no partial file is present
        elif len(skip) == 0 and files:
            skip.append(files.pop())

        # Build the list of WalFileInfo
        wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
        return WalArchiverBatch(wal_files, errors=errors, skip=skip)
예제 #9
0
    def get_next_batch(self):
        """
        Returns the next batch of WAL files that have been archived via
        streaming replication (in the 'streaming' directory)

        This method always leaves one file in the "streaming" directory,
        because the 'pg_receivexlog' process needs at least one file to
        detect the current streaming position after a restart.

        :return: WalArchiverBatch: list of WAL files
        """
        # List and sort all files in the incoming directory
        file_names = glob(
            os.path.join(self.config.streaming_wals_directory, '*'))
        file_names.sort()

        # Process anything that looks like a valid WAL file,
        # including partial ones.
        # Anything else is treated like an error/anomaly
        files = []
        skip = []
        errors = []
        for file_name in file_names:
            if xlog.is_wal_file(file_name) and os.path.isfile(file_name):
                files.append(file_name)
            elif xlog.is_partial_file(file_name) and os.path.isfile(file_name):
                skip.append(file_name)
            else:
                errors.append(file_name)
        # In case of more than a partial file, keep the last
        # and treat the rest as errors
        if len(skip) > 1:
            errors.extend(skip[:-1])
            skip = skip[-1:]

        # Keep the last full WAL file in case no partial file is present
        elif len(skip) == 0 and files:
            skip.append(files.pop())

        # Build the list of WalFileInfo
        wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
        return WalArchiverBatch(wal_files, errors=errors, skip=skip)