class PathDeletionCommand(with_metaclass(ABCMeta, object)): """ Stand-alone object that will execute delete operation on a self contained path """ @abstractmethod def delete(self): """
class KeepManager(with_metaclass(ABCMeta, object)): """Abstract base class which defines the KeepManager interface""" ANNOTATION_KEY = "keep" TARGET_FULL = "full" TARGET_STANDALONE = "standalone" supported_targets = (TARGET_FULL, TARGET_STANDALONE) @abstractmethod def should_keep_backup(self, backup_id): pass @abstractmethod def keep_backup(self, backup_id, target): pass @abstractmethod def get_keep_target(self, backup_id): pass @abstractmethod def release_keep(self, backup_id): pass
class AnnotationManager(with_metaclass(ABCMeta)): """ This abstract base class defines the AnnotationManager interface which provides methods for read, write and delete of annotations for a given backup. """ @abstractmethod def put_annotation(self, backup_id, key, value): """Add an annotation""" @abstractmethod def get_annotation(self, backup_id, key): """Get the value of an annotation""" @abstractmethod def delete_annotation(self, backup_id, key): """Delete an annotation"""
class ChunkedCompressor(with_metaclass(ABCMeta, object)): """ Base class for all ChunkedCompressors """ @abstractmethod def add_chunk(self, data): """ Compresses the supplied data and returns all the compressed bytes. :param bytes data: The chunk of data to be compressed :return: The compressed data :rtype: bytes """ @abstractmethod def decompress(self, data): """
class RemoteStatusMixin(with_metaclass(ABCMeta, object)): """ Abstract base class that implements remote status capabilities following the Mixin pattern. """ def __init__(self, *args, **kwargs): """ Base constructor (Mixin pattern) """ self._remote_status = None super(RemoteStatusMixin, self).__init__(*args, **kwargs) @abstractmethod def fetch_remote_status(self): """ Retrieve status information from the remote component The implementation of this method must not raise any exception in case of errors, but should set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ def get_remote_status(self): """ Get the status of the remote component This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ if self._remote_status is None: self._remote_status = self.fetch_remote_status() return self._remote_status def reset_remote_status(self): """ Reset the cached result """ self._remote_status = None
class FileManager(with_metaclass(ABCMeta)): @abstractmethod def file_exist(self, file_path): """ Tests if file exists :param file_path: File path :type file_path: string :return: True if file exists False otherwise :rtype: bool """ @abstractmethod def get_file_stats(self, file_path): """ Tests if file exists :param file_path: File path :type file_path: string :return: :rtype: FileStats """ @abstractmethod def get_file_list(self, path): """ List all files within a path, including subdirectories :param path: Path to analyze :type path: string :return: List of file path :rtype: list """ @abstractmethod def get_file_content(self, file_path, file_mode="rb"): """ """ @abstractmethod def save_content_to_file(self, file_path, content, file_mode="wb"): """ """
class Compressor(with_metaclass(ABCMeta, object)): """ Base class for all the compressors """ MAGIC = None def __init__(self, config, compression, path=None): self.config = config self.compression = compression self.path = path @classmethod def validate(cls, file_start): """ Guess if the first bytes of a file are compatible with the compression implemented by this class :param file_start: a binary string representing the first few bytes of a file :rtype: bool """ return cls.MAGIC and file_start.startswith(cls.MAGIC) @abstractmethod def compress(self, src, dst): """ Abstract Method for compression method :param str src: source file path :param str dst: destination file path """ @abstractmethod def decompress(self, src, dst): """
class PgBaseBackupCompression(with_metaclass(ABCMeta, object)): """ Represents the pg_basebackup compression options and provides functionality required by the backup process which depends on those options. """ def __init__(self, config): """ Constructor for the PgBaseBackupCompression abstract base class. :param barman.config.ServerConfig config: the server configuration """ self.type = config.backup_compression self.format = config.backup_compression_format self.level = config.backup_compression_level self.location = config.backup_compression_location @abstractproperty def suffix(self): """The filename suffix expected for this compression""" def with_suffix(self, basename): """ Append the suffix to the supplied basename. :param str basename: The basename (without compression suffix) of the file to be opened. """ return ".".join((basename, self.suffix)) @abstractmethod @contextmanager def open(self, basename): """ Open file at path/basename for reading. :param str basename: The basename (without compression suffix) of the file to be opened. """ def validate(self, server, remote_status): """ Validate pg_basebackup compression options. :param barman.server.Server server: the server for which the compression options should be validated. :param dict remote_status: the status of the pg_basebackup command """ if self.location is not None and self.location == "server": # "backup_location = server" requires pg_basebackup >= 15 if remote_status["pg_basebackup_version"] < Version("15"): server.config.disabled = True server.config.msg_list.append( "backup_compression_location = server requires " "pg_basebackup 15 or greater") # "backup_location = server" requires PostgreSQL >= 15 if server.postgres.server_version < 150000: server.config.disabled = True server.config.msg_list.append( "backup_compression_location = server requires " "PostgreSQL 15 or greater") # plain backup format is only allowed when compression is on the server if self.format == "plain" and self.location != "server": server.config.disabled = True server.config.msg_list.append( "backup_compression_format plain is not compatible with " "backup_compression_location %s" % self.location)
class PostgreSQL(with_metaclass(ABCMeta, RemoteStatusMixin)): """ This abstract class represents a generic interface to a PostgreSQL server. """ CHECK_QUERY = 'SELECT 1' def __init__(self, config, conninfo): """ Abstract base class constructor for PostgreSQL interface. :param barman.config.ServerConfig config: the server configuration :param str conninfo: Connection information (aka DSN) """ super(PostgreSQL, self).__init__() assert conninfo self.config = config self.conninfo = conninfo self._conn = None self.allow_reconnect = True # Build a dictionary with connection info parameters # This is mainly used to speed up search in conninfo try: self.conn_parameters = self.parse_dsn(conninfo) except (ValueError, TypeError) as e: _logger.debug(e) raise ConninfoException('Cannot connect to postgres: "%s" ' 'is not a valid connection string' % conninfo) @staticmethod def parse_dsn(dsn): """ Parse connection parameters from 'conninfo' :param str dsn: Connection information (aka DSN) :rtype: dict[str,str] """ # TODO: this might be made more robust in the future return dict(x.split('=', 1) for x in dsn.split()) @staticmethod def encode_dsn(parameters): """ Build a connection string from a dictionary of connection parameters :param dict[str,str] parameters: Connection parameters :rtype: str """ # TODO: this might be made more robust in the future return ' '.join( ["%s=%s" % (k, v) for k, v in sorted(parameters.items())]) def get_connection_string(self, application_name=None): """ Return the connection string, adding the application_name parameter if requested, unless already defined by user in the connection string :param str application_name: the application_name to add :return str: the connection string """ conn_string = self.conninfo # check if the application name is already defined by user if application_name and 'application_name' not in self.conn_parameters: # Then add the it to the connection string conn_string += ' application_name=%s' % application_name return conn_string def connect(self): """ Generic function for Postgres connection (using psycopg2) """ if not self._check_connection(): try: self._conn = psycopg2.connect(self.conninfo) # If psycopg2 fails to connect to the host, # raise the appropriate exception except psycopg2.DatabaseError as e: raise PostgresConnectionError(str(e).strip()) # Register the connection to the list of live connections _live_connections.append(self) return self._conn def _check_connection(self): """ Return false if the connection is broken :rtype: bool """ # If the connection is not present return False if not self._conn: return False # Check if the connection works by running 'SELECT 1' cursor = None try: cursor = self._conn.cursor() cursor.execute(self.CHECK_QUERY) except psycopg2.DatabaseError: # Connection is broken, so we need to reconnect self.close() # Raise an error if reconnect is not allowed if not self.allow_reconnect: raise PostgresConnectionError( "Connection lost, reconnection not allowed") return False finally: if cursor: cursor.close() return True def close(self): """ Close the connection to PostgreSQL """ if self._conn: # If the connection is still alive, rollback and close it if not self._conn.closed: if self._conn.status == STATUS_IN_TRANSACTION: self._conn.rollback() self._conn.close() # Remove the connection from the live connections list self._conn = None _live_connections.remove(self) def _cursor(self, *args, **kwargs): """ Return a cursor """ conn = self.connect() return conn.cursor(*args, **kwargs) @property def server_version(self): """ Version of PostgreSQL (returned by psycopg2) """ conn = self.connect() return conn.server_version @property def server_txt_version(self): """ Human readable version of PostgreSQL (calculated from server_version) :rtype: str|None """ try: conn = self.connect() major = int(conn.server_version / 10000) minor = int(conn.server_version / 100 % 100) patch = int(conn.server_version % 100) if major < 10: return "%d.%d.%d" % (major, minor, patch) if minor != 0: _logger.warning("Unexpected non zero minor version %s in %s", minor, conn.server_version) return "%d.%d" % (major, patch) except PostgresConnectionError as e: _logger.debug("Error retrieving PostgreSQL version: %s", str(e).strip()) return None @property def server_major_version(self): """ PostgreSQL major version (calculated from server_txt_version) :rtype: str|None """ result = self.server_txt_version if result is not None: return simplify_version(result) return None
class WalArchiver(with_metaclass(ABCMeta, RemoteStatusMixin)): """ Base class for WAL archiver objects """ def __init__(self, backup_manager, name): """ Base class init method. :param backup_manager: The backup manager :param name: The name of this archiver :return: """ self.backup_manager = backup_manager self.server = backup_manager.server self.config = backup_manager.config self.name = name super(WalArchiver, self).__init__() def receive_wal(self, reset=False): """ Manage reception of WAL files. Does nothing by default. Some archiver classes, like the StreamingWalArchiver, have a full implementation. :param bool reset: When set, resets the status of receive-wal :raise ArchiverFailure: when something goes wrong """ def archive(self, fxlogdb, verbose=True): """ Archive WAL files, discarding duplicates or those that are not valid. :param file fxlogdb: File object for xlogdb interactions :param boolean verbose: Flag for verbose output """ compressor = self.backup_manager.compression_manager.get_compressor() stamp = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') processed = 0 header = "Processing xlog segments from %s for %s" % (self.name, self.config.name) # Get the next batch of WAL files to be processed batch = self.get_next_batch() # Analyse the batch and properly log the information if batch.size: if batch.size > batch.run_size: # Batch mode enabled _logger.info( "Found %s xlog segments from %s for %s." " Archive a batch of %s segments in this run.", batch.size, self.name, self.config.name, batch.run_size) header += " (batch size: %s)" % batch.run_size else: # Single run mode (traditional) _logger.info( "Found %s xlog segments from %s for %s." " Archive all segments in one run.", batch.size, self.name, self.config.name) else: _logger.info("No xlog segments found from %s for %s.", self.name, self.config.name) # Print the header (verbose mode) if verbose: output.info(header, log=False) # Loop through all available WAL files for wal_info in batch: # Print the header (non verbose mode) if not processed and not verbose: output.info(header, log=False) # Exit when archive batch size is reached if processed >= batch.run_size: _logger.debug( "Batch size reached (%s) - " "Exit %s process for %s", batch.batch_size, self.name, self.config.name) break processed += 1 # Report to the user the WAL file we are archiving output.info("\t%s", wal_info.name, log=False) _logger.info("Archiving segment %s of %s from %s: %s/%s", processed, batch.run_size, self.name, self.config.name, wal_info.name) # Archive the WAL file try: self.archive_wal(compressor, wal_info) except MatchingDuplicateWalFile: # We already have this file. Simply unlink the file. os.unlink(wal_info.orig_filename) continue except DuplicateWalFile: output.info( "\tError: %s is already present in server %s. " "File moved to errors directory.", wal_info.name, self.config.name) error_dst = os.path.join( self.config.errors_directory, "%s.%s.duplicate" % (wal_info.name, stamp)) # TODO: cover corner case of duplication (unlikely, # but theoretically possible) shutil.move(wal_info.orig_filename, error_dst) continue except AbortedRetryHookScript as e: _logger.warning("Archiving of %s/%s aborted by " "pre_archive_retry_script." "Reason: %s" % (self.config.name, wal_info.name, e)) return # Updates the information of the WAL archive with # the latest segments fxlogdb.write(wal_info.to_xlogdb_line()) # flush and fsync for every line fxlogdb.flush() os.fsync(fxlogdb.fileno()) if processed: _logger.debug("Archived %s out of %s xlog segments from %s for %s", processed, batch.size, self.name, self.config.name) elif verbose: output.info("\tno file found", log=False) if batch.errors: output.info( "Some unknown objects have been found while " "processing xlog segments for %s. " "Objects moved to errors directory:", self.config.name, log=False) # Log unexpected files _logger.warning( "Archiver is about to move %s unexpected file(s) " "to errors directory for %s from %s", len(batch.errors), self.config.name, self.name) for error in batch.errors: basename = os.path.basename(error) output.info("\t%s", basename, log=False) # Print informative log line. _logger.warning("Moving unexpected file for %s from %s: %s", self.config.name, self.name, basename) error_dst = os.path.join(self.config.errors_directory, "%s.%s.unknown" % (basename, stamp)) try: shutil.move(error, error_dst) except IOError as e: if e.errno == errno.ENOENT: _logger.warning('%s not found' % error) def archive_wal(self, compressor, wal_info): """ Archive a WAL segment and update the wal_info object :param compressor: the compressor for the file (if any) :param WalFileInfo wal_info: the WAL file is being processed """ src_file = wal_info.orig_filename src_dir = os.path.dirname(src_file) dst_file = wal_info.fullpath(self.server) tmp_file = dst_file + '.tmp' dst_dir = os.path.dirname(dst_file) error = None try: # Run the pre_archive_script if present. script = HookScriptRunner(self.backup_manager, 'archive_script', 'pre') script.env_from_wal_info(wal_info, src_file) script.run() # Run the pre_archive_retry_script if present. retry_script = RetryHookScriptRunner(self.backup_manager, 'archive_retry_script', 'pre') retry_script.env_from_wal_info(wal_info, src_file) retry_script.run() # Check if destination already exists if os.path.exists(dst_file): src_uncompressed = src_file dst_uncompressed = dst_file dst_info = WalFileInfo.from_file(dst_file) try: comp_manager = self.backup_manager.compression_manager if dst_info.compression is not None: dst_uncompressed = dst_file + '.uncompressed' comp_manager.get_compressor( compression=dst_info.compression).decompress( dst_file, dst_uncompressed) if wal_info.compression: src_uncompressed = src_file + '.uncompressed' comp_manager.get_compressor( compression=wal_info.compression).decompress( src_file, src_uncompressed) # Directly compare files. # When the files are identical # raise a MatchingDuplicateWalFile exception, # otherwise raise a DuplicateWalFile exception. if filecmp.cmp(dst_uncompressed, src_uncompressed): raise MatchingDuplicateWalFile(wal_info) else: raise DuplicateWalFile(wal_info) finally: if src_uncompressed != src_file: os.unlink(src_uncompressed) if dst_uncompressed != dst_file: os.unlink(dst_uncompressed) mkpath(dst_dir) # Compress the file only if not already compressed if compressor and not wal_info.compression: compressor.compress(src_file, tmp_file) shutil.copystat(src_file, tmp_file) os.rename(tmp_file, dst_file) os.unlink(src_file) # Update wal_info stat = os.stat(dst_file) wal_info.size = stat.st_size wal_info.compression = compressor.compression else: # Try to atomically rename the file. If successful, # the renaming will be an atomic operation # (this is a POSIX requirement). try: os.rename(src_file, dst_file) except OSError: # Source and destination are probably on different # filesystems shutil.copy2(src_file, tmp_file) os.rename(tmp_file, dst_file) os.unlink(src_file) # At this point the original file has been removed wal_info.orig_filename = None # Execute fsync() on the archived WAL file file_fd = os.open(dst_file, os.O_RDONLY) os.fsync(file_fd) os.close(file_fd) # Execute fsync() on the archived WAL containing directory fsync_dir(dst_dir) # Execute fsync() also on the incoming directory fsync_dir(src_dir) except Exception as e: # In case of failure save the exception for the post scripts error = e raise # Ensure the execution of the post_archive_retry_script and # the post_archive_script finally: # Run the post_archive_retry_script if present. try: retry_script = RetryHookScriptRunner(self, 'archive_retry_script', 'post') retry_script.env_from_wal_info(wal_info, dst_file, error) retry_script.run() except AbortedRetryHookScript as e: # Ignore the ABORT_STOP as it is a post-hook operation _logger.warning( "Ignoring stop request after receiving " "abort (exit code %d) from post-archive " "retry hook script: %s", e.hook.exit_status, e.hook.script) # Run the post_archive_script if present. script = HookScriptRunner(self, 'archive_script', 'post', error) script.env_from_wal_info(wal_info, dst_file) script.run() @abstractmethod def get_next_batch(self): """ Return a WalArchiverQueue containing the WAL files to be archived. :rtype: WalArchiverQueue """ @abstractmethod def check(self, check_strategy): """ Perform specific checks for the archiver - invoked by server.check_postgres :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ @abstractmethod def status(self): """ Set additional status info - invoked by Server.status() """ @staticmethod def summarise_error_files(error_files): """ Summarise a error files list :param list[str] error_files: Error files list to summarise :return str: A summary, None if there are no error files """ if not error_files: return None # The default value for this dictionary will be 0 counters = collections.defaultdict(int) # Count the file types for name in error_files: if name.endswith(".error"): counters['not relevant'] += 1 elif name.endswith(".duplicate"): counters['duplicates'] += 1 elif name.endswith(".unknown"): counters['unknown'] += 1 else: counters['unknown failure'] += 1 # Return a summary list of the form: "item a: 2, item b: 5" return ', '.join("%s: %s" % entry for entry in counters.items())
class RetentionPolicy(with_metaclass(ABCMeta, object)): """Abstract base class for retention policies""" def __init__(self, mode, unit, value, context, server): """Constructor of the retention policy base class""" self.mode = mode self.unit = unit self.value = int(value) self.context = context self.server = server self._first_backup = None self._first_wal = None def report(self, source=None, context=None): """Report obsolete/valid objects according to the retention policy""" if context is None: context = self.context # Overrides the list of available backups if source is None: source = self.server.get_available_backups( BackupInfo.STATUS_NOT_EMPTY) if context == 'BASE': return self._backup_report(source) elif context == 'WAL': return self._wal_report() else: raise ValueError('Invalid context %s', context) def backup_status(self, backup_id): """Report the status of a backup according to the retention policy""" source = self.server.get_available_backups(BackupInfo.STATUS_NOT_EMPTY) if self.context == 'BASE': return self._backup_report(source)[backup_id] else: return BackupInfo.NONE def first_backup(self): """Returns the first valid backup according to retention policies""" if not self._first_backup: self.report(context='BASE') return self._first_backup def first_wal(self): """Returns the first valid WAL according to retention policies""" if not self._first_wal: self.report(context='WAL') return self._first_wal @abstractmethod def __str__(self): """String representation""" pass @abstractmethod def debug(self): """Debug information""" pass @abstractmethod def _backup_report(self, source): """Report obsolete/valid backups according to the retention policy""" pass @abstractmethod def _wal_report(self): """Report obsolete/valid WALs according to the retention policy""" pass @classmethod def create(cls, server, option, value): """ If given option and value from the configuration file match, creates the retention policy object for the given server """ # using @abstractclassmethod from python3 would be better here raise NotImplementedError( 'The class %s must override the create() class method', cls.__name__) def to_json(self): """ Output representation of the obj for JSON serialization """ return "%s %s %s" % (self.mode, self.value, self.unit)
class PostgreSQL(with_metaclass(ABCMeta, RemoteStatusMixin)): """ This abstract class represents a generic interface to a PostgreSQL server. """ def __init__(self, config, conninfo): """ Abstract base class constructor for PostgreSQL interface. :param barman.config.ServerConfig config: the server configuration :param str conninfo: Connection information (aka DSN) """ super(PostgreSQL, self).__init__() assert conninfo self.config = config self.conninfo = conninfo self._conn = None # Build a dictionary with connection info parameters # This is mainly used to speed up search in conninfo try: self.conn_parameters = self.parse_dsn(conninfo) except (ValueError, TypeError) as e: _logger.debug(e) raise ConninfoException('Cannot connect to postgres: "%s" ' 'is not a valid connection string' % conninfo) @staticmethod def parse_dsn(dsn): """ Parse connection parameters from 'conninfo' :param str dsn: Connection information (aka DSN) :rtype: dict[str,str] """ # TODO: this might be made more robust in the future return dict(x.split('=', 1) for x in dsn.split(' ')) def connect(self): """ Generic function for Postgres connection (using psycopg2) """ if not self._conn: try: self._conn = psycopg2.connect(self.conninfo) # If psycopg2 fails to connect to the host, # raise the appropriate exception except psycopg2.DatabaseError as e: raise PostgresConnectionError( "Cannot connect to postgres: %s" % str(e).strip()) # Register the connection to the live connections list _live_connections.append(self) return self._conn def close(self): """ Close the connection to PostgreSQL """ if self._conn: if self._conn.status == STATUS_IN_TRANSACTION: self._conn.rollback() self._conn.close() self._conn = None # Remove the connection from the live connections list _live_connections.remove(self) def _cursor(self, *args, **kwargs): """ Return a cursor """ conn = self.connect() return conn.cursor(*args, **kwargs) @property def server_version(self): """ Version of PostgreSQL (returned by psycopg2) """ conn = self.connect() return conn.server_version @property def server_txt_version(self): """ Human readable version of PostgreSQL (calculated from server_version) :rtype: str|None """ try: conn = self.connect() major = int(conn.server_version / 10000) minor = int(conn.server_version / 100 % 100) patch = int(conn.server_version % 100) return "%d.%d.%d" % (major, minor, patch) except PostgresConnectionError as e: _logger.debug("Error retrieving PostgreSQL version: %s", str(e).strip()) return None @property def server_major_version(self): """ PostgreSQL major version (calculated from server_txt_version) :rtype: str|None """ result = self.server_txt_version if result is not None: return simplify_version(result) return None