def test_get_version_info(self, find_command_mock): """ Test the `get_version_info` class method """ command_mock = find_command_mock.return_value command_mock.cmd = '/some/path/pg_receivewal' command_mock.out = \ 'pg_receivewal (PostgreSQL) 11.7 (ev1 12) (ev2 2:3.4)' # Test with normal output version_info = PgReceiveXlog.get_version_info() assert version_info['full_path'] == '/some/path/pg_receivewal' assert version_info['full_version'] == '11.7' assert version_info['major_version'] == '11' # Test with development branch command_mock.out = 'pg_receivewal 13devel' version_info = PgReceiveXlog.get_version_info() assert version_info['full_version'] == '13devel' assert version_info['major_version'] == '13' # Test with bad output command_mock.out = 'pg_receivewal' version_info = PgReceiveXlog.get_version_info() assert version_info['full_path'] == '/some/path/pg_receivewal' assert version_info['full_version'] is None assert version_info['major_version'] is None # Test with invocation error find_command_mock.side_effect = CommandFailedException version_info = PgReceiveXlog.get_version_info() assert version_info['full_path'] is None assert version_info['full_version'] is None assert version_info['major_version'] is None
def receive_wal(self, reset=False): """ Creates a PgReceiveXlog object and issues the pg_receivexlog command for a specific server :param bool reset: When set reset the status of receive-wal :raise ArchiverFailure: when something goes wrong """ # Ensure the presence of the destination directory mkpath(self.config.streaming_wals_directory) # Check if is a reset request if reset: self._reset_streaming_status() return # Execute basic sanity checks on PostgreSQL connection postgres_status = self.server.streaming.get_remote_status() if postgres_status["streaming_supported"] is None: raise ArchiverFailure( 'failed opening the PostgreSQL streaming connection') elif not postgres_status["streaming_supported"]: raise ArchiverFailure( 'PostgreSQL version too old (%s < 9.2)' % self.server.streaming.server_txt_version) # Execute basic sanity checks on pg_receivexlog remote_status = self.get_remote_status() if not remote_status["pg_receivexlog_installed"]: raise ArchiverFailure( 'pg_receivexlog not present in $PATH') if not remote_status['pg_receivexlog_compatible']: raise ArchiverFailure( 'pg_receivexlog version not compatible with ' 'PostgreSQL server version') # Make sure we are not wasting precious PostgreSQL resources self.server.postgres.close() self.server.streaming.close() _logger.info('Activating WAL archiving through streaming protocol') try: output_handler = PgReceiveXlog.make_output_handler( self.config.name + ': ') receive = PgReceiveXlog(remote_status['pg_receivexlog_path'], self.config.streaming_conninfo, self.config.streaming_wals_directory, out_handler=output_handler, err_handler=output_handler) receive.execute() except CommandFailedException as e: _logger.error(e) raise ArchiverFailure("pg_receivexlog exited with an error. " "Check the logs for more information.") except KeyboardInterrupt: # This is a normal termination, so there is nothing to do beside # informing the user. output.info('SIGINT received. Terminate gracefully.')
def fetch_remote_status(self): """ Execute checks for replication-based wal archiving This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ remote_status = dict.fromkeys( ('pg_receivexlog_compatible', 'pg_receivexlog_installed', 'pg_receivexlog_path', 'pg_receivexlog_version'), None) # Test pg_receivexlog existence version_info = PgReceiveXlog.get_version_info( self.backup_manager.server.path) if version_info['full_path']: remote_status["pg_receivexlog_installed"] = True remote_status["pg_receivexlog_path"] = version_info['full_path'] remote_status["pg_receivexlog_version"] = ( version_info['full_version']) pgreceivexlog_version = version_info['major_version'] else: remote_status["pg_receivexlog_installed"] = False return remote_status # Retrieve the PostgreSQL version pg_version = None if self.server.streaming is not None: pg_version = self.server.streaming.server_major_version # If one of the version is unknown we cannot compare them if pgreceivexlog_version is None or pg_version is None: return remote_status # pg_version is not None so transform into a Version object # for easier comparison between versions pg_version = Version(pg_version) # pg_receivexlog 9.2 is compatible only with PostgreSQL 9.2. if "9.2" == pg_version == pgreceivexlog_version: remote_status["pg_receivexlog_compatible"] = True # other versions are compatible with lesser versions of PostgreSQL # WARNING: The development versions of `pg_receivexlog` are considered # higher than the stable versions here, but this is not an issue # because it accepts everything that is less than # the `pg_receivexlog` version(e.g. '9.6' is less than '9.6devel') elif "9.2" < pg_version <= pgreceivexlog_version: remote_status["pg_receivexlog_compatible"] = True else: remote_status["pg_receivexlog_compatible"] = False return remote_status
def test_find_command(self, command_mock, which_mock): """ Test the `find_command` class method """ which_mapping = {} which_mock.side_effect = \ lambda cmd, path=None: which_mapping.get(cmd, None) # Neither pg_receivewal, neither pg_receivexlog are # available, and the result is a CommandFailedException with pytest.raises(CommandFailedException): PgReceiveXlog.find_command() # pg_receivexlog is available, but pg_receivewal is not which_mapping['pg_receivexlog'] = '/usr/bin/pg_receivexlog' command = PgReceiveXlog.find_command() assert command_mock.mock_calls == [ mock.call('/usr/bin/pg_receivexlog', check=True, path=None), mock.call()('--version'), ] assert command == command_mock.return_value # pg_receivewal is also available, but it's only a shim which_mapping['pg_receivewal'] = '/usr/bin/pg_receivewal' command_mock.reset_mock() command_mock.return_value.side_effect = [CommandFailedException, None] command = PgReceiveXlog.find_command() assert command_mock.mock_calls == [ mock.call('/usr/bin/pg_receivewal', check=True, path=None), mock.call()('--version'), mock.call('/usr/bin/pg_receivexlog', check=True, path=None), mock.call()('--version'), ] assert command == command_mock.return_value # pg_receivewal is available and works well command_mock.reset_mock() command_mock.return_value.side_effect = None command = PgReceiveXlog.find_command() assert command_mock.mock_calls == [ mock.call('/usr/bin/pg_receivewal', check=True, path=None), mock.call()('--version'), ] assert command == command_mock.return_value
def receive_wal(self): """ Creates a PgReceiveXlog object and issues the pg_receivexlog command for a specific server :raise ArchiverFailure: when something goes wrong """ # Execute basic sanity checks on PostgreSQL connection postgres_status = self.server.streaming.get_remote_status() if postgres_status["streaming_supported"] is None: raise ArchiverFailure( 'failed opening the PostgreSQL streaming connection') elif not postgres_status["streaming_supported"]: raise ArchiverFailure( 'PostgreSQL version too old (%s < 9.2)' % self.server.streaming.server_txt_version) # Execute basic sanity checks on pg_receivexlog remote_status = self.get_remote_status() if not remote_status["pg_receivexlog_installed"]: raise ArchiverFailure( 'pg_receivexlog not present in $PATH') if not remote_status['pg_receivexlog_compatible']: raise ArchiverFailure( 'pg_receivexlog version not compatible with ' 'PostgreSQL server version') # Make sure we are not wasting precious PostgreSQL resources self.server.postgres.close() self.server.streaming.close() _logger.info('Activating WAL archiving through streaming protocol') try: receive = PgReceiveXlog(remote_status['pg_receivexlog_path'], self.config.streaming_conninfo, self.config.streaming_wals_directory) receive.execute() except CommandFailedException as e: _logger.error(e) raise ArchiverFailure("pg_receivexlog exited with an error. " "Check the logs for more information.")
def receive_wal(self, reset=False): """ Creates a PgReceiveXlog object and issues the pg_receivexlog command for a specific server :param bool reset: When set reset the status of receive-wal :raise ArchiverFailure: when something goes wrong """ # Ensure the presence of the destination directory mkpath(self.config.streaming_wals_directory) # Check if is a reset request if reset: self._reset_streaming_status() return # Execute basic sanity checks on PostgreSQL connection streaming_status = self.server.streaming.get_remote_status() if streaming_status["streaming_supported"] is None: raise ArchiverFailure( 'failed opening the PostgreSQL streaming connection ' 'for server %s' % (self.config.name)) elif not streaming_status["streaming_supported"]: raise ArchiverFailure('PostgreSQL version too old (%s < 9.2)' % self.server.streaming.server_txt_version) # Execute basic sanity checks on pg_receivexlog remote_status = self.get_remote_status() if not remote_status["pg_receivexlog_installed"]: raise ArchiverFailure('pg_receivexlog not present in $PATH') if not remote_status['pg_receivexlog_compatible']: raise ArchiverFailure('pg_receivexlog version not compatible with ' 'PostgreSQL server version') # Execute sanity check on replication slot usage if self.config.slot_name: # Check if slots are supported if not remote_status['pg_receivexlog_supports_slots']: raise ArchiverFailure( 'Physical replication slot not supported by %s ' '(9.4 or higher is required)' % self.server.streaming.server_txt_version) # Check if the required slot exists postgres_status = self.server.postgres.get_remote_status() if postgres_status['replication_slot'] is None: raise ArchiverFailure( "replication slot '%s' doesn't exist. " "Please execute " "'barman receive-wal --create-slot %s'" % (self.config.slot_name, self.config.name)) # Check if the required slot is available if postgres_status['replication_slot'].active: raise ArchiverFailure( "replication slot '%s' is already in use" % (self.config.slot_name, )) # Make sure we are not wasting precious PostgreSQL resources self.server.close() _logger.info('Activating WAL archiving through streaming protocol') try: output_handler = PgReceiveXlog.make_output_handler( self.config.name + ': ') receive = PgReceiveXlog( connection=self.server.streaming, destination=self.config.streaming_wals_directory, command=remote_status['pg_receivexlog_path'], version=remote_status['pg_receivexlog_version'], app_name=self.config.streaming_archiver_name, path=self.server.path, slot_name=self.config.slot_name, synchronous=remote_status['pg_receivexlog_synchronous'], out_handler=output_handler, err_handler=output_handler) # Finally execute the pg_receivexlog process receive.execute() except CommandFailedException as e: # Retrieve the return code from the exception ret_code = e.args[0]['ret'] if ret_code < 0: # If the return code is negative, then pg_receivexlog # was terminated by a signal msg = "pg_receivexlog terminated by signal: %s" \ % abs(ret_code) else: # Otherwise terminated with an error msg = "pg_receivexlog terminated with error code: %s"\ % ret_code raise ArchiverFailure(msg) except KeyboardInterrupt: # This is a normal termination, so there is nothing to do beside # informing the user. output.info('SIGINT received. Terminate gracefully.')
def fetch_remote_status(self): """ Execute checks for replication-based wal archiving This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ remote_status = dict.fromkeys( ('pg_receivexlog_compatible', 'pg_receivexlog_installed', 'pg_receivexlog_path', 'pg_receivexlog_supports_slots', 'pg_receivexlog_synchronous', 'pg_receivexlog_version'), None) # Test pg_receivexlog existence version_info = PgReceiveXlog.get_version_info(self.server.path) if version_info['full_path']: remote_status["pg_receivexlog_installed"] = True remote_status["pg_receivexlog_path"] = version_info['full_path'] remote_status["pg_receivexlog_version"] = ( version_info['full_version']) pgreceivexlog_version = version_info['major_version'] else: remote_status["pg_receivexlog_installed"] = False return remote_status # Retrieve the PostgreSQL version pg_version = None if self.server.streaming is not None: pg_version = self.server.streaming.server_major_version # If one of the version is unknown we cannot compare them if pgreceivexlog_version is None or pg_version is None: return remote_status # pg_version is not None so transform into a Version object # for easier comparison between versions pg_version = Version(pg_version) # Set conservative default values (False) for modern features remote_status["pg_receivexlog_compatible"] = False remote_status['pg_receivexlog_supports_slots'] = False remote_status["pg_receivexlog_synchronous"] = False # pg_receivexlog 9.2 is compatible only with PostgreSQL 9.2. if "9.2" == pg_version == pgreceivexlog_version: remote_status["pg_receivexlog_compatible"] = True # other versions are compatible with lesser versions of PostgreSQL # WARNING: The development versions of `pg_receivexlog` are considered # higher than the stable versions here, but this is not an issue # because it accepts everything that is less than # the `pg_receivexlog` version(e.g. '9.6' is less than '9.6devel') elif "9.2" < pg_version <= pgreceivexlog_version: # At least PostgreSQL 9.3 is required here remote_status["pg_receivexlog_compatible"] = True # replication slots are supported starting from version 9.4 if "9.4" <= pg_version <= pgreceivexlog_version: remote_status['pg_receivexlog_supports_slots'] = True # Synchronous WAL streaming requires replication slots # and pg_receivexlog >= 9.5 if "9.4" <= pg_version and "9.5" <= pgreceivexlog_version: remote_status["pg_receivexlog_synchronous"] = ( self._is_synchronous()) return remote_status
def receive_wal(self, reset=False): """ Creates a PgReceiveXlog object and issues the pg_receivexlog command for a specific server :param bool reset: When set reset the status of receive-wal :raise ArchiverFailure: when something goes wrong """ # Ensure the presence of the destination directory mkpath(self.config.streaming_wals_directory) # Execute basic sanity checks on PostgreSQL connection streaming_status = self.server.streaming.get_remote_status() if streaming_status["streaming_supported"] is None: raise ArchiverFailure( "failed opening the PostgreSQL streaming connection " "for server %s" % (self.config.name)) elif not streaming_status["streaming_supported"]: raise ArchiverFailure("PostgreSQL version too old (%s < 9.2)" % self.server.streaming.server_txt_version) # Execute basic sanity checks on pg_receivexlog command = "pg_receivewal" if self.server.streaming.server_version < 100000: command = "pg_receivexlog" remote_status = self.get_remote_status() if not remote_status["pg_receivexlog_installed"]: raise ArchiverFailure("%s not present in $PATH" % command) if not remote_status["pg_receivexlog_compatible"]: raise ArchiverFailure( "%s version not compatible with PostgreSQL server version" % command) # Execute sanity check on replication slot usage postgres_status = self.server.postgres.get_remote_status() if self.config.slot_name: # Check if slots are supported if not remote_status["pg_receivexlog_supports_slots"]: raise ArchiverFailure( "Physical replication slot not supported by %s " "(9.4 or higher is required)" % self.server.streaming.server_txt_version) # Check if the required slot exists if postgres_status["replication_slot"] is None: if self.config.create_slot == "auto": if not reset: output.info("Creating replication slot '%s'", self.config.slot_name) self.server.create_physical_repslot() else: raise ArchiverFailure( "replication slot '%s' doesn't exist. " "Please execute " "'barman receive-wal --create-slot %s'" % (self.config.slot_name, self.config.name)) # Check if the required slot is available elif postgres_status["replication_slot"].active: raise ArchiverFailure( "replication slot '%s' is already in use" % (self.config.slot_name, )) # Check if is a reset request if reset: self._reset_streaming_status(postgres_status, streaming_status) return # Check the size of the .partial WAL file and truncate it if needed self._truncate_partial_file_if_needed( postgres_status["xlog_segment_size"]) # Make sure we are not wasting precious PostgreSQL resources self.server.close() _logger.info("Activating WAL archiving through streaming protocol") try: output_handler = PgReceiveXlog.make_output_handler( self.config.name + ": ") receive = PgReceiveXlog( connection=self.server.streaming, destination=self.config.streaming_wals_directory, command=remote_status["pg_receivexlog_path"], version=remote_status["pg_receivexlog_version"], app_name=self.config.streaming_archiver_name, path=self.server.path, slot_name=self.config.slot_name, synchronous=remote_status["pg_receivexlog_synchronous"], out_handler=output_handler, err_handler=output_handler, ) # Finally execute the pg_receivexlog process receive.execute() except CommandFailedException as e: # Retrieve the return code from the exception ret_code = e.args[0]["ret"] if ret_code < 0: # If the return code is negative, then pg_receivexlog # was terminated by a signal msg = "%s terminated by signal: %s" % (command, abs(ret_code)) else: # Otherwise terminated with an error msg = "%s terminated with error code: %s" % (command, ret_code) raise ArchiverFailure(msg) except KeyboardInterrupt: # This is a normal termination, so there is nothing to do beside # informing the user. output.info("SIGINT received. Terminate gracefully.")
def receive_wal(self, reset=False): """ Creates a PgReceiveXlog object and issues the pg_receivexlog command for a specific server :param bool reset: When set reset the status of receive-wal :raise ArchiverFailure: when something goes wrong """ # Ensure the presence of the destination directory mkpath(self.config.streaming_wals_directory) # Check if is a reset request if reset: self._reset_streaming_status() return # Execute basic sanity checks on PostgreSQL connection postgres_status = self.server.streaming.get_remote_status() if postgres_status["streaming_supported"] is None: raise ArchiverFailure( 'failed opening the PostgreSQL streaming connection') elif not postgres_status["streaming_supported"]: raise ArchiverFailure('PostgreSQL version too old (%s < 9.2)' % self.server.streaming.server_txt_version) # Execute basic sanity checks on pg_receivexlog remote_status = self.get_remote_status() if not remote_status["pg_receivexlog_installed"]: raise ArchiverFailure('pg_receivexlog not present in $PATH') if not remote_status['pg_receivexlog_compatible']: raise ArchiverFailure('pg_receivexlog version not compatible with ' 'PostgreSQL server version') # Make sure we are not wasting precious PostgreSQL resources self.server.close() _logger.info('Activating WAL archiving through streaming protocol') try: output_handler = PgReceiveXlog.make_output_handler( self.config.name + ': ') if remote_status['pg_receivexlog_version'] >= "9.3": conn_string = self.config.streaming_conninfo # Set a default application_name unless defined by user if ('application_name' not in self.server.streaming.conn_parameters): conn_string += ' application_name=%s' % ( self.config.streaming_archiver_name) # If pg_receivexlog version is >= 9.3 we use the connection # string because allows the user to use all the parameters # supported by the libpq library to create a connection receive = PgReceiveXlog( remote_status['pg_receivexlog_path'], conn_string=conn_string, dest=self.config.streaming_wals_directory, out_handler=output_handler, err_handler=output_handler) else: # 9.2 version of pg_receivexlog doesn't support # connection strings so the 'split' version of the conninfo # option is used instead. conn_params = self.server.streaming.conn_parameters receive = PgReceiveXlog( remote_status['pg_receivexlog_path'], host=conn_params.get('host', None), port=conn_params.get('port', None), user=conn_params.get('user', None), dest=self.config.streaming_wals_directory, out_handler=output_handler, err_handler=output_handler) # Finally execute the pg_receivexlog process receive.execute() except CommandFailedException as e: _logger.error(e) raise ArchiverFailure("pg_receivexlog exited with an error. " "Check the logs for more information.") except KeyboardInterrupt: # This is a normal termination, so there is nothing to do beside # informing the user. output.info('SIGINT received. Terminate gracefully.')
def receive_wal(self, reset=False): """ Creates a PgReceiveXlog object and issues the pg_receivexlog command for a specific server :param bool reset: When set reset the status of receive-wal :raise ArchiverFailure: when something goes wrong """ # Ensure the presence of the destination directory mkpath(self.config.streaming_wals_directory) # Check if is a reset request if reset: self._reset_streaming_status() return # Execute basic sanity checks on PostgreSQL connection postgres_status = self.server.streaming.get_remote_status() if postgres_status["streaming_supported"] is None: raise ArchiverFailure( 'failed opening the PostgreSQL streaming connection') elif not postgres_status["streaming_supported"]: raise ArchiverFailure( 'PostgreSQL version too old (%s < 9.2)' % self.server.streaming.server_txt_version) # Execute basic sanity checks on pg_receivexlog remote_status = self.get_remote_status() if not remote_status["pg_receivexlog_installed"]: raise ArchiverFailure( 'pg_receivexlog not present in $PATH') if not remote_status['pg_receivexlog_compatible']: raise ArchiverFailure( 'pg_receivexlog version not compatible with ' 'PostgreSQL server version') # Make sure we are not wasting precious PostgreSQL resources self.server.close() _logger.info('Activating WAL archiving through streaming protocol') try: output_handler = PgReceiveXlog.make_output_handler( self.config.name + ': ') if remote_status['pg_receivexlog_version'] >= "9.3": conn_string = self.config.streaming_conninfo # Set a default application_name unless defined by user if ('application_name' not in self.server.streaming.conn_parameters): conn_string += ' application_name=%s' % ( self.config.streaming_archiver_name ) # If pg_receivexlog version is >= 9.3 we use the connection # string because allows the user to use all the parameters # supported by the libpq library to create a connection receive = PgReceiveXlog( remote_status['pg_receivexlog_path'], conn_string=conn_string, dest=self.config.streaming_wals_directory, out_handler=output_handler, err_handler=output_handler) else: # 9.2 version of pg_receivexlog doesn't support # connection strings so the 'split' version of the conninfo # option is used instead. conn_params = self.server.streaming.conn_parameters receive = PgReceiveXlog( remote_status['pg_receivexlog_path'], host=conn_params.get('host', None), port=conn_params.get('port', None), user=conn_params.get('user', None), dest=self.config.streaming_wals_directory, out_handler=output_handler, err_handler=output_handler) # Finally execute the pg_receivexlog process receive.execute() except CommandFailedException as e: _logger.error(e) raise ArchiverFailure("pg_receivexlog exited with an error. " "Check the logs for more information.") except KeyboardInterrupt: # This is a normal termination, so there is nothing to do beside # informing the user. output.info('SIGINT received. Terminate gracefully.')
def receive_wal(self, reset=False): """ Creates a PgReceiveXlog object and issues the pg_receivexlog command for a specific server :param bool reset: When set reset the status of receive-wal :raise ArchiverFailure: when something goes wrong """ # Ensure the presence of the destination directory mkpath(self.config.streaming_wals_directory) # Check if is a reset request if reset: self._reset_streaming_status() return # Execute basic sanity checks on PostgreSQL connection streaming_status = self.server.streaming.get_remote_status() if streaming_status["streaming_supported"] is None: raise ArchiverFailure( 'failed opening the PostgreSQL streaming connection') elif not streaming_status["streaming_supported"]: raise ArchiverFailure( 'PostgreSQL version too old (%s < 9.2)' % self.server.streaming.server_txt_version) # Execute basic sanity checks on pg_receivexlog remote_status = self.get_remote_status() if not remote_status["pg_receivexlog_installed"]: raise ArchiverFailure( 'pg_receivexlog not present in $PATH') if not remote_status['pg_receivexlog_compatible']: raise ArchiverFailure( 'pg_receivexlog version not compatible with ' 'PostgreSQL server version') # Execute sanity check on replication slot usage if self.config.slot_name: # Check if slots are supported if not remote_status['pg_receivexlog_supports_slots']: raise ArchiverFailure( 'Physical replication slot not supported by %s ' '(9.4 or higher is required)' % self.server.streaming.server_txt_version) # Check if the required slot exists postgres_status = self.server.postgres.get_remote_status() if postgres_status['replication_slot'] is None: raise ArchiverFailure( "replication slot '%s' doesn't exist. " "Please execute " "'barman receive-wal --create-slot %s'" % (self.config.slot_name, self.config.name)) # Check if the required slot is available if postgres_status['replication_slot'].active: raise ArchiverFailure( "replication slot '%s' is already in use" % (self.config.slot_name,)) # Make sure we are not wasting precious PostgreSQL resources self.server.close() _logger.info('Activating WAL archiving through streaming protocol') try: output_handler = PgReceiveXlog.make_output_handler( self.config.name + ': ') receive = PgReceiveXlog( connection=self.server.streaming, destination=self.config.streaming_wals_directory, command=remote_status['pg_receivexlog_path'], version=remote_status['pg_receivexlog_version'], app_name=self.config.streaming_archiver_name, path=self.server.path, slot_name=self.config.slot_name, synchronous=remote_status['pg_receivexlog_synchronous'], out_handler=output_handler, err_handler=output_handler ) # Finally execute the pg_receivexlog process receive.execute() except CommandFailedException as e: # Retrieve the return code from the exception ret_code = e.args[0]['ret'] if ret_code < 0: # If the return code is negative, then pg_receivexlog # was terminated by a signal msg = "pg_receivexlog terminated by signal: %s" \ % abs(ret_code) else: # Otherwise terminated with an error msg = "pg_receivexlog terminated with error code: %s"\ % ret_code raise ArchiverFailure(msg) except KeyboardInterrupt: # This is a normal termination, so there is nothing to do beside # informing the user. output.info('SIGINT received. Terminate gracefully.')