Exemple #1
0
def check(args):
    """
    Check if the server configuration is working.

    This command returns success if every checks pass,
    or failure if any of these fails
    """
    if args.nagios:
        output.set_output_writer(output.NagiosOutputWriter())
    servers = get_server_list(args)
    for name in sorted(servers):
        server = servers[name]

        # Validate the returned server
        if not manage_server_command(
                server, name, skip_inactive=False,
                skip_disabled=False, disabled_is_error=False):
            continue

        # If the server has been manually disabled
        if not server.config.active:
            name += " (inactive)"
        # If server has configuration errors
        elif server.config.disabled:
            name += " (WARNING: disabled)"
        output.init('check', name, server.config.active)
        server.check()
    output.close_and_exit()
    def copy_temporary_config_files(self, dest, remote_command, recovery_info):
        """
        Copy modified configuration files using rsync in case of
        remote recovery

        :param str dest: destination directory of the recovery
        :param str remote_command: ssh command for remote connection
        :param dict recovery_info: Dictionary containing all the recovery
            parameters
        """
        if remote_command:
            # If this is a remote recovery, rsync the modified files from the
            # temporary local directory to the remote destination directory.
            file_list = []
            for conf_file in recovery_info['configuration_files']:
                file_list.append('%s' % conf_file)
                file_list.append('%s.origin' % conf_file)

            try:
                recovery_info['rsync'].from_file_list(file_list,
                                                      recovery_info['tempdir'],
                                                      ':%s' % dest)
            except CommandFailedException as e:
                output.exception(
                    'remote copy of configuration files failed: %s', e)
                output.close_and_exit()
Exemple #3
0
def backup(args):
    """
    Perform a full backup for the given server
    """
    servers = get_server_list(args, skip_disabled=True)
    for name in sorted(servers):
        server = servers[name]
        if server is None:
            output.error("Unknown server '%s'" % name)
            continue
        # If the server is disabled return an error message
        if not server.config.active:
            output.error(
                "Server '%s' is disabled.\n"
                "HINT: remove 'active=False' from server configuration "
                "to enable it.",
                name)
            continue
        if args.reuse_backup is not None:
            server.config.reuse_backup = args.reuse_backup
        if args.retry_sleep is not None:
            server.config.basebackup_retry_sleep = args.retry_sleep
        if args.retry_times is not None:
            server.config.basebackup_retry_times = args.retry_times
        if hasattr(args, 'immediate_checkpoint'):
            server.config.immediate_checkpoint = args.immediate_checkpoint
        server.backup()
    output.close_and_exit()
    def generate_archive_status(self, recovery_info, remote_command,
                                required_xlog_files):
        """
        Populate the archive_status directory

        :param dict recovery_info: Dictionary containing all the recovery
            parameters
        :param str remote_command: ssh command for remote connection
        :param tuple required_xlog_files: list of required WAL segments
        """
        if remote_command:
            status_dir = recovery_info['tempdir']
        else:
            status_dir = os.path.join(recovery_info['wal_dest'],
                                      'archive_status')
            mkpath(status_dir)
        for wal_info in required_xlog_files:
            with open(os.path.join(status_dir, "%s.done" % wal_info.name),
                      'a') as f:
                f.write('')
        if remote_command:
            try:
                recovery_info['rsync']('%s/' % status_dir,
                                       ':%s' % os.path.join(
                                           recovery_info['wal_dest'],
                                           'archive_status'))
            except CommandFailedException as e:
                output.exception(
                    "unable to populate pg_xlog/archive_status"
                    "directory: %s", e)
                output.close_and_exit()
Exemple #5
0
def diagnose():
    """
    Diagnostic command (for support and problems detection purpose)
    """
    servers = get_server_list(None)
    barman.diagnose.exec_diagnose(servers)
    output.close_and_exit()
    def setup(self, backup_info, remote_command, dest):
        """
        Prepare the recovery_info dictionary for the recovery, as well
        as temporary working directory

        :param barman.infofile.BackupInfo backup_info: representation of a
            backup
        :param str remote_command: ssh command for remote connection
        :return dict: recovery_info dictionary, holding the basic values for a
            recovery
        """
        recovery_info = {
            'cmd': None,
            'recovery_dest': 'local',
            'rsync': None,
            'configuration_files': [],
            'destination_path': dest,
            'temporary_configuration_files': [],
            'tempdir': tempfile.mkdtemp(prefix='barman_recovery-'),
            'is_pitr': False,
            'wal_dest': os.path.join(dest, 'pg_xlog'),
            'get_wal': RecoveryOptions.GET_WAL in self.config.recovery_options,
        }
        # A map that will keep track of the results of the recovery.
        # Used for output generation
        results = {
            'changes': [],
            'warnings': [],
            'delete_barman_xlog': False,
            'get_wal': False,
        }
        recovery_info['results'] = results

        # Set up a list of configuration files
        recovery_info['configuration_files'].append('postgresql.conf')
        if backup_info.version >= 90400:
            recovery_info['configuration_files'].append('postgresql.auto.conf')

        # Handle remote recovery options
        if remote_command:
            recovery_info['recovery_dest'] = 'remote'
            recovery_info['rsync'] = RsyncPgData(
                path=self.server.path,
                ssh=remote_command,
                bwlimit=self.config.bandwidth_limit,
                network_compression=self.config.network_compression)
            try:
                # create a UnixRemoteCommand obj if is a remote recovery
                recovery_info['cmd'] = UnixRemoteCommand(remote_command)
            except FsOperationFailed:
                output.error(
                    "Unable to connect to the target host using the command "
                    "'%s'", remote_command)
                output.close_and_exit()
        else:
            # if is a local recovery create a UnixLocalCommand
            recovery_info['cmd'] = UnixLocalCommand()

        return recovery_info
Exemple #7
0
    def test_close_and_exit(self, exit_mock):
        # preparation
        writer = self._mock_writer()

        output.close_and_exit()

        writer.close.assert_called_once_with()
        exit_mock.assert_called_once_with(0)
Exemple #8
0
def switch_xlog(args):
    """
    Execute the switch-xlog command on the target server
    """
    servers = get_server_list(args, skip_inactive=True)
    for name in sorted(servers):
        server = servers[name]
        server.switch_xlog(args.force)
    output.close_and_exit()
Exemple #9
0
def receive_wal(args):
    """
    Start a receive-wal process.
    The process uses the streaming protocol to receive WAL files
    from the PostgreSQL server.
    """
    server = get_server(args)
    server.receive_wal()
    output.close_and_exit()
Exemple #10
0
def delete(args):
    """
    Delete a backup
    """
    server = get_server(args)

    # Retrieves the backup
    backup_id = parse_backup_id(server, args)
    server.delete_backup(backup_id)
    output.close_and_exit()
Exemple #11
0
def archive_wal(args):
    """
    Execute maintenance operations on WAL files for a given server.
    This command processes any incoming WAL files for the server
    and archives them along the catalogue.

    """
    server = get_server(args)
    server.archive_wal()
    output.close_and_exit()
Exemple #12
0
def diagnose():
    """
    Diagnostic command (for support and problems detection purpose)
    """
    # Get every server (both inactive and temporarily disabled)
    servers = get_server_list(on_error_stop=False, suppress_error=True)
    # errors list with duplicate paths between servers
    errors_list = barman.__config__.servers_msg_list
    barman.diagnose.exec_diagnose(servers, errors_list)
    output.close_and_exit()
Exemple #13
0
def show_backup(args):
    """
    This method shows a single backup information
    """
    server = get_server(args)

    # Retrieves the backup
    backup_info = parse_backup_id(server, args)
    server.show_backup(backup_info)
    output.close_and_exit()
Exemple #14
0
def list_server(minimal=False):
    """
    List available servers, with useful information
    """
    servers = get_server_list()
    for name in sorted(servers):
        server = servers[name]
        output.init('list_server', name, minimal=minimal)
        output.result('list_server', name, server.config.description)
    output.close_and_exit()
Exemple #15
0
def list_files(args):
    """
    List all the files for a single backup
    """
    server = get_server(args)

    # Retrieves the backup
    backup_id = parse_backup_id(server, args)
    for line in backup_id.get_list_of_files(args.target):
        output.info(line, log=False)
    output.close_and_exit()
Exemple #16
0
    def test_close_and_exit_with_error(self, exit_mock):
        # preparation
        writer = self._mock_writer()
        output.error_occurred = True

        output.close_and_exit()

        writer.close.assert_called_once_with()
        assert exit_mock.called
        assert exit_mock.call_count == 1
        assert exit_mock.call_args[0] != 0
Exemple #17
0
def archive_wal(args):
    """
    Execute maintenance operations on WAL files for a given server.
    This command processes any incoming WAL files for the server
    and archives them along the catalogue.

    """
    server = get_server(args)
    output.debug("Starting archive-wal for server %s", server.config.name)
    server.archive_wal()
    output.close_and_exit()
Exemple #18
0
def rebuild_xlogdb(args):
    """
    Rebuild the WAL file database guessing it from the disk content.
    """
    servers = get_server_list(args)
    for name in sorted(servers):
        server = servers[name]
        if server is None:
            output.error("Unknown server '%s'", name)
            continue
        server.rebuild_xlogdb()
    output.close_and_exit()
Exemple #19
0
def main():
    """
    The main method of Barman
    """
    p = ArghParser(epilog='Barman by 2ndQuadrant (www.2ndQuadrant.com)')
    p.add_argument('-v', '--version', action='version',
                   version='%s\n\nBarman by 2ndQuadrant (www.2ndQuadrant.com)'
                           % barman.__version__)
    p.add_argument('-c', '--config',
                   help='uses a configuration file '
                        '(defaults: %s)'
                        % ', '.join(barman.config.Config.CONFIG_FILES),
                   default=SUPPRESS)
    p.add_argument('-q', '--quiet', help='be quiet', action='store_true')
    p.add_argument('-d', '--debug', help='debug output', action='store_true')
    p.add_argument('-f', '--format', help='output format',
                   choices=output.AVAILABLE_WRITERS.keys(),
                   default=output.DEFAULT_WRITER)
    p.add_commands(
        [
            archive_wal,
            backup,
            check,
            cron,
            delete,
            diagnose,
            get_wal,
            list_backup,
            list_files,
            list_server,
            rebuild_xlogdb,
            receive_wal,
            recover,
            show_backup,
            show_server,
            replication_status,
            status,
            switch_xlog,
        ]
    )
    # noinspection PyBroadException
    try:
        p.dispatch(pre_call=global_config)
    except KeyboardInterrupt:
        msg = "Process interrupted by user (KeyboardInterrupt)"
        output.error(msg)
    except Exception as e:
        msg = "%s\nSee log file for more details." % e
        output.exception(msg)

    # cleanup output API and exit honoring output.error_occurred and
    # output.error_exit_code
    output.close_and_exit()
Exemple #20
0
    def generate_recovery_conf(self, recovery_info, backup_info, dest,
                               exclusive, remote_command, target_name,
                               target_time, target_tli, target_xid):
        """
        Generate a recovery.conf file for PITR containing
        all the required configurations

        :param dict recovery_info: Dictionary containing all the recovery params
        :param barman.infofile.BackupInfo backup_info: representation of a
            backup
        :param str dest: destination directory of the recovery
        :param boolean exclusive: exclusive backup or concurrent
        :param str remote_command: ssh command for remote connection
        :param str target_name: recovery target name for PITR
        :param str target_time: recovery target time for PITR
        :param str target_tli: recovery target timeline for PITR
        :param str target_xid: recovery target transaction id for PITR
        """
        if remote_command:
            recovery = open(os.path.join(recovery_info['tempdir'],
                                         'recovery.conf'), 'w')
        else:
            recovery = open(os.path.join(dest, 'recovery.conf'), 'w')
        print >> recovery, "restore_command = 'cp barman_xlog/%f %p'"
        if backup_info.version >= 80400:
            print >> recovery, "recovery_end_command = 'rm -fr barman_xlog'"
        if target_time:
            print >> recovery, "recovery_target_time = '%s'" % target_time
        if target_tli:
            print >> recovery, "recovery_target_timeline = %s" % target_tli
        if target_xid:
            print >> recovery, "recovery_target_xid = '%s'" % target_xid
        if target_name:
            print >> recovery, "recovery_target_name = '%s'" % target_name
        if (target_xid or target_time) and exclusive:
            print >> recovery, "recovery_target_inclusive = '%s'" % (
                not exclusive)
        recovery.close()
        if remote_command:
            # Uses plain rsync (without exclusions) to ship recovery.conf
            plain_rsync = Rsync(
                ssh=remote_command,
                bwlimit=self.config.bandwidth_limit,
                network_compression=self.config.network_compression)
            try:
                plain_rsync.from_file_list(['recovery.conf'],
                                           recovery_info['tempdir'],
                                           ':%s' % dest)
            except CommandFailedException, e:
                output.exception(
                    'remote copy of recovery.conf failed: %s', e)
                output.close_and_exit()
Exemple #21
0
def show_server(args):
    """
    Show all configuration parameters for the specified servers
    """
    servers = get_server_list(args)
    for name in sorted(servers):
        server = servers[name]
        if server is None:
            output.error("Unknown server '%s'" % name)
            continue
        output.init('show_server', name)
        server.show()
    output.close_and_exit()
Exemple #22
0
def status(args):
    """
    Shows live information and status of the PostgreSQL server
    """
    servers = get_server_list(args)
    for name in sorted(servers):
        server = servers[name]
        if server is None:
            output.error("Unknown server '%s'" % name)
            continue
        output.init('status', name)
        server.status()
    output.close_and_exit()
Exemple #23
0
def list_backup(args):
    """
    List available backups for the given server (supports 'all')
    """
    servers = get_server_list(args)
    for name in sorted(servers):
        server = servers[name]
        output.init('list_backup', name, minimal=args.minimal)
        if server is None:
            output.error("Unknown server '%s'" % name)
            continue
        server.list_backups()
    output.close_and_exit()
Exemple #24
0
def switch_xlog(args):
    """
    Execute the switch-xlog command on the target server
    """
    servers = get_server_list(args, skip_inactive=True)
    for name in sorted(servers):
        server = servers[name]
        # Skip the server (apply general rule)
        if not manage_server_command(server, name):
            continue
        with closing(server):
            server.switch_xlog(args.force, args.archive, args.archive_timeout)
    output.close_and_exit()
    def prepare_tablespaces(self, backup_info, cmd, dest, tablespaces):
        """
        Prepare the directory structure for required tablespaces,
        taking care of tablespaces relocation, if requested.

        :param barman.infofile.BackupInfo backup_info: backup representation
        :param barman.fs.UnixLocalCommand cmd: Object for
            filesystem interaction
        :param str dest: destination dir for the recovery
        :param dict tablespaces: dict of all the tablespaces and their location
        """
        tblspc_dir = os.path.join(dest, 'pg_tblspc')
        try:
            # check for pg_tblspc dir into recovery destination folder.
            # if it does not exists, create it
            cmd.create_dir_if_not_exists(tblspc_dir)
        except FsOperationFailed as e:
            output.exception("unable to initialise tablespace directory "
                             "'%s': %s", tblspc_dir, e)
            output.close_and_exit()
        for item in backup_info.tablespaces:

            # build the filename of the link under pg_tblspc directory
            pg_tblspc_file = os.path.join(tblspc_dir, str(item.oid))

            # by default a tablespace goes in the same location where
            # it was on the source server when the backup was taken
            location = item.location

            # if a relocation has been requested for this tablespace,
            # use the target directory provided by the user
            if tablespaces and item.name in tablespaces:
                location = tablespaces[item.name]

            try:
                # remove the current link in pg_tblspc, if it exists
                # (raise an exception if it is a directory)
                cmd.delete_if_exists(pg_tblspc_file)
                # create tablespace location, if does not exist
                # (raise an exception if it is not possible)
                cmd.create_dir_if_not_exists(location)
                # check for write permissions on destination directory
                cmd.check_write_permission(location)
                # create symlink between tablespace and recovery folder
                cmd.create_symbolic_link(location, pg_tblspc_file)
            except FsOperationFailed as e:
                output.exception("unable to prepare '%s' tablespace "
                                 "(destination '%s'): %s",
                                 item.name, location, e)
                output.close_and_exit()
            output.info("\t%s, %s, %s", item.oid, item.name, location)
Exemple #26
0
def rebuild_xlogdb(args):
    """
    Rebuild the WAL file database guessing it from the disk content.
    """
    servers = get_server_list(args, skip_inactive=True)
    for name in sorted(servers):
        server = servers[name]

        # Skip the server (apply general rule)
        if not manage_server_command(server, name):
            continue

        server.rebuild_xlogdb()
    output.close_and_exit()
Exemple #27
0
def get_wal(args):
    """
    Retrieve WAL_NAME file from SERVER_NAME archive.
    The content will be streamed on standard output unless
    the --output-directory option is specified.
    """
    server = get_server(args)
    # Retrieve optional arguments. If an argument is not specified,
    # the namespace doesn't contain it due to SUPPRESS default.
    # In that case we pick 'None' using getattr third argument.
    compression = getattr(args, "compression", None)
    output_directory = getattr(args, "output_directory", None)
    server.get_wal(args.wal_name, compression=compression, output_directory=output_directory)
    output.close_and_exit()
Exemple #28
0
def receive_wal(args):
    """
    Start a receive-wal process.
    The process uses the streaming protocol to receive WAL files
    from the PostgreSQL server.
    """
    server = get_server(args)
    # If the caller requested to shutdown the receive-wal process deliver the
    # termination signal, otherwise attempt to start it
    if args.stop:
        server.kill('receive-wal')
    else:
        server.receive_wal()
    output.close_and_exit()
Exemple #29
0
def show_server(args):
    """
    Show all configuration parameters for the specified servers
    """
    servers = get_server_list(args, skip_inactive=True)
    for name in sorted(servers):
        server = servers[name]

        # Skip the server (apply general rule)
        if not manage_server_command(server, name):
            continue

        output.init('show_server', name)
        server.show()
    output.close_and_exit()
Exemple #30
0
def status(args):
    """
    Shows live information and status of the PostgreSQL server
    """
    servers = get_server_list(args, skip_inactive=True)
    for name in sorted(servers):
        server = servers[name]

        # Skip the server (apply general rule)
        if not manage_server_command(server, name):
            continue

        output.init('status', name)
        server.status()
    output.close_and_exit()
Exemple #31
0
def recover(args):
    """
    Recover a server at a given time, name, LSN or xid
    """
    server = get_server(args)

    # Retrieves the backup
    backup_id = parse_backup_id(server, args)
    if backup_id.status not in BackupInfo.STATUS_COPY_DONE:
        output.error(
            "Cannot recover from backup '%s' of server '%s': "
            "backup status is not DONE",
            args.backup_id, server.config.name)
        output.close_and_exit()

    # decode the tablespace relocation rules
    tablespaces = {}
    if args.tablespace:
        for rule in args.tablespace:
            try:
                tablespaces.update([rule.split(':', 1)])
            except ValueError:
                output.error(
                    "Invalid tablespace relocation rule '%s'\n"
                    "HINT: The valid syntax for a relocation rule is "
                    "NAME:LOCATION", rule)
                output.close_and_exit()

    # validate the rules against the tablespace list
    valid_tablespaces = []
    if backup_id.tablespaces:
        valid_tablespaces = [tablespace_data.name for tablespace_data in
                             backup_id.tablespaces]
    for item in tablespaces:
        if item not in valid_tablespaces:
            output.error("Invalid tablespace name '%s'\n"
                         "HINT: Please use any of the following "
                         "tablespaces: %s",
                         item, ', '.join(valid_tablespaces))
            output.close_and_exit()

    # explicitly disallow the rsync remote syntax (common mistake)
    if ':' in args.destination_directory:
        output.error(
            "The destination directory parameter "
            "cannot contain the ':' character\n"
            "HINT: If you want to do a remote recovery you have to use "
            "the --remote-ssh-command option")
        output.close_and_exit()
    if args.retry_sleep is not None:
        server.config.basebackup_retry_sleep = args.retry_sleep
    if args.retry_times is not None:
        server.config.basebackup_retry_times = args.retry_times
    if hasattr(args, 'get_wal'):
        if args.get_wal:
            server.config.recovery_options.add(RecoveryOptions.GET_WAL)
        else:
            server.config.recovery_options.remove(RecoveryOptions.GET_WAL)
    if args.jobs is not None:
        server.config.parallel_jobs = args.jobs
    if hasattr(args, 'bwlimit'):
        server.config.bandwidth_limit = args.bwlimit

    # PostgreSQL supports multiple parameters to specify when the recovery
    # process will end, and in that case the last entry in recovery
    # configuration files will be used. See [1]
    #
    # Since the meaning of the target options is not dependent on the order
    # of parameters, we decided to make the target options mutually exclusive.
    #
    # [1]: https://www.postgresql.org/docs/current/static/
    #   recovery-target-settings.html

    target_options = ['target_tli', 'target_time', 'target_xid',
                      'target_lsn', 'target_name', 'target_immediate']
    specified_target_options = len(
        [option for option in target_options if getattr(args, option)])
    if specified_target_options > 1:
        output.error(
            "You cannot specify multiple targets for the recovery operation")
        output.close_and_exit()

    if hasattr(args, 'network_compression'):
        if args.network_compression and args.remote_ssh_command is None:
            output.error(
                "Network compression can only be used with "
                "remote recovery.\n"
                "HINT: If you want to do a remote recovery "
                "you have to use the --remote-ssh-command option")
            output.close_and_exit()
        server.config.network_compression = args.network_compression

    with closing(server):
        try:
            server.recover(backup_id,
                           args.destination_directory,
                           tablespaces=tablespaces,
                           target_tli=args.target_tli,
                           target_time=args.target_time,
                           target_xid=args.target_xid,
                           target_lsn=args.target_lsn,
                           target_name=args.target_name,
                           target_immediate=args.target_immediate,
                           exclusive=args.exclusive,
                           remote_command=args.remote_ssh_command,
                           target_action=getattr(args, 'target_action', None),
                           standby_mode=getattr(args, 'standby_mode', None))
        except RecoveryException as exc:
            output.error(force_str(exc))

    output.close_and_exit()
Exemple #32
0
def recover(args):
    """
    Recover a server at a given time or xid
    """
    server = get_server(args)

    # Retrieves the backup
    backup_id = parse_backup_id(server, args)
    if backup_id.status != BackupInfo.DONE:
        output.error(
            "Cannot recover from backup '%s' of server '%s': "
            "backup status is not DONE", args.backup_id, server.config.name)
        output.close_and_exit()

    # decode the tablespace relocation rules
    tablespaces = {}
    if args.tablespace:
        for rule in args.tablespace:
            try:
                tablespaces.update([rule.split(':', 1)])
            except ValueError:
                output.error(
                    "Invalid tablespace relocation rule '%s'\n"
                    "HINT: The valid syntax for a relocation rule is "
                    "NAME:LOCATION", rule)
                output.close_and_exit()

    # validate the rules against the tablespace list
    valid_tablespaces = []
    if backup_id.tablespaces:
        valid_tablespaces = [
            tablespace_data.name for tablespace_data in backup_id.tablespaces
        ]
    for item in tablespaces:
        if item not in valid_tablespaces:
            output.error(
                "Invalid tablespace name '%s'\n"
                "HINT: Please use any of the following "
                "tablespaces: %s", item, ', '.join(valid_tablespaces))
            output.close_and_exit()

    # explicitly disallow the rsync remote syntax (common mistake)
    if ':' in args.destination_directory:
        output.error(
            "The destination directory parameter "
            "cannot contain the ':' character\n"
            "HINT: If you want to do a remote recovery you have to use "
            "the --remote-ssh-command option")
        output.close_and_exit()
    if args.retry_sleep is not None:
        server.config.basebackup_retry_sleep = args.retry_sleep
    if args.retry_times is not None:
        server.config.basebackup_retry_times = args.retry_times
    with closing(server):
        server.recover(backup_id,
                       args.destination_directory,
                       tablespaces=tablespaces,
                       target_tli=args.target_tli,
                       target_time=args.target_time,
                       target_xid=args.target_xid,
                       target_name=args.target_name,
                       exclusive=args.exclusive,
                       remote_command=args.remote_ssh_command)

    output.close_and_exit()
Exemple #33
0
def get_server(args,
               skip_inactive=True,
               skip_disabled=False,
               on_error_stop=True,
               suppress_error=False):
    """
    Get a single server retrieving its configuration (wraps get_server_list())

    Returns a Server object or None if the required server is unknown and
    on_error_stop is False.

    WARNING: this function modifies the 'args' parameter

    :param args: an argparse namespace containing a single
        server_name parameter
        WARNING: the function modifies the content of this parameter
    :param bool skip_inactive: skip inactive servers when 'all' is required
    :param bool skip_disabled: skip disabled servers when 'all' is required
    :param bool on_error_stop: stop if an error is found
    :param bool suppress_error: suppress display of errors (e.g. diagnose)
    :rtype: barman.server.Server|None
    """
    # This function must to be called with in a single-server context
    name = args.server_name
    assert isinstance(name, str)

    # The 'all' special name is forbidden in this context
    if name == 'all':
        output.error("You cannot use 'all' in a single server context")
        output.close_and_exit()
        # The following return statement will never be reached
        # but it is here for clarity
        return None

    # Builds a list from a single given name
    args.server_name = [name]

    # Retrieve the requested server
    servers = get_server_list(args, skip_inactive, skip_disabled,
                              on_error_stop, suppress_error)

    # The requested server has been excluded from get_server_list result
    if len(servers) == 0:
        output.close_and_exit()
        # The following return statement will never be reached
        # but it is here for clarity
        return None

    # retrieve the server object
    server = servers[name]

    # Apply standard validation control and skips
    # the server if inactive or disabled, displaying standard
    # error messages. If on_error_stop (default) exits
    if not manage_server_command(server, name) and on_error_stop:
        output.close_and_exit()
        # The following return statement will never be reached
        # but it is here for clarity
        return None

    # Returns the filtered server
    return server
Exemple #34
0
    def _set_pitr_targets(self, recovery_info, backup_info, dest, target_name,
                          target_time, target_tli, target_xid):
        """
        Set PITR targets - as specified by the user

        :param dict recovery_info: Dictionary containing all the recovery
            parameters
        :param barman.infofile.BackupInfo backup_info: representation of a
            backup
        :param str dest: destination directory of the recovery
        :param str|None target_name: recovery target name for PITR
        :param str|None target_time: recovery target time for PITR
        :param str|None target_tli: recovery target timeline for PITR
        :param str|None target_xid: recovery target transaction id for PITR
        """
        target_epoch = None
        target_datetime = None
        if (target_time or
                target_xid or
                (target_tli and target_tli != backup_info.timeline) or
                target_name or
                recovery_info['get_wal']):
            recovery_info['is_pitr'] = True
            targets = {}
            if target_time:
                # noinspection PyBroadException
                try:
                    target_datetime = dateutil.parser.parse(target_time)
                except ValueError as e:
                    output.error(
                        "unable to parse the target time parameter %r: %s",
                        target_time, e)
                    self._teardown(recovery_info)
                    output.close_and_exit()
                except Exception:
                    # this should not happen, but there is a known bug in
                    # dateutil.parser.parse() implementation
                    # ref: https://bugs.launchpad.net/dateutil/+bug/1247643
                    output.error(
                        "unable to parse the target time parameter %r",
                        target_time)
                    output.close_and_exit()

                target_epoch = (
                    time.mktime(target_datetime.timetuple()) +
                    (target_datetime.microsecond / 1000000.))
                targets['time'] = str(target_datetime)
            if target_xid:
                targets['xid'] = str(target_xid)
            if target_tli and target_tli != backup_info.timeline:
                targets['timeline'] = str(target_tli)
            if target_name:
                targets['name'] = str(target_name)
            output.info(
                "Doing PITR. Recovery target %s",
                (", ".join(["%s: %r" % (k, v) for k, v in targets.items()])))
            recovery_info['wal_dest'] = os.path.join(dest, 'barman_xlog')

            # With a PostgreSQL version older than 8.4, it is the user's
            # responsibility to delete the "barman_xlog" directory as the
            # restore_command option in recovery.conf is not supported
            if backup_info.version < 80400 and \
                    not recovery_info['get_wal']:
                recovery_info['results']['delete_barman_xlog'] = True
        recovery_info['target_epoch'] = target_epoch
        recovery_info['target_datetime'] = target_datetime
Exemple #35
0
def get_server_list(args=None, skip_inactive=False, skip_disabled=False,
                    skip_passive=False,
                    on_error_stop=True, suppress_error=False):
    """
    Get the server list from the configuration

    If args the parameter is None or arg.server_name is ['all']
    returns all defined servers

    :param args: an argparse namespace containing a list server_name parameter
    :param bool skip_inactive: skip inactive servers when 'all' is required
    :param bool skip_disabled: skip disabled servers when 'all' is required
    :param bool skip_passive: skip passive servers when 'all' is required
    :param bool on_error_stop: stop if an error is found
    :param bool suppress_error: suppress display of errors (e.g. diagnose)
    :rtype: dict[str,Server]
    """
    server_dict = {}

    # This function must to be called with in a multiple-server context
    assert not args or isinstance(args.server_name, list)

    # Generate the list of servers (required for global errors)
    available_servers = barman.__config__.server_names()

    # Get a list of configuration errors from all the servers
    global_error_list = barman.__config__.servers_msg_list

    # Global errors have higher priority
    if global_error_list:
        # Output the list of global errors
        if not suppress_error:
            for error in global_error_list:
                output.error(error)

        # If requested, exit on first error
        if on_error_stop:
            output.close_and_exit()
            # The following return statement will never be reached
            # but it is here for clarity
            return {}

    # Handle special 'all' server cases
    # - args is None
    # - 'all' special name
    if not args or 'all' in args.server_name:
        # When 'all' is used, it must be the only specified argument
        if args and len(args.server_name) != 1:
            output.error("You cannot use 'all' with other server names")
        servers = available_servers
    else:
        # Put servers in a set, so multiple occurrences are counted only once
        servers = set(args.server_name)

    # Loop through all the requested servers
    for server in servers:
        conf = barman.__config__.get_server(server)
        if conf is None:
            # Unknown server
            server_dict[server] = None
        else:
            server_object = Server(conf)
            # Skip inactive servers, if requested
            if skip_inactive and not server_object.config.active:
                output.info("Skipping inactive server '%s'"
                            % conf.name)
                continue
            # Skip disabled servers, if requested
            if skip_disabled and server_object.config.disabled:
                output.info("Skipping temporarily disabled server '%s'"
                            % conf.name)
                continue
            # Skip passive nodes, if requested
            if skip_passive and server_object.passive_node:
                output.info("Skipping passive server '%s'",
                            conf.name)
                continue
            server_dict[server] = server_object

    return server_dict
Exemple #36
0
def main():
    """
    The main method of Barman
    """
    p = ArghParser(epilog='Barman by 2ndQuadrant (www.2ndQuadrant.com)')
    p.add_argument('-v', '--version', action='version',
                   version='%s\n\nBarman by 2ndQuadrant (www.2ndQuadrant.com)'
                           % barman.__version__)
    p.add_argument('-c', '--config',
                   help='uses a configuration file '
                        '(defaults: %s)'
                        % ', '.join(barman.config.Config.CONFIG_FILES),
                   default=SUPPRESS)
    p.add_argument('--color', '--colour',
                   help='Whether to use colors in the output',
                   choices=['never', 'always', 'auto'],
                   default='auto')
    p.add_argument('--log-level',
                   help='Override the default log level',
                   choices=list(get_log_levels()),
                   default=SUPPRESS)
    p.add_argument('-q', '--quiet', help='be quiet', action='store_true')
    p.add_argument('-d', '--debug', help='debug output', action='store_true')
    p.add_argument('-f', '--format', help='output format',
                   choices=output.AVAILABLE_WRITERS.keys(),
                   default=output.DEFAULT_WRITER)
    p.add_commands(
        [
            archive_wal,
            backup,
            check,
            check_backup,
            cron,
            delete,
            diagnose,
            get_wal,
            list_backup,
            list_files,
            list_server,
            put_wal,
            rebuild_xlogdb,
            receive_wal,
            recover,
            show_backup,
            show_server,
            replication_status,
            status,
            switch_wal,
            switch_xlog,
            sync_info,
            sync_backup,
            sync_wals,
        ]
    )
    # noinspection PyBroadException
    try:
        p.dispatch(pre_call=global_config)
    except KeyboardInterrupt:
        msg = "Process interrupted by user (KeyboardInterrupt)"
        output.error(msg)
    except Exception as e:
        msg = "%s\nSee log file for more details." % e
        output.exception(msg)

    # cleanup output API and exit honoring output.error_occurred and
    # output.error_exit_code
    output.close_and_exit()
Exemple #37
0
    def _setup(self, backup_info, remote_command, dest):
        """
        Prepare the recovery_info dictionary for the recovery, as well
        as temporary working directory

        :param barman.infofile.BackupInfo backup_info: representation of a
            backup
        :param str remote_command: ssh command for remote connection
        :return dict: recovery_info dictionary, holding the basic values for a
            recovery
        """
        recovery_info = {
            'cmd': None,
            'recovery_dest': 'local',
            'rsync': None,
            'configuration_files': [],
            'destination_path': dest,
            'temporary_configuration_files': [],
            'tempdir': tempfile.mkdtemp(prefix='barman_recovery-'),
            'is_pitr': False,
            'wal_dest': os.path.join(dest, 'pg_xlog'),
            'get_wal': RecoveryOptions.GET_WAL in self.config.recovery_options,
        }
        # A map that will keep track of the results of the recovery.
        # Used for output generation
        results = {
            'changes': [],
            'warnings': [],
            'delete_barman_xlog': False,
            'missing_files': [],
            'get_wal': False,
        }
        recovery_info['results'] = results

        # Set up a list of configuration files
        recovery_info['configuration_files'].append('postgresql.conf')
        if backup_info.version >= 90400:
            recovery_info['configuration_files'].append('postgresql.auto.conf')

        # Handle remote recovery options
        if remote_command:
            recovery_info['recovery_dest'] = 'remote'
            try:
                recovery_info['rsync'] = RsyncPgData(
                    path=self.server.path,
                    ssh=remote_command,
                    bwlimit=self.config.bandwidth_limit,
                    network_compression=self.config.network_compression)
            except CommandFailedException:
                self._teardown(recovery_info)
                raise

            try:
                # create a UnixRemoteCommand obj if is a remote recovery
                recovery_info['cmd'] = UnixRemoteCommand(remote_command,
                                                         path=self.server.path)
            except FsOperationFailed:
                self._teardown(recovery_info)
                output.error(
                    "Unable to connect to the target host using the command "
                    "'%s'", remote_command)
                output.close_and_exit()
        else:
            # if is a local recovery create a UnixLocalCommand
            recovery_info['cmd'] = UnixLocalCommand()

        return recovery_info
Exemple #38
0
    def recover(self, backup_info, dest, tablespaces, target_tli,
                target_time, target_xid, target_name,
                exclusive, remote_command):
        """
        Performs a recovery of a backup

        :param barman.infofile.BackupInfo backup_info: the backup to recover
        :param str dest: the destination directory
        :param dict[str,str]|None tablespaces: a tablespace
            name -> location map (for relocation)
        :param str|None target_tli: the target timeline
        :param str|None target_time: the target time
        :param str|None target_xid: the target xid
        :param str|None target_name: the target name created previously with
                            pg_create_restore_point() function call
        :param bool exclusive: whether the recovery is exclusive or not
        :param str|None remote_command: The remote command to recover
                               the base backup, in case of remote backup.
        """

        # Run the cron to be sure the wal catalog is up to date
        # Prepare a map that contains all the objects required for a recovery
        recovery_info = self._setup(backup_info, remote_command, dest)
        output.info("Starting %s restore for server %s using backup %s",
                    recovery_info['recovery_dest'], self.server.config.name,
                    backup_info.backup_id)
        output.info("Destination directory: %s", dest)

        # Set targets for PITR
        self._set_pitr_targets(recovery_info,
                               backup_info, dest,
                               target_name,
                               target_time,
                               target_tli,
                               target_xid)

        # Retrieve the safe_horizon for smart copy
        self._retrieve_safe_horizon(recovery_info, backup_info, dest)

        # check destination directory. If doesn't exist create it
        try:
            recovery_info['cmd'].create_dir_if_not_exists(dest)
        except FsOperationFailed as e:
            output.error("unable to initialise destination directory "
                         "'%s': %s", dest, e)
            output.close_and_exit()

        # Initialize tablespace directories
        if backup_info.tablespaces:
            self._prepare_tablespaces(backup_info,
                                      recovery_info['cmd'],
                                      dest,
                                      tablespaces)
        # Copy the base backup
        output.info("Copying the base backup.")
        try:
            self._backup_copy(
                backup_info, dest,
                tablespaces, remote_command,
                recovery_info['safe_horizon'])
        except DataTransferFailure as e:
            output.error("Failure copying base backup: %s", e)
            output.close_and_exit()

        # Copy the backup.info file in the destination as
        # ".barman-recover.info"
        if remote_command:
            try:
                recovery_info['rsync'](backup_info.filename,
                                       ':%s/.barman-recover.info' % dest)
            except CommandFailedException as e:
                output.error(
                    'copy of recovery metadata file failed: %s', e)
                output.close_and_exit()
        else:
            backup_info.save(os.path.join(dest, '.barman-recover.info'))

        # Restore the WAL segments. If GET_WAL option is set, skip this phase
        # as they will be retrieved using the wal-get command.
        if not recovery_info['get_wal']:
            output.info("Copying required WAL segments.")

            try:
                # Retrieve a list of required log files
                required_xlog_files = tuple(
                    self.server.get_required_xlog_files(
                        backup_info, target_tli,
                        recovery_info['target_epoch']))

                # Restore WAL segments into the wal_dest directory
                self._xlog_copy(required_xlog_files,
                                recovery_info['wal_dest'],
                                remote_command)
            except DataTransferFailure as e:
                output.error("Failure copying WAL files: %s", e)
                output.close_and_exit()
            except BadXlogSegmentName as e:
                output.error(
                    "invalid xlog segment name %r\n"
                    "HINT: Please run \"barman rebuild-xlogdb %s\" "
                    "to solve this issue" %
                    str(e), self.config.name)
                output.close_and_exit()
            # If WAL files are put directly in the pg_xlog directory,
            # avoid shipping of just recovered files
            # by creating the corresponding archive status file
            if not recovery_info['is_pitr']:
                output.info("Generating archive status files")
                self._generate_archive_status(recovery_info,
                                              remote_command,
                                              required_xlog_files)

        # Generate recovery.conf file (only if needed by PITR)
        if recovery_info['is_pitr']:
            output.info("Generating recovery.conf")
            self._generate_recovery_conf(recovery_info, backup_info, dest,
                                         exclusive, remote_command,
                                         target_name, target_time,
                                         target_tli, target_xid)

        # Create archive_status directory if necessary
        archive_status_dir = os.path.join(dest, 'pg_xlog', 'archive_status')
        try:
            recovery_info['cmd'].create_dir_if_not_exists(archive_status_dir)
        except FsOperationFailed as e:
            output.error("unable to create the archive_status directory "
                         "'%s': %s", archive_status_dir, e)
            output.close_and_exit()

        # As last step, analyse configuration files in order to spot
        # harmful options. Barman performs automatic conversion of
        # some options as well as notifying users of their existence.
        #
        # This operation is performed in three steps:
        # 1) mapping
        # 2) analysis
        # 3) copy
        output.info("Identify dangerous settings in destination directory.")

        self._map_temporary_config_files(recovery_info,
                                         backup_info,
                                         remote_command)
        self._analyse_temporary_config_files(recovery_info)
        self._copy_temporary_config_files(dest,
                                          remote_command,
                                          recovery_info)

        # Cleanup operations
        self._teardown(recovery_info)

        return recovery_info
Exemple #39
0
def main():
    """
    The main method of Barman
    """
    p = ArghParser(epilog="Barman by EnterpriseDB (www.enterprisedb.com)")
    p.add_argument(
        "-v",
        "--version",
        action="version",
        version="%s\n\nBarman by EnterpriseDB (www.enterprisedb.com)" %
        barman.__version__,
    )
    p.add_argument(
        "-c",
        "--config",
        help="uses a configuration file "
        "(defaults: %s)" % ", ".join(barman.config.Config.CONFIG_FILES),
        default=SUPPRESS,
    )
    p.add_argument(
        "--color",
        "--colour",
        help="Whether to use colors in the output",
        choices=["never", "always", "auto"],
        default="auto",
    )
    p.add_argument(
        "--log-level",
        help="Override the default log level",
        choices=list(get_log_levels()),
        default=SUPPRESS,
    )
    p.add_argument("-q", "--quiet", help="be quiet", action="store_true")
    p.add_argument("-d", "--debug", help="debug output", action="store_true")
    p.add_argument(
        "-f",
        "--format",
        help="output format",
        choices=output.AVAILABLE_WRITERS.keys(),
        default=output.DEFAULT_WRITER,
    )
    p.add_commands([
        archive_wal,
        backup,
        check,
        check_backup,
        cron,
        delete,
        diagnose,
        get_wal,
        keep,
        list_backup,
        list_backups,
        list_files,
        list_server,
        list_servers,
        put_wal,
        rebuild_xlogdb,
        receive_wal,
        recover,
        show_backup,
        show_server,
        show_servers,
        replication_status,
        status,
        switch_wal,
        switch_xlog,
        sync_info,
        sync_backup,
        sync_wals,
    ])
    # noinspection PyBroadException
    try:
        p.dispatch(pre_call=global_config)
    except KeyboardInterrupt:
        msg = "Process interrupted by user (KeyboardInterrupt)"
        output.error(msg)
    except Exception as e:
        msg = "%s\nSee log file for more details." % e
        output.exception(msg)

    # cleanup output API and exit honoring output.error_occurred and
    # output.error_exit_code
    output.close_and_exit()
Exemple #40
0
    def _generate_recovery_conf(self, recovery_info, backup_info, dest,
                                exclusive, remote_command, target_name,
                                target_time, target_tli, target_xid):
        """
        Generate a recovery.conf file for PITR containing
        all the required configurations

        :param dict recovery_info: Dictionary containing all the recovery
            parameters
        :param barman.infofile.BackupInfo backup_info: representation of a
            backup
        :param str dest: destination directory of the recovery
        :param boolean exclusive: exclusive backup or concurrent
        :param str remote_command: ssh command for remote connection
        :param str target_name: recovery target name for PITR
        :param str target_time: recovery target time for PITR
        :param str target_tli: recovery target timeline for PITR
        :param str target_xid: recovery target transaction id for PITR
        """
        if remote_command:
            recovery = open(os.path.join(recovery_info['tempdir'],
                                         'recovery.conf'), 'w')
        else:
            recovery = open(os.path.join(dest, 'recovery.conf'), 'w')

        # If GET_WAL has been set, use the get-wal command to retrieve the
        # required wal files. Otherwise use the unix command "cp" to copy
        # them from the barman_xlog directory
        if recovery_info['get_wal']:
            # We need to create the right restore command.
            # If we are doing a remote recovery,
            # the barman-cli package is REQUIRED on the server that is hosting
            # the PostgreSQL server.
            # We use the machine FQDN and the barman_user
            # setting to call the barman-wal-restore correctly.
            # If local recovery, we use barman directly, assuming
            # the postgres process will be executed with the barman user.
            # It MUST to be reviewed by the user in any case.
            if remote_command:
                fqdn = socket.getfqdn()
                print("# The 'barman-wal-restore' command "
                      "is provided in the 'barman-cli' package",
                      file=recovery)
                print("restore_command = 'barman-wal-restore -U %s "
                      "%s %s %%f %%p'" % (self.config.config.user,
                                          fqdn, self.config.name),
                      file=recovery)
            else:
                print("# The 'barman get-wal' command "
                      "must run as '%s' user" % self.config.config.user,
                      file=recovery)
                print("restore_command = 'sudo -u %s "
                      "barman get-wal %s %%f > %%p'" % (
                          self.config.config.user,
                          self.config.name),
                      file=recovery)
            recovery_info['results']['get_wal'] = True
        else:
            print("restore_command = 'cp barman_xlog/%f %p'", file=recovery)
        if backup_info.version >= 80400 and \
                not recovery_info['get_wal']:
            print("recovery_end_command = 'rm -fr barman_xlog'", file=recovery)
        if target_time:
            print("recovery_target_time = '%s'" % target_time, file=recovery)
        if target_tli:
            print("recovery_target_timeline = %s" % target_tli, file=recovery)
        if target_xid:
            print("recovery_target_xid = '%s'" % target_xid, file=recovery)
        if target_name:
            print("recovery_target_name = '%s'" % target_name, file=recovery)
        if (target_xid or target_time) and exclusive:
            print("recovery_target_inclusive = '%s'" % (
                not exclusive), file=recovery)
        recovery.close()
        if remote_command:
            plain_rsync = RsyncPgData(
                path=self.server.path,
                ssh=remote_command,
                bwlimit=self.config.bandwidth_limit,
                network_compression=self.config.network_compression)
            try:
                plain_rsync.from_file_list(['recovery.conf'],
                                           recovery_info['tempdir'],
                                           ':%s' % dest)
            except CommandFailedException as e:
                output.error('remote copy of recovery.conf failed: %s', e)
                output.close_and_exit()