def check(args):
"""
Check if the server configuration is working.
This command returns success if every checks pass,
or failure if any of these fails
"""
if args.nagios:
output.set_output_writer(output.NagiosOutputWriter())
servers = get_server_list(args)
for name in sorted(servers):
server = servers[name]
# Validate the returned server
if not manage_server_command(
server, name, skip_inactive=False,
skip_disabled=False, disabled_is_error=False):
continue
# If the server has been manually disabled
if not server.config.active:
name += " (inactive)"
# If server has configuration errors
elif server.config.disabled:
name += " (WARNING: disabled)"
output.init('check', name, server.config.active)
server.check()
output.close_and_exit()
def copy_temporary_config_files(self, dest, remote_command, recovery_info):
"""
Copy modified configuration files using rsync in case of
remote recovery
:param str dest: destination directory of the recovery
:param str remote_command: ssh command for remote connection
:param dict recovery_info: Dictionary containing all the recovery
parameters
"""
if remote_command:
# If this is a remote recovery, rsync the modified files from the
# temporary local directory to the remote destination directory.
file_list = []
for conf_file in recovery_info['configuration_files']:
file_list.append('%s' % conf_file)
file_list.append('%s.origin' % conf_file)
try:
recovery_info['rsync'].from_file_list(file_list,
recovery_info['tempdir'],
':%s' % dest)
except CommandFailedException as e:
output.exception(
'remote copy of configuration files failed: %s', e)
output.close_and_exit()
def backup(args):
"""
Perform a full backup for the given server
"""
servers = get_server_list(args, skip_disabled=True)
for name in sorted(servers):
server = servers[name]
if server is None:
output.error("Unknown server '%s'" % name)
continue
# If the server is disabled return an error message
if not server.config.active:
output.error(
"Server '%s' is disabled.\n"
"HINT: remove 'active=False' from server configuration "
"to enable it.",
name)
continue
if args.reuse_backup is not None:
server.config.reuse_backup = args.reuse_backup
if args.retry_sleep is not None:
server.config.basebackup_retry_sleep = args.retry_sleep
if args.retry_times is not None:
server.config.basebackup_retry_times = args.retry_times
if hasattr(args, 'immediate_checkpoint'):
server.config.immediate_checkpoint = args.immediate_checkpoint
server.backup()
output.close_and_exit()
def generate_archive_status(self, recovery_info, remote_command,
required_xlog_files):
"""
Populate the archive_status directory
:param dict recovery_info: Dictionary containing all the recovery
parameters
:param str remote_command: ssh command for remote connection
:param tuple required_xlog_files: list of required WAL segments
"""
if remote_command:
status_dir = recovery_info['tempdir']
else:
status_dir = os.path.join(recovery_info['wal_dest'],
'archive_status')
mkpath(status_dir)
for wal_info in required_xlog_files:
with open(os.path.join(status_dir, "%s.done" % wal_info.name),
'a') as f:
f.write('')
if remote_command:
try:
recovery_info['rsync']('%s/' % status_dir,
':%s' % os.path.join(
recovery_info['wal_dest'],
'archive_status'))
except CommandFailedException as e:
output.exception(
"unable to populate pg_xlog/archive_status"
"directory: %s", e)
output.close_and_exit()
def diagnose():
"""
Diagnostic command (for support and problems detection purpose)
"""
servers = get_server_list(None)
barman.diagnose.exec_diagnose(servers)
output.close_and_exit()
def setup(self, backup_info, remote_command, dest):
"""
Prepare the recovery_info dictionary for the recovery, as well
as temporary working directory
:param barman.infofile.BackupInfo backup_info: representation of a
backup
:param str remote_command: ssh command for remote connection
:return dict: recovery_info dictionary, holding the basic values for a
recovery
"""
recovery_info = {
'cmd': None,
'recovery_dest': 'local',
'rsync': None,
'configuration_files': [],
'destination_path': dest,
'temporary_configuration_files': [],
'tempdir': tempfile.mkdtemp(prefix='barman_recovery-'),
'is_pitr': False,
'wal_dest': os.path.join(dest, 'pg_xlog'),
'get_wal': RecoveryOptions.GET_WAL in self.config.recovery_options,
}
# A map that will keep track of the results of the recovery.
# Used for output generation
results = {
'changes': [],
'warnings': [],
'delete_barman_xlog': False,
'get_wal': False,
}
recovery_info['results'] = results
# Set up a list of configuration files
recovery_info['configuration_files'].append('postgresql.conf')
if backup_info.version >= 90400:
recovery_info['configuration_files'].append('postgresql.auto.conf')
# Handle remote recovery options
if remote_command:
recovery_info['recovery_dest'] = 'remote'
recovery_info['rsync'] = RsyncPgData(
path=self.server.path,
ssh=remote_command,
bwlimit=self.config.bandwidth_limit,
network_compression=self.config.network_compression)
try:
# create a UnixRemoteCommand obj if is a remote recovery
recovery_info['cmd'] = UnixRemoteCommand(remote_command)
except FsOperationFailed:
output.error(
"Unable to connect to the target host using the command "
"'%s'", remote_command)
output.close_and_exit()
else:
# if is a local recovery create a UnixLocalCommand
recovery_info['cmd'] = UnixLocalCommand()
return recovery_info
def test_close_and_exit(self, exit_mock):
# preparation
writer = self._mock_writer()
output.close_and_exit()
writer.close.assert_called_once_with()
exit_mock.assert_called_once_with(0)
def switch_xlog(args):
"""
Execute the switch-xlog command on the target server
"""
servers = get_server_list(args, skip_inactive=True)
for name in sorted(servers):
server = servers[name]
server.switch_xlog(args.force)
output.close_and_exit()
def receive_wal(args):
"""
Start a receive-wal process.
The process uses the streaming protocol to receive WAL files
from the PostgreSQL server.
"""
server = get_server(args)
server.receive_wal()
output.close_and_exit()
def delete(args):
"""
Delete a backup
"""
server = get_server(args)
# Retrieves the backup
backup_id = parse_backup_id(server, args)
server.delete_backup(backup_id)
output.close_and_exit()
def archive_wal(args):
"""
Execute maintenance operations on WAL files for a given server.
This command processes any incoming WAL files for the server
and archives them along the catalogue.
"""
server = get_server(args)
server.archive_wal()
output.close_and_exit()
def diagnose():
"""
Diagnostic command (for support and problems detection purpose)
"""
# Get every server (both inactive and temporarily disabled)
servers = get_server_list(on_error_stop=False, suppress_error=True)
# errors list with duplicate paths between servers
errors_list = barman.__config__.servers_msg_list
barman.diagnose.exec_diagnose(servers, errors_list)
output.close_and_exit()
def show_backup(args):
"""
This method shows a single backup information
"""
server = get_server(args)
# Retrieves the backup
backup_info = parse_backup_id(server, args)
server.show_backup(backup_info)
output.close_and_exit()
def list_server(minimal=False):
"""
List available servers, with useful information
"""
servers = get_server_list()
for name in sorted(servers):
server = servers[name]
output.init('list_server', name, minimal=minimal)
output.result('list_server', name, server.config.description)
output.close_and_exit()
def list_files(args):
"""
List all the files for a single backup
"""
server = get_server(args)
# Retrieves the backup
backup_id = parse_backup_id(server, args)
for line in backup_id.get_list_of_files(args.target):
output.info(line, log=False)
output.close_and_exit()
def test_close_and_exit_with_error(self, exit_mock):
# preparation
writer = self._mock_writer()
output.error_occurred = True
output.close_and_exit()
writer.close.assert_called_once_with()
assert exit_mock.called
assert exit_mock.call_count == 1
assert exit_mock.call_args[0] != 0
def archive_wal(args):
"""
Execute maintenance operations on WAL files for a given server.
This command processes any incoming WAL files for the server
and archives them along the catalogue.
"""
server = get_server(args)
output.debug("Starting archive-wal for server %s", server.config.name)
server.archive_wal()
output.close_and_exit()
def rebuild_xlogdb(args):
"""
Rebuild the WAL file database guessing it from the disk content.
"""
servers = get_server_list(args)
for name in sorted(servers):
server = servers[name]
if server is None:
output.error("Unknown server '%s'", name)
continue
server.rebuild_xlogdb()
output.close_and_exit()
def main():
"""
The main method of Barman
"""
p = ArghParser(epilog='Barman by 2ndQuadrant (www.2ndQuadrant.com)')
p.add_argument('-v', '--version', action='version',
version='%s\n\nBarman by 2ndQuadrant (www.2ndQuadrant.com)'
% barman.__version__)
p.add_argument('-c', '--config',
help='uses a configuration file '
'(defaults: %s)'
% ', '.join(barman.config.Config.CONFIG_FILES),
default=SUPPRESS)
p.add_argument('-q', '--quiet', help='be quiet', action='store_true')
p.add_argument('-d', '--debug', help='debug output', action='store_true')
p.add_argument('-f', '--format', help='output format',
choices=output.AVAILABLE_WRITERS.keys(),
default=output.DEFAULT_WRITER)
p.add_commands(
[
archive_wal,
backup,
check,
cron,
delete,
diagnose,
get_wal,
list_backup,
list_files,
list_server,
rebuild_xlogdb,
receive_wal,
recover,
show_backup,
show_server,
replication_status,
status,
switch_xlog,
]
)
# noinspection PyBroadException
try:
p.dispatch(pre_call=global_config)
except KeyboardInterrupt:
msg = "Process interrupted by user (KeyboardInterrupt)"
output.error(msg)
except Exception as e:
msg = "%s\nSee log file for more details." % e
output.exception(msg)
# cleanup output API and exit honoring output.error_occurred and
# output.error_exit_code
output.close_and_exit()
def generate_recovery_conf(self, recovery_info, backup_info, dest,
exclusive, remote_command, target_name,
target_time, target_tli, target_xid):
"""
Generate a recovery.conf file for PITR containing
all the required configurations
:param dict recovery_info: Dictionary containing all the recovery params
:param barman.infofile.BackupInfo backup_info: representation of a
backup
:param str dest: destination directory of the recovery
:param boolean exclusive: exclusive backup or concurrent
:param str remote_command: ssh command for remote connection
:param str target_name: recovery target name for PITR
:param str target_time: recovery target time for PITR
:param str target_tli: recovery target timeline for PITR
:param str target_xid: recovery target transaction id for PITR
"""
if remote_command:
recovery = open(os.path.join(recovery_info['tempdir'],
'recovery.conf'), 'w')
else:
recovery = open(os.path.join(dest, 'recovery.conf'), 'w')
print >> recovery, "restore_command = 'cp barman_xlog/%f %p'"
if backup_info.version >= 80400:
print >> recovery, "recovery_end_command = 'rm -fr barman_xlog'"
if target_time:
print >> recovery, "recovery_target_time = '%s'" % target_time
if target_tli:
print >> recovery, "recovery_target_timeline = %s" % target_tli
if target_xid:
print >> recovery, "recovery_target_xid = '%s'" % target_xid
if target_name:
print >> recovery, "recovery_target_name = '%s'" % target_name
if (target_xid or target_time) and exclusive:
print >> recovery, "recovery_target_inclusive = '%s'" % (
not exclusive)
recovery.close()
if remote_command:
# Uses plain rsync (without exclusions) to ship recovery.conf
plain_rsync = Rsync(
ssh=remote_command,
bwlimit=self.config.bandwidth_limit,
network_compression=self.config.network_compression)
try:
plain_rsync.from_file_list(['recovery.conf'],
recovery_info['tempdir'],
':%s' % dest)
except CommandFailedException, e:
output.exception(
'remote copy of recovery.conf failed: %s', e)
output.close_and_exit()
def show_server(args):
"""
Show all configuration parameters for the specified servers
"""
servers = get_server_list(args)
for name in sorted(servers):
server = servers[name]
if server is None:
output.error("Unknown server '%s'" % name)
continue
output.init('show_server', name)
server.show()
output.close_and_exit()
def status(args):
"""
Shows live information and status of the PostgreSQL server
"""
servers = get_server_list(args)
for name in sorted(servers):
server = servers[name]
if server is None:
output.error("Unknown server '%s'" % name)
continue
output.init('status', name)
server.status()
output.close_and_exit()
def list_backup(args):
"""
List available backups for the given server (supports 'all')
"""
servers = get_server_list(args)
for name in sorted(servers):
server = servers[name]
output.init('list_backup', name, minimal=args.minimal)
if server is None:
output.error("Unknown server '%s'" % name)
continue
server.list_backups()
output.close_and_exit()
def switch_xlog(args):
"""
Execute the switch-xlog command on the target server
"""
servers = get_server_list(args, skip_inactive=True)
for name in sorted(servers):
server = servers[name]
# Skip the server (apply general rule)
if not manage_server_command(server, name):
continue
with closing(server):
server.switch_xlog(args.force, args.archive, args.archive_timeout)
output.close_and_exit()
def prepare_tablespaces(self, backup_info, cmd, dest, tablespaces):
"""
Prepare the directory structure for required tablespaces,
taking care of tablespaces relocation, if requested.
:param barman.infofile.BackupInfo backup_info: backup representation
:param barman.fs.UnixLocalCommand cmd: Object for
filesystem interaction
:param str dest: destination dir for the recovery
:param dict tablespaces: dict of all the tablespaces and their location
"""
tblspc_dir = os.path.join(dest, 'pg_tblspc')
try:
# check for pg_tblspc dir into recovery destination folder.
# if it does not exists, create it
cmd.create_dir_if_not_exists(tblspc_dir)
except FsOperationFailed as e:
output.exception("unable to initialise tablespace directory "
"'%s': %s", tblspc_dir, e)
output.close_and_exit()
for item in backup_info.tablespaces:
# build the filename of the link under pg_tblspc directory
pg_tblspc_file = os.path.join(tblspc_dir, str(item.oid))
# by default a tablespace goes in the same location where
# it was on the source server when the backup was taken
location = item.location
# if a relocation has been requested for this tablespace,
# use the target directory provided by the user
if tablespaces and item.name in tablespaces:
location = tablespaces[item.name]
try:
# remove the current link in pg_tblspc, if it exists
# (raise an exception if it is a directory)
cmd.delete_if_exists(pg_tblspc_file)
# create tablespace location, if does not exist
# (raise an exception if it is not possible)
cmd.create_dir_if_not_exists(location)
# check for write permissions on destination directory
cmd.check_write_permission(location)
# create symlink between tablespace and recovery folder
cmd.create_symbolic_link(location, pg_tblspc_file)
except FsOperationFailed as e:
output.exception("unable to prepare '%s' tablespace "
"(destination '%s'): %s",
item.name, location, e)
output.close_and_exit()
output.info("\t%s, %s, %s", item.oid, item.name, location)
def rebuild_xlogdb(args):
"""
Rebuild the WAL file database guessing it from the disk content.
"""
servers = get_server_list(args, skip_inactive=True)
for name in sorted(servers):
server = servers[name]
# Skip the server (apply general rule)
if not manage_server_command(server, name):
continue
server.rebuild_xlogdb()
output.close_and_exit()
def get_wal(args):
"""
Retrieve WAL_NAME file from SERVER_NAME archive.
The content will be streamed on standard output unless
the --output-directory option is specified.
"""
server = get_server(args)
# Retrieve optional arguments. If an argument is not specified,
# the namespace doesn't contain it due to SUPPRESS default.
# In that case we pick 'None' using getattr third argument.
compression = getattr(args, "compression", None)
output_directory = getattr(args, "output_directory", None)
server.get_wal(args.wal_name, compression=compression, output_directory=output_directory)
output.close_and_exit()
def receive_wal(args):
"""
Start a receive-wal process.
The process uses the streaming protocol to receive WAL files
from the PostgreSQL server.
"""
server = get_server(args)
# If the caller requested to shutdown the receive-wal process deliver the
# termination signal, otherwise attempt to start it
if args.stop:
server.kill('receive-wal')
else:
server.receive_wal()
output.close_and_exit()
def show_server(args):
"""
Show all configuration parameters for the specified servers
"""
servers = get_server_list(args, skip_inactive=True)
for name in sorted(servers):
server = servers[name]
# Skip the server (apply general rule)
if not manage_server_command(server, name):
continue
output.init('show_server', name)
server.show()
output.close_and_exit()
def status(args):
"""
Shows live information and status of the PostgreSQL server
"""
servers = get_server_list(args, skip_inactive=True)
for name in sorted(servers):
server = servers[name]
# Skip the server (apply general rule)
if not manage_server_command(server, name):
continue
output.init('status', name)
server.status()
output.close_and_exit()
def recover(args):
"""
Recover a server at a given time, name, LSN or xid
"""
server = get_server(args)
# Retrieves the backup
backup_id = parse_backup_id(server, args)
if backup_id.status not in BackupInfo.STATUS_COPY_DONE:
output.error(
"Cannot recover from backup '%s' of server '%s': "
"backup status is not DONE",
args.backup_id, server.config.name)
output.close_and_exit()
# decode the tablespace relocation rules
tablespaces = {}
if args.tablespace:
for rule in args.tablespace:
try:
tablespaces.update([rule.split(':', 1)])
except ValueError:
output.error(
"Invalid tablespace relocation rule '%s'\n"
"HINT: The valid syntax for a relocation rule is "
"NAME:LOCATION", rule)
output.close_and_exit()
# validate the rules against the tablespace list
valid_tablespaces = []
if backup_id.tablespaces:
valid_tablespaces = [tablespace_data.name for tablespace_data in
backup_id.tablespaces]
for item in tablespaces:
if item not in valid_tablespaces:
output.error("Invalid tablespace name '%s'\n"
"HINT: Please use any of the following "
"tablespaces: %s",
item, ', '.join(valid_tablespaces))
output.close_and_exit()
# explicitly disallow the rsync remote syntax (common mistake)
if ':' in args.destination_directory:
output.error(
"The destination directory parameter "
"cannot contain the ':' character\n"
"HINT: If you want to do a remote recovery you have to use "
"the --remote-ssh-command option")
output.close_and_exit()
if args.retry_sleep is not None:
server.config.basebackup_retry_sleep = args.retry_sleep
if args.retry_times is not None:
server.config.basebackup_retry_times = args.retry_times
if hasattr(args, 'get_wal'):
if args.get_wal:
server.config.recovery_options.add(RecoveryOptions.GET_WAL)
else:
server.config.recovery_options.remove(RecoveryOptions.GET_WAL)
if args.jobs is not None:
server.config.parallel_jobs = args.jobs
if hasattr(args, 'bwlimit'):
server.config.bandwidth_limit = args.bwlimit
# PostgreSQL supports multiple parameters to specify when the recovery
# process will end, and in that case the last entry in recovery
# configuration files will be used. See [1]
#
# Since the meaning of the target options is not dependent on the order
# of parameters, we decided to make the target options mutually exclusive.
#
# [1]: https://www.postgresql.org/docs/current/static/
# recovery-target-settings.html
target_options = ['target_tli', 'target_time', 'target_xid',
'target_lsn', 'target_name', 'target_immediate']
specified_target_options = len(
[option for option in target_options if getattr(args, option)])
if specified_target_options > 1:
output.error(
"You cannot specify multiple targets for the recovery operation")
output.close_and_exit()
if hasattr(args, 'network_compression'):
if args.network_compression and args.remote_ssh_command is None:
output.error(
"Network compression can only be used with "
"remote recovery.\n"
"HINT: If you want to do a remote recovery "
"you have to use the --remote-ssh-command option")
output.close_and_exit()
server.config.network_compression = args.network_compression
with closing(server):
try:
server.recover(backup_id,
args.destination_directory,
tablespaces=tablespaces,
target_tli=args.target_tli,
target_time=args.target_time,
target_xid=args.target_xid,
target_lsn=args.target_lsn,
target_name=args.target_name,
target_immediate=args.target_immediate,
exclusive=args.exclusive,
remote_command=args.remote_ssh_command,
target_action=getattr(args, 'target_action', None),
standby_mode=getattr(args, 'standby_mode', None))
except RecoveryException as exc:
output.error(force_str(exc))
output.close_and_exit()
def recover(args):
"""
Recover a server at a given time or xid
"""
server = get_server(args)
# Retrieves the backup
backup_id = parse_backup_id(server, args)
if backup_id.status != BackupInfo.DONE:
output.error(
"Cannot recover from backup '%s' of server '%s': "
"backup status is not DONE", args.backup_id, server.config.name)
output.close_and_exit()
# decode the tablespace relocation rules
tablespaces = {}
if args.tablespace:
for rule in args.tablespace:
try:
tablespaces.update([rule.split(':', 1)])
except ValueError:
output.error(
"Invalid tablespace relocation rule '%s'\n"
"HINT: The valid syntax for a relocation rule is "
"NAME:LOCATION", rule)
output.close_and_exit()
# validate the rules against the tablespace list
valid_tablespaces = []
if backup_id.tablespaces:
valid_tablespaces = [
tablespace_data.name for tablespace_data in backup_id.tablespaces
]
for item in tablespaces:
if item not in valid_tablespaces:
output.error(
"Invalid tablespace name '%s'\n"
"HINT: Please use any of the following "
"tablespaces: %s", item, ', '.join(valid_tablespaces))
output.close_and_exit()
# explicitly disallow the rsync remote syntax (common mistake)
if ':' in args.destination_directory:
output.error(
"The destination directory parameter "
"cannot contain the ':' character\n"
"HINT: If you want to do a remote recovery you have to use "
"the --remote-ssh-command option")
output.close_and_exit()
if args.retry_sleep is not None:
server.config.basebackup_retry_sleep = args.retry_sleep
if args.retry_times is not None:
server.config.basebackup_retry_times = args.retry_times
with closing(server):
server.recover(backup_id,
args.destination_directory,
tablespaces=tablespaces,
target_tli=args.target_tli,
target_time=args.target_time,
target_xid=args.target_xid,
target_name=args.target_name,
exclusive=args.exclusive,
remote_command=args.remote_ssh_command)
output.close_and_exit()
def get_server(args,
skip_inactive=True,
skip_disabled=False,
on_error_stop=True,
suppress_error=False):
"""
Get a single server retrieving its configuration (wraps get_server_list())
Returns a Server object or None if the required server is unknown and
on_error_stop is False.
WARNING: this function modifies the 'args' parameter
:param args: an argparse namespace containing a single
server_name parameter
WARNING: the function modifies the content of this parameter
:param bool skip_inactive: skip inactive servers when 'all' is required
:param bool skip_disabled: skip disabled servers when 'all' is required
:param bool on_error_stop: stop if an error is found
:param bool suppress_error: suppress display of errors (e.g. diagnose)
:rtype: barman.server.Server|None
"""
# This function must to be called with in a single-server context
name = args.server_name
assert isinstance(name, str)
# The 'all' special name is forbidden in this context
if name == 'all':
output.error("You cannot use 'all' in a single server context")
output.close_and_exit()
# The following return statement will never be reached
# but it is here for clarity
return None
# Builds a list from a single given name
args.server_name = [name]
# Retrieve the requested server
servers = get_server_list(args, skip_inactive, skip_disabled,
on_error_stop, suppress_error)
# The requested server has been excluded from get_server_list result
if len(servers) == 0:
output.close_and_exit()
# The following return statement will never be reached
# but it is here for clarity
return None
# retrieve the server object
server = servers[name]
# Apply standard validation control and skips
# the server if inactive or disabled, displaying standard
# error messages. If on_error_stop (default) exits
if not manage_server_command(server, name) and on_error_stop:
output.close_and_exit()
# The following return statement will never be reached
# but it is here for clarity
return None
# Returns the filtered server
return server
def _set_pitr_targets(self, recovery_info, backup_info, dest, target_name,
target_time, target_tli, target_xid):
"""
Set PITR targets - as specified by the user
:param dict recovery_info: Dictionary containing all the recovery
parameters
:param barman.infofile.BackupInfo backup_info: representation of a
backup
:param str dest: destination directory of the recovery
:param str|None target_name: recovery target name for PITR
:param str|None target_time: recovery target time for PITR
:param str|None target_tli: recovery target timeline for PITR
:param str|None target_xid: recovery target transaction id for PITR
"""
target_epoch = None
target_datetime = None
if (target_time or
target_xid or
(target_tli and target_tli != backup_info.timeline) or
target_name or
recovery_info['get_wal']):
recovery_info['is_pitr'] = True
targets = {}
if target_time:
# noinspection PyBroadException
try:
target_datetime = dateutil.parser.parse(target_time)
except ValueError as e:
output.error(
"unable to parse the target time parameter %r: %s",
target_time, e)
self._teardown(recovery_info)
output.close_and_exit()
except Exception:
# this should not happen, but there is a known bug in
# dateutil.parser.parse() implementation
# ref: https://bugs.launchpad.net/dateutil/+bug/1247643
output.error(
"unable to parse the target time parameter %r",
target_time)
output.close_and_exit()
target_epoch = (
time.mktime(target_datetime.timetuple()) +
(target_datetime.microsecond / 1000000.))
targets['time'] = str(target_datetime)
if target_xid:
targets['xid'] = str(target_xid)
if target_tli and target_tli != backup_info.timeline:
targets['timeline'] = str(target_tli)
if target_name:
targets['name'] = str(target_name)
output.info(
"Doing PITR. Recovery target %s",
(", ".join(["%s: %r" % (k, v) for k, v in targets.items()])))
recovery_info['wal_dest'] = os.path.join(dest, 'barman_xlog')
# With a PostgreSQL version older than 8.4, it is the user's
# responsibility to delete the "barman_xlog" directory as the
# restore_command option in recovery.conf is not supported
if backup_info.version < 80400 and \
not recovery_info['get_wal']:
recovery_info['results']['delete_barman_xlog'] = True
recovery_info['target_epoch'] = target_epoch
recovery_info['target_datetime'] = target_datetime
def get_server_list(args=None, skip_inactive=False, skip_disabled=False,
skip_passive=False,
on_error_stop=True, suppress_error=False):
"""
Get the server list from the configuration
If args the parameter is None or arg.server_name is ['all']
returns all defined servers
:param args: an argparse namespace containing a list server_name parameter
:param bool skip_inactive: skip inactive servers when 'all' is required
:param bool skip_disabled: skip disabled servers when 'all' is required
:param bool skip_passive: skip passive servers when 'all' is required
:param bool on_error_stop: stop if an error is found
:param bool suppress_error: suppress display of errors (e.g. diagnose)
:rtype: dict[str,Server]
"""
server_dict = {}
# This function must to be called with in a multiple-server context
assert not args or isinstance(args.server_name, list)
# Generate the list of servers (required for global errors)
available_servers = barman.__config__.server_names()
# Get a list of configuration errors from all the servers
global_error_list = barman.__config__.servers_msg_list
# Global errors have higher priority
if global_error_list:
# Output the list of global errors
if not suppress_error:
for error in global_error_list:
output.error(error)
# If requested, exit on first error
if on_error_stop:
output.close_and_exit()
# The following return statement will never be reached
# but it is here for clarity
return {}
# Handle special 'all' server cases
# - args is None
# - 'all' special name
if not args or 'all' in args.server_name:
# When 'all' is used, it must be the only specified argument
if args and len(args.server_name) != 1:
output.error("You cannot use 'all' with other server names")
servers = available_servers
else:
# Put servers in a set, so multiple occurrences are counted only once
servers = set(args.server_name)
# Loop through all the requested servers
for server in servers:
conf = barman.__config__.get_server(server)
if conf is None:
# Unknown server
server_dict[server] = None
else:
server_object = Server(conf)
# Skip inactive servers, if requested
if skip_inactive and not server_object.config.active:
output.info("Skipping inactive server '%s'"
% conf.name)
continue
# Skip disabled servers, if requested
if skip_disabled and server_object.config.disabled:
output.info("Skipping temporarily disabled server '%s'"
% conf.name)
continue
# Skip passive nodes, if requested
if skip_passive and server_object.passive_node:
output.info("Skipping passive server '%s'",
conf.name)
continue
server_dict[server] = server_object
return server_dict
def main():
"""
The main method of Barman
"""
p = ArghParser(epilog='Barman by 2ndQuadrant (www.2ndQuadrant.com)')
p.add_argument('-v', '--version', action='version',
version='%s\n\nBarman by 2ndQuadrant (www.2ndQuadrant.com)'
% barman.__version__)
p.add_argument('-c', '--config',
help='uses a configuration file '
'(defaults: %s)'
% ', '.join(barman.config.Config.CONFIG_FILES),
default=SUPPRESS)
p.add_argument('--color', '--colour',
help='Whether to use colors in the output',
choices=['never', 'always', 'auto'],
default='auto')
p.add_argument('--log-level',
help='Override the default log level',
choices=list(get_log_levels()),
default=SUPPRESS)
p.add_argument('-q', '--quiet', help='be quiet', action='store_true')
p.add_argument('-d', '--debug', help='debug output', action='store_true')
p.add_argument('-f', '--format', help='output format',
choices=output.AVAILABLE_WRITERS.keys(),
default=output.DEFAULT_WRITER)
p.add_commands(
[
archive_wal,
backup,
check,
check_backup,
cron,
delete,
diagnose,
get_wal,
list_backup,
list_files,
list_server,
put_wal,
rebuild_xlogdb,
receive_wal,
recover,
show_backup,
show_server,
replication_status,
status,
switch_wal,
switch_xlog,
sync_info,
sync_backup,
sync_wals,
]
)
# noinspection PyBroadException
try:
p.dispatch(pre_call=global_config)
except KeyboardInterrupt:
msg = "Process interrupted by user (KeyboardInterrupt)"
output.error(msg)
except Exception as e:
msg = "%s\nSee log file for more details." % e
output.exception(msg)
# cleanup output API and exit honoring output.error_occurred and
# output.error_exit_code
output.close_and_exit()
def _setup(self, backup_info, remote_command, dest):
"""
Prepare the recovery_info dictionary for the recovery, as well
as temporary working directory
:param barman.infofile.BackupInfo backup_info: representation of a
backup
:param str remote_command: ssh command for remote connection
:return dict: recovery_info dictionary, holding the basic values for a
recovery
"""
recovery_info = {
'cmd': None,
'recovery_dest': 'local',
'rsync': None,
'configuration_files': [],
'destination_path': dest,
'temporary_configuration_files': [],
'tempdir': tempfile.mkdtemp(prefix='barman_recovery-'),
'is_pitr': False,
'wal_dest': os.path.join(dest, 'pg_xlog'),
'get_wal': RecoveryOptions.GET_WAL in self.config.recovery_options,
}
# A map that will keep track of the results of the recovery.
# Used for output generation
results = {
'changes': [],
'warnings': [],
'delete_barman_xlog': False,
'missing_files': [],
'get_wal': False,
}
recovery_info['results'] = results
# Set up a list of configuration files
recovery_info['configuration_files'].append('postgresql.conf')
if backup_info.version >= 90400:
recovery_info['configuration_files'].append('postgresql.auto.conf')
# Handle remote recovery options
if remote_command:
recovery_info['recovery_dest'] = 'remote'
try:
recovery_info['rsync'] = RsyncPgData(
path=self.server.path,
ssh=remote_command,
bwlimit=self.config.bandwidth_limit,
network_compression=self.config.network_compression)
except CommandFailedException:
self._teardown(recovery_info)
raise
try:
# create a UnixRemoteCommand obj if is a remote recovery
recovery_info['cmd'] = UnixRemoteCommand(remote_command,
path=self.server.path)
except FsOperationFailed:
self._teardown(recovery_info)
output.error(
"Unable to connect to the target host using the command "
"'%s'", remote_command)
output.close_and_exit()
else:
# if is a local recovery create a UnixLocalCommand
recovery_info['cmd'] = UnixLocalCommand()
return recovery_info
def recover(self, backup_info, dest, tablespaces, target_tli,
target_time, target_xid, target_name,
exclusive, remote_command):
"""
Performs a recovery of a backup
:param barman.infofile.BackupInfo backup_info: the backup to recover
:param str dest: the destination directory
:param dict[str,str]|None tablespaces: a tablespace
name -> location map (for relocation)
:param str|None target_tli: the target timeline
:param str|None target_time: the target time
:param str|None target_xid: the target xid
:param str|None target_name: the target name created previously with
pg_create_restore_point() function call
:param bool exclusive: whether the recovery is exclusive or not
:param str|None remote_command: The remote command to recover
the base backup, in case of remote backup.
"""
# Run the cron to be sure the wal catalog is up to date
# Prepare a map that contains all the objects required for a recovery
recovery_info = self._setup(backup_info, remote_command, dest)
output.info("Starting %s restore for server %s using backup %s",
recovery_info['recovery_dest'], self.server.config.name,
backup_info.backup_id)
output.info("Destination directory: %s", dest)
# Set targets for PITR
self._set_pitr_targets(recovery_info,
backup_info, dest,
target_name,
target_time,
target_tli,
target_xid)
# Retrieve the safe_horizon for smart copy
self._retrieve_safe_horizon(recovery_info, backup_info, dest)
# check destination directory. If doesn't exist create it
try:
recovery_info['cmd'].create_dir_if_not_exists(dest)
except FsOperationFailed as e:
output.error("unable to initialise destination directory "
"'%s': %s", dest, e)
output.close_and_exit()
# Initialize tablespace directories
if backup_info.tablespaces:
self._prepare_tablespaces(backup_info,
recovery_info['cmd'],
dest,
tablespaces)
# Copy the base backup
output.info("Copying the base backup.")
try:
self._backup_copy(
backup_info, dest,
tablespaces, remote_command,
recovery_info['safe_horizon'])
except DataTransferFailure as e:
output.error("Failure copying base backup: %s", e)
output.close_and_exit()
# Copy the backup.info file in the destination as
# ".barman-recover.info"
if remote_command:
try:
recovery_info['rsync'](backup_info.filename,
':%s/.barman-recover.info' % dest)
except CommandFailedException as e:
output.error(
'copy of recovery metadata file failed: %s', e)
output.close_and_exit()
else:
backup_info.save(os.path.join(dest, '.barman-recover.info'))
# Restore the WAL segments. If GET_WAL option is set, skip this phase
# as they will be retrieved using the wal-get command.
if not recovery_info['get_wal']:
output.info("Copying required WAL segments.")
try:
# Retrieve a list of required log files
required_xlog_files = tuple(
self.server.get_required_xlog_files(
backup_info, target_tli,
recovery_info['target_epoch']))
# Restore WAL segments into the wal_dest directory
self._xlog_copy(required_xlog_files,
recovery_info['wal_dest'],
remote_command)
except DataTransferFailure as e:
output.error("Failure copying WAL files: %s", e)
output.close_and_exit()
except BadXlogSegmentName as e:
output.error(
"invalid xlog segment name %r\n"
"HINT: Please run \"barman rebuild-xlogdb %s\" "
"to solve this issue" %
str(e), self.config.name)
output.close_and_exit()
# If WAL files are put directly in the pg_xlog directory,
# avoid shipping of just recovered files
# by creating the corresponding archive status file
if not recovery_info['is_pitr']:
output.info("Generating archive status files")
self._generate_archive_status(recovery_info,
remote_command,
required_xlog_files)
# Generate recovery.conf file (only if needed by PITR)
if recovery_info['is_pitr']:
output.info("Generating recovery.conf")
self._generate_recovery_conf(recovery_info, backup_info, dest,
exclusive, remote_command,
target_name, target_time,
target_tli, target_xid)
# Create archive_status directory if necessary
archive_status_dir = os.path.join(dest, 'pg_xlog', 'archive_status')
try:
recovery_info['cmd'].create_dir_if_not_exists(archive_status_dir)
except FsOperationFailed as e:
output.error("unable to create the archive_status directory "
"'%s': %s", archive_status_dir, e)
output.close_and_exit()
# As last step, analyse configuration files in order to spot
# harmful options. Barman performs automatic conversion of
# some options as well as notifying users of their existence.
#
# This operation is performed in three steps:
# 1) mapping
# 2) analysis
# 3) copy
output.info("Identify dangerous settings in destination directory.")
self._map_temporary_config_files(recovery_info,
backup_info,
remote_command)
self._analyse_temporary_config_files(recovery_info)
self._copy_temporary_config_files(dest,
remote_command,
recovery_info)
# Cleanup operations
self._teardown(recovery_info)
return recovery_info
def main():
"""
The main method of Barman
"""
p = ArghParser(epilog="Barman by EnterpriseDB (www.enterprisedb.com)")
p.add_argument(
"-v",
"--version",
action="version",
version="%s\n\nBarman by EnterpriseDB (www.enterprisedb.com)" %
barman.__version__,
)
p.add_argument(
"-c",
"--config",
help="uses a configuration file "
"(defaults: %s)" % ", ".join(barman.config.Config.CONFIG_FILES),
default=SUPPRESS,
)
p.add_argument(
"--color",
"--colour",
help="Whether to use colors in the output",
choices=["never", "always", "auto"],
default="auto",
)
p.add_argument(
"--log-level",
help="Override the default log level",
choices=list(get_log_levels()),
default=SUPPRESS,
)
p.add_argument("-q", "--quiet", help="be quiet", action="store_true")
p.add_argument("-d", "--debug", help="debug output", action="store_true")
p.add_argument(
"-f",
"--format",
help="output format",
choices=output.AVAILABLE_WRITERS.keys(),
default=output.DEFAULT_WRITER,
)
p.add_commands([
archive_wal,
backup,
check,
check_backup,
cron,
delete,
diagnose,
get_wal,
keep,
list_backup,
list_backups,
list_files,
list_server,
list_servers,
put_wal,
rebuild_xlogdb,
receive_wal,
recover,
show_backup,
show_server,
show_servers,
replication_status,
status,
switch_wal,
switch_xlog,
sync_info,
sync_backup,
sync_wals,
])
# noinspection PyBroadException
try:
p.dispatch(pre_call=global_config)
except KeyboardInterrupt:
msg = "Process interrupted by user (KeyboardInterrupt)"
output.error(msg)
except Exception as e:
msg = "%s\nSee log file for more details." % e
output.exception(msg)
# cleanup output API and exit honoring output.error_occurred and
# output.error_exit_code
output.close_and_exit()
def _generate_recovery_conf(self, recovery_info, backup_info, dest,
exclusive, remote_command, target_name,
target_time, target_tli, target_xid):
"""
Generate a recovery.conf file for PITR containing
all the required configurations
:param dict recovery_info: Dictionary containing all the recovery
parameters
:param barman.infofile.BackupInfo backup_info: representation of a
backup
:param str dest: destination directory of the recovery
:param boolean exclusive: exclusive backup or concurrent
:param str remote_command: ssh command for remote connection
:param str target_name: recovery target name for PITR
:param str target_time: recovery target time for PITR
:param str target_tli: recovery target timeline for PITR
:param str target_xid: recovery target transaction id for PITR
"""
if remote_command:
recovery = open(os.path.join(recovery_info['tempdir'],
'recovery.conf'), 'w')
else:
recovery = open(os.path.join(dest, 'recovery.conf'), 'w')
# If GET_WAL has been set, use the get-wal command to retrieve the
# required wal files. Otherwise use the unix command "cp" to copy
# them from the barman_xlog directory
if recovery_info['get_wal']:
# We need to create the right restore command.
# If we are doing a remote recovery,
# the barman-cli package is REQUIRED on the server that is hosting
# the PostgreSQL server.
# We use the machine FQDN and the barman_user
# setting to call the barman-wal-restore correctly.
# If local recovery, we use barman directly, assuming
# the postgres process will be executed with the barman user.
# It MUST to be reviewed by the user in any case.
if remote_command:
fqdn = socket.getfqdn()
print("# The 'barman-wal-restore' command "
"is provided in the 'barman-cli' package",
file=recovery)
print("restore_command = 'barman-wal-restore -U %s "
"%s %s %%f %%p'" % (self.config.config.user,
fqdn, self.config.name),
file=recovery)
else:
print("# The 'barman get-wal' command "
"must run as '%s' user" % self.config.config.user,
file=recovery)
print("restore_command = 'sudo -u %s "
"barman get-wal %s %%f > %%p'" % (
self.config.config.user,
self.config.name),
file=recovery)
recovery_info['results']['get_wal'] = True
else:
print("restore_command = 'cp barman_xlog/%f %p'", file=recovery)
if backup_info.version >= 80400 and \
not recovery_info['get_wal']:
print("recovery_end_command = 'rm -fr barman_xlog'", file=recovery)
if target_time:
print("recovery_target_time = '%s'" % target_time, file=recovery)
if target_tli:
print("recovery_target_timeline = %s" % target_tli, file=recovery)
if target_xid:
print("recovery_target_xid = '%s'" % target_xid, file=recovery)
if target_name:
print("recovery_target_name = '%s'" % target_name, file=recovery)
if (target_xid or target_time) and exclusive:
print("recovery_target_inclusive = '%s'" % (
not exclusive), file=recovery)
recovery.close()
if remote_command:
plain_rsync = RsyncPgData(
path=self.server.path,
ssh=remote_command,
bwlimit=self.config.bandwidth_limit,
network_compression=self.config.network_compression)
try:
plain_rsync.from_file_list(['recovery.conf'],
recovery_info['tempdir'],
':%s' % dest)
except CommandFailedException as e:
output.error('remote copy of recovery.conf failed: %s', e)
output.close_and_exit()
