def test_exception_with_raise_class(self, caplog):
# preparation
writer = self._mock_writer()
msg = 'test format %02d %s'
args = (1, '2nd')
try:
raise ValueError('test exception')
except ValueError:
with pytest.raises(KeyError):
output.exception(msg, raise_exception=KeyError, *args)
assert msg % args in caplog.text
assert 'Traceback' in caplog.text
# logging test
for record in caplog.records:
assert record.levelname == 'ERROR'
assert record.name == __name__
# writer test
writer.error_occurred.assert_called_once_with()
writer.exception.assert_called_once_with(msg, *args)
# global status test
assert output.error_occurred
def test_exception_with_raise_object(self, caplog):
# preparation
writer = self._mock_writer()
msg = "test format %02d %s"
args = (1, "2nd")
try:
raise ValueError("test exception")
except ValueError:
with pytest.raises(KeyError):
output.exception(msg, raise_exception=KeyError(), *args)
# logging test
for record in caplog.records:
assert record.levelname == "ERROR"
assert record.name == __name__
assert msg % args in caplog.text
assert "Traceback" in caplog.text
# writer test
writer.error_occurred.assert_called_once_with()
writer.exception.assert_called_once_with(msg, *args)
# global status test
assert output.error_occurred
def test_exception_with_raise_class(self, caplog):
# preparation
writer = self._mock_writer()
msg = 'test format %02d %s'
args = (1, '2nd')
try:
raise ValueError('test exception')
except ValueError:
with pytest.raises(KeyError):
output.exception(msg, raise_exception=KeyError, *args)
assert msg % args in caplog.text()
assert 'Traceback' in caplog.text()
# logging test
for record in caplog.records():
assert record.levelname == 'ERROR'
assert record.name == __name__
# writer test
writer.error_occurred.assert_called_once_with()
writer.exception.assert_called_once_with(msg, *args)
# global status test
assert output.error_occurred
def copy_temporary_config_files(self, dest, remote_command, recovery_info):
"""
Copy modified configuration files using rsync in case of
remote recovery
:param str dest: destination directory of the recovery
:param str remote_command: ssh command for remote connection
:param dict recovery_info: Dictionary containing all the recovery
parameters
"""
if remote_command:
# If this is a remote recovery, rsync the modified files from the
# temporary local directory to the remote destination directory.
file_list = []
for conf_file in recovery_info['configuration_files']:
file_list.append('%s' % conf_file)
file_list.append('%s.origin' % conf_file)
try:
recovery_info['rsync'].from_file_list(file_list,
recovery_info['tempdir'],
':%s' % dest)
except CommandFailedException as e:
output.exception(
'remote copy of configuration files failed: %s', e)
output.close_and_exit()
def main():
"""
The main method of Barman
"""
p = ArghParser(epilog='Barman by 2ndQuadrant (www.2ndQuadrant.com)')
p.add_argument(
'-v',
'--version',
action='version',
version='%s\n\nBarman by 2ndQuadrant (www.2ndQuadrant.com)' %
barman.__version__)
p.add_argument('-c',
'--config',
help='uses a configuration file '
'(defaults: %s)' %
', '.join(barman.config.Config.CONFIG_FILES),
default=SUPPRESS)
p.add_argument('-q', '--quiet', help='be quiet', action='store_true')
p.add_argument('-d', '--debug', help='debug output', action='store_true')
p.add_argument('-f',
'--format',
help='output format',
choices=output.AVAILABLE_WRITERS.keys(),
default=output.DEFAULT_WRITER)
p.add_commands([
archive_wal,
backup,
check,
check_backup,
cron,
delete,
diagnose,
get_wal,
list_backup,
list_files,
list_server,
put_wal,
rebuild_xlogdb,
receive_wal,
recover,
show_backup,
show_server,
replication_status,
status,
switch_wal,
switch_xlog,
])
# noinspection PyBroadException
try:
p.dispatch(pre_call=global_config)
except KeyboardInterrupt:
msg = "Process interrupted by user (KeyboardInterrupt)"
output.error(msg)
except Exception as e:
msg = "%s\nSee log file for more details." % e
output.exception(msg)
# cleanup output API and exit honoring output.error_occurred and
# output.error_exit_code
output.close_and_exit()
def generate_archive_status(self, recovery_info, remote_command,
required_xlog_files):
"""
Populate the archive_status directory
:param dict recovery_info: Dictionary containing all the recovery
parameters
:param str remote_command: ssh command for remote connection
:param tuple required_xlog_files: list of required WAL segments
"""
if remote_command:
status_dir = recovery_info['tempdir']
else:
status_dir = os.path.join(recovery_info['wal_dest'],
'archive_status')
mkpath(status_dir)
for wal_info in required_xlog_files:
with open(os.path.join(status_dir, "%s.done" % wal_info.name),
'a') as f:
f.write('')
if remote_command:
try:
recovery_info['rsync'](
'%s/' % status_dir, ':%s' %
os.path.join(recovery_info['wal_dest'], 'archive_status'))
except CommandFailedException as e:
output.exception(
"unable to populate pg_xlog/archive_status"
"directory: %s", e)
output.close_and_exit()
def generate_archive_status(self, recovery_info, remote_command,
required_xlog_files):
"""
Populate the archive_status directory
:param dict recovery_info: Dictionary containing all the recovery
parameters
:param str remote_command: ssh command for remote connection
:param tuple required_xlog_files: list of required WAL segments
"""
if remote_command:
status_dir = recovery_info['tempdir']
else:
status_dir = os.path.join(recovery_info['wal_dest'],
'archive_status')
mkpath(status_dir)
for wal_info in required_xlog_files:
with open(os.path.join(status_dir, "%s.done" % wal_info.name),
'a') as f:
f.write('')
if remote_command:
try:
recovery_info['rsync']('%s/' % status_dir,
':%s' % os.path.join(
recovery_info['wal_dest'],
'archive_status'))
except CommandFailedException as e:
output.exception(
"unable to populate pg_xlog/archive_status"
"directory: %s", e)
output.close_and_exit()
def cron():
"""
Run maintenance tasks (global command)
"""
# Skip inactive and temporarily disabled servers
servers = get_server_list(skip_inactive=True, skip_disabled=True)
for name in sorted(servers):
server = servers[name]
# Exception: manage_server_command is not invoked here
# Normally you would call manage_server_command to check if the
# server is None and to report inactive and disabled servers,
# but here we have only active and well configured servers.
try:
server.cron()
except Exception:
# A cron should never raise an exception, so this code
# should never be executed. However, it is here to protect
# unrelated servers in case of unexpected failures.
output.exception(
"Unable to run cron on server '%s', "
"please look in the barman log file for more details.", name)
output.close_and_exit()
def copy_temporary_config_files(self, dest, remote_command, recovery_info):
"""
Copy modified configuration files using rsync in case of
remote recovery
:param str dest: destination directory of the recovery
:param str remote_command: ssh command for remote connection
:param dict recovery_info: Dictionary containing all the recovery
parameters
"""
if remote_command:
# If this is a remote recovery, rsync the modified files from the
# temporary local directory to the remote destination directory.
file_list = []
for conf_file in recovery_info['configuration_files']:
file_list.append('%s' % conf_file)
file_list.append('%s.origin' % conf_file)
try:
recovery_info['rsync'].from_file_list(file_list,
recovery_info['tempdir'],
':%s' % dest)
except CommandFailedException as e:
output.exception(
'remote copy of configuration files failed: %s', e)
output.close_and_exit()
def main():
"""
The main method of Barman
"""
p = ArghParser(epilog='Barman by 2ndQuadrant (www.2ndQuadrant.com)')
p.add_argument('-v', '--version', action='version',
version='%s\n\nBarman by 2ndQuadrant (www.2ndQuadrant.com)'
% barman.__version__)
p.add_argument('-c', '--config',
help='uses a configuration file '
'(defaults: %s)'
% ', '.join(barman.config.Config.CONFIG_FILES),
default=SUPPRESS)
p.add_argument('-q', '--quiet', help='be quiet', action='store_true')
p.add_argument('-d', '--debug', help='debug output', action='store_true')
p.add_argument('-f', '--format', help='output format',
choices=output.AVAILABLE_WRITERS.keys(),
default=output.DEFAULT_WRITER)
p.add_commands(
[
archive_wal,
backup,
check,
cron,
delete,
diagnose,
get_wal,
list_backup,
list_files,
list_server,
rebuild_xlogdb,
receive_wal,
recover,
show_backup,
show_server,
replication_status,
status,
switch_xlog,
]
)
# noinspection PyBroadException
try:
p.dispatch(pre_call=global_config)
except KeyboardInterrupt:
msg = "Process interrupted by user (KeyboardInterrupt)"
output.error(msg)
except Exception as e:
msg = "%s\nSee log file for more details." % e
output.exception(msg)
# cleanup output API and exit honoring output.error_occurred and
# output.error_exit_code
output.close_and_exit()
def prepare_tablespaces(self, backup_info, cmd, dest, tablespaces):
"""
Prepare the directory structure for required tablespaces,
taking care of tablespaces relocation, if requested.
:param barman.infofile.BackupInfo backup_info: backup representation
:param barman.fs.UnixLocalCommand cmd: Object for
filesystem interaction
:param str dest: destination dir for the recovery
:param dict tablespaces: dict of all the tablespaces and their location
"""
tblspc_dir = os.path.join(dest, 'pg_tblspc')
try:
# check for pg_tblspc dir into recovery destination folder.
# if it does not exists, create it
cmd.create_dir_if_not_exists(tblspc_dir)
except FsOperationFailed as e:
output.exception(
"unable to initialise tablespace directory "
"'%s': %s", tblspc_dir, e)
output.close_and_exit()
for item in backup_info.tablespaces:
# build the filename of the link under pg_tblspc directory
pg_tblspc_file = os.path.join(tblspc_dir, str(item.oid))
# by default a tablespace goes in the same location where
# it was on the source server when the backup was taken
location = item.location
# if a relocation has been requested for this tablespace,
# use the target directory provided by the user
if tablespaces and item.name in tablespaces:
location = tablespaces[item.name]
try:
# remove the current link in pg_tblspc, if it exists
# (raise an exception if it is a directory)
cmd.delete_if_exists(pg_tblspc_file)
# create tablespace location, if does not exist
# (raise an exception if it is not possible)
cmd.create_dir_if_not_exists(location)
# check for write permissions on destination directory
cmd.check_write_permission(location)
# create symlink between tablespace and recovery folder
cmd.create_symbolic_link(location, pg_tblspc_file)
except FsOperationFailed as e:
output.exception(
"unable to prepare '%s' tablespace "
"(destination '%s'): %s", item.name, location, e)
output.close_and_exit()
output.info("\t%s, %s, %s", item.oid, item.name, location)
def generate_recovery_conf(self, recovery_info, backup_info, dest,
exclusive, remote_command, target_name,
target_time, target_tli, target_xid):
"""
Generate a recovery.conf file for PITR containing
all the required configurations
:param dict recovery_info: Dictionary containing all the recovery params
:param barman.infofile.BackupInfo backup_info: representation of a
backup
:param str dest: destination directory of the recovery
:param boolean exclusive: exclusive backup or concurrent
:param str remote_command: ssh command for remote connection
:param str target_name: recovery target name for PITR
:param str target_time: recovery target time for PITR
:param str target_tli: recovery target timeline for PITR
:param str target_xid: recovery target transaction id for PITR
"""
if remote_command:
recovery = open(os.path.join(recovery_info['tempdir'],
'recovery.conf'), 'w')
else:
recovery = open(os.path.join(dest, 'recovery.conf'), 'w')
print >> recovery, "restore_command = 'cp barman_xlog/%f %p'"
if backup_info.version >= 80400:
print >> recovery, "recovery_end_command = 'rm -fr barman_xlog'"
if target_time:
print >> recovery, "recovery_target_time = '%s'" % target_time
if target_tli:
print >> recovery, "recovery_target_timeline = %s" % target_tli
if target_xid:
print >> recovery, "recovery_target_xid = '%s'" % target_xid
if target_name:
print >> recovery, "recovery_target_name = '%s'" % target_name
if (target_xid or target_time) and exclusive:
print >> recovery, "recovery_target_inclusive = '%s'" % (
not exclusive)
recovery.close()
if remote_command:
# Uses plain rsync (without exclusions) to ship recovery.conf
plain_rsync = Rsync(
ssh=remote_command,
bwlimit=self.config.bandwidth_limit,
network_compression=self.config.network_compression)
try:
plain_rsync.from_file_list(['recovery.conf'],
recovery_info['tempdir'],
':%s' % dest)
except CommandFailedException, e:
output.exception(
'remote copy of recovery.conf failed: %s', e)
output.close_and_exit()
def prepare_tablespaces(self, backup_info, cmd, dest, tablespaces):
"""
Prepare the directory structure for required tablespaces,
taking care of tablespaces relocation, if requested.
:param barman.infofile.BackupInfo backup_info: backup representation
:param barman.fs.UnixLocalCommand cmd: Object for
filesystem interaction
:param str dest: destination dir for the recovery
:param dict tablespaces: dict of all the tablespaces and their location
"""
tblspc_dir = os.path.join(dest, 'pg_tblspc')
try:
# check for pg_tblspc dir into recovery destination folder.
# if it does not exists, create it
cmd.create_dir_if_not_exists(tblspc_dir)
except FsOperationFailed as e:
output.exception("unable to initialise tablespace directory "
"'%s': %s", tblspc_dir, e)
output.close_and_exit()
for item in backup_info.tablespaces:
# build the filename of the link under pg_tblspc directory
pg_tblspc_file = os.path.join(tblspc_dir, str(item.oid))
# by default a tablespace goes in the same location where
# it was on the source server when the backup was taken
location = item.location
# if a relocation has been requested for this tablespace,
# use the target directory provided by the user
if tablespaces and item.name in tablespaces:
location = tablespaces[item.name]
try:
# remove the current link in pg_tblspc, if it exists
# (raise an exception if it is a directory)
cmd.delete_if_exists(pg_tblspc_file)
# create tablespace location, if does not exist
# (raise an exception if it is not possible)
cmd.create_dir_if_not_exists(location)
# check for write permissions on destination directory
cmd.check_write_permission(location)
# create symlink between tablespace and recovery folder
cmd.create_symbolic_link(location, pg_tblspc_file)
except FsOperationFailed as e:
output.exception("unable to prepare '%s' tablespace "
"(destination '%s'): %s",
item.name, location, e)
output.close_and_exit()
output.info("\t%s, %s, %s", item.oid, item.name, location)
def main():
"""
The main method of Barman
"""
p = ArghParser()
p.add_argument("-v", "--version", action="version", version=barman.__version__)
p.add_argument(
"-c",
"--config",
help="uses a configuration file " "(defaults: %s)" % ", ".join(barman.config.Config.CONFIG_FILES),
default=SUPPRESS,
)
p.add_argument("-q", "--quiet", help="be quiet", action="store_true")
p.add_argument("-d", "--debug", help="debug output", action="store_true")
p.add_argument(
"-f", "--format", help="output format", choices=output.AVAILABLE_WRITERS.keys(), default=output.DEFAULT_WRITER
)
p.add_commands(
[
archive_wal,
backup,
check,
cron,
delete,
diagnose,
get_wal,
list_backup,
list_files,
list_server,
rebuild_xlogdb,
receive_wal,
recover,
show_backup,
show_server,
status,
]
)
# noinspection PyBroadException
try:
p.dispatch(pre_call=global_config)
except KeyboardInterrupt:
msg = "Process interrupted by user (KeyboardInterrupt)"
output.exception(msg)
except Exception, e:
msg = "%s\nSee log file for more details." % e
output.exception(msg)
def recover(self, backup_info, dest, tablespaces, target_tli,
target_time, target_xid, target_name,
exclusive, remote_command):
"""
Performs a recovery of a backup
:param barman.infofile.BackupInfo backup_info: the backup to recover
:param str dest: the destination directory
:param dict[str,str]|None tablespaces: a tablespace name -> location map
(for relocation)
:param str|None target_tli: the target timeline
:param str|None target_time: the target time
:param str|None target_xid: the target xid
:param str|None target_name: the target name created previously with
pg_create_restore_point() function call
:param bool exclusive: whether the recovery is exclusive or not
:param str|None remote_command: The remote command to recover
the base backup, in case of remote backup.
"""
# Run the cron to be sure the wal catalog is up to date
# Prepare a map that contains all the objects required for a recovery
recovery_info = self.setup(backup_info, remote_command, dest)
output.info("Starting %s restore for server %s using backup %s",
recovery_info['recovery_dest'], self.server.config.name,
backup_info.backup_id)
output.info("Destination directory: %s", dest)
# Set targets for PITR
self.set_pitr_targets(recovery_info,
backup_info, dest,
target_name,
target_time,
target_tli,
target_xid)
# Retrieve the safe_horizon for smart copy
self.retrieve_safe_horizon(recovery_info, backup_info, dest)
# check destination directory. If doesn't exist create it
try:
recovery_info['cmd'].create_dir_if_not_exists(dest)
except FsOperationFailed, e:
output.exception("unable to initialise destination directory "
"'%s': %s", dest, e)
output.close_and_exit()
def main():
"""
The main method of Barman
"""
p = ArghParser()
p.add_argument('-v', '--version', action='version',
version=barman.__version__)
p.add_argument('-c', '--config',
help='uses a configuration file '
'(defaults: %s)'
% ', '.join(barman.config.Config.CONFIG_FILES),
default=SUPPRESS)
p.add_argument('-q', '--quiet', help='be quiet', action='store_true')
p.add_argument('-d', '--debug', help='debug output', action='store_true')
p.add_argument('-f', '--format', help='output format',
choices=output.AVAILABLE_WRITERS.keys(),
default=output.DEFAULT_WRITER)
p.add_commands(
[
archive_wal,
cron,
list_server,
show_server,
status,
check,
diagnose,
backup,
list_backup,
show_backup,
list_files,
get_wal,
recover,
delete,
rebuild_xlogdb,
]
)
# noinspection PyBroadException
try:
p.dispatch(pre_call=global_config)
except KeyboardInterrupt:
msg = "Process interrupted by user (KeyboardInterrupt)"
output.exception(msg)
except Exception, e:
msg = "%s\nSee log file for more details." % e
output.exception(msg)
def prepare_tablespaces(self, backup_info, cmd, dest, tablespaces):
"""
Prepare the directory structure for required tablespaces, taking care of
tablespaces relocation, if requested.
:param barman.infofile.BackupInfo backup_info: backup representation
:param barman.fs.UnixLocalCommand cmd: Object for filesystem interaction
:param str dest: destination dir for the recovery
:param dict tablespaces: dict of all the tablespaces and their location
"""
tblspc_dir = os.path.join(dest, 'pg_tblspc')
try:
# check for pg_tblspc dir into recovery destination folder.
# if it does not exists, create it
cmd.create_dir_if_not_exists(tblspc_dir)
except FsOperationFailed, e:
output.exception("unable to initialise tablespace directory "
"'%s': %s", tblspc_dir, e)
output.close_and_exit()
def test_exception(self, caplog):
# preparation
writer = self._mock_writer()
msg = 'test message'
try:
raise ValueError('test exception')
except ValueError:
output.exception(msg)
# logging test
for record in caplog.records():
assert record.levelname == 'ERROR'
assert record.name == __name__
assert msg in caplog.text()
assert 'Traceback' in caplog.text()
# writer test
writer.error_occurred.assert_called_once_with()
writer.exception.assert_called_once_with(msg)
# global status test
assert output.error_occurred
def test_exception(self, caplog):
# preparation
writer = self._mock_writer()
msg = 'test message'
try:
raise ValueError('test exception')
except ValueError:
output.exception(msg)
# logging test
for record in caplog.records:
assert record.levelname == 'ERROR'
assert record.name == __name__
assert msg in caplog.text
assert 'Traceback' in caplog.text
# writer test
writer.error_occurred.assert_called_once_with()
writer.exception.assert_called_once_with(msg)
# global status test
assert output.error_occurred
def test_exception(self, caplog):
# preparation
writer = self._mock_writer()
msg = "test message"
try:
raise ValueError("test exception")
except ValueError:
output.exception(msg)
# logging test
for record in caplog.records:
assert record.levelname == "ERROR"
assert record.name == __name__
assert msg in caplog.text
assert "Traceback" in caplog.text
# writer test
writer.error_occurred.assert_called_once_with()
writer.exception.assert_called_once_with(msg)
# global status test
assert output.error_occurred
def test_exception_with_ignore(self, caplog):
# preparation
writer = self._mock_writer()
msg = 'test format %02d %s'
args = (1, '2nd')
try:
raise ValueError('test exception')
except ValueError:
output.exception(msg, ignore=True, *args)
# logging test
for record in caplog.records():
assert record.levelname == 'ERROR'
assert record.name == __name__
assert msg % args in caplog.text()
assert 'Traceback' in caplog.text()
# writer test
assert not writer.error_occurred.called
writer.exception.assert_called_once_with(msg, *args)
# global status test
assert not output.error_occurred
def test_exception_with_ignore(self, caplog):
# preparation
writer = self._mock_writer()
msg = 'test format %02d %s'
args = (1, '2nd')
try:
raise ValueError('test exception')
except ValueError:
output.exception(msg, ignore=True, *args)
# logging test
for record in caplog.records:
assert record.levelname == 'ERROR'
assert record.name == __name__
assert msg % args in caplog.text
assert 'Traceback' in caplog.text
# writer test
assert not writer.error_occurred.called
writer.exception.assert_called_once_with(msg, *args)
# global status test
assert not output.error_occurred
def test_exception_with_ignore(self, caplog):
# preparation
writer = self._mock_writer()
msg = "test format %02d %s"
args = (1, "2nd")
try:
raise ValueError("test exception")
except ValueError:
output.exception(msg, ignore=True, *args)
# logging test
for record in caplog.records:
assert record.levelname == "ERROR"
assert record.name == __name__
assert msg % args in caplog.text
assert "Traceback" in caplog.text
# writer test
assert not writer.error_occurred.called
writer.exception.assert_called_once_with(msg, *args)
# global status test
assert not output.error_occurred
def recover(self, backup_info, dest, tablespaces, target_tli,
target_time, target_xid, target_name,
exclusive, remote_command):
"""
Performs a recovery of a backup
:param barman.infofile.BackupInfo backup_info: the backup to recover
:param str dest: the destination directory
:param dict[str,str]|None tablespaces: a tablespace
name -> location map (for relocation)
:param str|None target_tli: the target timeline
:param str|None target_time: the target time
:param str|None target_xid: the target xid
:param str|None target_name: the target name created previously with
pg_create_restore_point() function call
:param bool exclusive: whether the recovery is exclusive or not
:param str|None remote_command: The remote command to recover
the base backup, in case of remote backup.
"""
# Run the cron to be sure the wal catalog is up to date
# Prepare a map that contains all the objects required for a recovery
recovery_info = self.setup(backup_info, remote_command, dest)
output.info("Starting %s restore for server %s using backup %s",
recovery_info['recovery_dest'], self.server.config.name,
backup_info.backup_id)
output.info("Destination directory: %s", dest)
# Set targets for PITR
self.set_pitr_targets(recovery_info,
backup_info, dest,
target_name,
target_time,
target_tli,
target_xid)
# Retrieve the safe_horizon for smart copy
self.retrieve_safe_horizon(recovery_info, backup_info, dest)
# check destination directory. If doesn't exist create it
try:
recovery_info['cmd'].create_dir_if_not_exists(dest)
except FsOperationFailed as e:
output.exception("unable to initialise destination directory "
"'%s': %s", dest, e)
output.close_and_exit()
# Initialize tablespace directories
if backup_info.tablespaces:
self.prepare_tablespaces(backup_info,
recovery_info['cmd'],
dest,
tablespaces)
# Copy the base backup
output.info("Copying the base backup.")
try:
# perform the backup copy, honoring the retry option if set
self.backup_manager.retry_backup_copy(
self.basebackup_copy,
backup_info, dest,
tablespaces, remote_command,
recovery_info['safe_horizon'])
except DataTransferFailure as e:
output.exception("Failure copying base backup: %s", e)
output.close_and_exit()
# Copy the backup.info file in the destination as
# ".barman-recover.info"
if remote_command:
try:
recovery_info['rsync'](backup_info.filename,
':%s/.barman-recover.info' % dest)
except CommandFailedException as e:
output.exception(
'copy of recovery metadata file failed: %s', e)
output.close_and_exit()
else:
backup_info.save(os.path.join(dest, '.barman-recover.info'))
# Restore the WAL segments. If GET_WAL option is set, skip this phase
# as they will be retrieved using the wal-get command.
if not recovery_info['get_wal']:
output.info("Copying required WAL segments.")
try:
# Retrieve a list of required log files
required_xlog_files = tuple(
self.server.get_required_xlog_files(
backup_info, target_tli,
recovery_info['target_epoch']))
# Restore WAL segments into the wal_dest directory
self.xlog_copy(required_xlog_files,
recovery_info['wal_dest'],
remote_command)
except DataTransferFailure as e:
output.exception("Failure copying WAL files: %s", e)
output.close_and_exit()
except xlog.BadXlogSegmentName as e:
output.error(
"invalid xlog segment name %r\n"
"HINT: Please run \"barman rebuild-xlogdb %s\" "
"to solve this issue" %
str(e), self.config.name)
output.close_and_exit()
# If WAL files are put directly in the pg_xlog directory,
# avoid shipping of just recovered files
# by creating the corresponding archive status file
if not recovery_info['is_pitr']:
output.info("Generating archive status files")
self.generate_archive_status(recovery_info,
remote_command,
required_xlog_files)
# Generate recovery.conf file (only if needed by PITR)
if recovery_info['is_pitr']:
output.info("Generating recovery.conf")
self.generate_recovery_conf(recovery_info, backup_info, dest,
exclusive, remote_command, target_name,
target_time, target_tli, target_xid)
# Create archive_status directory if necessary
archive_status_dir = os.path.join(dest, 'pg_xlog', 'archive_status')
try:
recovery_info['cmd'].create_dir_if_not_exists(archive_status_dir)
except FsOperationFailed as e:
output.exception("unable to create the archive_status directory "
"'%s': %s", archive_status_dir, e)
output.close_and_exit()
# As last step, analyse configuration files in order to spot
# harmful options. Barman performs automatic conversion of
# some options as well as notifying users of their existence.
#
# This operation is performed in three steps:
# 1) mapping
# 2) analysis
# 3) copy
output.info("Identify dangerous settings in destination directory.")
self.map_temporary_config_files(recovery_info,
backup_info,
remote_command)
self.analyse_temporary_config_files(recovery_info)
self.copy_temporary_config_files(dest,
remote_command,
recovery_info)
# Cleanup operations
self.teardown(recovery_info)
return recovery_info
def set_pitr_targets(self, recovery_info, backup_info, dest, target_name,
target_time, target_tli, target_xid):
"""
Set PITR targets - as specified by the user
:param dict recovery_info: Dictionary containing all the recovery
parameters
:param barman.infofile.BackupInfo backup_info: representation of a
backup
:param str dest: destination directory of the recovery
:param str|None target_name: recovery target name for PITR
:param str|None target_time: recovery target time for PITR
:param str|None target_tli: recovery target timeline for PITR
:param str|None target_xid: recovery target transaction id for PITR
"""
target_epoch = None
target_datetime = None
if (target_time or
target_xid or
(target_tli and target_tli != backup_info.timeline) or
target_name or
recovery_info['get_wal']):
recovery_info['is_pitr'] = True
targets = {}
if target_time:
# noinspection PyBroadException
try:
target_datetime = dateutil.parser.parse(target_time)
except ValueError as e:
output.exception(
"unable to parse the target time parameter %r: %s",
target_time, e)
output.close_and_exit()
except Exception:
# this should not happen, but there is a known bug in
# dateutil.parser.parse() implementation
# ref: https://bugs.launchpad.net/dateutil/+bug/1247643
output.exception(
"unable to parse the target time parameter %r",
target_time)
output.close_and_exit()
target_epoch = (
time.mktime(target_datetime.timetuple()) +
(target_datetime.microsecond / 1000000.))
targets['time'] = str(target_datetime)
if target_xid:
targets['xid'] = str(target_xid)
if target_tli and target_tli != backup_info.timeline:
targets['timeline'] = str(target_tli)
if target_name:
targets['name'] = str(target_name)
output.info(
"Doing PITR. Recovery target %s",
(", ".join(["%s: %r" % (k, v) for k, v in targets.items()])))
recovery_info['wal_dest'] = os.path.join(dest, 'barman_xlog')
# With a PostgreSQL version older than 8.4, it is the user's
# responsibility to delete the "barman_xlog" directory as the
# restore_command option in recovery.conf is not supported
if backup_info.version < 80400 and \
not recovery_info['get_wal']:
recovery_info['results']['delete_barman_xlog'] = True
recovery_info['target_epoch'] = target_epoch
recovery_info['target_datetime'] = target_datetime
def generate_recovery_conf(self, recovery_info, backup_info, dest,
exclusive, remote_command, target_name,
target_time, target_tli, target_xid):
"""
Generate a recovery.conf file for PITR containing
all the required configurations
:param dict recovery_info: Dictionary containing all the recovery
parameters
:param barman.infofile.BackupInfo backup_info: representation of a
backup
:param str dest: destination directory of the recovery
:param boolean exclusive: exclusive backup or concurrent
:param str remote_command: ssh command for remote connection
:param str target_name: recovery target name for PITR
:param str target_time: recovery target time for PITR
:param str target_tli: recovery target timeline for PITR
:param str target_xid: recovery target transaction id for PITR
"""
if remote_command:
recovery = open(
os.path.join(recovery_info['tempdir'], 'recovery.conf'), 'w')
else:
recovery = open(os.path.join(dest, 'recovery.conf'), 'w')
# If GET_WAL has been set, use the get-wal command to retrieve the
# required wal files. Otherwise use the unix command "cp" to copy
# them from the barman_xlog directory
if recovery_info['get_wal']:
# We need to guess the right way to execute the "barman"
# command on the Barman server.
# If remote recovery we use the machine FQDN and the barman_user
# setting to build an ssh command.
# If local recovery, we use barman directly, assuming
# the postgres process will be executed with the barman user.
# It has to be reviewed by the user in any case.
if remote_command:
fqdn = socket.getfqdn()
barman_command = 'ssh "%s@%s" barman' % (
self.config.config.user, fqdn)
else:
barman_command = 'barman'
print("restore_command = '%s get-wal %s %%f > %%p'" %
(barman_command, self.config.name),
file=recovery)
recovery_info['results']['get_wal'] = True
else:
print("restore_command = 'cp barman_xlog/%f %p'", file=recovery)
if backup_info.version >= 80400 and \
not recovery_info['get_wal']:
print("recovery_end_command = 'rm -fr barman_xlog'", file=recovery)
if target_time:
print("recovery_target_time = '%s'" % target_time, file=recovery)
if target_tli:
print("recovery_target_timeline = %s" % target_tli, file=recovery)
if target_xid:
print("recovery_target_xid = '%s'" % target_xid, file=recovery)
if target_name:
print("recovery_target_name = '%s'" % target_name, file=recovery)
if (target_xid or target_time) and exclusive:
print("recovery_target_inclusive = '%s'" % (not exclusive),
file=recovery)
recovery.close()
if remote_command:
# Uses plain rsync (without exclusions) to ship recovery.conf
plain_rsync = Rsync(
path=self.server.path,
ssh=remote_command,
bwlimit=self.config.bandwidth_limit,
network_compression=self.config.network_compression)
try:
plain_rsync.from_file_list(['recovery.conf'],
recovery_info['tempdir'],
':%s' % dest)
except CommandFailedException as e:
output.exception('remote copy of recovery.conf failed: %s', e)
output.close_and_exit()
def generate_recovery_conf(self, recovery_info, backup_info, dest,
exclusive, remote_command, target_name,
target_time, target_tli, target_xid):
"""
Generate a recovery.conf file for PITR containing
all the required configurations
:param dict recovery_info: Dictionary containing all the recovery
parameters
:param barman.infofile.BackupInfo backup_info: representation of a
backup
:param str dest: destination directory of the recovery
:param boolean exclusive: exclusive backup or concurrent
:param str remote_command: ssh command for remote connection
:param str target_name: recovery target name for PITR
:param str target_time: recovery target time for PITR
:param str target_tli: recovery target timeline for PITR
:param str target_xid: recovery target transaction id for PITR
"""
if remote_command:
recovery = open(os.path.join(recovery_info['tempdir'],
'recovery.conf'), 'w')
else:
recovery = open(os.path.join(dest, 'recovery.conf'), 'w')
# If GET_WAL has been set, use the get-wal command to retrieve the
# required wal files. Otherwise use the unix command "cp" to copy
# them from the barman_xlog directory
if recovery_info['get_wal']:
# We need to guess the right way to execute the "barman"
# command on the Barman server.
# If remote recovery we use the machine FQDN and the barman_user
# setting to build an ssh command.
# If local recovery, we use barman directly, assuming
# the postgres process will be executed with the barman user.
# It has to be reviewed by the user in any case.
if remote_command:
fqdn = socket.getfqdn()
barman_command = 'ssh "%s@%s" barman' % (
self.config.config.user, fqdn)
else:
barman_command = 'barman'
print("restore_command = '%s get-wal %s %%f > %%p'" % (
barman_command, self.config.name), file=recovery)
recovery_info['results']['get_wal'] = True
else:
print("restore_command = 'cp barman_xlog/%f %p'", file=recovery)
if backup_info.version >= 80400 and \
not recovery_info['get_wal']:
print("recovery_end_command = 'rm -fr barman_xlog'", file=recovery)
if target_time:
print("recovery_target_time = '%s'" % target_time, file=recovery)
if target_tli:
print("recovery_target_timeline = %s" % target_tli, file=recovery)
if target_xid:
print("recovery_target_xid = '%s'" % target_xid, file=recovery)
if target_name:
print("recovery_target_name = '%s'" % target_name, file=recovery)
if (target_xid or target_time) and exclusive:
print("recovery_target_inclusive = '%s'" % (
not exclusive), file=recovery)
recovery.close()
if remote_command:
# Uses plain rsync (without exclusions) to ship recovery.conf
plain_rsync = Rsync(
path=self.server.path,
ssh=remote_command,
bwlimit=self.config.bandwidth_limit,
network_compression=self.config.network_compression)
try:
plain_rsync.from_file_list(['recovery.conf'],
recovery_info['tempdir'],
':%s' % dest)
except CommandFailedException as e:
output.exception(
'remote copy of recovery.conf failed: %s', e)
output.close_and_exit()
def set_pitr_targets(self, recovery_info, backup_info, dest, target_name,
target_time, target_tli, target_xid):
"""
Set PITR targets - as specified by the user
:param dict recovery_info: Dictionary containing all the recovery
parameters
:param barman.infofile.BackupInfo backup_info: representation of a
backup
:param str dest: destination directory of the recovery
:param str|None target_name: recovery target name for PITR
:param str|None target_time: recovery target time for PITR
:param str|None target_tli: recovery target timeline for PITR
:param str|None target_xid: recovery target transaction id for PITR
"""
target_epoch = None
target_datetime = None
if (target_time or target_xid
or (target_tli and target_tli != backup_info.timeline)
or target_name or recovery_info['get_wal']):
recovery_info['is_pitr'] = True
targets = {}
if target_time:
# noinspection PyBroadException
try:
target_datetime = dateutil.parser.parse(target_time)
except ValueError as e:
output.exception(
"unable to parse the target time parameter %r: %s",
target_time, e)
self.teardown(recovery_info)
output.close_and_exit()
except Exception:
# this should not happen, but there is a known bug in
# dateutil.parser.parse() implementation
# ref: https://bugs.launchpad.net/dateutil/+bug/1247643
output.exception(
"unable to parse the target time parameter %r",
target_time)
output.close_and_exit()
target_epoch = (time.mktime(target_datetime.timetuple()) +
(target_datetime.microsecond / 1000000.))
targets['time'] = str(target_datetime)
if target_xid:
targets['xid'] = str(target_xid)
if target_tli and target_tli != backup_info.timeline:
targets['timeline'] = str(target_tli)
if target_name:
targets['name'] = str(target_name)
output.info(
"Doing PITR. Recovery target %s",
(", ".join(["%s: %r" % (k, v) for k, v in targets.items()])))
recovery_info['wal_dest'] = os.path.join(dest, 'barman_xlog')
# With a PostgreSQL version older than 8.4, it is the user's
# responsibility to delete the "barman_xlog" directory as the
# restore_command option in recovery.conf is not supported
if backup_info.version < 80400 and \
not recovery_info['get_wal']:
recovery_info['results']['delete_barman_xlog'] = True
recovery_info['target_epoch'] = target_epoch
recovery_info['target_datetime'] = target_datetime
def recover(self, backup_info, dest, tablespaces, target_tli, target_time,
target_xid, target_name, exclusive, remote_command):
"""
Performs a recovery of a backup
:param barman.infofile.BackupInfo backup_info: the backup to recover
:param str dest: the destination directory
:param dict[str,str]|None tablespaces: a tablespace
name -> location map (for relocation)
:param str|None target_tli: the target timeline
:param str|None target_time: the target time
:param str|None target_xid: the target xid
:param str|None target_name: the target name created previously with
pg_create_restore_point() function call
:param bool exclusive: whether the recovery is exclusive or not
:param str|None remote_command: The remote command to recover
the base backup, in case of remote backup.
"""
# Run the cron to be sure the wal catalog is up to date
# Prepare a map that contains all the objects required for a recovery
recovery_info = self.setup(backup_info, remote_command, dest)
output.info("Starting %s restore for server %s using backup %s",
recovery_info['recovery_dest'], self.server.config.name,
backup_info.backup_id)
output.info("Destination directory: %s", dest)
# Set targets for PITR
self.set_pitr_targets(recovery_info, backup_info, dest, target_name,
target_time, target_tli, target_xid)
# Retrieve the safe_horizon for smart copy
self.retrieve_safe_horizon(recovery_info, backup_info, dest)
# check destination directory. If doesn't exist create it
try:
recovery_info['cmd'].create_dir_if_not_exists(dest)
except FsOperationFailed as e:
output.exception(
"unable to initialise destination directory "
"'%s': %s", dest, e)
output.close_and_exit()
# Initialize tablespace directories
if backup_info.tablespaces:
self.prepare_tablespaces(backup_info, recovery_info['cmd'], dest,
tablespaces)
# Copy the base backup
output.info("Copying the base backup.")
try:
# perform the backup copy, honoring the retry option if set
self.backup_manager.retry_backup_copy(
self.basebackup_copy, backup_info, dest, tablespaces,
remote_command, recovery_info['safe_horizon'])
except DataTransferFailure as e:
output.exception("Failure copying base backup: %s", e)
output.close_and_exit()
# Copy the backup.info file in the destination as
# ".barman-recover.info"
if remote_command:
try:
recovery_info['rsync'](backup_info.filename,
':%s/.barman-recover.info' % dest)
except CommandFailedException as e:
output.exception('copy of recovery metadata file failed: %s',
e)
output.close_and_exit()
else:
backup_info.save(os.path.join(dest, '.barman-recover.info'))
# Restore the WAL segments. If GET_WAL option is set, skip this phase
# as they will be retrieved using the wal-get command.
if not recovery_info['get_wal']:
output.info("Copying required WAL segments.")
try:
# Retrieve a list of required log files
required_xlog_files = tuple(
self.server.get_required_xlog_files(
backup_info, target_tli,
recovery_info['target_epoch']))
# Restore WAL segments into the wal_dest directory
self.xlog_copy(required_xlog_files, recovery_info['wal_dest'],
remote_command)
except DataTransferFailure as e:
output.exception("Failure copying WAL files: %s", e)
output.close_and_exit()
except BadXlogSegmentName as e:
output.error(
"invalid xlog segment name %r\n"
"HINT: Please run \"barman rebuild-xlogdb %s\" "
"to solve this issue" % str(e), self.config.name)
output.close_and_exit()
# If WAL files are put directly in the pg_xlog directory,
# avoid shipping of just recovered files
# by creating the corresponding archive status file
if not recovery_info['is_pitr']:
output.info("Generating archive status files")
self.generate_archive_status(recovery_info, remote_command,
required_xlog_files)
# Generate recovery.conf file (only if needed by PITR)
if recovery_info['is_pitr']:
output.info("Generating recovery.conf")
self.generate_recovery_conf(recovery_info, backup_info, dest,
exclusive, remote_command, target_name,
target_time, target_tli, target_xid)
# Create archive_status directory if necessary
archive_status_dir = os.path.join(dest, 'pg_xlog', 'archive_status')
try:
recovery_info['cmd'].create_dir_if_not_exists(archive_status_dir)
except FsOperationFailed as e:
output.exception(
"unable to create the archive_status directory "
"'%s': %s", archive_status_dir, e)
output.close_and_exit()
# As last step, analyse configuration files in order to spot
# harmful options. Barman performs automatic conversion of
# some options as well as notifying users of their existence.
#
# This operation is performed in three steps:
# 1) mapping
# 2) analysis
# 3) copy
output.info("Identify dangerous settings in destination directory.")
self.map_temporary_config_files(recovery_info, backup_info,
remote_command)
self.analyse_temporary_config_files(recovery_info)
self.copy_temporary_config_files(dest, remote_command, recovery_info)
# Cleanup operations
self.teardown(recovery_info)
return recovery_info
location = tablespaces[item.name]
try:
# remove the current link in pg_tblspc, if it exists
# (raise an exception if it is a directory)
cmd.delete_if_exists(pg_tblspc_file)
# create tablespace location, if does not exist
# (raise an exception if it is not possible)
cmd.create_dir_if_not_exists(location)
# check for write permissions on destination directory
cmd.check_write_permission(location)
# create symlink between tablespace and recovery folder
cmd.create_symbolic_link(location, pg_tblspc_file)
except FsOperationFailed, e:
output.exception("unable to prepare '%s' tablespace "
"(destination '%s'): %s",
item.name, location, e)
output.close_and_exit()
output.info("\t%s, %s, %s", item.oid, item.name, location)
def retrieve_safe_horizon(self, recovery_info, backup_info, dest):
"""
Retrieve the safe_horizon for smart copy
If the target directory contains a previous recovery, it is safe to
pick the least of the two backup "begin times" (the one we are
recovering now and the one previously recovered in the target
directory). Set the value in the given recovery_info dictionary.
:param dict recovery_info: Dictionary containing all the recovery params
:param barman.infofile.BackupInfo backup_info: a backup representation
location = tablespaces[item.name]
try:
# remove the current link in pg_tblspc, if it exists
# (raise an exception if it is a directory)
cmd.delete_if_exists(pg_tblspc_file)
# create tablespace location, if does not exist
# (raise an exception if it is not possible)
cmd.create_dir_if_not_exists(location)
# check for write permissions on destination directory
cmd.check_write_permission(location)
# create symlink between tablespace and recovery folder
cmd.create_symbolic_link(location, pg_tblspc_file)
except FsOperationFailed, e:
output.exception("unable to prepare '%s' tablespace "
"(destination '%s'): %s",
item.name, location, e)
output.close_and_exit()
output.info("\t%s, %s, %s", item.oid, item.name, location)
def retrieve_safe_horizon(self, recovery_info, backup_info, dest):
"""
Retrieve the safe_horizon for smart copy
If the target directory contains a previous recovery, it is safe to
pick the least of the two backup "begin times" (the one we are
recovering now and the one previously recovered in the target
directory). Set the value in the given recovery_info dictionary.
:param dict recovery_info: Dictionary containing all the recovery params
:param barman.infofile.BackupInfo backup_info: a backup representation
def main():
"""
The main method of Barman
"""
p = ArghParser(epilog="Barman by EnterpriseDB (www.enterprisedb.com)")
p.add_argument(
"-v",
"--version",
action="version",
version="%s\n\nBarman by EnterpriseDB (www.enterprisedb.com)"
% barman.__version__,
)
p.add_argument(
"-c",
"--config",
help="uses a configuration file "
"(defaults: %s)" % ", ".join(barman.config.Config.CONFIG_FILES),
default=SUPPRESS,
)
p.add_argument(
"--color",
"--colour",
help="Whether to use colors in the output",
choices=["never", "always", "auto"],
default="auto",
)
p.add_argument(
"--log-level",
help="Override the default log level",
choices=list(get_log_levels()),
default=SUPPRESS,
)
p.add_argument("-q", "--quiet", help="be quiet", action="store_true")
p.add_argument("-d", "--debug", help="debug output", action="store_true")
p.add_argument(
"-f",
"--format",
help="output format",
choices=output.AVAILABLE_WRITERS.keys(),
default=output.DEFAULT_WRITER,
)
p.add_commands(
[
archive_wal,
backup,
check,
check_backup,
cron,
delete,
diagnose,
get_wal,
list_backup,
list_files,
list_server,
put_wal,
rebuild_xlogdb,
receive_wal,
recover,
show_backup,
show_server,
replication_status,
status,
switch_wal,
switch_xlog,
sync_info,
sync_backup,
sync_wals,
]
)
# noinspection PyBroadException
try:
p.dispatch(pre_call=global_config)
except KeyboardInterrupt:
msg = "Process interrupted by user (KeyboardInterrupt)"
output.error(msg)
except Exception as e:
msg = "%s\nSee log file for more details." % e
output.exception(msg)
# cleanup output API and exit honoring output.error_occurred and
# output.error_exit_code
output.close_and_exit()
