def worker_process_main(self, process_number):
"""
Repeatedly grab a task from the queue and execute it, until a task
containing "None" is grabbed, indicating that the process must stop.
:param int process_number: the process number, used in the logging
output
"""
logging.info("Upload process started (worker %s)", process_number)
while True:
task = self.queue.get()
if not task:
self.queue.task_done()
break
try:
self.worker_process_execute_job(task, process_number)
except Exception as exc:
logging.error('Upload error: %s (worker %s)',
force_str(exc), process_number)
logging.debug('Exception details:', exc_info=exc)
self.errors_queue.put(force_str(exc))
except KeyboardInterrupt:
if not self.abort_requested:
logging.info('Got abort request: upload cancelled '
'(worker %s)', process_number)
self.abort_requested = True
finally:
self.queue.task_done()
logging.info("Upload process stopped (worker %s)", process_number)
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
try:
conninfo = build_conninfo(config)
postgres = PostgreSQLConnection(conninfo,
config.immediate_checkpoint,
application_name='barman_cloud_backup')
try:
postgres.connect()
except PostgresConnectionError as exc:
logging.error("Cannot connect to postgres: %s", force_str(exc))
logging.debug('Exception details:', exc_info=exc)
raise SystemExit(1)
with closing(postgres):
cloud_interface = CloudInterface(
destination_url=config.destination_url,
encryption=config.encryption,
jobs=config.jobs,
profile_name=config.profile)
if not cloud_interface.test_connectivity():
raise SystemExit(1)
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
with closing(cloud_interface):
# TODO: Should the setup be optional?
cloud_interface.setup_bucket()
uploader = S3BackupUploader(server_name=config.server_name,
compression=config.compression,
postgres=postgres,
cloud_interface=cloud_interface)
# Perform the backup
uploader.backup()
except KeyboardInterrupt as exc:
logging.error("Barman cloud backup was interrupted by the user")
logging.debug('Exception details:', exc_info=exc)
raise SystemExit(1)
except Exception as exc:
logging.error("Barman cloud backup exception: %s", force_str(exc))
logging.debug('Exception details:', exc_info=exc)
raise SystemExit(1)
def reset(self):
"""
Reset the status of the class.
"""
self.environment = dict(self.extra_env)
config_file = self.backup_manager.config.config.config_file
self.environment.update(
{
"BARMAN_VERSION": version.__version__,
"BARMAN_SERVER": self.backup_manager.config.name,
"BARMAN_CONFIGURATION": config_file,
"BARMAN_HOOK": self.name,
"BARMAN_RETRY": str(1 if self.retry else 0),
}
)
if self.error:
self.environment["BARMAN_ERROR"] = force_str(self.error)
if self.phase:
self.environment["BARMAN_PHASE"] = self.phase
script_config_name = "%s_%s" % (self.phase, self.name)
else:
script_config_name = self.name
self.script = getattr(self.backup_manager.config, script_config_name, None)
self.exit_status = None
self.exception = None
def __str__(self):
# Returns the first line
if self.args and self.args[0]:
from barman.utils import force_str
return force_str(self.args[0]).splitlines()[0].strip()
else:
return ''
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
try:
cloud_interface = CloudInterface(url=config.source_url,
encryption=config.encryption,
profile_name=config.profile,
endpoint_url=config.endpoint_url)
with closing(cloud_interface):
catalog = S3BackupCatalog(cloud_interface=cloud_interface,
server_name=config.server_name)
if not cloud_interface.test_connectivity():
raise SystemExit(1)
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
if not cloud_interface.bucket_exists:
logging.error("Bucket %s does not exist",
cloud_interface.bucket_name)
raise SystemExit(1)
backup_list = catalog.get_backup_list()
# Output
if config.format == 'console':
COLUMNS = "{:<20}{:<25}{:<30}"
print(COLUMNS.format("Backup ID", "End Time", "Begin Wal"))
for backup_id in sorted(backup_list):
item = backup_list[backup_id]
if item and item.status == BackupInfo.DONE:
print(
COLUMNS.format(
item.backup_id,
item.end_time.strftime("%Y-%m-%d %H:%M:%S"),
item.begin_wal))
else:
print(
json.dumps({
"backups_list": [
backup_list[backup_id].to_json()
for backup_id in sorted(backup_list)
]
}))
except Exception as exc:
logging.error("Barman cloud backup list exception: %s", force_str(exc))
logging.debug('Exception details:', exc_info=exc)
raise SystemExit(1)
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
# Read wal_path from environment if we're a hook script
if __is_hook_script():
if "BARMAN_FILE" not in os.environ:
raise BarmanException(
"Expected environment variable BARMAN_FILE not set")
config.wal_path = os.getenv("BARMAN_FILE")
else:
if config.wal_path is None:
raise BarmanException(
"the following arguments are required: wal_path")
# Validate the WAL file name before uploading it
if not is_any_xlog_file(config.wal_path):
logging.error("%s is an invalid name for a WAL file" % config.wal_path)
raise CLIErrorExit()
try:
cloud_interface = get_cloud_interface(config)
with closing(cloud_interface):
uploader = CloudWalUploader(
cloud_interface=cloud_interface,
server_name=config.server_name,
compression=config.compression,
)
if not cloud_interface.test_connectivity():
raise NetworkErrorExit()
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
# TODO: Should the setup be optional?
cloud_interface.setup_bucket()
upload_kwargs = {}
if is_history_file(config.wal_path):
upload_kwargs["override_tags"] = config.history_tags
uploader.upload_wal(config.wal_path, **upload_kwargs)
except Exception as exc:
logging.error("Barman cloud WAL archiver exception: %s",
force_str(exc))
logging.debug("Exception details:", exc_info=exc)
raise GeneralErrorExit()
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
try:
cloud_interface = get_cloud_interface(config)
if not cloud_interface.test_connectivity():
# Deliberately raise an error if we cannot connect
raise NetworkErrorExit()
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
if not cloud_interface.bucket_exists:
# If the bucket does not exist then the check should pass
return
catalog = CloudBackupCatalog(cloud_interface, config.server_name)
wals = list(catalog.get_wal_paths().keys())
check_archive_usable(
wals,
timeline=config.timeline,
)
except WalArchiveContentError as err:
logging.error(
"WAL archive check failed for server %s: %s",
config.server_name,
force_str(err),
)
raise OperationErrorExit()
except Exception as exc:
logging.error("Barman cloud WAL archive check exception: %s",
force_str(exc))
logging.debug("Exception details:", exc_info=exc)
raise GeneralErrorExit()
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
try:
cloud_interface = get_cloud_interface(config)
with closing(cloud_interface):
if not cloud_interface.test_connectivity():
raise NetworkErrorExit()
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
if not cloud_interface.bucket_exists:
logging.error("Bucket %s does not exist",
cloud_interface.bucket_name)
raise OperationErrorExit()
catalog = CloudBackupCatalog(cloud_interface, config.server_name)
if config.release:
catalog.release_keep(config.backup_id)
elif config.status:
target = catalog.get_keep_target(config.backup_id)
if target:
print("Keep: %s" % target)
else:
print("Keep: nokeep")
else:
backup_info = catalog.get_backup_info(config.backup_id)
if backup_info.status == BackupInfo.DONE:
catalog.keep_backup(config.backup_id, config.target)
else:
logging.error(
"Cannot add keep to backup %s because it has status %s. "
"Only backups with status DONE can be kept.",
config.backup_id,
backup_info.status,
)
raise OperationErrorExit()
except Exception as exc:
logging.error("Barman cloud keep exception: %s", force_str(exc))
logging.debug("Exception details:", exc_info=exc)
raise GeneralErrorExit()
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
# Validate the destination directory before starting recovery
if os.path.exists(config.recovery_dir) and os.listdir(config.recovery_dir):
logging.error(
"Destination %s already exists and it is not empty", config.recovery_dir
)
raise OperationErrorExit()
try:
cloud_interface = get_cloud_interface(config)
with closing(cloud_interface):
downloader = CloudBackupDownloader(
cloud_interface=cloud_interface, server_name=config.server_name
)
if not cloud_interface.test_connectivity():
raise NetworkErrorExit()
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
if not cloud_interface.bucket_exists:
logging.error("Bucket %s does not exist", cloud_interface.bucket_name)
raise OperationErrorExit()
downloader.download_backup(
config.backup_id,
config.recovery_dir,
tablespace_map(config.tablespace),
)
except KeyboardInterrupt as exc:
logging.error("Barman cloud restore was interrupted by the user")
logging.debug("Exception details:", exc_info=exc)
raise OperationErrorExit()
except Exception as exc:
logging.error("Barman cloud restore exception: %s", force_str(exc))
logging.debug("Exception details:", exc_info=exc)
raise GeneralErrorExit()
def decompress(self, src, dst):
"""
Decompress using the object defined in the subclass
:param src: source file to decompress
:param dst: destination of the decompression
"""
try:
with closing(self._decompressor(src)) as istream:
with open(dst, "wb") as ostream:
shutil.copyfileobj(istream, ostream)
except Exception as e:
# you won't get more information from the compressors anyway
raise CommandFailedException(
dict(ret=None, err=force_str(e), out=None))
return 0
def list_files(args):
"""
List all the files for a single backup
"""
server = get_server(args)
# Retrieves the backup
backup_info = parse_backup_id(server, args)
try:
for line in backup_info.get_list_of_files(args.target):
output.info(line, log=False)
except BadXlogSegmentName as e:
output.error(
"invalid xlog segment name %r\n"
"HINT: Please run \"barman rebuild-xlogdb %s\" "
"to solve this issue", force_str(e), server.config.name)
output.close_and_exit()
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
# Validate the WAL file name before downloading it
if not is_any_xlog_file(config.wal_name):
logging.error('%s is an invalid name for a WAL file' % config.wal_name)
raise SystemExit(1)
try:
cloud_interface = CloudInterface(
url=config.source_url,
encryption=config.encryption,
profile_name=config.profile,
endpoint_url=config.endpoint_url)
with closing(cloud_interface):
downloader = S3WalDownloader(
cloud_interface=cloud_interface,
server_name=config.server_name)
if not cloud_interface.test_connectivity():
raise SystemExit(1)
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
if not cloud_interface.bucket_exists:
logging.error("Bucket %s does not exist",
cloud_interface.bucket_name)
raise SystemExit(1)
downloader.download_wal(config.wal_name, config.wal_dest)
except Exception as exc:
logging.error("Barman cloud WAL restore exception: %s",
force_str(exc))
logging.debug('Exception details:', exc_info=exc)
raise SystemExit(1)
def connectivity_test(config):
"""
Invoke remote get-wal --test to test the connection with Barman server
:param argparse.Namespace config: the configuration from command line
"""
# Build the peek command
ssh_command = build_ssh_command(config, 'dummy_wal_name')
# Issue the command
try:
pipe = subprocess.Popen(ssh_command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = pipe.communicate()
print(force_str(output[0]))
sys.exit(pipe.returncode)
except subprocess.CalledProcessError as e:
exit_with_error("Impossible to invoke remote get-wal: %s" % e)
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
# Validate the destination directory before starting recovery
if os.path.exists(config.recovery_dir) and os.listdir(config.recovery_dir):
logging.error("Destination %s already exists and it is not empty",
config.recovery_dir)
raise SystemExit(1)
try:
cloud_interface = CloudInterface(url=config.source_url,
encryption=config.encryption,
profile_name=config.profile,
endpoint_url=config.endpoint_url)
with closing(cloud_interface):
downloader = S3BackupDownloader(cloud_interface=cloud_interface,
server_name=config.server_name)
if not cloud_interface.test_connectivity():
raise SystemExit(1)
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
if not cloud_interface.bucket_exists:
logging.error("Bucket %s does not exist",
cloud_interface.bucket_name)
raise SystemExit(1)
downloader.download_backup(config.backup_id, config.recovery_dir)
except Exception as exc:
logging.error("Barman cloud backup restore exception: %s",
force_str(exc))
logging.debug('Exception details:', exc_info=exc)
raise SystemExit(1)
def env_from_recover(self,
backup_info,
dest,
tablespaces,
remote_command,
error=None,
**kwargs):
"""
Prepare the environment for executing a script
:param BackupInfo backup_info: the backup metadata
:param str dest: the destination directory
:param dict[str,str]|None tablespaces: a tablespace name -> location
map (for relocation)
:param str|None remote_command: default None. The remote command
to recover the base backup, in case of remote backup.
:param str|Exception error: An error message in case of failure
"""
self.env_from_backup_info(backup_info)
# Prepare a JSON representation of tablespace map
tablespaces_map = ""
if tablespaces:
tablespaces_map = json.dumps(tablespaces, sort_keys=True)
# Prepare a JSON representation of additional recovery options
# Skip any empty argument
kwargs_filtered = dict([(k, v) for k, v in kwargs.items() if v])
recover_options = ""
if kwargs_filtered:
recover_options = json.dumps(kwargs_filtered, sort_keys=True)
self.environment.update({
"BARMAN_DESTINATION_DIRECTORY":
str(dest),
"BARMAN_TABLESPACES":
tablespaces_map,
"BARMAN_REMOTE_COMMAND":
str(remote_command or ""),
"BARMAN_RECOVER_OPTIONS":
recover_options,
"BARMAN_ERROR":
force_str(error or ""),
})
def __parse_tag(tag):
"""Parse key,value tag with csv reader"""
try:
rows = list(csv.reader([tag], delimiter=","))
except csv.Error as exc:
logging.error(
"Error parsing tag %s: %s",
tag,
force_str(exc),
)
raise CLIErrorExit()
if len(rows) != 1 or len(rows[0]) != 2:
logging.error(
"Invalid tag format: %s",
tag,
)
raise CLIErrorExit()
return tuple(rows[0])
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging()
# Validate the WAL file name before uploading it
if not is_any_xlog_file(config.wal_path):
logging.error('%s is an invalid name for a WAL file' % config.wal_path)
raise SystemExit(1)
try:
cloud_interface = CloudInterface(
destination_url=config.destination_url,
encryption=config.encryption,
profile_name=config.profile)
with closing(cloud_interface):
uploader = S3WalUploader(
cloud_interface=cloud_interface,
server_name=config.server_name,
compression=config.compression)
if not cloud_interface.test_connectivity():
raise SystemExit(1)
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
# TODO: Should the setup be optional?
cloud_interface.setup_bucket()
uploader.upload_wal(config.wal_path)
except Exception as exc:
logging.error("Barman cloud WAL archiver exception: %s",
force_str(exc))
logging.debug('Exception details:', exc_info=exc)
raise SystemExit(1)
def __init__(self, server):
"""
Constructor
"""
super(BackupManager, self).__init__()
self.server = server
self.config = server.config
self._backup_cache = None
self.compression_manager = CompressionManager(self.config, server.path)
self.executor = None
try:
if server.passive_node:
self.executor = PassiveBackupExecutor(self)
elif self.config.backup_method == "postgres":
self.executor = PostgresBackupExecutor(self)
else:
self.executor = RsyncBackupExecutor(self)
except SshCommandException as e:
self.config.disabled = True
self.config.msg_list.append(force_str(e).strip())
def from_command_error(cls, cmd, e, msg):
"""
This method build a DataTransferFailure exception and report the
provided message to the user (both console and log file) along with
the output of the failed command.
:param str cmd: The command that failed the transfer
:param CommandFailedException e: The exception we are handling
:param str msg: a descriptive message on what we are trying to do
:return DataTransferFailure: will contain the message provided in msg
"""
try:
details = msg
details += "\n%s error:\n" % cmd
details += e.args[0]['out']
details += e.args[0]['err']
return cls(details)
except (TypeError, NameError):
# If it is not a dictionary just convert it to a string
from barman.utils import force_str
return cls(force_str(e.args))
def handle_backup_errors(self, action, backup_info, exc):
"""
Mark the backup as failed and exit
:param str action: the upload phase that has failed
:param barman.infofile.BackupInfo backup_info: the backup info file
:param BaseException exc: the exception that caused the failure
"""
msg_lines = force_str(exc).strip().splitlines()
# If the exception has no attached message use the raw
# type name
if len(msg_lines) == 0:
msg_lines = [type(exc).__name__]
if backup_info:
# Use only the first line of exception message
# in backup_info error field
backup_info.set_attribute("status", "FAILED")
backup_info.set_attribute(
"error", "failure %s (%s)" % (action, msg_lines[0]))
logging.error("Backup failed %s (%s)", action, msg_lines[0])
logging.debug('Exception details:', exc_info=exc)
def env_from_wal_info(self, wal_info, full_path=None, error=None):
"""
Prepare the environment for executing a script
:param WalFileInfo wal_info: the backup metadata
:param str full_path: override wal_info.fullpath() result
:param str|Exception error: An error message in case of failure
"""
self.environment.update(
{
"BARMAN_SEGMENT": wal_info.name,
"BARMAN_FILE": str(
full_path
if full_path is not None
else wal_info.fullpath(self.backup_manager.server)
),
"BARMAN_SIZE": str(wal_info.size),
"BARMAN_TIMESTAMP": str(wal_info.time),
"BARMAN_COMPRESSION": wal_info.compression or "",
"BARMAN_ERROR": force_str(error or ""),
}
)
def _delete_backup(
cloud_interface,
catalog,
backup_id,
dry_run=True,
skip_wal_cleanup_if_standalone=True,
):
backup_info = catalog.get_backup_info(backup_id)
if not backup_info:
logging.warning("Backup %s does not exist", backup_id)
return
objects_to_delete = _get_files_for_backup(catalog, backup_info)
backup_info_path = os.path.join(
catalog.prefix, backup_info.backup_id, "backup.info"
)
logging.debug("Will delete backup.info file at %s" % backup_info_path)
if not dry_run:
try:
cloud_interface.delete_objects(objects_to_delete)
# Do not try to delete backup.info until we have successfully deleted
# everything else so that it is possible to retry the operation should
# we fail to delete any backup file
cloud_interface.delete_objects([backup_info_path])
except Exception as exc:
logging.error("Could not delete backup %s: %s", backup_id, force_str(exc))
raise SystemExit(2)
else:
print(
"Skipping deletion of objects %s due to --dry-run option"
% (objects_to_delete + [backup_info_path])
)
_remove_wals_for_backup(
cloud_interface, catalog, backup_info, dry_run, skip_wal_cleanup_if_standalone
)
# It is important that the backup is removed from the catalog after cleaning
# up the WALs because the code in _remove_wals_for_backup depends on the
# deleted backup existing in the backup catalog
catalog.remove_backup_from_cache(backup_id)
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
tempdir = tempfile.mkdtemp(prefix='barman-cloud-backup-')
try:
# Create any temporary file in the `tempdir` subdirectory
tempfile.tempdir = tempdir
conninfo = build_conninfo(config)
postgres = PostgreSQLConnection(conninfo, config.immediate_checkpoint,
application_name='barman_cloud_backup')
try:
postgres.connect()
except PostgresConnectionError as exc:
logging.error("Cannot connect to postgres: %s", force_str(exc))
logging.debug('Exception details:', exc_info=exc)
raise SystemExit(1)
with closing(postgres):
cloud_interface = CloudInterface(
url=config.destination_url,
encryption=config.encryption,
jobs=config.jobs,
profile_name=config.profile,
endpoint_url=config.endpoint_url)
if not cloud_interface.test_connectivity():
raise SystemExit(1)
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
with closing(cloud_interface):
# TODO: Should the setup be optional?
cloud_interface.setup_bucket()
uploader = S3BackupUploader(
server_name=config.server_name,
compression=config.compression,
postgres=postgres,
max_archive_size=config.max_archive_size,
cloud_interface=cloud_interface)
# Perform the backup
uploader.backup()
except KeyboardInterrupt as exc:
logging.error("Barman cloud backup was interrupted by the user")
logging.debug('Exception details:', exc_info=exc)
raise SystemExit(1)
except Exception as exc:
logging.error("Barman cloud backup exception: %s", force_str(exc))
logging.debug('Exception details:', exc_info=exc)
raise SystemExit(1)
finally:
# Remove the temporary directory and all the contained files
rmtree(tempdir, ignore_errors=True)
def _remove_wals_for_backup(
cloud_interface,
catalog,
deleted_backup,
dry_run,
skip_wal_cleanup_if_standalone=True,
):
# An implementation of BackupManager.remove_wal_before_backup which does not
# use xlogdb, since xlogdb is not available to barman-cloud
should_remove_wals, wal_ranges_to_protect = BackupManager.should_remove_wals(
deleted_backup,
catalog.get_backup_list(),
keep_manager=catalog,
skip_wal_cleanup_if_standalone=skip_wal_cleanup_if_standalone,
)
next_backup = BackupManager.find_next_backup_in(
catalog.get_backup_list(), deleted_backup.backup_id
)
wals_to_delete = {}
if should_remove_wals:
# There is no previous backup or all previous backups are archival
# standalone backups, so we can remove unused WALs (those WALs not
# required by standalone archival backups).
# If there is a next backup then all unused WALs up to the begin_wal
# of the next backup can be removed.
# If there is no next backup then there are no remaining backups,
# because we must assume non-exclusive backups are taken, we can only
# safely delete unused WALs up to begin_wal of the deleted backup.
# See comments in barman.backup.BackupManager.delete_backup.
if next_backup:
remove_until = next_backup
else:
remove_until = deleted_backup
# A WAL is only a candidate for deletion if it is on the same timeline so we
# use BackupManager to get a set of all other timelines with backups so that
# we can preserve all WALs on other timelines.
timelines_to_protect = BackupManager.get_timelines_to_protect(
remove_until=remove_until,
deleted_backup=deleted_backup,
available_backups=catalog.get_backup_list(),
)
try:
wal_paths = catalog.get_wal_paths()
except Exception as exc:
logging.error(
"Cannot clean up WALs for backup %s because an error occurred listing WALs: %s",
deleted_backup.backup_id,
force_str(exc),
)
return
for wal_name, wal in wal_paths.items():
if xlog.is_history_file(wal_name):
continue
if timelines_to_protect:
tli, _, _ = xlog.decode_segment_name(wal_name)
if tli in timelines_to_protect:
continue
# Check if the WAL is in a protected range, required by an archival
# standalone backup - so do not delete it
if xlog.is_backup_file(wal_name):
# If we have a backup file, truncate the name for the range check
range_check_wal_name = wal_name[:24]
else:
range_check_wal_name = wal_name
if any(
range_check_wal_name >= begin_wal and range_check_wal_name <= end_wal
for begin_wal, end_wal in wal_ranges_to_protect
):
continue
if wal_name < remove_until.begin_wal:
wals_to_delete[wal_name] = wal
# Explicitly sort because dicts are not ordered in python < 3.6
wal_paths_to_delete = sorted(wals_to_delete.values())
if len(wal_paths_to_delete) > 0:
if not dry_run:
try:
cloud_interface.delete_objects(wal_paths_to_delete)
except Exception as exc:
logging.error(
"Could not delete the following WALs for backup %s: %s, Reason: %s",
deleted_backup.backup_id,
wal_paths_to_delete,
force_str(exc),
)
# Return early so that we leave the WALs in the local cache so they
# can be cleaned up should there be a subsequent backup deletion.
return
else:
print(
"Skipping deletion of objects %s due to --dry-run option"
% wal_paths_to_delete
)
for wal_name in wals_to_delete.keys():
catalog.remove_wal_from_cache(wal_name)
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
try:
cloud_interface = get_cloud_interface(config)
with closing(cloud_interface):
if not cloud_interface.test_connectivity():
raise SystemExit(1)
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
if not cloud_interface.bucket_exists:
logging.error("Bucket %s does not exist", cloud_interface.bucket_name)
raise SystemExit(1)
catalog = CloudBackupCatalog(
cloud_interface=cloud_interface, server_name=config.server_name
)
# Call catalog.get_backup_list now so we know we can read the whole catalog
# (the results are cached so this does not result in extra calls to cloud
# storage)
catalog.get_backup_list()
if len(catalog.unreadable_backups) > 0:
logging.error(
"Cannot read the following backups: %s\n"
"Unsafe to proceed with deletion due to failure reading backup catalog"
% catalog.unreadable_backups
)
raise SystemExit(1)
if config.backup_id:
# Because we only care about one backup, skip the annotation cache
# because it is only helpful when dealing with multiple backups
if catalog.should_keep_backup(config.backup_id, use_cache=False):
logging.error(
"Skipping delete of backup %s for server %s "
"as it has a current keep request. If you really "
"want to delete this backup please remove the keep "
"and try again.",
config.backup_id,
config.server_name,
)
raise SystemExit(1)
_delete_backup(
cloud_interface, catalog, config.backup_id, config.dry_run
)
elif config.retention_policy:
retention_policy = RetentionPolicyFactory.create(
"retention_policy",
config.retention_policy,
server_name=config.server_name,
catalog=catalog,
)
# Sort to ensure that we delete the backups in ascending order, that is
# from oldest to newest. This ensures that the relevant WALs will be cleaned
# up after each backup is deleted.
backups_to_delete = sorted(
[
backup_id
for backup_id, status in retention_policy.report().items()
if status == "OBSOLETE"
]
)
for backup_id in backups_to_delete:
_delete_backup(
cloud_interface,
catalog,
backup_id,
config.dry_run,
skip_wal_cleanup_if_standalone=False,
)
except Exception as exc:
logging.error("Barman cloud backup delete exception: %s", force_str(exc))
logging.debug("Exception details:", exc_info=exc)
raise SystemExit(1)
def backup(self, wait=False, wait_timeout=None):
"""
Performs a backup for the server
:param bool wait: wait for all the required WAL files to be archived
:param int|None wait_timeout:
:return BackupInfo: the generated BackupInfo
"""
_logger.debug("initialising backup information")
self.executor.init()
backup_info = None
try:
# Create the BackupInfo object representing the backup
backup_info = LocalBackupInfo(
self.server,
backup_id=datetime.datetime.now().strftime('%Y%m%dT%H%M%S'))
backup_info.set_attribute('systemid', self.server.systemid)
backup_info.save()
self.backup_cache_add(backup_info)
output.info("Starting backup using %s method for server %s in %s",
self.mode, self.config.name,
backup_info.get_basebackup_directory())
# Run the pre-backup-script if present.
script = HookScriptRunner(self, 'backup_script', 'pre')
script.env_from_backup_info(backup_info)
script.run()
# Run the pre-backup-retry-script if present.
retry_script = RetryHookScriptRunner(self, 'backup_retry_script',
'pre')
retry_script.env_from_backup_info(backup_info)
retry_script.run()
# Do the backup using the BackupExecutor
self.executor.backup(backup_info)
# Create a restore point after a backup
target_name = 'barman_%s' % backup_info.backup_id
self.server.postgres.create_restore_point(target_name)
# Free the Postgres connection
self.server.postgres.close()
# Compute backup size and fsync it on disk
self.backup_fsync_and_set_sizes(backup_info)
# Mark the backup as WAITING_FOR_WALS
backup_info.set_attribute("status", BackupInfo.WAITING_FOR_WALS)
# Use BaseException instead of Exception to catch events like
# KeyboardInterrupt (e.g.: CTRL-C)
except BaseException as e:
msg_lines = force_str(e).strip().splitlines()
# If the exception has no attached message use the raw
# type name
if len(msg_lines) == 0:
msg_lines = [type(e).__name__]
if backup_info:
# Use only the first line of exception message
# in backup_info error field
backup_info.set_attribute("status", BackupInfo.FAILED)
backup_info.set_attribute(
"error", "failure %s (%s)" %
(self.executor.current_action, msg_lines[0]))
output.error("Backup failed %s.\nDETAILS: %s",
self.executor.current_action, '\n'.join(msg_lines))
else:
output.info("Backup end at LSN: %s (%s, %08X)",
backup_info.end_xlog, backup_info.end_wal,
backup_info.end_offset)
executor = self.executor
output.info(
"Backup completed (start time: %s, elapsed time: %s)",
self.executor.copy_start_time,
human_readable_timedelta(datetime.datetime.now() -
executor.copy_start_time))
# If requested, wait for end_wal to be archived
if wait:
try:
self.server.wait_for_wal(backup_info.end_wal, wait_timeout)
self.check_backup(backup_info)
except KeyboardInterrupt:
# Ignore CTRL-C pressed while waiting for WAL files
output.info(
"Got CTRL-C. Continuing without waiting for '%s' "
"to be archived", backup_info.end_wal)
finally:
if backup_info:
backup_info.save()
# Make sure we are not holding any PostgreSQL connection
# during the post-backup scripts
self.server.close()
# Run the post-backup-retry-script if present.
try:
retry_script = RetryHookScriptRunner(
self, 'backup_retry_script', 'post')
retry_script.env_from_backup_info(backup_info)
retry_script.run()
except AbortedRetryHookScript as e:
# Ignore the ABORT_STOP as it is a post-hook operation
_logger.warning(
"Ignoring stop request after receiving "
"abort (exit code %d) from post-backup "
"retry hook script: %s", e.hook.exit_status,
e.hook.script)
# Run the post-backup-script if present.
script = HookScriptRunner(self, 'backup_script', 'post')
script.env_from_backup_info(backup_info)
script.run()
output.result('backup', backup_info)
return backup_info
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
tempdir = tempfile.mkdtemp(prefix="barman-cloud-backup-")
try:
# Create any temporary file in the `tempdir` subdirectory
tempfile.tempdir = tempdir
cloud_interface = get_cloud_interface(config)
if not cloud_interface.test_connectivity():
raise NetworkErrorExit()
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
with closing(cloud_interface):
# TODO: Should the setup be optional?
cloud_interface.setup_bucket()
# Perform the backup
uploader_kwargs = {
"server_name": config.server_name,
"compression": config.compression,
"max_archive_size": config.max_archive_size,
"cloud_interface": cloud_interface,
}
if __is_hook_script():
if "BARMAN_BACKUP_DIR" not in os.environ:
raise BarmanException(
"BARMAN_BACKUP_DIR environment variable not set")
if "BARMAN_BACKUP_ID" not in os.environ:
raise BarmanException(
"BARMAN_BACKUP_ID environment variable not set")
if os.getenv("BARMAN_STATUS") != "DONE":
raise UnrecoverableHookScriptError(
"backup in '%s' has status '%s' (status should be: DONE)"
% (os.getenv("BARMAN_BACKUP_DIR"),
os.getenv("BARMAN_STATUS")))
uploader = CloudBackupUploaderBarman(
backup_dir=os.getenv("BARMAN_BACKUP_DIR"),
backup_id=os.getenv("BARMAN_BACKUP_ID"),
**uploader_kwargs)
uploader.backup()
else:
conninfo = build_conninfo(config)
postgres = PostgreSQLConnection(
conninfo,
config.immediate_checkpoint,
application_name="barman_cloud_backup",
)
try:
postgres.connect()
except PostgresConnectionError as exc:
logging.error("Cannot connect to postgres: %s",
force_str(exc))
logging.debug("Exception details:", exc_info=exc)
raise OperationErrorExit()
with closing(postgres):
uploader = CloudBackupUploaderPostgres(postgres=postgres,
**uploader_kwargs)
uploader.backup()
except KeyboardInterrupt as exc:
logging.error("Barman cloud backup was interrupted by the user")
logging.debug("Exception details:", exc_info=exc)
raise OperationErrorExit()
except UnrecoverableHookScriptError as exc:
logging.error("Barman cloud backup exception: %s", force_str(exc))
logging.debug("Exception details:", exc_info=exc)
raise SystemExit(63)
except Exception as exc:
logging.error("Barman cloud backup exception: %s", force_str(exc))
logging.debug("Exception details:", exc_info=exc)
raise GeneralErrorExit()
finally:
# Remove the temporary directory and all the contained files
rmtree(tempdir, ignore_errors=True)
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
try:
cloud_interface = get_cloud_interface(config)
with closing(cloud_interface):
catalog = CloudBackupCatalog(cloud_interface=cloud_interface,
server_name=config.server_name)
if not cloud_interface.test_connectivity():
raise NetworkErrorExit()
# If test is requested, just exit after connectivity test
elif config.test:
raise SystemExit(0)
if not cloud_interface.bucket_exists:
logging.error("Bucket %s does not exist",
cloud_interface.bucket_name)
raise OperationErrorExit()
backup_list = catalog.get_backup_list()
# Output
if config.format == "console":
COLUMNS = "{:<20}{:<25}{:<30}{:<16}"
print(
COLUMNS.format("Backup ID", "End Time", "Begin Wal",
"Archival Status"))
for backup_id in sorted(backup_list):
item = backup_list[backup_id]
if item and item.status == BackupInfo.DONE:
keep_target = catalog.get_keep_target(item.backup_id)
keep_status = (keep_target
and "KEEP:%s" % keep_target.upper()
or "")
print(
COLUMNS.format(
item.backup_id,
item.end_time.strftime("%Y-%m-%d %H:%M:%S"),
item.begin_wal,
keep_status,
))
else:
print(
json.dumps({
"backups_list": [
backup_list[backup_id].to_json()
for backup_id in sorted(backup_list)
]
}))
except Exception as exc:
logging.error("Barman cloud backup list exception: %s", force_str(exc))
logging.debug("Exception details:", exc_info=exc)
raise GeneralErrorExit()
def recover(self,
backup_info,
dest,
tablespaces=None,
remote_command=None,
target_tli=None,
target_time=None,
target_xid=None,
target_name=None,
target_immediate=False,
exclusive=False,
target_action=None,
standby_mode=None):
"""
Performs a recovery of a backup
This method should be called in a closing context
:param barman.infofile.BackupInfo backup_info: the backup to recover
:param str dest: the destination directory
:param dict[str,str]|None tablespaces: a tablespace
name -> location map (for relocation)
:param str|None remote_command: The remote command to recover
the base backup, in case of remote backup.
:param str|None target_tli: the target timeline
:param str|None target_time: the target time
:param str|None target_xid: the target xid
:param str|None target_name: the target name created previously with
pg_create_restore_point() function call
:param str|None target_immediate: end recovery as soon as consistency
is reached
:param bool exclusive: whether the recovery is exclusive or not
:param str|None target_action: The recovery target action
:param bool|None standby_mode: standby mode
"""
# Run the cron to be sure the wal catalog is up to date
# Prepare a map that contains all the objects required for a recovery
recovery_info = self._setup(backup_info, remote_command, dest)
output.info("Starting %s restore for server %s using backup %s",
recovery_info['recovery_dest'], self.server.config.name,
backup_info.backup_id)
output.info("Destination directory: %s", dest)
if remote_command:
output.info("Remote command: %s", remote_command)
# If the backup we are recovering is still not validated and we
# haven't requested the get-wal feature, display a warning message
if not recovery_info['get_wal']:
if backup_info.status == BackupInfo.WAITING_FOR_WALS:
output.warning(
"IMPORTANT: You have requested a recovery operation for "
"a backup that does not have yet all the WAL files that "
"are required for consistency.")
# Set targets for PITR
self._set_pitr_targets(recovery_info, backup_info, dest, target_name,
target_time, target_tli, target_xid,
target_immediate, target_action)
# Retrieve the safe_horizon for smart copy
self._retrieve_safe_horizon(recovery_info, backup_info, dest)
# check destination directory. If doesn't exist create it
try:
recovery_info['cmd'].create_dir_if_not_exists(dest)
except FsOperationFailed as e:
output.error(
"unable to initialise destination directory "
"'%s': %s", dest, e)
output.close_and_exit()
# Initialize tablespace directories
if backup_info.tablespaces:
self._prepare_tablespaces(backup_info, recovery_info['cmd'], dest,
tablespaces)
# Copy the base backup
output.info("Copying the base backup.")
try:
self._backup_copy(backup_info, dest, tablespaces, remote_command,
recovery_info['safe_horizon'])
except DataTransferFailure as e:
output.error("Failure copying base backup: %s", e)
output.close_and_exit()
# Copy the backup.info file in the destination as
# ".barman-recover.info"
if remote_command:
try:
recovery_info['rsync'](backup_info.filename,
':%s/.barman-recover.info' % dest)
except CommandFailedException as e:
output.error('copy of recovery metadata file failed: %s', e)
output.close_and_exit()
else:
backup_info.save(os.path.join(dest, '.barman-recover.info'))
# Standby mode is not available for PostgreSQL older than 9.0
if backup_info.version < 90000 and standby_mode:
raise RecoveryStandbyModeException(
'standby_mode is available only from PostgreSQL 9.0')
# Restore the WAL segments. If GET_WAL option is set, skip this phase
# as they will be retrieved using the wal-get command.
if not recovery_info['get_wal']:
# If the backup we restored is still waiting for WALS, read the
# backup info again and check whether it has been validated.
# Notify the user if it is still not DONE.
if backup_info.status == BackupInfo.WAITING_FOR_WALS:
data = BackupInfo(self.server, backup_info.filename)
if data.status == BackupInfo.WAITING_FOR_WALS:
output.warning(
"IMPORTANT: The backup we have recovered IS NOT "
"VALID. Required WAL files for consistency are "
"missing. Please verify that WAL archiving is "
"working correctly or evaluate using the 'get-wal' "
"option for recovery")
output.info("Copying required WAL segments.")
try:
# Retrieve a list of required log files
required_xlog_files = tuple(
self.server.get_required_xlog_files(
backup_info, target_tli,
recovery_info['target_epoch']))
# Restore WAL segments into the wal_dest directory
self._xlog_copy(required_xlog_files, recovery_info['wal_dest'],
remote_command)
except DataTransferFailure as e:
output.error("Failure copying WAL files: %s", e)
output.close_and_exit()
except BadXlogSegmentName as e:
output.error(
"invalid xlog segment name %r\n"
"HINT: Please run \"barman rebuild-xlogdb %s\" "
"to solve this issue", force_str(e), self.config.name)
output.close_and_exit()
# If WAL files are put directly in the pg_xlog directory,
# avoid shipping of just recovered files
# by creating the corresponding archive status file
if not recovery_info['is_pitr']:
output.info("Generating archive status files")
self._generate_archive_status(recovery_info, remote_command,
required_xlog_files)
# Generate recovery.conf file (only if needed by PITR or get_wal)
is_pitr = recovery_info['is_pitr']
get_wal = recovery_info['get_wal']
if is_pitr or get_wal or standby_mode:
output.info("Generating recovery.conf")
self._generate_recovery_conf(recovery_info, backup_info, dest,
target_immediate, exclusive,
remote_command, target_name,
target_time, target_tli, target_xid,
standby_mode)
# Create archive_status directory if necessary
archive_status_dir = os.path.join(recovery_info['wal_dest'],
'archive_status')
try:
recovery_info['cmd'].create_dir_if_not_exists(archive_status_dir)
except FsOperationFailed as e:
output.error(
"unable to create the archive_status directory "
"'%s': %s", archive_status_dir, e)
output.close_and_exit()
# As last step, analyse configuration files in order to spot
# harmful options. Barman performs automatic conversion of
# some options as well as notifying users of their existence.
#
# This operation is performed in three steps:
# 1) mapping
# 2) analysis
# 3) copy
output.info("Identify dangerous settings in destination directory.")
self._map_temporary_config_files(recovery_info, backup_info,
remote_command)
self._analyse_temporary_config_files(recovery_info)
self._copy_temporary_config_files(dest, remote_command, recovery_info)
return recovery_info
def recover(args):
"""
Recover a server at a given time, name, LSN or xid
"""
server = get_server(args)
# Retrieves the backup
backup_id = parse_backup_id(server, args)
if backup_id.status not in BackupInfo.STATUS_COPY_DONE:
output.error(
"Cannot recover from backup '%s' of server '%s': "
"backup status is not DONE", args.backup_id, server.config.name)
output.close_and_exit()
# decode the tablespace relocation rules
tablespaces = {}
if args.tablespace:
for rule in args.tablespace:
try:
tablespaces.update([rule.split(':', 1)])
except ValueError:
output.error(
"Invalid tablespace relocation rule '%s'\n"
"HINT: The valid syntax for a relocation rule is "
"NAME:LOCATION", rule)
output.close_and_exit()
# validate the rules against the tablespace list
valid_tablespaces = []
if backup_id.tablespaces:
valid_tablespaces = [
tablespace_data.name for tablespace_data in backup_id.tablespaces
]
for item in tablespaces:
if item not in valid_tablespaces:
output.error(
"Invalid tablespace name '%s'\n"
"HINT: Please use any of the following "
"tablespaces: %s", item, ', '.join(valid_tablespaces))
output.close_and_exit()
# explicitly disallow the rsync remote syntax (common mistake)
if ':' in args.destination_directory:
output.error(
"The destination directory parameter "
"cannot contain the ':' character\n"
"HINT: If you want to do a remote recovery you have to use "
"the --remote-ssh-command option")
output.close_and_exit()
if args.retry_sleep is not None:
server.config.basebackup_retry_sleep = args.retry_sleep
if args.retry_times is not None:
server.config.basebackup_retry_times = args.retry_times
if hasattr(args, 'get_wal'):
if args.get_wal:
server.config.recovery_options.add(RecoveryOptions.GET_WAL)
else:
server.config.recovery_options.remove(RecoveryOptions.GET_WAL)
if args.jobs is not None:
server.config.parallel_jobs = args.jobs
if hasattr(args, 'bwlimit'):
server.config.bandwidth_limit = args.bwlimit
# PostgreSQL supports multiple parameters to specify when the recovery
# process will end, and in that case the last entry in recovery
# configuration files will be used. See [1]
#
# Since the meaning of the target options is not dependent on the order
# of parameters, we decided to make the target options mutually exclusive.
#
# [1]: https://www.postgresql.org/docs/current/static/
# recovery-target-settings.html
target_options = [
'target_tli', 'target_time', 'target_xid', 'target_lsn', 'target_name',
'target_immediate'
]
specified_target_options = len(
[option for option in target_options if getattr(args, option)])
if specified_target_options > 1:
output.error(
"You cannot specify multiple targets for the recovery operation")
output.close_and_exit()
if hasattr(args, 'network_compression'):
if args.network_compression and args.remote_ssh_command is None:
output.error("Network compression can only be used with "
"remote recovery.\n"
"HINT: If you want to do a remote recovery "
"you have to use the --remote-ssh-command option")
output.close_and_exit()
server.config.network_compression = args.network_compression
with closing(server):
try:
server.recover(backup_id,
args.destination_directory,
tablespaces=tablespaces,
target_tli=args.target_tli,
target_time=args.target_time,
target_xid=args.target_xid,
target_lsn=args.target_lsn,
target_name=args.target_name,
target_immediate=args.target_immediate,
exclusive=args.exclusive,
remote_command=args.remote_ssh_command,
target_action=getattr(args, 'target_action', None),
standby_mode=getattr(args, 'standby_mode', None))
except RecoveryException as exc:
output.error(force_str(exc))
output.close_and_exit()