def worker_process_main(self, process_number): """ Repeatedly grab a task from the queue and execute it, until a task containing "None" is grabbed, indicating that the process must stop. :param int process_number: the process number, used in the logging output """ logging.info("Upload process started (worker %s)", process_number) while True: task = self.queue.get() if not task: self.queue.task_done() break try: self.worker_process_execute_job(task, process_number) except Exception as exc: logging.error('Upload error: %s (worker %s)', force_str(exc), process_number) logging.debug('Exception details:', exc_info=exc) self.errors_queue.put(force_str(exc)) except KeyboardInterrupt: if not self.abort_requested: logging.info('Got abort request: upload cancelled ' '(worker %s)', process_number) self.abort_requested = True finally: self.queue.task_done() logging.info("Upload process stopped (worker %s)", process_number)
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging(config) try: conninfo = build_conninfo(config) postgres = PostgreSQLConnection(conninfo, config.immediate_checkpoint, application_name='barman_cloud_backup') try: postgres.connect() except PostgresConnectionError as exc: logging.error("Cannot connect to postgres: %s", force_str(exc)) logging.debug('Exception details:', exc_info=exc) raise SystemExit(1) with closing(postgres): cloud_interface = CloudInterface( destination_url=config.destination_url, encryption=config.encryption, jobs=config.jobs, profile_name=config.profile) if not cloud_interface.test_connectivity(): raise SystemExit(1) # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) with closing(cloud_interface): # TODO: Should the setup be optional? cloud_interface.setup_bucket() uploader = S3BackupUploader(server_name=config.server_name, compression=config.compression, postgres=postgres, cloud_interface=cloud_interface) # Perform the backup uploader.backup() except KeyboardInterrupt as exc: logging.error("Barman cloud backup was interrupted by the user") logging.debug('Exception details:', exc_info=exc) raise SystemExit(1) except Exception as exc: logging.error("Barman cloud backup exception: %s", force_str(exc)) logging.debug('Exception details:', exc_info=exc) raise SystemExit(1)
def reset(self): """ Reset the status of the class. """ self.environment = dict(self.extra_env) config_file = self.backup_manager.config.config.config_file self.environment.update( { "BARMAN_VERSION": version.__version__, "BARMAN_SERVER": self.backup_manager.config.name, "BARMAN_CONFIGURATION": config_file, "BARMAN_HOOK": self.name, "BARMAN_RETRY": str(1 if self.retry else 0), } ) if self.error: self.environment["BARMAN_ERROR"] = force_str(self.error) if self.phase: self.environment["BARMAN_PHASE"] = self.phase script_config_name = "%s_%s" % (self.phase, self.name) else: script_config_name = self.name self.script = getattr(self.backup_manager.config, script_config_name, None) self.exit_status = None self.exception = None
def __str__(self): # Returns the first line if self.args and self.args[0]: from barman.utils import force_str return force_str(self.args[0]).splitlines()[0].strip() else: return ''
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging(config) try: cloud_interface = CloudInterface(url=config.source_url, encryption=config.encryption, profile_name=config.profile, endpoint_url=config.endpoint_url) with closing(cloud_interface): catalog = S3BackupCatalog(cloud_interface=cloud_interface, server_name=config.server_name) if not cloud_interface.test_connectivity(): raise SystemExit(1) # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) if not cloud_interface.bucket_exists: logging.error("Bucket %s does not exist", cloud_interface.bucket_name) raise SystemExit(1) backup_list = catalog.get_backup_list() # Output if config.format == 'console': COLUMNS = "{:<20}{:<25}{:<30}" print(COLUMNS.format("Backup ID", "End Time", "Begin Wal")) for backup_id in sorted(backup_list): item = backup_list[backup_id] if item and item.status == BackupInfo.DONE: print( COLUMNS.format( item.backup_id, item.end_time.strftime("%Y-%m-%d %H:%M:%S"), item.begin_wal)) else: print( json.dumps({ "backups_list": [ backup_list[backup_id].to_json() for backup_id in sorted(backup_list) ] })) except Exception as exc: logging.error("Barman cloud backup list exception: %s", force_str(exc)) logging.debug('Exception details:', exc_info=exc) raise SystemExit(1)
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging(config) # Read wal_path from environment if we're a hook script if __is_hook_script(): if "BARMAN_FILE" not in os.environ: raise BarmanException( "Expected environment variable BARMAN_FILE not set") config.wal_path = os.getenv("BARMAN_FILE") else: if config.wal_path is None: raise BarmanException( "the following arguments are required: wal_path") # Validate the WAL file name before uploading it if not is_any_xlog_file(config.wal_path): logging.error("%s is an invalid name for a WAL file" % config.wal_path) raise CLIErrorExit() try: cloud_interface = get_cloud_interface(config) with closing(cloud_interface): uploader = CloudWalUploader( cloud_interface=cloud_interface, server_name=config.server_name, compression=config.compression, ) if not cloud_interface.test_connectivity(): raise NetworkErrorExit() # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) # TODO: Should the setup be optional? cloud_interface.setup_bucket() upload_kwargs = {} if is_history_file(config.wal_path): upload_kwargs["override_tags"] = config.history_tags uploader.upload_wal(config.wal_path, **upload_kwargs) except Exception as exc: logging.error("Barman cloud WAL archiver exception: %s", force_str(exc)) logging.debug("Exception details:", exc_info=exc) raise GeneralErrorExit()
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging(config) try: cloud_interface = get_cloud_interface(config) if not cloud_interface.test_connectivity(): # Deliberately raise an error if we cannot connect raise NetworkErrorExit() # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) if not cloud_interface.bucket_exists: # If the bucket does not exist then the check should pass return catalog = CloudBackupCatalog(cloud_interface, config.server_name) wals = list(catalog.get_wal_paths().keys()) check_archive_usable( wals, timeline=config.timeline, ) except WalArchiveContentError as err: logging.error( "WAL archive check failed for server %s: %s", config.server_name, force_str(err), ) raise OperationErrorExit() except Exception as exc: logging.error("Barman cloud WAL archive check exception: %s", force_str(exc)) logging.debug("Exception details:", exc_info=exc) raise GeneralErrorExit()
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging(config) try: cloud_interface = get_cloud_interface(config) with closing(cloud_interface): if not cloud_interface.test_connectivity(): raise NetworkErrorExit() # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) if not cloud_interface.bucket_exists: logging.error("Bucket %s does not exist", cloud_interface.bucket_name) raise OperationErrorExit() catalog = CloudBackupCatalog(cloud_interface, config.server_name) if config.release: catalog.release_keep(config.backup_id) elif config.status: target = catalog.get_keep_target(config.backup_id) if target: print("Keep: %s" % target) else: print("Keep: nokeep") else: backup_info = catalog.get_backup_info(config.backup_id) if backup_info.status == BackupInfo.DONE: catalog.keep_backup(config.backup_id, config.target) else: logging.error( "Cannot add keep to backup %s because it has status %s. " "Only backups with status DONE can be kept.", config.backup_id, backup_info.status, ) raise OperationErrorExit() except Exception as exc: logging.error("Barman cloud keep exception: %s", force_str(exc)) logging.debug("Exception details:", exc_info=exc) raise GeneralErrorExit()
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging(config) # Validate the destination directory before starting recovery if os.path.exists(config.recovery_dir) and os.listdir(config.recovery_dir): logging.error( "Destination %s already exists and it is not empty", config.recovery_dir ) raise OperationErrorExit() try: cloud_interface = get_cloud_interface(config) with closing(cloud_interface): downloader = CloudBackupDownloader( cloud_interface=cloud_interface, server_name=config.server_name ) if not cloud_interface.test_connectivity(): raise NetworkErrorExit() # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) if not cloud_interface.bucket_exists: logging.error("Bucket %s does not exist", cloud_interface.bucket_name) raise OperationErrorExit() downloader.download_backup( config.backup_id, config.recovery_dir, tablespace_map(config.tablespace), ) except KeyboardInterrupt as exc: logging.error("Barman cloud restore was interrupted by the user") logging.debug("Exception details:", exc_info=exc) raise OperationErrorExit() except Exception as exc: logging.error("Barman cloud restore exception: %s", force_str(exc)) logging.debug("Exception details:", exc_info=exc) raise GeneralErrorExit()
def decompress(self, src, dst): """ Decompress using the object defined in the subclass :param src: source file to decompress :param dst: destination of the decompression """ try: with closing(self._decompressor(src)) as istream: with open(dst, "wb") as ostream: shutil.copyfileobj(istream, ostream) except Exception as e: # you won't get more information from the compressors anyway raise CommandFailedException( dict(ret=None, err=force_str(e), out=None)) return 0
def list_files(args): """ List all the files for a single backup """ server = get_server(args) # Retrieves the backup backup_info = parse_backup_id(server, args) try: for line in backup_info.get_list_of_files(args.target): output.info(line, log=False) except BadXlogSegmentName as e: output.error( "invalid xlog segment name %r\n" "HINT: Please run \"barman rebuild-xlogdb %s\" " "to solve this issue", force_str(e), server.config.name) output.close_and_exit()
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging(config) # Validate the WAL file name before downloading it if not is_any_xlog_file(config.wal_name): logging.error('%s is an invalid name for a WAL file' % config.wal_name) raise SystemExit(1) try: cloud_interface = CloudInterface( url=config.source_url, encryption=config.encryption, profile_name=config.profile, endpoint_url=config.endpoint_url) with closing(cloud_interface): downloader = S3WalDownloader( cloud_interface=cloud_interface, server_name=config.server_name) if not cloud_interface.test_connectivity(): raise SystemExit(1) # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) if not cloud_interface.bucket_exists: logging.error("Bucket %s does not exist", cloud_interface.bucket_name) raise SystemExit(1) downloader.download_wal(config.wal_name, config.wal_dest) except Exception as exc: logging.error("Barman cloud WAL restore exception: %s", force_str(exc)) logging.debug('Exception details:', exc_info=exc) raise SystemExit(1)
def connectivity_test(config): """ Invoke remote get-wal --test to test the connection with Barman server :param argparse.Namespace config: the configuration from command line """ # Build the peek command ssh_command = build_ssh_command(config, 'dummy_wal_name') # Issue the command try: pipe = subprocess.Popen(ssh_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = pipe.communicate() print(force_str(output[0])) sys.exit(pipe.returncode) except subprocess.CalledProcessError as e: exit_with_error("Impossible to invoke remote get-wal: %s" % e)
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging(config) # Validate the destination directory before starting recovery if os.path.exists(config.recovery_dir) and os.listdir(config.recovery_dir): logging.error("Destination %s already exists and it is not empty", config.recovery_dir) raise SystemExit(1) try: cloud_interface = CloudInterface(url=config.source_url, encryption=config.encryption, profile_name=config.profile, endpoint_url=config.endpoint_url) with closing(cloud_interface): downloader = S3BackupDownloader(cloud_interface=cloud_interface, server_name=config.server_name) if not cloud_interface.test_connectivity(): raise SystemExit(1) # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) if not cloud_interface.bucket_exists: logging.error("Bucket %s does not exist", cloud_interface.bucket_name) raise SystemExit(1) downloader.download_backup(config.backup_id, config.recovery_dir) except Exception as exc: logging.error("Barman cloud backup restore exception: %s", force_str(exc)) logging.debug('Exception details:', exc_info=exc) raise SystemExit(1)
def env_from_recover(self, backup_info, dest, tablespaces, remote_command, error=None, **kwargs): """ Prepare the environment for executing a script :param BackupInfo backup_info: the backup metadata :param str dest: the destination directory :param dict[str,str]|None tablespaces: a tablespace name -> location map (for relocation) :param str|None remote_command: default None. The remote command to recover the base backup, in case of remote backup. :param str|Exception error: An error message in case of failure """ self.env_from_backup_info(backup_info) # Prepare a JSON representation of tablespace map tablespaces_map = "" if tablespaces: tablespaces_map = json.dumps(tablespaces, sort_keys=True) # Prepare a JSON representation of additional recovery options # Skip any empty argument kwargs_filtered = dict([(k, v) for k, v in kwargs.items() if v]) recover_options = "" if kwargs_filtered: recover_options = json.dumps(kwargs_filtered, sort_keys=True) self.environment.update({ "BARMAN_DESTINATION_DIRECTORY": str(dest), "BARMAN_TABLESPACES": tablespaces_map, "BARMAN_REMOTE_COMMAND": str(remote_command or ""), "BARMAN_RECOVER_OPTIONS": recover_options, "BARMAN_ERROR": force_str(error or ""), })
def __parse_tag(tag): """Parse key,value tag with csv reader""" try: rows = list(csv.reader([tag], delimiter=",")) except csv.Error as exc: logging.error( "Error parsing tag %s: %s", tag, force_str(exc), ) raise CLIErrorExit() if len(rows) != 1 or len(rows[0]) != 2: logging.error( "Invalid tag format: %s", tag, ) raise CLIErrorExit() return tuple(rows[0])
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging() # Validate the WAL file name before uploading it if not is_any_xlog_file(config.wal_path): logging.error('%s is an invalid name for a WAL file' % config.wal_path) raise SystemExit(1) try: cloud_interface = CloudInterface( destination_url=config.destination_url, encryption=config.encryption, profile_name=config.profile) with closing(cloud_interface): uploader = S3WalUploader( cloud_interface=cloud_interface, server_name=config.server_name, compression=config.compression) if not cloud_interface.test_connectivity(): raise SystemExit(1) # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) # TODO: Should the setup be optional? cloud_interface.setup_bucket() uploader.upload_wal(config.wal_path) except Exception as exc: logging.error("Barman cloud WAL archiver exception: %s", force_str(exc)) logging.debug('Exception details:', exc_info=exc) raise SystemExit(1)
def __init__(self, server): """ Constructor """ super(BackupManager, self).__init__() self.server = server self.config = server.config self._backup_cache = None self.compression_manager = CompressionManager(self.config, server.path) self.executor = None try: if server.passive_node: self.executor = PassiveBackupExecutor(self) elif self.config.backup_method == "postgres": self.executor = PostgresBackupExecutor(self) else: self.executor = RsyncBackupExecutor(self) except SshCommandException as e: self.config.disabled = True self.config.msg_list.append(force_str(e).strip())
def from_command_error(cls, cmd, e, msg): """ This method build a DataTransferFailure exception and report the provided message to the user (both console and log file) along with the output of the failed command. :param str cmd: The command that failed the transfer :param CommandFailedException e: The exception we are handling :param str msg: a descriptive message on what we are trying to do :return DataTransferFailure: will contain the message provided in msg """ try: details = msg details += "\n%s error:\n" % cmd details += e.args[0]['out'] details += e.args[0]['err'] return cls(details) except (TypeError, NameError): # If it is not a dictionary just convert it to a string from barman.utils import force_str return cls(force_str(e.args))
def handle_backup_errors(self, action, backup_info, exc): """ Mark the backup as failed and exit :param str action: the upload phase that has failed :param barman.infofile.BackupInfo backup_info: the backup info file :param BaseException exc: the exception that caused the failure """ msg_lines = force_str(exc).strip().splitlines() # If the exception has no attached message use the raw # type name if len(msg_lines) == 0: msg_lines = [type(exc).__name__] if backup_info: # Use only the first line of exception message # in backup_info error field backup_info.set_attribute("status", "FAILED") backup_info.set_attribute( "error", "failure %s (%s)" % (action, msg_lines[0])) logging.error("Backup failed %s (%s)", action, msg_lines[0]) logging.debug('Exception details:', exc_info=exc)
def env_from_wal_info(self, wal_info, full_path=None, error=None): """ Prepare the environment for executing a script :param WalFileInfo wal_info: the backup metadata :param str full_path: override wal_info.fullpath() result :param str|Exception error: An error message in case of failure """ self.environment.update( { "BARMAN_SEGMENT": wal_info.name, "BARMAN_FILE": str( full_path if full_path is not None else wal_info.fullpath(self.backup_manager.server) ), "BARMAN_SIZE": str(wal_info.size), "BARMAN_TIMESTAMP": str(wal_info.time), "BARMAN_COMPRESSION": wal_info.compression or "", "BARMAN_ERROR": force_str(error or ""), } )
def _delete_backup( cloud_interface, catalog, backup_id, dry_run=True, skip_wal_cleanup_if_standalone=True, ): backup_info = catalog.get_backup_info(backup_id) if not backup_info: logging.warning("Backup %s does not exist", backup_id) return objects_to_delete = _get_files_for_backup(catalog, backup_info) backup_info_path = os.path.join( catalog.prefix, backup_info.backup_id, "backup.info" ) logging.debug("Will delete backup.info file at %s" % backup_info_path) if not dry_run: try: cloud_interface.delete_objects(objects_to_delete) # Do not try to delete backup.info until we have successfully deleted # everything else so that it is possible to retry the operation should # we fail to delete any backup file cloud_interface.delete_objects([backup_info_path]) except Exception as exc: logging.error("Could not delete backup %s: %s", backup_id, force_str(exc)) raise SystemExit(2) else: print( "Skipping deletion of objects %s due to --dry-run option" % (objects_to_delete + [backup_info_path]) ) _remove_wals_for_backup( cloud_interface, catalog, backup_info, dry_run, skip_wal_cleanup_if_standalone ) # It is important that the backup is removed from the catalog after cleaning # up the WALs because the code in _remove_wals_for_backup depends on the # deleted backup existing in the backup catalog catalog.remove_backup_from_cache(backup_id)
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging(config) tempdir = tempfile.mkdtemp(prefix='barman-cloud-backup-') try: # Create any temporary file in the `tempdir` subdirectory tempfile.tempdir = tempdir conninfo = build_conninfo(config) postgres = PostgreSQLConnection(conninfo, config.immediate_checkpoint, application_name='barman_cloud_backup') try: postgres.connect() except PostgresConnectionError as exc: logging.error("Cannot connect to postgres: %s", force_str(exc)) logging.debug('Exception details:', exc_info=exc) raise SystemExit(1) with closing(postgres): cloud_interface = CloudInterface( url=config.destination_url, encryption=config.encryption, jobs=config.jobs, profile_name=config.profile, endpoint_url=config.endpoint_url) if not cloud_interface.test_connectivity(): raise SystemExit(1) # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) with closing(cloud_interface): # TODO: Should the setup be optional? cloud_interface.setup_bucket() uploader = S3BackupUploader( server_name=config.server_name, compression=config.compression, postgres=postgres, max_archive_size=config.max_archive_size, cloud_interface=cloud_interface) # Perform the backup uploader.backup() except KeyboardInterrupt as exc: logging.error("Barman cloud backup was interrupted by the user") logging.debug('Exception details:', exc_info=exc) raise SystemExit(1) except Exception as exc: logging.error("Barman cloud backup exception: %s", force_str(exc)) logging.debug('Exception details:', exc_info=exc) raise SystemExit(1) finally: # Remove the temporary directory and all the contained files rmtree(tempdir, ignore_errors=True)
def _remove_wals_for_backup( cloud_interface, catalog, deleted_backup, dry_run, skip_wal_cleanup_if_standalone=True, ): # An implementation of BackupManager.remove_wal_before_backup which does not # use xlogdb, since xlogdb is not available to barman-cloud should_remove_wals, wal_ranges_to_protect = BackupManager.should_remove_wals( deleted_backup, catalog.get_backup_list(), keep_manager=catalog, skip_wal_cleanup_if_standalone=skip_wal_cleanup_if_standalone, ) next_backup = BackupManager.find_next_backup_in( catalog.get_backup_list(), deleted_backup.backup_id ) wals_to_delete = {} if should_remove_wals: # There is no previous backup or all previous backups are archival # standalone backups, so we can remove unused WALs (those WALs not # required by standalone archival backups). # If there is a next backup then all unused WALs up to the begin_wal # of the next backup can be removed. # If there is no next backup then there are no remaining backups, # because we must assume non-exclusive backups are taken, we can only # safely delete unused WALs up to begin_wal of the deleted backup. # See comments in barman.backup.BackupManager.delete_backup. if next_backup: remove_until = next_backup else: remove_until = deleted_backup # A WAL is only a candidate for deletion if it is on the same timeline so we # use BackupManager to get a set of all other timelines with backups so that # we can preserve all WALs on other timelines. timelines_to_protect = BackupManager.get_timelines_to_protect( remove_until=remove_until, deleted_backup=deleted_backup, available_backups=catalog.get_backup_list(), ) try: wal_paths = catalog.get_wal_paths() except Exception as exc: logging.error( "Cannot clean up WALs for backup %s because an error occurred listing WALs: %s", deleted_backup.backup_id, force_str(exc), ) return for wal_name, wal in wal_paths.items(): if xlog.is_history_file(wal_name): continue if timelines_to_protect: tli, _, _ = xlog.decode_segment_name(wal_name) if tli in timelines_to_protect: continue # Check if the WAL is in a protected range, required by an archival # standalone backup - so do not delete it if xlog.is_backup_file(wal_name): # If we have a backup file, truncate the name for the range check range_check_wal_name = wal_name[:24] else: range_check_wal_name = wal_name if any( range_check_wal_name >= begin_wal and range_check_wal_name <= end_wal for begin_wal, end_wal in wal_ranges_to_protect ): continue if wal_name < remove_until.begin_wal: wals_to_delete[wal_name] = wal # Explicitly sort because dicts are not ordered in python < 3.6 wal_paths_to_delete = sorted(wals_to_delete.values()) if len(wal_paths_to_delete) > 0: if not dry_run: try: cloud_interface.delete_objects(wal_paths_to_delete) except Exception as exc: logging.error( "Could not delete the following WALs for backup %s: %s, Reason: %s", deleted_backup.backup_id, wal_paths_to_delete, force_str(exc), ) # Return early so that we leave the WALs in the local cache so they # can be cleaned up should there be a subsequent backup deletion. return else: print( "Skipping deletion of objects %s due to --dry-run option" % wal_paths_to_delete ) for wal_name in wals_to_delete.keys(): catalog.remove_wal_from_cache(wal_name)
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging(config) try: cloud_interface = get_cloud_interface(config) with closing(cloud_interface): if not cloud_interface.test_connectivity(): raise SystemExit(1) # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) if not cloud_interface.bucket_exists: logging.error("Bucket %s does not exist", cloud_interface.bucket_name) raise SystemExit(1) catalog = CloudBackupCatalog( cloud_interface=cloud_interface, server_name=config.server_name ) # Call catalog.get_backup_list now so we know we can read the whole catalog # (the results are cached so this does not result in extra calls to cloud # storage) catalog.get_backup_list() if len(catalog.unreadable_backups) > 0: logging.error( "Cannot read the following backups: %s\n" "Unsafe to proceed with deletion due to failure reading backup catalog" % catalog.unreadable_backups ) raise SystemExit(1) if config.backup_id: # Because we only care about one backup, skip the annotation cache # because it is only helpful when dealing with multiple backups if catalog.should_keep_backup(config.backup_id, use_cache=False): logging.error( "Skipping delete of backup %s for server %s " "as it has a current keep request. If you really " "want to delete this backup please remove the keep " "and try again.", config.backup_id, config.server_name, ) raise SystemExit(1) _delete_backup( cloud_interface, catalog, config.backup_id, config.dry_run ) elif config.retention_policy: retention_policy = RetentionPolicyFactory.create( "retention_policy", config.retention_policy, server_name=config.server_name, catalog=catalog, ) # Sort to ensure that we delete the backups in ascending order, that is # from oldest to newest. This ensures that the relevant WALs will be cleaned # up after each backup is deleted. backups_to_delete = sorted( [ backup_id for backup_id, status in retention_policy.report().items() if status == "OBSOLETE" ] ) for backup_id in backups_to_delete: _delete_backup( cloud_interface, catalog, backup_id, config.dry_run, skip_wal_cleanup_if_standalone=False, ) except Exception as exc: logging.error("Barman cloud backup delete exception: %s", force_str(exc)) logging.debug("Exception details:", exc_info=exc) raise SystemExit(1)
def backup(self, wait=False, wait_timeout=None): """ Performs a backup for the server :param bool wait: wait for all the required WAL files to be archived :param int|None wait_timeout: :return BackupInfo: the generated BackupInfo """ _logger.debug("initialising backup information") self.executor.init() backup_info = None try: # Create the BackupInfo object representing the backup backup_info = LocalBackupInfo( self.server, backup_id=datetime.datetime.now().strftime('%Y%m%dT%H%M%S')) backup_info.set_attribute('systemid', self.server.systemid) backup_info.save() self.backup_cache_add(backup_info) output.info("Starting backup using %s method for server %s in %s", self.mode, self.config.name, backup_info.get_basebackup_directory()) # Run the pre-backup-script if present. script = HookScriptRunner(self, 'backup_script', 'pre') script.env_from_backup_info(backup_info) script.run() # Run the pre-backup-retry-script if present. retry_script = RetryHookScriptRunner(self, 'backup_retry_script', 'pre') retry_script.env_from_backup_info(backup_info) retry_script.run() # Do the backup using the BackupExecutor self.executor.backup(backup_info) # Create a restore point after a backup target_name = 'barman_%s' % backup_info.backup_id self.server.postgres.create_restore_point(target_name) # Free the Postgres connection self.server.postgres.close() # Compute backup size and fsync it on disk self.backup_fsync_and_set_sizes(backup_info) # Mark the backup as WAITING_FOR_WALS backup_info.set_attribute("status", BackupInfo.WAITING_FOR_WALS) # Use BaseException instead of Exception to catch events like # KeyboardInterrupt (e.g.: CTRL-C) except BaseException as e: msg_lines = force_str(e).strip().splitlines() # If the exception has no attached message use the raw # type name if len(msg_lines) == 0: msg_lines = [type(e).__name__] if backup_info: # Use only the first line of exception message # in backup_info error field backup_info.set_attribute("status", BackupInfo.FAILED) backup_info.set_attribute( "error", "failure %s (%s)" % (self.executor.current_action, msg_lines[0])) output.error("Backup failed %s.\nDETAILS: %s", self.executor.current_action, '\n'.join(msg_lines)) else: output.info("Backup end at LSN: %s (%s, %08X)", backup_info.end_xlog, backup_info.end_wal, backup_info.end_offset) executor = self.executor output.info( "Backup completed (start time: %s, elapsed time: %s)", self.executor.copy_start_time, human_readable_timedelta(datetime.datetime.now() - executor.copy_start_time)) # If requested, wait for end_wal to be archived if wait: try: self.server.wait_for_wal(backup_info.end_wal, wait_timeout) self.check_backup(backup_info) except KeyboardInterrupt: # Ignore CTRL-C pressed while waiting for WAL files output.info( "Got CTRL-C. Continuing without waiting for '%s' " "to be archived", backup_info.end_wal) finally: if backup_info: backup_info.save() # Make sure we are not holding any PostgreSQL connection # during the post-backup scripts self.server.close() # Run the post-backup-retry-script if present. try: retry_script = RetryHookScriptRunner( self, 'backup_retry_script', 'post') retry_script.env_from_backup_info(backup_info) retry_script.run() except AbortedRetryHookScript as e: # Ignore the ABORT_STOP as it is a post-hook operation _logger.warning( "Ignoring stop request after receiving " "abort (exit code %d) from post-backup " "retry hook script: %s", e.hook.exit_status, e.hook.script) # Run the post-backup-script if present. script = HookScriptRunner(self, 'backup_script', 'post') script.env_from_backup_info(backup_info) script.run() output.result('backup', backup_info) return backup_info
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging(config) tempdir = tempfile.mkdtemp(prefix="barman-cloud-backup-") try: # Create any temporary file in the `tempdir` subdirectory tempfile.tempdir = tempdir cloud_interface = get_cloud_interface(config) if not cloud_interface.test_connectivity(): raise NetworkErrorExit() # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) with closing(cloud_interface): # TODO: Should the setup be optional? cloud_interface.setup_bucket() # Perform the backup uploader_kwargs = { "server_name": config.server_name, "compression": config.compression, "max_archive_size": config.max_archive_size, "cloud_interface": cloud_interface, } if __is_hook_script(): if "BARMAN_BACKUP_DIR" not in os.environ: raise BarmanException( "BARMAN_BACKUP_DIR environment variable not set") if "BARMAN_BACKUP_ID" not in os.environ: raise BarmanException( "BARMAN_BACKUP_ID environment variable not set") if os.getenv("BARMAN_STATUS") != "DONE": raise UnrecoverableHookScriptError( "backup in '%s' has status '%s' (status should be: DONE)" % (os.getenv("BARMAN_BACKUP_DIR"), os.getenv("BARMAN_STATUS"))) uploader = CloudBackupUploaderBarman( backup_dir=os.getenv("BARMAN_BACKUP_DIR"), backup_id=os.getenv("BARMAN_BACKUP_ID"), **uploader_kwargs) uploader.backup() else: conninfo = build_conninfo(config) postgres = PostgreSQLConnection( conninfo, config.immediate_checkpoint, application_name="barman_cloud_backup", ) try: postgres.connect() except PostgresConnectionError as exc: logging.error("Cannot connect to postgres: %s", force_str(exc)) logging.debug("Exception details:", exc_info=exc) raise OperationErrorExit() with closing(postgres): uploader = CloudBackupUploaderPostgres(postgres=postgres, **uploader_kwargs) uploader.backup() except KeyboardInterrupt as exc: logging.error("Barman cloud backup was interrupted by the user") logging.debug("Exception details:", exc_info=exc) raise OperationErrorExit() except UnrecoverableHookScriptError as exc: logging.error("Barman cloud backup exception: %s", force_str(exc)) logging.debug("Exception details:", exc_info=exc) raise SystemExit(63) except Exception as exc: logging.error("Barman cloud backup exception: %s", force_str(exc)) logging.debug("Exception details:", exc_info=exc) raise GeneralErrorExit() finally: # Remove the temporary directory and all the contained files rmtree(tempdir, ignore_errors=True)
def main(args=None): """ The main script entry point :param list[str] args: the raw arguments list. When not provided it defaults to sys.args[1:] """ config = parse_arguments(args) configure_logging(config) try: cloud_interface = get_cloud_interface(config) with closing(cloud_interface): catalog = CloudBackupCatalog(cloud_interface=cloud_interface, server_name=config.server_name) if not cloud_interface.test_connectivity(): raise NetworkErrorExit() # If test is requested, just exit after connectivity test elif config.test: raise SystemExit(0) if not cloud_interface.bucket_exists: logging.error("Bucket %s does not exist", cloud_interface.bucket_name) raise OperationErrorExit() backup_list = catalog.get_backup_list() # Output if config.format == "console": COLUMNS = "{:<20}{:<25}{:<30}{:<16}" print( COLUMNS.format("Backup ID", "End Time", "Begin Wal", "Archival Status")) for backup_id in sorted(backup_list): item = backup_list[backup_id] if item and item.status == BackupInfo.DONE: keep_target = catalog.get_keep_target(item.backup_id) keep_status = (keep_target and "KEEP:%s" % keep_target.upper() or "") print( COLUMNS.format( item.backup_id, item.end_time.strftime("%Y-%m-%d %H:%M:%S"), item.begin_wal, keep_status, )) else: print( json.dumps({ "backups_list": [ backup_list[backup_id].to_json() for backup_id in sorted(backup_list) ] })) except Exception as exc: logging.error("Barman cloud backup list exception: %s", force_str(exc)) logging.debug("Exception details:", exc_info=exc) raise GeneralErrorExit()
def recover(self, backup_info, dest, tablespaces=None, remote_command=None, target_tli=None, target_time=None, target_xid=None, target_name=None, target_immediate=False, exclusive=False, target_action=None, standby_mode=None): """ Performs a recovery of a backup This method should be called in a closing context :param barman.infofile.BackupInfo backup_info: the backup to recover :param str dest: the destination directory :param dict[str,str]|None tablespaces: a tablespace name -> location map (for relocation) :param str|None remote_command: The remote command to recover the base backup, in case of remote backup. :param str|None target_tli: the target timeline :param str|None target_time: the target time :param str|None target_xid: the target xid :param str|None target_name: the target name created previously with pg_create_restore_point() function call :param str|None target_immediate: end recovery as soon as consistency is reached :param bool exclusive: whether the recovery is exclusive or not :param str|None target_action: The recovery target action :param bool|None standby_mode: standby mode """ # Run the cron to be sure the wal catalog is up to date # Prepare a map that contains all the objects required for a recovery recovery_info = self._setup(backup_info, remote_command, dest) output.info("Starting %s restore for server %s using backup %s", recovery_info['recovery_dest'], self.server.config.name, backup_info.backup_id) output.info("Destination directory: %s", dest) if remote_command: output.info("Remote command: %s", remote_command) # If the backup we are recovering is still not validated and we # haven't requested the get-wal feature, display a warning message if not recovery_info['get_wal']: if backup_info.status == BackupInfo.WAITING_FOR_WALS: output.warning( "IMPORTANT: You have requested a recovery operation for " "a backup that does not have yet all the WAL files that " "are required for consistency.") # Set targets for PITR self._set_pitr_targets(recovery_info, backup_info, dest, target_name, target_time, target_tli, target_xid, target_immediate, target_action) # Retrieve the safe_horizon for smart copy self._retrieve_safe_horizon(recovery_info, backup_info, dest) # check destination directory. If doesn't exist create it try: recovery_info['cmd'].create_dir_if_not_exists(dest) except FsOperationFailed as e: output.error( "unable to initialise destination directory " "'%s': %s", dest, e) output.close_and_exit() # Initialize tablespace directories if backup_info.tablespaces: self._prepare_tablespaces(backup_info, recovery_info['cmd'], dest, tablespaces) # Copy the base backup output.info("Copying the base backup.") try: self._backup_copy(backup_info, dest, tablespaces, remote_command, recovery_info['safe_horizon']) except DataTransferFailure as e: output.error("Failure copying base backup: %s", e) output.close_and_exit() # Copy the backup.info file in the destination as # ".barman-recover.info" if remote_command: try: recovery_info['rsync'](backup_info.filename, ':%s/.barman-recover.info' % dest) except CommandFailedException as e: output.error('copy of recovery metadata file failed: %s', e) output.close_and_exit() else: backup_info.save(os.path.join(dest, '.barman-recover.info')) # Standby mode is not available for PostgreSQL older than 9.0 if backup_info.version < 90000 and standby_mode: raise RecoveryStandbyModeException( 'standby_mode is available only from PostgreSQL 9.0') # Restore the WAL segments. If GET_WAL option is set, skip this phase # as they will be retrieved using the wal-get command. if not recovery_info['get_wal']: # If the backup we restored is still waiting for WALS, read the # backup info again and check whether it has been validated. # Notify the user if it is still not DONE. if backup_info.status == BackupInfo.WAITING_FOR_WALS: data = BackupInfo(self.server, backup_info.filename) if data.status == BackupInfo.WAITING_FOR_WALS: output.warning( "IMPORTANT: The backup we have recovered IS NOT " "VALID. Required WAL files for consistency are " "missing. Please verify that WAL archiving is " "working correctly or evaluate using the 'get-wal' " "option for recovery") output.info("Copying required WAL segments.") try: # Retrieve a list of required log files required_xlog_files = tuple( self.server.get_required_xlog_files( backup_info, target_tli, recovery_info['target_epoch'])) # Restore WAL segments into the wal_dest directory self._xlog_copy(required_xlog_files, recovery_info['wal_dest'], remote_command) except DataTransferFailure as e: output.error("Failure copying WAL files: %s", e) output.close_and_exit() except BadXlogSegmentName as e: output.error( "invalid xlog segment name %r\n" "HINT: Please run \"barman rebuild-xlogdb %s\" " "to solve this issue", force_str(e), self.config.name) output.close_and_exit() # If WAL files are put directly in the pg_xlog directory, # avoid shipping of just recovered files # by creating the corresponding archive status file if not recovery_info['is_pitr']: output.info("Generating archive status files") self._generate_archive_status(recovery_info, remote_command, required_xlog_files) # Generate recovery.conf file (only if needed by PITR or get_wal) is_pitr = recovery_info['is_pitr'] get_wal = recovery_info['get_wal'] if is_pitr or get_wal or standby_mode: output.info("Generating recovery.conf") self._generate_recovery_conf(recovery_info, backup_info, dest, target_immediate, exclusive, remote_command, target_name, target_time, target_tli, target_xid, standby_mode) # Create archive_status directory if necessary archive_status_dir = os.path.join(recovery_info['wal_dest'], 'archive_status') try: recovery_info['cmd'].create_dir_if_not_exists(archive_status_dir) except FsOperationFailed as e: output.error( "unable to create the archive_status directory " "'%s': %s", archive_status_dir, e) output.close_and_exit() # As last step, analyse configuration files in order to spot # harmful options. Barman performs automatic conversion of # some options as well as notifying users of their existence. # # This operation is performed in three steps: # 1) mapping # 2) analysis # 3) copy output.info("Identify dangerous settings in destination directory.") self._map_temporary_config_files(recovery_info, backup_info, remote_command) self._analyse_temporary_config_files(recovery_info) self._copy_temporary_config_files(dest, remote_command, recovery_info) return recovery_info
def recover(args): """ Recover a server at a given time, name, LSN or xid """ server = get_server(args) # Retrieves the backup backup_id = parse_backup_id(server, args) if backup_id.status not in BackupInfo.STATUS_COPY_DONE: output.error( "Cannot recover from backup '%s' of server '%s': " "backup status is not DONE", args.backup_id, server.config.name) output.close_and_exit() # decode the tablespace relocation rules tablespaces = {} if args.tablespace: for rule in args.tablespace: try: tablespaces.update([rule.split(':', 1)]) except ValueError: output.error( "Invalid tablespace relocation rule '%s'\n" "HINT: The valid syntax for a relocation rule is " "NAME:LOCATION", rule) output.close_and_exit() # validate the rules against the tablespace list valid_tablespaces = [] if backup_id.tablespaces: valid_tablespaces = [ tablespace_data.name for tablespace_data in backup_id.tablespaces ] for item in tablespaces: if item not in valid_tablespaces: output.error( "Invalid tablespace name '%s'\n" "HINT: Please use any of the following " "tablespaces: %s", item, ', '.join(valid_tablespaces)) output.close_and_exit() # explicitly disallow the rsync remote syntax (common mistake) if ':' in args.destination_directory: output.error( "The destination directory parameter " "cannot contain the ':' character\n" "HINT: If you want to do a remote recovery you have to use " "the --remote-ssh-command option") output.close_and_exit() if args.retry_sleep is not None: server.config.basebackup_retry_sleep = args.retry_sleep if args.retry_times is not None: server.config.basebackup_retry_times = args.retry_times if hasattr(args, 'get_wal'): if args.get_wal: server.config.recovery_options.add(RecoveryOptions.GET_WAL) else: server.config.recovery_options.remove(RecoveryOptions.GET_WAL) if args.jobs is not None: server.config.parallel_jobs = args.jobs if hasattr(args, 'bwlimit'): server.config.bandwidth_limit = args.bwlimit # PostgreSQL supports multiple parameters to specify when the recovery # process will end, and in that case the last entry in recovery # configuration files will be used. See [1] # # Since the meaning of the target options is not dependent on the order # of parameters, we decided to make the target options mutually exclusive. # # [1]: https://www.postgresql.org/docs/current/static/ # recovery-target-settings.html target_options = [ 'target_tli', 'target_time', 'target_xid', 'target_lsn', 'target_name', 'target_immediate' ] specified_target_options = len( [option for option in target_options if getattr(args, option)]) if specified_target_options > 1: output.error( "You cannot specify multiple targets for the recovery operation") output.close_and_exit() if hasattr(args, 'network_compression'): if args.network_compression and args.remote_ssh_command is None: output.error("Network compression can only be used with " "remote recovery.\n" "HINT: If you want to do a remote recovery " "you have to use the --remote-ssh-command option") output.close_and_exit() server.config.network_compression = args.network_compression with closing(server): try: server.recover(backup_id, args.destination_directory, tablespaces=tablespaces, target_tli=args.target_tli, target_time=args.target_time, target_xid=args.target_xid, target_lsn=args.target_lsn, target_name=args.target_name, target_immediate=args.target_immediate, exclusive=args.exclusive, remote_command=args.remote_ssh_command, target_action=getattr(args, 'target_action', None), standby_mode=getattr(args, 'standby_mode', None)) except RecoveryException as exc: output.error(force_str(exc)) output.close_and_exit()