def main(argv=None): if argv is None: argv = sys.argv parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=__doc__) parser.add_argument('-k', '--aws-access-key-id', help='public AWS access key. Can also be defined in ' 'an environment variable. If both are defined, ' 'the one defined in the programs arguments takes ' 'precedence.') parser.add_argument('--s3-prefix', help='S3 prefix to run all commands against. ' 'Can also be defined via environment variable ' 'WALE_S3_PREFIX') parser.add_argument( '--gpg-key-id', help='GPG key ID to encrypt to. (Also needed when decrypting.) ' 'Can also be defined via environment variable ' 'WALE_GPG_KEY_ID') subparsers = parser.add_subparsers(title='subcommands', dest='subcommand') # Common arguments for backup-fetch and backup-push backup_fetchpush_parent = argparse.ArgumentParser(add_help=False) backup_fetchpush_parent.add_argument('PG_CLUSTER_DIRECTORY', help="Postgres cluster path, " "such as '/var/lib/database'") backup_fetchpush_parent.add_argument('--pool-size', '-p', type=int, default=4, help='Download pooling size') # Common arguments for archive-fetch and archive-push archive_fetchpush_parent = argparse.ArgumentParser(add_help=False) archive_fetchpush_parent.add_argument('ARCHIVE_FILENAME', help="Database archive file") archive_fetchpush_parent.add_argument('--pool-size', '-p', type=int, default=4, help='Download pooling size') # operator to print the wal-e version subparsers.add_parser('version', help='print the wal-e version') # Common arguments for backup-list and backup-fetch # # NB: This does not include the --detail options because some # other commands use backup listing functionality in a way where # --detail is never required. backup_list_nodetail_parent = argparse.ArgumentParser(add_help=False) # Common arguments between wal-push and wal-fetch wal_fetchpush_parent = argparse.ArgumentParser(add_help=False) wal_fetchpush_parent.add_argument('WAL_SEGMENT', help='Path to a WAL segment to upload') backup_fetch_parser = subparsers.add_parser( 'backup-fetch', help='fetch a hot backup from S3', parents=[backup_fetchpush_parent, backup_list_nodetail_parent]) backup_list_parser = subparsers.add_parser( 'backup-list', parents=[backup_list_nodetail_parent], help='list backups in S3') backup_push_parser = subparsers.add_parser( 'backup-push', help='pushing a fresh hot backup to S3', parents=[backup_fetchpush_parent]) archive_push_parser = subparsers.add_parser( 'archive-push', help='pushing archive file to S3', parents=[archive_fetchpush_parent]) backup_push_parser.add_argument( '--cluster-read-rate-limit', help='Rate limit reading the PostgreSQL cluster directory to a ' 'tunable number of bytes per second', dest='rate_limit', metavar='BYTES_PER_SECOND', type=int, default=None) backup_push_parser.add_argument( '--while-offline', help=('Backup a Postgres cluster that is in a stopped state ' '(for example, a replica that you stop and restart ' 'when taking a backup)'), dest='while_offline', action='store_true', default=False) archive_push_parser.add_argument( '--cluster-read-rate-limit', help='Rate limit reading the archive to a ' 'tunable number of bytes per second', dest='rate_limit', metavar='BYTES_PER_SECOND', type=int, default=None) wal_fetch_parser = subparsers.add_parser('wal-fetch', help='fetch a WAL file from S3', parents=[wal_fetchpush_parent]) subparsers.add_parser('wal-push', help='push a WAL file to S3', parents=[wal_fetchpush_parent]) # backup-fetch operator section backup_fetch_parser.add_argument('BACKUP_NAME', help='the name of the backup to fetch') # backup-list operator section backup_list_parser.add_argument('QUERY', nargs='?', default=None, help='a string qualifying backups to list') backup_list_parser.add_argument( '--detail', default=False, action='store_true', help='show more detailed information about every backup') # wal-push operator section wal_fetch_parser.add_argument('WAL_DESTINATION', help='Path to download the WAL segment to') # delete subparser section delete_parser = subparsers.add_parser( 'delete', help=('operators to destroy specified data in S3')) delete_parser.add_argument('--dry-run', '-n', action='store_true', help=('Only print what would be deleted, ' 'do not actually delete anything')) delete_parser.add_argument('--confirm', action='store_true', help=('Actually delete data. ' 'By default, a dry run is performed. ' 'Overridden by --dry-run.')) delete_subparsers = delete_parser.add_subparsers( title='delete subcommands', description=('All operators that may delete data are contained ' 'in this subcommand.'), dest='delete_subcommand') # delete 'before' operator delete_before_parser = delete_subparsers.add_parser( 'before', help=('Delete all backups and WAL segments strictly before ' 'the given base backup name or WAL segment number. ' 'The passed backup is *not* deleted.')) delete_before_parser.add_argument( 'BEFORE_SEGMENT_EXCLUSIVE', help='A WAL segment number or base backup name') # delete old versions operator delete_subparsers.add_parser( 'old-versions', help=('Delete all old versions of WAL-E backup files. One probably ' 'wants to ensure that they take a new backup with the new ' 'format first. ' 'This is useful after a WAL-E major release upgrade.')) # delete *everything* operator delete_subparsers.add_parser( 'everything', help=('Delete all data in the current WAL-E context. ' 'Typically this is only appropriate when decommissioning an ' 'entire WAL-E archive.')) # Okay, parse some arguments, finally args = parser.parse_args() subcommand = args.subcommand # Handle version printing specially, because it doesn't need # credentials. if subcommand == 'version': import pkgutil print pkgutil.get_data('wal_e', 'VERSION').strip() sys.exit(0) # Attempt to read a few key parameters from environment variables # *or* the command line, enforcing a precedence order and # complaining should the required parameter not be defined in # either location. secret_key = os.getenv('AWS_SECRET_ACCESS_KEY') if secret_key is None: logger.error( msg='no AWS_SECRET_ACCESS_KEY defined', hint='Define the environment variable AWS_SECRET_ACCESS_KEY.') sys.exit(1) s3_prefix = args.s3_prefix or os.getenv('WALE_S3_PREFIX') if s3_prefix is None: logger.error(msg='no storage prefix defined', hint=('Either set the --s3-prefix option or define the ' 'environment variable WALE_S3_PREFIX.')) sys.exit(1) if args.aws_access_key_id is None: aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID') if aws_access_key_id is None: logger.error( msg='no storage prefix defined', hint=('Either set the --aws-access-key-id option or define ' 'the environment variable AWS_ACCESS_KEY_ID.')) sys.exit(1) else: aws_access_key_id = args.aws_access_key_id # This will be None if we're not encrypting gpg_key_id = args.gpg_key_id or os.getenv('WALE_GPG_KEY_ID') backup_cxt = s3_operator.S3Backup(aws_access_key_id, secret_key, s3_prefix, gpg_key_id) if gpg_key_id is not None: external_program_check([GPG_BIN]) try: if subcommand == 'backup-fetch': external_program_check([LZOP_BIN]) backup_cxt.database_s3_fetch(args.PG_CLUSTER_DIRECTORY, args.BACKUP_NAME, pool_size=args.pool_size) elif subcommand == 'backup-list': backup_cxt.backup_list(query=args.QUERY, detail=args.detail) elif subcommand == 'backup-push': if args.while_offline: # we need to query pg_config first for the # pg_controldata's bin location external_program_check([CONFIG_BIN]) parser = PgControlDataParser(args.PG_CLUSTER_DIRECTORY) controldata_bin = parser.controldata_bin() external_programs = [LZOP_BIN, PV_BIN, controldata_bin] else: external_programs = [LZOP_BIN, PSQL_BIN, PV_BIN] external_program_check(external_programs) rate_limit = args.rate_limit while_offline = args.while_offline backup_cxt.database_s3_backup(args.PG_CLUSTER_DIRECTORY, rate_limit=rate_limit, while_offline=while_offline, pool_size=args.pool_size) elif subcommand == 'archive-push': external_programs = [PV_BIN] external_program_check(external_programs) rate_limit = args.rate_limit backup_cxt.archive_s3_upload(args.ARCHIVE_FILENAME, rate_limit=rate_limit, pool_size=args.pool_size) elif subcommand == 'wal-fetch': external_program_check([LZOP_BIN]) res = backup_cxt.wal_s3_restore(args.WAL_SEGMENT, args.WAL_DESTINATION) if not res: sys.exit(1) elif subcommand == 'wal-push': external_program_check([LZOP_BIN]) backup_cxt.wal_s3_archive(args.WAL_SEGMENT) elif subcommand == 'delete': # Set up pruning precedence, optimizing for *not* deleting data # # Canonicalize the passed arguments into the value # "is_dry_run_really" if args.dry_run is False and args.confirm is True: # Actually delete data *only* if there are *no* --dry-runs # present and --confirm is present. logger.info(msg='deleting data in S3') is_dry_run_really = False else: logger.info(msg='performing dry run of S3 data deletion') is_dry_run_really = True import boto.s3.key import boto.s3.Bucket # This is not necessary, but "just in case" to find bugs. def just_error(*args, **kwargs): assert False, ('About to delete something in ' 'dry-run mode. Please report a bug.') boto.s3.key.Key.delete = just_error boto.s3.bucket.Bucket.delete_keys = just_error # Handle the subcommands and route them to the right # implementations. if args.delete_subcommand == 'old-versions': backup_cxt.delete_old_versions(is_dry_run_really) elif args.delete_subcommand == 'everything': backup_cxt.delete_all(is_dry_run_really) elif args.delete_subcommand == 'before': segment_info = extract_segment(args.BEFORE_SEGMENT_EXCLUSIVE) backup_cxt.delete_before(is_dry_run_really, segment_info) else: assert False, 'Should be rejected by argument parsing.' else: logger.error( msg='subcommand not implemented', detail=( 'The submitted subcommand was {0}.'.format(subcommand)), hint='Check for typos or consult wal-e --help.') sys.exit(127) # Report on all encountered exceptions, and raise the last one # to take advantage of the final catch-all reporting and exit # code management. if backup_cxt.exceptions: for exc in backup_cxt.exceptions[:-1]: logger.log(level=exc.severity, msg=log_help.WalELogger.fmt_logline( exc.msg, exc.detail, exc.hint)) raise backup_cxt.exceptions[-1] except UserException, e: logger.log(level=e.severity, msg=log_help.WalELogger.fmt_logline(e.msg, e.detail, e.hint)) sys.exit(1)
def configure_backup_cxt(args): # Try to find some WAL-E prefix to store data in. prefix = (args.file_prefix or args.gs_prefix or args.s3_prefix or args.wabs_prefix or os.getenv('WALE_FILE_PREFIX') or os.getenv('WALE_GS_PREFIX') or os.getenv('WALE_S3_PREFIX') or os.getenv('WALE_SWIFT_PREFIX') or os.getenv('WALE_WABS_PREFIX')) if prefix is None: raise UserException(msg='no storage prefix defined', hint=('Either set one of the' ' --file-prefix,' ' --gs-prefix,' ' --s3-prefix or' ' --wabs-prefix options' ' or define one of the' ' WALE_FILE_PREFIX,' ' WALE_GS_PREFIX,' ' WALE_S3_PREFIX,' ' WALE_SWIFT_PREFIX or' ' WALE_WABS_PREFIX,' ' environment variables.')) store = storage.StorageLayout(prefix) # GPG can be optionally layered atop of every backend, so a common # code path suffices. gpg_key_id = args.gpg_key_id or os.getenv('WALE_GPG_KEY_ID') if gpg_key_id is not None: external_program_check([GPG_BIN]) # Enumeration of reading in configuration for all supported # backend data stores, yielding value adhering to the # 'operator.Backup' protocol. if store.is_s3: use_instance_profile = args.aws_instance_profile or \ parse_boolean_envvar(os.getenv('AWS_INSTANCE_PROFILE')) if use_instance_profile: creds = s3_instance_profile() else: creds = s3_explicit_creds(args) from wal_e.blobstore import s3 s3.sigv4_check_apply() from wal_e.operator import s3_operator return s3_operator.S3Backup(store, creds, gpg_key_id) elif store.is_wabs: account_name = args.wabs_account_name or os.getenv('WABS_ACCOUNT_NAME') if account_name is None: raise UserException(msg='WABS account name is undefined', hint=_config_hint_generate( 'wabs-account-name', True)) access_key = os.getenv('WABS_ACCESS_KEY') access_token = os.getenv('WABS_SAS_TOKEN') if not (access_key or access_token): raise UserException( msg='WABS access credentials is required but not provided', hint=('Define one of the WABS_ACCESS_KEY or ' 'WABS_SAS_TOKEN environment variables.')) from wal_e.blobstore import wabs from wal_e.operator.wabs_operator import WABSBackup creds = wabs.Credentials(account_name, access_key, access_token) return WABSBackup(store, creds, gpg_key_id) elif store.is_swift: from wal_e.blobstore import swift from wal_e.operator.swift_operator import SwiftBackup creds = swift.Credentials( os.getenv('SWIFT_AUTHURL'), os.getenv('SWIFT_USER'), os.getenv('SWIFT_PASSWORD'), os.getenv('SWIFT_TENANT'), os.getenv('SWIFT_REGION'), os.getenv('SWIFT_ENDPOINT_TYPE', 'publicURL'), os.getenv('SWIFT_AUTH_VERSION', '2'), os.getenv('SWIFT_DOMAIN_ID'), os.getenv('SWIFT_DOMAIN_NAME'), os.getenv('SWIFT_TENANT_ID'), os.getenv('SWIFT_USER_ID'), os.getenv('SWIFT_USER_DOMAIN_ID'), os.getenv('SWIFT_USER_DOMAIN_NAME'), os.getenv('SWIFT_PROJECT_ID'), os.getenv('SWIFT_PROJECT_NAME'), os.getenv('SWIFT_PROJECT_DOMAIN_ID'), os.getenv('SWIFT_PROJECT_DOMAIN_NAME'), ) return SwiftBackup(store, creds, gpg_key_id) elif store.is_gs: from wal_e.operator.gs_operator import GSBackup return GSBackup(store, gpg_key_id) elif store.is_file: from wal_e.blobstore import file from wal_e.operator.file_operator import FileBackup creds = file.Credentials() return FileBackup(store, creds, gpg_key_id) else: raise UserCritical(msg='no unsupported blob stores should get here', hint='Report a bug.')
def configure_backup_cxt(args): # Try to find some WAL-E prefix to store data in. prefix = (args.s3_prefix or args.wabs_prefix or os.getenv('WALE_S3_PREFIX') or os.getenv('WALE_WABS_PREFIX') or os.getenv('WALE_SWIFT_PREFIX')) if prefix is None: raise UserException( msg='no storage prefix defined', hint=( 'Either set one of the --wabs-prefix or --s3-prefix options or' ' define one of the WALE_WABS_PREFIX, WALE_S3_PREFIX, or ' 'WALE_SWIFT_PREFIX environment variables.' ) ) store = storage.StorageLayout(prefix) # GPG can be optionally layered atop of every backend, so a common # code path suffices. gpg_key_id = args.gpg_key_id or os.getenv('WALE_GPG_KEY_ID') if gpg_key_id is not None: external_program_check([GPG_BIN]) # Enumeration of reading in configuration for all supported # backend data stores, yielding value adhering to the # 'operator.Backup' protocol. if store.is_s3: if args.aws_instance_profile: creds = s3_instance_profile(args) else: creds = s3_explicit_creds(args) from wal_e.operator import s3_operator return s3_operator.S3Backup(store, creds, gpg_key_id) elif store.is_wabs: account_name = args.wabs_account_name or os.getenv('WABS_ACCOUNT_NAME') if account_name is None: raise UserException( msg='WABS account name is undefined', hint=_config_hint_generate('wabs-account-name', True)) access_key = os.getenv('WABS_ACCESS_KEY') if access_key is None: raise UserException( msg='WABS access key credential is required but not provided', hint=_config_hint_generate('wabs-access-key', False)) from wal_e.blobstore import wabs from wal_e.operator.wabs_operator import WABSBackup creds = wabs.Credentials(account_name, access_key) return WABSBackup(store, creds, gpg_key_id) elif store.is_swift: from wal_e.blobstore import swift from wal_e.operator.swift_operator import SwiftBackup creds = swift.Credentials( os.getenv('SWIFT_AUTHURL'), os.getenv('SWIFT_USER'), os.getenv('SWIFT_PASSWORD'), os.getenv('SWIFT_TENANT'), os.getenv('SWIFT_REGION'), os.getenv('SWIFT_ENDPOINT_TYPE', 'publicURL'), ) return SwiftBackup(store, creds, gpg_key_id) else: raise UserCritical( msg='no unsupported blob stores should get here', hint='Report a bug.')