def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=__doc__)
parser.add_argument('-k',
'--aws-access-key-id',
help='public AWS access key. Can also be defined in '
'an environment variable. If both are defined, '
'the one defined in the programs arguments takes '
'precedence.')
parser.add_argument('--s3-prefix',
help='S3 prefix to run all commands against. '
'Can also be defined via environment variable '
'WALE_S3_PREFIX')
parser.add_argument(
'--gpg-key-id',
help='GPG key ID to encrypt to. (Also needed when decrypting.) '
'Can also be defined via environment variable '
'WALE_GPG_KEY_ID')
subparsers = parser.add_subparsers(title='subcommands', dest='subcommand')
# Common arguments for backup-fetch and backup-push
backup_fetchpush_parent = argparse.ArgumentParser(add_help=False)
backup_fetchpush_parent.add_argument('PG_CLUSTER_DIRECTORY',
help="Postgres cluster path, "
"such as '/var/lib/database'")
backup_fetchpush_parent.add_argument('--pool-size',
'-p',
type=int,
default=4,
help='Download pooling size')
# Common arguments for archive-fetch and archive-push
archive_fetchpush_parent = argparse.ArgumentParser(add_help=False)
archive_fetchpush_parent.add_argument('ARCHIVE_FILENAME',
help="Database archive file")
archive_fetchpush_parent.add_argument('--pool-size',
'-p',
type=int,
default=4,
help='Download pooling size')
# operator to print the wal-e version
subparsers.add_parser('version', help='print the wal-e version')
# Common arguments for backup-list and backup-fetch
#
# NB: This does not include the --detail options because some
# other commands use backup listing functionality in a way where
# --detail is never required.
backup_list_nodetail_parent = argparse.ArgumentParser(add_help=False)
# Common arguments between wal-push and wal-fetch
wal_fetchpush_parent = argparse.ArgumentParser(add_help=False)
wal_fetchpush_parent.add_argument('WAL_SEGMENT',
help='Path to a WAL segment to upload')
backup_fetch_parser = subparsers.add_parser(
'backup-fetch',
help='fetch a hot backup from S3',
parents=[backup_fetchpush_parent, backup_list_nodetail_parent])
backup_list_parser = subparsers.add_parser(
'backup-list',
parents=[backup_list_nodetail_parent],
help='list backups in S3')
backup_push_parser = subparsers.add_parser(
'backup-push',
help='pushing a fresh hot backup to S3',
parents=[backup_fetchpush_parent])
archive_push_parser = subparsers.add_parser(
'archive-push',
help='pushing archive file to S3',
parents=[archive_fetchpush_parent])
backup_push_parser.add_argument(
'--cluster-read-rate-limit',
help='Rate limit reading the PostgreSQL cluster directory to a '
'tunable number of bytes per second',
dest='rate_limit',
metavar='BYTES_PER_SECOND',
type=int,
default=None)
backup_push_parser.add_argument(
'--while-offline',
help=('Backup a Postgres cluster that is in a stopped state '
'(for example, a replica that you stop and restart '
'when taking a backup)'),
dest='while_offline',
action='store_true',
default=False)
archive_push_parser.add_argument(
'--cluster-read-rate-limit',
help='Rate limit reading the archive to a '
'tunable number of bytes per second',
dest='rate_limit',
metavar='BYTES_PER_SECOND',
type=int,
default=None)
wal_fetch_parser = subparsers.add_parser('wal-fetch',
help='fetch a WAL file from S3',
parents=[wal_fetchpush_parent])
subparsers.add_parser('wal-push',
help='push a WAL file to S3',
parents=[wal_fetchpush_parent])
# backup-fetch operator section
backup_fetch_parser.add_argument('BACKUP_NAME',
help='the name of the backup to fetch')
# backup-list operator section
backup_list_parser.add_argument('QUERY',
nargs='?',
default=None,
help='a string qualifying backups to list')
backup_list_parser.add_argument(
'--detail',
default=False,
action='store_true',
help='show more detailed information about every backup')
# wal-push operator section
wal_fetch_parser.add_argument('WAL_DESTINATION',
help='Path to download the WAL segment to')
# delete subparser section
delete_parser = subparsers.add_parser(
'delete', help=('operators to destroy specified data in S3'))
delete_parser.add_argument('--dry-run',
'-n',
action='store_true',
help=('Only print what would be deleted, '
'do not actually delete anything'))
delete_parser.add_argument('--confirm',
action='store_true',
help=('Actually delete data. '
'By default, a dry run is performed. '
'Overridden by --dry-run.'))
delete_subparsers = delete_parser.add_subparsers(
title='delete subcommands',
description=('All operators that may delete data are contained '
'in this subcommand.'),
dest='delete_subcommand')
# delete 'before' operator
delete_before_parser = delete_subparsers.add_parser(
'before',
help=('Delete all backups and WAL segments strictly before '
'the given base backup name or WAL segment number. '
'The passed backup is *not* deleted.'))
delete_before_parser.add_argument(
'BEFORE_SEGMENT_EXCLUSIVE',
help='A WAL segment number or base backup name')
# delete old versions operator
delete_subparsers.add_parser(
'old-versions',
help=('Delete all old versions of WAL-E backup files. One probably '
'wants to ensure that they take a new backup with the new '
'format first. '
'This is useful after a WAL-E major release upgrade.'))
# delete *everything* operator
delete_subparsers.add_parser(
'everything',
help=('Delete all data in the current WAL-E context. '
'Typically this is only appropriate when decommissioning an '
'entire WAL-E archive.'))
# Okay, parse some arguments, finally
args = parser.parse_args()
subcommand = args.subcommand
# Handle version printing specially, because it doesn't need
# credentials.
if subcommand == 'version':
import pkgutil
print pkgutil.get_data('wal_e', 'VERSION').strip()
sys.exit(0)
# Attempt to read a few key parameters from environment variables
# *or* the command line, enforcing a precedence order and
# complaining should the required parameter not be defined in
# either location.
secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
if secret_key is None:
logger.error(
msg='no AWS_SECRET_ACCESS_KEY defined',
hint='Define the environment variable AWS_SECRET_ACCESS_KEY.')
sys.exit(1)
s3_prefix = args.s3_prefix or os.getenv('WALE_S3_PREFIX')
if s3_prefix is None:
logger.error(msg='no storage prefix defined',
hint=('Either set the --s3-prefix option or define the '
'environment variable WALE_S3_PREFIX.'))
sys.exit(1)
if args.aws_access_key_id is None:
aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
if aws_access_key_id is None:
logger.error(
msg='no storage prefix defined',
hint=('Either set the --aws-access-key-id option or define '
'the environment variable AWS_ACCESS_KEY_ID.'))
sys.exit(1)
else:
aws_access_key_id = args.aws_access_key_id
# This will be None if we're not encrypting
gpg_key_id = args.gpg_key_id or os.getenv('WALE_GPG_KEY_ID')
backup_cxt = s3_operator.S3Backup(aws_access_key_id, secret_key, s3_prefix,
gpg_key_id)
if gpg_key_id is not None:
external_program_check([GPG_BIN])
try:
if subcommand == 'backup-fetch':
external_program_check([LZOP_BIN])
backup_cxt.database_s3_fetch(args.PG_CLUSTER_DIRECTORY,
args.BACKUP_NAME,
pool_size=args.pool_size)
elif subcommand == 'backup-list':
backup_cxt.backup_list(query=args.QUERY, detail=args.detail)
elif subcommand == 'backup-push':
if args.while_offline:
# we need to query pg_config first for the
# pg_controldata's bin location
external_program_check([CONFIG_BIN])
parser = PgControlDataParser(args.PG_CLUSTER_DIRECTORY)
controldata_bin = parser.controldata_bin()
external_programs = [LZOP_BIN, PV_BIN, controldata_bin]
else:
external_programs = [LZOP_BIN, PSQL_BIN, PV_BIN]
external_program_check(external_programs)
rate_limit = args.rate_limit
while_offline = args.while_offline
backup_cxt.database_s3_backup(args.PG_CLUSTER_DIRECTORY,
rate_limit=rate_limit,
while_offline=while_offline,
pool_size=args.pool_size)
elif subcommand == 'archive-push':
external_programs = [PV_BIN]
external_program_check(external_programs)
rate_limit = args.rate_limit
backup_cxt.archive_s3_upload(args.ARCHIVE_FILENAME,
rate_limit=rate_limit,
pool_size=args.pool_size)
elif subcommand == 'wal-fetch':
external_program_check([LZOP_BIN])
res = backup_cxt.wal_s3_restore(args.WAL_SEGMENT,
args.WAL_DESTINATION)
if not res:
sys.exit(1)
elif subcommand == 'wal-push':
external_program_check([LZOP_BIN])
backup_cxt.wal_s3_archive(args.WAL_SEGMENT)
elif subcommand == 'delete':
# Set up pruning precedence, optimizing for *not* deleting data
#
# Canonicalize the passed arguments into the value
# "is_dry_run_really"
if args.dry_run is False and args.confirm is True:
# Actually delete data *only* if there are *no* --dry-runs
# present and --confirm is present.
logger.info(msg='deleting data in S3')
is_dry_run_really = False
else:
logger.info(msg='performing dry run of S3 data deletion')
is_dry_run_really = True
import boto.s3.key
import boto.s3.Bucket
# This is not necessary, but "just in case" to find bugs.
def just_error(*args, **kwargs):
assert False, ('About to delete something in '
'dry-run mode. Please report a bug.')
boto.s3.key.Key.delete = just_error
boto.s3.bucket.Bucket.delete_keys = just_error
# Handle the subcommands and route them to the right
# implementations.
if args.delete_subcommand == 'old-versions':
backup_cxt.delete_old_versions(is_dry_run_really)
elif args.delete_subcommand == 'everything':
backup_cxt.delete_all(is_dry_run_really)
elif args.delete_subcommand == 'before':
segment_info = extract_segment(args.BEFORE_SEGMENT_EXCLUSIVE)
backup_cxt.delete_before(is_dry_run_really, segment_info)
else:
assert False, 'Should be rejected by argument parsing.'
else:
logger.error(
msg='subcommand not implemented',
detail=(
'The submitted subcommand was {0}.'.format(subcommand)),
hint='Check for typos or consult wal-e --help.')
sys.exit(127)
# Report on all encountered exceptions, and raise the last one
# to take advantage of the final catch-all reporting and exit
# code management.
if backup_cxt.exceptions:
for exc in backup_cxt.exceptions[:-1]:
logger.log(level=exc.severity,
msg=log_help.WalELogger.fmt_logline(
exc.msg, exc.detail, exc.hint))
raise backup_cxt.exceptions[-1]
except UserException, e:
logger.log(level=e.severity,
msg=log_help.WalELogger.fmt_logline(e.msg, e.detail,
e.hint))
sys.exit(1)
def configure_backup_cxt(args):
# Try to find some WAL-E prefix to store data in.
prefix = (args.file_prefix or args.gs_prefix or args.s3_prefix
or args.wabs_prefix or os.getenv('WALE_FILE_PREFIX')
or os.getenv('WALE_GS_PREFIX') or os.getenv('WALE_S3_PREFIX')
or os.getenv('WALE_SWIFT_PREFIX')
or os.getenv('WALE_WABS_PREFIX'))
if prefix is None:
raise UserException(msg='no storage prefix defined',
hint=('Either set one of the'
' --file-prefix,'
' --gs-prefix,'
' --s3-prefix or'
' --wabs-prefix options'
' or define one of the'
' WALE_FILE_PREFIX,'
' WALE_GS_PREFIX,'
' WALE_S3_PREFIX,'
' WALE_SWIFT_PREFIX or'
' WALE_WABS_PREFIX,'
' environment variables.'))
store = storage.StorageLayout(prefix)
# GPG can be optionally layered atop of every backend, so a common
# code path suffices.
gpg_key_id = args.gpg_key_id or os.getenv('WALE_GPG_KEY_ID')
if gpg_key_id is not None:
external_program_check([GPG_BIN])
# Enumeration of reading in configuration for all supported
# backend data stores, yielding value adhering to the
# 'operator.Backup' protocol.
if store.is_s3:
use_instance_profile = args.aws_instance_profile or \
parse_boolean_envvar(os.getenv('AWS_INSTANCE_PROFILE'))
if use_instance_profile:
creds = s3_instance_profile()
else:
creds = s3_explicit_creds(args)
from wal_e.blobstore import s3
s3.sigv4_check_apply()
from wal_e.operator import s3_operator
return s3_operator.S3Backup(store, creds, gpg_key_id)
elif store.is_wabs:
account_name = args.wabs_account_name or os.getenv('WABS_ACCOUNT_NAME')
if account_name is None:
raise UserException(msg='WABS account name is undefined',
hint=_config_hint_generate(
'wabs-account-name', True))
access_key = os.getenv('WABS_ACCESS_KEY')
access_token = os.getenv('WABS_SAS_TOKEN')
if not (access_key or access_token):
raise UserException(
msg='WABS access credentials is required but not provided',
hint=('Define one of the WABS_ACCESS_KEY or '
'WABS_SAS_TOKEN environment variables.'))
from wal_e.blobstore import wabs
from wal_e.operator.wabs_operator import WABSBackup
creds = wabs.Credentials(account_name, access_key, access_token)
return WABSBackup(store, creds, gpg_key_id)
elif store.is_swift:
from wal_e.blobstore import swift
from wal_e.operator.swift_operator import SwiftBackup
creds = swift.Credentials(
os.getenv('SWIFT_AUTHURL'),
os.getenv('SWIFT_USER'),
os.getenv('SWIFT_PASSWORD'),
os.getenv('SWIFT_TENANT'),
os.getenv('SWIFT_REGION'),
os.getenv('SWIFT_ENDPOINT_TYPE', 'publicURL'),
os.getenv('SWIFT_AUTH_VERSION', '2'),
os.getenv('SWIFT_DOMAIN_ID'),
os.getenv('SWIFT_DOMAIN_NAME'),
os.getenv('SWIFT_TENANT_ID'),
os.getenv('SWIFT_USER_ID'),
os.getenv('SWIFT_USER_DOMAIN_ID'),
os.getenv('SWIFT_USER_DOMAIN_NAME'),
os.getenv('SWIFT_PROJECT_ID'),
os.getenv('SWIFT_PROJECT_NAME'),
os.getenv('SWIFT_PROJECT_DOMAIN_ID'),
os.getenv('SWIFT_PROJECT_DOMAIN_NAME'),
)
return SwiftBackup(store, creds, gpg_key_id)
elif store.is_gs:
from wal_e.operator.gs_operator import GSBackup
return GSBackup(store, gpg_key_id)
elif store.is_file:
from wal_e.blobstore import file
from wal_e.operator.file_operator import FileBackup
creds = file.Credentials()
return FileBackup(store, creds, gpg_key_id)
else:
raise UserCritical(msg='no unsupported blob stores should get here',
hint='Report a bug.')
def configure_backup_cxt(args):
# Try to find some WAL-E prefix to store data in.
prefix = (args.s3_prefix or args.wabs_prefix
or os.getenv('WALE_S3_PREFIX') or os.getenv('WALE_WABS_PREFIX')
or os.getenv('WALE_SWIFT_PREFIX'))
if prefix is None:
raise UserException(
msg='no storage prefix defined',
hint=(
'Either set one of the --wabs-prefix or --s3-prefix options or'
' define one of the WALE_WABS_PREFIX, WALE_S3_PREFIX, or '
'WALE_SWIFT_PREFIX environment variables.'
)
)
store = storage.StorageLayout(prefix)
# GPG can be optionally layered atop of every backend, so a common
# code path suffices.
gpg_key_id = args.gpg_key_id or os.getenv('WALE_GPG_KEY_ID')
if gpg_key_id is not None:
external_program_check([GPG_BIN])
# Enumeration of reading in configuration for all supported
# backend data stores, yielding value adhering to the
# 'operator.Backup' protocol.
if store.is_s3:
if args.aws_instance_profile:
creds = s3_instance_profile(args)
else:
creds = s3_explicit_creds(args)
from wal_e.operator import s3_operator
return s3_operator.S3Backup(store, creds, gpg_key_id)
elif store.is_wabs:
account_name = args.wabs_account_name or os.getenv('WABS_ACCOUNT_NAME')
if account_name is None:
raise UserException(
msg='WABS account name is undefined',
hint=_config_hint_generate('wabs-account-name', True))
access_key = os.getenv('WABS_ACCESS_KEY')
if access_key is None:
raise UserException(
msg='WABS access key credential is required but not provided',
hint=_config_hint_generate('wabs-access-key', False))
from wal_e.blobstore import wabs
from wal_e.operator.wabs_operator import WABSBackup
creds = wabs.Credentials(account_name, access_key)
return WABSBackup(store, creds, gpg_key_id)
elif store.is_swift:
from wal_e.blobstore import swift
from wal_e.operator.swift_operator import SwiftBackup
creds = swift.Credentials(
os.getenv('SWIFT_AUTHURL'),
os.getenv('SWIFT_USER'),
os.getenv('SWIFT_PASSWORD'),
os.getenv('SWIFT_TENANT'),
os.getenv('SWIFT_REGION'),
os.getenv('SWIFT_ENDPOINT_TYPE', 'publicURL'),
)
return SwiftBackup(store, creds, gpg_key_id)
else:
raise UserCritical(
msg='no unsupported blob stores should get here',
hint='Report a bug.')