def uri_put_file(creds, uri, fp, content_encoding=None):
blobstore = get_blobstore(storage.StorageLayout(uri))
return blobstore.uri_put_file(creds,
uri,
fp,
content_encoding=content_encoding)
def configure_backup_cxt(args):
# Try to find some WAL-E prefix to store data in.
prefix = (args.file_prefix or args.gs_prefix or args.s3_prefix
or args.wabs_prefix or os.getenv('WALE_FILE_PREFIX')
or os.getenv('WALE_GS_PREFIX') or os.getenv('WALE_S3_PREFIX')
or os.getenv('WALE_SWIFT_PREFIX')
or os.getenv('WALE_WABS_PREFIX'))
if prefix is None:
raise UserException(msg='no storage prefix defined',
hint=('Either set one of the'
' --file-prefix,'
' --gs-prefix,'
' --s3-prefix or'
' --wabs-prefix options'
' or define one of the'
' WALE_FILE_PREFIX,'
' WALE_GS_PREFIX,'
' WALE_S3_PREFIX,'
' WALE_SWIFT_PREFIX or'
' WALE_WABS_PREFIX,'
' environment variables.'))
store = storage.StorageLayout(prefix)
# GPG can be optionally layered atop of every backend, so a common
# code path suffices.
gpg_key_id = args.gpg_key_id or os.getenv('WALE_GPG_KEY_ID')
if gpg_key_id is not None:
external_program_check([GPG_BIN])
# Enumeration of reading in configuration for all supported
# backend data stores, yielding value adhering to the
# 'operator.Backup' protocol.
if store.is_s3:
use_instance_profile = args.aws_instance_profile or \
parse_boolean_envvar(os.getenv('AWS_INSTANCE_PROFILE'))
if use_instance_profile:
creds = s3_instance_profile()
else:
creds = s3_explicit_creds(args)
from wal_e.blobstore import s3
s3.sigv4_check_apply()
from wal_e.operator import s3_operator
return s3_operator.S3Backup(store, creds, gpg_key_id)
elif store.is_wabs:
account_name = args.wabs_account_name or os.getenv('WABS_ACCOUNT_NAME')
if account_name is None:
raise UserException(msg='WABS account name is undefined',
hint=_config_hint_generate(
'wabs-account-name', True))
access_key = os.getenv('WABS_ACCESS_KEY')
access_token = os.getenv('WABS_SAS_TOKEN')
if not (access_key or access_token):
raise UserException(
msg='WABS access credentials is required but not provided',
hint=('Define one of the WABS_ACCESS_KEY or '
'WABS_SAS_TOKEN environment variables.'))
from wal_e.blobstore import wabs
from wal_e.operator.wabs_operator import WABSBackup
creds = wabs.Credentials(account_name, access_key, access_token)
return WABSBackup(store, creds, gpg_key_id)
elif store.is_swift:
from wal_e.blobstore import swift
from wal_e.operator.swift_operator import SwiftBackup
creds = swift.Credentials(
os.getenv('SWIFT_AUTHURL'),
os.getenv('SWIFT_USER'),
os.getenv('SWIFT_PASSWORD'),
os.getenv('SWIFT_TENANT'),
os.getenv('SWIFT_REGION'),
os.getenv('SWIFT_ENDPOINT_TYPE', 'publicURL'),
os.getenv('SWIFT_AUTH_VERSION', '2'),
os.getenv('SWIFT_DOMAIN_ID'),
os.getenv('SWIFT_DOMAIN_NAME'),
os.getenv('SWIFT_TENANT_ID'),
os.getenv('SWIFT_USER_ID'),
os.getenv('SWIFT_USER_DOMAIN_ID'),
os.getenv('SWIFT_USER_DOMAIN_NAME'),
os.getenv('SWIFT_PROJECT_ID'),
os.getenv('SWIFT_PROJECT_NAME'),
os.getenv('SWIFT_PROJECT_DOMAIN_ID'),
os.getenv('SWIFT_PROJECT_DOMAIN_NAME'),
)
return SwiftBackup(store, creds, gpg_key_id)
elif store.is_gs:
from wal_e.operator.gs_operator import GSBackup
return GSBackup(store, gpg_key_id)
elif store.is_file:
from wal_e.blobstore import file
from wal_e.operator.file_operator import FileBackup
creds = file.Credentials()
return FileBackup(store, creds, gpg_key_id)
else:
raise UserCritical(msg='no unsupported blob stores should get here',
hint='Report a bug.')
def configure_backup_cxt(args):
# Try to find some WAL-E prefix to store data in.
prefix = (args.s3_prefix or args.wabs_prefix
or os.getenv('WALE_S3_PREFIX') or os.getenv('WALE_WABS_PREFIX')
or os.getenv('WALE_SWIFT_PREFIX'))
if prefix is None:
raise UserException(
msg='no storage prefix defined',
hint=(
'Either set one of the --wabs-prefix or --s3-prefix options or'
' define one of the WALE_WABS_PREFIX, WALE_S3_PREFIX, or '
'WALE_SWIFT_PREFIX environment variables.'
)
)
store = storage.StorageLayout(prefix)
# GPG can be optionally layered atop of every backend, so a common
# code path suffices.
gpg_key_id = args.gpg_key_id or os.getenv('WALE_GPG_KEY_ID')
if gpg_key_id is not None:
external_program_check([GPG_BIN])
# Define some hint-text generator to help the user with consistent
# language between storage backends when possible.
def _opt_env_hint(optname):
option = '--' + optname.lower()
env = optname.replace('-', '_').upper()
return ('Pass "{0}" or set the environment variable "{1}".'
.format(option, env))
def _env_hint(optname):
env = optname.replace('-', '_').upper()
return 'Set the environment variable {0}.'.format(env)
# Enumeration of reading in configuration for all supported
# backend data stores, yielding value adhering to the
# 'operator.Backup' protocol.
if store.is_s3:
access_key = args.aws_access_key_id or os.getenv('AWS_ACCESS_KEY_ID')
if access_key is None:
raise UserException(
msg='AWS Access Key credential is required but not provided',
hint=(_opt_env_hint('aws-access-key-id')))
secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
if secret_key is None:
raise UserException(
msg='AWS Secret Key credential is required but not provided',
hint=_env_hint('aws-secret-access-key'))
security_token = os.getenv('AWS_SECURITY_TOKEN')
from wal_e.blobstore import s3
from wal_e.operator.s3_operator import S3Backup
creds = s3.Credentials(access_key, secret_key, security_token)
return S3Backup(store, creds, gpg_key_id)
elif store.is_wabs:
account_name = args.wabs_account_name or os.getenv('WABS_ACCOUNT_NAME')
if account_name is None:
raise UserException(
msg='WABS account name is undefined',
hint=_opt_env_hint('wabs-account-name'))
access_key = os.getenv('WABS_ACCESS_KEY')
if access_key is None:
raise UserException(
msg='WABS access key credential is required but not provided',
hint=_env_hint('wabs-access-key'))
from wal_e.blobstore import wabs
from wal_e.operator.wabs_operator import WABSBackup
creds = wabs.Credentials(account_name, access_key)
return WABSBackup(store, creds, gpg_key_id)
elif store.is_swift:
from wal_e.blobstore import swift
from wal_e.operator.swift_operator import SwiftBackup
creds = swift.Credentials(
os.getenv('SWIFT_AUTHURL'),
os.getenv('SWIFT_USER'),
os.getenv('SWIFT_PASSWORD'),
os.getenv('SWIFT_TENANT'),
os.getenv('SWIFT_REGION'),
os.getenv('SWIFT_ENDPOINT_TYPE', 'publicURL'),
)
return SwiftBackup(store, creds, gpg_key_id)
else:
raise UserCritical(
msg='no unsupported blob stores should get here',
hint='Report a bug.')
def __init__(self, creds, backup_prefix, rate_limit, gpg_key):
self.creds = creds
self.backup_prefix = backup_prefix
self.rate_limit = rate_limit
self.gpg_key = gpg_key
self.blobstore = get_blobstore(storage.StorageLayout(backup_prefix))
def configure_backup_cxt(args):
# Try to find some WAL-E prefix to store data in.
prefix = (args.s3_prefix or args.wabs_prefix
or os.getenv('WALE_S3_PREFIX') or os.getenv('WALE_WABS_PREFIX')
or os.getenv('WALE_SWIFT_PREFIX'))
if prefix is None:
raise UserException(
msg='no storage prefix defined',
hint=(
'Either set one of the --wabs-prefix or --s3-prefix options or'
' define one of the WALE_WABS_PREFIX, WALE_S3_PREFIX, or '
'WALE_SWIFT_PREFIX environment variables.'
)
)
store = storage.StorageLayout(prefix)
# GPG can be optionally layered atop of every backend, so a common
# code path suffices.
gpg_key_id = args.gpg_key_id or os.getenv('WALE_GPG_KEY_ID')
if gpg_key_id is not None:
external_program_check([GPG_BIN])
# Enumeration of reading in configuration for all supported
# backend data stores, yielding value adhering to the
# 'operator.Backup' protocol.
if store.is_s3:
if args.aws_instance_profile:
creds = s3_instance_profile(args)
else:
creds = s3_explicit_creds(args)
from wal_e.operator import s3_operator
return s3_operator.S3Backup(store, creds, gpg_key_id)
elif store.is_wabs:
account_name = args.wabs_account_name or os.getenv('WABS_ACCOUNT_NAME')
if account_name is None:
raise UserException(
msg='WABS account name is undefined',
hint=_config_hint_generate('wabs-account-name', True))
access_key = os.getenv('WABS_ACCESS_KEY')
if access_key is None:
raise UserException(
msg='WABS access key credential is required but not provided',
hint=_config_hint_generate('wabs-access-key', False))
from wal_e.blobstore import wabs
from wal_e.operator.wabs_operator import WABSBackup
creds = wabs.Credentials(account_name, access_key)
return WABSBackup(store, creds, gpg_key_id)
elif store.is_swift:
from wal_e.blobstore import swift
from wal_e.operator.swift_operator import SwiftBackup
creds = swift.Credentials(
os.getenv('SWIFT_AUTHURL'),
os.getenv('SWIFT_USER'),
os.getenv('SWIFT_PASSWORD'),
os.getenv('SWIFT_TENANT'),
os.getenv('SWIFT_REGION'),
os.getenv('SWIFT_ENDPOINT_TYPE', 'publicURL'),
)
return SwiftBackup(store, creds, gpg_key_id)
else:
raise UserCritical(
msg='no unsupported blob stores should get here',
hint='Report a bug.')
