def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=__doc__)
parser.add_argument('-k', '--aws-access-key-id',
help='public AWS access key. Can also be defined in an '
'environment variable. If both are defined, '
'the one defined in the programs arguments takes '
'precedence.')
parser.add_argument('--s3-prefix',
help='S3 prefix to run all commands against. '
'Can also be defined via environment variable '
'WALE_S3_PREFIX')
subparsers = parser.add_subparsers(title='subcommands',
dest='subcommand')
# Common options for backup-fetch and backup-push
backup_fetchpush_parent = argparse.ArgumentParser(add_help=False)
backup_fetchpush_parent.add_argument('PG_CLUSTER_DIRECTORY',
help="Postgres cluster path, "
"such as '/var/lib/database'")
backup_fetchpush_parent.add_argument('--pool-size', '-p',
type=int, default=4,
help='Download pooling size')
wal_fetchpush_parent = argparse.ArgumentParser(add_help=False)
wal_fetchpush_parent.add_argument('WAL_SEGMENT',
help='Path to a WAL segment to upload')
backup_fetch_parser = subparsers.add_parser(
'backup-fetch', help='fetch a hot backup from S3',
parents=[backup_fetchpush_parent])
subparsers.add_parser('backup-list', help='list backups in S3')
backup_push_parser = subparsers.add_parser(
'backup-push', help='pushing a fresh hot backup to S3',
parents=[backup_fetchpush_parent])
backup_push_parser.add_argument(
'--cluster-read-rate-limit',
help='Rate limit reading the PostgreSQL cluster directory to a '
'tunable number of bytes per second', dest='rate_limit',
metavar='BYTES_PER_SECOND',
type=int, default=None)
wal_fetch_parser = subparsers.add_parser(
'wal-fetch', help='fetch a WAL file from S3',
parents=[wal_fetchpush_parent])
subparsers.add_parser('wal-push', help='push a WAL file to S3',
parents=[wal_fetchpush_parent])
wal_fark_parser = subparsers.add_parser('wal-fark',
help='The FAke Arkiver')
# XXX: Partial copy paste, because no parallel archiving support
# is supported and to have the --pool option would be confusing.
wal_fark_parser.add_argument('PG_CLUSTER_DIRECTORY',
help="Postgres cluster path, "
"such as '/var/lib/database'")
# backup-fetch operator section
backup_fetch_parser.add_argument('BACKUP_NAME',
help='the name of the backup to fetch')
# wal-push operator section
wal_fetch_parser.add_argument('WAL_DESTINATION',
help='Path to download the WAL segment to')
args = parser.parse_args()
secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
if secret_key is None:
print >>sys.stderr, ('Must define AWS_SECRET_ACCESS_KEY ask S3 to do '
'anything')
sys.exit(1)
s3_prefix = args.s3_prefix or os.getenv('WALE_S3_PREFIX')
if s3_prefix is None:
print >>sys.stderr, ('Must pass --s3-prefix or define environment '
'variable WALE_S3_PREFIX')
sys.exit(1)
if args.aws_access_key_id is None:
aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
if aws_access_key_id is None:
print >>sys.stderr, ('Must define an AWS_ACCESS_KEY_ID, '
'using environment variable or '
'--aws_access_key_id')
else:
aws_access_key_id = args.aws_access_key_id
backup_cxt = S3Backup(aws_access_key_id, secret_key, s3_prefix)
subcommand = args.subcommand
try:
if subcommand == 'backup-fetch':
external_program_check([S3CMD_BIN, DECODE_BIN])
backup_cxt.database_s3_fetch(args.PG_CLUSTER_DIRECTORY,
args.BACKUP_NAME,
pool_size=args.pool_size)
elif subcommand == 'backup-list':
external_program_check([S3CMD_BIN])
backup_cxt.backup_list()
elif subcommand == 'backup-push':
external_program_check([S3CMD_BIN, ENCODE_BIN, PSQL_BIN, MBUFFER_BIN])
rate_limit = args.rate_limit
if rate_limit is not None and rate_limit < 8192:
print >>sys.stderr, ('--cluster-read-rate-limit must be a '
'positive integer over or equal to 8192')
sys.exit(1)
backup_cxt.database_s3_backup(
args.PG_CLUSTER_DIRECTORY, rate_limit=rate_limit,
pool_size=args.pool_size)
elif subcommand == 'wal-fetch':
external_program_check([S3CMD_BIN, DECODE_BIN])
backup_cxt.wal_s3_restore(args.WAL_SEGMENT, args.WAL_DESTINATION)
elif subcommand == 'wal-push':
external_program_check([S3CMD_BIN, ENCODE_BIN])
backup_cxt.wal_s3_archive(args.WAL_SEGMENT)
elif subcommand == 'wal-fark':
external_program_check([S3CMD_BIN, ENCODE_BIN])
backup_cxt.wal_fark(args.PG_CLUSTER_DIRECTORY)
else:
print >>sys.stderr, ('Subcommand {0} not implemented!'
.format(subcommand))
sys.exit(127)
except UserException, e:
logger.log(level=e.severity,
msg=log_help.fmt_logline(e.msg, e.detail, e.hint))
sys.exit(1)
def database_s3_backup(self, *args, **kwargs):
"""
Uploads a PostgreSQL file cluster to S3
Mechanism: just wraps _s3_upload_pg_cluster_dir with
start/stop backup actions with exception handling.
In particular there is a 'finally' block to stop the backup in
most situations.
"""
upload_good = False
backup_stop_good = False
try:
start_backup_info = PgBackupStatements.run_start_backup()
version = PgBackupStatements.pg_version()['version']
uploaded_to, expanded_size_bytes = \
self._s3_upload_pg_cluster_dir(
start_backup_info, version=version,
*args, **kwargs)
upload_good = True
finally:
if not upload_good:
logger.warning(
log_help.fmt_logline(
'blocking on sending WAL segments',
'The backup was not completed successfully, '
'but we have to wait anyway. '
'See README: TODO about pg_cancel_backup'))
stop_backup_info = PgBackupStatements.run_stop_backup()
backup_stop_good = True
if upload_good and backup_stop_good:
# Make a best-effort attempt to write a sentinel file to
# the cluster backup directory that indicates that the
# base backup upload has definitely run its course (it may
# have, even without this file, though) and also
# communicates what WAL segments are needed to get to
# consistency.
try:
with self.s3cmd_temp_config as s3cmd_config:
with tempfile.NamedTemporaryFile(mode='w') as sentinel:
json.dump(
{'wal_segment_backup_stop':
stop_backup_info['file_name'],
'wal_segment_offset_backup_stop':
stop_backup_info['file_offset'],
'expanded_size_bytes': expanded_size_bytes},
sentinel)
sentinel.flush()
# Avoid using do_lzo_aes_s3_put to store
# uncompressed: easier to read/double click
# on/dump to terminal
check_call_wait_sigint(
[S3CMD_BIN, '-c', s3cmd_config.name,
'--mime-type=application/json', 'put',
sentinel.name,
uploaded_to + '_backup_stop_sentinel.json'])
except KeyboardInterrupt, e:
# Specially re-raise exception on SIGINT to allow
# propagation.
raise
except:
