def database_s3_backup(self, data_directory, *args, **kwargs):
"""Uploads a PostgreSQL file cluster to S3
Mechanism: just wraps _s3_upload_pg_cluster_dir with
start/stop backup actions with exception handling.
In particular there is a 'finally' block to stop the backup in
most situations.
"""
upload_good = False
backup_stop_good = False
while_offline = False
start_backup_info = None
if 'while_offline' in kwargs:
while_offline = kwargs.pop('while_offline')
try:
if not while_offline:
start_backup_info = PgBackupStatements.run_start_backup()
version = PgBackupStatements.pg_version()['version']
else:
if os.path.exists(os.path.join(data_directory,
'postmaster.pid')):
hint = ('Shut down postgres. '
'If there is a stale lockfile, '
'then remove it after being very sure postgres '
'is not running.')
raise UserException(
msg='while_offline set, but pg looks to be running',
detail='Found a postmaster.pid lockfile, and aborting',
hint=hint)
controldata = PgControlDataParser(data_directory)
start_backup_info = \
controldata.last_xlog_file_name_and_offset()
version = controldata.pg_version()
uploaded_to, expanded_size_bytes = self._s3_upload_pg_cluster_dir(
start_backup_info, data_directory, version=version,
*args, **kwargs)
upload_good = True
finally:
if not upload_good:
logger.warning(
'blocking on sending WAL segments',
detail=('The backup was not completed successfully, '
'but we have to wait anyway. '
'See README: TODO about pg_cancel_backup'))
if not while_offline:
stop_backup_info = PgBackupStatements.run_stop_backup()
else:
stop_backup_info = start_backup_info
backup_stop_good = True
# XXX: Ugly, this is more of a 'worker' task because it might
# involve retries and error messages, something that is not
# treated by the "operator" category of modules. So
# basically, if this small upload fails, the whole upload
# fails!
if upload_good and backup_stop_good:
# Try to write a sentinel file to the cluster backup
# directory that indicates that the base backup upload has
# definitely run its course and also communicates what WAL
# segments are needed to get to consistency.
sentinel_content = StringIO()
json.dump(
{'wal_segment_backup_stop':
stop_backup_info['file_name'],
'wal_segment_offset_backup_stop':
stop_backup_info['file_offset'],
'expanded_size_bytes': expanded_size_bytes},
sentinel_content)
# XXX: should use the storage.s3_storage operators.
#
# XXX: distinguish sentinels by *PREFIX* not suffix,
# which makes searching harder. (For the next version
# bump).
sentinel_content.seek(0)
s3_worker.uri_put_file(
uploaded_to + '_backup_stop_sentinel.json',
sentinel_content, content_encoding='application/json')
else:
# NB: Other exceptions should be raised before this that
# have more informative results, it is intended that this
# exception never will get raised.
raise UserCritical('could not complete backup process')
def _s3_upload_pg_cluster_dir(self, start_backup_info, pg_cluster_dir,
version, pool_size, rate_limit=None):
"""
Upload to s3_url_prefix from pg_cluster_dir
This function ignores the directory pg_xlog, which contains WAL
files and are not generally part of a base backup.
Note that this is also lzo compresses the files: thus, the number
of pooled processes involves doing a full sequential scan of the
uncompressed Postgres heap file that is pipelined into lzo. Once
lzo is completely finished (necessary to have access to the file
size) the file is sent to S3.
TODO: Investigate an optimization to decouple the compression and
upload steps to make sure that the most efficient possible use of
pipelining of network and disk resources occurs. Right now it
possible to bounce back and forth between bottlenecking on reading
from the database block device and subsequently the S3 sending
steps should the processes be at the same stage of the upload
pipeline: this can have a very negative impact on being able to
make full use of system resources.
Furthermore, it desirable to overflowing the page cache: having
separate tunables for number of simultanious compression jobs
(which occupy /tmp space and page cache) and number of uploads
(which affect upload throughput) would help.
"""
# Get a manifest of files first.
matches = []
def raise_walk_error(e):
raise e
walker = os.walk(pg_cluster_dir, onerror=raise_walk_error)
for root, dirnames, filenames in walker:
is_cluster_toplevel = (os.path.abspath(root) ==
os.path.abspath(pg_cluster_dir))
# Do not capture any WAL files, although we do want to
# capture the WAL directory or symlink
if is_cluster_toplevel:
if 'pg_xlog' in dirnames:
dirnames.remove('pg_xlog')
matches.append(os.path.join(root, 'pg_xlog'))
for filename in filenames:
if is_cluster_toplevel and filename in ('postmaster.pid',
'postgresql.conf'):
# Do not include the postmaster pid file or the
# configuration file in the backup.
pass
else:
matches.append(os.path.join(root, filename))
# Special case for empty directories
if not filenames:
matches.append(root)
backup_s3_prefix = ('{0}/basebackups_{1}/'
'base_{file_name}_{file_offset}'
.format(self.s3_prefix, FILE_STRUCTURE_VERSION,
**start_backup_info))
# absolute upload paths are used for telling lzop what to compress
local_abspaths = [os.path.abspath(match) for match in matches]
# computed to subtract out extra extraneous absolute path
# information when storing on S3
common_local_prefix = os.path.commonprefix(local_abspaths)
partitions = tar_partition.tar_partitions_plan(
common_local_prefix, local_abspaths,
# 1610612736 bytes == 1.5 gigabytes, per partition,
# non-tunable
1610612736)
if rate_limit is None:
per_process_limit = None
else:
per_process_limit = int(rate_limit / pool_size)
# Reject tiny per-process rate limits. They should be
# rejected more nicely elsewhere.
assert per_process_limit > 0 or per_process_limit is None
# a list to accumulate async upload jobs
uploads = []
total_size = 0
# Make an attempt to upload extended version metadata
extended_version_url = backup_s3_prefix + '/extended_version.txt'
logger.info(
msg='start upload postgres version metadata',
detail=('Uploading to {extended_version_url}.'
.format(extended_version_url=extended_version_url)))
s3_worker.uri_put_file(extended_version_url, StringIO(version),
content_encoding='text/plain')
logger.info(msg='postgres version metadata upload complete')
pool = gevent.pool.Pool(size=pool_size)
# Enqueue uploads for parallel execution
try:
for tpart in partitions:
total_size += tpart.total_member_size
uploads.append(pool.apply_async(
s3_worker.do_partition_put,
[backup_s3_prefix, tpart, per_process_limit,
self.gpg_key_id]))
finally:
while uploads:
uploads.pop().get()
pool.join()
return backup_s3_prefix, total_size
def _s3_upload_pg_cluster_dir(self, start_backup_info, pg_cluster_dir,
version, pool_size, rate_limit=None):
"""
Upload to s3_url_prefix from pg_cluster_dir
This function ignores the directory pg_xlog, which contains WAL
files and are not generally part of a base backup.
Note that this is also lzo compresses the files: thus, the number
of pooled processes involves doing a full sequential scan of the
uncompressed Postgres heap file that is pipelined into lzo. Once
lzo is completely finished (necessary to have access to the file
size) the file is sent to S3.
TODO: Investigate an optimization to decouple the compression and
upload steps to make sure that the most efficient possible use of
pipelining of network and disk resources occurs. Right now it
possible to bounce back and forth between bottlenecking on reading
from the database block device and subsequently the S3 sending
steps should the processes be at the same stage of the upload
pipeline: this can have a very negative impact on being able to
make full use of system resources.
Furthermore, it desirable to overflowing the page cache: having
separate tunables for number of simultanious compression jobs
(which occupy /tmp space and page cache) and number of uploads
(which affect upload throughput) would help.
"""
parts = tar_partition.partition(pg_cluster_dir)
backup_s3_prefix = ('{0}/basebackups_{1}/'
'base_{file_name}_{file_offset}'
.format(self.s3_prefix, FILE_STRUCTURE_VERSION,
**start_backup_info))
if rate_limit is None:
per_process_limit = None
else:
per_process_limit = int(rate_limit / pool_size)
# Reject tiny per-process rate limits. They should be
# rejected more nicely elsewhere.
assert per_process_limit > 0 or per_process_limit is None
total_size = 0
# Make an attempt to upload extended version metadata
extended_version_url = backup_s3_prefix + '/extended_version.txt'
logger.info(
msg='start upload postgres version metadata',
detail=('Uploading to {extended_version_url}.'
.format(extended_version_url=extended_version_url)))
s3_worker.uri_put_file(self.aws_access_key_id,
self.aws_secret_access_key,
extended_version_url, StringIO(version),
content_encoding='text/plain')
logger.info(msg='postgres version metadata upload complete')
uploader = s3_worker.PartitionUploader(
self.aws_access_key_id, self.aws_secret_access_key,
backup_s3_prefix, per_process_limit, self.gpg_key_id)
pool = worker.TarUploadPool(uploader, pool_size)
# Enqueue uploads for parallel execution
for tpart in parts:
total_size += tpart.total_member_size
# 'put' can raise an exception for a just-failed upload,
# aborting the process.
pool.put(tpart)
# Wait for remaining parts to upload. An exception can be
# raised to signal failure of the upload.
pool.join()
return backup_s3_prefix, total_size
def database_s3_backup(self, *args, **kwargs):
"""
Uploads a PostgreSQL file cluster to S3
Mechanism: just wraps _s3_upload_pg_cluster_dir with
start/stop backup actions with exception handling.
In particular there is a 'finally' block to stop the backup in
most situations.
"""
upload_good = False
backup_stop_good = False
try:
start_backup_info = PgBackupStatements.run_start_backup()
version = PgBackupStatements.pg_version()['version']
uploaded_to, expanded_size_bytes = self._s3_upload_pg_cluster_dir(
start_backup_info, version=version, *args, **kwargs)
upload_good = True
finally:
if not upload_good:
logger.warning(
'blocking on sending WAL segments',
detail=('The backup was not completed successfully, '
'but we have to wait anyway. '
'See README: TODO about pg_cancel_backup'))
stop_backup_info = PgBackupStatements.run_stop_backup()
backup_stop_good = True
if upload_good and backup_stop_good:
# Make a best-effort attempt to write a sentinel file to
# the cluster backup directory that indicates that the
# base backup upload has definitely run its course (it may
# have, even without this file, though) and also
# communicates what WAL segments are needed to get to
# consistency.
try:
sentinel_content = StringIO()
json.dump(
{'wal_segment_backup_stop':
stop_backup_info['file_name'],
'wal_segment_offset_backup_stop':
stop_backup_info['file_offset'],
'expanded_size_bytes': expanded_size_bytes},
sentinel_content)
# XXX: distinguish sentinels by *PREFIX* not suffix,
# which makes searching harder. (For the next version
# bump).
s3_worker.uri_put_file(
uploaded_to + '_backup_stop_sentinel.json',
sentinel_content, content_encoding='application/json')
except KeyboardInterrupt, e:
# Specially re-raise exception on SIGINT to allow
# propagation.
raise
except:
def _s3_upload_pg_cluster_dir(self, start_backup_info, pg_cluster_dir,
version, pool_size, rate_limit=None):
"""
Upload to s3_url_prefix from pg_cluster_dir
This function ignores the directory pg_xlog, which contains WAL
files and are not generally part of a base backup.
Note that this is also lzo compresses the files: thus, the number
of pooled processes involves doing a full sequential scan of the
uncompressed Postgres heap file that is pipelined into lzo. Once
lzo is completely finished (necessary to have access to the file
size) the file is sent to S3.
TODO: Investigate an optimization to decouple the compression and
upload steps to make sure that the most efficient possible use of
pipelining of network and disk resources occurs. Right now it
possible to bounce back and forth between bottlenecking on reading
from the database block device and subsequently the S3 sending
steps should the processes be at the same stage of the upload
pipeline: this can have a very negative impact on being able to
make full use of system resources.
Furthermore, it desirable to overflowing the page cache: having
separate tunables for number of simultanious compression jobs
(which occupy /tmp space and page cache) and number of uploads
(which affect upload throughput) would help.
"""
# Get a manifest of files first.
matches = []
def raise_walk_error(e):
raise e
walker = os.walk(pg_cluster_dir, onerror=raise_walk_error)
for root, dirnames, filenames in walker:
is_cluster_toplevel = (os.path.abspath(root) ==
os.path.abspath(pg_cluster_dir))
# Do not capture any WAL files, although we do want to
# capture the WAL directory or symlink
if is_cluster_toplevel:
if 'pg_xlog' in dirnames:
dirnames.remove('pg_xlog')
matches.append(os.path.join(root, 'pg_xlog'))
for filename in filenames:
if is_cluster_toplevel and filename in ('postmaster.pid',
'postgresql.conf'):
# Do not include the postmaster pid file or the
# configuration file in the backup.
pass
else:
matches.append(os.path.join(root, filename))
# Special case for empty directories
if not filenames:
matches.append(root)
backup_s3_prefix = ('{0}/basebackups_{1}/'
'base_{file_name}_{file_offset}'
.format(self.s3_prefix, FILE_STRUCTURE_VERSION,
**start_backup_info))
# absolute upload paths are used for telling lzop what to compress
local_abspaths = [os.path.abspath(match) for match in matches]
# computed to subtract out extra extraneous absolute path
# information when storing on S3
common_local_prefix = os.path.commonprefix(local_abspaths)
partitions = tar_partition.tar_partitions_plan(
common_local_prefix, local_abspaths,
# 1610612736 bytes == 1.5 gigabytes, per partition,
# non-tunable
1610612736)
if rate_limit is None:
per_process_limit = None
else:
per_process_limit = int(rate_limit / pool_size)
# Reject tiny per-process rate limits. They should be
# rejected more nicely elsewhere.
assert per_process_limit > 0 or per_process_limit is None
# a list to accumulate async upload jobs
uploads = []
total_size = 0
# Make an attempt to upload extended version metadata
extended_version_url = backup_s3_prefix + '/extended_version.txt'
logger.info(msg='start upload postgres version metadata',
detail=('Uploading to {extended_version_url}.'
.format(extended_version_url=extended_version_url)))
s3_worker.uri_put_file(extended_version_url, StringIO(version),
content_encoding='text/plain')
logger.info(msg='postgres version metadata upload complete')
pool = gevent.pool.Pool(size=pool_size)
# Enqueue uploads for parallel execution
try:
for tpart in partitions:
total_size += tpart.total_member_size
uploads.append(pool.apply_async(
s3_worker.do_partition_put,
[backup_s3_prefix, tpart, per_process_limit, self.gpg_key_id]))
finally:
while uploads:
uploads.pop().get()
pool.join()
return backup_s3_prefix, total_size
def database_s3_backup(self, data_directory, *args, **kwargs):
"""
Uploads a PostgreSQL file cluster to S3
Mechanism: just wraps _s3_upload_pg_cluster_dir with
start/stop backup actions with exception handling.
In particular there is a 'finally' block to stop the backup in
most situations.
"""
upload_good = False
backup_stop_good = False
while_offline = False
start_backup_info = None
if 'while_offline' in kwargs:
while_offline = kwargs.pop('while_offline')
try:
if not while_offline:
start_backup_info = PgBackupStatements.run_start_backup()
version = PgBackupStatements.pg_version()['version']
else:
if os.path.exists(os.path.join(data_directory, 'postmaster.pid')):
raise UserException(
msg='while_offline set, but pg looks to be running',
detail='Found a postmaster.pid lockfile, and aborting',
hint='Shut down postgres. If there is a stale lockfile, '
'then remove it after being very sure postgres is not '
'running.')
controldata = PgControlDataParser(data_directory)
start_backup_info = controldata.last_xlog_file_name_and_offset()
version = controldata.pg_version()
uploaded_to, expanded_size_bytes = self._s3_upload_pg_cluster_dir(
start_backup_info, data_directory, version=version, *args, **kwargs)
upload_good = True
finally:
if not upload_good:
logger.warning(
'blocking on sending WAL segments',
detail=('The backup was not completed successfully, '
'but we have to wait anyway. '
'See README: TODO about pg_cancel_backup'))
if not while_offline:
stop_backup_info = PgBackupStatements.run_stop_backup()
else:
stop_backup_info = start_backup_info
backup_stop_good = True
# XXX: Ugly, this is more of a 'worker' task because it might
# involve retries and error messages, something that is not
# treated by the "operator" category of modules. So
# basically, if this small upload fails, the whole upload
# fails!
if upload_good and backup_stop_good:
# Try to write a sentinel file to the cluster backup
# directory that indicates that the base backup upload has
# definitely run its course and also communicates what WAL
# segments are needed to get to consistency.
sentinel_content = StringIO()
json.dump(
{'wal_segment_backup_stop':
stop_backup_info['file_name'],
'wal_segment_offset_backup_stop':
stop_backup_info['file_offset'],
'expanded_size_bytes': expanded_size_bytes},
sentinel_content)
# XXX: should use the storage.s3_storage operators.
#
# XXX: distinguish sentinels by *PREFIX* not suffix,
# which makes searching harder. (For the next version
# bump).
sentinel_content.seek(0)
s3_worker.uri_put_file(
uploaded_to + '_backup_stop_sentinel.json',
sentinel_content, content_encoding='application/json')
else:
# NB: Other exceptions should be raised before this that
# have more informative results, it is intended that this
# exception never will get raised.
raise UserCritical('could not complete backup process')
def _s3_upload_pg_cluster_dir(self,
start_backup_info,
pg_cluster_dir,
version,
pool_size,
rate_limit=None):
"""
Upload to s3_url_prefix from pg_cluster_dir
This function ignores the directory pg_xlog, which contains WAL
files and are not generally part of a base backup.
Note that this is also lzo compresses the files: thus, the number
of pooled processes involves doing a full sequential scan of the
uncompressed Postgres heap file that is pipelined into lzo. Once
lzo is completely finished (necessary to have access to the file
size) the file is sent to S3.
TODO: Investigate an optimization to decouple the compression and
upload steps to make sure that the most efficient possible use of
pipelining of network and disk resources occurs. Right now it
possible to bounce back and forth between bottlenecking on reading
from the database block device and subsequently the S3 sending
steps should the processes be at the same stage of the upload
pipeline: this can have a very negative impact on being able to
make full use of system resources.
Furthermore, it desirable to overflowing the page cache: having
separate tunables for number of simultanious compression jobs
(which occupy /tmp space and page cache) and number of uploads
(which affect upload throughput) would help.
"""
parts = tar_partition.partition(pg_cluster_dir)
backup_s3_prefix = ('{0}/basebackups_{1}/'
'base_{file_name}_{file_offset}'.format(
self.s3_prefix, FILE_STRUCTURE_VERSION,
**start_backup_info))
if rate_limit is None:
per_process_limit = None
else:
per_process_limit = int(rate_limit / pool_size)
# Reject tiny per-process rate limits. They should be
# rejected more nicely elsewhere.
assert per_process_limit > 0 or per_process_limit is None
total_size = 0
# Make an attempt to upload extended version metadata
extended_version_url = backup_s3_prefix + '/extended_version.txt'
logger.info(msg='start upload postgres version metadata',
detail=('Uploading to {extended_version_url}.'.format(
extended_version_url=extended_version_url)))
s3_worker.uri_put_file(self.aws_access_key_id,
self.aws_secret_access_key,
extended_version_url,
StringIO(version),
content_encoding='text/plain')
logger.info(msg='postgres version metadata upload complete')
uploader = s3_worker.PartitionUploader(self.aws_access_key_id,
self.aws_secret_access_key,
backup_s3_prefix,
per_process_limit,
self.gpg_key_id)
pool = worker.TarUploadPool(uploader, pool_size)
# Enqueue uploads for parallel execution
for tpart in parts:
total_size += tpart.total_member_size
# 'put' can raise an exception for a just-failed upload,
# aborting the process.
pool.put(tpart)
# Wait for remaining parts to upload. An exception can be
# raised to signal failure of the upload.
pool.join()
return backup_s3_prefix, total_size