def wal_s3_archive(self, wal_path, concurrency=1):
"""
Uploads a WAL file to S3
This code is intended to typically be called from Postgres's
archive_command feature.
"""
# Upload the segment expressly indicated. It's special
# relative to other uploads when parallel wal-push is enabled,
# in that it's not desirable to tweak its .ready/.done files
# in archive_status.
xlog_dir = path.dirname(wal_path)
segment = worker.WalSegment(wal_path, explicit=True)
uploader = s3_worker.WalUploader(self.aws_access_key_id,
self.aws_secret_access_key,
self.s3_prefix, self.gpg_key_id)
group = worker.WalTransferGroup(uploader)
group.start(segment)
# Upload any additional wal segments up to the specified
# concurrency by scanning the Postgres archive_status
# directory.
started = 1
seg_stream = worker.WalSegment.from_ready_archive_status(xlog_dir)
while started < concurrency:
try:
other_segment = seg_stream.next()
except StopIteration:
break
if other_segment.path != wal_path:
group.start(other_segment)
started += 1
# Wait for uploads to finish.
group.join()
def make_segment(num, **kwargs):
return worker.WalSegment('pg_xlog/' + str(num) * 8 * 3, **kwargs)
def seg():
return worker.WalSegment('0' * 8 * 3)