def wal_archive(self, wal_path, concurrency=1): """ Uploads a WAL file to S3 or Windows Azure Blob Service This code is intended to typically be called from Postgres's archive_command feature. """ # Upload the segment expressly indicated. It's special # relative to other uploads when parallel wal-push is enabled, # in that it's not desirable to tweak its .ready/.done files # in archive_status. xlog_dir = os.path.dirname(wal_path) segment = WalSegment(wal_path, explicit=True) uploader = WalUploader(self.layout, self.creds, self.gpg_key_id) group = WalTransferGroup(uploader) group.start(segment) # Upload any additional wal segments up to the specified # concurrency by scanning the Postgres archive_status # directory. started = 1 seg_stream = WalSegment.from_ready_archive_status(xlog_dir) while started < concurrency: try: other_segment = seg_stream.next() except StopIteration: break if other_segment.path != wal_path: group.start(other_segment) started += 1 # Wait for uploads to finish. group.join()
def wal_archive(self, wal_path, concurrency=1): """ Uploads a WAL file to S3 or Windows Azure Blob Service This code is intended to typically be called from Postgres's archive_command feature. """ # Upload the segment expressly indicated. It's special # relative to other uploads when parallel wal-push is enabled, # in that it's not desirable to tweak its .ready/.done files # in archive_status. xlog_dir = os.path.dirname(wal_path) segment = WalSegment(wal_path, explicit=True) uploader = WalUploader(self.layout, self.creds, self.gpg_key_id) group = WalTransferGroup(uploader) group.start(segment) # Upload any additional wal segments up to the specified # concurrency by scanning the Postgres archive_status # directory. started = 1 seg_stream = WalSegment.from_ready_archive_status(xlog_dir) while started < concurrency: try: other_segment = next(seg_stream) except StopIteration: break if other_segment.path != wal_path: group.start(other_segment) started += 1 try: # Wait for uploads to finish. group.join() except EnvironmentError as e: if e.errno == errno.ENOENT: print(e) raise UserException( msg='could not find file for wal-push', detail=('The operating system reported: {0} {1}'.format( e.strerror, repr(e.filename)))) raise
def wal_archive(self, wal_path, concurrency=1): """ Uploads a WAL file to S3 or Windows Azure Blob Service This code is intended to typically be called from Postgres's archive_command feature. """ # Upload the segment expressly indicated. It's special # relative to other uploads when parallel wal-push is enabled, # in that it's not desirable to tweak its .ready/.done files # in archive_status. xlog_dir = os.path.dirname(wal_path) segment = WalSegment(wal_path, explicit=True) uploader = WalUploader(self.layout, self.creds, self.gpg_key_id) group = WalTransferGroup(uploader) group.start(segment) # Upload any additional wal segments up to the specified # concurrency by scanning the Postgres archive_status # directory. started = 1 seg_stream = WalSegment.from_ready_archive_status(xlog_dir) while started < concurrency: try: other_segment = next(seg_stream) except StopIteration: break if other_segment.path != wal_path: group.start(other_segment) started += 1 try: # Wait for uploads to finish. group.join() except EnvironmentError as e: if e.errno == errno.ENOENT: print(e) raise UserException( msg='could not find file for wal-push', detail=('The operating system reported: {0} {1}' .format(e.strerror, repr(e.filename)))) raise