def wal_restore(self, wal_name, wal_destination):
"""
Downloads a WAL file from S3 or Windows Azure Blob Service
This code is intended to typically be called from Postgres's
restore_command feature.
NB: Postgres doesn't guarantee that wal_name ==
basename(wal_path), so both are required.
"""
# TODO :: Move arbitray path construction to StorageLayout Object
url = '{0}/wal_{1}/{2}.lzo'.format(
self.layout.prefix.rstrip('/'), FILE_STRUCTURE_VERSION, wal_name)
logger.info(
msg='begin wal restore',
structured={'action': 'wal-fetch',
'key': url,
'seg': wal_name,
'prefix': self.layout.path_prefix,
'state': 'begin'})
ret = do_lzop_get(self.creds, url, wal_destination,
self.gpg_key_id is not None)
logger.info(
msg='complete wal restore',
structured={'action': 'wal-fetch',
'key': url,
'seg': wal_name,
'prefix': self.layout.path_prefix,
'state': 'complete'})
return ret
def wal_restore(self, wal_name, wal_destination):
"""
Downloads a WAL file from S3 or Windows Azure Blob Service
This code is intended to typically be called from Postgres's
restore_command feature.
NB: Postgres doesn't guarantee that wal_name ==
basename(wal_path), so both are required.
"""
# TODO :: Move arbitray path construction to StorageLayout Object
url = '{0}/wal_{1}/{2}.lzo'.format(
self.layout.prefix.rstrip('/'), FILE_STRUCTURE_VERSION, wal_name)
logger.info(
msg='begin wal restore',
structured={'action': 'wal-fetch',
'key': url,
'seg': wal_name,
'prefix': self.layout.path_prefix,
'state': 'begin'})
ret = do_lzop_get(self.creds, url, wal_destination,
self.gpg_key_id is not None)
logger.info(
msg='complete wal restore',
structured={'action': 'wal-fetch',
'key': url,
'seg': wal_name,
'prefix': self.layout.path_prefix,
'state': 'complete'})
return ret
def wal_prefetch(self, base, segment_name):
url = '{0}://{1}/{2}'.format(self.layout.scheme,
self.layout.store_name(),
self.layout.wal_path(segment_name))
pd = prefetch.Dirs(base)
seg = WalSegment(segment_name)
pd.create(seg)
with pd.download(seg) as d:
logger.info(msg='begin wal restore',
structured={
'action': 'wal-prefetch',
'key': url,
'seg': segment_name,
'prefix': self.layout.path_prefix,
'state': 'begin'
})
ret = do_lzop_get(self.creds,
url,
d.dest,
self.gpg_key_id is not None,
do_retry=False)
logger.info(msg='complete wal restore',
structured={
'action': 'wal-prefetch',
'key': url,
'seg': segment_name,
'prefix': self.layout.path_prefix,
'state': 'complete'
})
return ret
def wal_prefetch(self, base, segment_name):
url = '{0}://{1}/{2}'.format(
self.layout.scheme, self.layout.store_name(),
self.layout.wal_path(segment_name))
pd = prefetch.Dirs(base)
seg = WalSegment(segment_name)
pd.create(seg)
with pd.download(seg) as d:
logger.info(
msg='begin wal restore',
structured={'action': 'wal-prefetch',
'key': url,
'seg': segment_name,
'prefix': self.layout.path_prefix,
'state': 'begin'})
ret = do_lzop_get(self.creds, url, d.dest,
self.gpg_key_id is not None, do_retry=False)
logger.info(
msg='complete wal restore',
structured={'action': 'wal-prefetch',
'key': url,
'seg': segment_name,
'prefix': self.layout.path_prefix,
'state': 'complete'})
return ret
def wal_prefetch(self, base, segment_name):
url = '{0}://{1}/{2}'.format(self.layout.scheme,
self.layout.store_name(),
self.layout.wal_path(segment_name))
pd = prefetch.Dirs(base)
seg = WalSegment(segment_name)
pd.create(seg)
with pd.download(seg) as d:
logger.info(msg='begin wal restore',
structured={
'action': 'wal-prefetch',
'key': url,
'seg': segment_name,
'prefix': self.layout.path_prefix,
'state': 'begin'
})
ret = do_lzop_get(self.creds,
url,
d.dest,
self.gpg_key_id is not None,
do_retry=False)
if not ret:
# If the download failed, AtomicDownload.__exit__()
# must be informed so that it does not link an empty
# archive file into place.
#
# We thus raise SystemExit. This is acceptable for
# prefetch since prefetch execution is daemonized.
# I.e., PostgreSQL has no knowledge of prefetch
# exit codes.
raise SystemExit('Failed to prefetch %s' % segment_name)
logger.info(msg='complete wal restore',
structured={
'action': 'wal-prefetch',
'key': url,
'seg': segment_name,
'prefix': self.layout.path_prefix,
'state': 'complete'
})
return ret
def wal_prefetch(self, base, segment_name):
url = '{0}://{1}/{2}'.format(
self.layout.scheme, self.layout.store_name(),
self.layout.wal_path(segment_name))
pd = prefetch.Dirs(base)
seg = WalSegment(segment_name)
pd.create(seg)
with pd.download(seg) as d:
logger.info(
msg='begin wal restore',
structured={'action': 'wal-prefetch',
'key': url,
'seg': segment_name,
'prefix': self.layout.path_prefix,
'state': 'begin'})
ret = do_lzop_get(self.creds, url, d.dest,
self.gpg_key_id is not None, do_retry=False)
if not ret:
# If the download failed, AtomicDownload.__exit__()
# must be informed so that it does not link an empty
# archive file into place.
#
# We thus raise SystemExit. This is acceptable for
# prefetch since prefetch execution is daemonized.
# I.e., PostgreSQL has no knowledge of prefetch
# exit codes.
raise SystemExit('Failed to prefetch %s' % segment_name)
logger.info(
msg='complete wal restore',
structured={'action': 'wal-prefetch',
'key': url,
'seg': segment_name,
'prefix': self.layout.path_prefix,
'state': 'complete'})
return ret
def wal_restore(self, wal_name, wal_destination, prefetch_max):
"""
Downloads a WAL file from S3 or Windows Azure Blob Service
This code is intended to typically be called from Postgres's
restore_command feature.
NB: Postgres doesn't guarantee that wal_name ==
basename(wal_path), so both are required.
"""
url = '{0}://{1}/{2}'.format(self.layout.scheme,
self.layout.store_name(),
self.layout.wal_path(wal_name))
if prefetch_max > 0:
# Check for prefetch-hit.
base = os.path.dirname(os.path.realpath(wal_destination))
pd = prefetch.Dirs(base)
seg = WalSegment(wal_name)
started = start_prefetches(seg, pd, prefetch_max)
last_size = 0
while True:
if pd.contains(seg):
pd.promote(seg, wal_destination)
logger.info(msg='promoted prefetched wal segment',
structured={
'action': 'wal-fetch',
'key': url,
'seg': wal_name,
'prefix': self.layout.path_prefix
})
pd.clear_except(started)
return True
# If there is a 'running' download, wait a bit for it
# to make progress or finish. However, if it doesn't
# make progress in some amount of time, assume that
# the prefetch process has died and go on with the
# in-band downloading code.
sz = pd.running_size(seg)
if sz <= last_size:
break
last_size = sz
gevent.sleep(0.5)
pd.clear_except(started)
logger.info(msg='begin wal restore',
structured={
'action': 'wal-fetch',
'key': url,
'seg': wal_name,
'prefix': self.layout.path_prefix,
'state': 'begin'
})
ret = do_lzop_get(self.creds, url, wal_destination, self.gpg_key_id
is not None)
logger.info(msg='complete wal restore',
structured={
'action': 'wal-fetch',
'key': url,
'seg': wal_name,
'prefix': self.layout.path_prefix,
'state': 'complete'
})
return ret
def wal_restore(self, wal_name, wal_destination, prefetch_max):
"""
Downloads a WAL file from S3 or Windows Azure Blob Service
This code is intended to typically be called from Postgres's
restore_command feature.
NB: Postgres doesn't guarantee that wal_name ==
basename(wal_path), so both are required.
"""
url = '{0}://{1}/{2}'.format(
self.layout.scheme, self.layout.store_name(),
self.layout.wal_path(wal_name))
if prefetch_max > 0:
# Check for prefetch-hit.
base = os.path.dirname(os.path.realpath(wal_destination))
pd = prefetch.Dirs(base)
seg = WalSegment(wal_name)
started = start_prefetches(seg, pd, prefetch_max)
last_size = 0
while True:
if pd.contains(seg):
pd.promote(seg, wal_destination)
logger.info(
msg='promoted prefetched wal segment',
structured={'action': 'wal-fetch',
'key': url,
'seg': wal_name,
'prefix': self.layout.path_prefix})
pd.clear_except(started)
return True
# If there is a 'running' download, wait a bit for it
# to make progress or finish. However, if it doesn't
# make progress in some amount of time, assume that
# the prefetch process has died and go on with the
# in-band downloading code.
sz = pd.running_size(seg)
if sz <= last_size:
break
last_size = sz
gevent.sleep(0.5)
pd.clear_except(started)
logger.info(
msg='begin wal restore',
structured={'action': 'wal-fetch',
'key': url,
'seg': wal_name,
'prefix': self.layout.path_prefix,
'state': 'begin'})
ret = do_lzop_get(self.creds, url, wal_destination,
self.gpg_key_id is not None)
logger.info(
msg='complete wal restore',
structured={'action': 'wal-fetch',
'key': url,
'seg': wal_name,
'prefix': self.layout.path_prefix,
'state': 'complete'})
return ret