def download():
with open(path, 'wb') as decomp_out:
with get_download_pipeline(PIPE, decomp_out, decrypt) as pl:
conn = calling_format.connect(creds)
g = gevent.spawn(write_and_return_error, uri, conn, pl.stdin)
# Raise any exceptions from write_and_return_error
try:
exc = g.get()
if exc is not None:
raise exc
except ClientException as e:
if e.http_status == 404:
# Do not retry if the key not present, this
# can happen under normal situations.
pl.abort()
logger.warning(
msg=('could no longer locate object while '
'performing wal restore'),
detail=('The absolute URI that could not be '
'located is {uri}.'.format(uri=uri)),
hint=('This can be normal when Postgres is trying '
'to detect what timelines are available '
'during restoration.'))
return False
else:
raise
logger.info(
msg='completed download and decompression',
detail='Downloaded and decompressed "{uri}" to "{path}"'
.format(uri=uri, path=path))
return True
def uri_get_file(creds, uri, conn=None):
assert uri.startswith('swift://')
url_tup = urlparse(uri)
container_name = url_tup.netloc
object_name = url_tup.path
if conn is None:
conn = calling_format.connect(creds)
_, content = conn.get_object(container_name, object_name)
return content
def uri_get_file(creds, uri, conn=None):
assert uri.startswith('swift://')
url_tup = urlparse(uri)
container_name = url_tup.netloc
object_name = url_tup.path
if conn is None:
conn = calling_format.connect(creds)
_, content = conn.get_object(container_name, object_name)
return content
def uri_put_file(creds, uri, fp, content_encoding=None):
assert fp.tell() == 0
assert uri.startswith('swift://')
url_tup = urlparse(uri)
container_name = url_tup.netloc
conn = calling_format.connect(creds)
data = fp.read()
conn.put_object(
container_name, url_tup.path, data, content_type=content_encoding
)
return SwiftKey(url_tup.path, len(data))
def uri_put_file(creds, uri, fp, content_encoding=None):
assert fp.tell() == 0
assert uri.startswith('swift://')
url_tup = urlparse(uri)
container_name = url_tup.netloc
conn = calling_format.connect(creds)
data = fp.read()
conn.put_object(container_name,
url_tup.path,
data,
content_type=content_encoding)
return SwiftKey(url_tup.path, len(data))
def uri_put_file(creds, uri, fp, content_encoding=None):
assert fp.tell() == 0
assert uri.startswith('swift://')
url_tup = urlparse(uri)
container_name = url_tup.netloc
conn = calling_format.connect(creds)
conn.put_object(
container_name, url_tup.path, fp, content_type=content_encoding
)
# Swiftclient doesn't return us the total file size, we see how much of the
# file swiftclient read in order to determine the file size.
return SwiftKey(url_tup.path, size=fp.tell())
def download():
with open(path, 'wb') as decomp_out:
pipeline = get_download_pipeline(PIPE, decomp_out, decrypt)
conn = calling_format.connect(creds)
g = gevent.spawn(write_and_return_error, uri, conn, pipeline.stdin)
# Raise any exceptions from write_and_return_error
exc = g.get()
if exc is not None:
raise exc
pipeline.finish()
logger.info(
msg='completed download and decompression',
detail='Downloaded and decompressed "{uri}" to "{path}"'.
format(uri=uri, path=path))
return True
def download():
with open(path, 'wb') as decomp_out:
pipeline = get_download_pipeline(PIPE, decomp_out, decrypt)
conn = calling_format.connect(creds)
g = gevent.spawn(write_and_return_error, uri, conn, pipeline.stdin)
# Raise any exceptions from write_and_return_error
exc = g.get()
if exc is not None:
raise exc
pipeline.finish()
logger.info(
msg='completed download and decompression',
detail='Downloaded and decompressed "{uri}" to "{path}"'
.format(uri=uri, path=path))
return True