def download(): with open(path, 'wb') as decomp_out: with get_download_pipeline(PIPE, decomp_out, decrypt) as pl: conn = calling_format.connect(creds) g = gevent.spawn(write_and_return_error, uri, conn, pl.stdin) # Raise any exceptions from write_and_return_error try: exc = g.get() if exc is not None: raise exc except ClientException as e: if e.http_status == 404: # Do not retry if the key not present, this # can happen under normal situations. pl.abort() logger.warning( msg=('could no longer locate object while ' 'performing wal restore'), detail=('The absolute URI that could not be ' 'located is {uri}.'.format(uri=uri)), hint=('This can be normal when Postgres is trying ' 'to detect what timelines are available ' 'during restoration.')) return False else: raise logger.info( msg='completed download and decompression', detail='Downloaded and decompressed "{uri}" to "{path}"' .format(uri=uri, path=path)) return True
def uri_get_file(creds, uri, conn=None): assert uri.startswith('swift://') url_tup = urlparse(uri) container_name = url_tup.netloc object_name = url_tup.path if conn is None: conn = calling_format.connect(creds) _, content = conn.get_object(container_name, object_name) return content
def uri_put_file(creds, uri, fp, content_encoding=None): assert fp.tell() == 0 assert uri.startswith('swift://') url_tup = urlparse(uri) container_name = url_tup.netloc conn = calling_format.connect(creds) data = fp.read() conn.put_object( container_name, url_tup.path, data, content_type=content_encoding ) return SwiftKey(url_tup.path, len(data))
def uri_put_file(creds, uri, fp, content_encoding=None): assert fp.tell() == 0 assert uri.startswith('swift://') url_tup = urlparse(uri) container_name = url_tup.netloc conn = calling_format.connect(creds) data = fp.read() conn.put_object(container_name, url_tup.path, data, content_type=content_encoding) return SwiftKey(url_tup.path, len(data))
def uri_put_file(creds, uri, fp, content_encoding=None): assert fp.tell() == 0 assert uri.startswith('swift://') url_tup = urlparse(uri) container_name = url_tup.netloc conn = calling_format.connect(creds) conn.put_object( container_name, url_tup.path, fp, content_type=content_encoding ) # Swiftclient doesn't return us the total file size, we see how much of the # file swiftclient read in order to determine the file size. return SwiftKey(url_tup.path, size=fp.tell())
def download(): with open(path, 'wb') as decomp_out: pipeline = get_download_pipeline(PIPE, decomp_out, decrypt) conn = calling_format.connect(creds) g = gevent.spawn(write_and_return_error, uri, conn, pipeline.stdin) # Raise any exceptions from write_and_return_error exc = g.get() if exc is not None: raise exc pipeline.finish() logger.info( msg='completed download and decompression', detail='Downloaded and decompressed "{uri}" to "{path}"'. format(uri=uri, path=path)) return True
def download(): with open(path, 'wb') as decomp_out: pipeline = get_download_pipeline(PIPE, decomp_out, decrypt) conn = calling_format.connect(creds) g = gevent.spawn(write_and_return_error, uri, conn, pipeline.stdin) # Raise any exceptions from write_and_return_error exc = g.get() if exc is not None: raise exc pipeline.finish() logger.info( msg='completed download and decompression', detail='Downloaded and decompressed "{uri}" to "{path}"' .format(uri=uri, path=path)) return True