def do_lzop_put(creds, url, local_path, gpg_key): """ Compress and upload a given local path. :type url: string :param url: A (s3|wabs)://bucket/key style URL that is the destination :type local_path: string :param local_path: a path to a file to be compressed """ assert url.endswith('.lzo') blobstore = get_blobstore(storage.StorageLayout(url)) with tempfile.NamedTemporaryFile(mode='r+b') as tf: pipeline = get_upload_pipeline( open(local_path, 'r'), tf, gpg_key=gpg_key) pipeline.finish() tf.flush() clock_start = time.time() tf.seek(0) k = blobstore.uri_put_file(creds, url, tf) clock_finish = time.time() kib_per_second = format_kib_per_second( clock_start, clock_finish, k.size) return kib_per_second
def __init__(self, access_key, secret_key, backup_prefix, rate_limit, gpg_key): self.backup_prefix = backup_prefix self.access_key = access_key self.secret_key = secret_key self.backup_prefix = backup_prefix self.rate_limit = rate_limit self.gpg_key = gpg_key self.blobstore = get_blobstore(storage.StorageLayout(backup_prefix))
def do_lzop_get(creds, url, path, decrypt): """ Get and decompress an S3 or WABS URL This streams the content directly to lzop; the compressed version is never stored on disk. """ blobstore = get_blobstore(storage.StorageLayout(url)) return blobstore.do_lzop_get(creds, url, path, decrypt)
def __init__(self, creds, backup_prefix, rate_limit, gpg_key): self.creds = creds self.backup_prefix = backup_prefix self.rate_limit = rate_limit self.gpg_key = gpg_key self.blobstore = get_blobstore(storage.StorageLayout(backup_prefix))
def __init__(self, layout, creds, gpg_key_id): self.layout = layout self.creds = creds self.gpg_key_id = gpg_key_id self.blobstore = get_blobstore(layout)
def uri_put_file(creds, uri, fp, content_encoding=None): blobstore = get_blobstore(storage.StorageLayout(uri)) return blobstore.uri_put_file(creds, uri, fp, content_encoding=content_encoding)
def __init__(self, access_key, secret_key, prefix, gpg_key_id): self.access_key = access_key self.secret_key = secret_key self.prefix = prefix self.gpg_key_id = gpg_key_id self.blobstore = get_blobstore(storage.StorageLayout(prefix))