Esempio n. 1
0
def do_lzop_put(creds, url, local_path, gpg_key):
    """
    Compress and upload a given local path.

    :type url: string
    :param url: A (s3|wabs)://bucket/key style URL that is the destination

    :type local_path: string
    :param local_path: a path to a file to be compressed

    """
    assert url.endswith('.lzo')
    blobstore = get_blobstore(storage.StorageLayout(url))

    with tempfile.NamedTemporaryFile(mode='r+b') as tf:
        pipeline = get_upload_pipeline(
            open(local_path, 'r'), tf, gpg_key=gpg_key)
        pipeline.finish()

        tf.flush()

        clock_start = time.time()
        tf.seek(0)
        k = blobstore.uri_put_file(creds, url, tf)
        clock_finish = time.time()

        kib_per_second = format_kib_per_second(
            clock_start, clock_finish, k.size)

        return kib_per_second
Esempio n. 2
0
 def __init__(self,
              access_key, secret_key, backup_prefix, rate_limit, gpg_key):
     self.backup_prefix = backup_prefix
     self.access_key = access_key
     self.secret_key = secret_key
     self.backup_prefix = backup_prefix
     self.rate_limit = rate_limit
     self.gpg_key = gpg_key
     self.blobstore = get_blobstore(storage.StorageLayout(backup_prefix))
Esempio n. 3
0
def do_lzop_get(creds, url, path, decrypt):
    """
    Get and decompress an S3 or WABS URL

    This streams the content directly to lzop; the compressed version
    is never stored on disk.

    """
    blobstore = get_blobstore(storage.StorageLayout(url))
    return blobstore.do_lzop_get(creds, url, path, decrypt)
Esempio n. 4
0
 def __init__(self, creds, backup_prefix, rate_limit, gpg_key):
     self.creds = creds
     self.backup_prefix = backup_prefix
     self.rate_limit = rate_limit
     self.gpg_key = gpg_key
     self.blobstore = get_blobstore(storage.StorageLayout(backup_prefix))
Esempio n. 5
0
 def __init__(self, layout, creds, gpg_key_id):
     self.layout = layout
     self.creds = creds
     self.gpg_key_id = gpg_key_id
     self.blobstore = get_blobstore(layout)
Esempio n. 6
0
 def __init__(self, creds, backup_prefix, rate_limit, gpg_key):
     self.creds = creds
     self.backup_prefix = backup_prefix
     self.rate_limit = rate_limit
     self.gpg_key = gpg_key
     self.blobstore = get_blobstore(storage.StorageLayout(backup_prefix))
Esempio n. 7
0
 def __init__(self, layout, creds, gpg_key_id):
     self.layout = layout
     self.creds = creds
     self.gpg_key_id = gpg_key_id
     self.blobstore = get_blobstore(layout)
Esempio n. 8
0
def uri_put_file(creds, uri, fp, content_encoding=None):
    blobstore = get_blobstore(storage.StorageLayout(uri))
    return blobstore.uri_put_file(creds, uri, fp,
                                  content_encoding=content_encoding)
Esempio n. 9
0
 def __init__(self, access_key, secret_key, prefix, gpg_key_id):
     self.access_key = access_key
     self.secret_key = secret_key
     self.prefix = prefix
     self.gpg_key_id = gpg_key_id
     self.blobstore = get_blobstore(storage.StorageLayout(prefix))