예제 #1
0
def upload_rotate(file_path, s3_bucket, s3_key_prefix, aws_key=None, aws_secret=None):
    '''
    Upload file_path to s3 bucket with prefix
    Ex. upload('/tmp/file-2015-01-01.tar.bz2', 'backups', 'foo.net/')
    would upload file to bucket backups with key=foo.net/file-2015-01-01.tar.bz2
    and then rotate all files starting with foo.net/file and with extension .tar.bz2
    Timestamps need to be present between the file root and the extension and in the same format as strftime("%Y-%m-%d").
    Ex file-2015-12-28.tar.bz2
    '''
    key = ''.join([s3_key_prefix, os.path.basename(file_path)])
    logger.debug("Uploading {0} to {1}".format(file_path, key))
    multipart_upload(s3_bucket, aws_key, aws_secret, file_path, key, False, False, None, 0)

    file_root, file_ext = splitext(os.path.basename(file_path))
    # strip timestamp from file_base
    regex = '(?P<filename>.*)-(?P<year>[\d]+?)-(?P<month>[\d]+?)-(?P<day>[\d]+?)'
    match = re.match(regex, file_root)
    if not match:
        raise Exception('File does not contain a timestamp')
    key_prefix = ''.join([s3_key_prefix, match.group('filename')])
    logger.debug('Rotating files on S3 with key prefix {0} and extension {1}'.format(key_prefix, file_ext))
    rotate(key_prefix, file_ext, s3_bucket, aws_key=aws_key, aws_secret=aws_secret)
예제 #2
0
#!/usr/bin/env python

import argparse
import os
import boto
from datetime import datetime
from dcu.active_memory.rotate import rotate, splitext
from dcu.active_memory.upload import multipart_upload

parser = argparse.ArgumentParser(description='Upload a file to Amazon S3 and rotate old backups.')
parser.add_argument('bucket', help="Name of the Amazon S3 bucket to save the backup file to.")
parser.add_argument('prefix', help="The prefix to add before the filename for the key.")
parser.add_argument('file', help="Path to the file to upload.")
args = parser.parse_args()

file_path = args.file
basename = os.path.basename(file_path)
key_base, key_ext = list(splitext(basename))
key_prefix = "".join([args.prefix, key_base])
key_datestamp = datetime.utcnow().date().strftime("-%Y-%m-%d")
key = "".join([key_prefix, key_datestamp, key_ext])

print("Uploading {0} to {1}".format(file_path, key))
multipart_upload(args.bucket, None, None, file_path, key, False, 0, None, 0)

print('Rotating file on S3 with key prefix {0} and extension {1}'.format(key_prefix, key_ext))
rotate(key_prefix, key_ext, args.bucket)