def upload_rotate(file_path, s3_bucket, s3_key_prefix, aws_key=None, aws_secret=None):
'''
Upload file_path to s3 bucket with prefix
Ex. upload('/tmp/file-2015-01-01.tar.bz2', 'backups', 'foo.net/')
would upload file to bucket backups with key=foo.net/file-2015-01-01.tar.bz2
and then rotate all files starting with foo.net/file and with extension .tar.bz2
Timestamps need to be present between the file root and the extension and in the same format as strftime("%Y-%m-%d").
Ex file-2015-12-28.tar.bz2
'''
key = ''.join([s3_key_prefix, os.path.basename(file_path)])
logger.debug("Uploading {0} to {1}".format(file_path, key))
multipart_upload(s3_bucket, aws_key, aws_secret, file_path, key, False, False, None, 0)
file_root, file_ext = splitext(os.path.basename(file_path))
# strip timestamp from file_base
regex = '(?P<filename>.*)-(?P<year>[\d]+?)-(?P<month>[\d]+?)-(?P<day>[\d]+?)'
match = re.match(regex, file_root)
if not match:
raise Exception('File does not contain a timestamp')
key_prefix = ''.join([s3_key_prefix, match.group('filename')])
logger.debug('Rotating files on S3 with key prefix {0} and extension {1}'.format(key_prefix, file_ext))
rotate(key_prefix, file_ext, s3_bucket, aws_key=aws_key, aws_secret=aws_secret)
#!/usr/bin/env python
import argparse
import os
import boto
from datetime import datetime
from dcu.active_memory.rotate import rotate, splitext
from dcu.active_memory.upload import multipart_upload
parser = argparse.ArgumentParser(description='Upload a file to Amazon S3 and rotate old backups.')
parser.add_argument('bucket', help="Name of the Amazon S3 bucket to save the backup file to.")
parser.add_argument('prefix', help="The prefix to add before the filename for the key.")
parser.add_argument('file', help="Path to the file to upload.")
args = parser.parse_args()
file_path = args.file
basename = os.path.basename(file_path)
key_base, key_ext = list(splitext(basename))
key_prefix = "".join([args.prefix, key_base])
key_datestamp = datetime.utcnow().date().strftime("-%Y-%m-%d")
key = "".join([key_prefix, key_datestamp, key_ext])
print("Uploading {0} to {1}".format(file_path, key))
multipart_upload(args.bucket, None, None, file_path, key, False, 0, None, 0)
print('Rotating file on S3 with key prefix {0} and extension {1}'.format(key_prefix, key_ext))
rotate(key_prefix, key_ext, args.bucket)
