def create_distribution(access_key_id, secret_access_key, origin, comment="", cnames=None):
import time
from boto.cloudfront import CloudFrontConnection
"""utility function to create a new distribution"""
c = CloudFrontConnection(
access_key_id,
secret_access_key
)
d = c.create_distribution(origin, True, '', cnames, comment)
print """Created distribution
- domain name: %s
- origin: %s
- status: %s
- comment: %s
- id: %s
Over the next few minutes, the distribution will become active. This
function will keep running until that happens.
""" % (d.domain_name, d.config.origin, d.status, d.config.comment, d.id)
# Keep polling CloudFront every 5 seconds until the status changes from
# "InProgress" to (hopefully) "Deployed".
print "\n"
id = d.id
while d.status == "InProgress":
d = c.get_distribution_info(id)
print "."
time.sleep(5)
print "\nThe distribution has been deployed!"
def invalidate(paths):
try:
print("Invalidating...")
conn = CloudFrontConnection()
conn.create_invalidation_request(AWS_CF_DISTRIBUTION_ID, paths)
return True
except:
return False
def handle_noargs(self, **options):
try:
access_key = settings.AWS_ACCESS_KEY_ID
secret_key = settings.AWS_SECRET_ACCESS_KEY
distribution = settings.AWS_CF_DISTRIBUTION_ID
static_url = settings.STATIC_URL
except AttributeError:
raise ImproperlyConfigured(
'Please specify AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,'
' AWS_CF_DISTRIBUTION_ID in settings\n')
super(Command, self).handle_noargs(**options)
invalid_files = self.copied_files + self.symlinked_files
if not len(invalid_files):
self.stdout.write('Nothing to invalidate\n')
return
self.stdout.write('Invalidating files...\n')
# we need add static prefix to each address
static_prefix = urlsplit(static_url).path
invalid_files = \
[urljoin(static_prefix, path) for path in invalid_files if path]
invalidation_step = 1000
invalication_max_num = 3
invalidation_max_files = invalidation_step * invalication_max_num
def dump(invalid_files):
import tempfile
from django.utils import simplejson
dump = tempfile.NamedTemporaryFile(delete=False)
dump.write(simplejson.dumps(invalid_files, indent=2))
dump.close()
return dump.name
if len(invalid_files) > invalidation_max_files:
raise AssertionError(
"There are more than %s files to invalidate, "
"AWS doesn't support it at the moment.\n"
"File names to be invalidated dumped to JSON file %s, "
"please invalidate manually\n"""
% (invalidation_max_files, dump(invalid_files)))
conn = CloudFrontConnection(access_key, secret_key)
try:
for i in xrange(invalication_max_num):
thousand = invalid_files[i:i + invalidation_step]
if not thousand: break
conn.create_invalidation_request(distribution, thousand)
except exception.CloudFrontServerError, message:
self.stderr.write('Error while processing:\n\n%s\n\n'
'Unprocessed files dumped to JSON file %s, '
'please invalidate manually\n'
% (message, dump(invalid_files)))
def do_invalidate(distrib_id, inval_keys, creds):
'''
Send a CloudFront invalidation request for the given objects.
'''
cf = CloudFrontConnection(**creds.as_dict())
distrib = get_distribution(cf, distrib_id)
req = cf.create_invalidation_request(distrib.id, inval_keys)
pbar = s3pub.progress.InvalidationProgressBar(req.id)
for _ in pbar(Monitor(cf, distrib_id, req.id)):
pass
print('Done.')
def distribute(s3_url):
c = CloudFrontConnection(config['aws-access-key'].strip(), config['aws-secret-access-key'].strip())
rs = c.get_all_streaming_distributions()
try:
ds = rs[0]
distro = ds.get_distribution()
except IndexError:
origin = '{}.s3.amazonaws.com'.format(config['aws-bucket-name'])
s3_origin = S3Origin(dns_name=origin)
distro = c.create_streaming_distribution(origin=s3_origin, enabled=True, comment="Streaming Videos")
key_name = s3_url.split('/')[-1]
return 'rtmp://{}/cfx/st/mp4:{}'.format(distro.domain_name, key_name)
def do_invalidation(self):
files = sys.argv[1:]
if len(files) > 0:
print 'Sending invalidation request for the following files:'
for f in files:
print f
conn = CloudFrontConnection(settings.AWS_ACCESS_KEY, settings.AWS_SECRET_ACCESS_KEY)
req = conn.create_invalidation_request(settings.AWS_CF_DISTRIBUTION_ID, files)
print req.status
else:
self.help()
sys.exit()
def to_cdn(c, slug):
"Create a new Distribution object on CloudFront"
from boto.cloudfront import CloudFrontConnection
from boto.cloudfront.origin import CustomOrigin
c = CloudFrontConnection(env.aws_access_key_id, env.aws_secret_access_key)
d = c.create_distribution(
origin=CustomOrigin(slug + ".cdn.readthedocs.org", origin_protocol_policy="http-only"),
enabled=True,
comment="Slug: " + slug,
cnames=[slug + ".readthedocs.org"],
)
print "Created: " + d.domain_name + " for " + slug
list_cdn()
def create_cloudfront_distribution(aws_access_key, aws_secret_key, bucket_endpoint, hostname):
connection = CloudFrontConnection(aws_access_key, aws_secret_key)
origin = CustomOrigin(dns_name=bucket_endpoint, origin_protocol_policy="http-only")
distribution = connection.create_distribution(origin=origin, enabled=True, cnames=[hostname])
print("A CloudFront distribution has been created.")
print("You need to do two things:")
print("1. Go to the DNS provider for {hostname} and set up a CNAME to map it to {distribution_domain}".format(
hostname=hostname, distribution_domain=distribution.domain_name
))
print("2. Go to the AWS control panel, and associate the appropriate SSL cert with distribution {id}".format(
id=distribution.id
))
print("(The latter step is required because boto currently doesn't support setting certificates.)")
class AwsCloudFrontTools(object):
''' Aws CloudFront Tools
:param str aws_access_key_id: aws_access_key_id
:param str aws_secret_access_key: aws_secret_access_key
'''
def __init__(self, aws_access_key_id, aws_secret_access_key):
self.conn = CloudFrontConnection(aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
def get_all_distributions(self, fields=['id', 'cnames', 'domain_name',
'origin']):
''' Get all distributions
:param list fields: Get distributions info fields.
:rtype: dict in list
.. todo::
- Add show all distributions data.
'''
result = []
for target in self.conn.get_all_distributions():
result.append({key: getattr(target, key) for key in fields})
return result
class AmazonCloudFrontDistribution:
"""
Represents a Cloud Front Distribution and its supported actions
"""
def __init__(self, distribution_id):
if (cf_2_gcs.conf.AWS_ACCESS_KEY and cf_2_gcs.conf.AWS_SECRET_KEY):
self.cf_conn = CloudFrontConnection(cf_2_gcs.conf.AWS_ACCESS_KEY, cf_2_gcs.conf.AWS_SECRET_KEY)
try:
self.distribution = Distribution(connection=self.cf_conn, config=None, id=distribution_id)
except Exception:
log.error("Unable to associate distribution %s. Is the ID correct?" % distribution_id)
self.distribution = None
self.distribution_id = distribution_id
def get_root_url(self):
"""
Returns the root URL domain for the instance distribution id
"""
info = self.cf_conn.get_distribution_info(self.distribution_id)
protocol = str(info.config.Protocol)
distribution = str(info.domain_name)
return protocol + "://" + distribution
def get_public_url(self, path):
"""
Returns the public URL for a path
Args:
path (str): The pathname for the file relative to the CloudFront distribution
method and domain. Example: /test/video.mp4.
"""
return self.get_root_url() + path
def get_signed_url(self, path, expire_time=3600):
"""
Signs an URL with a CloudFront private key (for private content)
Args:
path (str): The pathname for the file relative to the CloudFront distribution
method and domain. Example: /test/video.mp4.
expire_time (Optional|int): How many seconds the signed URL will
be valid. Defaults to 3600 (1 hour).
"""
# setup cloudfront trusted keypair
keypair_id = cf_2_gcs.conf.CLOUDFRONT_KEYPAIR_ID
private_key_file = cf_2_gcs.conf.CLOUDFRONT_PRIVATE_KEY_FILE
# setup expire time from now
expire_time = int(time.time()+expire_time)
# append root domain to the url
url = self.get_root_url() + path
return self.distribution.create_signed_url(url=url, keypair_id=keypair_id, private_key_file=private_key_file, expire_time=expire_time)
def get_distribution(is_music=True):
"""
Get CloudFront distribution
"""
try:
cnn = CloudFrontConnection(aws_access_key_id=settings.PINNA_AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.PINNA_AWS_SECRET_KEY_ID)
if is_music:
dis = cnn.get_distribution_info(settings.PINNA_AWS_MUSIC_DATA_ID)
else:
dis = cnn.get_distribution_info(settings.PINNA_AWS_USER_PROFILE_ID)
return dis
except:
msg = traceback.format_exc()
LOG.error(msg)
return None
def __init__(self, distribution_id):
if (cf_2_gcs.conf.AWS_ACCESS_KEY and cf_2_gcs.conf.AWS_SECRET_KEY):
self.cf_conn = CloudFrontConnection(cf_2_gcs.conf.AWS_ACCESS_KEY, cf_2_gcs.conf.AWS_SECRET_KEY)
try:
self.distribution = Distribution(connection=self.cf_conn, config=None, id=distribution_id)
except Exception:
log.error("Unable to associate distribution %s. Is the ID correct?" % distribution_id)
self.distribution = None
self.distribution_id = distribution_id
def invalidate_paths(deploy_type, paths, *args, **kwargs):
"""
Helper function, create an invalidation request for CloudFront distribution
:param deploy_type: The deploy type
:param paths: The paths array
"""
from boto.cloudfront import CloudFrontConnection
# TODO chunking invalidation to prevent error
distributions = get_conf('AWS_DISTRIBUTION_ID', deploy_type=deploy_type)
if isinstance(distributions, list):
distributions = distributions
else:
distributions = [distributions]
for distribution in distributions:
conn_cf = CloudFrontConnection(get_conf('AWS_ACCESS_KEY_ID', deploy_type=deploy_type),
get_conf('AWS_SECRET_ACCESS_KEY', deploy_type=deploy_type))
conn_cf.create_invalidation_request(distribution, paths)
def invalidate_cache(ec2, env, source):
""" Invalidate CloudFront cache for each machine with a Cloudfront Distribution ID"""
# NOTE: Creating distributions is not yet supported, only cache invalidation
cfc = CloudFrontConnection(ec2.access_key,ec2.secret_key)
for machine in env:
if 'cloudfront' in machine.keys():
print 'Invalidating cache for %s' % machine['name']
cloudfront = machine['cloudfront'] # Cloudfront Distribution ID
media_files = []
static_dir = os.path.join(source,'project','static')
for root, dirs, files in os.walk(static_dir):
if '.svn' in dirs: dirs.remove('.svn')
key_root = root.split('static')[1]
for file in files:
filename = os.path.join(root,file)
if os.path.isfile(filename):
media_files.append(os.path.join(key_root,file))
cfc.create_invalidation_request(cloudfront, media_files)
def clear_cloudfront_cache(base_dir,index_file,dir_prefix='',passn=0):
base_dir = os.path.abspath(base_dir)
all_files = get_files_from_dir(base_dir)
if(os.path.exists(index_file)):
data = file(index_file).read()
os.unlink(index_file+'.back')
file(index_file+'.back','w').write(data)
else:
data = ''
file(index_file+'.back','w').write('')
index = {}
data = data.split('\n')
for line in data:
if not line:
continue
path,md5 = line.split('\t#\t')
index[path] = md5
new_files,new_files_raw = get_modified_files(base_dir,all_files,index,dir_prefix)
for filename in index.iterkeys():
if filename not in all_files:
next = False
for it in IGNORE:
if it in filename:
next = True
if next:
continue
new_files.append(os.path.join(dir_prefix,filename.strip('/')))
new_files_raw.append(filename)
if new_files:
for filename in new_files:
print 'Modified: %s' % filename
else:
print 'No files were modified.\n'
sys.exit()
print '\nUploading %s files\n' % len(new_files)
inp = ''
while (inp != 'y' and inp != 'n'):
inp = raw_input('Upload changes to CloudFront(y/n): ')
if inp == 'y':
try:
conn = CloudFrontConnection(AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY)
k = 0
for i in xrange(0,len(new_files),800):
if k < passn:
k += 1
continue
res = True
res = conn.create_invalidation_request(AWS_CF_DISTRIBUTION_ID, new_files[i:i+900])
if res:
print '\nInvalidation request created'
for filename in new_files_raw[i:i+800]:
fc = file(base_dir+filename).read()
index[filename] = hashlib.md5(fc).hexdigest()
if k >= (passn+2):
print '\nToo many files. Repeat update after 15 minutes.'
break
k += 1
except Exception,e:
save_index(index_file,index)
sys.exit('\nError: %s' % e)
save_index(index_file,index)
import sys, datetime, settings
from level3client import Level3Service
from boto.cloudfront import CloudFrontConnection
urls = sys.argv
urls.remove('invcache.py')
scope = settings.level3scope
key_id = settings.level3key
secret = settings.level3secret
service = Level3Service(key_id, secret, method='POST')
result = service('invalidations/%s' % scope, post_data="""
<paths>
%s
</paths>
""" % ('\n'.join(['<path>%s</path>' % url for url in urls]))
)
conn = CloudFrontConnection( settings.awscfkey, settings.awscfsecret )
invreq = conn.create_invalidation_request( settings.awscfdistribution, urls)
print "Level3 Invalidation Request:"
print result.dom.toprettyxml()
print "CloudFront Invalidation Status:"
print invreq.status
#!/usr/bin/env python __author__ = "Stamped ([email protected])" __version__ = "1.0" __copyright__ = "Copyright (c) 2011-2012 Stamped.com" __license__ = "TODO" import Globals import datetime, os, sys, urllib2, utils import keys.aws from boto.cloudfront import CloudFrontConnection if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__) args = parser.parse_args() files = [ '/assets/css/screen.css', '/assets/css/styles.css' ] conn = CloudFrontConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY) # NOTE: this cloudfront distribution id is hardcoded to the static.stamped.com distro distribution_id = 'E17S1XTPOFC1EQ' print conn.create_invalidation_request(distribution_id, files)
def _get_cf():
from boto.cloudfront import CloudFrontConnection
return CloudFrontConnection(
aws_access_key_id=settings.AWS_ACCESS_KEY,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY
)
def invalidate_all(aws_access_key, aws_secret_key, cloudfront_distribution):
print("Invalidating all")
cloudfront_connection = CloudFrontConnection(aws_access_key, aws_secret_key)
cloudfront_connection.create_invalidation_request(cloudfront_distribution, "/*")
target = filename.replace('./target', '')
k = Key(bucket)
k.key = target
print 'uploading', target
if target.split('.')[-1] in ['jpg', 'png', 'gif', 'ico', 'css', 'js', 'txt']:
k.set_contents_from_filename(filename, headers={'Cache-Control': 'max-age=31536000'})
elif target == '/feed/index.html':
k.set_contents_from_filename(filename, headers={'Cache-Control': 'max-age=86400', 'Content-Type': 'application/xml'})
else:
k.set_contents_from_filename(filename, headers={'Cache-Control': 'max-age=86400'})
elif action == 'invalidate':
try:
extension = '.' + sys.argv[2]
except:
usage()
indexes = []
for filename in dirwalk('./target/'):
target = filename.replace('./target', '')
if extension in target:
indexes.append(target)
print 'creating cache invalidation request for %d files ending in "%s"' % (len(indexes), extension)
connection = CloudFrontConnection()
connection.create_invalidation_request(cf_distribution, indexes)
files = ['/css/style.css',
'/favicon.ico',
'/images/harper_6.jpg',
'/images/glider.png',
'/js/scripts.js',
'/css/style.css',
'/images/DateHeaderIco.png',
'/images/PostComLeft.png',
'/images/PostComRight.png',
'/images/PostDetailsTagsIco.png',
'/images/RelatedPostsHeading.png',
'/images/RelatedPostsLi.png',
'/images/CommentsListHeading.png',
'/images/CommentsFormHeading.png',
'/images/SubmitComment.png',
'/images/SubmitComment.png',
'/images/SubmitComment.png',
'/images/CategIco.png',
'/images/tr.png',
'/images/tl.png',
'/images/br.png',
'/images/bl.png',
'/images/SRLBkg',
'/images/SRRBkg',
'/images/close2.png',]
conn = CloudFrontConnection()
print conn.create_invalidation_request(AWS_CF_DISTRIBUTION_ID, files)
def create_invalidation(article, settings):
cloudfront = CloudFrontConnection(settings.aws_access_key_id, settings.aws_secret_access_key)
inval_req = cloudfront.create_invalidation_request(settings.cloudfront_distribution_id_cdn,
["/articles/" + article + "/*"])
assert isinstance(inval_req, InvalidationBatch), \
"Invalidation request did not return expected object."
def invalidate_paths(paths):
cx = CloudFrontConnection(settings.get('aws_access_key'),
settings.get('aws_secret_key'))
distros = cx.get_all_distributions()
return [cx.create_invalidation_request(d.id, paths) for d in distros]
def __init__(self, aws_access_key_id, aws_secret_access_key):
self.conn = CloudFrontConnection(aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
def invalidate_cloudfront():
conn = CloudFrontConnection(env.cdn['aws_access_key'], env.cdn['aws_secret_key'])
print conn.create_invalidation_request(env.cdn['aws_cloudfront_id'], env.cdn['files'])
print 'Invalidated cloudfront cache for ...%s' % '\n\t '.join(env.cdn['files'])
env.cdn['files'] = []
def get_cloudfront_connection(cls):
if cls._connection is None:
cls._connection = CloudFrontConnection(
aws_access_key_id=app.config['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=app.config['AWS_SECRET_ACCESS_KEY'])
return cls._connection
