def main():
argument_spec = aws_common_argument_spec()
argument_spec.update(
dict(
profile_name=dict(required=True),
distribution_id=dict(required=True),
path=dict(required=True),
))
module = AnsibleModule(argument_spec=argument_spec)
if not HAS_BOTO:
module.fail_json(msg='boto 2/3 required for this module')
distribution_id = module.params.get('distribution_id')
path = module.params.get('path')
profile_name = module.params.get('profile_name')
session = Session(profile_name=profile_name).get_credentials()
# connect to Cloudfront
try:
conn = CloudFrontConnection(session.access_key, session.secret_key)
except boto.exception.BotoServerError as e:
module.fail_json(msg=e.error_message)
# Make the invalidation request
invalidation = [path]
conn.create_invalidation_request(distribution_id, invalidation)
module.exit_json(msg="Path %s scheduled for invalidation." %
(distribution_id),
changed=True)
def create_distribution(access_key_id,
secret_access_key,
origin,
comment="",
cnames=None):
import time
from boto.cloudfront import CloudFrontConnection
"""utility function to create a new distribution"""
c = CloudFrontConnection(access_key_id, secret_access_key)
d = c.create_distribution(origin, True, '', cnames, comment)
print """Created distribution
- domain name: %s
- origin: %s
- status: %s
- comment: %s
- id: %s
Over the next few minutes, the distribution will become active. This
function will keep running until that happens.
""" % (d.domain_name, d.config.origin, d.status, d.config.comment, d.id)
# Keep polling CloudFront every 5 seconds until the status changes from
# "InProgress" to (hopefully) "Deployed".
print "\n"
id = d.id
while d.status == "InProgress":
d = c.get_distribution_info(id)
print "."
time.sleep(5)
print "\nThe distribution has been deployed!"
def cloudfront(self):
if self._cloudfront is None:
try:
self._cloudfront = CloudFrontConnection(**self.credentials)
except Exception as e:
msg = 'Error connecting to cloudfront: {}'.format(e)
print(msg)
raise ConfigurationError(msg)
return self._cloudfront
def handle_noargs(self, **options):
try:
access_key = settings.AWS_ACCESS_KEY_ID
secret_key = settings.AWS_SECRET_ACCESS_KEY
distribution = settings.AWS_CF_DISTRIBUTION_ID
static_url = settings.STATIC_URL
except AttributeError:
raise ImproperlyConfigured(
'Please specify AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,'
' AWS_CF_DISTRIBUTION_ID in settings\n')
super(Command, self).handle_noargs(**options)
invalid_files = self.copied_files + self.symlinked_files
if not len(invalid_files):
self.stdout.write('Nothing to invalidate\n')
return
self.stdout.write('Invalidating files...\n')
# we need add static prefix to each address
static_prefix = urlsplit(static_url).path
invalid_files = \
[urljoin(static_prefix, path) for path in invalid_files if path]
invalidation_step = 1000
invalication_max_num = 3
invalidation_max_files = invalidation_step * invalication_max_num
def dump(invalid_files):
import tempfile
from django.utils import simplejson
dump = tempfile.NamedTemporaryFile(delete=False)
dump.write(simplejson.dumps(invalid_files, indent=2))
dump.close()
return dump.name
if len(invalid_files) > invalidation_max_files:
raise AssertionError(
"There are more than %s files to invalidate, "
"AWS doesn't support it at the moment.\n"
"File names to be invalidated dumped to JSON file %s, "
"please invalidate manually\n"
"" % (invalidation_max_files, dump(invalid_files)))
conn = CloudFrontConnection(access_key, secret_key)
try:
for i in xrange(invalication_max_num):
thousand = invalid_files[i:i + invalidation_step]
if not thousand: break
conn.create_invalidation_request(distribution, thousand)
except exception.CloudFrontServerError, message:
self.stderr.write('Error while processing:\n\n%s\n\n'
'Unprocessed files dumped to JSON file %s, '
'please invalidate manually\n' %
(message, dump(invalid_files)))
def do_invalidate(distrib_id, inval_keys, creds):
'''
Send a CloudFront invalidation request for the given objects.
'''
cf = CloudFrontConnection(**creds.as_dict())
distrib = get_distribution(cf, distrib_id)
req = cf.create_invalidation_request(distrib.id, inval_keys)
pbar = s3pub.progress.InvalidationProgressBar(req.id)
for _ in pbar(Monitor(cf, distrib_id, req.id)):
pass
print('Done.')
def do_invalidation(self):
files = sys.argv[1:]
if len(files) > 0:
print 'Sending invalidation request for the following files:'
for f in files:
print f
conn = CloudFrontConnection(settings.AWS_ACCESS_KEY, settings.AWS_SECRET_ACCESS_KEY)
req = conn.create_invalidation_request(settings.AWS_CF_DISTRIBUTION_ID, files)
print req.status
else:
self.help()
sys.exit()
def wrapped(*args, **kwargs):
from boto.cloudfront.exception import CloudFrontServerError
from boto.cloudfront import CloudFrontConnection
c = CloudFrontConnection(env.aws_access_key_id,
env.aws_secret_access_key)
if not hasattr(env, 'aws_access_key_id'):
prompt('AWS Access Key ID: ', key='aws_access_key_id')
if not hasattr(env, 'aws_secret_access_key'):
prompt('AWS Secret Access Key: ', key='aws_secret_access_key')
try:
return f(c, *args, **kwargs)
except CloudFrontServerError as e:
print "Error: \n", e.error_message
def to_cdn(c, slug):
"Create a new Distribution object on CloudFront"
from boto.cloudfront import CloudFrontConnection
from boto.cloudfront.origin import CustomOrigin
c = CloudFrontConnection(env.aws_access_key_id, env.aws_secret_access_key)
d = c.create_distribution(origin=CustomOrigin(
slug + '.cdn.readthedocs.org', origin_protocol_policy='http-only'),
enabled=True,
comment='Slug: ' + slug,
cnames=[slug + '.readthedocs.org'])
print "Created: " + d.domain_name + " for " + slug
list_cdn()
def connect_cloudfront(aws_access_key_id=None, aws_secret_access_key=None, **kwargs):
"""
:type aws_access_key_id: string
:param aws_access_key_id: Your AWS Access Key ID
:type aws_secret_access_key: string
:param aws_secret_access_key: Your AWS Secret Access Key
:rtype: :class:`boto.fps.connection.FPSConnection`
:return: A connection to FPS
"""
from boto.cloudfront import CloudFrontConnection
return CloudFrontConnection(aws_access_key_id, aws_secret_access_key, **kwargs)
def connect_cloudfront(aws_access_key_id=None,
aws_secret_access_key=None,
**kwargs):
"""
@type aws_access_key_id: string
@param aws_access_key_id: Your AWS Access Key ID
@type aws_secret_access_key: string
@param aws_secret_access_key: Your AWS Secret Access Key
@rtype: L{FPSConnection<boto.fps.connection.FPSConnection>}
@return: A connection to FPS
"""
from boto.cloudfront import CloudFrontConnection
return CloudFrontConnection(aws_access_key_id, aws_secret_access_key,
**kwargs)
def invalidate_paths(deploy_type, paths, *args, **kwargs):
"""
Helper function, create an invalidation request for CloudFront distribution
:param deploy_type: The deploy type
:param paths: The paths array
"""
from boto.cloudfront import CloudFrontConnection
# TODO chunking invalidation to prevent error
distributions = get_conf('AWS_DISTRIBUTION_ID', deploy_type=deploy_type)
if isinstance(distributions, list):
distributions = distributions
else:
distributions = [distributions]
for distribution in distributions:
conn_cf = CloudFrontConnection(
get_conf('AWS_ACCESS_KEY_ID', deploy_type=deploy_type),
get_conf('AWS_SECRET_ACCESS_KEY', deploy_type=deploy_type))
conn_cf.create_invalidation_request(distribution, paths)
def get_playlist_with_signed_url(playlist_path):
# Get playlist file from S3
s3_conn = connect_s3(settings.ACCESS_KEY_ID, settings.SECRET_ACCESS_KEY)
bucket = s3_conn.get_bucket(settings.PLAYLIST_BUCKET_NAME)
key = bucket.get_key(playlist_path)
if key is None:
raise Exception("No such key was found. key={}".format(key))
fp = StringIO()
key.get_contents_to_file(fp)
fp.seek(0)
# Convert with signed url
cf_conn = CloudFrontConnection(settings.ACCESS_KEY_ID,
settings.SECRET_ACCESS_KEY)
dist = Distribution(cf_conn)
expire_time = int(time.time() + 60 * 60) # 60 mins
outlines = []
for line in fp.readlines():
line = line.rstrip()
matchObj = re.search(TS_PATTERN, line)
if matchObj is not None:
file_name = matchObj.group()
url = os.path.join(
os.path.dirname(
os.path.join(settings.CLOUDFRONT_URL_PREFIX,
playlist_path)), file_name)
signed_url = dist.create_signed_url(
url,
settings.CLOUDFRONT_KEYPAIR_ID,
expire_time,
private_key_file=settings.CLOUDFRONT_PRIVATE_KEY_FILE_LOCATION)
outlines.append(signed_url)
else:
outlines.append(line)
fp.close()
return '\n'.join(outlines)
def create_cloudfront_distribution(aws_access_key, aws_secret_key,
bucket_endpoint, hostname):
connection = CloudFrontConnection(aws_access_key, aws_secret_key)
origin = CustomOrigin(dns_name=bucket_endpoint,
origin_protocol_policy="http-only")
distribution = connection.create_distribution(origin=origin,
enabled=True,
cnames=[hostname])
print("A CloudFront distribution has been created.")
print("You need to do two things:")
print(
"1. Go to the DNS provider for {hostname} and set up a CNAME to map it to {distribution_domain}"
.format(hostname=hostname,
distribution_domain=distribution.domain_name))
print(
"2. Go to the AWS control panel, and associate the appropriate SSL cert with distribution {id}"
.format(id=distribution.id))
print(
"(The latter step is required because boto currently doesn't support setting certificates.)"
)
def invalidate_paths(paths):
cx = CloudFrontConnection(settings.get('aws_access_key'),
settings.get('aws_secret_key'))
distros = cx.get_all_distributions()
return [cx.create_invalidation_request(d.id, paths) for d in distros]
def clear_cloudfront_cache(base_dir,index_file,dir_prefix='',passn=0):
base_dir = os.path.abspath(base_dir)
all_files = get_files_from_dir(base_dir)
if(os.path.exists(index_file)):
data = file(index_file).read()
os.unlink(index_file+'.back')
file(index_file+'.back','w').write(data)
else:
data = ''
file(index_file+'.back','w').write('')
index = {}
data = data.split('\n')
for line in data:
if not line:
continue
path,md5 = line.split('\t#\t')
index[path] = md5
new_files,new_files_raw = get_modified_files(base_dir,all_files,index,dir_prefix)
for filename in index.iterkeys():
if filename not in all_files:
next = False
for it in IGNORE:
if it in filename:
next = True
if next:
continue
new_files.append(os.path.join(dir_prefix,filename.strip('/')))
new_files_raw.append(filename)
if new_files:
for filename in new_files:
print 'Modified: %s' % filename
else:
print 'No files were modified.\n'
sys.exit()
print '\nUploading %s files\n' % len(new_files)
inp = ''
while (inp != 'y' and inp != 'n'):
inp = raw_input('Upload changes to CloudFront(y/n): ')
if inp == 'y':
try:
conn = CloudFrontConnection(AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY)
k = 0
for i in xrange(0,len(new_files),800):
if k < passn:
k += 1
continue
res = True
res = conn.create_invalidation_request(AWS_CF_DISTRIBUTION_ID, new_files[i:i+900])
if res:
print '\nInvalidation request created'
for filename in new_files_raw[i:i+800]:
fc = file(base_dir+filename).read()
index[filename] = hashlib.md5(fc).hexdigest()
if k >= (passn+2):
print '\nToo many files. Repeat update after 15 minutes.'
break
k += 1
except Exception,e:
save_index(index_file,index)
sys.exit('\nError: %s' % e)
save_index(index_file,index)
import sys, datetime, settings
from level3client import Level3Service
from boto.cloudfront import CloudFrontConnection
urls = sys.argv
urls.remove('invcache.py')
scope = settings.level3scope
key_id = settings.level3key
secret = settings.level3secret
service = Level3Service(key_id, secret, method='POST')
result = service('invalidations/%s' % scope,
post_data="""
<paths>
%s
</paths>
""" % ('\n'.join(['<path>%s</path>' % url for url in urls])))
conn = CloudFrontConnection(settings.awscfkey, settings.awscfsecret)
invreq = conn.create_invalidation_request(settings.awscfdistribution, urls)
print "Level3 Invalidation Request:"
print result.dom.toprettyxml()
print "CloudFront Invalidation Status:"
print invreq.status
def invalidate_all(aws_access_key, aws_secret_key, cloudfront_distribution):
print("Invalidating all")
cloudfront_connection = CloudFrontConnection(aws_access_key,
aws_secret_key)
cloudfront_connection.create_invalidation_request(cloudfront_distribution,
"/*")
def _get_cf():
from boto.cloudfront import CloudFrontConnection
return CloudFrontConnection(
aws_access_key_id=settings.AWS_ACCESS_KEY,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY)
def get_cloudfront_connection(cls):
if cls._connection is None:
cls._connection = CloudFrontConnection(
aws_access_key_id=app.config['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=app.config['AWS_SECRET_ACCESS_KEY'])
return cls._connection
