def upload_notes_build_directory_to_s3(config): """ !!AI TODO need a manifest, and a saner way of tracking files.""" logger = logging.getLogger("%s.upload_notes_build_directory_to_s3" % APP_NAME) logger.debug("entry. config.notes_build_directory: %s, config.notes_s3_bucket: %s" % (config.notes_build_directory, config.notes_s3_bucket)) with contextlib.closing(boto.connect_s3()) as conn: with contextlib.closing(boto.connect_cloudfront()) as conn_cloudfront: cloudfront_distribution = [elem for elem in conn_cloudfront.get_all_distributions() if config.s3_bucket in elem.origin.dns_name][0] cloudfront_distribution = cloudfront_distribution.get_distribution() bucket = conn.get_bucket(config.notes_s3_bucket) output_subpaths = [] for subpath in sorted(config.notes_input_to_output.values()): posix_subpath = posixpath.join(*subpath.split(os.sep)) posix_subdirectory = posixpath.split(posix_subpath)[0] output_subpaths.append(subpath) output_subpaths.append(posixpath.join(posix_subdirectory, "_pandoc.css")) output_filepaths = [os.path.normpath(os.path.join(config.notes_build_directory, subpath)) for subpath in output_subpaths] for (subpath, filepath) in zip(output_subpaths, output_filepaths): logger.debug("uploading subpath: '%s'" % subpath) key = bucket.delete_key(subpath) key = bucket.new_key(subpath) if is_gzip_file(filepath): logger.debug("mark as a gzipped file") key.set_metadata("Content-Encoding", "gzip") key.set_contents_from_filename(filepath) key.make_public() logger.debug("creating cloudfront invalidation request") conn_cloudfront.create_invalidation_request(cloudfront_distribution.id, output_subpaths)
def generate_url(self, _object): assert isinstance(_object, Object) bucket = _object.s3_bucket resource = u'%s%s' % ( bucket.origin, _object.s3_key ) conn = boto.connect_cloudfront() dist = conn.get_distribution_info(bucket.distribution_id) signers = filter(lambda signer: signer.id == u'Self', dist.active_signers) if len(signers) < 1: raise SignerNotFound() key_pair = random.choice(filter( lambda key_pair: key_pair is not None, [ self._get_key_pair_by_id(key_pair_id, False) for key_pair_id in signers[0].key_pair_ids ] )) signed_url = dist.create_signed_url( resource, key_pair.id, expire_time=int(time.time() + 60 * 60 * 6), # 6hours private_key_string=key_pair.private ) return signed_url
def setup_s3(bucket_name=None): """ Setup s3 instance with cloudfront distribution. """ if not bucket_name: bucket_name = '{0}-media'.format(PROJECT_NAME) conn = boto.connect_s3() try: conn.create_bucket(bucket_name, location=_Location.EU, policy='public-read') except _S3CreateError: pass print('AWS_STORAGE_BUCKET_NAME={0}'.format(bucket_name)) if CLOUDFRONT_ENABLED: try: origin = CLOUDFRONT_CUSTOM_ORIGIN except NameError: origin = None if not origin: origin = '{0}.s3.amazonaws.com'.format(bucket_name) origin = _CustomOrigin(origin, origin_protocol_policy='http-only') conn = boto.connect_cloudfront() distro = conn.create_distribution(origin=origin, enabled=True) print('MEDIA_DOMAIN={0}'.format(distro.domain_name)) else: bucket_url = '{0}.s3.amazonaws.com'.format(bucket_name) print('MEDIA_DOMAIN={0}'.format(bucket_url))
def main(): paths = open(invalidation_path, "r+") conn = boto.connect_cloudfront() inval_req = conn.create_invalidation_request(dist_id, paths) print inval_req touch = open(invalidation_path, "w") touch.write("")
def __init__(self): handlers = [ (r"/", HomeHandler), (r"/login", LoginHandler), (r"/logout", LogoutHandler), (r"/register", RegisterHandler), (r"/video/([0-9]+)", VideoHandler), (r"/purchase/([a-zA-Z0-9-_]+)", PurchaseHandler), (r"/bucket", BucketHandler), ] settings = dict( site_title=u"Site Title", template_path=os.path.join(os.path.dirname(__file__), "templates"), static_path=os.path.join(os.path.dirname(__file__), "static"), static_url_prefix="https://s3-us-west-2.amazonaws.com/assets/", xsrf_cookies=True, cookie_secret="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", login_url="/login", debug=False, ) tornado.web.Application.__init__(self, handlers, **settings) self.db = torndb.Connection( host=options.mysql_host, database=options.mysql_database, user=options.mysql_user, password=options.mysql_password ) self.cf = boto.connect_cloudfront()
def invalidate_paths(dry_run=False): paths = _get_paths() num_paths = len(paths) if num_paths > 0: cloudfront = boto.connect_cloudfront(AWS_KEY, AWS_SECRET) # Can only do 1000 invalidates at a time, so send them in chunks # Note: 1st 1000 invalidations per month are free, anything over # that costs $$$ chunk_begin = 0 chunk_end = min(num_paths, 1000) while chunk_begin < num_paths: paths_chunk = paths[chunk_begin:chunk_end] if dry_run: print "Would've created an invalidation request." else: req = cloudfront.create_invalidation_request(DISTRIBUTION_ID, paths_chunk) print "Created invalidation request." chunk_begin = chunk_end chunk_end = min(chunk_end + 1000, num_paths) if dry_run: print "Without the dry_run arg, would've invalidated {0} paths.".format(num_paths) else: print "Invalidated {0} paths.".format(num_paths)
def get_or_create_distribution(self, s3_bucket_name): assert isinstance(s3_bucket_name, six.string_types) boto = get_boto() origin_dns = '%s.s3.amazonaws.com' % s3_bucket_name if not self.dryrun: conn = boto.connect_cloudfront( self.genv.aws_access_key_id, self.genv.aws_secret_access_key ) origin = boto.cloudfront.origin.S3Origin(origin_dns) distro = None if self.verbose: # Loop over all distributions to determine whether this one exists already. dists = conn.get_all_distributions() for d in dists: print('Checking existing Cloudfront distribution %s...' % d.get_distribution().config.origin.dns_name) if origin_dns == d.get_distribution().config.origin.dns_name: print('Found existing distribution!') distro = d break # Necessary to avoid "Rate exceeded" errors. time.sleep(0.4) if not distro: print('Creating new distribution from %s...' % origin) distro = conn.create_distribution(origin=origin, enabled=True) return distro else: print('boto.connect_cloudfront().create_distribution(%s)' % repr(origin_dns))
def push_json_to_s3(lineage, resolution, directory='../auspice/data/', bucket='nextflu-dev', cloudfront='E1XKGZG0ZTX4YN'): """Upload JSON files to S3 bucket""" """Boto expects environmental variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY""" directory = directory.rstrip('/') + '/' import boto conn = boto.connect_s3() b = conn.get_bucket(bucket) k = boto.s3.key.Key(b) paths = [] print "Uploading JSONs for", lineage, resolution for postfix in [ 'tree.json', 'sequences.json', 'frequencies.json', 'meta.json' ]: json = lineage + '_' + resolution + '_' + postfix k.key = 'data/' + json k.set_contents_from_filename(directory + json) print json, "uploaded" paths.append('data/' + json) c = boto.connect_cloudfront() c.create_invalidation_request(cloudfront, paths)
def foia_file_delete_s3(sender, **kwargs): """Delete file from S3 after the model is deleted""" # pylint: disable=unused-argument if settings.CLEAN_S3_ON_FOIA_DELETE: # only delete if we are using s3 foia_file = kwargs['instance'] conn = S3Connection( settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, ) bucket = conn.get_bucket(settings.AWS_STORAGE_BUCKET_NAME) key = bucket.get_key(foia_file.ffile.name) if key: key.delete() # also clear the cloudfront cache cloudfront = boto.connect_cloudfront( settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, ) # find the current distribution distributions = [ d for d in cloudfront.get_all_distributions() if settings.AWS_S3_CUSTOM_DOMAIN in d.cnames ] if distributions: distribution = distributions[0] cloudfront.create_invalidation_request( distribution.id, [foia_file.ffile.name], )
def get_or_create_distribution(self, s3_bucket_name): assert isinstance(s3_bucket_name, six.string_types) boto = get_boto() origin_dns = '%s.s3.amazonaws.com' % s3_bucket_name if not self.dryrun: conn = boto.connect_cloudfront(self.genv.aws_access_key_id, self.genv.aws_secret_access_key) origin = boto.cloudfront.origin.S3Origin(origin_dns) distro = None if self.verbose: # Loop over all distributions to determine whether this one exists already. dists = conn.get_all_distributions() for d in dists: print('Checking existing Cloudfront distribution %s...' % d.get_distribution().config.origin.dns_name) if origin_dns == d.get_distribution( ).config.origin.dns_name: print('Found existing distribution!') distro = d break # Necessary to avoid "Rate exceeded" errors. time.sleep(0.4) if not distro: print('Creating new distribution from %s...' % origin) distro = conn.create_distribution(origin=origin, enabled=True) return distro else: print('boto.connect_cloudfront().create_distribution(%s)' % repr(origin_dns))
def purge_cloudfront_caches(page, request): try: distribution = CloudfrontDistribution.objects.all()[0] cf = boto.connect_cloudfront() cf.create_invalidation_request(distribution.distribution_id, []) except: #TODO: This is a broad exception - but we are having issues with the cache. Will tighten down once we start using it. pass
def do_activity(self, data = None): """ Do the work """ if(self.logger): self.logger.info('data: %s' % json.dumps(data, sort_keys=True, indent=4)) self.db.connect() # cdn.elifesciences.org CDN ID distribution_id = self.settings.cdn_distribution_id invalidation_list = self.get_invalidation_list() # Connect to S3 c_conn = boto.connect_cloudfront(self.settings.aws_access_key_id, self.settings.aws_secret_access_key) # Limit of 1000 URLs to invalidate at one time try: count = int(len(invalidation_list) / 1000) + 1 except: # Divide by zero or something else return False array_of_invalidation_list = self.split_array(invalidation_list, count) for i_list in array_of_invalidation_list: inval_req = c_conn.create_invalidation_request(distribution_id, invalidation_list) if(self.logger): self.logger.info('LensCDNInvalidation: %s' % "") return True
def s3_export(data, name): raw_data = json.dumps(data) script_content = 'window.EVENTS_DATA=' + raw_data with gzip.open(name + '.js.gz', 'wb') as f: f.write(str(script_content).encode('utf-8')) with open(name + '.json', 'w') as f: f.write(raw_data) aws_host = os.environ.get('AWS_HOST') aws_bucket = os.environ.get('S3_BUCKET') cloudfront_id = os.environ.get('CLOUDFRONT_ID') aws_region = os.environ.get('AWS_REGION') access_key_id = os.environ.get('AWS_ACCESS_KEY_ID') secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY') conn = boto.s3.connect_to_region( aws_region, aws_access_key_id=access_key_id, aws_secret_access_key=secret_access_key, is_secure=True, # uncomment if you are not using ssl calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) bucket = conn.get_bucket(aws_bucket) key = bucket.get_key('output/' + name + '.js.gz') key_raw = bucket.get_key('raw/' + name + '.json') if key is None: print("Creating New Bucket") key = bucket.new_key('output/' + name + '.js.gz') if key_raw is None: print("Creating New Raw File") key_raw = bucket.new_key('raw/' + name + '.json') # Upload data to S3 print("Uploading RAW to S3") key_raw.set_contents_from_filename(name + '.json') key_raw.set_acl('public-read') print("Uploading GZIP to S3") key.set_metadata('Content-Type', 'text/plain') key.set_metadata('Content-Encoding', 'gzip') key.set_contents_from_filename(name + '.js.gz') key.set_acl('public-read') # Cloudfront Invalidation requests print("Invalidating Output") cloudfront = boto.connect_cloudfront() paths = ['/output/*'] inval_req = cloudfront.create_invalidation_request( cloudfront_id, paths) #Delete all files os.remove(name + ".js.gz") os.remove(name + ".json")
def __init__(self, conf): super(CDNProvider, self).__init__(conf) self._conf.register_opts(CLOUDFRONT_OPTIONS, group=CLOUDFRONT_GROUP) self.cloudfront_conf = self._conf[CLOUDFRONT_GROUP] self.cloudfront_client = boto.connect_cloudfront( aws_access_key_id=self.cloudfront_conf.aws_access_key_id, aws_secret_access_key=self.cloudfront_conf.aws_secret_access_key)
def run(self, terms, variables=None, **kwargs): distributionId = terms[0] cloudfront_conn = boto.connect_cloudfront(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) distributionConfig = cloudfront_conn.get_distribution_config(distributionId) if distributionConfig: # zz = dir(distributionConfig) return [distributionConfig.etag] return None
def publish_program(host, program): import boto c = boto.connect_cloudfront() for d in c.get_all_distributions(): if d.origin.dns_name == host: print 'good' else: print 'bad', d.origin.dns_name
def perform_create_cloudfront_invalidation(dispatcher, intent): """ See :class:`CreateCloudFrontInvalidation`. """ cf = boto.connect_cloudfront() distribution = [dist for dist in cf.get_all_distributions() if intent.cname in dist.cnames][0] cf.create_invalidation_request(distribution.id, intent.paths)
def main(): distid = 'xxxxxxxxxx' invalidationfilepath = '/home/ec2-user/invalid.txt' paths = open(invalidationfilepath,"r+") conn = boto.connect_cloudfront() inval_req = conn.create_invalidation_request(distid, paths) print inval_req touch = open(invalidationfilepath,"w") touch.write("")
def __init__(self, server_name, distribution_id, key_pair_id, private_key_string, aws_access_key_id=None, aws_secret_access_key=None): super(AmazonCloudFrontS3Storage, self).__init__(server_name) self.cloudfront_connection = boto.connect_cloudfront(aws_access_key_id, aws_secret_access_key) self.s3_connection = boto.connect_s3(aws_access_key_id, aws_secret_access_key) self.distribution_id = distribution_id self.distribution = self.cloudfront_connection.get_distribution_info(self.distribution_id) self.key_pair_id = key_pair_id self.private_key_string = private_key_string
def run(self, terms, variables=None, **kwargs): distributionId = terms[0] cloudfront_conn = boto.connect_cloudfront(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) distributionConfig = cloudfront_conn.get_distribution_config( distributionId) if distributionConfig: # zz = dir(distributionConfig) return [distributionConfig.etag] return None
def perform_create_cloudfront_invalidation(dispatcher, intent): """ See :class:`CreateCloudFrontInvalidation`. """ cf = boto.connect_cloudfront() distribution = [ dist for dist in cf.get_all_distributions() if intent.cname in dist.cnames ][0] cf.create_invalidation_request(distribution.id, intent.paths)
def __init__(self, bucket_name, aws_key, aws_secret): self.job_id = str(uuid.uuid1()) self.bucket_name = bucket_name self.aws_key = aws_key self.aws_secret = aws_secret # TODO: clean up bucket name string? What's a valid AWS username? self.user_name = "bhuket-" + bucket_name self.iam = boto.connect_iam(aws_key, aws_secret) self.s3 = boto.connect_s3(aws_key, aws_secret) self.cloudfront = boto.connect_cloudfront(aws_key, aws_secret)
def run(): indiv_groups = indivisible_group.grab_data() indiv_groupmtg = indivisible_groupmtg.grab_data() indiv_action = indivisible_action.grab_data() data = indiv_groupmtg + indiv_action + indiv_groups content = 'window.INDIVISIBLE_EVENTS=' + json.dumps(data) # Locally Store Data with gzip.open('data/indivisible-data.js.gz', 'wb') as f: f.write(str(content).encode('utf-8')) with open('data/indivisible.json', 'w') as f: f.write(content) # START aws_host = os.environ.get('AWS_HOST') conn = S3Connection(host=aws_host) bucket = conn.get_bucket('pplsmap-data') key = bucket.get_key('output/indivisible.js.gz') key_raw = bucket.get_key('raw/indivisible.json') # Retrieve Keys if key is None: print("Creating New Bucket") key = bucket.new_key('output/indivisible.js.gz') if key_raw is None: print("Creating New Raw File") key_raw = bucket.new_key('raw/indivisible.json') # Upload data to S3 print("Uploading RAW to S3") key_raw.set_contents_from_filename('data/indivisible.json') key_raw.set_acl('public-read') print("Uploading GZIP to S3") key.set_metadata('Content-Type', 'text/plain') key.set_metadata('Content-Encoding', 'gzip') key.set_contents_from_filename('data/indivisible-data.js.gz') key.set_acl('public-read') # Cloudfront Invalidation requests print("Invalidating Indivisible Output") cloudfront = boto.connect_cloudfront() paths = ['/output/*'] inval_req = cloudfront.create_invalidation_request(u'EXFHJXIFH495H', paths) #Delete all files os.remove("data/indivisible-data.js.gz") os.remove("data/indivisible.json") os.remove("data/indivisible.csv")
def connect_cloudfront(self, aws_access_key_id=None, aws_secret_access_key=None, **kwargs): if aws_access_key_id == None: aws_access_key_id = self.config.get("Credentials", "aws_access_key_id") if aws_secret_access_key == None: aws_secret_access_key = self.config.get("Credentials", "aws_secret_access_key") return boto.connect_cloudfront(aws_access_key_id, aws_secret_access_key, **kwargs)
def s3_export(data, name): raw_data = json.dumps(data) script_content = 'window.EVENTS_DATA=' + raw_data with gzip.open(name + '.js.gz', 'wb') as f: f.write(str(script_content).encode('utf-8')) with open(name + '.json', 'w') as f: f.write(raw_data) aws_host = os.environ.get('AWS_HOST') aws_bucket = os.environ.get('S3_BUCKET') cloudfront_id = os.environ.get('CLOUDFRONT_ID') conn = S3Connection(host=aws_host) bucket = conn.get_bucket(aws_bucket) key = bucket.get_key('output/' + name + '.js.gz') key_raw = bucket.get_key('raw/' + name + '.json') if key is None: print("Creating New Bucket") key = bucket.new_key('output/' + name + '.js.gz') if key_raw is None: print("Creating New Raw File") key_raw = bucket.new_key('raw/' + name + '.json') # Upload data to S3 print("Uploading RAW to S3") key_raw.set_contents_from_filename(name + '.json') key_raw.set_acl('public-read') print("Uploading GZIP to S3") key.set_metadata('Content-Type', 'text/plain') key.set_metadata('Content-Encoding', 'gzip') key.set_contents_from_filename(name + '.js.gz') key.set_acl('public-read') # Cloudfront Invalidation requests print("Invalidating Output") cloudfront = boto.connect_cloudfront() paths = ['/output/*'] inval_req = cloudfront.create_invalidation_request( cloudfront_id, paths) #Delete all files os.remove(name + ".js.gz") os.remove(name + ".json")
def create_signed_url(url, **kwargs): """Create new signed CloudFront URL for given URL See <http://boto.cloudhackers.com/en/latest/ref/cloudfront.html> """ conn = boto.connect_cloudfront(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) dist = conn.get_distribution_info(settings.AWS_CF_ID) signed = dist.create_signed_url(url, settings.AWS_KEYPAIR_ID, private_key_string=settings.AWS_KEYPAIR_PRIVATE_KEY, **kwargs) return signed
def invalidate_cloudfront_objects(profile, distribution, objects): """Chunk the invalidation request into 1000 objects""" chunk_size = 3000 # upto 3000 according to this guide: # http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html conn = boto.connect_cloudfront(profile_name=profile) print('distribution info: %s' % conn.get_distribution_info(distribution)) # Splitting the object list into chunks chunks = [objects[i:i + chunk_size] for i in range(0, len(objects), chunk_size)] # Create invalidation requests for paths in chunks: req = conn.create_invalidation_request(distribution, paths) print('invalidation request: %s' % req)
def do_invalidate(paths, origin): print 'do invalidate' import boto c = boto.connect_cloudfront() for d in c.get_all_distributions(): print 'distribution:', d.origin.dns_name, d.cnames, origin if d.cnames[0] == str(sys.argv[1]): print d.id, d.domain_name, d.status, d.comment for ir in c.get_invalidation_requests(d.id): if ir.status != 'Completed': return 'no can do!!!' print 'invalidate request:', ir.id, ir.status print 'create invalidation' c.create_invalidation_request(d.id, paths)
def run(): data = peoplepower_action.grab_data() raw = json.dumps(data) content = 'window.PEOPLEPOWER_EVENTS=' + json.dumps(data) # Locally Store Data with gzip.open('data/peoplepower.js.gz', 'wb') as f: f.write(str(content).encode('utf-8')) with open('data/peoplepower.json', 'w') as f: f.write(raw) # START aws_host = os.environ.get('AWS_HOST') conn = S3Connection(host=aws_host) bucket = conn.get_bucket('pplsmap-data') key = bucket.get_key('output/peoplepower.js.gz') key_raw = bucket.get_key('raw/peoplepower.json') # Retrieve Keys if key is None: print("Creating New Bucket") key = bucket.new_key('output/peoplepower.js.gz') if key_raw is None: print("Creating New Raw File") key_raw = bucket.new_key('raw/peoplepower.json') # Upload data to S3 print("Uploading RAW to S3") key_raw.set_contents_from_filename('data/peoplepower.json') key_raw.set_acl('public-read') print("Uploading GZIP to S3") key.set_metadata('Content-Type', 'text/plain') key.set_metadata('Content-Encoding', 'gzip') key.set_contents_from_filename('data/peoplepower.js.gz') key.set_acl('public-read') # Cloudfront Invalidation requests print("Invalidating ACLU Output") cloudfront = boto.connect_cloudfront() paths = ['/output/*'] inval_req = cloudfront.create_invalidation_request(u'EXFHJXIFH495H', paths) os.remove("data/peoplepower.js.gz") os.remove("data/peoplepower.json")
def _upload_to_cloudfront(filepath): global _cf_connection global _cf_distribution if _cf_connection is None: _cf_connection = boto.connect_cloudfront(settings.AWS_ACCESS_KEY, settings.AWS_ACCESS_SECRET) if _cf_distribution is None: _cf_distribution = _cf_connection.create_distribution( origin='%s.s3.amazonaws.com' % settings.AWS_STORAGE_BUCKET_NAME, enabled=True, comment=settings.AWS_CLOUDFRONT_DISTRIBUTION_COMMENT) # now we can delete any old versions of the same file that have the # same name but a different timestamp basename = os.path.basename(filepath) object_regex = re.compile('%s\.(\d+)\.%s' % \ (re.escape('.'.join(basename.split('.')[:-2])), re.escape(basename.split('.')[-1]))) for obj in _cf_distribution.get_objects(): match = object_regex.findall(obj.name) if match: old_timestamp = int(match[0]) new_timestamp = int(object_regex.findall(basename)[0]) if new_timestamp == old_timestamp: # an exact copy already exists return obj.url() elif new_timestamp > old_timestamp: # we've come across the same file but with an older timestamp #print "DELETE!", obj_.name obj.delete() break # Still here? That means that the file wasn't already in the distribution fp = open(filepath) # Because the name will always contain a timestamp we set faaar future # caching headers. Doesn't matter exactly as long as it's really far future. headers = {'Cache-Control':'max-age=315360000, public', 'Expires': 'Thu, 31 Dec 2037 23:55:55 GMT', } #print "\t\t\tAWS upload(%s)" % basename obj = _cf_distribution.add_object(basename, fp, headers=headers) return obj.url()
def get_cache(force_rebuild=False): if not settings.AWS_ACCESS_KEY_ID: return {} if force_rebuild or not hasattr(get_cache, "cache"): if force_rebuild or not CACHE_FILE.exists(): CACHE_FILE.dirname().makedirs_p() connection = connect_cloudfront(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) distributions = connection.get_all_distributions() cache = {distribution.origin.dns_name: distribution.domain_name for distribution in distributions} with open(CACHE_FILE, "w") as handle: json.dump(cache, handle) else: with open(CACHE_FILE) as handle: cache = json.load(handle) get_cache.cache = cache return get_cache.cache
def do_invalidate(): c = boto.connect_cloudfront() for d in c.get_all_distributions(): if d.origin.dns_name == app_util.app_bucket: print 'domain id:', d.id, 'domain name:', d.domain_name, 'domain status:', d.status, 'comment:', d.comment for ir in c.get_invalidation_requests(d.id): if ir.status != 'Completed': print 'invalidate request:', ir.id, ir.status exit(1) paths = [res[len(publish.build_dir):] for res in publish.get_publish_list()] if paths: print 'invalidate paths:', paths c.create_invalidation_request(d.id, paths) else: print 'invalidate skipped'
def get_cache(force_rebuild=False): if not settings.AWS_ACCESS_KEY_ID: return {} if force_rebuild or not hasattr(get_cache, 'cache'): if force_rebuild or not CACHE_FILE.exists(): CACHE_FILE.dirname().makedirs_p() connection = connect_cloudfront(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) distributions = connection.get_all_distributions() cache = {distribution.origin.dns_name: distribution.domain_name for distribution in distributions} with open(CACHE_FILE, 'w') as handle: json.dump(cache, handle) else: with open(CACHE_FILE) as handle: cache = json.load(handle) get_cache.cache = cache return get_cache.cache
def invalidate(*paths): """ Issues invalidation requests to a Cloudfront distribution for the current static media bucket, triggering it to reload the specified paths from the origin. Note, only 1000 paths can be issued in a request at any one time. """ from burlap.dj import get_settings if not paths: return # http://boto.readthedocs.org/en/latest/cloudfront_tut.html _settings = get_settings() if not _settings.AWS_STATIC_BUCKET_NAME: print('No static media bucket set.') return if isinstance(paths, basestring): paths = paths.split(',') all_paths = map(str.strip, paths) # assert len(paths) <= 1000, \ # 'Cloudfront invalidation request limited to 1000 paths or less.' i = 0 while 1: paths = all_paths[i:i+1000] if not paths: break #print 'paths:',paths c = boto.connect_cloudfront() rs = c.get_all_distributions() target_dist = None for dist in rs: print(dist.domain_name, dir(dist), dist.__dict__) bucket_name = dist.origin.dns_name.replace('.s3.amazonaws.com', '') if bucket_name == _settings.AWS_STATIC_BUCKET_NAME: target_dist = dist break if not target_dist: raise Exception(('Target distribution %s could not be found ' 'in the AWS account.') \ % (settings.AWS_STATIC_BUCKET_NAME,)) print('Using distribution %s associated with origin %s.' \ % (target_dist.id, _settings.AWS_STATIC_BUCKET_NAME)) inval_req = c.create_invalidation_request(target_dist.id, paths) print('Issue invalidation request %s.' % (inval_req,)) i += 1000
def invalidate(*paths): """ Issues invalidation requests to a Cloudfront distribution for the current static media bucket, triggering it to reload the specified paths from the origin. Note, only 1000 paths can be issued in a request at any one time. """ from burlap.dj import get_settings if not paths: return # http://boto.readthedocs.org/en/latest/cloudfront_tut.html _settings = get_settings() if not _settings.AWS_STATIC_BUCKET_NAME: print('No static media bucket set.') return if isinstance(paths, basestring): paths = paths.split(',') all_paths = map(str.strip, paths) # assert len(paths) <= 1000, \ # 'Cloudfront invalidation request limited to 1000 paths or less.' i = 0 while 1: paths = all_paths[i:i + 1000] if not paths: break #print 'paths:',paths c = boto.connect_cloudfront() rs = c.get_all_distributions() target_dist = None for dist in rs: print(dist.domain_name, dir(dist), dist.__dict__) bucket_name = dist.origin.dns_name.replace('.s3.amazonaws.com', '') if bucket_name == _settings.AWS_STATIC_BUCKET_NAME: target_dist = dist break if not target_dist: raise Exception(('Target distribution %s could not be found ' 'in the AWS account.') \ % (settings.AWS_STATIC_BUCKET_NAME,)) print('Using distribution %s associated with origin %s.' \ % (target_dist.id, _settings.AWS_STATIC_BUCKET_NAME)) inval_req = c.create_invalidation_request(target_dist.id, paths) print('Issue invalidation request %s.' % (inval_req, )) i += 1000
def connect(): global connected global AWS_ACCESS_KEY global AWS_SECRET_ACCESS_KEY global iam global s3 global cloudfront if not connected: iam = boto.connect_iam(AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY) s3 = boto.connect_s3(AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY) cloudfront = boto.connect_cloudfront(AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY) connected = True return connected
def upload_build_directory_to_s3(config): logger = logging.getLogger("%s.upload_build_directory_to_s3" % APP_NAME) logger.debug("entry. config.build_directory: %s, config.s3_bucket: %s" % (config.build_directory, config.s3_bucket)) with contextlib.closing(boto.s3.connect_to_region('eu-west-1')) as conn: with contextlib.closing(boto.connect_cloudfront()) as conn_cloudfront: cloudfront_distribution = [elem for elem in conn_cloudfront.get_all_distributions() if config.s3_bucket in elem.origin.dns_name][0] cloudfront_distribution = cloudfront_distribution.get_distribution() all_subpaths = [] bucket = conn.get_bucket(config.s3_bucket) manifest_key = bucket.get_key(config.manifest_filename) existing_manifest_hashes = {} if not manifest_key: logger.debug("manifest does not exist, upload everything.") else: logger.debug("manifest exists, may be duplicate files.") manifest_contents = manifest_key.get_contents_as_string() for line in manifest_contents.splitlines(): (subpath, digest) = line.strip().split() existing_manifest_hashes[subpath] = digest build_path_length = len(config.build_directory.split(os.sep)) for root, dirs, files in os.walk(config.build_directory): for filename in files: filepath = os.path.join(root, filename) subpath = posixpath.join(*filepath.split(os.sep)[build_path_length:]) digest = calculate_hash(filepath) if existing_manifest_hashes.get(subpath, None) == digest: logger.debug("file '%s' already exists and is identical, skipping." % subpath) continue logger.debug("uploading subpath '%s'" % subpath) all_subpaths.append(subpath) key = bucket.delete_key(subpath) key = bucket.new_key(subpath) if is_gzip_file(filepath): logger.debug("mark as a gzipped file") key.set_metadata("Content-Encoding", "gzip") key.set_contents_from_filename(filepath) key.make_public() logger.debug("creating cloudfront invalidation request") conn_cloudfront.create_invalidation_request(cloudfront_distribution.id, all_subpaths)
def s3_simple_file_export(dataStr, bucketKey): """ dataStr = "Is a string" bucketKey = 'folder/subfolder/filename.ext' """ aws_host = os.environ.get('AWS_HOST') aws_bucket = os.environ.get('S3_BUCKET') cloudfront_id = os.environ.get('CLOUDFRONT_ID') aws_region = os.environ.get('AWS_REGION') access_key_id = os.environ.get('AWS_ACCESS_KEY_ID') secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY') tempFile = "__" + bucketKey.split('/')[-1] with open(tempFile, 'w') as f: f.write(dataStr) conn = boto.s3.connect_to_region( aws_region, aws_access_key_id=access_key_id, aws_secret_access_key=secret_access_key, is_secure=True, # uncomment if you are not using ssl calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) bucket = conn.get_bucket(aws_bucket) key = bucket.get_key(bucketKey) if key is None: print("Creating New Bucket") key = bucket.new_key(bucketKey) key.set_contents_from_filename(tempFile) key.set_acl('public-read') print("Refreshing Export") cloudfront = boto.connect_cloudfront() paths = [bucketKey] inval_req = cloudfront.create_invalidation_request( cloudfront_id, paths) os.remove(tempFile)
def setup(self): """setup AWS connection""" if "." in self.bucket_name: # work around for a boto bug self.s3_connection = S3Connection( self.aws_access_key, self.aws_secret_key, calling_format=OrdinaryCallingFormat(), host=self.s3_host, ) else: self.s3_connection = S3Connection( self.aws_access_key, self.aws_secret_key, ) self.bucket = self.s3_connection.get_bucket(self.bucket_name) if self.cloudfront_id: self.cf_connection = boto.connect_cloudfront( self.aws_access_key, self.aws_secret_key, )
def push_json_to_s3(lineage, resolution, directory = '../auspice/data/', bucket = 'nextflu-dev', cloudfront = 'E1XKGZG0ZTX4YN'): """Upload JSON files to S3 bucket""" """Boto expects environmental variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY""" directory = directory.rstrip('/')+'/' import boto conn = boto.connect_s3() b = conn.get_bucket(bucket) k = boto.s3.key.Key(b) paths = [] print "Uploading JSONs for", lineage, resolution for postfix in ['tree.json', 'sequences.json', 'frequencies.json', 'meta.json']: json = lineage + '_' + resolution + '_' + postfix k.key = 'data/'+json k.set_contents_from_filename(directory+json) print json,"uploaded" paths.append('data/'+json) c = boto.connect_cloudfront() c.create_invalidation_request(cloudfront, paths)
def get_or_create_distribution(s3_bucket): if not get_dryrun(): conn = boto.connect_cloudfront() origin_dns = '%s.s3.amazonaws.com' % s3_bucket.name origin = boto.cloudfront.origin\ .S3Origin(origin_dns) # origin = boto.cloudfront.origin\ # .S3Origin(s3_bucket.get_website_endpoint()) distro = None dists = conn.get_all_distributions() for d in dists: if origin_dns == d.get_distribution().config.origin.dns_name: distro = d break if not distro: distro = conn.create_distribution(origin=origin, enabled=True) return distro else: print('boto.connect_cloudfront().create_distribution(%s)' % repr(name))
def do_activity(self, data=None): """ Do the work """ if (self.logger): self.logger.info('data: %s' % json.dumps(data, sort_keys=True, indent=4)) self.db.connect() # cdn.elifesciences.org CDN ID distribution_id = self.settings.cdn_distribution_id invalidation_list = self.get_invalidation_list() # Connect to S3 c_conn = boto.connect_cloudfront(self.settings.aws_access_key_id, self.settings.aws_secret_access_key) # Limit of 1000 URLs to invalidate at one time try: count = int(len(invalidation_list) / 1000) + 1 except: # Divide by zero or something else return False array_of_invalidation_list = self.split_array(invalidation_list, count) for i_list in array_of_invalidation_list: inval_req = c_conn.create_invalidation_request( distribution_id, invalidation_list) if (self.logger): self.logger.info('LensCDNInvalidation: %s' % "") return True
def open_cf(self): """ Returns an open connection to CloudFront """ return boto.connect_cloudfront( self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY)
access_key = os.getenv("AWS_ACCESS_KEY") access_secret = os.getenv("AWS_SECRET_KEY") cloudfront_dist = os.getenv("AWS_CF_DISTRIBUTION_ID") bucket = os.getenv("AWS_S3_BUCKET") if access_key == "" or access_key is None: print "Please set AWS_ACCESS_KEY env variable." sys.exit(1) elif access_secret == "" or access_secret is None: print "Please set AWS_SECRET_KEY env variable." sys.exit(1) elif cloudfront_dist == "" or cloudfront_dist is None: print "Please set AWS_CF_DISTRIBUTION_ID env variable." sys.exit(1) elif bucket == "" or bucket is None: print "Please set AWS_S3_BUCKET env variable." sys.exit(1) # get the paths from s3 s3_conn = boto.connect_s3(access_key, access_secret) docs = s3_conn.get_bucket(bucket) items = [] for key in docs.list(): items.append(key.name) cf_conn = boto.connect_cloudfront(access_key, access_secret) inval_req = cf_conn.create_invalidation_request(cloudfront_dist, items) print inval_req sys.exit(0)
def create_distro(): origin = boto.cloudfront.origin.S3Origin(app_util.app_bucket) distro = boto.connect_cloudfront().create_distribution(cnames=[app_util.app_name], origin=origin, enabled=False, comment='Snapyelp Distribution') return distro
def sign_s3_url(url, timeout=None): """create a signed url for amazon s3 authetication, requires that ``boto`` be installed""" c = boto.connect_cloudfront(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) d = c.get_streaming_distribution_info(settings.CLOUDFRONT_DISTRIBUTION_ID) e = int(time.time()+timeout if timeout else getattr(settings, 'CLOUDFRONT_URL_TIMEOUT', 10)) return d.create_signed_url(url, settings.CLOUDFRONT_KEY_PAIR_ID, private_key_file=settings.CLOUDFRONT_PEM)
def __init__(self): self.conn = boto.connect_s3(aws_access_key_id=config.my_access_key, aws_secret_access_key =config.my_secret_key) self.c = boto.connect_cloudfront(config.my_access_key, config.my_secret_key) print 'establish connection for s3'
elif key.endswith('exe'): latest = 'latest.exe' elif key.endswith('32-bit.deb'): latest = 'latest-32.deb' elif key.endswith('64-bit.deb'): latest = 'latest-64.deb' else: print 'File name with full version required. .deb files should end in 32-bit.deb or 64-bit.deb' sys.exit(1) conn = boto.connect_s3() b = conn.get_bucket('lantern') k = Key(b) k.key = key k.copy('lantern', latest, preserve_acl=True) # Since we've just updated the fixed name 'lantest.x' file in our bucket, # we need to make sure to invalidate it on cloudfront in case anyone's # using it. print 'Invalidating latest installers on CloudFront...' c = boto.connect_cloudfront() #rs = c.get_all_distributions() #ds = rs[1] #distro = ds.get_distribution() #print distro.domain_name #print distro.id paths = [latest] inval_req = c.create_invalidation_request(u'E1D7VOTZEUYRZT', paths) status = c.invalidation_request_status(u'E1D7VOTZEUYRZT', inval_req.id)
def connect_cloudfront(self): "Connect to Cloud Front. This is done automatically for you when needed." self.conn_cloudfront = connect_cloudfront(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, debug=self.S3UTILS_DEBUG_LEVEL)
def main(): logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser( description='AWS S3 website deployment tool') parser.add_argument('-f', '--force', action='store_true', dest='force', help='force upload of all files') parser.add_argument('-n', '--dry-run', action='store_true', dest='dry', help='run without uploading any files') parser.add_argument( 'path', help='the .s3_website.yaml configuration file or directory', default='.', nargs='?') args = parser.parse_args() # Open configuration file conf, base_path = config.load_config_file(args.path) bucket_name = conf['s3_bucket'] cache_rules = conf.get('cache_rules', []) if 's3_reduced_redundancy' in conf.keys(): reduced_redundancy = conf['s3_reduced_redundancy'] else: reduced_redundancy = False logger.info('Connecting to bucket {}...'.format(bucket_name)) conn = S3Connection(calling_format=OrdinaryCallingFormat()) bucket = conn.get_bucket(bucket_name, validate=False) site_dir = os.path.join(base_path, conf['site']) logger.info('Site: {}'.format(site_dir)) processed_keys = set() updated_keys = set() for key in bucket: processed_keys.add(key.key) path = os.path.join(site_dir, key.key) # Delete keys that have been deleted locally if not os.path.isfile(path): logger.info('Deleting {}...'.format(key.key)) if not args.dry: key.delete() updated_keys.add(key.key) continue # Skip keys that have not been updated mtime = int(os.path.getmtime(path)) if not args.force: # Update key metadata if not available. # The bucket list() call that is executed through the bucket # iteration above actually does obtain the last modified date # from the server, but boto currently does not update the key # variables based on that. We need to do an additional get_key() # request to get the field populated. key = bucket.get_key(key.key) key_mtime = mktime_tz(parsedate_tz(key.last_modified)) if mtime <= key_mtime: logger.info('Not modified, skipping {}.'.format(key.key)) continue upload_key(key, path, cache_rules, args.dry, replace=True, reduced_redundancy=reduced_redundancy) updated_keys.add(key.key) for dirpath, dirnames, filenames in os.walk(site_dir): key_base = os.path.relpath(dirpath, site_dir) for name in filenames: path = os.path.join(dirpath, name) key_name = key_name_from_path(os.path.join(key_base, name)) if key_name in processed_keys: continue # Create new key key = Key(bucket) key.key = key_name logger.info('Creating key {}...'.format(key_name)) upload_key(key, path, cache_rules, args.dry, replace=False, reduced_redundancy=reduced_redundancy) updated_keys.add(key_name) logger.info('Bucket update done.') # Invalidate files in cloudfront distribution if 'cloudfront_distribution_id' in conf: logger.info('Connecting to Cloudfront distribution {}...'.format( conf['cloudfront_distribution_id'])) index_pattern = None if 'index_document' in conf: index_doc = conf['index_document'] index_pattern = r'(^(?:.*/)?)' + re.escape(index_doc) + '$' def path_from_key_name(key_name): if index_pattern is not None: m = re.match(index_pattern, key_name) if m: return m.group(1) return key_name t = PrefixCoverTree() for key_name in updated_keys: t.include(path_from_key_name(key_name)) for key_name in processed_keys - updated_keys: t.exclude(path_from_key_name(key_name)) paths = [] for prefix, exact in t.matches(): path = '/' + prefix + ('' if exact else '*') logger.info('Preparing to invalidate {}...'.format(path)) paths.append(path) conn = boto.connect_cloudfront() if len(paths) > 0: dist_id = conf['cloudfront_distribution_id'] if not args.dry: logger.info('Creating invalidation request...') conn.create_invalidation_request(dist_id, paths) else: logger.info('Nothing updated, skipping invalidation...') logger.info('Cloudfront invalidation done.')
import boto, os conn = boto.connect_cloudfront(os.environ['AWS_ACCESS_KEY_ID'], os.environ['AWS_SECRET_ACCESS_KEY']) folder_name = os.path.split(os.getcwd())[-1] # same as folder name print "Invalidating files: " paths = [ 'en/' + folder_name + '/index.html', 'en/' + folder_name + '/game.js', 'en/' + folder_name + '/api.js', 'en/' + folder_name + '/promo.zip', # Add more files ] for path in paths: print path inval_req = conn.create_invalidation_request(u'E1OS1XR1ELDPOJ', paths) print 'Cloudfront invalidation done ... please check again after 5 minutes'
def open_cf(self): """ Returns an open connection to CloudFront """ return boto.connect_cloudfront(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY)
import boto c = boto.connect_cloudfront() all_distributions = c.get_all_distributions() print all_distributions for distribution in all_distributions: print distribution.domain_name, "-->", distribution.comment, "-->", distribution.origin, "-->", distribution.status #print distribution_id.domain_name
def main(): cloudfront_connection = boto.connect_cloudfront() s3_connection = connect_to_s3() bucket = Bucket(s3_connection, S3_BUCKET) publish(bucket, cloudfront_connection)