def aws_update_status(message, S3_BUCKET, S3_KEYBASE): if S3_BUCKET == None: print "\n. Error, you haven't defined an S3 bucket." exit() if S3_KEYBASE == None: print "\n. Error, you haven't defined an S3 key base." exit() """Update the status field in AWS S3 for this job""" #s3 = boto.connect_s3() s3 = S3Connection() bucket = s3.lookup(S3_BUCKET) if bucket == None: bucket = s3.create_bucket(S3_BUCKET, location=S3LOCATION) bucket.set_acl('public-read') STATUS_KEY = S3_KEYBASE + "/status" key = bucket.get_key(STATUS_KEY) if key == None: key = bucket.new_key(STATUS_KEY) #key = bucket.get_key(STATUS_KEY) if key == None: print "\n. Error 39 - the key is None" exit() key.set_contents_from_string(message) key.set_acl('public-read') print "\n. S3 Status Update:", key.get_contents_as_string()
def put_into_s3(bucket_name, bundle, t, f): key_name = '/'.join(map(str, bundle) + [t]) b = s3.lookup(bucket_name) key = b.lookup(key_name) if key is None: key = b.new_key(key_name) return key.set_contents_from_file(f)
def get_file_from_cache_or_s3(bucket, fn, dst, cache=True): hostname, _, _ = socket.gethostname().partition('.') look_dirs = get_property('s3_cache', hostname) if cache: cache_dir = look_dirs[-1:] else: cache_dir = [] for d in look_dirs: path = (LocalPath(d) / bucket / fn) if path.exists(): mkdir_p(dst) path.copy(dst) break else: b = s3.lookup(bucket) key = b.lookup(fn) if key is None: raise NoFileOnS3( "Key missing from bucket {bucket}: {key_name}".format( bucket=bucket, key_name=fn)) mkdir_p(dst) with open(dst, 'w') as out_f: out_f.write(key.read()) for d in cache_dir: path = (LocalPath(d) / bucket / fn) if not path.exists(): mkdir_p(path) LocalPath(dst).copy(path)
def push_database_to_s3(dbpath, S3_BUCKET, S3_KEYBASE): """Pushes the startup files for a job to S3. jobid is the ID of the Job object, filepath is the path on this server to the file.""" #s3 = S3Connection() print "\n. Pushing the database to S3" s3 = S3Connection() #s3 = boto.connect_s3() #print "46: connected to", s3.Location # allBuckets = s3.get_all_buckets() # for bucket in allBuckets: # print(str(bucket.name)) # allKeys = bucket.get_all_keys() # for key in allKeys: # print "\t", key.name bucket = s3.lookup(S3_BUCKET) if bucket == None: bucket = s3.create_bucket(S3_BUCKET, location=S3LOCATION) bucket.set_acl('public-read') SQLDB_KEY = S3_KEYBASE + "/sqldb" key = bucket.get_key(SQLDB_KEY) if key == None: key = bucket.new_key(SQLDB_KEY) key.set_contents_from_filename(dbpath) key.set_acl('public-read')
def aws_checkpoint(checkpoint, S3_BUCKET, S3_KEYBASE): if S3_BUCKET == None: print "\n. Error, you haven't defined an S3 bucket." exit() if S3_KEYBASE == None: print "\n. Error, you haven't defined an S3 key base." exit() """Update the status field in AWS S3 for this job""" #s3 = boto.connect_s3() s3 = S3Connection() bucket = s3.lookup(S3_BUCKET) if bucket == None: bucket = s3.create_bucket(S3_BUCKET, location=S3LOCATION) bucket.set_acl('public-read') CHECKPOINT_KEY = S3_KEYBASE + "/checkpoint" key = bucket.get_key(CHECKPOINT_KEY) if key == None: key = bucket.new_key(CHECKPOINT_KEY) #key = bucket.get_key(STATUS_KEY) if key == None: print "\n. Error 67 - the key is None" exit() key.set_contents_from_string(checkpoint.__str__()) key.set_acl('public-read') print "\n. S3 Checkpoint:", key.get_contents_as_string()
def get_s3_bucket(): s3 = boto.connect_s3() bucket = s3.lookup(S3BUCKET) # bucket names must be unique if bucket == None: bucket = s3.create_bucket(S3BUCKET, location=S3LOCATION) bucket.set_acl('public-read') return bucket
def configure_s3_buckets(account_name: str, cfg: dict): account_id = get_account_id() for _, config in cfg.get("s3_buckets", {}).items(): for region in config.get("regions", []): bucket_name = config["name"] bucket_name = bucket_name.replace("{account_id}", account_id).replace("{region}", region) s3 = boto.s3.connect_to_region(region) with Action("Checking S3 bucket {}..".format(bucket_name)): bucket = s3.lookup(bucket_name) if not bucket: with Action("Creating S3 bucket {}..".format(bucket_name)): s3.create_bucket(bucket_name, location=region) with Action("Updating policy for S3 bucket {}..".format(bucket_name)): bucket = s3.lookup(bucket_name) policy_json = json.dumps(config.get("policy")) policy_json = policy_json.replace("{bucket_name}", bucket_name) bucket.set_policy(policy_json)
def clean_queue(bkt): ### We use boto because for boto3 there is no easy way to get a complete list of all multipart in progress s3 = boto.connect_s3() bucket = s3.lookup(bkt) mp_list = bucket.list_multipart_uploads() for mp in mp_list: print "Key: %s ID: %s" % (mp.key_name, mp.id) bucket.cancel_multipart_upload(mp.key_name, mp.id)
def get_or_create_bucket(): s3 = get_s3_connection() b = s3.lookup(AWS_BUCKET) if b is None: print('Creating bucket: ' + AWS_BUCKET + ' in region: ' + AWS_REGION + '...') b = s3.create_bucket(AWS_BUCKET, location=AWS_REGION) else: print('Found bucket: ' + AWS_BUCKET + '.') return b
def get_or_create_bucket(): s3 = get_s3_connection() b = s3.lookup(AWS_BUCKET) if b is None: print('Creating bucket: ' + AWS_BUCKET + ' in region: ' + AWS_REGION + '...') LOCATION = AWS_REGION if AWS_REGION != 'us-east-1' else '' b = s3.create_bucket(AWS_BUCKET, location=LOCATION) else: print('Found bucket: ' + AWS_BUCKET + '.') return b
def check_s3_bucket(bucket_name: str, region: str): with Action("Checking S3 bucket {}..".format(bucket_name)): exists = False try: s3 = boto.s3.connect_to_region(region) exists = s3.lookup(bucket_name, validate=True) except: pass if not exists: with Action("Creating S3 bucket {}...".format(bucket_name)): s3.create_bucket(bucket_name, location=region)
def get_job_exe(jobid): s3 = S3Connection() bucket = s3.lookup(S3BUCKET) if bucket == None: return None EXE_KEY = jobid.__str__() + "/exe" print "attempting to get key", EXE_KEY key = bucket.get_key(EXE_KEY) if key == None: return None return key.get_contents_as_string(exe)
def find_bucket(bucket_name): # Use access key to access secure buckets! # s3 = boto.connect_s3(aws_access_key_id='your_access_key', aws_secret_access_key='your_secret_key') s3 = boto.connect_s3() bucket = s3.lookup(bucket_name) if bucket: print 'Bucket (%s) found' % bucket_name else: print 'Bucket (%s) not found' % bucket_name return bucket
def get_or_create_bucket(): s3 = get_s3_connection() b = s3.lookup(AWS_BUCKET) if b is None: print('Creating bucket: ' + AWS_BUCKET + ' in region: ' + AWS_REGION + '...') LOCATION = AWS_REGION if AWS_REGION != 'us-east-1' else '' b = s3.create_bucket(AWS_BUCKET, location=LOCATION, policy='public-read') b.set_acl('public-read') b.configure_website('index.html', 'error.html') set_bucket_policy(b) set_bucket_role_policy() else: print('Found bucket: ' + AWS_BUCKET + '.') return b
def convert(bucket_name, aws_id, aws_key): s3 = boto.connect_s3(aws_id, aws_key) bucket = s3.lookup(bucket_name) if not bucket: sys.stderr.write('Invalid authentication, or bucketname. Try again.\n') sys.exit(1) print 'Found bucket: %s' % bucket_name sys.stdout.write('Converting: ') sys.stdout.flush() found = converted = 0 try: for key in bucket.list(): found += 1 if key.storage_class != 'REDUCED_REDUNDANCY': key.change_storage_class('REDUCED_REDUNDANCY') converted += 1 if found % 100 == 0: sys.stdout.write('.') sys.stdout.flush() except KeyboardInterrupt: pass print '\nConverted %d items out of %d to reduced redundancy storage.' % \ (converted, found)
def setup_slave_startup_script(jobid): s3 = S3Connection() bucket = s3.lookup(S3BUCKET) SLAVE_STARTUP_SCRIPT_KEY = jobid.__str__() + "/slave_startup_script" key = bucket.get_key(SLAVE_STARTUP_SCRIPT_KEY) if key == None: key = bucket.new_key(SLAVE_STARTUP_SCRIPT_KEY) if key == None: print "\n. Error 115 - the key is None" exit() """The startup script:""" l = "" l += "cd ~/\n" l += "sudo rm -rf repository/asr-pipeline\n" l += "cd repository\n" l += "sudo git clone https://github.com/vhsvhs/asr-pipeline\n" l += "sudo rm -rf lazarus\n" l += "sudo git clone https://github.com/vhsvhs/lazarus lazarus\n" l += "cd ~/\n" key.set_contents_from_string(l) key.set_acl('public-read')
def list_from_s3(bucket_name, bundle, prefix=''): key_name = '/'.join(map(str, bundle)) + '/' + prefix b = s3.lookup(bucket_name) return b.list(key_name)
def create_archive_from_directory(root_directory, *args, **kwargs): aws_access_key_id = kwargs.get('aws_access_key_id', kwargs.get('access', None)) aws_secret_access_key = kwargs.get('aws_secret_access_key', kwargs.get('secret', None)) region_name = kwargs.get('region', 'us-east-1') stage = kwargs.get('stage', 'alpha') stripped_root_directory = root_directory[:-1] if root_directory[-1] == '/' else root_directory project_name = kwargs.get('project_name', stripped_root_directory.split('/')[-1]) autoconfigure = kwargs.get('autoconfigure', False) iam = boto.connect_iam(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) account_id = iam.get_user()['get_user_response']['get_user_result']['user']['arn'].split(':')[4] bucket_name = kwargs.get('bucket_name', '{}-dosidicus'.format(account_id)) m = hashlib.md5() m.update('{}{}{}'.format(account_id,project_name,time.time())) version_id = m.hexdigest() archive_name = '{}-{}'.format(project_name, version_id) archive_file = os.path.join('/tmp','{}.zip'.format(archive_name)) archive_key = os.path.split(archive_file)[1] root_length = len(stripped_root_directory) with zipfile.ZipFile(file=archive_file, mode='a') as newzip: for root, dirnames, filenames in os.walk(root_directory): for filename in filenames: file_path = os.path.join(root, filename) arc_path = file_path[root_length:] newzip.write(filename=file_path, arcname=arc_path) apply_relevant_configs(project_root=root_directory, region=region_name, stage=stage, zip=newzip) if autoconfigure: # Could just assign each one to a new variable, but grouping them together like this kinda feels good. values = {'STAGE': stage} for z in newzip.infolist(): if z.filename.endswith('wsgi.py'): values['WSGIPath'] = z.filename if z.filename.endswith('settings.py'): values['DJANGO_SETTINGS_MODULE'] = z.filename[:-3].replace('/','.') if 'static/' in z.filename: start_index = z.filename.find('static/') end_index = start_index + 7 values['/static/'] = z.filename[:end_index] if values.get('DJANGO_SETTINGS_MODULE'): values['appname'] = values['DJANGO_SETTINGS_MODULE'].split('/')[0] config_contents = """option_settings: "aws:elasticbeanstalk:application:environment": STAGE: "{}" DJANGO_SETTINGS_MODULE: "{}" "PYTHONPATH": "/opt/python/current/app/{}:$PYTHONPATH" "aws:elasticbeanstalk:container:python": WSGIPath: {} NumProcesses: 3 NumThreads: 20 "aws:elasticbeanstalk:container:python:staticfiles": "/static/": "{}" container_commands: 01_migrate: command: "python manage.py migrate --noinput" leader_only: true """.format(values['STAGE'], values['DJANGO_SETTINGS_MODULE'], values['appname'], values['WSGIPath'], values['/static/']) # if a user happens to already have a config file at .ebextensions/dosidicus.auto.config, well, that's really weird. newzip.writestr('.ebextensions/dosidicus.auto.config', config_contents) s3 = boto.s3.connect_to_region(region_name=region_name, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) if not s3.lookup(bucket_name=bucket_name): s3.create_bucket(bucket_name=bucket_name) bucket = s3.get_bucket(bucket_name=bucket_name) key = boto.s3.key.Key(bucket=bucket, name=archive_key) key.set_contents_from_filename(filename=archive_file) return bucket_name, archive_key, archive_name
def s3_bucket_exists(bucket_name): s3 = boto.connect_s3() if s3.lookup(bucket_name): return True else: return False