def aws_update_status(message, S3_BUCKET, S3_KEYBASE):
if S3_BUCKET == None:
print "\n. Error, you haven't defined an S3 bucket."
exit()
if S3_KEYBASE == None:
print "\n. Error, you haven't defined an S3 key base."
exit()
"""Update the status field in AWS S3 for this job"""
#s3 = boto.connect_s3()
s3 = S3Connection()
bucket = s3.lookup(S3_BUCKET)
if bucket == None:
bucket = s3.create_bucket(S3_BUCKET, location=S3LOCATION)
bucket.set_acl('public-read')
STATUS_KEY = S3_KEYBASE + "/status"
key = bucket.get_key(STATUS_KEY)
if key == None:
key = bucket.new_key(STATUS_KEY)
#key = bucket.get_key(STATUS_KEY)
if key == None:
print "\n. Error 39 - the key is None"
exit()
key.set_contents_from_string(message)
key.set_acl('public-read')
print "\n. S3 Status Update:", key.get_contents_as_string()
def put_into_s3(bucket_name, bundle, t, f):
key_name = '/'.join(map(str, bundle) + [t])
b = s3.lookup(bucket_name)
key = b.lookup(key_name)
if key is None:
key = b.new_key(key_name)
return key.set_contents_from_file(f)
def get_file_from_cache_or_s3(bucket, fn, dst, cache=True):
hostname, _, _ = socket.gethostname().partition('.')
look_dirs = get_property('s3_cache', hostname)
if cache:
cache_dir = look_dirs[-1:]
else:
cache_dir = []
for d in look_dirs:
path = (LocalPath(d) / bucket / fn)
if path.exists():
mkdir_p(dst)
path.copy(dst)
break
else:
b = s3.lookup(bucket)
key = b.lookup(fn)
if key is None:
raise NoFileOnS3(
"Key missing from bucket {bucket}: {key_name}".format(
bucket=bucket, key_name=fn))
mkdir_p(dst)
with open(dst, 'w') as out_f:
out_f.write(key.read())
for d in cache_dir:
path = (LocalPath(d) / bucket / fn)
if not path.exists():
mkdir_p(path)
LocalPath(dst).copy(path)
def push_database_to_s3(dbpath, S3_BUCKET, S3_KEYBASE):
"""Pushes the startup files for a job to S3.
jobid is the ID of the Job object,
filepath is the path on this server to the file."""
#s3 = S3Connection()
print "\n. Pushing the database to S3"
s3 = S3Connection()
#s3 = boto.connect_s3()
#print "46: connected to", s3.Location
# allBuckets = s3.get_all_buckets()
# for bucket in allBuckets:
# print(str(bucket.name))
# allKeys = bucket.get_all_keys()
# for key in allKeys:
# print "\t", key.name
bucket = s3.lookup(S3_BUCKET)
if bucket == None:
bucket = s3.create_bucket(S3_BUCKET, location=S3LOCATION)
bucket.set_acl('public-read')
SQLDB_KEY = S3_KEYBASE + "/sqldb"
key = bucket.get_key(SQLDB_KEY)
if key == None:
key = bucket.new_key(SQLDB_KEY)
key.set_contents_from_filename(dbpath)
key.set_acl('public-read')
def aws_checkpoint(checkpoint, S3_BUCKET, S3_KEYBASE):
if S3_BUCKET == None:
print "\n. Error, you haven't defined an S3 bucket."
exit()
if S3_KEYBASE == None:
print "\n. Error, you haven't defined an S3 key base."
exit()
"""Update the status field in AWS S3 for this job"""
#s3 = boto.connect_s3()
s3 = S3Connection()
bucket = s3.lookup(S3_BUCKET)
if bucket == None:
bucket = s3.create_bucket(S3_BUCKET, location=S3LOCATION)
bucket.set_acl('public-read')
CHECKPOINT_KEY = S3_KEYBASE + "/checkpoint"
key = bucket.get_key(CHECKPOINT_KEY)
if key == None:
key = bucket.new_key(CHECKPOINT_KEY)
#key = bucket.get_key(STATUS_KEY)
if key == None:
print "\n. Error 67 - the key is None"
exit()
key.set_contents_from_string(checkpoint.__str__())
key.set_acl('public-read')
print "\n. S3 Checkpoint:", key.get_contents_as_string()
def get_s3_bucket():
s3 = boto.connect_s3()
bucket = s3.lookup(S3BUCKET) # bucket names must be unique
if bucket == None:
bucket = s3.create_bucket(S3BUCKET, location=S3LOCATION)
bucket.set_acl('public-read')
return bucket
def configure_s3_buckets(account_name: str, cfg: dict):
account_id = get_account_id()
for _, config in cfg.get("s3_buckets", {}).items():
for region in config.get("regions", []):
bucket_name = config["name"]
bucket_name = bucket_name.replace("{account_id}", account_id).replace("{region}", region)
s3 = boto.s3.connect_to_region(region)
with Action("Checking S3 bucket {}..".format(bucket_name)):
bucket = s3.lookup(bucket_name)
if not bucket:
with Action("Creating S3 bucket {}..".format(bucket_name)):
s3.create_bucket(bucket_name, location=region)
with Action("Updating policy for S3 bucket {}..".format(bucket_name)):
bucket = s3.lookup(bucket_name)
policy_json = json.dumps(config.get("policy"))
policy_json = policy_json.replace("{bucket_name}", bucket_name)
bucket.set_policy(policy_json)
def clean_queue(bkt):
### We use boto because for boto3 there is no easy way to get a complete list of all multipart in progress
s3 = boto.connect_s3()
bucket = s3.lookup(bkt)
mp_list = bucket.list_multipart_uploads()
for mp in mp_list:
print "Key: %s ID: %s" % (mp.key_name, mp.id)
bucket.cancel_multipart_upload(mp.key_name, mp.id)
def get_or_create_bucket():
s3 = get_s3_connection()
b = s3.lookup(AWS_BUCKET)
if b is None:
print('Creating bucket: ' + AWS_BUCKET + ' in region: ' + AWS_REGION + '...')
b = s3.create_bucket(AWS_BUCKET, location=AWS_REGION)
else:
print('Found bucket: ' + AWS_BUCKET + '.')
return b
def get_or_create_bucket():
s3 = get_s3_connection()
b = s3.lookup(AWS_BUCKET)
if b is None:
print('Creating bucket: ' + AWS_BUCKET + ' in region: ' + AWS_REGION + '...')
LOCATION = AWS_REGION if AWS_REGION != 'us-east-1' else ''
b = s3.create_bucket(AWS_BUCKET, location=LOCATION)
else:
print('Found bucket: ' + AWS_BUCKET + '.')
return b
def check_s3_bucket(bucket_name: str, region: str):
with Action("Checking S3 bucket {}..".format(bucket_name)):
exists = False
try:
s3 = boto.s3.connect_to_region(region)
exists = s3.lookup(bucket_name, validate=True)
except:
pass
if not exists:
with Action("Creating S3 bucket {}...".format(bucket_name)):
s3.create_bucket(bucket_name, location=region)
def check_s3_bucket(bucket_name: str, region: str):
with Action("Checking S3 bucket {}..".format(bucket_name)):
exists = False
try:
s3 = boto.s3.connect_to_region(region)
exists = s3.lookup(bucket_name, validate=True)
except:
pass
if not exists:
with Action("Creating S3 bucket {}...".format(bucket_name)):
s3.create_bucket(bucket_name, location=region)
def get_job_exe(jobid):
s3 = S3Connection()
bucket = s3.lookup(S3BUCKET)
if bucket == None:
return None
EXE_KEY = jobid.__str__() + "/exe"
print "attempting to get key", EXE_KEY
key = bucket.get_key(EXE_KEY)
if key == None:
return None
return key.get_contents_as_string(exe)
def find_bucket(bucket_name):
# Use access key to access secure buckets!
# s3 = boto.connect_s3(aws_access_key_id='your_access_key', aws_secret_access_key='your_secret_key')
s3 = boto.connect_s3()
bucket = s3.lookup(bucket_name)
if bucket:
print 'Bucket (%s) found' % bucket_name
else:
print 'Bucket (%s) not found' % bucket_name
return bucket
def find_bucket(bucket_name):
# Use access key to access secure buckets!
# s3 = boto.connect_s3(aws_access_key_id='your_access_key', aws_secret_access_key='your_secret_key')
s3 = boto.connect_s3()
bucket = s3.lookup(bucket_name)
if bucket:
print 'Bucket (%s) found' % bucket_name
else:
print 'Bucket (%s) not found' % bucket_name
return bucket
def get_or_create_bucket():
s3 = get_s3_connection()
b = s3.lookup(AWS_BUCKET)
if b is None:
print('Creating bucket: ' + AWS_BUCKET + ' in region: ' + AWS_REGION + '...')
LOCATION = AWS_REGION if AWS_REGION != 'us-east-1' else ''
b = s3.create_bucket(AWS_BUCKET, location=LOCATION, policy='public-read')
b.set_acl('public-read')
b.configure_website('index.html', 'error.html')
set_bucket_policy(b)
set_bucket_role_policy()
else:
print('Found bucket: ' + AWS_BUCKET + '.')
return b
def convert(bucket_name, aws_id, aws_key):
s3 = boto.connect_s3(aws_id, aws_key)
bucket = s3.lookup(bucket_name)
if not bucket:
sys.stderr.write('Invalid authentication, or bucketname. Try again.\n')
sys.exit(1)
print 'Found bucket: %s' % bucket_name
sys.stdout.write('Converting: ')
sys.stdout.flush()
found = converted = 0
try:
for key in bucket.list():
found += 1
if key.storage_class != 'REDUCED_REDUNDANCY':
key.change_storage_class('REDUCED_REDUNDANCY')
converted += 1
if found % 100 == 0:
sys.stdout.write('.')
sys.stdout.flush()
except KeyboardInterrupt: pass
print '\nConverted %d items out of %d to reduced redundancy storage.' % \
(converted, found)
def setup_slave_startup_script(jobid):
s3 = S3Connection()
bucket = s3.lookup(S3BUCKET)
SLAVE_STARTUP_SCRIPT_KEY = jobid.__str__() + "/slave_startup_script"
key = bucket.get_key(SLAVE_STARTUP_SCRIPT_KEY)
if key == None:
key = bucket.new_key(SLAVE_STARTUP_SCRIPT_KEY)
if key == None:
print "\n. Error 115 - the key is None"
exit()
"""The startup script:"""
l = ""
l += "cd ~/\n"
l += "sudo rm -rf repository/asr-pipeline\n"
l += "cd repository\n"
l += "sudo git clone https://github.com/vhsvhs/asr-pipeline\n"
l += "sudo rm -rf lazarus\n"
l += "sudo git clone https://github.com/vhsvhs/lazarus lazarus\n"
l += "cd ~/\n"
key.set_contents_from_string(l)
key.set_acl('public-read')
def convert(bucket_name, aws_id, aws_key):
s3 = boto.connect_s3(aws_id, aws_key)
bucket = s3.lookup(bucket_name)
if not bucket:
sys.stderr.write('Invalid authentication, or bucketname. Try again.\n')
sys.exit(1)
print 'Found bucket: %s' % bucket_name
sys.stdout.write('Converting: ')
sys.stdout.flush()
found = converted = 0
try:
for key in bucket.list():
found += 1
if key.storage_class != 'REDUCED_REDUNDANCY':
key.change_storage_class('REDUCED_REDUNDANCY')
converted += 1
if found % 100 == 0:
sys.stdout.write('.')
sys.stdout.flush()
except KeyboardInterrupt:
pass
print '\nConverted %d items out of %d to reduced redundancy storage.' % \
(converted, found)
def list_from_s3(bucket_name, bundle, prefix=''):
key_name = '/'.join(map(str, bundle)) + '/' + prefix
b = s3.lookup(bucket_name)
return b.list(key_name)
def create_archive_from_directory(root_directory, *args, **kwargs):
aws_access_key_id = kwargs.get('aws_access_key_id', kwargs.get('access', None))
aws_secret_access_key = kwargs.get('aws_secret_access_key', kwargs.get('secret', None))
region_name = kwargs.get('region', 'us-east-1')
stage = kwargs.get('stage', 'alpha')
stripped_root_directory = root_directory[:-1] if root_directory[-1] == '/' else root_directory
project_name = kwargs.get('project_name', stripped_root_directory.split('/')[-1])
autoconfigure = kwargs.get('autoconfigure', False)
iam = boto.connect_iam(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
account_id = iam.get_user()['get_user_response']['get_user_result']['user']['arn'].split(':')[4]
bucket_name = kwargs.get('bucket_name', '{}-dosidicus'.format(account_id))
m = hashlib.md5()
m.update('{}{}{}'.format(account_id,project_name,time.time()))
version_id = m.hexdigest()
archive_name = '{}-{}'.format(project_name, version_id)
archive_file = os.path.join('/tmp','{}.zip'.format(archive_name))
archive_key = os.path.split(archive_file)[1]
root_length = len(stripped_root_directory)
with zipfile.ZipFile(file=archive_file, mode='a') as newzip:
for root, dirnames, filenames in os.walk(root_directory):
for filename in filenames:
file_path = os.path.join(root, filename)
arc_path = file_path[root_length:]
newzip.write(filename=file_path, arcname=arc_path)
apply_relevant_configs(project_root=root_directory, region=region_name, stage=stage, zip=newzip)
if autoconfigure:
# Could just assign each one to a new variable, but grouping them together like this kinda feels good.
values = {'STAGE': stage}
for z in newzip.infolist():
if z.filename.endswith('wsgi.py'):
values['WSGIPath'] = z.filename
if z.filename.endswith('settings.py'):
values['DJANGO_SETTINGS_MODULE'] = z.filename[:-3].replace('/','.')
if 'static/' in z.filename:
start_index = z.filename.find('static/')
end_index = start_index + 7
values['/static/'] = z.filename[:end_index]
if values.get('DJANGO_SETTINGS_MODULE'):
values['appname'] = values['DJANGO_SETTINGS_MODULE'].split('/')[0]
config_contents = """option_settings:
"aws:elasticbeanstalk:application:environment":
STAGE: "{}"
DJANGO_SETTINGS_MODULE: "{}"
"PYTHONPATH": "/opt/python/current/app/{}:$PYTHONPATH"
"aws:elasticbeanstalk:container:python":
WSGIPath: {}
NumProcesses: 3
NumThreads: 20
"aws:elasticbeanstalk:container:python:staticfiles":
"/static/": "{}"
container_commands:
01_migrate:
command: "python manage.py migrate --noinput"
leader_only: true
""".format(values['STAGE'], values['DJANGO_SETTINGS_MODULE'], values['appname'], values['WSGIPath'], values['/static/'])
# if a user happens to already have a config file at .ebextensions/dosidicus.auto.config, well, that's really weird.
newzip.writestr('.ebextensions/dosidicus.auto.config', config_contents)
s3 = boto.s3.connect_to_region(region_name=region_name, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
if not s3.lookup(bucket_name=bucket_name):
s3.create_bucket(bucket_name=bucket_name)
bucket = s3.get_bucket(bucket_name=bucket_name)
key = boto.s3.key.Key(bucket=bucket, name=archive_key)
key.set_contents_from_filename(filename=archive_file)
return bucket_name, archive_key, archive_name
def s3_bucket_exists(bucket_name):
s3 = boto.connect_s3()
if s3.lookup(bucket_name):
return True
else:
return False
