def get_ami_from_profile(profile_id):
with app.app_context():
profile = profiles.get_profile(profile_id)
if 'ami' in profile and profile['ami']:
return profile['ami']
client = boto3.client('ec2',
region_name=current_app.config['aws']['region'])
filters = yaml.load(profile['filter'])
if 'owner' in profile:
response = client.describe_images(Owners=[profile['owner']],
Filters=filters)
else:
response = client.describe_images(Filters=filters)
if 'Images' not in response:
raise LookupError('Unable to find AMI with required filters')
response['Images']
latest = None
for image in response['Images']:
if not latest:
latest = image
continue
if parser.parse(image['CreationDate']) > parser.parse(
latest['CreationDate']):
latest = image
return latest['ImageId']
def get_launch_instance_arguments_from_profile(profile_id,
owner,
group_id,
size,
label=False,
shutdown=False,
gpuidle=False):
# within this block, current_app points to app.
profile = profiles.get_profile(profile_id)
userdata = profile['userdata']
ImageID = get_ami_from_profile(profile_id)
startArgs = {
'DryRun': False,
'ImageId': ImageID,
'MinCount': 1,
'MaxCount': 1,
'UserData': userdata,
'Monitoring': {
'Enabled': True
},
'DisableApiTermination': False,
'InstanceInitiatedShutdownBehavior': 'stop',
'EbsOptimized': app.config['aws'].get('ebs_optimized', False),
}
if 'subnets' not in current_app.config['aws']:
raise ValueError("SUBNET_ID must be saved in configuration")
if 'security_group' in current_app.config['aws']:
startArgs['SecurityGroupIds'] = [
current_app.config['aws']['security_group']
]
if 'iam_instance_profile' in current_app.config['aws']:
startArgs['IamInstanceProfile'] = {
'Arn': current_app.config['aws']['iam_instance_profile']
}
tags = get_base_tags(profile['name'], owner, group_id, size, label,
shutdown, gpuidle)
if profile['tags']:
for line in profile['tags'].splitlines():
if len(line) > 3 and '=' in line:
tag_name, tag_value = line.split('=', 1)
tags.append({'Key': tag_name, 'Value': tag_value})
startArgs['TagSpecifications'] = [
{
'ResourceType': 'instance',
'Tags': tags
},
{
'ResourceType': 'volume',
'Tags': tags
},
]
return startArgs
def api_profiles_delete(profile_id):
if request.method == 'DELETE':
# Remove profile stored at specified id.
if not profiles.get_profile(profile_id):
# Profile does not exist
return abort(404)
profiles.remove_profile(profile_id)
return jsonify({'status': 'ok'})
def profiles_update(profile_id):
profile = profiles.get_profile(profile_id)
print(profile)
if request.method == 'POST':
name = request.form.get('name', profile['name'])
ami = request.form.get('ami', profile['ami'])
owner = request.form.get('owner', profile['owner'])
filter = request.form.get('filter', profile['filter'])
userdata = request.form.get('userdata', None)
profiles.update_profile(profile_id, name, ami, owner, filter, userdata)
return redirect(url_for('profiles_list'))
return render_template("profile_form.html", profile=profile)
def api_profiles_update(profile_id):
"""Update or delete specified profile id."""
if request.method == 'PUT':
existing_profile = profiles.get_profile(profile_id)
# Update existing profile with new profile object.
if not existing_profile:
# Profile does not exist
return abort(404)
profile = request.get_json()
name = profile.get('name', existing_profile['name'])
ami = profile.get('ami', existing_profile['ami'])
userdata = profile.get('userdata', existing_profile['userdata'])
profiles.update_profile(profile_id, name, ami, userdata)
return jsonify({'status': 'ok', 'id': profile_id})
def launch_instance(group_id,
profile_id,
instancetype,
owner,
size=120,
label=False,
shutdown=False,
gpuidle=False):
with app.app_context():
# within this block, current_app points to app.
profile = profiles.get_profile(profile_id)
print('Launching %s for %s with profile "%s"' %
(instancetype, owner, profile))
userdata = profile['userdata']
ImageID = get_ami_from_profile(profile_id)
startArgs = {
'DryRun':
False,
'ImageId':
ImageID,
'MinCount':
1,
'MaxCount':
1,
'UserData':
userdata,
'InstanceType':
instancetype,
'Monitoring': {
'Enabled': True
},
'DisableApiTermination':
False,
'InstanceInitiatedShutdownBehavior':
'stop',
'EbsOptimized':
app.config['aws'].get('ebs_optimized', False),
'BlockDeviceMappings': [{
'DeviceName': '/dev/sda1',
'Ebs': {
'VolumeSize': size,
'VolumeType': 'gp2'
}
}]
}
if 'subnets' not in current_app.config['aws']:
raise ValueError("SUBNET_ID must be saved in configuration")
if 'security_group' in current_app.config['aws']:
startArgs['SecurityGroupIds'] = [
current_app.config['aws']['security_group']
]
if 'iam_instance_profile' in current_app.config['aws']:
startArgs['IamInstanceProfile'] = {
'Arn': current_app.config['aws']['iam_instance_profile']
}
ec2 = get_ec2_client()
# Attempt on all available subnets
subnets = current_app.config['aws']['subnets'][:]
while True:
startArgs['SubnetId'] = subnets.pop(0)
try:
instances = ec2.create_instances(**startArgs)
break
except ClientError as e:
if len(subnets) < 1:
raise e
# Wait for all machine requests to process so we can tag them.
while True:
launched = True
for instance in instances:
if instance.state == 16:
launched = False
if launched:
break
time.sleep(5)
autolive = app.config['aws'].get('auto_live', False)
sitetag = app.config['general'].get('site_name', 'nebula')
for instance in instances:
print('Cluster start - tag')
tags = [{
'Key': sitetag,
'Value': 'true'
}, {
'Key': 'User',
'Value': owner
}, {
'Key': 'Profile',
'Value': profile['name']
}, {
'Key': 'Group',
'Value': group_id
}]
# Tag network devices- useful for cost exploration.
for eni in instance.network_interfaces:
print('tagging network interface')
eni.create_tags(Tags=tags)
# Tag attached devices. Volumes initialize slowly so schedule another task.
tag_instance_volumes.delay(instance.instance_id, tags)
tags.append({'Key': 'Disk_Space', 'Value': str(size)})
if label:
tags.append({'Key': 'Label', 'Value': label})
if shutdown:
tags.append({'Key': 'Shutdown', 'Value': shutdown})
if gpuidle:
tags.append({'Key': 'GPU_Shutdown', 'Value': gpuidle})
if autolive:
tags.append({'Key': 'Status', 'Value': 'Live'})
instance.create_tags(Tags=tags)
return True