def get_ec2_client():
with app.app_context():
conn = getattr(g, '_ec2', None)
if conn is None:
g._ec2 = boto3.resource(
'ec2', region_name=current_app.config['aws']['region'])
return g._ec2
def get_ami_from_profile(profile_id):
with app.app_context():
profile = profiles.get_profile(profile_id)
if 'ami' in profile and profile['ami']:
return profile['ami']
client = boto3.client('ec2',
region_name=current_app.config['aws']['region'])
filters = yaml.load(profile['filter'])
if 'owner' in profile:
response = client.describe_images(Owners=[profile['owner']],
Filters=filters)
else:
response = client.describe_images(Filters=filters)
if 'Images' not in response:
raise LookupError('Unable to find AMI with required filters')
response['Images']
latest = None
for image in response['Images']:
if not latest:
latest = image
continue
if parser.parse(image['CreationDate']) > parser.parse(
latest['CreationDate']):
latest = image
return latest['ImageId']
def terminate_expired_instances():
print('Processing Autoterminations')
with app.app_context():
if not current_app.config['aws'].get('autoterminate_after_stopped',
False):
return False
curtimestamp = int(datetime.now(pytz.utc).timestamp())
instances = get_instance_list(state='stopped',
terminated=False,
tag_keys=['LastOnline', 'TerminateAfter'])
for instance in instances:
tags = get_tags_from_aws_object(instance)
# LastOnline is updated every five minutes for running machines.
if 'LastOnline' not in tags or not tags['LastOnline'].isdigit():
continue
# TerminateAfter comes from the provisioner- typically a launch template. It is in hours.
if 'TerminateAfter' not in tags or not tags['TerminateAfter'].isdigit(
):
continue
# Cutoff is taken by subtracting the number of days in TerminateAfter from the current date.
# Machines launched before then will be terminated.
cutoff_timestamp = curtimestamp - (int(tags['TerminateAfter']) * 60 *
60)
last_online = int(tags['LastOnline'])
if last_online < cutoff_timestamp:
terminate_instance.delay(instance.instance_id)
def notify_users():
"""Notify users of billing information."""
user_bill = get_total_costs()
for user, info in user_bill.items():
if info['total_cost'] > int(NOTIFICATION_THRESHOLD):
# Prepare body of message
with app.app_context():
text = render_template('email_notification.html',
name=user,
num_instances=info['num_instances'],
total_cost=as_currency(
info['total_cost']))
# Send notification email
email = '*****@*****.**' % (user, )
send_notification_email('Nebula Billing Notification', text, email)
def launch_instance(group_id,
profile_id,
instancetype,
owner,
size=120,
label=False,
shutdown=False,
gpuidle=False):
with app.app_context():
# within this block, current_app points to app.
profile = profiles.get_profile(profile_id)
print('Launching %s for %s with profile "%s"' %
(instancetype, owner, profile))
userdata = profile['userdata']
ImageID = get_ami_from_profile(profile_id)
startArgs = {
'DryRun':
False,
'ImageId':
ImageID,
'MinCount':
1,
'MaxCount':
1,
'UserData':
userdata,
'InstanceType':
instancetype,
'Monitoring': {
'Enabled': True
},
'DisableApiTermination':
False,
'InstanceInitiatedShutdownBehavior':
'stop',
'EbsOptimized':
app.config['aws'].get('ebs_optimized', False),
'BlockDeviceMappings': [{
'DeviceName': '/dev/sda1',
'Ebs': {
'VolumeSize': size,
'VolumeType': 'gp2'
}
}]
}
if 'subnets' not in current_app.config['aws']:
raise ValueError("SUBNET_ID must be saved in configuration")
if 'security_group' in current_app.config['aws']:
startArgs['SecurityGroupIds'] = [
current_app.config['aws']['security_group']
]
if 'iam_instance_profile' in current_app.config['aws']:
startArgs['IamInstanceProfile'] = {
'Arn': current_app.config['aws']['iam_instance_profile']
}
ec2 = get_ec2_client()
# Attempt on all available subnets
subnets = current_app.config['aws']['subnets'][:]
while True:
startArgs['SubnetId'] = subnets.pop(0)
try:
instances = ec2.create_instances(**startArgs)
break
except ClientError as e:
if len(subnets) < 1:
raise e
# Wait for all machine requests to process so we can tag them.
while True:
launched = True
for instance in instances:
if instance.state == 16:
launched = False
if launched:
break
time.sleep(5)
autolive = app.config['aws'].get('auto_live', False)
sitetag = app.config['general'].get('site_name', 'nebula')
for instance in instances:
print('Cluster start - tag')
tags = [{
'Key': sitetag,
'Value': 'true'
}, {
'Key': 'User',
'Value': owner
}, {
'Key': 'Profile',
'Value': profile['name']
}, {
'Key': 'Group',
'Value': group_id
}]
# Tag network devices- useful for cost exploration.
for eni in instance.network_interfaces:
print('tagging network interface')
eni.create_tags(Tags=tags)
# Tag attached devices. Volumes initialize slowly so schedule another task.
tag_instance_volumes.delay(instance.instance_id, tags)
tags.append({'Key': 'Disk_Space', 'Value': str(size)})
if label:
tags.append({'Key': 'Label', 'Value': label})
if shutdown:
tags.append({'Key': 'Shutdown', 'Value': shutdown})
if gpuidle:
tags.append({'Key': 'GPU_Shutdown', 'Value': gpuidle})
if autolive:
tags.append({'Key': 'Status', 'Value': 'Live'})
instance.create_tags(Tags=tags)
return True
def launch_instance(group_id,
profile_id,
instancetype,
owner,
size=120,
label=False,
shutdown=False,
gpuidle=False):
print('Launching %s for %s with profile "%s"' %
(instancetype, owner, profile_id))
with app.app_context():
# within this block, current_app points to app.
if current_app.config['aws'].get('use_launch_templates', False):
startArgs = get_launch_instance_arguments_from_launch_template(
profile_id, owner, group_id, size, label, shutdown, gpuidle)
else:
startArgs = get_launch_instance_arguments_from_profile(
profile_id, owner, group_id, size, label, shutdown, gpuidle)
startArgs['InstanceType'] = instancetype
startArgs['BlockDeviceMappings'] = [{
'DeviceName': '/dev/sda1',
'Ebs': {
'VolumeSize': size,
'VolumeType': 'gp2'
}
}]
ec2 = get_ec2_resource()
# Attempt on all available subnets
subnets = current_app.config['aws']['subnets'][:]
while True:
startArgs['SubnetId'] = subnets.pop(0)
try:
instances = ec2.create_instances(**startArgs)
break
except ClientError as e:
if len(subnets) < 1:
raise e
# Wait for all machine requests to process so we can tag the network interfaces.
while True:
launched = True
for instance in instances:
if instance.state == 16:
launched = False
if launched:
break
time.sleep(5)
# Pull out "instance" tags and apply them to the network interface
existing_instance_tags = [
x for x in startArgs['TagSpecifications']
if x['ResourceType'] == 'instance'
]
if len(existing_instance_tags) > 0:
for instance in instances:
# Tag network devices- useful for cost exploration.
for eni in instance.network_interfaces:
print('tagging network interface')
eni.create_tags(Tags=existing_instance_tags[0]['Tags'])
return True
