def index():
ec2 = get_ec2_connection()
hourly_rate = get_spot_prices(ec2)
instances = get_instance_counts(ec2)
instance_table = []
total = 0
for instance_type, count in instances.items():
cpus = cpus_per_instance[instance_type] * count
total += cpus
instance_table.append((instance_type, count, cpus))
instance_table.append(("Total", "", total))
active_terminals = terminal_manager.get_active_terminals()
active_terminals.sort(lambda a, b: cmp(a.start_time, b.start_time))
cluster_state = get_cluster_state(ec2)
manager_state = cluster_manager.get_manager_state()
request_counts = collections.defaultdict(lambda: 0)
for request in ec2.get_all_spot_instance_requests(filters={"state":"open"}):
name = (request.launch_specification.instance_type, request.price, request.status.message)
request_counts[name] += 1
open_spot_requests = []
for k in request_counts.keys():
instance_type, price, status = k
count = request_counts[k]
cpus = cpus_per_instance[instance_type] * count
open_spot_requests.append( (instance_type, status, price, count, cpus) )
return flask.render_template("index.html", terminals=active_terminals, instances=instance_table, cluster_state=cluster_state,
manager_state=manager_state,
hourly_rate=hourly_rate, open_spot_requests=open_spot_requests)
def get_spot_prices(ec2):
hourly_rate = 0.0
requests = ec2.get_all_spot_instance_requests(filters={"state":"active"})
for spot_request in requests:
if spot_request.state == "active":
print "spot code ", spot_request.id, spot_request.status.code
hourly_rate += spot_request.price
return hourly_rate
def get_launch_time(conf, spot_request_id):
ec2 = get_ec2_conn(conf)
try:
spot_requests = ec2.get_all_spot_instance_requests(request_ids=[spot_request_id])
if len(spot_requests) > 0:
return spot_requests[0].create_time
except boto.exception.EC2ResponseError:
pass
return None
def get_status():
ec2 = conn_region
reservations = ec2.get_all_spot_instance_requests(
filters={'instance-state-name': 'running'})
instances = [i for r in reservations for i in r.instances]
for instance in instances:
if instance.__dict__['tags'].__dict__['_current_key'] == tag_name:
instanceip = instance.__dict__['ip_address']
print "%s \tIP:%s" % (instance, instanceip)
def check_spot_requests():
ec2 = boto.ec2.connect_to_region(REGION)
spot_requests = ec2.get_all_spot_instance_requests(
filters={'status-code':'price-too-low', 'state':'open'}
)
#spot_requests = ec2.get_all_spot_instance_requests(
# filters={
# 'status-code':'instance-terminated-by-price',
# 'state':'closed'})
print spot_requests
# if are spot instances with state on price too low then go forward
if len(spot_requests) > 0:
get_best_option()
else:
print "No open spot requests with status code price-too-low"
def check_spot_requests():
ec2 = boto.ec2.connect_to_region(REGION)
spot_requests = ec2.get_all_spot_instance_requests(filters={
'status-code': 'price-too-low',
'state': 'open'
})
#spot_requests = ec2.get_all_spot_instance_requests(
# filters={
# 'status-code':'instance-terminated-by-price',
# 'state':'closed'})
print spot_requests
# if are spot instances with state on price too low then go forward
if len(spot_requests) > 0:
get_best_option()
else:
print "No open spot requests with status code price-too-low"
def whoAmI():
ec2 = boto.ec2.connect_to_region(
"us-west-1",
aws_access_key_id="AKIAINZ2TABEYCFH7SMQ",
aws_secret_access_key="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")
myID = requests.get("http://169.254.169.254/latest/meta-data/instance-id",
timeout=1.0).text
instance = ec2.get_only_instances([myID])[0]
spotRequestId = instance.spot_instance_request_id
if spotRequestId is not None:
spotRequest = ec2.get_all_spot_instance_requests([spotRequestId])[0]
tags = spotRequest.tags
for key, value in tags.items():
if key not in instance.tags:
instance.add_tag(key, value)
return ec2.get_only_instances([myID])[0].tags
elif placement_group :
module.fail_json(
msg="placement_group parameter requires Boto version 2.3.0 or higher.")
params.update(dict(
count = count_remaining,
type = spot_type,
))
res = ec2.request_spot_instances(spot_price, **params)
# Now we have to do the intermediate waiting
if wait:
spot_req_inst_ids = dict()
spot_wait_timeout = time.time() + spot_wait_timeout
while spot_wait_timeout > time.time():
reqs = ec2.get_all_spot_instance_requests()
for sirb in res:
if sirb.id in spot_req_inst_ids:
continue
for sir in reqs:
if sir.id == sirb.id and sir.instance_id is not None:
spot_req_inst_ids[sirb.id] = sir.instance_id
if len(spot_req_inst_ids) < count:
time.sleep(5)
else:
break
if spot_wait_timeout <= time.time():
module.fail_json(msg = "wait for spot requests timeout on %s" % time.asctime())
instids = spot_req_inst_ids.values()
except boto.exception.BotoServerError, e:
module.fail_json(msg = "Instance creation failed => %s: %s" % (e.error_code, e.error_message))
price=spotBid,
image_id=amiId,
count=numInstancesToLaunch,
key_name=keyName,
instance_type=instanceType,
security_groups=[securityGroupName],
instance_profile_arn=iam_profile_resource_name,
instance_profile_name=iam_profile_name,
dry_run=dryRun,
)
requestIDs = [request.id for request in spotRequests]
fulfilled = []
while requestIDs:
requests = ec2.get_all_spot_instance_requests(request_ids=requestIDs, )
for request in requests:
if request.instance_id:
requestIDs.remove(request.id)
fulfilled.append(request.instance_id)
# unrecoverable error without increasing the spot bid
elif request.status.code == u'price-too-low':
print request.status.message
print 'Cancelling Spot requests...'
# get original request ids since some requests may have already been fulfilled
ec2.cancel_spot_instance_requests(
request_ids=[request.id for request in spotRequests], )
print 'Exiting...'
price="0.005",
count=1,
image_id=ami,
key_name=key_name,
security_groups=[group_name],
instance_type=instance_type,
user_data=user_data,
)[0]
while True:
eprint("Waiting. spot request status: '%s', state: '%s'" %
(spot_request.state, spot_request.status.code))
if spot_request.state == 'active' and spot_request.status.code == 'fulfilled':
break
time.sleep(10)
spot_request = ec2.get_all_spot_instance_requests(
request_ids=[spot_request.id])[0]
while True:
instance = ec2.get_all_instances(
instance_ids=[spot_request.instance_id])[0].instances[0]
eprint("Waiting. spot instance state: '%s'" % instance.state)
if instance.state == 'running':
break
time.sleep(10)
ec2.create_tags([instance.id], {tag: ""})
global host
instance = ec2.get_all_instances(
instance_ids=[spot_request.instance_id])[0].instances[0]
host = instance.ip_address
print("%s" % host)
def get_spot_request_dict(conf):
ec2 = get_ec2_conn(conf)
requests = ec2.get_all_spot_instance_requests()
return dict([(sir.id, sir) for sir in requests])
def get_spot_request_dict(conf):
ec2 = get_ec2_conn(conf)
requests = ec2.get_all_spot_instance_requests()
return dict([(sir.id, sir) for sir in requests])
params['placement_group'] = placement_group
elif placement_group :
module.fail_json(
msg="placement_group parameter requires Boto version 2.3.0 or higher.")
params.update(dict(
count = count_remaining,
))
res = ec2.request_spot_instances(spot_price, **params)
# Now we have to do the intermediate waiting
if wait:
spot_req_inst_ids = dict()
spot_wait_timeout = time.time() + spot_wait_timeout
while spot_wait_timeout > time.time():
reqs = ec2.get_all_spot_instance_requests()
for sirb in res:
if sirb.id in spot_req_inst_ids:
continue
for sir in reqs:
if sir.id == sirb.id and sir.instance_id is not None:
spot_req_inst_ids[sirb.id] = sir.instance_id
if len(spot_req_inst_ids) < count:
time.sleep(5)
else:
break
if spot_wait_timeout <= time.time():
module.fail_json(msg = "wait for spot requests timeout on %s" % time.asctime())
instids = spot_req_inst_ids.values()
except boto.exception.BotoServerError, e:
module.fail_json(msg = "Instance creation failed => %s: %s" % (e.error_code, e.error_message))
def get_all_spot_instance_requests(opts, conf, filters={}):
ec2 = get_ec2_conn(conf)
filters.update(get_filter_tags(dict(opts.tags)))
logging.debug('Spot request filters: %s', filters)
return ec2.get_all_spot_instance_requests(filters=filters)
image_id=amiID,
count=numInstancesToLaunch,
key_name=keyName,
instance_type=instanceType,
security_groups=[securityGroupName],
instance_profile_arn=iam_profile_resource_name,
instance_profile_name=iam_profile_name,
dry_run=dryRun,
)
requestIDs = [request.id for request in spotRequests]
fulfilled = []
while requestIDs:
requests = ec2.get_all_spot_instance_requests(
request_ids=requestIDs,
)
for request in requests:
if request.instance_id:
requestIDs.remove(request.id)
fulfilled.append(request.instance_id)
# unrecoverable error without increasing the spot bid
elif request.status.code == u'price-too-low':
print request.status.message
print 'Cancelling Spot requests...'
# get original request ids since some requests may have already been fulfilled
ec2.cancel_spot_instance_requests(
request_ids=[request.id for request in spotRequests],
)
def _launch_instances_abort_on_error(ami,
bid,
count,
instance_type,
subnet,
assign_public_ip,
source_dest_check,
placement_group,
disks,
security_group_ids,
tags=None):
ephemeral_idx = 0
bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping()
if disks is not None:
for disk in disks:
bdm[disk] = boto.ec2.blockdevicemapping.BlockDeviceType(
ephemeral_name='ephemeral%d' % ephemeral_idx)
ephemeral_idx += 1
nics = boto.ec2.networkinterface.NetworkInterfaceCollection()
nics.append(boto.ec2.networkinterface.NetworkInterfaceSpecification(
subnet_id=subnet,
groups=security_group_ids,
associate_public_ip_address=assign_public_ip))
ec2 = boto.ec2.connect_to_region(env.ec2_region)
sirs = ec2.request_spot_instances(
price=bid,
image_id=ami,
count=count,
type='one-time',
key_name=env.ec2_key_pair_name,
instance_type=instance_type,
monitoring_enabled=True,
placement_group=placement_group,
block_device_map=bdm,
network_interfaces=nics)
instance_ids = set()
while True:
time.sleep(10)
done = True
for sir in ec2.get_all_spot_instance_requests(map(lambda x: x.id, sirs)):
print 'State: %s' % sir.state
print 'Fault: %s' % sir.fault
print 'Status: %s' % sir.status.message
if sir.state not in ('open', 'active'):
abort('Failed to launch instances')
if sir.state == 'open':
done = False
if sir.state == 'active':
instance_ids.add(sir.instance_id)
if done:
break
for instance_id in instance_ids:
ec2.modify_instance_attribute(
instance_id=instance_id,
attribute='sourceDestCheck',
value=source_dest_check)
print ''
print 'Instances:'
for reservation in ec2.get_all_instances(list(instance_ids)):
for instance in reservation.instances:
if tags is not None:
instance.add_tags(tags)
print ' %s' % instance.id
print ' type: %s' % instance.instance_type
print ' internal ip: %s' % instance.private_ip_address
print ' public ip: %s' % instance.ip_address
print ' tags:'
for tag, value in sorted(instance.tags.iteritems(), lambda a, b: cmp(a[0], b[0])):
print ' %s: %s' % (tag, value)
def await_spot_requests(module, ec2, spot_requests, count):
"""
Wait for a group of spot requests to be fulfilled, or fail.
module: Ansible module object
ec2: authenticated ec2 connection object
spot_requests: boto.ec2.spotinstancerequest.SpotInstanceRequest object returned by ec2.request_spot_instances
count: Total number of instances to be created by the spot requests
Returns:
list of instance ID's created by the spot request(s)
"""
spot_wait_timeout = int(module.params.get('spot_wait_timeout'))
wait_complete = time.time() + spot_wait_timeout
spot_req_inst_ids = dict()
while time.time() < wait_complete:
reqs = ec2.get_all_spot_instance_requests()
for sirb in spot_requests:
if sirb.id in spot_req_inst_ids:
continue
for sir in reqs:
if sir.id != sirb.id:
continue # this is not our spot instance
if sir.instance_id is not None:
spot_req_inst_ids[sirb.id] = sir.instance_id
elif sir.state == 'open':
continue # still waiting, nothing to do here
elif sir.state == 'active':
continue # Instance is created already, nothing to do here
elif sir.state == 'failed':
module.fail_json(
msg=
"Spot instance request %s failed with status %s and fault %s:%s"
% (sir.id, sir.status.code, sir.fault.code,
sir.fault.message))
elif sir.state == 'cancelled':
module.fail_json(
msg=
"Spot instance request %s was cancelled before it could be fulfilled."
% sir.id)
elif sir.state == 'closed':
# instance is terminating or marked for termination
# this may be intentional on the part of the operator,
# or it may have been terminated by AWS due to capacity,
# price, or group constraints in this case, we'll fail
# the module if the reason for the state is anything
# other than termination by user. Codes are documented at
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-bid-status.html
if sir.status.code == 'instance-terminated-by-user':
# do nothing, since the user likely did this on purpose
pass
else:
spot_msg = "Spot instance request %s was closed by AWS with the status %s and fault %s:%s"
module.fail_json(msg=spot_msg %
(sir.id, sir.status.code,
sir.fault.code, sir.fault.message))
if len(spot_req_inst_ids) < count:
time.sleep(5)
else:
return spot_req_inst_ids.values()
module.fail_json(msg="wait for spot requests timeout on %s" %
time.asctime())
def show_instances():
ec2 = get_ec2_connection()
spots = ec2.get_all_spot_instance_requests(filters={"state":"open"})
instances = ec2.get_only_instances()
instances = filter_stopped_instances(instances)
return flask.render_template("show-instances.html", instances=instances, spots=spots)
def launch_spot_instance(id,
profile,
spot_wait_sleep=5,
instance_wait_sleep=3):
ec2 = boto.ec2.connect_to_region(profile['region'])
if not 'key_pair' in profile:
print('key pair {0} does not exist'.format(profile['key_pair'][0]))
profile['key_pair'] = ('KP-' + id, 'KP-' + id + '.pem')
try:
print >> sys.stderr, 'Creating key pair...',
keypair = ec2.create_key_pair('KP-' + id)
keypair.save('.')
print >> sys.stderr, 'created'
except boto.exception.EC2ResponseError as e:
if e.code == 'InvalidKeyPair.Duplicate':
print >> sys.stderr, 'already exists'
else:
raise e
if not 'security_group' in profile:
try:
print >> sys.stderr, 'Creating security group...',
sc = ec2.create_security_group('SG-' + id,
'Security Group for ' + id)
for proto, fromport, toport, ip in profile['firewall']:
sc.authorize(proto, fromport, toport, ip)
profile['security_group'] = (sc.id, sc.name)
print >> sys.stderr, 'created'
except boto.exception.EC2ResponseError as e:
if e.code == 'InvalidGroup.Duplicate':
print >> sys.stderr, 'already exists'
sc = ec2.get_all_security_groups(groupnames=['SG-' + id])[0]
profile['security_group'] = (sc.id, sc.name)
else:
raise e
existing_requests = ec2.get_all_spot_instance_requests(
filters={
'launch.group-id': profile['security_group'][0],
'state': ['open', 'active']
})
if existing_requests:
if len(existing_requests) > 1:
raise Exception('Too many existing spot requests')
print >> sys.stderr, 'Reusing existing spot request'
spot_req_id = existing_requests[0].id
else:
bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping()
bdm['/dev/sda1'] = boto.ec2.blockdevicemapping.BlockDeviceType(
volume_type='gp2',
size=profile['disk_size'],
delete_on_termination=profile['disk_delete_on_termination'])
bdm['/dev/sdb'] = boto.ec2.blockdevicemapping.BlockDeviceType(
ephemeral_name='ephemeral0')
print >> sys.stderr, 'Requesting spot instance'
spot_reqs = ec2.request_spot_instances(
price=profile['price'],
image_id=profile['image_id'],
instance_type=profile['type'],
placement=profile['region'] + profile['availability_zone'],
security_groups=[profile['security_group'][1]],
key_name=profile['key_pair'][0],
block_device_map=bdm,
instance_profile_arn=
'arn:aws:iam::720533437540:instance-profile/ec2_ml')
spot_req_id = spot_reqs[0].id
print >> sys.stderr, 'Waiting for launch',
instance_id = None
spot_tag_added = False
while not instance_id:
spot_req = ec2.get_all_spot_instance_requests(
request_ids=[spot_req_id])[0]
if not spot_tag_added:
spot_req.add_tag('Name', id)
spot_tag_added = True
if spot_req.state == 'failed':
# print(dir(spot_req))
raise Exception('spto request failed')
print('Spot request failed - {0}'.format(spot_req.status))
sys.exit(0)
instance_id = spot_req.instance_id
if not instance_id:
print >> sys.stderr, '.',
time.sleep(spot_wait_sleep)
print >> sys.stderr
print >> sys.stderr, 'Retrieving instance by id'
reservations = ec2.get_all_instances(instance_ids=[instance_id])
instance = reservations[0].instances[0]
instance.add_tag('Name', id)
print >> sys.stderr, 'Got instance: ' + str(
instance.id) + ' [' + instance.state + ']'
print >> sys.stderr, 'Waiting for instance to boot',
while not instance.state in ['running', 'terminated', 'shutting-down']:
print >> sys.stderr, '.',
time.sleep(instance_wait_sleep)
instance.update()
print >> sys.stderr
if instance.state != 'running':
raise Exception('Instance was terminated')
return instance