def index(): ec2 = get_ec2_connection() hourly_rate = get_spot_prices(ec2) instances = get_instance_counts(ec2) instance_table = [] total = 0 for instance_type, count in instances.items(): cpus = cpus_per_instance[instance_type] * count total += cpus instance_table.append((instance_type, count, cpus)) instance_table.append(("Total", "", total)) active_terminals = terminal_manager.get_active_terminals() active_terminals.sort(lambda a, b: cmp(a.start_time, b.start_time)) cluster_state = get_cluster_state(ec2) manager_state = cluster_manager.get_manager_state() request_counts = collections.defaultdict(lambda: 0) for request in ec2.get_all_spot_instance_requests(filters={"state":"open"}): name = (request.launch_specification.instance_type, request.price, request.status.message) request_counts[name] += 1 open_spot_requests = [] for k in request_counts.keys(): instance_type, price, status = k count = request_counts[k] cpus = cpus_per_instance[instance_type] * count open_spot_requests.append( (instance_type, status, price, count, cpus) ) return flask.render_template("index.html", terminals=active_terminals, instances=instance_table, cluster_state=cluster_state, manager_state=manager_state, hourly_rate=hourly_rate, open_spot_requests=open_spot_requests)
def get_spot_prices(ec2): hourly_rate = 0.0 requests = ec2.get_all_spot_instance_requests(filters={"state":"active"}) for spot_request in requests: if spot_request.state == "active": print "spot code ", spot_request.id, spot_request.status.code hourly_rate += spot_request.price return hourly_rate
def get_launch_time(conf, spot_request_id): ec2 = get_ec2_conn(conf) try: spot_requests = ec2.get_all_spot_instance_requests(request_ids=[spot_request_id]) if len(spot_requests) > 0: return spot_requests[0].create_time except boto.exception.EC2ResponseError: pass return None
def get_status(): ec2 = conn_region reservations = ec2.get_all_spot_instance_requests( filters={'instance-state-name': 'running'}) instances = [i for r in reservations for i in r.instances] for instance in instances: if instance.__dict__['tags'].__dict__['_current_key'] == tag_name: instanceip = instance.__dict__['ip_address'] print "%s \tIP:%s" % (instance, instanceip)
def check_spot_requests(): ec2 = boto.ec2.connect_to_region(REGION) spot_requests = ec2.get_all_spot_instance_requests( filters={'status-code':'price-too-low', 'state':'open'} ) #spot_requests = ec2.get_all_spot_instance_requests( # filters={ # 'status-code':'instance-terminated-by-price', # 'state':'closed'}) print spot_requests # if are spot instances with state on price too low then go forward if len(spot_requests) > 0: get_best_option() else: print "No open spot requests with status code price-too-low"
def check_spot_requests(): ec2 = boto.ec2.connect_to_region(REGION) spot_requests = ec2.get_all_spot_instance_requests(filters={ 'status-code': 'price-too-low', 'state': 'open' }) #spot_requests = ec2.get_all_spot_instance_requests( # filters={ # 'status-code':'instance-terminated-by-price', # 'state':'closed'}) print spot_requests # if are spot instances with state on price too low then go forward if len(spot_requests) > 0: get_best_option() else: print "No open spot requests with status code price-too-low"
def whoAmI(): ec2 = boto.ec2.connect_to_region( "us-west-1", aws_access_key_id="AKIAINZ2TABEYCFH7SMQ", aws_secret_access_key="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") myID = requests.get("http://169.254.169.254/latest/meta-data/instance-id", timeout=1.0).text instance = ec2.get_only_instances([myID])[0] spotRequestId = instance.spot_instance_request_id if spotRequestId is not None: spotRequest = ec2.get_all_spot_instance_requests([spotRequestId])[0] tags = spotRequest.tags for key, value in tags.items(): if key not in instance.tags: instance.add_tag(key, value) return ec2.get_only_instances([myID])[0].tags
elif placement_group : module.fail_json( msg="placement_group parameter requires Boto version 2.3.0 or higher.") params.update(dict( count = count_remaining, type = spot_type, )) res = ec2.request_spot_instances(spot_price, **params) # Now we have to do the intermediate waiting if wait: spot_req_inst_ids = dict() spot_wait_timeout = time.time() + spot_wait_timeout while spot_wait_timeout > time.time(): reqs = ec2.get_all_spot_instance_requests() for sirb in res: if sirb.id in spot_req_inst_ids: continue for sir in reqs: if sir.id == sirb.id and sir.instance_id is not None: spot_req_inst_ids[sirb.id] = sir.instance_id if len(spot_req_inst_ids) < count: time.sleep(5) else: break if spot_wait_timeout <= time.time(): module.fail_json(msg = "wait for spot requests timeout on %s" % time.asctime()) instids = spot_req_inst_ids.values() except boto.exception.BotoServerError, e: module.fail_json(msg = "Instance creation failed => %s: %s" % (e.error_code, e.error_message))
price=spotBid, image_id=amiId, count=numInstancesToLaunch, key_name=keyName, instance_type=instanceType, security_groups=[securityGroupName], instance_profile_arn=iam_profile_resource_name, instance_profile_name=iam_profile_name, dry_run=dryRun, ) requestIDs = [request.id for request in spotRequests] fulfilled = [] while requestIDs: requests = ec2.get_all_spot_instance_requests(request_ids=requestIDs, ) for request in requests: if request.instance_id: requestIDs.remove(request.id) fulfilled.append(request.instance_id) # unrecoverable error without increasing the spot bid elif request.status.code == u'price-too-low': print request.status.message print 'Cancelling Spot requests...' # get original request ids since some requests may have already been fulfilled ec2.cancel_spot_instance_requests( request_ids=[request.id for request in spotRequests], ) print 'Exiting...'
price="0.005", count=1, image_id=ami, key_name=key_name, security_groups=[group_name], instance_type=instance_type, user_data=user_data, )[0] while True: eprint("Waiting. spot request status: '%s', state: '%s'" % (spot_request.state, spot_request.status.code)) if spot_request.state == 'active' and spot_request.status.code == 'fulfilled': break time.sleep(10) spot_request = ec2.get_all_spot_instance_requests( request_ids=[spot_request.id])[0] while True: instance = ec2.get_all_instances( instance_ids=[spot_request.instance_id])[0].instances[0] eprint("Waiting. spot instance state: '%s'" % instance.state) if instance.state == 'running': break time.sleep(10) ec2.create_tags([instance.id], {tag: ""}) global host instance = ec2.get_all_instances( instance_ids=[spot_request.instance_id])[0].instances[0] host = instance.ip_address print("%s" % host)
def get_spot_request_dict(conf): ec2 = get_ec2_conn(conf) requests = ec2.get_all_spot_instance_requests() return dict([(sir.id, sir) for sir in requests])
params['placement_group'] = placement_group elif placement_group : module.fail_json( msg="placement_group parameter requires Boto version 2.3.0 or higher.") params.update(dict( count = count_remaining, )) res = ec2.request_spot_instances(spot_price, **params) # Now we have to do the intermediate waiting if wait: spot_req_inst_ids = dict() spot_wait_timeout = time.time() + spot_wait_timeout while spot_wait_timeout > time.time(): reqs = ec2.get_all_spot_instance_requests() for sirb in res: if sirb.id in spot_req_inst_ids: continue for sir in reqs: if sir.id == sirb.id and sir.instance_id is not None: spot_req_inst_ids[sirb.id] = sir.instance_id if len(spot_req_inst_ids) < count: time.sleep(5) else: break if spot_wait_timeout <= time.time(): module.fail_json(msg = "wait for spot requests timeout on %s" % time.asctime()) instids = spot_req_inst_ids.values() except boto.exception.BotoServerError, e: module.fail_json(msg = "Instance creation failed => %s: %s" % (e.error_code, e.error_message))
def get_all_spot_instance_requests(opts, conf, filters={}): ec2 = get_ec2_conn(conf) filters.update(get_filter_tags(dict(opts.tags))) logging.debug('Spot request filters: %s', filters) return ec2.get_all_spot_instance_requests(filters=filters)
image_id=amiID, count=numInstancesToLaunch, key_name=keyName, instance_type=instanceType, security_groups=[securityGroupName], instance_profile_arn=iam_profile_resource_name, instance_profile_name=iam_profile_name, dry_run=dryRun, ) requestIDs = [request.id for request in spotRequests] fulfilled = [] while requestIDs: requests = ec2.get_all_spot_instance_requests( request_ids=requestIDs, ) for request in requests: if request.instance_id: requestIDs.remove(request.id) fulfilled.append(request.instance_id) # unrecoverable error without increasing the spot bid elif request.status.code == u'price-too-low': print request.status.message print 'Cancelling Spot requests...' # get original request ids since some requests may have already been fulfilled ec2.cancel_spot_instance_requests( request_ids=[request.id for request in spotRequests], )
def _launch_instances_abort_on_error(ami, bid, count, instance_type, subnet, assign_public_ip, source_dest_check, placement_group, disks, security_group_ids, tags=None): ephemeral_idx = 0 bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping() if disks is not None: for disk in disks: bdm[disk] = boto.ec2.blockdevicemapping.BlockDeviceType( ephemeral_name='ephemeral%d' % ephemeral_idx) ephemeral_idx += 1 nics = boto.ec2.networkinterface.NetworkInterfaceCollection() nics.append(boto.ec2.networkinterface.NetworkInterfaceSpecification( subnet_id=subnet, groups=security_group_ids, associate_public_ip_address=assign_public_ip)) ec2 = boto.ec2.connect_to_region(env.ec2_region) sirs = ec2.request_spot_instances( price=bid, image_id=ami, count=count, type='one-time', key_name=env.ec2_key_pair_name, instance_type=instance_type, monitoring_enabled=True, placement_group=placement_group, block_device_map=bdm, network_interfaces=nics) instance_ids = set() while True: time.sleep(10) done = True for sir in ec2.get_all_spot_instance_requests(map(lambda x: x.id, sirs)): print 'State: %s' % sir.state print 'Fault: %s' % sir.fault print 'Status: %s' % sir.status.message if sir.state not in ('open', 'active'): abort('Failed to launch instances') if sir.state == 'open': done = False if sir.state == 'active': instance_ids.add(sir.instance_id) if done: break for instance_id in instance_ids: ec2.modify_instance_attribute( instance_id=instance_id, attribute='sourceDestCheck', value=source_dest_check) print '' print 'Instances:' for reservation in ec2.get_all_instances(list(instance_ids)): for instance in reservation.instances: if tags is not None: instance.add_tags(tags) print ' %s' % instance.id print ' type: %s' % instance.instance_type print ' internal ip: %s' % instance.private_ip_address print ' public ip: %s' % instance.ip_address print ' tags:' for tag, value in sorted(instance.tags.iteritems(), lambda a, b: cmp(a[0], b[0])): print ' %s: %s' % (tag, value)
def await_spot_requests(module, ec2, spot_requests, count): """ Wait for a group of spot requests to be fulfilled, or fail. module: Ansible module object ec2: authenticated ec2 connection object spot_requests: boto.ec2.spotinstancerequest.SpotInstanceRequest object returned by ec2.request_spot_instances count: Total number of instances to be created by the spot requests Returns: list of instance ID's created by the spot request(s) """ spot_wait_timeout = int(module.params.get('spot_wait_timeout')) wait_complete = time.time() + spot_wait_timeout spot_req_inst_ids = dict() while time.time() < wait_complete: reqs = ec2.get_all_spot_instance_requests() for sirb in spot_requests: if sirb.id in spot_req_inst_ids: continue for sir in reqs: if sir.id != sirb.id: continue # this is not our spot instance if sir.instance_id is not None: spot_req_inst_ids[sirb.id] = sir.instance_id elif sir.state == 'open': continue # still waiting, nothing to do here elif sir.state == 'active': continue # Instance is created already, nothing to do here elif sir.state == 'failed': module.fail_json( msg= "Spot instance request %s failed with status %s and fault %s:%s" % (sir.id, sir.status.code, sir.fault.code, sir.fault.message)) elif sir.state == 'cancelled': module.fail_json( msg= "Spot instance request %s was cancelled before it could be fulfilled." % sir.id) elif sir.state == 'closed': # instance is terminating or marked for termination # this may be intentional on the part of the operator, # or it may have been terminated by AWS due to capacity, # price, or group constraints in this case, we'll fail # the module if the reason for the state is anything # other than termination by user. Codes are documented at # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-bid-status.html if sir.status.code == 'instance-terminated-by-user': # do nothing, since the user likely did this on purpose pass else: spot_msg = "Spot instance request %s was closed by AWS with the status %s and fault %s:%s" module.fail_json(msg=spot_msg % (sir.id, sir.status.code, sir.fault.code, sir.fault.message)) if len(spot_req_inst_ids) < count: time.sleep(5) else: return spot_req_inst_ids.values() module.fail_json(msg="wait for spot requests timeout on %s" % time.asctime())
def show_instances(): ec2 = get_ec2_connection() spots = ec2.get_all_spot_instance_requests(filters={"state":"open"}) instances = ec2.get_only_instances() instances = filter_stopped_instances(instances) return flask.render_template("show-instances.html", instances=instances, spots=spots)
def launch_spot_instance(id, profile, spot_wait_sleep=5, instance_wait_sleep=3): ec2 = boto.ec2.connect_to_region(profile['region']) if not 'key_pair' in profile: print('key pair {0} does not exist'.format(profile['key_pair'][0])) profile['key_pair'] = ('KP-' + id, 'KP-' + id + '.pem') try: print >> sys.stderr, 'Creating key pair...', keypair = ec2.create_key_pair('KP-' + id) keypair.save('.') print >> sys.stderr, 'created' except boto.exception.EC2ResponseError as e: if e.code == 'InvalidKeyPair.Duplicate': print >> sys.stderr, 'already exists' else: raise e if not 'security_group' in profile: try: print >> sys.stderr, 'Creating security group...', sc = ec2.create_security_group('SG-' + id, 'Security Group for ' + id) for proto, fromport, toport, ip in profile['firewall']: sc.authorize(proto, fromport, toport, ip) profile['security_group'] = (sc.id, sc.name) print >> sys.stderr, 'created' except boto.exception.EC2ResponseError as e: if e.code == 'InvalidGroup.Duplicate': print >> sys.stderr, 'already exists' sc = ec2.get_all_security_groups(groupnames=['SG-' + id])[0] profile['security_group'] = (sc.id, sc.name) else: raise e existing_requests = ec2.get_all_spot_instance_requests( filters={ 'launch.group-id': profile['security_group'][0], 'state': ['open', 'active'] }) if existing_requests: if len(existing_requests) > 1: raise Exception('Too many existing spot requests') print >> sys.stderr, 'Reusing existing spot request' spot_req_id = existing_requests[0].id else: bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping() bdm['/dev/sda1'] = boto.ec2.blockdevicemapping.BlockDeviceType( volume_type='gp2', size=profile['disk_size'], delete_on_termination=profile['disk_delete_on_termination']) bdm['/dev/sdb'] = boto.ec2.blockdevicemapping.BlockDeviceType( ephemeral_name='ephemeral0') print >> sys.stderr, 'Requesting spot instance' spot_reqs = ec2.request_spot_instances( price=profile['price'], image_id=profile['image_id'], instance_type=profile['type'], placement=profile['region'] + profile['availability_zone'], security_groups=[profile['security_group'][1]], key_name=profile['key_pair'][0], block_device_map=bdm, instance_profile_arn= 'arn:aws:iam::720533437540:instance-profile/ec2_ml') spot_req_id = spot_reqs[0].id print >> sys.stderr, 'Waiting for launch', instance_id = None spot_tag_added = False while not instance_id: spot_req = ec2.get_all_spot_instance_requests( request_ids=[spot_req_id])[0] if not spot_tag_added: spot_req.add_tag('Name', id) spot_tag_added = True if spot_req.state == 'failed': # print(dir(spot_req)) raise Exception('spto request failed') print('Spot request failed - {0}'.format(spot_req.status)) sys.exit(0) instance_id = spot_req.instance_id if not instance_id: print >> sys.stderr, '.', time.sleep(spot_wait_sleep) print >> sys.stderr print >> sys.stderr, 'Retrieving instance by id' reservations = ec2.get_all_instances(instance_ids=[instance_id]) instance = reservations[0].instances[0] instance.add_tag('Name', id) print >> sys.stderr, 'Got instance: ' + str( instance.id) + ' [' + instance.state + ']' print >> sys.stderr, 'Waiting for instance to boot', while not instance.state in ['running', 'terminated', 'shutting-down']: print >> sys.stderr, '.', time.sleep(instance_wait_sleep) instance.update() print >> sys.stderr if instance.state != 'running': raise Exception('Instance was terminated') return instance