Ejemplo n.º 1
0
def index():
    ec2 = get_ec2_connection()
    hourly_rate = get_spot_prices(ec2)
    instances = get_instance_counts(ec2)
    instance_table = []
    total = 0
    for instance_type, count in instances.items():
        cpus = cpus_per_instance[instance_type] * count
        total += cpus
        instance_table.append((instance_type, count, cpus))
    instance_table.append(("Total", "", total))

    active_terminals = terminal_manager.get_active_terminals()
    active_terminals.sort(lambda a, b: cmp(a.start_time, b.start_time))

    cluster_state = get_cluster_state(ec2)
    manager_state = cluster_manager.get_manager_state()

    request_counts = collections.defaultdict(lambda: 0)
    for request in ec2.get_all_spot_instance_requests(filters={"state":"open"}):
        name = (request.launch_specification.instance_type, request.price, request.status.message)
        request_counts[name] += 1

    open_spot_requests = []
    for k in request_counts.keys():
        instance_type, price, status = k
        count = request_counts[k]
        cpus = cpus_per_instance[instance_type] * count
        open_spot_requests.append( (instance_type, status, price, count, cpus) )

    return flask.render_template("index.html", terminals=active_terminals, instances=instance_table, cluster_state=cluster_state,
                                 manager_state=manager_state,
                                 hourly_rate=hourly_rate, open_spot_requests=open_spot_requests)
Ejemplo n.º 2
0
def get_spot_prices(ec2):
    hourly_rate = 0.0
    requests = ec2.get_all_spot_instance_requests(filters={"state":"active"})
    for spot_request in requests:
        if spot_request.state == "active":
            print "spot code ", spot_request.id, spot_request.status.code
            hourly_rate += spot_request.price
    return hourly_rate
Ejemplo n.º 3
0
Archivo: aws.py Proyecto: bioid/brenda
def get_launch_time(conf, spot_request_id):
    ec2 = get_ec2_conn(conf)
    try:
        spot_requests = ec2.get_all_spot_instance_requests(request_ids=[spot_request_id])
        if len(spot_requests) > 0:
            return spot_requests[0].create_time
    except boto.exception.EC2ResponseError:
        pass
    return None
Ejemplo n.º 4
0
def get_status():

    ec2 = conn_region
    reservations = ec2.get_all_spot_instance_requests(
        filters={'instance-state-name': 'running'})
    instances = [i for r in reservations for i in r.instances]

    for instance in instances:
        if instance.__dict__['tags'].__dict__['_current_key'] == tag_name:
            instanceip = instance.__dict__['ip_address']
            print "%s \tIP:%s" % (instance, instanceip)
Ejemplo n.º 5
0
def check_spot_requests():
    ec2 = boto.ec2.connect_to_region(REGION)
    spot_requests = ec2.get_all_spot_instance_requests(
                        filters={'status-code':'price-too-low', 'state':'open'}
                        )
    #spot_requests = ec2.get_all_spot_instance_requests(
    #                filters={
    #                    'status-code':'instance-terminated-by-price',
    #                    'state':'closed'})
    print spot_requests
    # if are spot instances with state on price too low then go forward
    if len(spot_requests) > 0:
        get_best_option()
    else:
        print "No open spot requests with status code price-too-low"
Ejemplo n.º 6
0
def check_spot_requests():
    ec2 = boto.ec2.connect_to_region(REGION)
    spot_requests = ec2.get_all_spot_instance_requests(filters={
        'status-code': 'price-too-low',
        'state': 'open'
    })
    #spot_requests = ec2.get_all_spot_instance_requests(
    #                filters={
    #                    'status-code':'instance-terminated-by-price',
    #                    'state':'closed'})
    print spot_requests
    # if are spot instances with state on price too low then go forward
    if len(spot_requests) > 0:
        get_best_option()
    else:
        print "No open spot requests with status code price-too-low"
Ejemplo n.º 7
0
def whoAmI():
    ec2 = boto.ec2.connect_to_region(
        "us-west-1",
        aws_access_key_id="AKIAINZ2TABEYCFH7SMQ",
        aws_secret_access_key="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")
    myID = requests.get("http://169.254.169.254/latest/meta-data/instance-id",
                        timeout=1.0).text
    instance = ec2.get_only_instances([myID])[0]
    spotRequestId = instance.spot_instance_request_id
    if spotRequestId is not None:
        spotRequest = ec2.get_all_spot_instance_requests([spotRequestId])[0]
        tags = spotRequest.tags
        for key, value in tags.items():
            if key not in instance.tags:
                instance.add_tag(key, value)
    return ec2.get_only_instances([myID])[0].tags
Ejemplo n.º 8
0
                elif placement_group :
                        module.fail_json(
                            msg="placement_group parameter requires Boto version 2.3.0 or higher.")

                params.update(dict(
                    count = count_remaining,
                    type = spot_type,
                ))
                res = ec2.request_spot_instances(spot_price, **params)

                # Now we have to do the intermediate waiting
                if wait:
                    spot_req_inst_ids = dict()
                    spot_wait_timeout = time.time() + spot_wait_timeout
                    while spot_wait_timeout > time.time():
                        reqs = ec2.get_all_spot_instance_requests()
                        for sirb in res:
                            if sirb.id in spot_req_inst_ids:
                                continue
                            for sir in reqs:
                                if sir.id == sirb.id and sir.instance_id is not None:
                                    spot_req_inst_ids[sirb.id] = sir.instance_id
                        if len(spot_req_inst_ids) < count:
                            time.sleep(5)
                        else:
                            break
                    if spot_wait_timeout <= time.time():
                        module.fail_json(msg = "wait for spot requests timeout on %s" % time.asctime())
                    instids = spot_req_inst_ids.values()
        except boto.exception.BotoServerError, e:
            module.fail_json(msg = "Instance creation failed => %s: %s" % (e.error_code, e.error_message))
        price=spotBid,
        image_id=amiId,
        count=numInstancesToLaunch,
        key_name=keyName,
        instance_type=instanceType,
        security_groups=[securityGroupName],
        instance_profile_arn=iam_profile_resource_name,
        instance_profile_name=iam_profile_name,
        dry_run=dryRun,
    )

    requestIDs = [request.id for request in spotRequests]
    fulfilled = []

    while requestIDs:
        requests = ec2.get_all_spot_instance_requests(request_ids=requestIDs, )

        for request in requests:
            if request.instance_id:
                requestIDs.remove(request.id)
                fulfilled.append(request.instance_id)

            # unrecoverable error without increasing the spot bid
            elif request.status.code == u'price-too-low':
                print request.status.message
                print 'Cancelling Spot requests...'
                # get original request ids since some requests may have already been fulfilled
                ec2.cancel_spot_instance_requests(
                    request_ids=[request.id for request in spotRequests], )

                print 'Exiting...'
Ejemplo n.º 10
0
        price="0.005",
        count=1,
        image_id=ami,
        key_name=key_name,
        security_groups=[group_name],
        instance_type=instance_type,
        user_data=user_data,
    )[0]

    while True:
        eprint("Waiting. spot request status: '%s', state: '%s'" %
               (spot_request.state, spot_request.status.code))
        if spot_request.state == 'active' and spot_request.status.code == 'fulfilled':
            break
        time.sleep(10)
        spot_request = ec2.get_all_spot_instance_requests(
            request_ids=[spot_request.id])[0]
    while True:
        instance = ec2.get_all_instances(
            instance_ids=[spot_request.instance_id])[0].instances[0]
        eprint("Waiting. spot instance state: '%s'" % instance.state)
        if instance.state == 'running':
            break
        time.sleep(10)

    ec2.create_tags([instance.id], {tag: ""})

    global host
    instance = ec2.get_all_instances(
        instance_ids=[spot_request.instance_id])[0].instances[0]
    host = instance.ip_address
    print("%s" % host)
Ejemplo n.º 11
0
Archivo: aws.py Proyecto: SenH/brenda
def get_spot_request_dict(conf):
    ec2 = get_ec2_conn(conf)
    requests = ec2.get_all_spot_instance_requests()
    return dict([(sir.id, sir) for sir in requests])
Ejemplo n.º 12
0
def get_spot_request_dict(conf):
    ec2 = get_ec2_conn(conf)
    requests = ec2.get_all_spot_instance_requests()
    return dict([(sir.id, sir) for sir in requests])
Ejemplo n.º 13
0
                    params['placement_group'] = placement_group
                elif placement_group :
                        module.fail_json(
                            msg="placement_group parameter requires Boto version 2.3.0 or higher.")

                params.update(dict(
                    count = count_remaining,
                ))
                res = ec2.request_spot_instances(spot_price, **params)

                # Now we have to do the intermediate waiting
                if wait:
                    spot_req_inst_ids = dict()
                    spot_wait_timeout = time.time() + spot_wait_timeout
                    while spot_wait_timeout > time.time():
                        reqs = ec2.get_all_spot_instance_requests()
                        for sirb in res:
                            if sirb.id in spot_req_inst_ids:
                                continue
                            for sir in reqs:
                                if sir.id == sirb.id and sir.instance_id is not None:
                                    spot_req_inst_ids[sirb.id] = sir.instance_id
                        if len(spot_req_inst_ids) < count:
                            time.sleep(5)
                        else:
                            break
                    if spot_wait_timeout <= time.time():
                        module.fail_json(msg = "wait for spot requests timeout on %s" % time.asctime())
                    instids = spot_req_inst_ids.values()
        except boto.exception.BotoServerError, e:
            module.fail_json(msg = "Instance creation failed => %s: %s" % (e.error_code, e.error_message))
Ejemplo n.º 14
0
Archivo: aws.py Proyecto: SenH/brenda
def get_all_spot_instance_requests(opts, conf, filters={}):
    ec2 = get_ec2_conn(conf)
    filters.update(get_filter_tags(dict(opts.tags)))
    logging.debug('Spot request filters: %s', filters)
    return ec2.get_all_spot_instance_requests(filters=filters)
Ejemplo n.º 15
0
        image_id=amiID,
        count=numInstancesToLaunch,
        key_name=keyName,
        instance_type=instanceType,
        security_groups=[securityGroupName],
        instance_profile_arn=iam_profile_resource_name,
        instance_profile_name=iam_profile_name,
        dry_run=dryRun,
    )

    requestIDs = [request.id for request in spotRequests]
    fulfilled = []

    while requestIDs:
        requests = ec2.get_all_spot_instance_requests(
            request_ids=requestIDs,
        )

        for request in requests:
            if request.instance_id:
                requestIDs.remove(request.id)
                fulfilled.append(request.instance_id)

            # unrecoverable error without increasing the spot bid
            elif request.status.code == u'price-too-low':
                print request.status.message
                print 'Cancelling Spot requests...'
                # get original request ids since some requests may have already been fulfilled
                ec2.cancel_spot_instance_requests(
                    request_ids=[request.id for request in spotRequests],
                )
Ejemplo n.º 16
0
def _launch_instances_abort_on_error(ami,
                                     bid,
                                     count,
                                     instance_type,
                                     subnet,
                                     assign_public_ip,
                                     source_dest_check,
                                     placement_group,
                                     disks,
                                     security_group_ids,
                                     tags=None):
    ephemeral_idx = 0
    bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping()
    if disks is not None:
        for disk in disks:
            bdm[disk] = boto.ec2.blockdevicemapping.BlockDeviceType(
                ephemeral_name='ephemeral%d' % ephemeral_idx)
            ephemeral_idx += 1

    nics = boto.ec2.networkinterface.NetworkInterfaceCollection()
    nics.append(boto.ec2.networkinterface.NetworkInterfaceSpecification(
        subnet_id=subnet,
        groups=security_group_ids,
        associate_public_ip_address=assign_public_ip))

    ec2 = boto.ec2.connect_to_region(env.ec2_region)
    sirs = ec2.request_spot_instances(
        price=bid,
        image_id=ami,
        count=count,
        type='one-time',
        key_name=env.ec2_key_pair_name,
        instance_type=instance_type,
        monitoring_enabled=True,
        placement_group=placement_group,
        block_device_map=bdm,
        network_interfaces=nics)

    instance_ids = set()
    while True:
        time.sleep(10)
        done = True
        for sir in ec2.get_all_spot_instance_requests(map(lambda x: x.id, sirs)):
            print 'State:  %s' % sir.state
            print 'Fault:  %s' % sir.fault
            print 'Status: %s' % sir.status.message
            if sir.state not in ('open', 'active'):
                abort('Failed to launch instances')
            if sir.state == 'open':
                done = False
            if sir.state == 'active':
                instance_ids.add(sir.instance_id)

        if done:
            break

    for instance_id in instance_ids:
        ec2.modify_instance_attribute(
            instance_id=instance_id,
            attribute='sourceDestCheck',
            value=source_dest_check)

    print ''
    print 'Instances:'
    for reservation in ec2.get_all_instances(list(instance_ids)):
        for instance in reservation.instances:
            if tags is not None:
                instance.add_tags(tags)

            print '    %s' % instance.id
            print '        type:        %s' % instance.instance_type
            print '        internal ip: %s' % instance.private_ip_address
            print '        public ip:   %s' % instance.ip_address
            print '        tags:'
            for tag, value in sorted(instance.tags.iteritems(), lambda a, b: cmp(a[0], b[0])):
                print '            %s: %s' % (tag, value)
Ejemplo n.º 17
0
def await_spot_requests(module, ec2, spot_requests, count):
    """
    Wait for a group of spot requests to be fulfilled, or fail.
    module: Ansible module object
    ec2: authenticated ec2 connection object
    spot_requests: boto.ec2.spotinstancerequest.SpotInstanceRequest object returned by ec2.request_spot_instances
    count: Total number of instances to be created by the spot requests
    Returns:
        list of instance ID's created by the spot request(s)
    """
    spot_wait_timeout = int(module.params.get('spot_wait_timeout'))
    wait_complete = time.time() + spot_wait_timeout

    spot_req_inst_ids = dict()
    while time.time() < wait_complete:
        reqs = ec2.get_all_spot_instance_requests()
        for sirb in spot_requests:
            if sirb.id in spot_req_inst_ids:
                continue
            for sir in reqs:
                if sir.id != sirb.id:
                    continue  # this is not our spot instance
                if sir.instance_id is not None:
                    spot_req_inst_ids[sirb.id] = sir.instance_id
                elif sir.state == 'open':
                    continue  # still waiting, nothing to do here
                elif sir.state == 'active':
                    continue  # Instance is created already, nothing to do here
                elif sir.state == 'failed':
                    module.fail_json(
                        msg=
                        "Spot instance request %s failed with status %s and fault %s:%s"
                        % (sir.id, sir.status.code, sir.fault.code,
                           sir.fault.message))
                elif sir.state == 'cancelled':
                    module.fail_json(
                        msg=
                        "Spot instance request %s was cancelled before it could be fulfilled."
                        % sir.id)
                elif sir.state == 'closed':
                    # instance is terminating or marked for termination
                    # this may be intentional on the part of the operator,
                    # or it may have been terminated by AWS due to capacity,
                    # price, or group constraints in this case, we'll fail
                    # the module if the reason for the state is anything
                    # other than termination by user. Codes are documented at
                    # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-bid-status.html
                    if sir.status.code == 'instance-terminated-by-user':
                        # do nothing, since the user likely did this on purpose
                        pass
                    else:
                        spot_msg = "Spot instance request %s was closed by AWS with the status %s and fault %s:%s"
                        module.fail_json(msg=spot_msg %
                                         (sir.id, sir.status.code,
                                          sir.fault.code, sir.fault.message))

        if len(spot_req_inst_ids) < count:
            time.sleep(5)
        else:
            return spot_req_inst_ids.values()
    module.fail_json(msg="wait for spot requests timeout on %s" %
                     time.asctime())
Ejemplo n.º 18
0
def show_instances():
    ec2 = get_ec2_connection()
    spots = ec2.get_all_spot_instance_requests(filters={"state":"open"})
    instances = ec2.get_only_instances()
    instances = filter_stopped_instances(instances)
    return flask.render_template("show-instances.html", instances=instances, spots=spots)
Ejemplo n.º 19
0
def launch_spot_instance(id,
                         profile,
                         spot_wait_sleep=5,
                         instance_wait_sleep=3):
    ec2 = boto.ec2.connect_to_region(profile['region'])

    if not 'key_pair' in profile:
        print('key pair {0} does not exist'.format(profile['key_pair'][0]))
        profile['key_pair'] = ('KP-' + id, 'KP-' + id + '.pem')
        try:
            print >> sys.stderr, 'Creating key pair...',
            keypair = ec2.create_key_pair('KP-' + id)
            keypair.save('.')
            print >> sys.stderr, 'created'
        except boto.exception.EC2ResponseError as e:
            if e.code == 'InvalidKeyPair.Duplicate':
                print >> sys.stderr, 'already exists'
            else:
                raise e

    if not 'security_group' in profile:
        try:
            print >> sys.stderr, 'Creating security group...',
            sc = ec2.create_security_group('SG-' + id,
                                           'Security Group for ' + id)
            for proto, fromport, toport, ip in profile['firewall']:
                sc.authorize(proto, fromport, toport, ip)
            profile['security_group'] = (sc.id, sc.name)
            print >> sys.stderr, 'created'
        except boto.exception.EC2ResponseError as e:
            if e.code == 'InvalidGroup.Duplicate':
                print >> sys.stderr, 'already exists'
                sc = ec2.get_all_security_groups(groupnames=['SG-' + id])[0]
                profile['security_group'] = (sc.id, sc.name)
            else:
                raise e

    existing_requests = ec2.get_all_spot_instance_requests(
        filters={
            'launch.group-id': profile['security_group'][0],
            'state': ['open', 'active']
        })
    if existing_requests:
        if len(existing_requests) > 1:
            raise Exception('Too many existing spot requests')
        print >> sys.stderr, 'Reusing existing spot request'
        spot_req_id = existing_requests[0].id
    else:
        bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping()
        bdm['/dev/sda1'] = boto.ec2.blockdevicemapping.BlockDeviceType(
            volume_type='gp2',
            size=profile['disk_size'],
            delete_on_termination=profile['disk_delete_on_termination'])
        bdm['/dev/sdb'] = boto.ec2.blockdevicemapping.BlockDeviceType(
            ephemeral_name='ephemeral0')
        print >> sys.stderr, 'Requesting spot instance'
        spot_reqs = ec2.request_spot_instances(
            price=profile['price'],
            image_id=profile['image_id'],
            instance_type=profile['type'],
            placement=profile['region'] + profile['availability_zone'],
            security_groups=[profile['security_group'][1]],
            key_name=profile['key_pair'][0],
            block_device_map=bdm,
            instance_profile_arn=
            'arn:aws:iam::720533437540:instance-profile/ec2_ml')
        spot_req_id = spot_reqs[0].id

    print >> sys.stderr, 'Waiting for launch',
    instance_id = None
    spot_tag_added = False
    while not instance_id:
        spot_req = ec2.get_all_spot_instance_requests(
            request_ids=[spot_req_id])[0]
        if not spot_tag_added:
            spot_req.add_tag('Name', id)
            spot_tag_added = True
        if spot_req.state == 'failed':
            # print(dir(spot_req))
            raise Exception('spto request failed')
            print('Spot request failed - {0}'.format(spot_req.status))
            sys.exit(0)
        instance_id = spot_req.instance_id
        if not instance_id:
            print >> sys.stderr, '.',
            time.sleep(spot_wait_sleep)
    print >> sys.stderr

    print >> sys.stderr, 'Retrieving instance by id'
    reservations = ec2.get_all_instances(instance_ids=[instance_id])
    instance = reservations[0].instances[0]
    instance.add_tag('Name', id)
    print >> sys.stderr, 'Got instance: ' + str(
        instance.id) + ' [' + instance.state + ']'
    print >> sys.stderr, 'Waiting for instance to boot',
    while not instance.state in ['running', 'terminated', 'shutting-down']:
        print >> sys.stderr, '.',
        time.sleep(instance_wait_sleep)
        instance.update()
    print >> sys.stderr
    if instance.state != 'running':
        raise Exception('Instance was terminated')
    return instance