def tag_spot_resources(conf, reservation, tags): """ Tag spot resources http://boto.cloudhackers.com/en/latest/ref/ec2.html#module-boto.ec2.spotinstancerequest """ ec2 = aws.get_ec2_conn(conf) tag_ids = [] # Tag reservations for res in reservation: logging.info('Tagging %s with %s', res, tags) ec2.create_tags(res.id, tags) # Wait until request is fulfilled while True: logging.debug('Waiting for spot request to be fulfilled...') time.sleep(3) spot_request = ec2.get_all_spot_instance_requests(res.id)[0] if spot_request.state != 'open': break # Tag instance tag_ids.extend(get_tag_instance_ids(conf, spot_request.instance_id)) logging.info('Tagging %s with %s', tag_ids, tags) ec2.create_tags(tag_ids, tags)
def init(opts, conf): ec2 = aws.get_ec2_conn(conf) # create ssh key pair if not opts.no_ssh_keys: # get new ssh public key pair from AWS ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") ssh_local_fn = utils.get_local_ssh_id_fn(opts, conf, mkdir=True) logging.info("Creating AWS ssh key pair %r.", ssh_key_name) keypair = ec2.create_key_pair(ssh_key_name, opts.dry_run) with open(ssh_local_fn, 'w') as f: pass os.chmod(ssh_local_fn, 0600) with open(ssh_local_fn, 'w') as f: f.write(keypair.material) logging.info("Created local ssh id in %r from %r key pair.", ssh_local_fn, ssh_key_name) # create security group if not opts.no_security_group: sec_group = conf.get("SECURITY_GROUP", "brenda") logging.info("Creating Brenda security group %r", sec_group) sg = ec2.create_security_group(sec_group, 'Brenda security group', dry_run=opts.dry_run) logging.debug('Tagging %s with %s', sg, dict(opts.tags)) ec2.create_tags(sg.id, dict(opts.tags)) sg.authorize('tcp', 22, 22, '0.0.0.0/0') # ssh sg.authorize('icmp', -1, -1, '0.0.0.0/0') # all ICMP
def init(opts, conf): ec2 = aws.get_ec2_conn(conf) # create ssh key pair if not opts.no_ssh_keys: try: ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") if not opts.aws_ssh_pull and aws.local_ssh_keys_exist(opts, conf): # push local ssh public key to AWS pubkey_fn = aws.get_ssh_pubkey_fn(opts, conf) print "Pushing ssh public key %r to AWS under %r key pair." % (pubkey_fn, ssh_key_name) with open(pubkey_fn) as f: pubkey = f.read() res = ec2.import_key_pair(ssh_key_name, pubkey) print res else: # get new ssh public key pair from AWS brenda_ssh_ident_fn = aws.get_brenda_ssh_identity_fn(opts, conf, mkdir=True) print "Fetching ssh private key from AWS into %r under %r key pair." % (brenda_ssh_ident_fn, ssh_key_name) keypair = ec2.create_key_pair(key_name=ssh_key_name) with open(brenda_ssh_ident_fn, 'w') as f: pass os.chmod(brenda_ssh_ident_fn, 0600) with open(brenda_ssh_ident_fn, 'w') as f: f.write(keypair.material) except Exception, e: print "Error creating ssh key pair", e
def demand(opts, conf): run_args = { 'image_id': utils.get_opt(opts.ami, conf, 'AMI_ID', must_exist=True), 'max_count': opts.n_instances, 'instance_type': utils.get_opt(opts.instance_type, conf, 'INSTANCE_TYPE', must_exist=True), 'instance_profile_name': utils.get_opt(opts.instance_profile, conf, 'INSTANCE_PROFILE', must_exist=True), 'user_data': startup_script(opts, conf) if not opts.idle else None, 'key_name': conf.get("SSH_KEY_NAME", "brenda"), 'security_groups': [conf.get("SECURITY_GROUP", "brenda")], 'monitoring_enabled': opts.monitoring, 'dry_run': opts.dry_run, } logging.debug('Instance parameters: %s', run_args) node.get_done(opts, conf) # sanity check on DONE var # Request instances ec2 = aws.get_ec2_conn(conf) reservation = ec2.run_instances(**run_args) logging.info(reservation) if not opts.dry_run: tag_demand_resources(conf, reservation, dict(opts.tags))
def init(opts, conf): ec2 = aws.get_ec2_conn(conf) # create ssh key pair if not opts.no_ssh_keys: try: ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") if not opts.aws_ssh_pull and aws.local_ssh_keys_exist(opts, conf): # push local ssh public key to AWS pubkey_fn = aws.get_ssh_pubkey_fn(opts, conf) print "Pushing ssh public key %r to AWS under %r key pair." % ( pubkey_fn, ssh_key_name) with open(pubkey_fn) as f: pubkey = f.read() res = ec2.import_key_pair(ssh_key_name, pubkey) print res else: # get new ssh public key pair from AWS brenda_ssh_ident_fn = aws.get_brenda_ssh_identity_fn( opts, conf, mkdir=True) print "Fetching ssh private key from AWS into %r under %r key pair." % ( brenda_ssh_ident_fn, ssh_key_name) keypair = ec2.create_key_pair(key_name=ssh_key_name) with open(brenda_ssh_ident_fn, 'w') as f: pass os.chmod(brenda_ssh_ident_fn, 0600) with open(brenda_ssh_ident_fn, 'w') as f: f.write(keypair.material) except Exception, e: print "Error creating ssh key pair", e
def cancel(opts, conf): ec2 = aws.get_ec2_conn(conf) requests = aws.get_all_spot_instance_requests( opts, conf, {'state': ['open', 'active']}) requests = [r.id for r in requests] logging.info('Cancel %s', requests) ec2.cancel_spot_instance_requests(requests, opts.dry_run)
def create_instance_with_ebs(opts, conf, new): ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True) itype = utils.get_opt(opts.ebs_manage_instance_type, conf, 'EBS_MANAGE_INSTANCE_TYPE', default="t2.micro") zone = utils.get_opt(opts.ebs_manage_availability_zone, conf, 'EBS_MANAGE_AVAILABILITY_ZONE') ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") sec_groups = (conf.get("SECURITY_GROUP", "brenda"), ) blkprops = {} if new: blkprops['size'] = opts.size else: if opts.size > 1: blkprops['size'] = opts.size if not opts.snapshot: raise ValueError("--snapshot must be specified") blkprops['snapshot_id'] = aws.translate_snapshot_name( conf, opts.snapshot) bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping() bdm[utils.blkdev(0)] = boto.ec2.blockdevicemapping.EBSBlockDeviceType( delete_on_termination=False, **blkprops) istore_dev = aws.add_instance_store(opts, conf, bdm, itype) script = None if opts.mount: dev = utils.blkdev(0, mount_form=True) script = "#!/bin/bash\n" if new: script += "/sbin/mkfs -t ext4 %s\n" % (dev, ) script += "/bin/mount %s /mnt\n" % (dev, ) run_args = { 'image_id': ami_id, 'instance_type': itype, 'key_name': ssh_key_name, 'security_groups': sec_groups, 'placement': zone, 'block_device_map': bdm, } if script: run_args['user_data'] = script print "RUN ARGS" for k, v in sorted(run_args.items()): print " %s : %r" % (k, v) print "BLK DEV PROPS", blkprops print "ISTORE DEV", istore_dev if not opts.dry_run: ec2 = aws.get_ec2_conn(conf) reservation = ec2.run_instances(**run_args) print reservation
def spot(opts, conf): ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True) price = utils.get_opt(opts.price, conf, 'BID_PRICE', must_exist=True) reqtype = 'persistent' if opts.persistent else 'one-time' itype = brenda_instance_type(opts, conf) snapshots = aws.get_snapshots(conf) bdm, snap_description, istore_dev = aws.blk_dev_map( opts, conf, itype, snapshots) script = startup_script(opts, conf, istore_dev) user_data = None if not opts.idle: user_data = script ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") sec_groups = (conf.get("SECURITY_GROUP", "brenda"), ) run_args = { 'image_id': ami_id, 'price': price, 'type': reqtype, 'count': opts.n_instances, 'instance_type': itype, 'user_data': user_data, 'key_name': ssh_key_name, 'security_groups': sec_groups, 'block_device_map': bdm, } if opts.availability_zone: run_args['placement'] = opts.availability_zone print "----------------------------" print "AMI ID:", ami_id print "Max bid price", price print "Request type:", reqtype print "Instance type:", itype print "Instance count:", opts.n_instances if opts.availability_zone: print "Availability zone:", opts.availability_zone if snap_description: print "Project EBS snapshot:", snap_description if istore_dev: print "Instance store device:", istore_dev print "SSH key name:", ssh_key_name print "Security groups:", sec_groups print_script(opts, conf, script) aws.get_done(opts, conf) # sanity check on DONE var if not opts.dry_run: ec2 = aws.get_ec2_conn(conf) reservation = ec2.request_spot_instances(**run_args) print reservation
def tag_demand_resources(conf, reservation, tags): """ Tag demand resources http://boto.cloudhackers.com/en/latest/ref/ec2.html#boto.ec2.instance.Reservation """ ec2 = aws.get_ec2_conn(conf) tag_ids = [] # Tag instances for instance in reservation.instances: tag_ids.extend(get_tag_instance_ids(conf, instance.id)) logging.info('Tagging %s with %s', tag_ids, tags) ec2.create_tags(tag_ids, tags)
def create_instance_with_ebs(opts, conf, new): ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True) itype = utils.get_opt(opts.ebs_manage_instance_type, conf, 'EBS_MANAGE_INSTANCE_TYPE', default="t1.micro") zone = utils.get_opt(opts.ebs_manage_availability_zone, conf, 'EBS_MANAGE_AVAILABILITY_ZONE') ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") sec_groups = (conf.get("SECURITY_GROUP", "brenda"),) blkprops = {} if new: blkprops['size'] = opts.size else: if opts.size > 1: blkprops['size'] = opts.size if not opts.snapshot: raise ValueError("--snapshot must be specified") blkprops['snapshot_id'] = aws.translate_snapshot_name(conf, opts.snapshot) bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping() bdm[utils.blkdev(0)] = boto.ec2.blockdevicemapping.EBSBlockDeviceType(delete_on_termination=False, **blkprops) istore_dev = aws.add_instance_store(opts, conf, bdm, itype) script = None if opts.mount: dev = utils.blkdev(0, mount_form=True) script = "#!/bin/bash\n" if new: script += "/sbin/mkfs -t ext4 %s\n" % (dev,) script += "/bin/mount %s /mnt\n" % (dev,) run_args = { 'image_id' : ami_id, 'instance_type' : itype, 'key_name' : ssh_key_name, 'security_groups' : sec_groups, 'placement' : zone, 'block_device_map' : bdm, } if script: run_args['user_data'] = script print "RUN ARGS" for k, v in sorted(run_args.items()): print " %s : %r" % (k, v) print "BLK DEV PROPS", blkprops print "ISTORE DEV", istore_dev if not opts.dry_run: ec2 = aws.get_ec2_conn(conf) reservation = ec2.run_instances(**run_args) print reservation
def reset_keys(opts, conf): ec2 = aws.get_ec2_conn(conf) # remove ssh keys if not opts.no_ssh_keys: try: ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") print "Removing AWS ssh key pair %r." % (ssh_key_name,) ec2.delete_key_pair(key_name=ssh_key_name) brenda_ssh_ident_fn = aws.get_brenda_ssh_identity_fn(opts, conf) if os.path.exists(brenda_ssh_ident_fn): print "Removing AWS local ssh identity %r." % (brenda_ssh_ident_fn,) os.remove(brenda_ssh_ident_fn) except Exception, e: print "Error removing ssh key pair", e
def reset_keys(opts, conf): ec2 = aws.get_ec2_conn(conf) # remove ssh keys if not opts.no_ssh_keys: try: ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") print "Removing AWS ssh key pair %r." % (ssh_key_name,) ec2.delete_key_pair(key_name=ssh_key_name) brenda_ssh_ident_fn = aws.get_brenda_ssh_identity_fn(opts, conf) if os.path.exists(brenda_ssh_ident_fn): print "Removing AWS local ssh identity %r." % (brenda_ssh_ident_fn,) os.remove(brenda_ssh_ident_fn) except Exception, e: print "Error removing ssh key pair", e
def status(opts, conf): ec2 = aws.get_ec2_conn(conf) instances = aws.filter_instances(opts, conf) if instances: print "Active Instances" now = time.time() for i in instances: uptime = aws.get_uptime(now, i.launch_time) print ' ', i.image_id, aws.format_uptime(uptime), i.public_dns_name requests = ec2.get_all_spot_instance_requests() if requests: print "Spot Requests" for r in requests: dns_name = '' print " %s %s %s %s $%s %s %s" % (r.id, r.region, r.type, r.create_time, r.price, r.state, r.status)
def status(opts, conf): ec2 = aws.get_ec2_conn(conf) instances = aws.filter_instances(opts, conf) if instances: print "Active Instances" now = time.time() for i in instances: uptime = aws.get_uptime(now, i.launch_time) print ' ', i.image_id, aws.format_uptime(uptime), i.public_dns_name requests = ec2.get_all_spot_instance_requests() if requests: print "Spot Requests" for r in requests: dns_name = '' print " %s %s %s %s $%s %s %s" % (r.id, r.region, r.type, r.create_time, r.price, r.state, r.status)
def spot(opts, conf): ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True) price = utils.get_opt(opts.price, conf, 'BID_PRICE', must_exist=True) reqtype = 'persistent' if opts.persistent else 'one-time' itype = brenda_instance_type(opts, conf) snapshots = aws.get_snapshots(conf) bdm, snap_description, istore_dev = aws.blk_dev_map(opts, conf, itype, snapshots) script = startup_script(opts, conf, istore_dev) user_data = None if not opts.idle: user_data = script ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") sec_groups = (conf.get("SECURITY_GROUP", "brenda"),) run_args = { 'image_id' : ami_id, 'price' : price, 'type' : reqtype, 'count' : opts.n_instances, 'instance_type' : itype, 'user_data' : user_data, 'key_name' : ssh_key_name, 'security_groups' : sec_groups, 'block_device_map' : bdm, } if opts.availability_zone: run_args['placement'] = opts.availability_zone print "----------------------------" print "AMI ID:", ami_id print "Max bid price", price print "Request type:", reqtype print "Instance type:", itype print "Instance count:", opts.n_instances if opts.availability_zone: print "Availability zone:", opts.availability_zone if snap_description: print "Project EBS snapshot:", snap_description if istore_dev: print "Instance store device:", istore_dev print "SSH key name:", ssh_key_name print "Security groups:", sec_groups print_script(opts, conf, script) aws.get_done(opts, conf) # sanity check on DONE var if not opts.dry_run: ec2 = aws.get_ec2_conn(conf) reservation = ec2.request_spot_instances(**run_args) print reservation
def price(opts, conf): ec2 = aws.get_ec2_conn(conf) itype = brenda_instance_type(opts, conf) data = {} for item in ec2.get_spot_price_history(instance_type=itype, product_description="Linux/UNIX"): # show the most recent price for each availability zone if item.availability_zone in data: if item.timestamp > data[item.availability_zone].timestamp: data[item.availability_zone] = item else: data[item.availability_zone] = item print "Spot price data for instance", itype for k, v in sorted(data.items()): print "%s %s $%s" % (v.availability_zone, v.timestamp, v.price)
def price(opts, conf): ec2 = aws.get_ec2_conn(conf) itype = brenda_instance_type(opts, conf) data = {} for item in ec2.get_spot_price_history(instance_type=itype, product_description="Linux/UNIX"): # show the most recent price for each availability zone if item.availability_zone in data: if item.timestamp > data[item.availability_zone].timestamp: data[item.availability_zone] = item else: data[item.availability_zone] = item print "Spot price data for instance", itype for k, v in sorted(data.items()): print "%s %s $%s" % (v.availability_zone, v.timestamp, v.price)
def demand(opts, conf): ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True) itype = brenda_instance_type(opts, conf) snapshots = aws.get_snapshots(conf) bdm, snap_description, istore_dev = aws.blk_dev_map( opts, conf, itype, snapshots) script = startup_script(opts, conf, istore_dev) user_data = None if not opts.idle: user_data = script ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") sec_groups = (conf.get("SECURITY_GROUP", "brenda"), ) run_args = { 'image_id': ami_id, 'max_count': opts.n_instances, 'instance_type': itype, 'user_data': user_data, 'key_name': ssh_key_name, 'security_groups': sec_groups, 'block_device_map': bdm, } print "----------------------------" print "AMI ID:", ami_id print "Instance type:", itype print "Max instances:", opts.n_instances if snap_description: print "Project EBS snapshot:", snap_description if istore_dev: print "Instance store device:", istore_dev print "SSH key name:", ssh_key_name print "Security groups:", sec_groups print_script(opts, conf, script) aws.get_done(opts, conf) # sanity check on DONE var if not opts.dry_run: ec2 = aws.get_ec2_conn(conf) reservation = ec2.run_instances(**run_args) print reservation
def reset(opts, conf): ec2 = aws.get_ec2_conn(conf) # remove ssh keys if not opts.no_ssh_keys: ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") logging.info("Deleting AWS ssh key pair %r.", ssh_key_name) ec2.delete_key_pair(ssh_key_name, opts.dry_run) ssh_local_fn = utils.get_local_ssh_id_fn(opts, conf) if os.path.exists(ssh_local_fn): logging.info("Removing local ssh id %r.", ssh_local_fn) os.remove(ssh_local_fn) else: logging.info("Local ssh id %r does not exist", ssh_local_fn) # remove security group if not opts.no_security_group: sec_group = conf.get("SECURITY_GROUP", "brenda") logging.info("Removing Brenda security group %r", sec_group) ec2.delete_security_group(sec_group, dry_run=opts.dry_run)
def get_tag_instance_ids(conf, instance_id): ec2 = aws.get_ec2_conn(conf) tag_ids = [] logging.info('Getting instance %s', instance_id) tag_ids.append(instance_id) # Wait for block volumes while True: logging.debug('Waiting for block volumes...') time.sleep(3) inst_attr = ec2.get_instance_attribute(instance_id=instance_id, attribute='blockDeviceMapping') if len(inst_attr['blockDeviceMapping']) > 0: for block_device in inst_attr['blockDeviceMapping'].itervalues(): # print block_device.__dict__ logging.info('Getting block volume %s', block_device.volume_id) tag_ids.append(block_device.volume_id) break return tag_ids
def demand(opts, conf): ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True) itype = brenda_instance_type(opts, conf) snapshots = aws.get_snapshots(conf) bdm, snap_description, istore_dev = aws.blk_dev_map(opts, conf, itype, snapshots) script = startup_script(opts, conf, istore_dev) user_data = None if not opts.idle: user_data = script ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") sec_groups = (conf.get("SECURITY_GROUP", "brenda"),) run_args = { 'image_id' : ami_id, 'max_count' : opts.n_instances, 'instance_type' : itype, 'user_data' : user_data, 'key_name' : ssh_key_name, 'security_groups' : sec_groups, 'block_device_map' : bdm, } print "----------------------------" print "AMI ID:", ami_id print "Instance type:", itype print "Max instances:", opts.n_instances if snap_description: print "Project EBS snapshot:", snap_description if istore_dev: print "Instance store device:", istore_dev print "SSH key name:", ssh_key_name print "Security groups:", sec_groups print_script(opts, conf, script) aws.get_done(opts, conf) # sanity check on DONE var if not opts.dry_run: ec2 = aws.get_ec2_conn(conf) reservation = ec2.run_instances(**run_args) print reservation
def cancel(opts, conf): ec2 = aws.get_ec2_conn(conf) requests = [r.id for r in ec2.get_all_spot_instance_requests()] print "CANCEL", requests if not opts.dry_run: ec2.cancel_spot_instance_requests(requests)
def cancel(opts, conf): ec2 = aws.get_ec2_conn(conf) requests = [r.id for r in ec2.get_all_spot_instance_requests()] print "CANCEL", requests if not opts.dry_run: ec2.cancel_spot_instance_requests(requests)