def tag_spot_resources(conf, reservation, tags):
"""
Tag spot resources
http://boto.cloudhackers.com/en/latest/ref/ec2.html#module-boto.ec2.spotinstancerequest
"""
ec2 = aws.get_ec2_conn(conf)
tag_ids = []
# Tag reservations
for res in reservation:
logging.info('Tagging %s with %s', res, tags)
ec2.create_tags(res.id, tags)
# Wait until request is fulfilled
while True:
logging.debug('Waiting for spot request to be fulfilled...')
time.sleep(3)
spot_request = ec2.get_all_spot_instance_requests(res.id)[0]
if spot_request.state != 'open':
break
# Tag instance
tag_ids.extend(get_tag_instance_ids(conf, spot_request.instance_id))
logging.info('Tagging %s with %s', tag_ids, tags)
ec2.create_tags(tag_ids, tags)
def init(opts, conf):
ec2 = aws.get_ec2_conn(conf)
# create ssh key pair
if not opts.no_ssh_keys:
# get new ssh public key pair from AWS
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
ssh_local_fn = utils.get_local_ssh_id_fn(opts, conf, mkdir=True)
logging.info("Creating AWS ssh key pair %r.", ssh_key_name)
keypair = ec2.create_key_pair(ssh_key_name, opts.dry_run)
with open(ssh_local_fn, 'w') as f:
pass
os.chmod(ssh_local_fn, 0600)
with open(ssh_local_fn, 'w') as f:
f.write(keypair.material)
logging.info("Created local ssh id in %r from %r key pair.",
ssh_local_fn, ssh_key_name)
# create security group
if not opts.no_security_group:
sec_group = conf.get("SECURITY_GROUP", "brenda")
logging.info("Creating Brenda security group %r", sec_group)
sg = ec2.create_security_group(sec_group,
'Brenda security group',
dry_run=opts.dry_run)
logging.debug('Tagging %s with %s', sg, dict(opts.tags))
ec2.create_tags(sg.id, dict(opts.tags))
sg.authorize('tcp', 22, 22, '0.0.0.0/0') # ssh
sg.authorize('icmp', -1, -1, '0.0.0.0/0') # all ICMP
def init(opts, conf):
ec2 = aws.get_ec2_conn(conf)
# create ssh key pair
if not opts.no_ssh_keys:
try:
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
if not opts.aws_ssh_pull and aws.local_ssh_keys_exist(opts, conf):
# push local ssh public key to AWS
pubkey_fn = aws.get_ssh_pubkey_fn(opts, conf)
print "Pushing ssh public key %r to AWS under %r key pair." % (pubkey_fn, ssh_key_name)
with open(pubkey_fn) as f:
pubkey = f.read()
res = ec2.import_key_pair(ssh_key_name, pubkey)
print res
else:
# get new ssh public key pair from AWS
brenda_ssh_ident_fn = aws.get_brenda_ssh_identity_fn(opts, conf, mkdir=True)
print "Fetching ssh private key from AWS into %r under %r key pair." % (brenda_ssh_ident_fn, ssh_key_name)
keypair = ec2.create_key_pair(key_name=ssh_key_name)
with open(brenda_ssh_ident_fn, 'w') as f:
pass
os.chmod(brenda_ssh_ident_fn, 0600)
with open(brenda_ssh_ident_fn, 'w') as f:
f.write(keypair.material)
except Exception, e:
print "Error creating ssh key pair", e
def demand(opts, conf):
run_args = {
'image_id':
utils.get_opt(opts.ami, conf, 'AMI_ID', must_exist=True),
'max_count':
opts.n_instances,
'instance_type':
utils.get_opt(opts.instance_type,
conf,
'INSTANCE_TYPE',
must_exist=True),
'instance_profile_name':
utils.get_opt(opts.instance_profile,
conf,
'INSTANCE_PROFILE',
must_exist=True),
'user_data':
startup_script(opts, conf) if not opts.idle else None,
'key_name':
conf.get("SSH_KEY_NAME", "brenda"),
'security_groups': [conf.get("SECURITY_GROUP", "brenda")],
'monitoring_enabled':
opts.monitoring,
'dry_run':
opts.dry_run,
}
logging.debug('Instance parameters: %s', run_args)
node.get_done(opts, conf) # sanity check on DONE var
# Request instances
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.run_instances(**run_args)
logging.info(reservation)
if not opts.dry_run:
tag_demand_resources(conf, reservation, dict(opts.tags))
def init(opts, conf):
ec2 = aws.get_ec2_conn(conf)
# create ssh key pair
if not opts.no_ssh_keys:
try:
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
if not opts.aws_ssh_pull and aws.local_ssh_keys_exist(opts, conf):
# push local ssh public key to AWS
pubkey_fn = aws.get_ssh_pubkey_fn(opts, conf)
print "Pushing ssh public key %r to AWS under %r key pair." % (
pubkey_fn, ssh_key_name)
with open(pubkey_fn) as f:
pubkey = f.read()
res = ec2.import_key_pair(ssh_key_name, pubkey)
print res
else:
# get new ssh public key pair from AWS
brenda_ssh_ident_fn = aws.get_brenda_ssh_identity_fn(
opts, conf, mkdir=True)
print "Fetching ssh private key from AWS into %r under %r key pair." % (
brenda_ssh_ident_fn, ssh_key_name)
keypair = ec2.create_key_pair(key_name=ssh_key_name)
with open(brenda_ssh_ident_fn, 'w') as f:
pass
os.chmod(brenda_ssh_ident_fn, 0600)
with open(brenda_ssh_ident_fn, 'w') as f:
f.write(keypair.material)
except Exception, e:
print "Error creating ssh key pair", e
def cancel(opts, conf):
ec2 = aws.get_ec2_conn(conf)
requests = aws.get_all_spot_instance_requests(
opts, conf, {'state': ['open', 'active']})
requests = [r.id for r in requests]
logging.info('Cancel %s', requests)
ec2.cancel_spot_instance_requests(requests, opts.dry_run)
def create_instance_with_ebs(opts, conf, new):
ami_id = utils.get_opt(opts.ami,
conf,
'AMI_ID',
default=AMI_ID,
must_exist=True)
itype = utils.get_opt(opts.ebs_manage_instance_type,
conf,
'EBS_MANAGE_INSTANCE_TYPE',
default="t2.micro")
zone = utils.get_opt(opts.ebs_manage_availability_zone, conf,
'EBS_MANAGE_AVAILABILITY_ZONE')
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
sec_groups = (conf.get("SECURITY_GROUP", "brenda"), )
blkprops = {}
if new:
blkprops['size'] = opts.size
else:
if opts.size > 1:
blkprops['size'] = opts.size
if not opts.snapshot:
raise ValueError("--snapshot must be specified")
blkprops['snapshot_id'] = aws.translate_snapshot_name(
conf, opts.snapshot)
bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping()
bdm[utils.blkdev(0)] = boto.ec2.blockdevicemapping.EBSBlockDeviceType(
delete_on_termination=False, **blkprops)
istore_dev = aws.add_instance_store(opts, conf, bdm, itype)
script = None
if opts.mount:
dev = utils.blkdev(0, mount_form=True)
script = "#!/bin/bash\n"
if new:
script += "/sbin/mkfs -t ext4 %s\n" % (dev, )
script += "/bin/mount %s /mnt\n" % (dev, )
run_args = {
'image_id': ami_id,
'instance_type': itype,
'key_name': ssh_key_name,
'security_groups': sec_groups,
'placement': zone,
'block_device_map': bdm,
}
if script:
run_args['user_data'] = script
print "RUN ARGS"
for k, v in sorted(run_args.items()):
print " %s : %r" % (k, v)
print "BLK DEV PROPS", blkprops
print "ISTORE DEV", istore_dev
if not opts.dry_run:
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.run_instances(**run_args)
print reservation
def spot(opts, conf):
ami_id = utils.get_opt(opts.ami,
conf,
'AMI_ID',
default=AMI_ID,
must_exist=True)
price = utils.get_opt(opts.price, conf, 'BID_PRICE', must_exist=True)
reqtype = 'persistent' if opts.persistent else 'one-time'
itype = brenda_instance_type(opts, conf)
snapshots = aws.get_snapshots(conf)
bdm, snap_description, istore_dev = aws.blk_dev_map(
opts, conf, itype, snapshots)
script = startup_script(opts, conf, istore_dev)
user_data = None
if not opts.idle:
user_data = script
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
sec_groups = (conf.get("SECURITY_GROUP", "brenda"), )
run_args = {
'image_id': ami_id,
'price': price,
'type': reqtype,
'count': opts.n_instances,
'instance_type': itype,
'user_data': user_data,
'key_name': ssh_key_name,
'security_groups': sec_groups,
'block_device_map': bdm,
}
if opts.availability_zone:
run_args['placement'] = opts.availability_zone
print "----------------------------"
print "AMI ID:", ami_id
print "Max bid price", price
print "Request type:", reqtype
print "Instance type:", itype
print "Instance count:", opts.n_instances
if opts.availability_zone:
print "Availability zone:", opts.availability_zone
if snap_description:
print "Project EBS snapshot:", snap_description
if istore_dev:
print "Instance store device:", istore_dev
print "SSH key name:", ssh_key_name
print "Security groups:", sec_groups
print_script(opts, conf, script)
aws.get_done(opts, conf) # sanity check on DONE var
if not opts.dry_run:
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.request_spot_instances(**run_args)
print reservation
def tag_demand_resources(conf, reservation, tags):
"""
Tag demand resources
http://boto.cloudhackers.com/en/latest/ref/ec2.html#boto.ec2.instance.Reservation
"""
ec2 = aws.get_ec2_conn(conf)
tag_ids = []
# Tag instances
for instance in reservation.instances:
tag_ids.extend(get_tag_instance_ids(conf, instance.id))
logging.info('Tagging %s with %s', tag_ids, tags)
ec2.create_tags(tag_ids, tags)
def create_instance_with_ebs(opts, conf, new):
ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True)
itype = utils.get_opt(opts.ebs_manage_instance_type, conf, 'EBS_MANAGE_INSTANCE_TYPE', default="t1.micro")
zone = utils.get_opt(opts.ebs_manage_availability_zone, conf, 'EBS_MANAGE_AVAILABILITY_ZONE')
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
sec_groups = (conf.get("SECURITY_GROUP", "brenda"),)
blkprops = {}
if new:
blkprops['size'] = opts.size
else:
if opts.size > 1:
blkprops['size'] = opts.size
if not opts.snapshot:
raise ValueError("--snapshot must be specified")
blkprops['snapshot_id'] = aws.translate_snapshot_name(conf, opts.snapshot)
bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping()
bdm[utils.blkdev(0)] = boto.ec2.blockdevicemapping.EBSBlockDeviceType(delete_on_termination=False, **blkprops)
istore_dev = aws.add_instance_store(opts, conf, bdm, itype)
script = None
if opts.mount:
dev = utils.blkdev(0, mount_form=True)
script = "#!/bin/bash\n"
if new:
script += "/sbin/mkfs -t ext4 %s\n" % (dev,)
script += "/bin/mount %s /mnt\n" % (dev,)
run_args = {
'image_id' : ami_id,
'instance_type' : itype,
'key_name' : ssh_key_name,
'security_groups' : sec_groups,
'placement' : zone,
'block_device_map' : bdm,
}
if script:
run_args['user_data'] = script
print "RUN ARGS"
for k, v in sorted(run_args.items()):
print " %s : %r" % (k, v)
print "BLK DEV PROPS", blkprops
print "ISTORE DEV", istore_dev
if not opts.dry_run:
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.run_instances(**run_args)
print reservation
def reset_keys(opts, conf):
ec2 = aws.get_ec2_conn(conf)
# remove ssh keys
if not opts.no_ssh_keys:
try:
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
print "Removing AWS ssh key pair %r." % (ssh_key_name,)
ec2.delete_key_pair(key_name=ssh_key_name)
brenda_ssh_ident_fn = aws.get_brenda_ssh_identity_fn(opts, conf)
if os.path.exists(brenda_ssh_ident_fn):
print "Removing AWS local ssh identity %r." % (brenda_ssh_ident_fn,)
os.remove(brenda_ssh_ident_fn)
except Exception, e:
print "Error removing ssh key pair", e
def reset_keys(opts, conf):
ec2 = aws.get_ec2_conn(conf)
# remove ssh keys
if not opts.no_ssh_keys:
try:
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
print "Removing AWS ssh key pair %r." % (ssh_key_name,)
ec2.delete_key_pair(key_name=ssh_key_name)
brenda_ssh_ident_fn = aws.get_brenda_ssh_identity_fn(opts, conf)
if os.path.exists(brenda_ssh_ident_fn):
print "Removing AWS local ssh identity %r." % (brenda_ssh_ident_fn,)
os.remove(brenda_ssh_ident_fn)
except Exception, e:
print "Error removing ssh key pair", e
def status(opts, conf):
ec2 = aws.get_ec2_conn(conf)
instances = aws.filter_instances(opts, conf)
if instances:
print "Active Instances"
now = time.time()
for i in instances:
uptime = aws.get_uptime(now, i.launch_time)
print ' ', i.image_id, aws.format_uptime(uptime), i.public_dns_name
requests = ec2.get_all_spot_instance_requests()
if requests:
print "Spot Requests"
for r in requests:
dns_name = ''
print " %s %s %s %s $%s %s %s" % (r.id, r.region, r.type, r.create_time, r.price, r.state, r.status)
def status(opts, conf):
ec2 = aws.get_ec2_conn(conf)
instances = aws.filter_instances(opts, conf)
if instances:
print "Active Instances"
now = time.time()
for i in instances:
uptime = aws.get_uptime(now, i.launch_time)
print ' ', i.image_id, aws.format_uptime(uptime), i.public_dns_name
requests = ec2.get_all_spot_instance_requests()
if requests:
print "Spot Requests"
for r in requests:
dns_name = ''
print " %s %s %s %s $%s %s %s" % (r.id, r.region, r.type, r.create_time, r.price, r.state, r.status)
def spot(opts, conf):
ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True)
price = utils.get_opt(opts.price, conf, 'BID_PRICE', must_exist=True)
reqtype = 'persistent' if opts.persistent else 'one-time'
itype = brenda_instance_type(opts, conf)
snapshots = aws.get_snapshots(conf)
bdm, snap_description, istore_dev = aws.blk_dev_map(opts, conf, itype, snapshots)
script = startup_script(opts, conf, istore_dev)
user_data = None
if not opts.idle:
user_data = script
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
sec_groups = (conf.get("SECURITY_GROUP", "brenda"),)
run_args = {
'image_id' : ami_id,
'price' : price,
'type' : reqtype,
'count' : opts.n_instances,
'instance_type' : itype,
'user_data' : user_data,
'key_name' : ssh_key_name,
'security_groups' : sec_groups,
'block_device_map' : bdm,
}
if opts.availability_zone:
run_args['placement'] = opts.availability_zone
print "----------------------------"
print "AMI ID:", ami_id
print "Max bid price", price
print "Request type:", reqtype
print "Instance type:", itype
print "Instance count:", opts.n_instances
if opts.availability_zone:
print "Availability zone:", opts.availability_zone
if snap_description:
print "Project EBS snapshot:", snap_description
if istore_dev:
print "Instance store device:", istore_dev
print "SSH key name:", ssh_key_name
print "Security groups:", sec_groups
print_script(opts, conf, script)
aws.get_done(opts, conf) # sanity check on DONE var
if not opts.dry_run:
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.request_spot_instances(**run_args)
print reservation
def price(opts, conf):
ec2 = aws.get_ec2_conn(conf)
itype = brenda_instance_type(opts, conf)
data = {}
for item in ec2.get_spot_price_history(instance_type=itype,
product_description="Linux/UNIX"):
# show the most recent price for each availability zone
if item.availability_zone in data:
if item.timestamp > data[item.availability_zone].timestamp:
data[item.availability_zone] = item
else:
data[item.availability_zone] = item
print "Spot price data for instance", itype
for k, v in sorted(data.items()):
print "%s %s $%s" % (v.availability_zone, v.timestamp, v.price)
def price(opts, conf):
ec2 = aws.get_ec2_conn(conf)
itype = brenda_instance_type(opts, conf)
data = {}
for item in ec2.get_spot_price_history(instance_type=itype,
product_description="Linux/UNIX"):
# show the most recent price for each availability zone
if item.availability_zone in data:
if item.timestamp > data[item.availability_zone].timestamp:
data[item.availability_zone] = item
else:
data[item.availability_zone] = item
print "Spot price data for instance", itype
for k, v in sorted(data.items()):
print "%s %s $%s" % (v.availability_zone, v.timestamp, v.price)
def demand(opts, conf):
ami_id = utils.get_opt(opts.ami,
conf,
'AMI_ID',
default=AMI_ID,
must_exist=True)
itype = brenda_instance_type(opts, conf)
snapshots = aws.get_snapshots(conf)
bdm, snap_description, istore_dev = aws.blk_dev_map(
opts, conf, itype, snapshots)
script = startup_script(opts, conf, istore_dev)
user_data = None
if not opts.idle:
user_data = script
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
sec_groups = (conf.get("SECURITY_GROUP", "brenda"), )
run_args = {
'image_id': ami_id,
'max_count': opts.n_instances,
'instance_type': itype,
'user_data': user_data,
'key_name': ssh_key_name,
'security_groups': sec_groups,
'block_device_map': bdm,
}
print "----------------------------"
print "AMI ID:", ami_id
print "Instance type:", itype
print "Max instances:", opts.n_instances
if snap_description:
print "Project EBS snapshot:", snap_description
if istore_dev:
print "Instance store device:", istore_dev
print "SSH key name:", ssh_key_name
print "Security groups:", sec_groups
print_script(opts, conf, script)
aws.get_done(opts, conf) # sanity check on DONE var
if not opts.dry_run:
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.run_instances(**run_args)
print reservation
def reset(opts, conf):
ec2 = aws.get_ec2_conn(conf)
# remove ssh keys
if not opts.no_ssh_keys:
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
logging.info("Deleting AWS ssh key pair %r.", ssh_key_name)
ec2.delete_key_pair(ssh_key_name, opts.dry_run)
ssh_local_fn = utils.get_local_ssh_id_fn(opts, conf)
if os.path.exists(ssh_local_fn):
logging.info("Removing local ssh id %r.", ssh_local_fn)
os.remove(ssh_local_fn)
else:
logging.info("Local ssh id %r does not exist", ssh_local_fn)
# remove security group
if not opts.no_security_group:
sec_group = conf.get("SECURITY_GROUP", "brenda")
logging.info("Removing Brenda security group %r", sec_group)
ec2.delete_security_group(sec_group, dry_run=opts.dry_run)
def get_tag_instance_ids(conf, instance_id):
ec2 = aws.get_ec2_conn(conf)
tag_ids = []
logging.info('Getting instance %s', instance_id)
tag_ids.append(instance_id)
# Wait for block volumes
while True:
logging.debug('Waiting for block volumes...')
time.sleep(3)
inst_attr = ec2.get_instance_attribute(instance_id=instance_id,
attribute='blockDeviceMapping')
if len(inst_attr['blockDeviceMapping']) > 0:
for block_device in inst_attr['blockDeviceMapping'].itervalues():
# print block_device.__dict__
logging.info('Getting block volume %s', block_device.volume_id)
tag_ids.append(block_device.volume_id)
break
return tag_ids
def demand(opts, conf):
ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True)
itype = brenda_instance_type(opts, conf)
snapshots = aws.get_snapshots(conf)
bdm, snap_description, istore_dev = aws.blk_dev_map(opts, conf, itype, snapshots)
script = startup_script(opts, conf, istore_dev)
user_data = None
if not opts.idle:
user_data = script
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
sec_groups = (conf.get("SECURITY_GROUP", "brenda"),)
run_args = {
'image_id' : ami_id,
'max_count' : opts.n_instances,
'instance_type' : itype,
'user_data' : user_data,
'key_name' : ssh_key_name,
'security_groups' : sec_groups,
'block_device_map' : bdm,
}
print "----------------------------"
print "AMI ID:", ami_id
print "Instance type:", itype
print "Max instances:", opts.n_instances
if snap_description:
print "Project EBS snapshot:", snap_description
if istore_dev:
print "Instance store device:", istore_dev
print "SSH key name:", ssh_key_name
print "Security groups:", sec_groups
print_script(opts, conf, script)
aws.get_done(opts, conf) # sanity check on DONE var
if not opts.dry_run:
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.run_instances(**run_args)
print reservation
def cancel(opts, conf):
ec2 = aws.get_ec2_conn(conf)
requests = [r.id for r in ec2.get_all_spot_instance_requests()]
print "CANCEL", requests
if not opts.dry_run:
ec2.cancel_spot_instance_requests(requests)
def cancel(opts, conf):
ec2 = aws.get_ec2_conn(conf)
requests = [r.id for r in ec2.get_all_spot_instance_requests()]
print "CANCEL", requests
if not opts.dry_run:
ec2.cancel_spot_instance_requests(requests)
