def demand(opts, conf):
run_args = {
'image_id':
utils.get_opt(opts.ami, conf, 'AMI_ID', must_exist=True),
'max_count':
opts.n_instances,
'instance_type':
utils.get_opt(opts.instance_type,
conf,
'INSTANCE_TYPE',
must_exist=True),
'instance_profile_name':
utils.get_opt(opts.instance_profile,
conf,
'INSTANCE_PROFILE',
must_exist=True),
'user_data':
startup_script(opts, conf) if not opts.idle else None,
'key_name':
conf.get("SSH_KEY_NAME", "brenda"),
'security_groups': [conf.get("SECURITY_GROUP", "brenda")],
'monitoring_enabled':
opts.monitoring,
'dry_run':
opts.dry_run,
}
logging.debug('Instance parameters: %s', run_args)
node.get_done(opts, conf) # sanity check on DONE var
# Request instances
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.run_instances(**run_args)
logging.info(reservation)
if not opts.dry_run:
tag_demand_resources(conf, reservation, dict(opts.tags))
def create_instance_with_ebs(opts, conf, new):
ami_id = utils.get_opt(opts.ami,
conf,
'AMI_ID',
default=AMI_ID,
must_exist=True)
itype = utils.get_opt(opts.ebs_manage_instance_type,
conf,
'EBS_MANAGE_INSTANCE_TYPE',
default="t2.micro")
zone = utils.get_opt(opts.ebs_manage_availability_zone, conf,
'EBS_MANAGE_AVAILABILITY_ZONE')
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
sec_groups = (conf.get("SECURITY_GROUP", "brenda"), )
blkprops = {}
if new:
blkprops['size'] = opts.size
else:
if opts.size > 1:
blkprops['size'] = opts.size
if not opts.snapshot:
raise ValueError("--snapshot must be specified")
blkprops['snapshot_id'] = aws.translate_snapshot_name(
conf, opts.snapshot)
bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping()
bdm[utils.blkdev(0)] = boto.ec2.blockdevicemapping.EBSBlockDeviceType(
delete_on_termination=False, **blkprops)
istore_dev = aws.add_instance_store(opts, conf, bdm, itype)
script = None
if opts.mount:
dev = utils.blkdev(0, mount_form=True)
script = "#!/bin/bash\n"
if new:
script += "/sbin/mkfs -t ext4 %s\n" % (dev, )
script += "/bin/mount %s /mnt\n" % (dev, )
run_args = {
'image_id': ami_id,
'instance_type': itype,
'key_name': ssh_key_name,
'security_groups': sec_groups,
'placement': zone,
'block_device_map': bdm,
}
if script:
run_args['user_data'] = script
print "RUN ARGS"
for k, v in sorted(run_args.items()):
print " %s : %r" % (k, v)
print "BLK DEV PROPS", blkprops
print "ISTORE DEV", istore_dev
if not opts.dry_run:
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.run_instances(**run_args)
print reservation
def spot(opts, conf):
ami_id = utils.get_opt(opts.ami,
conf,
'AMI_ID',
default=AMI_ID,
must_exist=True)
price = utils.get_opt(opts.price, conf, 'BID_PRICE', must_exist=True)
reqtype = 'persistent' if opts.persistent else 'one-time'
itype = brenda_instance_type(opts, conf)
snapshots = aws.get_snapshots(conf)
bdm, snap_description, istore_dev = aws.blk_dev_map(
opts, conf, itype, snapshots)
script = startup_script(opts, conf, istore_dev)
user_data = None
if not opts.idle:
user_data = script
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
sec_groups = (conf.get("SECURITY_GROUP", "brenda"), )
run_args = {
'image_id': ami_id,
'price': price,
'type': reqtype,
'count': opts.n_instances,
'instance_type': itype,
'user_data': user_data,
'key_name': ssh_key_name,
'security_groups': sec_groups,
'block_device_map': bdm,
}
if opts.availability_zone:
run_args['placement'] = opts.availability_zone
print "----------------------------"
print "AMI ID:", ami_id
print "Max bid price", price
print "Request type:", reqtype
print "Instance type:", itype
print "Instance count:", opts.n_instances
if opts.availability_zone:
print "Availability zone:", opts.availability_zone
if snap_description:
print "Project EBS snapshot:", snap_description
if istore_dev:
print "Instance store device:", istore_dev
print "SSH key name:", ssh_key_name
print "Security groups:", sec_groups
print_script(opts, conf, script)
aws.get_done(opts, conf) # sanity check on DONE var
if not opts.dry_run:
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.request_spot_instances(**run_args)
print reservation
def create_instance_with_ebs(opts, conf, new):
ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True)
itype = utils.get_opt(opts.ebs_manage_instance_type, conf, 'EBS_MANAGE_INSTANCE_TYPE', default="t1.micro")
zone = utils.get_opt(opts.ebs_manage_availability_zone, conf, 'EBS_MANAGE_AVAILABILITY_ZONE')
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
sec_groups = (conf.get("SECURITY_GROUP", "brenda"),)
blkprops = {}
if new:
blkprops['size'] = opts.size
else:
if opts.size > 1:
blkprops['size'] = opts.size
if not opts.snapshot:
raise ValueError("--snapshot must be specified")
blkprops['snapshot_id'] = aws.translate_snapshot_name(conf, opts.snapshot)
bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping()
bdm[utils.blkdev(0)] = boto.ec2.blockdevicemapping.EBSBlockDeviceType(delete_on_termination=False, **blkprops)
istore_dev = aws.add_instance_store(opts, conf, bdm, itype)
script = None
if opts.mount:
dev = utils.blkdev(0, mount_form=True)
script = "#!/bin/bash\n"
if new:
script += "/sbin/mkfs -t ext4 %s\n" % (dev,)
script += "/bin/mount %s /mnt\n" % (dev,)
run_args = {
'image_id' : ami_id,
'instance_type' : itype,
'key_name' : ssh_key_name,
'security_groups' : sec_groups,
'placement' : zone,
'block_device_map' : bdm,
}
if script:
run_args['user_data'] = script
print "RUN ARGS"
for k, v in sorted(run_args.items()):
print " %s : %r" % (k, v)
print "BLK DEV PROPS", blkprops
print "ISTORE DEV", istore_dev
if not opts.dry_run:
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.run_instances(**run_args)
print reservation
def spot(opts, conf):
ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True)
price = utils.get_opt(opts.price, conf, 'BID_PRICE', must_exist=True)
reqtype = 'persistent' if opts.persistent else 'one-time'
itype = brenda_instance_type(opts, conf)
snapshots = aws.get_snapshots(conf)
bdm, snap_description, istore_dev = aws.blk_dev_map(opts, conf, itype, snapshots)
script = startup_script(opts, conf, istore_dev)
user_data = None
if not opts.idle:
user_data = script
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
sec_groups = (conf.get("SECURITY_GROUP", "brenda"),)
run_args = {
'image_id' : ami_id,
'price' : price,
'type' : reqtype,
'count' : opts.n_instances,
'instance_type' : itype,
'user_data' : user_data,
'key_name' : ssh_key_name,
'security_groups' : sec_groups,
'block_device_map' : bdm,
}
if opts.availability_zone:
run_args['placement'] = opts.availability_zone
print "----------------------------"
print "AMI ID:", ami_id
print "Max bid price", price
print "Request type:", reqtype
print "Instance type:", itype
print "Instance count:", opts.n_instances
if opts.availability_zone:
print "Availability zone:", opts.availability_zone
if snap_description:
print "Project EBS snapshot:", snap_description
if istore_dev:
print "Instance store device:", istore_dev
print "SSH key name:", ssh_key_name
print "Security groups:", sec_groups
print_script(opts, conf, script)
aws.get_done(opts, conf) # sanity check on DONE var
if not opts.dry_run:
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.request_spot_instances(**run_args)
print reservation
def ssh_args(opts, conf):
user = utils.get_opt(opts.user, conf, "AWS_USER", default="root")
args = ["ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "LogLevel=quiet"]
if user:
args.extend(["-o", "User="******"-i", aws.get_adaptive_ssh_identity_fn(opts, conf)])
return args
def ssh_args(opts, conf):
user = utils.get_opt(opts.user, conf, 'AWS_USER', default='root')
args = ['ssh', '-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no',
'-o', 'LogLevel=quiet']
if user:
args.extend(['-o', 'User='******'-i', aws.get_adaptive_ssh_identity_fn(opts, conf)])
return args
def worker():
while True:
try:
item = q.get(block=False)
except Queue.Empty, e:
break
else:
node, cmd = item
#output = utils.system_return_output(cmd, capture_stderr=capture_stderr)
#new ssh on windows code-------------------------------------
import paramiko
#read ssh args from end of cmd object immediately following the node hostname entry
sshArgs = " ".join(cmd[cmd.index(node)+1:])
#print "sshargs: " +ssArgs
#get username
#print "trying new user"
user = utils.get_opt(opts.user, conf, 'AWS_USER', default='ubuntu')
#print "user: "******"cd /mnt/brenda", sshArgs]
for command in commands:
stdin , stdout, stderr = c.exec_command(command)
output = stdout.read()
c.close()
data = (node, output)
with lock:
if show_output:
print "------- %s\n%s" % data,
ret.append(data)
q.task_done()
except Exception as e:
print e
q.task_done()
def ssh_args(opts, conf):
user = utils.get_opt(opts.user, conf, 'AMI_USER', default='root')
args = [
'ssh', '-o', 'UserKnownHostsFile=/dev/null', '-o',
'StrictHostKeyChecking=no', '-o', 'LogLevel=quiet'
]
if user:
args.extend(['-o', 'User='******'brenda-run init'?")
args.extend(['-i', ssh_local_fn])
return args
def price(opts, conf):
ec2 = aws.get_ec2_conn(conf)
itype = utils.get_opt(opts.instance_type,
conf,
'INSTANCE_TYPE',
must_exist=True)
data = {}
for item in ec2.get_spot_price_history(instance_type=itype,
product_description="Linux/UNIX"):
# show the most recent price for each availability zone
if item.availability_zone in data:
if item.timestamp > data[item.availability_zone].timestamp:
data[item.availability_zone] = item
else:
data[item.availability_zone] = item
print "Spot price data for instance", itype
for k, v in sorted(data.items()):
print "%s %s $%s" % (v.availability_zone, v.timestamp, v.price)
def demand(opts, conf):
ami_id = utils.get_opt(opts.ami,
conf,
'AMI_ID',
default=AMI_ID,
must_exist=True)
itype = brenda_instance_type(opts, conf)
snapshots = aws.get_snapshots(conf)
bdm, snap_description, istore_dev = aws.blk_dev_map(
opts, conf, itype, snapshots)
script = startup_script(opts, conf, istore_dev)
user_data = None
if not opts.idle:
user_data = script
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
sec_groups = (conf.get("SECURITY_GROUP", "brenda"), )
run_args = {
'image_id': ami_id,
'max_count': opts.n_instances,
'instance_type': itype,
'user_data': user_data,
'key_name': ssh_key_name,
'security_groups': sec_groups,
'block_device_map': bdm,
}
print "----------------------------"
print "AMI ID:", ami_id
print "Instance type:", itype
print "Max instances:", opts.n_instances
if snap_description:
print "Project EBS snapshot:", snap_description
if istore_dev:
print "Instance store device:", istore_dev
print "SSH key name:", ssh_key_name
print "Security groups:", sec_groups
print_script(opts, conf, script)
aws.get_done(opts, conf) # sanity check on DONE var
if not opts.dry_run:
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.run_instances(**run_args)
print reservation
def demand(opts, conf):
ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True)
itype = brenda_instance_type(opts, conf)
snapshots = aws.get_snapshots(conf)
bdm, snap_description, istore_dev = aws.blk_dev_map(opts, conf, itype, snapshots)
script = startup_script(opts, conf, istore_dev)
user_data = None
if not opts.idle:
user_data = script
ssh_key_name = conf.get("SSH_KEY_NAME", "brenda")
sec_groups = (conf.get("SECURITY_GROUP", "brenda"),)
run_args = {
'image_id' : ami_id,
'max_count' : opts.n_instances,
'instance_type' : itype,
'user_data' : user_data,
'key_name' : ssh_key_name,
'security_groups' : sec_groups,
'block_device_map' : bdm,
}
print "----------------------------"
print "AMI ID:", ami_id
print "Instance type:", itype
print "Max instances:", opts.n_instances
if snap_description:
print "Project EBS snapshot:", snap_description
if istore_dev:
print "Instance store device:", istore_dev
print "SSH key name:", ssh_key_name
print "Security groups:", sec_groups
print_script(opts, conf, script)
aws.get_done(opts, conf) # sanity check on DONE var
if not opts.dry_run:
ec2 = aws.get_ec2_conn(conf)
reservation = ec2.run_instances(**run_args)
print reservation
def brenda_instance_type(opts, conf):
return utils.get_opt(opts.instance_type,
conf,
'INSTANCE_TYPE',
default="m2.xlarge")
def brenda_instance_type(opts, conf):
return utils.get_opt(opts.instance_type, conf, 'INSTANCE_TYPE', default="m2.xlarge")