def build_block_device_map(ephemeral,
number_ephemeral_disks=1,
ebs_size=None,
iops=None,
number_ebs_volumes=1):
bdm = blockdevicemapping.BlockDeviceMapping()
if ephemeral:
# The ephemeral disk
xvdb = BlockDeviceType()
xvdb.ephemeral_name = 'ephemeral0'
bdm['/dev/xvdb'] = xvdb
if number_ephemeral_disks == 2:
xvdc = BlockDeviceType()
xvdc.ephemeral_name = 'ephemeral1'
bdm['/dev/xvdc'] = xvdc
if ebs_size:
for disks in range(0, number_ebs_volumes):
xvd_n = blockdevicemapping.EBSBlockDeviceType(
delete_on_termination=True)
xvd_n.size = int(ebs_size) # size in Gigabytes
if iops:
xvd_n.iops = 500
xvd_n.volume_type = 'io1'
else:
xvd_n.volume_type = 'gp2'
last_char = chr(ord('f') + disks)
bdm['/dev/xvd' + last_char] = xvd_n
return bdm
def build_block_device_map(ephemeral, number_ephemeral_disks=1, ebs_size=None, iops=None, number_ebs_volumes=1):
bdm = blockdevicemapping.BlockDeviceMapping()
if ephemeral:
# The ephemeral disk
xvdb = BlockDeviceType()
xvdb.ephemeral_name = 'ephemeral0'
bdm['/dev/xvdb'] = xvdb
if number_ephemeral_disks == 2:
xvdc = BlockDeviceType()
xvdc.ephemeral_name = 'ephemeral1'
bdm['/dev/xvdc'] = xvdc
if ebs_size:
for disks in range(0, number_ebs_volumes):
xvd_n = blockdevicemapping.EBSBlockDeviceType(delete_on_termination=True)
xvd_n.size = int(ebs_size) # size in Gigabytes
if iops:
xvd_n.iops = 500
xvd_n.volume_type = 'io1'
else:
xvd_n.volume_type = 'gp2'
last_char = chr(ord('f') + disks)
bdm['/dev/xvd' + last_char] = xvd_n
return bdm
def spotrequest(ec2, nodetype, count, testname, testdate, threads, userdata = None):
global selected_availability_zone
maxprice = "0.40"
if nodetype == 'client':
instancetype = 'c3.xlarge'
else:
# For data nodes
instancetype = 'c3.4xlarge'
# Allow an explicit selection if needed...
#selected_availability_zone = "us-east-1e"
if not selected_availability_zone:
selected_availability_zone = random.choice([
'us-east-1a',
#'us-east-1b',
'us-east-1d',
'us-east-1e',
])
availability_zone = selected_availability_zone
if userdata == None:
userdata = """#!/bin/bash
echo {0} > /etc/node_testname
echo {1} > /etc/node_testdate
echo {2} > /etc/node_threads
echo {3} > /etc/node_role
#echo 10.136.71.116 > /etc/node_headnode
echo 400 > /etc/node_swap # MB of swap created
echo 1 > /etc/node_mongo_uselocal # Use local mongos shard server on each client
""".format(testname, testdate, threads, nodetype)
# For some tests we may not need any nodes of this type
if count == 0:
return []
# Default AMI
ami = 'ami-XXXXXXXX' # Current versions
# Specify ephemeral block devices...
bdmap = BlockDeviceMapping()
sdb = BlockDeviceType()
sdb.ephemeral_name = 'ephemeral0'
bdmap['/dev/sdb'] = sdb
sdc = BlockDeviceType()
sdc.ephemeral_name = 'ephemeral1'
bdmap['/dev/sdc'] = sdc
#sdd = BlockDeviceType()
#sdd.ephemeral_name = 'ephemeral2'
#bdmap['/dev/sdd'] = sdd
#sde = BlockDeviceType()
#sde.ephemeral_name = 'ephemeral3'
#bdmap['/dev/sde'] = sde
return ec2.request_spot_instances(maxprice, ami, count=count, launch_group=testdate, availability_zone_group=testdate, security_groups=['epstatic'], user_data=userdata, instance_type=instancetype, block_device_map=bdmap)
def get_block_device_map(bdmapping_json=None):
"""Parse block_device_mapping JSON and return a configured BlockDeviceMapping object
Mapping JSON structure...
{"/dev/sda":
{"snapshot_id": "snap-23E93E09", "volume_type": null, "delete_on_termination": true, "size": 1} }
"""
if bdmapping_json:
mapping = json.loads(bdmapping_json)
if mapping:
bdm = BlockDeviceMapping()
for key, val in mapping.items():
device = BlockDeviceType()
if val.get('virtual_name') is not None and val.get(
'virtual_name').startswith('ephemeral'):
device.ephemeral_name = val.get('virtual_name')
else:
device.volume_type = 'standard'
device.snapshot_id = val.get('snapshot_id') or None
device.size = val.get('size')
device.delete_on_termination = val.get(
'delete_on_termination', False)
bdm[key] = device
return bdm
return None
return None
def get_block_device_mapping(instance_type):
'''
Given instance type, return the block device mapping for m3 instances.
We need to do this explicitly because AWS tends to ignore the
because AMI-specified block device mapping.
Later on, we could extend this functionality to allow customer to specify
attaching external EBS volumes.
'''
block_map = BlockDeviceMapping()
# AWS ignores the AMI-specified block device mapping for M3
# See:
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
### For M3 instances, you must specify instance store volumes in the block
### device mapping for the instance when you launch it. When you launch
### an M3 instance, instance store volumes specified in the block device
### mapping for the AMI may be ignored if they are not specified as part of
### the instance block device mapping.
if instance_type.startswith('m3.'):
for i in range(get_num_disks(instance_type)):
dev = BlockDeviceType()
dev.ephemeral_name = 'ephemeral%d' % i
# The first ephemeral drive is /dev/sdb.
name = '/dev/sd' + string.ascii_letters[i + 1]
block_map[name] = dev
return block_map
def create_block_device_mapping(ami, device_map):
bdm = BlockDeviceMapping()
for device, device_info in device_map.items():
if ami.root_device_type == "instance-store" and \
not device_info.get("ephemeral_name"):
# EBS is not supported by S3-backed AMIs at request time
# EBS volumes can be attached when an instance is running
continue
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
if ami.root_device_name == device:
ami_size = ami.block_device_mapping[device].size
if ami.virtualization_type == "hvm":
# Overwrite root device size for HVM instances, since they
# cannot be resized online
bd.size = ami_size
elif device_info.get('size'):
# make sure that size is enough for this AMI
assert ami_size <= device_info['size'], \
"Instance root device size cannot be smaller than AMI " \
"root device"
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
if device_info.get("volume_type"):
bd.volume_type = device_info["volume_type"]
if device_info["volume_type"] == "io1" \
and device_info.get("iops"):
bd.iops = device_info["iops"]
bdm[device] = bd
return bdm
def create_block_device_mapping(ami, device_map):
bdm = BlockDeviceMapping()
for device, device_info in device_map.items():
if ami.root_device_type == "instance-store" and \
not device_info.get("ephemeral_name"):
# EBS is not supported by S3-backed AMIs at request time
# EBS volumes can be attached when an instance is running
continue
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
if ami.root_device_name == device:
ami_size = ami.block_device_mapping[device].size
if ami.virtualization_type == "hvm":
# Overwrite root device size for HVM instances, since they
# cannot be resized online
bd.size = ami_size
elif device_info.get('size'):
# make sure that size is enough for this AMI
assert ami_size <= device_info['size'], \
"Instance root device size cannot be smaller than AMI " \
"root device"
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
if device_info.get("volume_type"):
bd.volume_type = device_info["volume_type"]
if device_info["volume_type"] == "io1" \
and device_info.get("iops"):
bd.iops = device_info["iops"]
bdm[device] = bd
return bdm
def launch_ondemand_request(conn, request, tenant, job):
try:
mapping = BlockDeviceMapping()
sda1 = BlockDeviceType()
eph0 = BlockDeviceType()
eph1 = BlockDeviceType()
eph2 = BlockDeviceType()
eph3 = BlockDeviceType()
sda1.size = 10
eph0.ephemeral_name = 'ephemeral0'
eph1.ephemeral_name = 'ephemeral1'
eph2.ephemeral_name = 'ephemeral2'
eph3.ephemeral_name = 'ephemeral3'
mapping['/dev/sda1'] = sda1
mapping['/dev/sdb'] = eph0
mapping['/dev/sdc'] = eph1
mapping['/dev/sdd'] = eph2
mapping['/dev/sde'] = eph3
# issue a run_instances command for this request
res = conn.run_instances(min_count=request.count,
max_count=request.count,
key_name=tenant.key_pair,
image_id=request.ami,
security_group_ids=[tenant.security_group],
user_data=customise_cloudinit(tenant, job),
instance_type=request.instance_type,
subnet_id=tenant.subnet,
block_device_map=mapping)
my_req_ids = [req.id for req in res.instances]
# address = ""
for req in my_req_ids:
# tag each request
tag_requests(req, tenant.name, conn)
# update the database to include the new request
ProvisionerConfig().dbconn.execute(
("insert into instance_request (tenant, instance_type, " +
"price, job_runner_id, request_type, request_id, " +
"subnet) values ('%s', '%s', %s, %s, '%s', '%s', %s)") %
(tenant.db_id, request.instance.db_id,
request.instance.ondemand, job.id, "ondemand", req,
tenant.subnet_id))
# ProvisionerConfig().dbconn.commit()
return
except boto.exception.EC2ResponseError:
logger.exception("There was an error communicating with EC2.")
def _parse_block_device_mappings(user_input):
"""
Parse block device mappings per AWS CLI tools syntax (modified to add IOPS)
http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
Syntax:
/dev/xvd[a-z]=[snapshot-id|ephemeral]:[size in GB]:[Delete on Term]:[IOPS]
- Leave inapplicable fields blank
- Delete on Termination defaults to True
- IOPS limits are not validated
- EBS sizing is not validated
Mount an Ephemeral Drive:
/dev/xvdb1=ephemeral0
Mount multiple Ephemeral Drives:
/dev/xvdb1=ephemeral0,/dev/xvdb2=ephemeral1
Mount a Snapshot:
/dev/xvdp=snap-1234abcd
Mount a Snapshot to a 100GB drive:
/dev/xvdp=snap-1234abcd:100
Mount a Snapshot to a 100GB drive and do not delete on termination:
/dev/xvdp=snap-1234abcd:100:false
Mount a Fresh 100GB EBS device
/dev/xvdp=:100
Mount a Fresh 100GB EBS Device and do not delete on termination:
/dev/xvdp=:100:false
Mount a Fresh 100GB EBS Device with 1000 IOPS
/dev/xvdp=:100::1000
"""
block_device_map = BlockDeviceMapping()
mappings = user_input.split(",")
for mapping in mappings:
block_type = BlockDeviceType()
mount_point, drive_type, size, delete, iops = _parse_drive_mapping(
mapping)
if 'ephemeral' in drive_type:
block_type.ephemeral_name = drive_type
elif 'snap' in drive_type:
block_type.snapshot_id = drive_type
block_type.volume_type = "standard"
else:
block_type.volume_type = "standard"
block_type.size = size
block_type.delete_on_termination = delete
if iops:
block_type.iops = iops
block_type.volume_type = "io1"
block_device_map[mount_point] = block_type
return block_device_map
def test_create_launch_configuration_with_block_device_mappings():
block_device_mapping = BlockDeviceMapping()
ephemeral_drive = BlockDeviceType()
ephemeral_drive.ephemeral_name = 'ephemeral0'
block_device_mapping['/dev/xvdb'] = ephemeral_drive
snapshot_drive = BlockDeviceType()
snapshot_drive.snapshot_id = "snap-1234abcd"
snapshot_drive.volume_type = "standard"
block_device_mapping['/dev/xvdp'] = snapshot_drive
ebs_drive = BlockDeviceType()
ebs_drive.volume_type = "io1"
ebs_drive.size = 100
ebs_drive.iops = 1000
ebs_drive.delete_on_termination = False
block_device_mapping['/dev/xvdh'] = ebs_drive
conn = boto.connect_autoscale(use_block_device_types=True)
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='m1.small',
key_name='the_keys',
security_groups=["default", "default2"],
user_data="This is some user_data",
instance_monitoring=True,
instance_profile_name='arn:aws:iam::123456789012:instance-profile/testing',
spot_price=0.1,
block_device_mappings=[block_device_mapping]
)
conn.create_launch_configuration(config)
launch_config = conn.get_all_launch_configurations()[0]
launch_config.name.should.equal('tester')
launch_config.image_id.should.equal('ami-abcd1234')
launch_config.instance_type.should.equal('m1.small')
launch_config.key_name.should.equal('the_keys')
set(launch_config.security_groups).should.equal(set(['default', 'default2']))
launch_config.user_data.should.equal("This is some user_data")
launch_config.instance_monitoring.enabled.should.equal('true')
launch_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/testing')
launch_config.spot_price.should.equal(0.1)
len(launch_config.block_device_mappings).should.equal(3)
returned_mapping = launch_config.block_device_mappings
set(returned_mapping.keys()).should.equal(set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh']))
returned_mapping['/dev/xvdh'].iops.should.equal(1000)
returned_mapping['/dev/xvdh'].size.should.equal(100)
returned_mapping['/dev/xvdh'].volume_type.should.equal("io1")
returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false
returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd")
returned_mapping['/dev/xvdp'].volume_type.should.equal("standard")
returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')
def _parse_block_device_mappings(user_input):
"""
Parse block device mappings per AWS CLI tools syntax (modified to add IOPS)
http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
Syntax:
/dev/xvd[a-z]=[snapshot-id|ephemeral]:[size in GB]:[Delete on Term]:[IOPS]
- Leave inapplicable fields blank
- Delete on Termination defaults to True
- IOPS limits are not validated
- EBS sizing is not validated
Mount an Ephemeral Drive:
/dev/xvdb1=ephemeral0
Mount multiple Ephemeral Drives:
/dev/xvdb1=ephemeral0,/dev/xvdb2=ephemeral1
Mount a Snapshot:
/dev/xvdp=snap-1234abcd
Mount a Snapshot to a 100GB drive:
/dev/xvdp=snap-1234abcd:100
Mount a Snapshot to a 100GB drive and do not delete on termination:
/dev/xvdp=snap-1234abcd:100:false
Mount a Fresh 100GB EBS device
/dev/xvdp=:100
Mount a Fresh 100GB EBS Device and do not delete on termination:
/dev/xvdp=:100:false
Mount a Fresh 100GB EBS Device with 1000 IOPS
/dev/xvdp=:100::1000
"""
block_device_map = BlockDeviceMapping()
mappings = user_input.split(",")
for mapping in mappings:
block_type = BlockDeviceType()
mount_point, drive_type, size, delete, iops = _parse_drive_mapping(mapping)
if 'ephemeral' in drive_type:
block_type.ephemeral_name = drive_type
elif 'snap' in drive_type:
block_type.snapshot_id = drive_type
block_type.volume_type = "standard"
else:
block_type.volume_type = "standard"
block_type.size = size
block_type.delete_on_termination = delete
if iops:
block_type.iops = iops
block_type.volume_type = "io1"
block_device_map[mount_point] = block_type
return block_device_map
def create_mapping(config):
if 'mapping' not in config:
return None
mapping = BlockDeviceMapping()
for ephemeral_name, device_path in config['mapping'].iteritems():
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = ephemeral_name
mapping[device_path] = ephemeral
return mapping
def launch(name, ami='ami-3d4ff254', instance_type='t1.micro', key_name='amazon2',
zone='us-east-1d', security_group='quicklaunch-1', user='******', job=None):
'''Launch a single instance of the provided ami '''
conn = EC2Connection()
# Declare the block device mapping for ephemeral disks
mapping = BlockDeviceMapping()
eph0 = BlockDeviceType()
eph1 = BlockDeviceType()
eph0.ephemeral_name = 'ephemeral0'
eph1.ephemeral_name = 'ephemeral1'
mapping['/dev/sdb'] = eph0
mapping['/dev/sdc'] = eph1
# Now, ask for a reservation
reservation = conn.run_instances(ami, instance_type=instance_type,
key_name=key_name, placement=zone,
block_device_map=mapping, security_groups=[security_group])
# And assume that the instance we're talking about is the first in the list
# This is not always a good assumption, and will likely depend on the specifics
# of your launching situation. For launching an isolated instance while no
# other actions are taking place, this is sufficient.
instance = reservation.instances[0]
print('Waiting for instance to start...')
# Check up on its status every so often
status = instance.update()
while status == 'pending':
time.sleep(5)
status = instance.update()
if status == 'running':
print('New instance "' + instance.id + '" accessible at ' + instance.public_dns_name)
# Name the instance
conn.create_tags([instance.id], {'Name': name})
n = Node(name, instance.id, instance.image_id, instance.key_name, instance.placement,
instance.instance_type, instance.dns_name, instance.private_dns_name,
instance.ip_address, instance.private_ip_address, user, job)
pprint.pprint(n.to_dict())
addNode(n)
else:
print('Instance status: ' + status)
return
def _create_device_map(ephemeral_disk_count):
"""Creates a block device out of the ephemeral disks on this instance."""
device_map = BlockDeviceMapping()
device_paths = _get_device_paths(ephemeral_disk_count)
for index, device_path in enumerate(device_paths):
device = BlockDeviceType()
device.ephemeral_name = "ephemeral{}".format(index)
device_map[device_path] = device
return device_map
def block_mappings(flavor):
mapping = BlockDeviceMapping()
if flavor == "hs1.8xlarge":
for i in range(0, 24):
eph = BlockDeviceType()
eph.ephemeral_name = "ephemeral%d" % i
device = "/dev/sd%c1" % chr(ord('b') + i)
mapping[device] = eph
elif flavor == "hi1.4xlarge":
for i in range(0, 2):
eph = BlockDeviceType()
eph.ephemeral_name = "ephemeral%d" % i
device = "/dev/sd%c1" % chr(ord('b') + i)
mapping[device] = eph
elif flavor == "m1.xlarge":
for i in range(0, 4):
eph = BlockDeviceType()
eph.ephemeral_name = "ephemeral%d" % i
device = "/dev/sd%c1" % chr(ord('b') + i)
mapping[device] = eph
return mapping
def __call__(self, config, sectionname):
from boto.ec2.blockdevicemapping import BlockDeviceMapping
from boto.ec2.blockdevicemapping import BlockDeviceType
value = BaseMassager.__call__(self, config, sectionname)
device_map = BlockDeviceMapping()
for mapping in value.split():
device_path, ephemeral_name = mapping.split(':')
device = BlockDeviceType()
device.ephemeral_name = ephemeral_name
device_map[device_path] = device
return device_map
def __call__(self, config, sectionname):
from boto.ec2.blockdevicemapping import BlockDeviceMapping
from boto.ec2.blockdevicemapping import BlockDeviceType
value = BaseMassager.__call__(self, config, sectionname)
device_map = BlockDeviceMapping()
for mapping in value.split():
device_path, ephemeral_name = mapping.split(':')
device = BlockDeviceType()
device.ephemeral_name = ephemeral_name
device_map[device_path] = device
return device_map
def register(snapshot_id,
region,
arch,
size=None,
name=None,
desc=None,
pvm=False):
conn = utils.connect(region)
if None in (name, size):
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
virt = 'hvm'
kernel_id = None
device_base = '/dev/xvd'
ec2_arch = "x86_64" if arch == "amd64" else arch
if pvm:
kernel_id = utils.get_kernel(region, arch)
virt = 'paravirtual'
device_base = '/dev/sd'
name += '-pvm'
log.debug('creating block_device_map')
block_device_map = BlockDeviceMapping()
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
rootfs_device_name = device_base + 'a'
block_device_map[rootfs_device_name] = rootfs
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
ephemeral_device_name = device_base + 'b'
block_device_map[ephemeral_device_name] = ephemeral
log.debug('registering image - %s', name)
ami_id = conn.register_image(name=name,
description=desc,
kernel_id=kernel_id,
architecture=ec2_arch,
root_device_name=rootfs_device_name,
block_device_map=block_device_map,
virtualization_type=virt)
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id, name
def launch_instance(self):
if not self.verify_settings():
return
block_map = BlockDeviceMapping()
root_device = self.config["ec2_root_device"]
block_map[root_device] = EBSBlockDeviceType()
if self.config["ec2_size"]:
block_map[root_device].size = self.config["ec2_size"]
block_map[root_device].delete_on_termination = True
for num, device_location in enumerate(self.config["ec2_ephemeral_devices"]):
device = BlockDeviceType()
device.ephemeral_name = "ephemeral%d" % num
block_map[device_location] = device
reservation = self.conn.run_instances(
self.config["ec2_ami_id"],
key_name=self.config["ec2_key_name"],
security_groups=self.config["ec2_security_groups"] or [self.config["ec2_security_group"]],
instance_type=self.config["ec2_instance_type"],
placement=self.config["ec2_zone"],
monitoring_enabled=self.config["ec2_monitoring_enabled"],
block_device_map=block_map,
user_data=self.user_data,
)
self.instance = reservation.instances[0]
secs = RUN_INSTANCE_TIMEOUT
rest_interval = 5
while secs and not self.instance.state == "running":
time.sleep(rest_interval)
secs = secs - rest_interval
try:
self.instance.update()
except boto.exception.EC2ResponseError:
pass
if secs <= 0:
errmsg = "run instance %s failed after %d seconds" % (self.instance.id, RUN_INSTANCE_TIMEOUT)
LOG.error(errmsg)
else:
if self.config["hostname"]:
self.assign_name_tag()
msg1 = "Started Instance: {0}\n".format(self.instance.id)
LOG.info(msg1)
print msg1
p = int(self.config["ssh_port"])
port = "-p {0} ".format(p) if p and not p == 22 else ""
## change user to 'root' for all non-Ubuntu systems
user = self.config["sudouser"] if self.config["sudouser"] and self.config["ssh_import"] else "ubuntu"
# XXX - TODO: replace public dns with fqdn, where appropriate
msg2 = "To access: ssh {0}{1}@{2}\n" "To terminate: shaker-terminate {3}".format(
port, user, self.instance.public_dns_name, self.instance.id
)
LOG.info(msg2)
print msg2
def demandrequest(ec2, nodetype, testname, testdate, threads, userdata = None):
global selected_availability_zone
if nodetype == 'client':
instancetype = 'c3.xlarge'
else:
instancetype = 'i2.xlarge'
# Allow an explicit selection if needed...
#selected_availability_zone = 'us-east-1e'
if not selected_availability_zone:
selected_availability_zone = random.choice([
'us-east-1a',
#'us-east-1b',
'us-east-1d',
'us-east-1e',
])
availability_zone = selected_availability_zone
if userdata == None:
userdata = """#!/bin/bash
echo {0} > /etc/node_testname
echo {1} > /etc/node_testdate
echo {2} > /etc/node_threads
echo {3} > /etc/node_role
echo 10.136.71.116 > /etc/node_headnode
echo 400 > /etc/node_swap # MB of swap created
echo 1 > /etc/node_mongo_uselocal # Use local mongos shard server on each client
""".format(testname, testdate, threads, nodetype)
# Default AMI
ami = 'ami-XXXXXXXX' # Current versions
# Specify ephemeral block devices...
bdmap = BlockDeviceMapping()
sdb = BlockDeviceType()
sdb.ephemeral_name = 'ephemeral0'
bdmap['/dev/sdb'] = sdb
#sdc = BlockDeviceType()
#sdc.ephemeral_name = 'ephemeral1'
#bdmap['/dev/sdc'] = sdc
#sdd = BlockDeviceType()
#sdd.ephemeral_name = 'ephemeral2'
#bdmap['/dev/sdd'] = sdd
#sde = BlockDeviceType()
#sde.ephemeral_name = 'ephemeral3'
#bdmap['/dev/sde'] = sde
return ec2.run_instances(ami, placement=availability_zone, security_groups=['epstatic'], user_data=userdata, instance_type=instancetype, block_device_map=bdmap)
def _process_block_device_mappings(self, launch_config, zone=None):
"""
Processes block device mapping information
and returns a Boto BlockDeviceMapping object. If new volumes
are requested (source is None and destination is VOLUME), they will be
created and the relevant volume ids included in the mapping.
"""
bdm = BlockDeviceMapping()
# Assign letters from f onwards
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
next_letter = iter(list(string.ascii_lowercase[6:]))
# assign ephemeral devices from 0 onwards
ephemeral_counter = 0
for device in launch_config.block_devices:
bd_type = BlockDeviceType()
if device.is_volume:
if device.is_root:
bdm['/dev/sda1'] = bd_type
else:
bdm['sd' + next(next_letter)] = bd_type
if isinstance(device.source, Snapshot):
bd_type.snapshot_id = device.source.id
elif isinstance(device.source, Volume):
bd_type.volume_id = device.source.id
elif isinstance(device.source, MachineImage):
# Not supported
pass
else:
# source is None, but destination is volume, therefore
# create a blank volume. If the Zone is None, this
# could fail since the volume and instance may be created
# in two different zones.
if not zone:
raise InvalidConfigurationException(
"A zone must be specified when launching with a"
" new blank volume block device mapping.")
new_vol = self.provider.block_store.volumes.create(
'',
device.size,
zone)
bd_type.volume_id = new_vol.id
bd_type.delete_on_terminate = device.delete_on_terminate
if device.size:
bd_type.size = device.size
else: # device is ephemeral
bd_type.ephemeral_name = 'ephemeral%s' % ephemeral_counter
return bdm
def _get_bmap(self, params):
bmap = BlockDeviceMapping()
for device in params['bmap']:
if not 'name' in device.keys():
self.logger.debug('bad device ' + str(device))
continue
dev = BlockDeviceType()
if 'size' in device.keys():
dev.size = device['size']
if 'delete_on_termination' in device.keys():
dev.delete_on_termination = device['delete_on_termination']
if 'ephemeral_name' in device.keys():
dev.ephemeral_name = device['ephemeral_name']
bmap[device['name']] = dev
return bmap
def _parse_block_device_mappings(self):
block_device_map = BlockDeviceMapping()
for mapping in self.block_device_mapping_dict:
block_type = BlockDeviceType()
mount_point = mapping.get('device_name')
if 'ephemeral' in mapping.get('virtual_name', ''):
block_type.ephemeral_name = mapping.get('virtual_name')
else:
block_type.volume_type = mapping.get('ebs._volume_type')
block_type.snapshot_id = mapping.get('ebs._snapshot_id')
block_type.delete_on_termination = mapping.get('ebs._delete_on_termination')
block_type.size = mapping.get('ebs._volume_size')
block_type.iops = mapping.get('ebs._iops')
block_device_map[mount_point] = block_type
return block_device_map
def _parse_block_device_mappings(self):
block_device_map = BlockDeviceMapping()
for mapping in self.block_device_mapping_dict:
block_type = BlockDeviceType()
mount_point = mapping.get('device_name')
if 'ephemeral' in mapping.get('virtual_name', ''):
block_type.ephemeral_name = mapping.get('virtual_name')
else:
block_type.volume_type = mapping.get('ebs._volume_type')
block_type.snapshot_id = mapping.get('ebs._snapshot_id')
block_type.delete_on_termination = mapping.get('ebs._delete_on_termination')
block_type.size = mapping.get('ebs._volume_size')
block_type.iops = mapping.get('ebs._iops')
block_device_map[mount_point] = block_type
return block_device_map
def _get_bmap(self, params):
bmap = BlockDeviceMapping()
for device in params['bmap']:
if not 'name' in device.keys():
self.logger.debug('bad device ' + str(device))
continue
dev = BlockDeviceType()
if 'size' in device.keys():
dev.size = device['size']
if 'delete_on_termination' in device.keys():
dev.delete_on_termination = device['delete_on_termination']
if 'ephemeral_name' in device.keys():
dev.ephemeral_name = device['ephemeral_name']
bmap[device['name']] = dev
return bmap
def _process_block_device_mappings(self, launch_config, zone=None):
"""
Processes block device mapping information
and returns a Boto BlockDeviceMapping object. If new volumes
are requested (source is None and destination is VOLUME), they will be
created and the relevant volume ids included in the mapping.
"""
bdm = BlockDeviceMapping()
# Assign letters from f onwards
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
next_letter = iter(list(string.ascii_lowercase[6:]))
# assign ephemeral devices from 0 onwards
ephemeral_counter = 0
for device in launch_config.block_devices:
bd_type = BlockDeviceType()
if device.is_volume:
if device.is_root:
bdm['/dev/sda1'] = bd_type
else:
bdm['sd' + next(next_letter)] = bd_type
if isinstance(device.source, Snapshot):
bd_type.snapshot_id = device.source.id
elif isinstance(device.source, Volume):
bd_type.volume_id = device.source.id
elif isinstance(device.source, MachineImage):
# Not supported
pass
else:
# source is None, but destination is volume, therefore
# create a blank volume. If the Zone is None, this
# could fail since the volume and instance may be created
# in two different zones.
if not zone:
raise InvalidConfigurationException(
"A zone must be specified when launching with a"
" new blank volume block device mapping.")
new_vol = self.provider.block_store.volumes.create(
'', device.size, zone)
bd_type.volume_id = new_vol.id
bd_type.delete_on_terminate = device.delete_on_terminate
if device.size:
bd_type.size = device.size
else: # device is ephemeral
bd_type.ephemeral_name = 'ephemeral%s' % ephemeral_counter
return bdm
def start_instances(ec2cxn,
instance_count,
image_id,
use_ephemeral=False,
instance_type="c1.xlarge"):
print "Attempting to start ", instance_count, " instances of image: ", image_id
if use_ephemeral:
dev_map = BlockDeviceMapping()
sdb1 = BlockDeviceType()
sdb1.ephemeral_name = 'ephemeral0'
dev_map['/dev/sdb1'] = sdb1
reservation = ec2cxn.run_instances(image_id,
min_count=1,
max_count=instance_count,
block_device_map=dev_map,
security_groups=SECURITY_GROUPS,
key_name="capk",
instance_type=instance_type)
else:
reservation = ec2cxn.run_instances(image_id,
min_count=1,
max_count=instance_count,
security_groups=SECURITY_GROUPS,
key_name="capk",
instance_type=instance_type)
instances = reservation.instances
# never leave instances running at the end of the script
def kill_instances():
for instance in instances:
instance.update()
if instance.state != 'terminated':
print "Killing ", instance
instance.terminate()
atexit.register(kill_instances)
print "Started ", instance_count, " instances"
for i in instances:
print " =>", i.id
if len(instances) != instance_count:
print "Expected %d instances, got %d" % (instance_count,
len(instances))
return instances
def startCluster(self, argv):
if len(argv) != 0:
print "ec2 startCluster"
sys.exit(-1)
regions = boto.ec2.regions()
regionInfo = '\n'.join(str(region).split(':')[1] for region in regions)
regionName = raw_input("select region:\n%s\n>>"%regionInfo)
region = boto.ec2.get_region(regionName)
conn = region.connect()
print "region connected successfully"
images = conn.get_all_images(owners='self')
imageInfo = '\n'.join(
str(image).split(':')[1] + ":" + image.name for image in images)
imageId = raw_input("enter imageId:\nself-created images:\n%s\n>>"%imageInfo)
instanceTypeInfo = ("m1.small, " "m1.large, " "m1.xlarge\n"
"c1.medium, " "c1.xlarge\n"
"m2.xlarge, " "m2.2xlarge, " "m2.4xlarge\n"
"cc1.4xlarge, " "t1.micro\n")
instanceType = raw_input("enter instanceType:\n%s\n>>"%instanceTypeInfo)
availZone = raw_input("enter placement[a,b,c]:\n>>")
availZone = regionName + availZone
diskSize = int(raw_input("enter disk size[G]:\n>>"))
rootDev = BlockDeviceType()
rootDev.name = 'root'
rootDev.size = diskSize
rootDev.delete_on_termination = True
instStorage = bool(raw_input("mount inst storage?\n>>"))
mapping = BlockDeviceMapping()
mapping['/dev/sda1'] = rootDev
if (instStorage == True):
eph0 = BlockDeviceType()
eph0.ephemeral_name = 'ephemeral0'
mapping['/dev/sdb'] = eph0
groups = conn.get_all_security_groups()
groupInfo = '\n'.join(str(group).split(':')[1] for group in groups)
group = raw_input("enter securityGroup:\n%s\n>>"%groupInfo)
keys = conn.get_all_key_pairs()
if len(keys) == 1:
key = keys[0].name
print 'using default key: ' + key
else:
keyInfo = '\n'.join(str(key).split(':')[1] for key in keys)
key = raw_input("enter key name:\n%s\n>>"%keyInfo)
numNodes = int(raw_input("number of nodes:\n>>"))
conn.run_instances(
imageId, min_count=numNodes, max_count=numNodes, placement=availZone,
security_groups = [group], instance_type=instanceType,
block_device_map=mapping, key_name=key)
def _parse_block_device_mappings(self):
block_device_map = BlockDeviceMapping()
for mapping in self.block_device_mapping_dict:
block_type = BlockDeviceType()
mount_point = mapping.get("device_name")
if "ephemeral" in mapping.get("virtual_name", ""):
block_type.ephemeral_name = mapping.get("virtual_name")
else:
block_type.volume_type = mapping.get("ebs._volume_type")
block_type.snapshot_id = mapping.get("ebs._snapshot_id")
block_type.delete_on_termination = mapping.get(
"ebs._delete_on_termination")
block_type.size = mapping.get("ebs._volume_size")
block_type.iops = mapping.get("ebs._iops")
block_device_map[mount_point] = block_type
return block_device_map
def register(snapshot_id, region, arch, size=None, name=None, desc=None, pvm=False):
conn = utils.connect(region)
if None in (name, size):
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
virt = 'hvm'
kernel_id = None
device_base = '/dev/xvd'
ec2_arch = "x86_64" if arch == "amd64" else arch
if pvm:
kernel_id = utils.get_kernel(region, arch)
virt = 'paravirtual'
device_base = '/dev/sd'
name += '-pvm'
log.debug('creating block_device_map')
block_device_map = BlockDeviceMapping()
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
rootfs_device_name = device_base + 'a'
block_device_map[rootfs_device_name] = rootfs
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
ephemeral_device_name = device_base + 'b'
block_device_map[ephemeral_device_name] = ephemeral
log.debug('registering image - %s', name)
ami_id = conn.register_image(
name=name,
description=desc,
kernel_id=kernel_id,
architecture=ec2_arch,
root_device_name=rootfs_device_name,
block_device_map=block_device_map,
virtualization_type=virt)
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id, name
def get_block_device(instance_type, ebs_vol_size):
block_map = BlockDeviceMapping()
if ebs_vol_size > 0:
device = EBSBlockDeviceType()
device.size = ebs_vol_size
device.delete_on_termination = True
block_map["/dev/sdv"] = device
for i in range(get_num_disks(instance_type)):
dev = BlockDeviceType()
dev.ephemeral_name = 'ephemeral%d' % i
# The first ephemeral drive is /dev/sdb.
name = '/dev/sd' + string.ascii_letters[i + 1]
block_map[name] = dev
return block_map
def get_block_device(instance_type, ebs_vol_size):
block_map = BlockDeviceMapping()
if ebs_vol_size > 0:
device = EBSBlockDeviceType()
device.size = ebs_vol_size
device.delete_on_termination = True
block_map['/dev/sdv'] = device
for i in range(get_num_disks(instance_type)):
dev = BlockDeviceType()
dev.ephemeral_name = 'ephemeral%d' % i
# The first ephemeral drive is /dev/sdb.
name = '/dev/sd' + string.ascii_letters[i + 1]
block_map[name] = dev
return block_map
def start_instances(ec2cxn, instance_count, image_id, use_ephemeral = False, instance_type="c1.xlarge"):
print "Attempting to start ", instance_count, " instances of image: ", image_id
if use_ephemeral:
dev_map = BlockDeviceMapping()
sdb1 = BlockDeviceType()
sdb1.ephemeral_name = 'ephemeral0'
dev_map['/dev/sdb1'] = sdb1
reservation = ec2cxn.run_instances(
image_id, min_count=1,
max_count=instance_count,
block_device_map=dev_map,
security_groups=SECURITY_GROUPS,
key_name="capk",
instance_type=instance_type)
else:
reservation = ec2cxn.run_instances(
image_id,
min_count=1,
max_count=instance_count,
security_groups=SECURITY_GROUPS,
key_name="capk",
instance_type=instance_type)
instances = reservation.instances
# never leave instances running at the end of the script
def kill_instances():
for instance in instances:
instance.update()
if instance.state != 'terminated':
print "Killing ", instance
instance.terminate()
atexit.register(kill_instances)
print "Started ", instance_count, " instances"
for i in instances:
print " =>", i.id
if len(instances) != instance_count:
print "Expected %d instances, got %d" % (instance_count, len(instances))
return instances
def register(snapshot_id, region, arch, size=None, name=None, desc=None):
conn = utils.connect(region)
if None in (name, size):
log.debug("getting snapshot - %s", snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
ec2_arch = "x86_64" if arch == "amd64" else arch
log.debug("creating block_device_map")
block_device_map = BlockDeviceMapping()
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
rootfs_device_name = "/dev/xvda"
block_device_map[rootfs_device_name] = rootfs
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = "ephemeral0"
ephemeral_device_name = "/dev/xvdb"
block_device_map[ephemeral_device_name] = ephemeral
log.debug("registering image - %s", name)
ami_id = conn.register_image(
name=name,
description=desc,
architecture=ec2_arch,
root_device_name=rootfs_device_name,
block_device_map=block_device_map,
virtualization_type="hvm",
)
log.info("registered image - %s %s %s", ami_id, name, region)
return ami_id, name
def register(snapshot_id, region, size=None, arch=None, name=None, desc=None):
conn = utils.connect(region)
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
desc = desc if desc else utils.parse_imagename(name)['url']
arch = arch if arch else utils.parse_imagename(name)['architecture']
kernel_id = utils.get_kernel(region, arch)
arch_ec2 = "x86_64" if arch == "amd64" else arch
log.debug('creating block_device_map')
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
block_device_map = BlockDeviceMapping()
block_device_map['/dev/sda1'] = rootfs
block_device_map['/dev/sda2'] = ephemeral
log.debug('registering image - %s', name)
ami_id = conn.register_image(
name=name,
description=desc,
architecture=arch_ec2,
kernel_id=kernel_id,
root_device_name="/dev/sda1",
block_device_map=block_device_map)
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id
def get_block_device_map(bdmapping_json=None):
"""Parse block_device_mapping JSON and return a configured BlockDeviceMapping object
Mapping JSON structure...
{"/dev/sda":
{"snapshot_id": "snap-23E93E09", "volume_type": null, "delete_on_termination": true, "size": 1} }
"""
if bdmapping_json:
mapping = json.loads(bdmapping_json)
if mapping:
bdm = BlockDeviceMapping()
for key, val in mapping.items():
device = BlockDeviceType()
if val.get('virtual_name') is not None and val.get('virtual_name').startswith('ephemeral'):
device.ephemeral_name = val.get('virtual_name')
else:
device.volume_type = 'standard'
device.snapshot_id = val.get('snapshot_id') or None
device.size = val.get('size')
device.delete_on_termination = val.get('delete_on_termination', False)
bdm[key] = device
return bdm
return None
return None
def parse_block_device_args(self, block_device_maps_args):
block_device_map = BlockDeviceMapping()
for block_device_map_arg in block_device_maps_args:
parts = block_device_map_arg.split('=')
if len(parts) > 1:
device_name = parts[0]
block_dev_type = BlockDeviceType()
value_parts = parts[1].split(':')
if value_parts[0].startswith('snap'):
block_dev_type.snapshot_id = value_parts[0]
else:
if value_parts[0].startswith('ephemeral'):
block_dev_type.ephemeral_name = value_parts[0]
if len(value_parts) > 1:
try:
block_dev_type.size = int(value_parts[1])
except ValueError:
pass
if len(value_parts) > 2:
if value_parts[2] == 'true':
block_dev_type.delete_on_termination = True
block_device_map[device_name] = block_dev_type
return block_device_map
def parse_block_device_args(self, block_device_maps_args):
block_device_map = BlockDeviceMapping()
for block_device_map_arg in block_device_maps_args:
parts = block_device_map_arg.split('=')
if len(parts) > 1:
device_name = parts[0]
block_dev_type = BlockDeviceType()
value_parts = parts[1].split(':')
if value_parts[0].startswith('snap'):
block_dev_type.snapshot_id = value_parts[0]
else:
if value_parts[0].startswith('ephemeral'):
block_dev_type.ephemeral_name = value_parts[0]
if len(value_parts) > 1:
try:
block_dev_type.size = int(value_parts[1])
except ValueError:
pass
if len(value_parts) > 2:
if value_parts[2] == 'true':
block_dev_type.delete_on_termination = True
block_device_map[device_name] = block_dev_type
return block_device_map
def register(snapshot_id, region, size=None, arch=None, name=None, desc=None):
conn = utils.connect(region)
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
desc = desc if desc else utils.parse_imagename(name)['url']
arch = arch if arch else utils.parse_imagename(name)['architecture']
kernel_id = utils.get_kernel(region, arch)
arch_ec2 = "x86_64" if arch == "amd64" else arch
log.debug('creating block_device_map')
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
block_device_map = BlockDeviceMapping()
block_device_map['/dev/sda1'] = rootfs
block_device_map['/dev/sda2'] = ephemeral
log.debug('registering image - %s', name)
ami_id = conn.register_image(name=name,
description=desc,
architecture=arch_ec2,
kernel_id=kernel_id,
root_device_name="/dev/sda1",
block_device_map=block_device_map)
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id
def __update_bdm(self, bdm, bd_spec):
"""Update the BlockDeviceMapping bdm with the block device
spec bd_spec.
"""
try:
dev_name, dev_value = bd_spec.split('=', 1)
except Exception:
raise CommandError(
"Block device spec missing '=' : %s" % (bd_spec,))
dot = None
bdt = BlockDeviceType()
if ':' in dev_value:
blockdev_origin, dot = dev_value.split(':', 1)
else:
blockdev_origin = dev_value
if blockdev_origin is None:
raise CommandError("No source specified for %s" % (dev_name,))
if blockdev_origin.startswith('ephemeral'):
bdt.ephemeral_name = blockdev_origin
elif blockdev_origin.startswith('snap-'):
bdt.snapshot_id = blockdev_origin
else:
raise CommandError("Bad source specified for %s: %s" % (dev_name,
blockdev_origin))
if dot is not None:
if dot == 'delete':
bdt.delete_on_termination = True
elif dot == 'nodelete':
bdt.delete_on_termination = False
else:
raise CommandError(
"Bad delete-on-termination specified for %s: %s" %
(dev_name, dot))
else:
bdt.delete_on_termination = False
dev_path = '/dev/' + dev_name
bdm[dev_path] = bdt
def handleImages(self, action, clc, callback=None):
if action == "DescribeImages":
owner = self.get_argument("Owner", None)
if not owner:
owners = None
else:
owners = [owner]
filters = self.get_filter_args()
return clc.get_all_images(owners, filters, callback)
elif action == "DescribeImageAttribute":
imageid = self.get_argument("ImageId")
attribute = self.get_argument("Attribute")
return clc.get_image_attribute(imageid, attribute, callback)
elif action == "ModifyImageAttribute":
imageid = self.get_argument("ImageId")
attribute = self.get_argument("Attribute")
operation = self.get_argument("OperationType")
users = self.get_argument_list("UserId")
groups = self.get_argument_list("UserGroup")
return clc.modify_image_attribute(imageid, attribute, operation, users, groups, callback)
elif action == "ResetImageAttribute":
imageid = self.get_argument("ImageId")
attribute = self.get_argument("Attribute")
return clc.reset_image_attribute(imageid, attribute, callback)
elif action == "DeregisterImage":
image_id = self.get_argument("ImageId")
return clc.deregister_image(image_id, callback)
elif action == "RegisterImage":
image_location = self.get_argument("ImageLocation", None)
name = self.get_argument("Name")
description = self.get_argument("Description", None)
if description != None:
description = base64.b64decode(description)
architecture = self.get_argument("Architecture", None)
kernel_id = self.get_argument("KernelId", None)
ramdisk_id = self.get_argument("RamdiskId", None)
root_dev_name = self.get_argument("RootDeviceName", None)
snapshot_id = self.get_argument("SnapshotId", None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument("BlockDeviceMapping.1.DeviceName", None)
idx = 1
while mapping:
pre = "BlockDeviceMapping.%d" % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument("%s.VirtualName" % pre, None)
if not (block_dev_type.ephemeral_name):
block_dev_type.no_device = self.get_argument("%s.NoDevice" % pre, "") == "true"
block_dev_type.snapshot_id = self.get_argument("%s.Ebs.SnapshotId" % pre, None)
block_dev_type.size = self.get_argument("%s.Ebs.VolumeSize" % pre, None)
block_dev_type.delete_on_termination = (
self.get_argument("%s.Ebs.DeleteOnTermination" % pre, "") == "true"
)
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument("BlockDeviceMapping.%d.DeviceName" % idx, None)
if snapshot_id:
rootbdm = BlockDeviceType()
rootbdm.snapshot_id = snapshot_id
bdm["/dev/sda1"] = rootbdm
if len(bdm) == 0:
bdm = None
return clc.register_image(
name, image_location, description, architecture, kernel_id, ramdisk_id, root_dev_name, bdm, callback
)
def handleRunInstances(self, action, clc, user_data_file, callback):
image_id = self.get_argument("ImageId")
min = self.get_argument("MinCount", "1")
max = self.get_argument("MaxCount", "1")
key = self.get_argument("KeyName", None)
groups = self.get_argument_list("SecurityGroup")
sec_group_ids = self.get_argument_list("SecurityGroupId")
if user_data_file:
user_data = user_data_file
else:
user_data = self.get_argument("UserData", "")
user_data = base64.b64decode(user_data)
addr_type = self.get_argument("AddressingType", None)
vm_type = self.get_argument("InstanceType", None)
placement = self.get_argument("Placement.AvailabilityZone", None)
placement_group = self.get_argument("Placement.GroupName", None)
tenancy = self.get_argument("Placement.Tenancy", None)
kernel = self.get_argument("KernelId", None)
ramdisk = self.get_argument("RamdiskId", None)
monitoring = False
if self.get_argument("Monitoring.Enabled", "") == "true":
monitoring = True
subnet = self.get_argument("SubnetId", None)
private_ip = self.get_argument("PrivateIpAddress", None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument("BlockDeviceMapping.1.DeviceName", None)
idx = 1
while mapping:
pre = "BlockDeviceMapping.%d" % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument("%s.VirtualName" % pre, None)
if not (block_dev_type.ephemeral_name):
block_dev_type.no_device = self.get_argument("%s.NoDevice" % pre, "") == "true"
block_dev_type.snapshot_id = self.get_argument("%s.Ebs.SnapshotId" % pre, None)
block_dev_type.size = self.get_argument("%s.Ebs.VolumeSize" % pre, None)
block_dev_type.delete_on_termination = (
self.get_argument("%s.Ebs.DeleteOnTermination" % pre, "") == "true"
)
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument("BlockDeviceMapping.%d.DeviceName" % idx, None)
if len(bdm) == 0:
bdm = None
api_termination = False
if self.get_argument("DisableApiTermination", "") == "true":
api_termination = True
instance_shutdown = False
if self.get_argument("InstanceInitiatedShutdownBehavior", "") == "true":
instance_shutdown = True
token = self.get_argument("ClientToken", None)
addition_info = self.get_argument("AdditionInfo", None)
instance_profile_name = self.get_argument("IamInstanceProfile.Name", None)
instance_profile_arn = self.get_argument("IamInstanceProfile.Arn", None)
return clc.run_instances(
image_id,
min_count=min,
max_count=max,
key_name=key,
security_groups=groups,
user_data=user_data,
addressing_type=addr_type,
instance_type=vm_type,
placement=placement,
kernel_id=kernel,
ramdisk_id=ramdisk,
monitoring_enabled=monitoring,
subnet_id=subnet,
block_device_map=bdm,
disable_api_termination=api_termination,
instance_initiated_shutdown_behavior=instance_shutdown,
private_ip_address=private_ip,
placement_group=placement_group,
client_token=token,
security_group_ids=sec_group_ids,
additional_info=addition_info,
instance_profile_name=instance_profile_name,
instance_profile_arn=instance_profile_arn,
tenancy=tenancy,
callback=callback,
)
def create_cluster(conn, args):
if args.identity_file is None:
print("ERROR: Must provide an identity file (-i) for ssh connections.",
file=stderr)
sys.exit(1)
if args.key_pair is None:
print("ERROR: Must provide a key pair name (-k) to use on instances.",
file=stderr)
sys.exit(1)
# make or get the security group.
security_group = get_or_make_group(conn, args.name, args.vpc_id)
# set the inbound permission rules
if len(security_group.rules) == 0:
if __name__ == '__main__':
if args.vpc_id is None:
security_group.authorize(src_group=security_group)
else:
security_group.authorize('tcp', 22, 22,
args.authorized_address)
security_group.authorize('tcp', 8888, 8888,
args.authorized_address)
security_group.authorize('tcp', 7000, 7000,
args.authorized_address)
security_group.authorize('tcp', 7001, 7001,
args.authorized_address)
security_group.authorize('tcp', 7199, 7199,
args.authorized_address)
security_group.authorize('tcp', 9042, 9042,
args.authorized_address)
security_group.authorize('tcp', 9160, 9160,
args.authorized_address)
else:
print("Security group already exists, skipping creation.")
instances = cluster_nodes(conn, args.name)
if any(instances):
additional_tags = {}
for i in instances:
i.add_tags(
dict(additional_tags,
Name="{cn}-node-{iid}".format(cn=args.name, iid=i.id)))
return instances
else:
print(
"Launching {m} instances for cluster...".format(m=args.node_count))
try:
image = conn.get_all_images(image_ids=args.ami)[0]
block_map = BlockDeviceMapping()
if args.ebs_vol_size > 0:
if args.instance_type.startswith('m3.'):
for i in range(get_num_disks(args.instance_type)):
device = BlockDeviceType()
device.ephemeral_name = "ephemeral%d" % i
name = "/dev/sd" + string.ascii_letters[i + 1]
block_map[name] = device
else:
device = EBSBlockDeviceType()
device.size = args.ebs_vol_size
device.volume_type = args.ebs_vol_type
device.delete_on_termination = True
key = "/dev/sd" + chr(ord('s') + 1)
block_map[key] = device
nodes = image.run(key_name=args.key_pair,
security_group_ids=[security_group.id],
instance_type="",
placement=args.zone,
min_count=args.node_count,
max_count=args.node_count,
block_device_map=block_map,
subnet_id=None,
placement_group=None,
user_data=None,
instance_initiated_shutdown_behavior="stop",
instance_profile_name=None)
print("Waiting for AWS to propagate instance metadata...")
time.sleep(15)
additional_tags = {}
for node in nodes.instances:
node.add_tags(
dict(additional_tags,
Name="{cn}-node-{iid}".format(cn=args.name,
iid=node.id)))
return nodes.instances
except Exception as e:
print("Caught exception: ", e)
print("ERROR: Could not find AMI " + args.ami, file=stderr)
sys.exit(1)
parser.add_argument('--server', help='Packaging serfver to use', default = None);
parser.add_argument('--instance-type', help='Instance Type', default = 'm3.xlarge');
parser.add_argument('--version', help='Version number to build', required = True, default = None);
args = parser.parse_args()
os.environ['AWS_ACCESS_KEY_ID'] = AWS_ACCESS_KEY_ID
os.environ['AWS_SECRET_ACCESS_KEY'] = AWS_SECRET_ACCESS_KEY
if args.server:
print("Using server %s\n" % args.server)
env.hosts = [args.server]
else:
print("Creating server\n")
conn = boto.ec2.connect_to_region(region)
dev_sdb = BlockDeviceType()
dev_sdb.ephemeral_name = 'ephemeral0'
bdm = BlockDeviceMapping()
bdm['/dev/sdb'] = dev_sdb
# bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping({'/dev/xvdb': 'ephemeral0', '/dev/xvdc': 'ephemeral1'})
res = conn.run_instances(
ami_id,
key_name = key_name,
instance_type = args.instance_type,
security_groups=[security_group],
block_device_map = bdm)
timeout = 60
inst = res.instances[0]
def handleRunInstances(self, action, clc, user_data_file):
image_id = self.get_argument('ImageId')
min = self.get_argument('MinCount', '1')
max = self.get_argument('MaxCount', '1')
key = self.get_argument('KeyName', None)
groups = self.get_argument_list('SecurityGroup')
sec_group_ids = self.get_argument_list('SecurityGroupId')
if user_data_file:
user_data = user_data_file
else:
user_data = self.get_argument('UserData', None)
addr_type = self.get_argument('AddressingType', None)
vm_type = self.get_argument('InstanceType', None)
placement = self.get_argument('Placement.AvailabilityZone', None)
placement_group = self.get_argument('Placement.GroupName', None)
tenancy = self.get_argument('Placement.Tenancy', None)
kernel = self.get_argument('KernelId', None)
ramdisk = self.get_argument('RamdiskId', None)
monitoring = False
if self.get_argument('Monitoring.Enabled', '') == 'true':
monitoring = True
subnet = self.get_argument('SubnetId', None)
private_ip = self.get_argument('PrivateIpAddress', None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None)
idx = 1
while mapping:
pre = 'BlockDeviceMapping.%d' % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument(
'%s.VirtualName' % pre, None)
if not (block_dev_type.ephemeral_name):
block_dev_type.no_device = \
(self.get_argument('%s.NoDevice' % pre, '') == 'true')
block_dev_type.snapshot_id = \
self.get_argument('%s.Ebs.SnapshotId' % pre, None)
block_dev_type.size = \
self.get_argument('%s.Ebs.VolumeSize' % pre, None)
block_dev_type.delete_on_termination = \
(self.get_argument('%s.DeleteOnTermination' % pre, '') == 'true')
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument(
'BlockDeviceMapping.%d.DeviceName' % idx, None)
if len(bdm) == 0:
bdm = None
api_termination = False
if self.get_argument('DisableApiTermination', '') == 'true':
api_termination = True
instance_shutdown = False
if self.get_argument('InstanceInitiatedShutdownBehavior',
'') == 'true':
instance_shutdown = True
token = self.get_argument('ClientToken', None)
addition_info = self.get_argument('AdditionInfo', None)
instance_profile_name = self.get_argument('IamInstanceProfile.Name',
None)
instance_profile_arn = self.get_argument('IamInstanceProfile.Arn',
None)
return self.__normalize_instances__([
clc.run_instances(
image_id,
min_count=min,
max_count=max,
key_name=key,
security_groups=groups,
user_data=user_data,
addressing_type=addr_type,
instance_type=vm_type,
placement=placement,
kernel_id=kernel,
ramdisk_id=ramdisk,
monitoring_enabled=monitoring,
subnet_id=subnet,
block_device_map=bdm,
disable_api_termination=api_termination,
instance_initiated_shutdown_behavior=instance_shutdown,
private_ip_address=private_ip,
placement_group=placement_group,
client_token=token,
security_group_ids=sec_group_ids,
additional_info=addition_info,
instance_profile_name=instance_profile_name,
instance_profile_arn=instance_profile_arn,
tenancy=tenancy)
])
def launch_cluster(conn, opts, cluster_name):
if opts.identity_file is None:
print("ERROR: Must provide an identity file (-i) for ssh connections.", file=stderr)
sys.exit(1)
if opts.key_pair is None:
print("ERROR: Must provide a key pair name (-k) to use on instances.", file=stderr)
sys.exit(1)
user_data_content = None
if opts.user_data:
with open(opts.user_data) as user_data_file:
user_data_content = user_data_file.read()
print("Setting up security groups...")
master_group = get_or_make_group(conn, cluster_name + "-master", opts.vpc_id)
slave_group = get_or_make_group(conn, cluster_name + "-slaves", opts.vpc_id)
authorized_address = opts.authorized_address
if master_group.rules == []: # Group was just now created
if opts.vpc_id is None:
master_group.authorize(src_group=master_group)
master_group.authorize(src_group=slave_group)
else:
master_group.authorize(ip_protocol='icmp', from_port=-1, to_port=-1,
src_group=master_group)
master_group.authorize(ip_protocol='tcp', from_port=0, to_port=65535,
src_group=master_group)
master_group.authorize(ip_protocol='udp', from_port=0, to_port=65535,
src_group=master_group)
master_group.authorize(ip_protocol='icmp', from_port=-1, to_port=-1,
src_group=slave_group)
master_group.authorize(ip_protocol='tcp', from_port=0, to_port=65535,
src_group=slave_group)
master_group.authorize(ip_protocol='udp', from_port=0, to_port=65535,
src_group=slave_group)
master_group.authorize('tcp', 22, 22, authorized_address)
master_group.authorize('tcp', 8080, 8081, authorized_address)
master_group.authorize('tcp', 18080, 18080, authorized_address)
master_group.authorize('tcp', 19999, 19999, authorized_address)
master_group.authorize('tcp', 50030, 50030, authorized_address)
master_group.authorize('tcp', 50070, 50070, authorized_address)
master_group.authorize('tcp', 60070, 60070, authorized_address)
master_group.authorize('tcp', 4040, 4045, authorized_address)
# Rstudio (GUI for R) needs port 8787 for web access
master_group.authorize('tcp', 8787, 8787, authorized_address)
# HDFS NFS gateway requires 111,2049,4242 for tcp & udp
master_group.authorize('tcp', 111, 111, authorized_address)
master_group.authorize('udp', 111, 111, authorized_address)
master_group.authorize('tcp', 2049, 2049, authorized_address)
master_group.authorize('udp', 2049, 2049, authorized_address)
master_group.authorize('tcp', 4242, 4242, authorized_address)
master_group.authorize('udp', 4242, 4242, authorized_address)
# RM in YARN mode uses 8088
master_group.authorize('tcp', 8088, 8088, authorized_address)
if opts.ganglia:
master_group.authorize('tcp', 5080, 5080, authorized_address)
if slave_group.rules == []: # Group was just now created
if opts.vpc_id is None:
slave_group.authorize(src_group=master_group)
slave_group.authorize(src_group=slave_group)
else:
slave_group.authorize(ip_protocol='icmp', from_port=-1, to_port=-1,
src_group=master_group)
slave_group.authorize(ip_protocol='tcp', from_port=0, to_port=65535,
src_group=master_group)
slave_group.authorize(ip_protocol='udp', from_port=0, to_port=65535,
src_group=master_group)
slave_group.authorize(ip_protocol='icmp', from_port=-1, to_port=-1,
src_group=slave_group)
slave_group.authorize(ip_protocol='tcp', from_port=0, to_port=65535,
src_group=slave_group)
slave_group.authorize(ip_protocol='udp', from_port=0, to_port=65535,
src_group=slave_group)
slave_group.authorize('tcp', 22, 22, authorized_address)
slave_group.authorize('tcp', 8080, 8081, authorized_address)
slave_group.authorize('tcp', 50060, 50060, authorized_address)
slave_group.authorize('tcp', 50075, 50075, authorized_address)
slave_group.authorize('tcp', 60060, 60060, authorized_address)
slave_group.authorize('tcp', 60075, 60075, authorized_address)
#Kylix
slave_group.authorize(ip_protocol='tcp', from_port=50050, to_port=50060,
src_group=slave_group)
slave_group.authorize(ip_protocol='udp', from_port=50050, to_port=50060,
src_group=slave_group)
slave_group.authorize(ip_protocol='tcp', from_port=50050, to_port=50060,
src_group=master_group)
slave_group.authorize(ip_protocol='udp', from_port=50050, to_port=50060,
src_group=master_group)
master_group.authorize(ip_protocol='tcp', from_port=50050, to_port=50060,
src_group=slave_group)
master_group.authorize(ip_protocol='udp', from_port=50050, to_port=50060,
src_group=slave_group)
# Check if instances are already running in our groups
existing_masters, existing_slaves = get_existing_cluster(conn, opts, cluster_name,
die_on_error=False)
if existing_slaves or (existing_masters and not opts.use_existing_master):
print("ERROR: There are already instances running in group %s or %s" %
(master_group.name, slave_group.name), file=stderr)
sys.exit(1)
# we use group ids to work around https://github.com/boto/boto/issues/350
additional_group_ids = []
if opts.additional_security_group:
additional_group_ids = [sg.id
for sg in conn.get_all_security_groups()
if opts.additional_security_group in (sg.name, sg.id)]
print("Launching instances...")
try:
image = conn.get_all_images(image_ids=[opts.ami])[0]
except:
print("Could not find AMI " + opts.ami, file=stderr)
sys.exit(1)
# Create block device mapping so that we can add EBS volumes if asked to.
# The first drive is attached as /dev/sds, 2nd as /dev/sdt, ... /dev/sdz
block_map = BlockDeviceMapping()
if opts.ebs_vol_size > 0:
for i in range(opts.ebs_vol_num):
device = EBSBlockDeviceType()
device.size = opts.ebs_vol_size
device.volume_type = opts.ebs_vol_type
device.delete_on_termination = True
block_map["/dev/sd" + chr(ord('s') + i)] = device
# AWS ignores the AMI-specified block device mapping for M3 (see SPARK-3342).
if opts.instance_type.startswith('m3.'):
for i in range(get_num_disks(opts.instance_type)):
dev = BlockDeviceType()
dev.ephemeral_name = 'ephemeral%d' % i
# The first ephemeral drive is /dev/sdb.
name = '/dev/sd' + string.letters[i + 1]
block_map[name] = dev
# Launch slaves
if opts.spot_price is not None:
# Launch spot instances with the requested price
print("Requesting %d slaves as spot instances with price $%.3f" %
(opts.slaves, opts.spot_price))
zones = get_zones(conn, opts)
num_zones = len(zones)
i = 0
my_req_ids = []
for zone in zones:
num_slaves_this_zone = get_partition(opts.slaves, num_zones, i)
slave_reqs = conn.request_spot_instances(
price=opts.spot_price,
image_id=opts.ami,
launch_group="launch-group-%s" % cluster_name,
placement=zone,
count=num_slaves_this_zone,
key_name=opts.key_pair,
security_group_ids=[slave_group.id] + additional_group_ids,
instance_type=opts.instance_type,
block_device_map=block_map,
subnet_id=opts.subnet_id,
placement_group=opts.placement_group,
user_data=user_data_content,
instance_profile_name=opts.instance_profile_name)
my_req_ids += [req.id for req in slave_reqs]
i += 1
print("Waiting for spot instances to be granted...")
try:
while True:
time.sleep(10)
reqs = conn.get_all_spot_instance_requests()
id_to_req = {}
for r in reqs:
id_to_req[r.id] = r
active_instance_ids = []
for i in my_req_ids:
if i in id_to_req and id_to_req[i].state == "active":
active_instance_ids.append(id_to_req[i].instance_id)
if len(active_instance_ids) == opts.slaves:
print("All %d slaves granted" % opts.slaves)
reservations = conn.get_all_reservations(active_instance_ids)
slave_nodes = []
for r in reservations:
slave_nodes += r.instances
break
else:
print("%d of %d slaves granted, waiting longer" % (
len(active_instance_ids), opts.slaves))
except:
print("Canceling spot instance requests")
conn.cancel_spot_instance_requests(my_req_ids)
# Log a warning if any of these requests actually launched instances:
(master_nodes, slave_nodes) = get_existing_cluster(
conn, opts, cluster_name, die_on_error=False)
running = len(master_nodes) + len(slave_nodes)
if running:
print(("WARNING: %d instances are still running" % running), file=stderr)
sys.exit(0)
else:
# Launch non-spot instances
zones = get_zones(conn, opts)
num_zones = len(zones)
i = 0
slave_nodes = []
for zone in zones:
num_slaves_this_zone = get_partition(opts.slaves, num_zones, i)
if num_slaves_this_zone > 0:
slave_res = image.run(
key_name=opts.key_pair,
security_group_ids=[slave_group.id] + additional_group_ids,
instance_type=opts.instance_type,
placement=zone,
min_count=num_slaves_this_zone,
max_count=num_slaves_this_zone,
block_device_map=block_map,
subnet_id=opts.subnet_id,
placement_group=opts.placement_group,
user_data=user_data_content,
instance_initiated_shutdown_behavior=opts.instance_initiated_shutdown_behavior,
instance_profile_name=opts.instance_profile_name)
slave_nodes += slave_res.instances
print("Launched {s} slave{plural_s} in {z}, regid = {r}".format(
s=num_slaves_this_zone,
plural_s=('' if num_slaves_this_zone == 1 else 's'),
z=zone,
r=slave_res.id))
i += 1
# Launch or resume masters
if existing_masters:
print("Starting master...")
for inst in existing_masters:
if inst.state not in ["shutting-down", "terminated"]:
inst.start()
master_nodes = existing_masters
else:
master_type = opts.master_instance_type
if master_type == "":
master_type = opts.instance_type
if opts.zone == 'all':
opts.zone = random.choice(conn.get_all_zones()).name
master_res = image.run(
key_name=opts.key_pair,
security_group_ids=[master_group.id] + additional_group_ids,
instance_type=master_type,
placement=opts.zone,
min_count=1,
max_count=1,
block_device_map=block_map,
subnet_id=opts.subnet_id,
placement_group=opts.placement_group,
user_data=user_data_content,
instance_initiated_shutdown_behavior=opts.instance_initiated_shutdown_behavior,
instance_profile_name=opts.instance_profile_name)
master_nodes = master_res.instances
print("Launched master in %s, regid = %s" % (zone, master_res.id))
# This wait time corresponds to SPARK-4983
print("Waiting for AWS to propagate instance metadata...")
time.sleep(15)
# Give the instances descriptive names and set additional tags
additional_tags = {}
if opts.additional_tags.strip():
additional_tags = dict(
map(str.strip, tag.split(':', 1)) for tag in opts.additional_tags.split(',')
)
for master in master_nodes:
master.add_tags(
dict(additional_tags, Name='{cn}-master-{iid}'.format(cn=cluster_name, iid=master.id))
)
for slave in slave_nodes:
slave.add_tags(
dict(additional_tags, Name='{cn}-slave-{iid}'.format(cn=cluster_name, iid=slave.id))
)
# Return all the instances
return (master_nodes, slave_nodes)
def launch_cluster(conn, opts, cluster_name):
if opts.identity_file is None:
print >> stderr, "ERROR: Must provide an identity file (-i) for ssh connections."
sys.exit(1)
if opts.key_pair is None:
print >> stderr, "ERROR: Must provide a key pair name (-k) to use on instances."
sys.exit(1)
user_data_content = None
if opts.user_data:
with open(opts.user_data) as user_data_file:
user_data_content = user_data_file.read()
print "Setting up security groups..."
master_group = get_or_make_group(conn, cluster_name + "-master")
slave_group = get_or_make_group(conn, cluster_name + "-slaves")
if master_group.rules == []: # Group was just now created
master_group.authorize(src_group=master_group)
master_group.authorize(src_group=slave_group)
master_group.authorize('tcp', 22, 22, '0.0.0.0/0')
master_group.authorize('tcp', 8080, 8081, '0.0.0.0/0')
master_group.authorize('tcp', 18080, 18080, '0.0.0.0/0')
master_group.authorize('tcp', 19999, 19999, '0.0.0.0/0')
master_group.authorize('tcp', 50030, 50030, '0.0.0.0/0')
master_group.authorize('tcp', 50070, 50070, '0.0.0.0/0')
master_group.authorize('tcp', 60070, 60070, '0.0.0.0/0')
master_group.authorize('tcp', 4040, 4045, '0.0.0.0/0')
if opts.ganglia:
master_group.authorize('tcp', 5080, 5080, '0.0.0.0/0')
if slave_group.rules == []: # Group was just now created
slave_group.authorize(src_group=master_group)
slave_group.authorize(src_group=slave_group)
slave_group.authorize('tcp', 22, 22, '0.0.0.0/0')
slave_group.authorize('tcp', 8080, 8081, '0.0.0.0/0')
slave_group.authorize('tcp', 50060, 50060, '0.0.0.0/0')
slave_group.authorize('tcp', 50075, 50075, '0.0.0.0/0')
slave_group.authorize('tcp', 60060, 60060, '0.0.0.0/0')
slave_group.authorize('tcp', 60075, 60075, '0.0.0.0/0')
# Check if instances are already running in our groups
existing_masters, existing_slaves = get_existing_cluster(conn, opts, cluster_name,
die_on_error=False)
if existing_slaves or (existing_masters and not opts.use_existing_master):
print >> stderr, ("ERROR: There are already instances running in " +
"group %s or %s" % (master_group.name, slave_group.name))
sys.exit(1)
# Figure out Spark AMI
if opts.ami is None:
opts.ami = get_spark_ami(opts)
print "Launching instances..."
try:
image = conn.get_all_images(image_ids=[opts.ami])[0]
except:
print >> stderr, "Could not find AMI " + opts.ami
sys.exit(1)
# Create block device mapping so that we can add an EBS volume if asked to
block_map = BlockDeviceMapping()
if opts.ebs_vol_size > 0:
device = EBSBlockDeviceType()
device.size = opts.ebs_vol_size
device.delete_on_termination = True
block_map["/dev/sdv"] = device
# AWS ignores the AMI-specified block device mapping for M3 (see SPARK-3342).
if opts.instance_type.startswith('m3.'):
for i in range(get_num_disks(opts.instance_type)):
dev = BlockDeviceType()
dev.ephemeral_name = 'ephemeral%d' % i
# The first ephemeral drive is /dev/sdb.
name = '/dev/sd' + string.letters[i + 1]
block_map[name] = dev
# Launch slaves
if opts.spot_price is not None:
# Launch spot instances with the requested price
print ("Requesting %d slaves as spot instances with price $%.3f" %
(opts.slaves, opts.spot_price))
zones = get_zones(conn, opts)
num_zones = len(zones)
i = 0
my_req_ids = []
for zone in zones:
num_slaves_this_zone = get_partition(opts.slaves, num_zones, i)
slave_reqs = conn.request_spot_instances(
price=opts.spot_price,
image_id=opts.ami,
launch_group="launch-group-%s" % cluster_name,
placement=zone,
count=num_slaves_this_zone,
key_name=opts.key_pair,
security_groups=[slave_group],
instance_type=opts.instance_type,
block_device_map=block_map,
user_data=user_data_content)
my_req_ids += [req.id for req in slave_reqs]
i += 1
print "Waiting for spot instances to be granted..."
try:
while True:
time.sleep(10)
reqs = conn.get_all_spot_instance_requests()
id_to_req = {}
for r in reqs:
id_to_req[r.id] = r
active_instance_ids = []
for i in my_req_ids:
if i in id_to_req and id_to_req[i].state == "active":
active_instance_ids.append(id_to_req[i].instance_id)
if len(active_instance_ids) == opts.slaves:
print "All %d slaves granted" % opts.slaves
reservations = conn.get_all_instances(active_instance_ids)
slave_nodes = []
for r in reservations:
slave_nodes += r.instances
break
else:
print "%d of %d slaves granted, waiting longer" % (
len(active_instance_ids), opts.slaves)
except:
print "Canceling spot instance requests"
conn.cancel_spot_instance_requests(my_req_ids)
# Log a warning if any of these requests actually launched instances:
(master_nodes, slave_nodes) = get_existing_cluster(
conn, opts, cluster_name, die_on_error=False)
running = len(master_nodes) + len(slave_nodes)
if running:
print >> stderr, ("WARNING: %d instances are still running" % running)
sys.exit(0)
else:
# Launch non-spot instances
zones = get_zones(conn, opts)
num_zones = len(zones)
i = 0
slave_nodes = []
for zone in zones:
num_slaves_this_zone = get_partition(opts.slaves, num_zones, i)
if num_slaves_this_zone > 0:
slave_res = image.run(key_name=opts.key_pair,
security_groups=[slave_group],
instance_type=opts.instance_type,
placement=zone,
min_count=num_slaves_this_zone,
max_count=num_slaves_this_zone,
block_device_map=block_map,
user_data=user_data_content)
slave_nodes += slave_res.instances
print "Launched %d slaves in %s, regid = %s" % (num_slaves_this_zone,
zone, slave_res.id)
i += 1
# Launch or resume masters
if existing_masters:
print "Starting master..."
for inst in existing_masters:
if inst.state not in ["shutting-down", "terminated"]:
inst.start()
master_nodes = existing_masters
else:
master_type = opts.master_instance_type
if master_type == "":
master_type = opts.instance_type
if opts.zone == 'all':
opts.zone = random.choice(conn.get_all_zones()).name
master_res = image.run(key_name=opts.key_pair,
security_groups=[master_group],
instance_type=master_type,
placement=opts.zone,
min_count=1,
max_count=1,
block_device_map=block_map,
user_data=user_data_content)
master_nodes = master_res.instances
print "Launched master in %s, regid = %s" % (zone, master_res.id)
# Give the instances descriptive names
for master in master_nodes:
master.add_tag(
key='Name',
value='{cn}-master-{iid}'.format(cn=cluster_name, iid=master.id))
for slave in slave_nodes:
slave.add_tag(
key='Name',
value='{cn}-slave-{iid}'.format(cn=cluster_name, iid=slave.id))
# Return all the instances
return (master_nodes, slave_nodes)
import boto
ec2conn = boto.connect_ec2()
from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping
map = BlockDeviceMapping()
sdb1 = BlockDeviceType()
sdc1 = BlockDeviceType()
sdd1 = BlockDeviceType()
sde1 = BlockDeviceType()
sdb1.ephemeral_name = 'ephemeral0'
sdc1.ephemeral_name = 'ephemeral1'
sdd1.ephemeral_name = 'ephemeral2'
sde1.ephemeral_name = 'ephemeral3'
map['/dev/sdb1'] = sdb1
map['/dev/sdc1'] = sdc1
map['/dev/sdd1'] = sdd1
map['/dev/sde1'] = sde1
img = ec2conn.get_all_images(image_ids=['ami-f61dfd9f'])[0]
img.run(key_name='id_bv-keypair',
instance_type='c1.xlarge',
block_device_map=map)
def test_create_launch_configuration_with_block_device_mappings():
block_device_mapping = BlockDeviceMapping()
ephemeral_drive = BlockDeviceType()
ephemeral_drive.ephemeral_name = "ephemeral0"
block_device_mapping["/dev/xvdb"] = ephemeral_drive
snapshot_drive = BlockDeviceType()
snapshot_drive.snapshot_id = "snap-1234abcd"
snapshot_drive.volume_type = "standard"
block_device_mapping["/dev/xvdp"] = snapshot_drive
ebs_drive = BlockDeviceType()
ebs_drive.volume_type = "io1"
ebs_drive.size = 100
ebs_drive.iops = 1000
ebs_drive.delete_on_termination = False
block_device_mapping["/dev/xvdh"] = ebs_drive
conn = boto.connect_autoscale(use_block_device_types=True)
config = LaunchConfiguration(
name="tester",
image_id="ami-abcd1234",
instance_type="m1.small",
key_name="the_keys",
security_groups=["default", "default2"],
user_data=b"This is some user_data",
instance_monitoring=True,
instance_profile_name="arn:aws:iam::{}:instance-profile/testing".format(
ACCOUNT_ID
),
spot_price=0.1,
block_device_mappings=[block_device_mapping],
)
conn.create_launch_configuration(config)
launch_config = conn.get_all_launch_configurations()[0]
launch_config.name.should.equal("tester")
launch_config.image_id.should.equal("ami-abcd1234")
launch_config.instance_type.should.equal("m1.small")
launch_config.key_name.should.equal("the_keys")
set(launch_config.security_groups).should.equal(set(["default", "default2"]))
launch_config.user_data.should.equal(b"This is some user_data")
launch_config.instance_monitoring.enabled.should.equal("true")
launch_config.instance_profile_name.should.equal(
"arn:aws:iam::{}:instance-profile/testing".format(ACCOUNT_ID)
)
launch_config.spot_price.should.equal(0.1)
len(launch_config.block_device_mappings).should.equal(3)
returned_mapping = launch_config.block_device_mappings
set(returned_mapping.keys()).should.equal(
set(["/dev/xvdb", "/dev/xvdp", "/dev/xvdh"])
)
returned_mapping["/dev/xvdh"].iops.should.equal(1000)
returned_mapping["/dev/xvdh"].size.should.equal(100)
returned_mapping["/dev/xvdh"].volume_type.should.equal("io1")
returned_mapping["/dev/xvdh"].delete_on_termination.should.be.false
returned_mapping["/dev/xvdp"].snapshot_id.should.equal("snap-1234abcd")
returned_mapping["/dev/xvdp"].volume_type.should.equal("standard")
returned_mapping["/dev/xvdb"].ephemeral_name.should.equal("ephemeral0")
def create_instance(name, config, region, key_name, ssh_key, instance_data,
deploypass, loaned_to, loan_bug, create_ami,
ignore_subnet_check, max_attempts):
"""Creates an AMI instance with the given name and config. The config must
specify things like ami id."""
conn = get_aws_connection(region)
# Make sure we don't request the same things twice
token = str(uuid.uuid4())[:16]
instance_data = instance_data.copy()
instance_data['name'] = name
instance_data['domain'] = config['domain']
instance_data['hostname'] = '{name}.{domain}'.format(
name=name, domain=config['domain'])
ami = conn.get_all_images(image_ids=[config["ami"]])[0]
bdm = None
if 'device_map' in config:
bdm = BlockDeviceMapping()
for device, device_info in config['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
# Overwrite root device size for HVM instances, since they cannot
# be resized online
if ami.virtualization_type == "hvm" and \
ami.root_device_name == device:
bd.size = ami.block_device_mapping[ami.root_device_name].size
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
bdm[device] = bd
interfaces = make_instance_interfaces(region, instance_data['hostname'],
ignore_subnet_check,
config.get('subnet_ids'),
config.get('security_group_ids', []),
config.get("use_public_ip"))
keep_going, attempt = True, 1
while keep_going:
try:
if 'user_data_file' in config:
user_data = open(config['user_data_file']).read()
else:
user_data = get_user_data_tmpl(config['type'])
if user_data:
user_data = user_data.format(
puppet_server=instance_data.get('default_puppet_server'),
fqdn=instance_data['hostname'],
hostname=instance_data['name'],
domain=instance_data['domain'],
dns_search_domain=config.get('dns_search_domain'),
password=deploypass,
moz_instance_type=config['type'],
region_dns_atom=get_region_dns_atom(region),
)
reservation = conn.run_instances(
image_id=config['ami'],
key_name=key_name,
instance_type=config['instance_type'],
block_device_map=bdm,
client_token=token,
disable_api_termination=config.get('disable_api_termination'),
user_data=user_data,
instance_profile_name=config.get('instance_profile_name'),
network_interfaces=interfaces,
)
break
except boto.exception.BotoServerError:
log.exception("Cannot start an instance")
time.sleep(10)
if max_attempts:
attempt += 1
keep_going = max_attempts >= attempt
instance = reservation.instances[0]
log.info("instance %s created, waiting to come up", instance)
# Wait for the instance to come up
wait_for_status(instance, "state", "running", "update")
instance.add_tag('Name', name)
instance.add_tag('FQDN', instance_data['hostname'])
instance.add_tag('created',
time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime()))
instance.add_tag('moz-type', config['type'])
if loaned_to:
instance.add_tag("moz-loaned-to", loaned_to)
if loan_bug:
instance.add_tag("moz-bug", loan_bug)
log.info("assimilating %s", instance)
instance.add_tag('moz-state', 'pending')
keep_going, attempt = True, 1
while keep_going:
try:
# Don't reboot if need to create ami
reboot = not create_ami
assimilate_instance(instance=instance,
config=config,
ssh_key=ssh_key,
instance_data=instance_data,
deploypass=deploypass,
reboot=reboot)
break
except NetworkError as e:
# it takes a while for the machine to start/reboot so the
# NetworkError exception is quite common, just log the error,
# without the full stack trace
log.warn(
"cannot connect; instance may still be starting %s (%s, %s) - %s,"
"retrying in %d sec ...", instance_data['hostname'],
instance.id, instance.private_ip_address, e, FAILURE_TIMEOUT)
time.sleep(FAILURE_TIMEOUT)
except:
# any other exception
log.warn(
"problem assimilating %s (%s, %s), retrying in "
"%d sec ...",
instance_data['hostname'],
instance.id,
instance.private_ip_address,
FAILURE_TIMEOUT,
exc_info=True)
time.sleep(FAILURE_TIMEOUT)
if max_attempts:
attempt += 1
keep_going = max_attempts >= attempt
instance.add_tag('moz-state', 'ready')
if create_ami:
ami_name = "spot-%s-%s" % (
config['type'], time.strftime("%Y-%m-%d-%H-%M", time.gmtime()))
log.info("Generating AMI %s", ami_name)
ami_cleanup(mount_point="/", distro=config["distro"])
root_bd = instance.block_device_mapping[instance.root_device_name]
volume = instance.connection.get_all_volumes(
volume_ids=[root_bd.volume_id])[0]
# The instance has to be stopped to flush EBS caches
instance.stop()
wait_for_status(instance, 'state', 'stopped', 'update')
ami = volume_to_ami(volume=volume,
ami_name=ami_name,
arch=instance.architecture,
virtualization_type=instance.virtualization_type,
kernel_id=instance.kernel,
root_device_name=instance.root_device_name,
tags=config["tags"])
log.info("AMI %s (%s) is ready", ami_name, ami.id)
log.warn("Terminating %s", instance)
instance.terminate()
def launch_cluster(conn, config):
key_name = config.get('ec2', 'key_name')
if not key_name:
exit('ERROR - key.name is not set zetten.props')
cur_nodes = get_active_cluster(conn, config)
if cur_nodes:
exit('ERROR - There are already instances running for {0} cluster'.format(config.cluster_name))
if isfile(config.hosts_path):
exit("ERROR - A hosts file already exists at {0}. Please delete before running launch again".format(config.hosts_path))
print "Launching {0} cluster".format(config.cluster_name)
vpc_id = None
if config.has_option('ec2', 'vpc_id'):
vpc_id = config.get('ec2', 'vpc_id')
subnet_id = None
if config.has_option('ec2', 'subnet_id'):
subnet_id = config.get('ec2', 'subnet_id')
security_group = get_or_make_group(conn, config.cluster_name + "-group", vpc_id)
if security_group.rules == []: # Group was just now created
if vpc_id is None:
security_group.authorize(src_group=security_group)
else:
security_group.authorize(ip_protocol='icmp', from_port=-1, to_port=-1, src_group=security_group)
security_group.authorize(ip_protocol='tcp', from_port=0, to_port=65535, src_group=security_group)
security_group.authorize(ip_protocol='udp', from_port=0, to_port=65535, src_group=security_group)
security_group.authorize('tcp', 22, 22, '0.0.0.0/0')
instance_d = {}
for (hostname, services) in config.nodes().items():
if 'worker' in services:
instance_type = config.get('ec2', 'worker_instance_type')
num_ephemeral = config.worker_num_ephemeral()
else:
instance_type = config.get('ec2', 'default_instance_type')
num_ephemeral = config.default_num_ephemeral()
if config.has_option('ec2', 'aws_ami'):
host_ami = config.get('ec2', 'aws_ami')
else:
host_ami = config.get_image_id(instance_type)
if not host_ami:
exit('ERROR - Image not found for instance type: '+instance_type)
bdm = BlockDeviceMapping()
bdm['/dev/sda1'] = BlockDeviceType(delete_on_termination=True)
for i in range(0, num_ephemeral):
bdt = BlockDeviceType()
bdt.ephemeral_name=config.ephemeral_root + str(i)
bdm[config.device_root + chr(ord('b') + i)] = bdt
try:
resv = conn.run_instances(key_name=key_name,
image_id=host_ami,
security_group_ids=[security_group.id],
instance_type=instance_type,
subnet_id=subnet_id,
min_count=1,
max_count=1,
block_device_map=bdm)
except EC2ResponseError as e:
ami_help = """PLEASE NOTE - If you have accepted the software terms for CentOS 7 and still get an error,
this could be due to CentOS releasing new images of CentOS 7. When this occurs, the old images
are no longer available to new users. If you think this is the case, go to the CentOS 7 product
page on AWS Marketplace at the URL below to find the latest AMI:
https://aws.amazon.com/marketplace/ordering?productId=b7ee8a69-ee97-4a49-9e68-afaee216db2e
On the product page, click 'Manual Launch' to find the latest AMI ID for your EC2 region.
This should be used to set the 'aws_ami' property in your zetten.props which will override
the default AMI IDs used by Zetten. After setting the 'aws_ami' property, run the launch
command again.
Also, let us know that this has occured by creating an issue on the Zetten's GitHub page
and we'll upgrade the defaults AMIs used by Zetten to be the latest CentOS images.
"""
exit("ERROR - Failed to launch EC2 instance due to exception below:\n\n{0}\n\n{1}".format(e, ami_help))
if len(resv.instances) != 1:
exit('ERROR - Failed to start {0} node'.format(hostname))
instance = resv.instances[0]
instance_d[hostname] = instance.id
print 'Launching {0} node using {1}'.format(hostname, host_ami)
while True:
time.sleep(5)
nodes = get_cluster(conn, config, ['running'])
num_actual = len(nodes)
num_expected = len(config.nodes())
if num_actual == num_expected:
# Tag instances and create hosts file
with open(config.hosts_path, 'w') as hosts_file:
for (hostname, services) in config.nodes().items():
instance = get_instance(nodes, instance_d[hostname])
instance.add_tag(key='Name', value='{cn}-{id}'.format(cn=config.cluster_name, id=hostname))
for tkey, tval in config.instance_tags().iteritems():
instance.add_tag(key=tkey, value=tval)
public_ip = ''
if instance.ip_address:
public_ip = instance.ip_address
private_ip = instance.private_ip_address
print >>hosts_file, hostname, private_ip, public_ip
print "All {0} nodes have started. Created hosts file at {1}".format(num_actual, config.hosts_path)
break
else:
print "{0} of {1} nodes have started. Waiting another 5 sec..".format(num_actual, num_expected)
def launch_cluster(conn, opts, cluster_name):
if opts.identity_file is None:
print >> stderr, "ERROR: Must provide an identity file (-i) for ssh connections."
sys.exit(1)
if opts.key_pair is None:
print >> stderr, "ERROR: Must provide a key pair name (-k) to use on instances."
sys.exit(1)
user_data_content = None
if opts.user_data:
with open(opts.user_data) as user_data_file:
user_data_content = user_data_file.read()
print "Setting up security groups..."
master_group = get_or_make_group(conn, cluster_name + "-master")
slave_group = get_or_make_group(conn, cluster_name + "-slaves")
if master_group.rules == []: # Group was just now created
master_group.authorize(src_group=master_group)
master_group.authorize(src_group=slave_group)
master_group.authorize('tcp', 22, 22, '0.0.0.0/0')
master_group.authorize('tcp', 8080, 8081, '0.0.0.0/0')
master_group.authorize('tcp', 18080, 18080, '0.0.0.0/0')
master_group.authorize('tcp', 19999, 19999, '0.0.0.0/0')
master_group.authorize('tcp', 50030, 50030, '0.0.0.0/0')
master_group.authorize('tcp', 50070, 50070, '0.0.0.0/0')
master_group.authorize('tcp', 60070, 60070, '0.0.0.0/0')
master_group.authorize('tcp', 4040, 4045, '0.0.0.0/0')
if opts.ganglia:
master_group.authorize('tcp', 5080, 5080, '0.0.0.0/0')
if slave_group.rules == []: # Group was just now created
slave_group.authorize(src_group=master_group)
slave_group.authorize(src_group=slave_group)
slave_group.authorize('tcp', 22, 22, '0.0.0.0/0')
slave_group.authorize('tcp', 8080, 8081, '0.0.0.0/0')
slave_group.authorize('tcp', 50060, 50060, '0.0.0.0/0')
slave_group.authorize('tcp', 50075, 50075, '0.0.0.0/0')
slave_group.authorize('tcp', 60060, 60060, '0.0.0.0/0')
slave_group.authorize('tcp', 60075, 60075, '0.0.0.0/0')
# Check if instances are already running in our groups
existing_masters, existing_slaves = get_existing_cluster(
conn, opts, cluster_name, die_on_error=False)
if existing_slaves or (existing_masters and not opts.use_existing_master):
print >> stderr, ("ERROR: There are already instances running in " +
"group %s or %s" %
(master_group.name, slave_group.name))
sys.exit(1)
# Figure out Spark AMI
if opts.ami is None:
opts.ami = get_spark_ami(opts)
print "Launching instances..."
try:
image = conn.get_all_images(image_ids=[opts.ami])[0]
except:
print >> stderr, "Could not find AMI " + opts.ami
sys.exit(1)
# Create block device mapping so that we can add an EBS volume if asked to
block_map = BlockDeviceMapping()
if opts.ebs_vol_size > 0:
device = EBSBlockDeviceType()
device.size = opts.ebs_vol_size
device.delete_on_termination = True
block_map["/dev/sdv"] = device
# AWS ignores the AMI-specified block device mapping for M3 (see SPARK-3342).
if opts.instance_type.startswith('m3.'):
for i in range(get_num_disks(opts.instance_type)):
dev = BlockDeviceType()
dev.ephemeral_name = 'ephemeral%d' % i
# The first ephemeral drive is /dev/sdb.
name = '/dev/sd' + string.letters[i + 1]
block_map[name] = dev
# Launch slaves
if opts.spot_price is not None:
# Launch spot instances with the requested price
print("Requesting %d slaves as spot instances with price $%.3f" %
(opts.slaves, opts.spot_price))
zones = get_zones(conn, opts)
num_zones = len(zones)
i = 0
my_req_ids = []
for zone in zones:
num_slaves_this_zone = get_partition(opts.slaves, num_zones, i)
slave_reqs = conn.request_spot_instances(
price=opts.spot_price,
image_id=opts.ami,
launch_group="launch-group-%s" % cluster_name,
placement=zone,
count=num_slaves_this_zone,
key_name=opts.key_pair,
security_groups=[slave_group],
instance_type=opts.instance_type,
block_device_map=block_map,
user_data=user_data_content)
my_req_ids += [req.id for req in slave_reqs]
i += 1
print "Waiting for spot instances to be granted..."
try:
while True:
time.sleep(10)
reqs = conn.get_all_spot_instance_requests()
id_to_req = {}
for r in reqs:
id_to_req[r.id] = r
active_instance_ids = []
for i in my_req_ids:
if i in id_to_req and id_to_req[i].state == "active":
active_instance_ids.append(id_to_req[i].instance_id)
if len(active_instance_ids) == opts.slaves:
print "All %d slaves granted" % opts.slaves
reservations = conn.get_all_instances(active_instance_ids)
slave_nodes = []
for r in reservations:
slave_nodes += r.instances
break
else:
print "%d of %d slaves granted, waiting longer" % (
len(active_instance_ids), opts.slaves)
except:
print "Canceling spot instance requests"
conn.cancel_spot_instance_requests(my_req_ids)
# Log a warning if any of these requests actually launched instances:
(master_nodes,
slave_nodes) = get_existing_cluster(conn,
opts,
cluster_name,
die_on_error=False)
running = len(master_nodes) + len(slave_nodes)
if running:
print >> stderr, ("WARNING: %d instances are still running" %
running)
sys.exit(0)
else:
# Launch non-spot instances
zones = get_zones(conn, opts)
num_zones = len(zones)
i = 0
slave_nodes = []
for zone in zones:
num_slaves_this_zone = get_partition(opts.slaves, num_zones, i)
if num_slaves_this_zone > 0:
slave_res = image.run(key_name=opts.key_pair,
security_groups=[slave_group],
instance_type=opts.instance_type,
placement=zone,
min_count=num_slaves_this_zone,
max_count=num_slaves_this_zone,
block_device_map=block_map,
user_data=user_data_content)
slave_nodes += slave_res.instances
print "Launched %d slaves in %s, regid = %s" % (
num_slaves_this_zone, zone, slave_res.id)
i += 1
# Launch or resume masters
if existing_masters:
print "Starting master..."
for inst in existing_masters:
if inst.state not in ["shutting-down", "terminated"]:
inst.start()
master_nodes = existing_masters
else:
master_type = opts.master_instance_type
if master_type == "":
master_type = opts.instance_type
if opts.zone == 'all':
opts.zone = random.choice(conn.get_all_zones()).name
master_res = image.run(key_name=opts.key_pair,
security_groups=[master_group],
instance_type=master_type,
placement=opts.zone,
min_count=1,
max_count=1,
block_device_map=block_map,
user_data=user_data_content)
master_nodes = master_res.instances
print "Launched master in %s, regid = %s" % (zone, master_res.id)
# Give the instances descriptive names
for master in master_nodes:
master.add_tag(key='Name',
value='{cn}-master-{iid}'.format(cn=cluster_name,
iid=master.id))
for slave in slave_nodes:
slave.add_tag(key='Name',
value='{cn}-slave-{iid}'.format(cn=cluster_name,
iid=slave.id))
# Return all the instances
return (master_nodes, slave_nodes)
def handleImages(self, action, clc):
if action == 'DescribeImages':
owner = self.get_argument('Owner', None)
if not owner:
owners = None
else:
owners = [owner]
return clc.get_all_images(owners)
elif action == 'DescribeImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
return clc.get_image_attribute(imageid, attribute)
elif action == 'ModifyImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
operation = self.get_argument('OperationType')
users = self.get_argument_list('UserId')
groups = self.get_argument_list('UserGroup')
return clc.modify_image_attribute(imageid, attribute, operation,
users, groups)
elif action == 'ResetImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
return clc.reset_image_attribute(imageid, attribute)
elif action == 'RegisterImage':
image_location = self.get_argument('ImageLocation', None)
name = self.get_argument('Name')
description = self.get_argument('Description', None)
description = base64.b64decode(description)
architecture = self.get_argument('Architecture', None)
kernel_id = self.get_argument('KernelId', None)
ramdisk_id = self.get_argument('RamdiskId', None)
root_dev_name = self.get_argument('RootDeviceName', None)
snapshot_id = self.get_argument('SnapshotId', None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument('BlockDeviceMapping.1.DeviceName',
None)
idx = 1
while mapping:
pre = 'BlockDeviceMapping.%d' % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument(
'%s.VirtualName' % pre, None)
if not (block_dev_type.ephemeral_name):
block_dev_type.no_device = \
(self.get_argument('%s.NoDevice' % pre, '') == 'true')
block_dev_type.snapshot_id = \
self.get_argument('%s.Ebs.SnapshotId' % pre, None)
block_dev_type.size = \
self.get_argument('%s.Ebs.VolumeSize' % pre, None)
block_dev_type.delete_on_termination = \
(self.get_argument('%s.DeleteOnTermination' % pre, '') == 'true')
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument(
'BlockDeviceMapping.%d.DeviceName' % idx, None)
if snapshot_id:
rootbdm = BlockDeviceType()
rootbdm.snapshot_id = snapshot_id
bdm['/dev/sda1'] = rootbdm
if len(bdm) == 0:
bdm = None
return clc.register_image(name, image_location, description,
architecture, kernel_id, ramdisk_id,
root_dev_name, bdm)
import os
os.environ["BOTO_CONFIG"]='C:/Dropbox/AWS/.boto'
import boto
from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping
#connecting to the AWS s3
#s3 = boto.connect_s3()
#bucket = s3.get_bucket('aws-publicdatasets')
#k = bucket.get_key("common-crawl/parse-output/valid_segments.txt")
#s = k.get_contents_as_string()
#list = s.split("\n")
#connecting to the AWS ec2
ec2 = boto.connect_ec2()
xvdb = BlockDeviceType()
xvdb.ephemeral_name='ephemeral0'
bdm = BlockDeviceMapping()
bdm['/dev/xvdb'] = xvdb
image_id = 'ami-fb8e9292'
image_name = 'Amazon Linux AMI 2014.03.1'
new_reservation = ec2.run_instances(image_id=image_id, key_name='ec2-key', security_groups=['web'], instance_type='m1.large', block_device_map=bdm)
instance = new_reservation.instances[0]
# Wait a minute or two while it boots
print "Spinning up instance for '%s' - %s. Waiting for it to boot up." % (image_id, image_name)
while instance.state != 'running':
print ".",
time.sleep(1)
instance.update()
print " "
def launch_cluster(conn, OPTS, cluster_name):
print "Setting up security groups..."
master_group = get_or_make_group(conn, cluster_name + "-master")
slave_group = get_or_make_group(conn, cluster_name + "-slaves")
ambari_group = get_or_make_group(conn, cluster_name + "-ambari")
if master_group.rules == []: # Group was just now created
master_group.authorize(src_group=master_group)
master_group.authorize(src_group=slave_group)
master_group.authorize(src_group=ambari_group)
# TODO: Currently Group is completely open
master_group.authorize('tcp', 0, 65535, '0.0.0.0/0')
if slave_group.rules == []: # Group was just now created
slave_group.authorize(src_group=master_group)
slave_group.authorize(src_group=slave_group)
slave_group.authorize(src_group=ambari_group)
# TODO: Currently Group is completely open
slave_group.authorize('tcp', 0, 65535, '0.0.0.0/0')
if ambari_group.rules == []: # Group was just now created
ambari_group.authorize(src_group=master_group)
ambari_group.authorize(src_group=slave_group)
ambari_group.authorize(src_group=ambari_group)
# TODO: Currently Group is completely open
ambari_group.authorize('tcp', 0, 65535, '0.0.0.0/0')
# Check if instances are already running in our groups
if OPTS.resume:
return get_existing_cluster(conn,
OPTS,
cluster_name,
die_on_error=False)
else:
active_nodes = get_existing_cluster(conn,
OPTS,
cluster_name,
die_on_error=False)
if any(active_nodes):
print >> stderr, (
"ERROR: There are already instances running in " +
"group %s or %s" % (master_group.name, slave_group.name))
sys.exit(1)
print "Launching instances..."
try:
image = conn.get_all_images(image_ids=[OPTS.ami])[0]
except:
print >> stderr, "Could not find AMI " + OPTS.ami
sys.exit(1)
# Create block device mapping so that we can add an EBS volume if asked to
block_map = BlockDeviceMapping()
device = BlockDeviceType()
device.ephemeral_name = 'ephemeral0'
device.delete_on_termination = True
block_map["/dev/sdv"] = device
# assume master and ambari hosts have the same instance type
master_type = OPTS.master_instance_type
if master_type == "":
master_type = OPTS.instance_type
# Launch slaves
if OPTS.spot_price != None:
# Launch spot instances with the requested price
num_spot_instances = OPTS.slaves + 2 # slaves, ambari host, master
print("Requesting %d slaves as spot instances with price $%.3f" %
(num_spot_instances, OPTS.spot_price))
zones = get_zones(conn, OPTS)
num_zones = len(zones)
i = 0
my_req_ids = []
ambari_req_ids = []
master_req_ids = []
for zone in zones:
num_slaves_this_zone = get_partition(OPTS.slaves, num_zones, i)
ambari_reqs = conn.request_spot_instances(
price=OPTS.spot_price,
image_id=OPTS.ami,
launch_group="launch-group-%s" % cluster_name,
placement=zone,
count=1,
key_name=OPTS.key_pair,
security_groups=[ambari_group],
instance_type=master_type,
block_device_map=block_map)
master_reqs = conn.request_spot_instances(
price=OPTS.spot_price,
image_id=OPTS.ami,
launch_group="launch-group-%s" % cluster_name,
placement=zone,
count=1,
key_name=OPTS.key_pair,
security_groups=[master_group],
instance_type=master_type,
block_device_map=block_map)
slave_reqs = conn.request_spot_instances(
price=OPTS.spot_price,
image_id=OPTS.ami,
launch_group="launch-group-%s" % cluster_name,
placement=zone,
count=num_slaves_this_zone,
key_name=OPTS.key_pair,
security_groups=[slave_group],
instance_type=OPTS.instance_type,
block_device_map=block_map)
my_req_ids += [req.id for req in slave_reqs]
ambari_req_ids += [req.id for req in ambari_reqs]
master_req_ids += [req.id for req in master_reqs]
i += 1
print "Waiting for spot instances to be granted..."
try:
while True:
time.sleep(10)
reqs = conn.get_all_spot_instance_requests()
id_to_req = {}
for r in reqs:
id_to_req[r.id] = r
active_instance_ids = []
ambari_instance_ids = []
master_instance_ids = []
for i in my_req_ids:
if i in id_to_req and id_to_req[i].state == "active":
active_instance_ids.append(
id_to_req[i].instance_id)
for i in master_req_ids:
if i in id_to_req and id_to_req[i].state == "active":
master_instance_ids.append(
id_to_req[i].instance_id)
for i in ambari_req_ids:
if i in id_to_req and id_to_req[i].state == "active":
ambari_instance_ids.append(
id_to_req[i].instance_id)
if len(active_instance_ids) == OPTS.slaves and len(
master_instance_ids) == 1 and len(
ambari_instance_ids) == 1:
print "All %d slaves, 1 master, 1 ambari host granted" % OPTS.slaves
slave_nodes = []
master_nodes = []
ambari_nodes = []
for r in conn.get_all_instances(active_instance_ids):
slave_nodes += r.instances
for r in conn.get_all_instances(master_instance_ids):
master_nodes += r.instances
for r in conn.get_all_instances(ambari_instance_ids):
ambari_nodes += r.instances
break
else:
print "%d of %d spot instance requests granted, waiting longer" % (
len(active_instance_ids), num_spot_instances)
except Exception as e:
print e
print "Canceling spot instance requests"
conn.cancel_spot_instance_requests(my_req_ids)
# Log a warning if any of these requests actually launched instances:
(master_nodes, slave_nodes,
ambari_nodes) = get_existing_cluster(conn,
OPTS,
cluster_name,
die_on_error=False)
running = len(master_nodes) + len(slave_nodes) + len(
ambari_nodes)
if running:
print >> stderr, (
"WARNING: %d instances are still running" % running)
sys.exit(0)
else:
# Launch non-spot instances
zones = get_zones(conn, OPTS)
num_zones = len(zones)
i = 0
slave_nodes = []
for zone in zones:
num_slaves_this_zone = get_partition(OPTS.slaves, num_zones, i)
if num_slaves_this_zone > 0:
slave_res = image.run(key_name=OPTS.key_pair,
security_groups=[slave_group],
instance_type=OPTS.instance_type,
placement=zone,
min_count=num_slaves_this_zone,
max_count=num_slaves_this_zone,
block_device_map=block_map)
slave_nodes += slave_res.instances
print "Launched %d slaves in %s, regid = %s" % (
num_slaves_this_zone, zone, slave_res.id)
i += 1
# Launch masters
if OPTS.zone == 'all':
OPTS.zone = random.choice(conn.get_all_zones()).name
master_res = image.run(key_name=OPTS.key_pair,
security_groups=[master_group],
instance_type=master_type,
placement=OPTS.zone,
min_count=1,
max_count=1,
block_device_map=block_map)
master_nodes = master_res.instances
print "Launched master in %s, regid = %s" % (zone, master_res.id)
ambari_type = master_type
if OPTS.zone == 'all':
OPTS.zone = random.choice(conn.get_all_zones()).name
ambari_res = image.run(key_name=OPTS.key_pair,
security_groups=[ambari_group],
instance_type=ambari_type,
placement=OPTS.zone,
min_count=1,
max_count=1,
block_device_map=block_map)
ambari_nodes = ambari_res.instances
print "Launched ambari in %s, regid = %s" % (zone, ambari_res.id)
# Return all the instances
return (master_nodes, slave_nodes, ambari_nodes)
def start_instances(conn, imageId, instanceType, num_instances, master=False):
cur_instances = Set([])
if os.path.isfile('musketeer_ec2_workers'):
for line in open('musketeer_ec2_workers').readlines():
fields = [x.strip() for x in line.split()]
cur_instances.add(fields[0])
if os.path.isfile('musketeer_ec2_master'):
for line in open('musketeer_ec2_master').readlines():
fields = [x.strip() for x in line.split()]
cur_instances.add(fields[0])
reservations = conn.get_all_reservations()
for reservation in reservations:
for instance in reservation.instances:
instance.update()
if instance.state == u'running':
cur_instances.add(instance.id)
if master is True:
instances_file = open("musketeer_ec2_master", "w")
else:
instances_file = open("musketeer_ec2_workers", "a")
instances = []
bdm = BlockDeviceMapping()
eph0 = BlockDeviceType()
eph0.ephemeral_name = 'ephemeral0'
bdm['/dev/sdb'] = eph0
spot_request = conn.request_spot_instances(
price="0.8",
image_id=imageId,
count=num_instances,
launch_group="us-east-1a",
availability_zone_group="us-east-1a",
key_name="Musketeer",
security_groups=["Musketeer"],
instance_type=instanceType,
block_device_map=bdm,
user_data="")
spot_request = conn.get_all_spot_instance_requests(
request_ids=[spot_request[0].id])[0]
# Wait until requests gets fulfilled
open_request = True
while open_request:
time.sleep(5)
open_request = False
spot_requests = conn.get_all_spot_instance_requests(
request_ids=[spot_request.id])
for spot_request in spot_requests:
print 'Spot request status: ', spot_request.state
if spot_request.state == 'open':
open_request = True
# Get info about instances
instance_num = 0
reservations = conn.get_all_reservations()
for reservation in reservations:
for instance in reservation.instances:
instance.update()
while instance.state == u'pending':
time.sleep(1)
instance.update()
if instance.state == u'running':
if instance.id not in cur_instances:
instance_num += 1
print 'Started instance %d: %s' % (instance_num, instance)
instances.append(instance.public_dns_name)
instances_file.write(instance.id + ' ' +
instance.public_dns_name + '\n')
else:
print 'Could not start instance: ', instance
instances_file.close()
return instances
def rebundle(reboot_if_needed=False):
"""
Rebundles the EC2 instance that is passed as the -H parameter
This script handles all aspects of the rebundling process and is (almost) fully automated.
Two things should be edited and provided before invoking it: AWS account information
and the desired size of the root volume for the new instance.
:rtype: bool
:return: If instance was successfully rebundled and an AMI ID was received,
return True.
False, otherwise.
"""
_check_fabric_version()
time_start = dt.datetime.utcnow()
print "Rebundling instance '%s'. Start time: %s" % (env.hosts[0], time_start)
_amazon_ec2_environment()
if boto:
# Select appropriate region:
availability_zone = run("curl --silent http://169.254.169.254/latest/meta-data/placement/availability-zone")
instance_region = availability_zone[:-1] # Truncate zone letter to get region name
ec2_conn = _get_ec2_conn(instance_region)
# hostname = env.hosts[0] # -H flag to fab command sets this variable so get only 1st hostname
instance_id = run("curl --silent http://169.254.169.254/latest/meta-data/instance-id")
# Get the size (in GB) of the root partition for the new image
vol_size = _get_root_vol_size(ec2_conn, instance_id)
# Handle reboot if required
if not _reboot(instance_id, reboot_if_needed):
return False # Indicates that rebundling was not completed and should be restarted
_clean() # Clean up the environment before rebundling
image_id = None
kernel_id = run("curl --silent http://169.254.169.254/latest/meta-data/kernel-id")
if ec2_conn and instance_id and availability_zone and kernel_id:
print "Rebundling instance with ID '%s' in region '%s'" % (instance_id, ec2_conn.region.name)
try:
# Need 2 volumes - one for image (rsync) and the other for the snapshot (see instance-to-ebs-ami.sh)
vol = ec2_conn.create_volume(vol_size, availability_zone)
vol2 = ec2_conn.create_volume(vol_size, availability_zone)
# TODO: wait until it becomes 'available'
print "Created 2 new volumes of size '%s' with IDs '%s' and '%s'" % (vol_size, vol.id, vol2.id)
except EC2ResponseError, e:
print(red("Error creating volume: %s" % e))
return False
if vol:
try:
# Attach newly created volumes to the instance
dev_id = '/dev/sdh'
if not _attach(ec2_conn, instance_id, vol.id, dev_id):
print(red("Error attaching volume '%s' to the instance. Aborting." % vol.id))
return False
dev_id = '/dev/sdj'
if not _attach(ec2_conn, instance_id, vol2.id, dev_id):
print(red("Error attaching volume '%s' to the instance. Aborting." % vol2.id))
return False
# Move the file system onto the new volume (with a help of a script)
url = os.path.join(REPO_ROOT_URL, "instance-to-ebs-ami.sh")
# with contextlib.nested(cd('/tmp'), settings(hide('stdout', 'stderr'))):
with cd('/tmp'):
if exists('/tmp/'+os.path.split(url)[1]):
sudo('rm /tmp/'+os.path.split(url)[1])
sudo('wget %s' % url)
sudo('chmod u+x /tmp/%s' % os.path.split(url)[1])
sudo('./%s' % os.path.split(url)[1])
# Detach the new volume
_detach(ec2_conn, instance_id, vol.id)
_detach(ec2_conn, instance_id, vol2.id)
answer = confirm("Would you like to terminate the instance used during rebundling?", default=False)
if answer:
ec2_conn.terminate_instances([instance_id])
# Create a snapshot of the new volume
commit_num = local('cd %s; hg tip | grep changeset | cut -d: -f2' % os.getcwd()).strip()
snap_id = _create_snapshot(ec2_conn, vol.id, "AMI: galaxy-cloudman (using mi-deployment at commit %s)" % commit_num)
# Register the snapshot of the new volume as a machine image (i.e., AMI)
arch = 'x86_64'
root_device_name = '/dev/sda1'
# Extra info on how EBS image registration is done: http://markmail.org/message/ofgkyecjktdhofgz
# http://www.elastician.com/2009/12/creating-ebs-backed-ami-from-s3-backed.html
# http://www.shlomoswidler.com/2010/01/creating-consistent-snapshots-of-live.html
ebs = BlockDeviceType()
ebs.snapshot_id = snap_id
ebs.delete_on_termination = True
ephemeral0_device_name = '/dev/sdb'
ephemeral0 = BlockDeviceType()
ephemeral0.ephemeral_name = 'ephemeral0'
ephemeral1_device_name = '/dev/sdc'
ephemeral1 = BlockDeviceType()
ephemeral1.ephemeral_name = 'ephemeral1'
# ephemeral2_device_name = '/dev/sdd' # Needed for instances w/ 3 ephemeral disks
# ephemeral2 = BlockDeviceType()
# ephemeral2.ephemeral_name = 'ephemeral2'
# ephemeral3_device_name = '/dev/sde' # Needed for instances w/ 4 ephemeral disks
# ephemeral3 = BlockDeviceType()
# ephemeral3.ephemeral_name = 'ephemeral3'
block_map = BlockDeviceMapping()
block_map[root_device_name] = ebs
block_map[ephemeral0_device_name] = ephemeral0
block_map[ephemeral1_device_name] = ephemeral1
print(yellow('galaxy-cloudman-%s' % time_start.strftime("%Y-%m-%d")))
name, desc = _get_image_name()
image_id = ec2_conn.register_image(name, description=desc, architecture=arch,
kernel_id=kernel_id, root_device_name=root_device_name, block_device_map=block_map)
answer = confirm("Volume with ID '%s' was created and used to make this AMI but is not longer needed. Would you like to delete it?" % vol.id)
if answer:
ec2_conn.delete_volume(vol.id)
print "Deleting the volume (%s) used for rsync only" % vol2.id
ec2_conn.delete_volume(vol2.id)
print(green("--------------------------"))
print(green("Finished creating new machine image. Image ID: '%s'" % (image_id)))
print(green("--------------------------"))
answer = confirm("Would you like to make this machine image public?", default=False)
if image_id and answer:
ec2_conn.modify_image_attribute(image_id, attribute='launchPermission', operation='add', groups=['all'])
except EC2ResponseError, e:
print(red("Error creating image: %s" % e))
return False
else:
print(red("Error creating new volume"))
return False
def AwsStartFromConfigSpec(args, configSpec):
if configSpec['Verbose']:
log.info("[AwsStartInstanceImpl]")
k, s = AwsLoadCreds(configSpec)
conn = AwsConnect(configSpec, k, s)
reservation = None
subnetId = None
securityGroups = None
securityGroupIds = None
shutdownBehavior = None
#NOTE: shutdown behavior can only be defined for EBS backed instances
if 'EbsInstance' in configSpec:
if configSpec['EbsInstance']:
shutdownBehavior = 'terminate'
else:
shutdownBehavior = 'stop'
if configSpec['InVpc']:
securityGroupIds = configSpec['SecurityGroups']
else:
securityGroups = configSpec['SecurityGroups']
deviceMap = None
ebsOptimized = False
if 'EBSRaid' in configSpec and configSpec["EBSRaid"]:
ebsOptimized = True
deviceMap = BlockDeviceMapping()
if configSpec['InstanceType'] == 'm3.2xlarge':
ep0 = BlockDeviceType()
ep0.ephemeral_name = 'ephemeral0'
deviceMap['/dev/xvdb'] = ep0
ep1 = BlockDeviceType()
ep1.ephemeral_name = 'ephemeral1'
deviceMap['/dev/xvdc'] = ep1
if configSpec['InstanceType'] == 'c3.4xlarge':
ep0 = BlockDeviceType()
ep0.ephemeral_name = 'ephemeral0'
deviceMap['/dev/xvdb'] = ep0
ep1 = BlockDeviceType()
ep1.ephemeral_name = 'ephemeral1'
deviceMap['/dev/xvdc'] = ep1
if 'EphemeralRaid' in configSpec and configSpec['EphemeralRaid']:
if configSpec['Verbose']:
log.info(
"AwsStartFromConfigSpec(): configSpec['EphemeralRaid']=%s (%s)"
% (configSpec['EphemeralRaid'], configSpec['InstanceType']))
deviceMap = BlockDeviceMapping()
if configSpec['InstanceType'] == 'm1.xlarge':
ep0 = BlockDeviceType()
ep0.ephemeral_name = 'ephemeral0'
deviceMap['/dev/sdb'] = ep0
ep1 = BlockDeviceType()
ep1.ephemeral_name = 'ephemeral1'
deviceMap['/dev/sdc'] = ep1
ep2 = BlockDeviceType()
ep2.ephemeral_name = 'ephemeral2'
deviceMap['/dev/sdd'] = ep2
ep3 = BlockDeviceType()
ep3.ephemeral_name = 'ephemeral3'
deviceMap['/dev/sde'] = ep3
elif configSpec['InstanceType'] == 'm1.large':
ep0 = BlockDeviceType()
ep0.ephemeral_name = 'ephemeral0'
deviceMap['/dev/sdb'] = ep0
ep1 = BlockDeviceType()
ep1.ephemeral_name = 'ephemeral1'
deviceMap['/dev/sdc'] = ep1
elif configSpec['InstanceType'] == 'c3.8xlarge':
ep0 = BlockDeviceType()
ep0.ephemeral_name = 'ephemeral0'
deviceMap['/dev/sdb'] = ep0
ep1 = BlockDeviceType()
ep1.ephemeral_name = 'ephemeral1'
deviceMap['/dev/sdc'] = ep1
elif configSpec['InstanceType'] == 'i2.2xlarge':
ep0 = BlockDeviceType()
ep0.ephemeral_name = 'ephemeral0'
deviceMap['/dev/xvdb'] = ep0
ep1 = BlockDeviceType()
ep1.ephemeral_name = 'ephemeral1'
deviceMap['/dev/xvdc'] = ep1
else:
log.error(
"AwsStartFromConfigSpec(): unexpected instance type => %s" %
configSpec['InstanceType'])
sys.exit(-1)
ami_image = configSpec['AmiId']
aws_az = args.az
subnet = configSpec['Subnets'][aws_az]['subnet']
try:
log.info("Launching in AZ: {0}".format(aws_az))
if args.dryrun == True:
log.warn("DRY RUN, NOT LAUNCHING....")
sys.exit()
else:
reservation = conn.run_instances(
ami_image,
placement=aws_az,
min_count=settings['total-nodes'],
max_count=settings['total-nodes'],
instance_initiated_shutdown_behavior=shutdownBehavior,
instance_type=configSpec['InstanceType'],
key_name=configSpec['SshKeys']['KeyPairName'],
subnet_id=subnet,
security_groups=securityGroups,
security_group_ids=securityGroupIds,
ebs_optimized=ebsOptimized,
block_device_map=deviceMap)
except boto.exception.EC2ResponseError as x:
log.error("Failed to start an AWS instance: %s" % x)
return
except Exception as e:
log.error("Got reservation error", e)
return
if reservation:
log.info('Waiting for VM instances to start...')
time.sleep(15)
instanceSetInfo = []
instanceIds = [
] # stores a list of all the active instanceids we can use to attach ebs volumes to
instanceIps = []
isFirst = True
firstNodeIp = None
for i, instance in enumerate(reservation.instances):
status = instance.update()
while not status == 'running':
log.info("Instance status: %s" % status)
if status == 'terminated':
sys.exit(-1)
time.sleep(4)
status = instance.update()
if configSpec['Verbose']:
log.info("Instance ID: %s" % instance.id)
log.info("Instance Private IP: %s" % instance.private_ip_address)
log.info("Instance Public DNS: %s" % instance.public_dns_name)
if isFirst:
firstNodeIp = instance.private_ip_address
info = {
'Id': instance.id,
'PrivateIp': instance.private_ip_address,
'PublicDnsName': instance.public_dns_name,
'FirstNode': isFirst
}
isFirst = False
instanceSetInfo.append(info)
instanceIds.append(instance.id)
instanceIps.append(instance.private_ip_address)
tags = configSpec['Tags']
for instance in reservation.instances:
conn.create_tags([instance.id], tags)
ips = ",".join(instanceIps)
print "IPS ARE"
print "-" * 10
print ips
print "-" * 10
print
for ip in instanceIps:
print ip
print
