def create_block_device_mapping(ami, device_map):
bdm = BlockDeviceMapping()
for device, device_info in device_map.items():
if ami.root_device_type == "instance-store" and \
not device_info.get("ephemeral_name"):
# EBS is not supported by S3-backed AMIs at request time
# EBS volumes can be attached when an instance is running
continue
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
if ami.root_device_name == device:
ami_size = ami.block_device_mapping[device].size
if ami.virtualization_type == "hvm":
# Overwrite root device size for HVM instances, since they
# cannot be resized online
bd.size = ami_size
elif device_info.get('size'):
# make sure that size is enough for this AMI
assert ami_size <= device_info['size'], \
"Instance root device size cannot be smaller than AMI " \
"root device"
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
if device_info.get("volume_type"):
bd.volume_type = device_info["volume_type"]
if device_info["volume_type"] == "io1" \
and device_info.get("iops"):
bd.iops = device_info["iops"]
bdm[device] = bd
return bdm
def create_block_device_mapping(ami, device_map):
bdm = BlockDeviceMapping()
for device, device_info in device_map.items():
if ami.root_device_type == "instance-store" and \
not device_info.get("ephemeral_name"):
# EBS is not supported by S3-backed AMIs at request time
# EBS volumes can be attached when an instance is running
continue
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
if ami.root_device_name == device:
ami_size = ami.block_device_mapping[device].size
if ami.virtualization_type == "hvm":
# Overwrite root device size for HVM instances, since they
# cannot be resized online
bd.size = ami_size
elif device_info.get('size'):
# make sure that size is enough for this AMI
assert ami_size <= device_info['size'], \
"Instance root device size cannot be smaller than AMI " \
"root device"
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
if device_info.get("volume_type"):
bd.volume_type = device_info["volume_type"]
if device_info["volume_type"] == "io1" \
and device_info.get("iops"):
bd.iops = device_info["iops"]
bdm[device] = bd
return bdm
def run_instance(conn, ami_id, key_name, instance_type,
sec_group, zone="us-east-1d", vol_size=None):
"""
@param connection: python boto connection
@param ami_id: AMI ID
@param key_name: SSH key name
@param instance_type: instance type, example 'm1.large'
@param sec_group: security group
@param zone: optional, defaults to 'us-east-1d'
@param vol_size: optional integer, if specified will change size of root volume to this size
@return boto.ec2.instance.Instance
"""
bdm = None
if vol_size:
# Create block device mapping info
dev_sda1 = BlockDeviceType()
dev_sda1.size = int(vol_size)
dev_sda1.delete_on_termination = True
bdm = BlockDeviceMapping()
bdm['/dev/sda1'] = dev_sda1
# Run instance
reservation = conn.run_instances(
ami_id,
key_name=key_name,
instance_type=instance_type,
placement=zone,
instance_initiated_shutdown_behavior="stop",
security_groups=[sec_group],
block_device_map=bdm)
return reservation.instances[0]
def get_block_device_map(bdmapping_json=None):
"""Parse block_device_mapping JSON and return a configured BlockDeviceMapping object
Mapping JSON structure...
{"/dev/sda":
{"snapshot_id": "snap-23E93E09", "volume_type": null, "delete_on_termination": true, "size": 1} }
"""
if bdmapping_json:
mapping = json.loads(bdmapping_json)
if mapping:
bdm = BlockDeviceMapping()
for key, val in mapping.items():
device = BlockDeviceType()
if val.get('virtual_name') is not None and val.get(
'virtual_name').startswith('ephemeral'):
device.ephemeral_name = val.get('virtual_name')
else:
device.volume_type = 'standard'
device.snapshot_id = val.get('snapshot_id') or None
device.size = val.get('size')
device.delete_on_termination = val.get(
'delete_on_termination', False)
bdm[key] = device
return bdm
return None
return None
def test_create_launch_configuration_with_block_device_mappings():
block_device_mapping = BlockDeviceMapping()
ephemeral_drive = BlockDeviceType()
ephemeral_drive.ephemeral_name = 'ephemeral0'
block_device_mapping['/dev/xvdb'] = ephemeral_drive
snapshot_drive = BlockDeviceType()
snapshot_drive.snapshot_id = "snap-1234abcd"
snapshot_drive.volume_type = "standard"
block_device_mapping['/dev/xvdp'] = snapshot_drive
ebs_drive = BlockDeviceType()
ebs_drive.volume_type = "io1"
ebs_drive.size = 100
ebs_drive.iops = 1000
ebs_drive.delete_on_termination = False
block_device_mapping['/dev/xvdh'] = ebs_drive
conn = boto.connect_autoscale(use_block_device_types=True)
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='m1.small',
key_name='the_keys',
security_groups=["default", "default2"],
user_data="This is some user_data",
instance_monitoring=True,
instance_profile_name='arn:aws:iam::123456789012:instance-profile/testing',
spot_price=0.1,
block_device_mappings=[block_device_mapping]
)
conn.create_launch_configuration(config)
launch_config = conn.get_all_launch_configurations()[0]
launch_config.name.should.equal('tester')
launch_config.image_id.should.equal('ami-abcd1234')
launch_config.instance_type.should.equal('m1.small')
launch_config.key_name.should.equal('the_keys')
set(launch_config.security_groups).should.equal(set(['default', 'default2']))
launch_config.user_data.should.equal("This is some user_data")
launch_config.instance_monitoring.enabled.should.equal('true')
launch_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/testing')
launch_config.spot_price.should.equal(0.1)
len(launch_config.block_device_mappings).should.equal(3)
returned_mapping = launch_config.block_device_mappings
set(returned_mapping.keys()).should.equal(set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh']))
returned_mapping['/dev/xvdh'].iops.should.equal(1000)
returned_mapping['/dev/xvdh'].size.should.equal(100)
returned_mapping['/dev/xvdh'].volume_type.should.equal("io1")
returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false
returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd")
returned_mapping['/dev/xvdp'].volume_type.should.equal("standard")
returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')
def _parse_block_device_mappings(user_input):
"""
Parse block device mappings per AWS CLI tools syntax (modified to add IOPS)
http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
Syntax:
/dev/xvd[a-z]=[snapshot-id|ephemeral]:[size in GB]:[Delete on Term]:[IOPS]
- Leave inapplicable fields blank
- Delete on Termination defaults to True
- IOPS limits are not validated
- EBS sizing is not validated
Mount an Ephemeral Drive:
/dev/xvdb1=ephemeral0
Mount multiple Ephemeral Drives:
/dev/xvdb1=ephemeral0,/dev/xvdb2=ephemeral1
Mount a Snapshot:
/dev/xvdp=snap-1234abcd
Mount a Snapshot to a 100GB drive:
/dev/xvdp=snap-1234abcd:100
Mount a Snapshot to a 100GB drive and do not delete on termination:
/dev/xvdp=snap-1234abcd:100:false
Mount a Fresh 100GB EBS device
/dev/xvdp=:100
Mount a Fresh 100GB EBS Device and do not delete on termination:
/dev/xvdp=:100:false
Mount a Fresh 100GB EBS Device with 1000 IOPS
/dev/xvdp=:100::1000
"""
block_device_map = BlockDeviceMapping()
mappings = user_input.split(",")
for mapping in mappings:
block_type = BlockDeviceType()
mount_point, drive_type, size, delete, iops = _parse_drive_mapping(
mapping)
if 'ephemeral' in drive_type:
block_type.ephemeral_name = drive_type
elif 'snap' in drive_type:
block_type.snapshot_id = drive_type
block_type.volume_type = "standard"
else:
block_type.volume_type = "standard"
block_type.size = size
block_type.delete_on_termination = delete
if iops:
block_type.iops = iops
block_type.volume_type = "io1"
block_device_map[mount_point] = block_type
return block_device_map
def _parse_block_device_mappings(user_input):
"""
Parse block device mappings per AWS CLI tools syntax (modified to add IOPS)
http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
Syntax:
/dev/xvd[a-z]=[snapshot-id|ephemeral]:[size in GB]:[Delete on Term]:[IOPS]
- Leave inapplicable fields blank
- Delete on Termination defaults to True
- IOPS limits are not validated
- EBS sizing is not validated
Mount an Ephemeral Drive:
/dev/xvdb1=ephemeral0
Mount multiple Ephemeral Drives:
/dev/xvdb1=ephemeral0,/dev/xvdb2=ephemeral1
Mount a Snapshot:
/dev/xvdp=snap-1234abcd
Mount a Snapshot to a 100GB drive:
/dev/xvdp=snap-1234abcd:100
Mount a Snapshot to a 100GB drive and do not delete on termination:
/dev/xvdp=snap-1234abcd:100:false
Mount a Fresh 100GB EBS device
/dev/xvdp=:100
Mount a Fresh 100GB EBS Device and do not delete on termination:
/dev/xvdp=:100:false
Mount a Fresh 100GB EBS Device with 1000 IOPS
/dev/xvdp=:100::1000
"""
block_device_map = BlockDeviceMapping()
mappings = user_input.split(",")
for mapping in mappings:
block_type = BlockDeviceType()
mount_point, drive_type, size, delete, iops = _parse_drive_mapping(mapping)
if 'ephemeral' in drive_type:
block_type.ephemeral_name = drive_type
elif 'snap' in drive_type:
block_type.snapshot_id = drive_type
block_type.volume_type = "standard"
else:
block_type.volume_type = "standard"
block_type.size = size
block_type.delete_on_termination = delete
if iops:
block_type.iops = iops
block_type.volume_type = "io1"
block_device_map[mount_point] = block_type
return block_device_map
def register(snapshot_id,
region,
arch,
size=None,
name=None,
desc=None,
pvm=False):
conn = utils.connect(region)
if None in (name, size):
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
virt = 'hvm'
kernel_id = None
device_base = '/dev/xvd'
ec2_arch = "x86_64" if arch == "amd64" else arch
if pvm:
kernel_id = utils.get_kernel(region, arch)
virt = 'paravirtual'
device_base = '/dev/sd'
name += '-pvm'
log.debug('creating block_device_map')
block_device_map = BlockDeviceMapping()
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
rootfs_device_name = device_base + 'a'
block_device_map[rootfs_device_name] = rootfs
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
ephemeral_device_name = device_base + 'b'
block_device_map[ephemeral_device_name] = ephemeral
log.debug('registering image - %s', name)
ami_id = conn.register_image(name=name,
description=desc,
kernel_id=kernel_id,
architecture=ec2_arch,
root_device_name=rootfs_device_name,
block_device_map=block_device_map,
virtualization_type=virt)
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id, name
def _process_block_device_mappings(self, launch_config, zone=None):
"""
Processes block device mapping information
and returns a Boto BlockDeviceMapping object. If new volumes
are requested (source is None and destination is VOLUME), they will be
created and the relevant volume ids included in the mapping.
"""
bdm = BlockDeviceMapping()
# Assign letters from f onwards
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
next_letter = iter(list(string.ascii_lowercase[6:]))
# assign ephemeral devices from 0 onwards
ephemeral_counter = 0
for device in launch_config.block_devices:
bd_type = BlockDeviceType()
if device.is_volume:
if device.is_root:
bdm['/dev/sda1'] = bd_type
else:
bdm['sd' + next(next_letter)] = bd_type
if isinstance(device.source, Snapshot):
bd_type.snapshot_id = device.source.id
elif isinstance(device.source, Volume):
bd_type.volume_id = device.source.id
elif isinstance(device.source, MachineImage):
# Not supported
pass
else:
# source is None, but destination is volume, therefore
# create a blank volume. If the Zone is None, this
# could fail since the volume and instance may be created
# in two different zones.
if not zone:
raise InvalidConfigurationException(
"A zone must be specified when launching with a"
" new blank volume block device mapping.")
new_vol = self.provider.block_store.volumes.create(
'',
device.size,
zone)
bd_type.volume_id = new_vol.id
bd_type.delete_on_terminate = device.delete_on_terminate
if device.size:
bd_type.size = device.size
else: # device is ephemeral
bd_type.ephemeral_name = 'ephemeral%s' % ephemeral_counter
return bdm
def _parse_block_device_mappings(self):
block_device_map = BlockDeviceMapping()
for mapping in self.block_device_mapping_dict:
block_type = BlockDeviceType()
mount_point = mapping.get('device_name')
if 'ephemeral' in mapping.get('virtual_name', ''):
block_type.ephemeral_name = mapping.get('virtual_name')
else:
block_type.volume_type = mapping.get('ebs._volume_type')
block_type.snapshot_id = mapping.get('ebs._snapshot_id')
block_type.delete_on_termination = mapping.get('ebs._delete_on_termination')
block_type.size = mapping.get('ebs._volume_size')
block_type.iops = mapping.get('ebs._iops')
block_device_map[mount_point] = block_type
return block_device_map
def _getBlockDeviceMapping(cls, instanceType, rootVolSize=50):
# determine number of ephemeral drives via cgcloud-lib
bdtKeys = ['', '/dev/xvdb', '/dev/xvdc', '/dev/xvdd']
bdm = BlockDeviceMapping()
# Change root volume size to allow for bigger Docker instances
root_vol = BlockDeviceType(delete_on_termination=True)
root_vol.size = rootVolSize
bdm["/dev/xvda"] = root_vol
# the first disk is already attached for us so start with 2nd.
for disk in xrange(1, instanceType.disks + 1):
bdm[bdtKeys[disk]] = BlockDeviceType(
ephemeral_name='ephemeral{}'.format(disk - 1)) # ephemeral counts start at 0
logger.debug('Device mapping: %s', bdm)
return bdm
def _getBlockDeviceMapping(cls, instanceType, rootVolSize=50):
# determine number of ephemeral drives via cgcloud-lib
bdtKeys = [''] + ['/dev/xvd{}'.format(c) for c in string.lowercase[1:]]
bdm = BlockDeviceMapping()
# Change root volume size to allow for bigger Docker instances
root_vol = BlockDeviceType(delete_on_termination=True)
root_vol.size = rootVolSize
bdm["/dev/xvda"] = root_vol
# the first disk is already attached for us so start with 2nd.
for disk in range(1, instanceType.disks + 1):
bdm[bdtKeys[disk]] = BlockDeviceType(
ephemeral_name='ephemeral{}'.format(disk - 1)) # ephemeral counts start at 0
logger.debug('Device mapping: %s', bdm)
return bdm
def _get_bmap(self, params):
bmap = BlockDeviceMapping()
for device in params['bmap']:
if not 'name' in device.keys():
self.logger.debug('bad device ' + str(device))
continue
dev = BlockDeviceType()
if 'size' in device.keys():
dev.size = device['size']
if 'delete_on_termination' in device.keys():
dev.delete_on_termination = device['delete_on_termination']
if 'ephemeral_name' in device.keys():
dev.ephemeral_name = device['ephemeral_name']
bmap[device['name']] = dev
return bmap
def _parse_block_device_mappings(self):
block_device_map = BlockDeviceMapping()
for mapping in self.block_device_mapping_dict:
block_type = BlockDeviceType()
mount_point = mapping.get('device_name')
if 'ephemeral' in mapping.get('virtual_name', ''):
block_type.ephemeral_name = mapping.get('virtual_name')
else:
block_type.volume_type = mapping.get('ebs._volume_type')
block_type.snapshot_id = mapping.get('ebs._snapshot_id')
block_type.delete_on_termination = mapping.get('ebs._delete_on_termination')
block_type.size = mapping.get('ebs._volume_size')
block_type.iops = mapping.get('ebs._iops')
block_device_map[mount_point] = block_type
return block_device_map
def _getBlockDeviceMapping(cls, instanceType):
# determine number of ephemeral drives via cgcloud-lib
bdtKeys = ['', '/dev/xvdb', '/dev/xvdc', '/dev/xvdd']
bdm = BlockDeviceMapping()
# Change root volume size to allow for bigger Docker instances
root_vol = BlockDeviceType()
root_vol.size = 50
bdm["/dev/xvda"] = root_vol
# the first disk is already attached for us so start with 2nd.
for disk in xrange(1, instanceType.disks + 1):
bdm[bdtKeys[disk]] = BlockDeviceType(
ephemeral_name='ephemeral{}'.format(disk - 1)) # ephemeral counts start at 0
logger.debug('Device mapping: %s', bdm)
return bdm
def _get_bmap(self, params):
bmap = BlockDeviceMapping()
for device in params['bmap']:
if not 'name' in device.keys():
self.logger.debug('bad device ' + str(device))
continue
dev = BlockDeviceType()
if 'size' in device.keys():
dev.size = device['size']
if 'delete_on_termination' in device.keys():
dev.delete_on_termination = device['delete_on_termination']
if 'ephemeral_name' in device.keys():
dev.ephemeral_name = device['ephemeral_name']
bmap[device['name']] = dev
return bmap
def startCluster(self, argv):
if len(argv) != 0:
print "ec2 startCluster"
sys.exit(-1)
regions = boto.ec2.regions()
regionInfo = '\n'.join(str(region).split(':')[1] for region in regions)
regionName = raw_input("select region:\n%s\n>>"%regionInfo)
region = boto.ec2.get_region(regionName)
conn = region.connect()
print "region connected successfully"
images = conn.get_all_images(owners='self')
imageInfo = '\n'.join(
str(image).split(':')[1] + ":" + image.name for image in images)
imageId = raw_input("enter imageId:\nself-created images:\n%s\n>>"%imageInfo)
instanceTypeInfo = ("m1.small, " "m1.large, " "m1.xlarge\n"
"c1.medium, " "c1.xlarge\n"
"m2.xlarge, " "m2.2xlarge, " "m2.4xlarge\n"
"cc1.4xlarge, " "t1.micro\n")
instanceType = raw_input("enter instanceType:\n%s\n>>"%instanceTypeInfo)
availZone = raw_input("enter placement[a,b,c]:\n>>")
availZone = regionName + availZone
diskSize = int(raw_input("enter disk size[G]:\n>>"))
rootDev = BlockDeviceType()
rootDev.name = 'root'
rootDev.size = diskSize
rootDev.delete_on_termination = True
instStorage = bool(raw_input("mount inst storage?\n>>"))
mapping = BlockDeviceMapping()
mapping['/dev/sda1'] = rootDev
if (instStorage == True):
eph0 = BlockDeviceType()
eph0.ephemeral_name = 'ephemeral0'
mapping['/dev/sdb'] = eph0
groups = conn.get_all_security_groups()
groupInfo = '\n'.join(str(group).split(':')[1] for group in groups)
group = raw_input("enter securityGroup:\n%s\n>>"%groupInfo)
keys = conn.get_all_key_pairs()
if len(keys) == 1:
key = keys[0].name
print 'using default key: ' + key
else:
keyInfo = '\n'.join(str(key).split(':')[1] for key in keys)
key = raw_input("enter key name:\n%s\n>>"%keyInfo)
numNodes = int(raw_input("number of nodes:\n>>"))
conn.run_instances(
imageId, min_count=numNodes, max_count=numNodes, placement=availZone,
security_groups = [group], instance_type=instanceType,
block_device_map=mapping, key_name=key)
def _process_block_device_mappings(self, launch_config, zone=None):
"""
Processes block device mapping information
and returns a Boto BlockDeviceMapping object. If new volumes
are requested (source is None and destination is VOLUME), they will be
created and the relevant volume ids included in the mapping.
"""
bdm = BlockDeviceMapping()
# Assign letters from f onwards
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
next_letter = iter(list(string.ascii_lowercase[6:]))
# assign ephemeral devices from 0 onwards
ephemeral_counter = 0
for device in launch_config.block_devices:
bd_type = BlockDeviceType()
if device.is_volume:
if device.is_root:
bdm['/dev/sda1'] = bd_type
else:
bdm['sd' + next(next_letter)] = bd_type
if isinstance(device.source, Snapshot):
bd_type.snapshot_id = device.source.id
elif isinstance(device.source, Volume):
bd_type.volume_id = device.source.id
elif isinstance(device.source, MachineImage):
# Not supported
pass
else:
# source is None, but destination is volume, therefore
# create a blank volume. If the Zone is None, this
# could fail since the volume and instance may be created
# in two different zones.
if not zone:
raise InvalidConfigurationException(
"A zone must be specified when launching with a"
" new blank volume block device mapping.")
new_vol = self.provider.block_store.volumes.create(
'', device.size, zone)
bd_type.volume_id = new_vol.id
bd_type.delete_on_terminate = device.delete_on_terminate
if device.size:
bd_type.size = device.size
else: # device is ephemeral
bd_type.ephemeral_name = 'ephemeral%s' % ephemeral_counter
return bdm
def _parse_block_device_mappings(self):
block_device_map = BlockDeviceMapping()
for mapping in self.block_device_mapping_dict:
block_type = BlockDeviceType()
mount_point = mapping.get("device_name")
if "ephemeral" in mapping.get("virtual_name", ""):
block_type.ephemeral_name = mapping.get("virtual_name")
else:
block_type.volume_type = mapping.get("ebs._volume_type")
block_type.snapshot_id = mapping.get("ebs._snapshot_id")
block_type.delete_on_termination = mapping.get(
"ebs._delete_on_termination")
block_type.size = mapping.get("ebs._volume_size")
block_type.iops = mapping.get("ebs._iops")
block_device_map[mount_point] = block_type
return block_device_map
def _getBlockDeviceMapping(cls, instanceType, rootVolSize=50):
# determine number of ephemeral drives via cgcloud-lib (actually this is moved into toil's lib
bdtKeys = [''] + ['/dev/xvd{}'.format(c) for c in string.ascii_lowercase[1:]]
bdm = BlockDeviceMapping()
# Change root volume size to allow for bigger Docker instances
root_vol = BlockDeviceType(delete_on_termination=True)
root_vol.size = rootVolSize
bdm["/dev/xvda"] = root_vol
# The first disk is already attached for us so start with 2nd.
# Disk count is weirdly a float in our instance database, so make it an int here.
for disk in range(1, int(instanceType.disks) + 1):
bdm[bdtKeys[disk]] = BlockDeviceType(
ephemeral_name='ephemeral{}'.format(disk - 1)) # ephemeral counts start at 0
logger.debug('Device mapping: %s', bdm)
return bdm
def _get_block_device_mapping(device_name, size):
""" Returns a block device mapping object for the specified device and size.
Block Device Mapping is used to associate a device on the VM with an EBS Volume.
parameters:
device_name -- The name of the device in the VM, such as /dev/sda1, /dev/sdb1. etc
size -- The amount of space to allocate for the EBS drive.
"""
block_device = BlockDeviceType()
block_device.size = size
bdm = BlockDeviceMapping()
bdm[device_name] = block_device
return bdm
def _get_block_device_mapping(device_name, size):
""" Returns a block device mapping object for the specified device and size.
Block Device Mapping is used to associate a device on the VM with an EBS Volume.
parameters:
device_name -- The name of the device in the VM, such as /dev/sda1, /dev/sdb1. etc
size -- The amount of space to allocate for the EBS drive.
"""
block_device = BlockDeviceType()
block_device.size = size
bdm = BlockDeviceMapping()
bdm[device_name] = block_device
return bdm
def register(snapshot_id, region, arch, size=None, name=None, desc=None, pvm=False):
conn = utils.connect(region)
if None in (name, size):
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
virt = 'hvm'
kernel_id = None
device_base = '/dev/xvd'
ec2_arch = "x86_64" if arch == "amd64" else arch
if pvm:
kernel_id = utils.get_kernel(region, arch)
virt = 'paravirtual'
device_base = '/dev/sd'
name += '-pvm'
log.debug('creating block_device_map')
block_device_map = BlockDeviceMapping()
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
rootfs_device_name = device_base + 'a'
block_device_map[rootfs_device_name] = rootfs
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
ephemeral_device_name = device_base + 'b'
block_device_map[ephemeral_device_name] = ephemeral
log.debug('registering image - %s', name)
ami_id = conn.register_image(
name=name,
description=desc,
kernel_id=kernel_id,
architecture=ec2_arch,
root_device_name=rootfs_device_name,
block_device_map=block_device_map,
virtualization_type=virt)
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id, name
def launch_ondemand_request(conn, request, tenant, job):
try:
mapping = BlockDeviceMapping()
sda1 = BlockDeviceType()
eph0 = BlockDeviceType()
eph1 = BlockDeviceType()
eph2 = BlockDeviceType()
eph3 = BlockDeviceType()
sda1.size = 10
eph0.ephemeral_name = 'ephemeral0'
eph1.ephemeral_name = 'ephemeral1'
eph2.ephemeral_name = 'ephemeral2'
eph3.ephemeral_name = 'ephemeral3'
mapping['/dev/sda1'] = sda1
mapping['/dev/sdb'] = eph0
mapping['/dev/sdc'] = eph1
mapping['/dev/sdd'] = eph2
mapping['/dev/sde'] = eph3
# issue a run_instances command for this request
res = conn.run_instances(min_count=request.count,
max_count=request.count,
key_name=tenant.key_pair,
image_id=request.ami,
security_group_ids=[tenant.security_group],
user_data=customise_cloudinit(tenant, job),
instance_type=request.instance_type,
subnet_id=tenant.subnet,
block_device_map=mapping)
my_req_ids = [req.id for req in res.instances]
# address = ""
for req in my_req_ids:
# tag each request
tag_requests(req, tenant.name, conn)
# update the database to include the new request
ProvisionerConfig().dbconn.execute(
("insert into instance_request (tenant, instance_type, " +
"price, job_runner_id, request_type, request_id, " +
"subnet) values ('%s', '%s', %s, %s, '%s', '%s', %s)") %
(tenant.db_id, request.instance.db_id,
request.instance.ondemand, job.id, "ondemand", req,
tenant.subnet_id))
# ProvisionerConfig().dbconn.commit()
return
except boto.exception.EC2ResponseError:
logger.exception("There was an error communicating with EC2.")
def _get_block_device_mapping(device_name, size, delete_on_terminate = False):
""" Returns a block device mapping object for the specified device and size.
Block Device Mapping is used to associate a device on the VM with an EBS Volume.
parameters:
device_name -- The name of the device in the VM, such as /dev/sda1, /dev/sdb1. etc
size -- The amount of space to allocate for the EBS drive.
delete_on_terminate -- Whether the volume should be deleted when the instance is terminated
"""
block_device = BlockDeviceType(delete_on_termination=delete_on_terminate)
block_device.size = size
bdm = BlockDeviceMapping()
bdm[device_name] = block_device
return bdm
def create_root_block_device(self):
# if the root volume size is not the same as the AMI default value:
if self.root_volume_size is not None:
# sda rather than xvda (for Windows)
dev_sda1 = BlockDeviceType()
dev_sda1.size = self.root_volume_size
dev_sda1.delete_on_termination = True
volume = BlockDeviceMapping()
# Check the OS type, if its windows we use sda, linux: xvda
images = self.ec2.get_all_images(image_ids=[self.node_obj.ami])
image = images[0]
if image.platform is None:
volume['/dev/xvda'] = dev_sda1
else:
volume['/dev/sda1'] = dev_sda1
self.volumes = volume
def start_remote(self):
self.get_bootstrap_kernel()
self.get_bootstrap_image()
# create EBS volume for /mnt/gentoo
device = BlockDeviceType()
device.size = self.settings["ec2/instance/device/size"]
device.delete_on_termination = True
mapping = BlockDeviceMapping()
self.root_device = "/dev/" + self.settings["ec2/instance/device/name"]
mapping[self.root_device] = device
# start bootstrapping instance
reservation = self.ec2.run_instances(
self.bootstrap_image.id,
kernel_id=self.bootstrap_kernel.id,
instance_type=self.settings["ec2/instance/type"],
security_groups=[self.name],
key_name=self.name,
block_device_map=mapping)
self.instance = reservation.instances[0]
sys.stdout.write("waiting for instance to come up ..")
while self.instance.update() != 'running':
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(5)
sys.stdout.write("\n")
time.sleep(120)
self.ssh_uri = "ec2-user@" + self.instance.public_dns_name
self.remote_upload_path = "/tmp"
# enable sudo without a tty
cmd = "sudo sed -i -e '/requiretty/d' /etc/sudoers"
cmd = ["ssh", "-t"] + self.ssh_options() + [self.ssh_uri, cmd]
ssh = subprocess.Popen(cmd)
ssh.wait()
self.run_script_at_remote("steps/remote/postboot")
def start_remote(self):
self.get_bootstrap_kernel()
self.get_bootstrap_image()
# create EBS volume for /mnt/gentoo
device = BlockDeviceType()
device.size = self.settings["ec2/instance/device/size"]
device.delete_on_termination = True
mapping = BlockDeviceMapping()
self.root_device = "/dev/" + self.settings["ec2/instance/device/name"]
mapping[self.root_device] = device
# start bootstrapping instance
reservation = self.ec2.run_instances(self.bootstrap_image.id,
kernel_id=self.bootstrap_kernel.id,
instance_type=self.settings["ec2/instance/type"],
security_groups=[self.name],
key_name=self.name,
block_device_map=mapping)
self.instance = reservation.instances[0]
sys.stdout.write("waiting for instance to come up ..")
while self.instance.update() != 'running':
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(5)
sys.stdout.write("\n")
time.sleep(120)
self.ssh_uri = "ec2-user@" + self.instance.public_dns_name
self.ssh_port = "22"
self.remote_upload_path = "/tmp"
# enable sudo without a tty
cmd = "sudo sed -i -e '/requiretty/d' /etc/sudoers"
cmd = ["ssh", "-t"] + self.ssh_options() + [self.ssh_uri, cmd]
ssh = subprocess.Popen(cmd)
ssh.wait()
self.run_script_at_remote("steps/remote/postboot")
def run_instance(conn, ami_id, key_name, instance_type, sec_group, zone, vol_size=None):
bdm = None
if vol_size:
# Create block device mapping info
dev_sda1 = BlockDeviceType()
dev_sda1.size = int(vol_size)
dev_sda1.delete_on_termination = True
bdm = BlockDeviceMapping()
bdm["/dev/sda1"] = dev_sda1
# Run instance
reservation = conn.run_instances(
ami_id,
key_name=key_name,
instance_type=instance_type,
placement=zone,
instance_initiated_shutdown_behavior="stop",
security_groups=[sec_group],
block_device_map=bdm,
)
return reservation.instances[0]
def register(snapshot_id, region, arch, size=None, name=None, desc=None):
conn = utils.connect(region)
if None in (name, size):
log.debug("getting snapshot - %s", snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
ec2_arch = "x86_64" if arch == "amd64" else arch
log.debug("creating block_device_map")
block_device_map = BlockDeviceMapping()
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
rootfs_device_name = "/dev/xvda"
block_device_map[rootfs_device_name] = rootfs
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = "ephemeral0"
ephemeral_device_name = "/dev/xvdb"
block_device_map[ephemeral_device_name] = ephemeral
log.debug("registering image - %s", name)
ami_id = conn.register_image(
name=name,
description=desc,
architecture=ec2_arch,
root_device_name=rootfs_device_name,
block_device_map=block_device_map,
virtualization_type="hvm",
)
log.info("registered image - %s %s %s", ami_id, name, region)
return ami_id, name
def register(snapshot_id, region, size=None, arch=None, name=None, desc=None):
conn = utils.connect(region)
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
desc = desc if desc else utils.parse_imagename(name)['url']
arch = arch if arch else utils.parse_imagename(name)['architecture']
kernel_id = utils.get_kernel(region, arch)
arch_ec2 = "x86_64" if arch == "amd64" else arch
log.debug('creating block_device_map')
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
block_device_map = BlockDeviceMapping()
block_device_map['/dev/sda1'] = rootfs
block_device_map['/dev/sda2'] = ephemeral
log.debug('registering image - %s', name)
ami_id = conn.register_image(
name=name,
description=desc,
architecture=arch_ec2,
kernel_id=kernel_id,
root_device_name="/dev/sda1",
block_device_map=block_device_map)
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id
def parse_block_device_args(self, block_device_maps_args):
block_device_map = BlockDeviceMapping()
for block_device_map_arg in block_device_maps_args:
parts = block_device_map_arg.split('=')
if len(parts) > 1:
device_name = parts[0]
block_dev_type = BlockDeviceType()
value_parts = parts[1].split(':')
if value_parts[0].startswith('snap'):
block_dev_type.snapshot_id = value_parts[0]
else:
if value_parts[0].startswith('ephemeral'):
block_dev_type.ephemeral_name = value_parts[0]
if len(value_parts) > 1:
try:
block_dev_type.size = int(value_parts[1])
except ValueError:
pass
if len(value_parts) > 2:
if value_parts[2] == 'true':
block_dev_type.delete_on_termination = True
block_device_map[device_name] = block_dev_type
return block_device_map
def get_block_device_map(bdmapping_json=None):
"""Parse block_device_mapping JSON and return a configured BlockDeviceMapping object
Mapping JSON structure...
{"/dev/sda":
{"snapshot_id": "snap-23E93E09", "volume_type": null, "delete_on_termination": true, "size": 1} }
"""
if bdmapping_json:
mapping = json.loads(bdmapping_json)
if mapping:
bdm = BlockDeviceMapping()
for key, val in mapping.items():
device = BlockDeviceType()
if val.get('virtual_name') is not None and val.get('virtual_name').startswith('ephemeral'):
device.ephemeral_name = val.get('virtual_name')
else:
device.volume_type = 'standard'
device.snapshot_id = val.get('snapshot_id') or None
device.size = val.get('size')
device.delete_on_termination = val.get('delete_on_termination', False)
bdm[key] = device
return bdm
return None
return None
def parse_block_device_args(self, block_device_maps_args):
block_device_map = BlockDeviceMapping()
for block_device_map_arg in block_device_maps_args:
parts = block_device_map_arg.split('=')
if len(parts) > 1:
device_name = parts[0]
block_dev_type = BlockDeviceType()
value_parts = parts[1].split(':')
if value_parts[0].startswith('snap'):
block_dev_type.snapshot_id = value_parts[0]
else:
if value_parts[0].startswith('ephemeral'):
block_dev_type.ephemeral_name = value_parts[0]
if len(value_parts) > 1:
try:
block_dev_type.size = int(value_parts[1])
except ValueError:
pass
if len(value_parts) > 2:
if value_parts[2] == 'true':
block_dev_type.delete_on_termination = True
block_device_map[device_name] = block_dev_type
return block_device_map
def run_instance(conn,
ami_id,
key_name,
instance_type,
sec_group,
zone="us-east-1d",
vol_size=None):
"""
@param connection: python boto connection
@param ami_id: AMI ID
@param key_name: SSH key name
@param instance_type: instance type, example 'm1.large'
@param sec_group: security group
@param zone: optional, defaults to 'us-east-1d'
@param vol_size: optional integer, if specified will change size of root volume to this size
@return boto.ec2.instance.Instance
"""
bdm = None
if vol_size:
# Create block device mapping info
dev_sda1 = BlockDeviceType()
dev_sda1.size = int(vol_size)
dev_sda1.delete_on_termination = True
bdm = BlockDeviceMapping()
bdm['/dev/sda1'] = dev_sda1
# Run instance
reservation = conn.run_instances(
ami_id,
key_name=key_name,
instance_type=instance_type,
placement=zone,
instance_initiated_shutdown_behavior="stop",
security_groups=[sec_group],
block_device_map=bdm)
return reservation.instances[0]
def register(snapshot_id, region, size=None, arch=None, name=None, desc=None):
conn = utils.connect(region)
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
desc = desc if desc else utils.parse_imagename(name)['url']
arch = arch if arch else utils.parse_imagename(name)['architecture']
kernel_id = utils.get_kernel(region, arch)
arch_ec2 = "x86_64" if arch == "amd64" else arch
log.debug('creating block_device_map')
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
block_device_map = BlockDeviceMapping()
block_device_map['/dev/sda1'] = rootfs
block_device_map['/dev/sda2'] = ephemeral
log.debug('registering image - %s', name)
ami_id = conn.register_image(name=name,
description=desc,
architecture=arch_ec2,
kernel_id=kernel_id,
root_device_name="/dev/sda1",
block_device_map=block_device_map)
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id
def create_instance(name, config, region, secrets, key_name, instance_data,
deploypass, loaned_to, loan_bug):
"""Creates an AMI instance with the given name and config. The config must
specify things like ami id."""
conn = get_connection(
region,
aws_access_key_id=secrets['aws_access_key_id'],
aws_secret_access_key=secrets['aws_secret_access_key'])
vpc = get_vpc(connection=conn,
aws_access_key_id=secrets['aws_access_key_id'],
aws_secret_access_key=secrets['aws_secret_access_key'])
# Make sure we don't request the same things twice
token = str(uuid.uuid4())[:16]
instance_data = instance_data.copy()
instance_data['name'] = name
instance_data['hostname'] = '{name}.{domain}'.format(
name=name, domain=config['domain'])
ami = conn.get_all_images(image_ids=[config["ami"]])[0]
bdm = None
if 'device_map' in config:
bdm = BlockDeviceMapping()
for device, device_info in config['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
# Overwrite root device size for HVM instances, since they cannot
# be resized online
if ami.virtualization_type == "hvm" and \
ami.root_device_name == device:
bd.size = ami.block_device_mapping[ami.root_device_name].size
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
bdm[device] = bd
ip_address = get_ip(instance_data['hostname'])
subnet_id = None
if ip_address:
s_id = get_subnet_id(vpc, ip_address)
if s_id in config['subnet_ids']:
if ip_available(conn, ip_address):
subnet_id = s_id
else:
log.warning("%s already assigned" % ip_address)
if not ip_address or not subnet_id:
ip_address = None
subnet_id = choice(config.get('subnet_ids'))
interface = NetworkInterfaceSpecification(
subnet_id=subnet_id,
private_ip_address=ip_address,
delete_on_termination=True,
groups=config.get('security_group_ids', []),
associate_public_ip_address=config.get("use_public_ip"))
interfaces = NetworkInterfaceCollection(interface)
while True:
try:
reservation = conn.run_instances(
image_id=config['ami'],
key_name=key_name,
instance_type=config['instance_type'],
block_device_map=bdm,
client_token=token,
disable_api_termination=bool(
config.get('disable_api_termination')),
network_interfaces=interfaces,
instance_profile_name=config.get("instance_profile_name"),
)
break
except boto.exception.BotoServerError:
log.exception("Cannot start an instance")
time.sleep(10)
instance = reservation.instances[0]
log.info("instance %s created, waiting to come up", instance)
# Wait for the instance to come up
while True:
try:
instance.update()
if instance.state == 'running':
break
except Exception:
log.warn("waiting for instance to come up, retrying in 10 sec...")
time.sleep(10)
instance.add_tag('Name', name)
instance.add_tag('FQDN', instance_data['hostname'])
instance.add_tag('created',
time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime()))
instance.add_tag('moz-type', config['type'])
if loaned_to:
instance.add_tag("moz-loaned-to", loaned_to)
if loan_bug:
instance.add_tag("moz-bug", loan_bug)
log.info("assimilating %s", instance)
instance.add_tag('moz-state', 'pending')
while True:
try:
assimilate(instance.private_ip_address, config, instance_data,
deploypass)
break
except:
log.warn("problem assimilating %s (%s), retrying in 10 sec ...",
instance_data['hostname'], instance.id)
time.sleep(10)
instance.add_tag('moz-state', 'ready')
def vm(install_list):
region = install_list["region"]
subnet_id = install_list["subnet_id"]
ami_id = install_list["ami_id"]
snap_id = install_list["snap_id"]
volume_capacity = install_list["volume_capacity"]
key_name = install_list["key_name"]
instance_type = install_list["instance_type"]
sg_id = install_list["sg_id"]
user_data = install_list["user_data"]
usage = install_list["usage"]
block_device_map = BlockDeviceMapping()
block_dev_type = BlockDeviceType()
block_dev_type.snapshot_id = snap_id
block_dev_type.delete_on_termination = False
block_dev_type.size = volume_capacity
block_device_map['/dev/sdb'] = block_dev_type
if region not in EXTRA_INFO:
aws_access_key_id = EXTRA_INFO["common"]["aws_access_key_id"]
aws_secret_access_key = EXTRA_INFO["common"]["aws_secret_access_key"]
name_key = EXTRA_INFO["common"]["name_key"]
else:
aws_access_key_id = EXTRA_INFO[region]["aws_access_key_id"]
aws_secret_access_key = EXTRA_INFO[region]["aws_secret_access_key"]
name_key = EXTRA_INFO[region]["name_key"]
ret = asset_hostname.get(region, usage)
hostname = ret["hostname"].split(".")[0]
user_data = user_data.replace("region=","region=%s" % region)
user_data = user_data.replace("hostname=","hostname=%s" % hostname)
user_data = user_data.replace("dns_vip=","dns_vip=%s" % \
DNS_INFO[region]["dns_vip"])
user_data = user_data.replace("ns_servers=","ns_servers='%s'" % \
" ".join(DNS_INFO[region]["ns_servers"]) )
network_interface = NetworkInterfaceSpecification(subnet_id=subnet_id, \
groups=[sg_id])
network_interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(\
network_interface)
conn = boto.ec2.connect_to_region(region, \
aws_access_key_id=aws_access_key_id, \
aws_secret_access_key=aws_secret_access_key)
reservation = conn.run_instances(ami_id,
key_name=key_name,
network_interfaces=network_interfaces,
instance_type=instance_type,
block_device_map=block_device_map,
# security_groups=[security_groups],
min_count=1,
max_count=1,
user_data=user_data,
)
instance = reservation.instances[0]
time_init = 0
time_total = 300
time_interval = 5
while time_init < time_total:
status = instance.update()
if status == 'running':
instance.add_tag(u"名称", hostname)
instance.add_tag("Name", hostname)
break
else:
time.sleep(time_interval)
time_init += time_interval
install_list["instance_id"] = str(instance).split(":")[-1]
install_list["placement"] = instance.placement
install_list["status"] = instance.update()
install_list["hostname"] = hostname
return install_list
def do_request_spot_instance(region, secrets, moz_instance_type, price, ami,
instance_config, cached_cert_dir, instance_type,
availability_zone, dryrun):
conn = aws_connect_to_region(region, secrets)
interface = get_available_interface(conn=conn,
moz_instance_type=moz_instance_type,
availability_zone=availability_zone)
if not interface:
raise RuntimeError("No free network interfaces left in %s" % region)
# TODO: check DNS
fqdn = interface.tags.get("FQDN")
if not fqdn:
raise RuntimeError("Skipping %s without FQDN" % interface)
log.debug("Spot request for %s (%s)", fqdn, price)
if dryrun:
log.info("Dry run. skipping")
return
spec = NetworkInterfaceSpecification(network_interface_id=interface.id)
nc = NetworkInterfaceCollection(spec)
ip = interface.private_ip_address
certs = get_puppet_certs(ip, secrets, cached_cert_dir)
user_data = """
FQDN="%(fqdn)s"
cd /var/lib/puppet/ssl || exit 1
%(certs)s
cd -
""" % dict(fqdn=fqdn, certs=certs)
if instance_config[region].get("lvm"):
user_data += """
mkdir -p /etc/lvm-init/
cat <<EOF > /etc/lvm-init/lvm-init.json
%s
EOF
/sbin/lvm-init
""" % json.dumps(instance_config[region])
bdm = BlockDeviceMapping()
for device, device_info in instance_config[region]['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
if ami.root_device_name == device:
ami_size = ami.block_device_mapping[device].size
if ami.virtualization_type == "hvm":
# Overwrite root device size for HVM instances, since they
# cannot be resized online
bd.size = ami_size
elif device_info.get('size'):
# make sure that size is enough for this AMI
assert ami_size <= device_info['size'], \
"Instance root device size cannot be smaller than AMI " \
"root device"
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
bdm[device] = bd
sir = conn.request_spot_instances(
price=str(price),
image_id=ami.id,
count=1,
instance_type=instance_type,
key_name=instance_config[region]["ssh_key"],
user_data=user_data,
block_device_map=bdm,
network_interfaces=nc,
instance_profile_name=instance_config[region].get(
"instance_profile_name"),
)
sir[0].add_tag("moz-type", moz_instance_type)
def handleRunInstances(self, action, clc, user_data_file, callback):
image_id = self.get_argument('ImageId')
min = self.get_argument('MinCount', '1')
max = self.get_argument('MaxCount', '1')
key = self.get_argument('KeyName', None)
groups = self.get_argument_list('SecurityGroup')
sec_group_ids = self.get_argument_list('SecurityGroupId')
if user_data_file:
user_data = user_data_file
else:
user_data = self.get_argument('UserData', "")
user_data = base64.b64decode(user_data)
addr_type = self.get_argument('AddressingType', None)
vm_type = self.get_argument('InstanceType', None)
placement = self.get_argument('Placement.AvailabilityZone', None)
placement_group = self.get_argument('Placement.GroupName', None)
tenancy = self.get_argument('Placement.Tenancy', None)
kernel = self.get_argument('KernelId', None)
ramdisk = self.get_argument('RamdiskId', None)
monitoring=False
if self.get_argument('Monitoring.Enabled', '') == 'true':
monitoring=True
subnet = self.get_argument('SubnetId', None);
private_ip = self.get_argument('PrivateIpAddress', None);
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None)
idx = 1
while mapping:
pre = 'BlockDeviceMapping.%d' % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument('%s.VirtualName' % pre, None)
if not(block_dev_type.ephemeral_name):
block_dev_type.no_device = \
(self.get_argument('%s.NoDevice' % pre, '') == 'true')
block_dev_type.snapshot_id = \
self.get_argument('%s.Ebs.SnapshotId' % pre, None)
block_dev_type.size = \
self.get_argument('%s.Ebs.VolumeSize' % pre, None)
block_dev_type.delete_on_termination = \
(self.get_argument('%s.Ebs.DeleteOnTermination' % pre, '') == 'true')
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument('BlockDeviceMapping.%d.DeviceName' % idx, None)
if len(bdm) == 0:
bdm = None
api_termination=False
if self.get_argument('DisableApiTermination', '') == 'true':
api_termination=True
instance_shutdown=False
if self.get_argument('InstanceInitiatedShutdownBehavior', '') == 'true':
instance_shutdown=True
token = self.get_argument('ClientToken', None);
addition_info = self.get_argument('AdditionInfo', None);
instance_profile_name = self.get_argument('IamInstanceProfile.Name', None);
instance_profile_arn = self.get_argument('IamInstanceProfile.Arn', None);
return clc.run_instances(image_id, min_count=min, max_count=max,
key_name=key, security_groups=groups,
user_data=user_data, addressing_type=addr_type,
instance_type=vm_type, placement=placement,
kernel_id=kernel, ramdisk_id=ramdisk,
monitoring_enabled=monitoring, subnet_id=subnet,
block_device_map=bdm,
disable_api_termination=api_termination,
instance_initiated_shutdown_behavior=instance_shutdown,
private_ip_address=private_ip,
placement_group=placement_group, client_token=token,
security_group_ids=sec_group_ids,
additional_info=addition_info,
instance_profile_name=instance_profile_name,
instance_profile_arn=instance_profile_arn,
tenancy=tenancy, callback=callback)
def handleImages(self, action, clc):
if action == 'DescribeImages':
owner = self.get_argument('Owner', None)
if not owner:
owners = None
else:
owners = [owner]
return clc.get_all_images(owners)
elif action == 'DescribeImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
return clc.get_image_attribute(imageid, attribute)
elif action == 'ModifyImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
operation = self.get_argument('OperationType')
users = self.get_argument_list('UserId')
groups = self.get_argument_list('UserGroup')
return clc.modify_image_attribute(imageid, attribute, operation,
users, groups)
elif action == 'ResetImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
return clc.reset_image_attribute(imageid, attribute)
elif action == 'RegisterImage':
image_location = self.get_argument('ImageLocation', None)
name = self.get_argument('Name')
description = self.get_argument('Description', None)
description = base64.b64decode(description)
architecture = self.get_argument('Architecture', None)
kernel_id = self.get_argument('KernelId', None)
ramdisk_id = self.get_argument('RamdiskId', None)
root_dev_name = self.get_argument('RootDeviceName', None)
snapshot_id = self.get_argument('SnapshotId', None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument('BlockDeviceMapping.1.DeviceName',
None)
idx = 1
while mapping:
pre = 'BlockDeviceMapping.%d' % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument(
'%s.VirtualName' % pre, None)
if not (block_dev_type.ephemeral_name):
block_dev_type.no_device = \
(self.get_argument('%s.NoDevice' % pre, '') == 'true')
block_dev_type.snapshot_id = \
self.get_argument('%s.Ebs.SnapshotId' % pre, None)
block_dev_type.size = \
self.get_argument('%s.Ebs.VolumeSize' % pre, None)
block_dev_type.delete_on_termination = \
(self.get_argument('%s.DeleteOnTermination' % pre, '') == 'true')
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument(
'BlockDeviceMapping.%d.DeviceName' % idx, None)
if snapshot_id:
rootbdm = BlockDeviceType()
rootbdm.snapshot_id = snapshot_id
bdm['/dev/sda1'] = rootbdm
if len(bdm) == 0:
bdm = None
return clc.register_image(name, image_location, description,
architecture, kernel_id, ramdisk_id,
root_dev_name, bdm)
def handleImages(self, action, clc, callback=None):
if action == "DescribeImages":
owner = self.get_argument("Owner", None)
if not owner:
owners = None
else:
owners = [owner]
filters = self.get_filter_args()
return clc.get_all_images(owners, filters, callback)
elif action == "DescribeImageAttribute":
imageid = self.get_argument("ImageId")
attribute = self.get_argument("Attribute")
return clc.get_image_attribute(imageid, attribute, callback)
elif action == "ModifyImageAttribute":
imageid = self.get_argument("ImageId")
attribute = self.get_argument("Attribute")
operation = self.get_argument("OperationType")
users = self.get_argument_list("UserId")
groups = self.get_argument_list("UserGroup")
return clc.modify_image_attribute(imageid, attribute, operation, users, groups, callback)
elif action == "ResetImageAttribute":
imageid = self.get_argument("ImageId")
attribute = self.get_argument("Attribute")
return clc.reset_image_attribute(imageid, attribute, callback)
elif action == "DeregisterImage":
image_id = self.get_argument("ImageId")
return clc.deregister_image(image_id, callback)
elif action == "RegisterImage":
image_location = self.get_argument("ImageLocation", None)
name = self.get_argument("Name")
description = self.get_argument("Description", None)
if description != None:
description = base64.b64decode(description)
architecture = self.get_argument("Architecture", None)
kernel_id = self.get_argument("KernelId", None)
ramdisk_id = self.get_argument("RamdiskId", None)
root_dev_name = self.get_argument("RootDeviceName", None)
snapshot_id = self.get_argument("SnapshotId", None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument("BlockDeviceMapping.1.DeviceName", None)
idx = 1
while mapping:
pre = "BlockDeviceMapping.%d" % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument("%s.VirtualName" % pre, None)
if not (block_dev_type.ephemeral_name):
block_dev_type.no_device = self.get_argument("%s.NoDevice" % pre, "") == "true"
block_dev_type.snapshot_id = self.get_argument("%s.Ebs.SnapshotId" % pre, None)
block_dev_type.size = self.get_argument("%s.Ebs.VolumeSize" % pre, None)
block_dev_type.delete_on_termination = (
self.get_argument("%s.Ebs.DeleteOnTermination" % pre, "") == "true"
)
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument("BlockDeviceMapping.%d.DeviceName" % idx, None)
if snapshot_id:
rootbdm = BlockDeviceType()
rootbdm.snapshot_id = snapshot_id
bdm["/dev/sda1"] = rootbdm
if len(bdm) == 0:
bdm = None
return clc.register_image(
name, image_location, description, architecture, kernel_id, ramdisk_id, root_dev_name, bdm, callback
)
def create_instance(name, config, region, key_name, ssh_key, instance_data,
deploypass, loaned_to, loan_bug, create_ami,
ignore_subnet_check, max_attempts):
"""Creates an AMI instance with the given name and config. The config must
specify things like ami id."""
conn = get_aws_connection(region)
# Make sure we don't request the same things twice
token = str(uuid.uuid4())[:16]
instance_data = instance_data.copy()
instance_data['name'] = name
instance_data['domain'] = config['domain']
instance_data['hostname'] = '{name}.{domain}'.format(
name=name, domain=config['domain'])
ami = conn.get_all_images(image_ids=[config["ami"]])[0]
bdm = None
if 'device_map' in config:
bdm = BlockDeviceMapping()
for device, device_info in config['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
# Overwrite root device size for HVM instances, since they cannot
# be resized online
if ami.virtualization_type == "hvm" and \
ami.root_device_name == device:
bd.size = ami.block_device_mapping[ami.root_device_name].size
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
if device_info.get("volume_type"):
bd.volume_type = device_info["volume_type"]
if device_info["volume_type"] == "io1" \
and device_info.get("iops"):
bd.iops = device_info["iops"]
bdm[device] = bd
interfaces = make_instance_interfaces(
region, instance_data['hostname'], ignore_subnet_check,
config.get('subnet_ids'), config.get('security_group_ids', []),
config.get("use_public_ip"))
keep_going, attempt = True, 1
while keep_going:
try:
puppet_master = pick_puppet_master(instance_data.get('puppet_masters'))
user_data = user_data_from_template(config['type'], {
"puppet_server": puppet_master,
"fqdn": instance_data['hostname'],
"hostname": instance_data['name'],
"domain": instance_data['domain'],
"dns_search_domain": config.get('dns_search_domain'),
"password": deploypass,
"moz_instance_type": config['type'],
"region_dns_atom": get_region_dns_atom(region)})
reservation = conn.run_instances(
image_id=config['ami'],
key_name=key_name,
instance_type=config['instance_type'],
block_device_map=bdm,
client_token=token,
disable_api_termination=config.get('disable_api_termination'),
user_data=user_data,
instance_profile_name=config.get('instance_profile_name'),
network_interfaces=interfaces,
)
break
except boto.exception.BotoServerError:
log.exception("Cannot start an instance")
time.sleep(10)
if max_attempts:
attempt += 1
keep_going = max_attempts >= attempt
instance = reservation.instances[0]
log.info("instance %s created, waiting to come up", instance)
# Wait for the instance to come up
wait_for_status(instance, "state", "running", "update")
instance.add_tag('Name', name)
instance.add_tag('FQDN', instance_data['hostname'])
instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z",
time.gmtime()))
instance.add_tag('moz-type', config['type'])
if loaned_to:
instance.add_tag("moz-loaned-to", loaned_to)
if loan_bug:
instance.add_tag("moz-bug", loan_bug)
log.info("assimilating %s", instance)
instance.add_tag('moz-state', 'pending')
keep_going, attempt = True, 1
while keep_going:
try:
# Don't reboot if need to create ami
reboot = not create_ami
assimilate_instance(instance=instance, config=config,
ssh_key=ssh_key, instance_data=instance_data,
deploypass=deploypass, reboot=reboot)
break
except NetworkError as e:
# it takes a while for the machine to start/reboot so the
# NetworkError exception is quite common, just log the error,
# without the full stack trace
log.warn("cannot connect; instance may still be starting %s (%s, %s) - %s,"
"retrying in %d sec ...", instance_data['hostname'], instance.id,
instance.private_ip_address, e, FAILURE_TIMEOUT)
time.sleep(FAILURE_TIMEOUT)
except:
# any other exception
log.warn("problem assimilating %s (%s, %s), retrying in "
"%d sec ...", instance_data['hostname'], instance.id,
instance.private_ip_address, FAILURE_TIMEOUT, exc_info=True)
time.sleep(FAILURE_TIMEOUT)
if max_attempts:
attempt += 1
keep_going = max_attempts >= attempt
instance.add_tag('moz-state', 'ready')
if create_ami:
ami_name = "spot-%s-%s" % (
config['type'], time.strftime("%Y-%m-%d-%H-%M", time.gmtime()))
log.info("Generating AMI %s", ami_name)
ami_cleanup(mount_point="/", distro=config["distro"])
root_bd = instance.block_device_mapping[instance.root_device_name]
volume = instance.connection.get_all_volumes(
volume_ids=[root_bd.volume_id])[0]
# The instance has to be stopped to flush EBS caches
instance.stop()
wait_for_status(instance, 'state', 'stopped', 'update')
ami = volume_to_ami(volume=volume, ami_name=ami_name,
arch=instance.architecture,
virtualization_type=instance.virtualization_type,
kernel_id=instance.kernel,
root_device_name=instance.root_device_name,
tags=config["tags"])
log.info("AMI %s (%s) is ready", ami_name, ami.id)
log.warn("Terminating %s", instance)
instance.terminate()
def launch_instance(self):
"""
Launch an instance for the profiler to run on.
"""
# Read in the cloudinit file
cfgfile = open('deploy.cfg')
user_data = cfgfile.read()
# Create a block mapping to get instance storage for the machine
mapping = BlockDeviceMapping()
sda1 = BlockDeviceType()
eph0 = BlockDeviceType()
eph1 = BlockDeviceType()
eph2 = BlockDeviceType()
eph3 = BlockDeviceType()
sda1.size = 10
eph0.ephemeral_name = 'ephemeral0'
eph1.ephemeral_name = 'ephemeral1'
eph2.ephemeral_name = 'ephemeral2'
eph3.ephemeral_name = 'ephemeral3'
mapping['/dev/sda1'] = sda1
mapping['/dev/sdb'] = eph0
mapping['/dev/sdc'] = eph1
mapping['/dev/sdd'] = eph2
mapping['/dev/sde'] = eph3
conn = boto.connect_ec2(self.access_key, self.secret_key)
inst_req = None
if self.ondemand:
inst_req= conn.run_instances(
min_count=1, max_count=1,
key_name=self.key_pair, image_id=self.ami,
security_group_ids=[self.security_group],
user_data=user_data,
instance_type=self.instance_type,
block_device_map=mapping)
else:
inst_req = conn.request_spot_instances(
price=self.bid, image_id=self.ami,
subnet_id=self.subnet, count=1,
key_name=self.key_pair,
security_group_ids=[self.security_group],
instance_type=self.type,
user_data=user_data,
block_device_map=mapping)
# Reorder the list of id's
my_req_ids = [req.id for req in inst_req]
# Tag the request
for req in my_req_ids:
self.tag_requests(req, 'profiler', conn)
self.address = ""
# Wait for the resource to become active
while self.address == "":
while True:
# Only check every 10 seconds
time.sleep(10)
# Get all spot requests
reqs = conn.get_all_spot_instance_requests()
id_to_req = {}
for r in reqs:
id_to_req[r.id] = r
active = 0
instance_ids = []
for i in my_req_ids:
try:
# Once it is active add it to the list
if id_to_req[i].state == "active":
active += 1
instance_ids.append(id_to_req[i].instance_id)
except Exception, e:
self.logger.debug("Hmm, error, skipping for now.")
if active == 1:
# Once it is granted we can leave this loop
self.logger.debug("All %d slaves granted" % 1)
reservations = conn.get_all_instances(instance_ids)
slave_nodes = []
for r in reservations:
slave_nodes += r.instances
break
self.address = slave_nodes[0].private_dns_name
def test_create_launch_configuration_with_block_device_mappings():
block_device_mapping = BlockDeviceMapping()
ephemeral_drive = BlockDeviceType()
ephemeral_drive.ephemeral_name = "ephemeral0"
block_device_mapping["/dev/xvdb"] = ephemeral_drive
snapshot_drive = BlockDeviceType()
snapshot_drive.snapshot_id = "snap-1234abcd"
snapshot_drive.volume_type = "standard"
block_device_mapping["/dev/xvdp"] = snapshot_drive
ebs_drive = BlockDeviceType()
ebs_drive.volume_type = "io1"
ebs_drive.size = 100
ebs_drive.iops = 1000
ebs_drive.delete_on_termination = False
block_device_mapping["/dev/xvdh"] = ebs_drive
conn = boto.connect_autoscale(use_block_device_types=True)
config = LaunchConfiguration(
name="tester",
image_id="ami-abcd1234",
instance_type="m1.small",
key_name="the_keys",
security_groups=["default", "default2"],
user_data=b"This is some user_data",
instance_monitoring=True,
instance_profile_name="arn:aws:iam::{}:instance-profile/testing".format(
ACCOUNT_ID
),
spot_price=0.1,
block_device_mappings=[block_device_mapping],
)
conn.create_launch_configuration(config)
launch_config = conn.get_all_launch_configurations()[0]
launch_config.name.should.equal("tester")
launch_config.image_id.should.equal("ami-abcd1234")
launch_config.instance_type.should.equal("m1.small")
launch_config.key_name.should.equal("the_keys")
set(launch_config.security_groups).should.equal(set(["default", "default2"]))
launch_config.user_data.should.equal(b"This is some user_data")
launch_config.instance_monitoring.enabled.should.equal("true")
launch_config.instance_profile_name.should.equal(
"arn:aws:iam::{}:instance-profile/testing".format(ACCOUNT_ID)
)
launch_config.spot_price.should.equal(0.1)
len(launch_config.block_device_mappings).should.equal(3)
returned_mapping = launch_config.block_device_mappings
set(returned_mapping.keys()).should.equal(
set(["/dev/xvdb", "/dev/xvdp", "/dev/xvdh"])
)
returned_mapping["/dev/xvdh"].iops.should.equal(1000)
returned_mapping["/dev/xvdh"].size.should.equal(100)
returned_mapping["/dev/xvdh"].volume_type.should.equal("io1")
returned_mapping["/dev/xvdh"].delete_on_termination.should.be.false
returned_mapping["/dev/xvdp"].snapshot_id.should.equal("snap-1234abcd")
returned_mapping["/dev/xvdp"].volume_type.should.equal("standard")
returned_mapping["/dev/xvdb"].ephemeral_name.should.equal("ephemeral0")
def main():
n_arg = len(sys.argv)
if 1 < n_arg:
if sys.argv[1] == 'unixbench':
u_data_model = 'unixbench/unixbench_ec2_userscript_model.dat'
elif sys.argv[1] == 'x264':
u_data_model = 'x264/x264_userscript_model.dat'
else:
print "usage: %s [unixbench|x264]" % sys.argv[0]
sys.exit(0)
# Block device storage size
if n_arg == 3:
dev_sda1 = BlockDeviceType()
dev_sda1.size = int(sys.argv[2])
bdm = BlockDeviceMapping()
bdm['/dev/sda1'] = dev_sda1
else:
bdm = None
# Lists of instance types to be benchmarked and already completed
instances = []
completed = []
try:
instances_dict = json.load(open("web/data/instances.json", "r"))
except IOError:
print "*** web/data/instances.json not found! Try ./update_instances.py ***"
sys.exit(1)
for k, v in instances_dict.iteritems():
if v['cloud'] == 'EC2':
if u_data_model == 'unixbench/unixbench_userscript_model.dat':
try:
instance_logs = Table(k)
instance_logs.describe()
completed.append(k)
except JSONResponseError:
instances.append(k)
elif u_data_model == 'x264/x264_userscript_model.dat':
instances.append(k)
else:
print 'Nothing to do'
sys.exit(0)
# Start all the benchmark at once will most likely exceeds the quota limit per user
# Better to execute the benchmark on a category to category basis
conn = boto.ec2.connect_to_region(region)
#instances = []
#completed = ['t1.micro_paravirtual']
num_instances = len(instances)
while 0 < len(instances):
for i in instances:
print '%s is waiting for launch' % i
for i in completed:
if i in instances:
instances.remove(i)
for i in instances:
# Generate an user-script per instance
userscript = ''
if u_data_model == 'unixbench/unixbench_userscript_model.dat':
userscript = "#!/bin/sh\nTRIAL=%d\nINSTANCE_NAME=%s\n"%(trial,i) + open(u_data_model,'r').read()
elif u_data_model == 'x264/x264_userscript_model.dat':
userscript = "#!/bin/sh\nTRIAL=%d\necho %s > /var/local/instance_name\n"%(trial,i) + open(u_data_model,'r').read()
u_data = base64.b64encode(userscript)
res = start_benchmark_instance(conn, i, u_data, bdm)
if res is not None and not res in completed:
completed.append(res)
sleep(5)
if len(completed) == num_instances:
break
else:
print '*** Cooling down...'
# 30 mins interval
sleep(60*30)
def do_create(self, ntry, params):
"""
Create stage of testing
@param ntry: number of try
@type ntry: int
@param params: list of testing parameters
@type params: list
"""
result = None
logging.debug(self.getName() + ': trying to create instance ' + params['iname'] + ', ntry ' + str(ntry))
ntry += 1
try:
bmap = BlockDeviceMapping()
for device in params['bmap']:
if not 'name' in device.keys():
logging.debug(self.getName() + ': bad device ' + str(device))
continue
d = BlockDeviceType()
if 'size' in device.keys():
d.size = device['size']
if 'delete_on_termination' in device.keys():
d.delete_on_termination = device['delete_on_termination']
if 'ephemeral_name' in device.keys():
d.ephemeral_name = device['ephemeral_name']
bmap[device['name']] = d
reg = boto.ec2.get_region(params['region'], aws_access_key_id=ec2_key, aws_secret_access_key=ec2_secret_key)
connection = reg.connect(aws_access_key_id=ec2_key, aws_secret_access_key=ec2_secret_key)
(ssh_key_name, ssh_key) = yamlconfig['ssh'][params['region']]
# all handled params to be put in here
boto_params = ['ami', 'subnet_id']
for param in boto_params:
params.setdefault(param)
reservation = connection.run_instances(
params['ami'],
instance_type=params['ec2name'],
key_name=ssh_key_name,
block_device_map=bmap,
subnet_id=params['subnet_id'],
user_data=params['userdata']
)
myinstance = reservation.instances[0]
count = 0
# Sometimes EC2 failes to return something meaningful without small timeout between run_instances() and update()
time.sleep(10)
while myinstance.update() == 'pending' and count < maxwait / 5:
# Waiting out instance to appear
logging.debug(params['iname'] + '... waiting...' + str(count))
time.sleep(5)
count += 1
connection.close()
instance_state = myinstance.update()
if instance_state == 'running':
# Instance appeared - scheduling 'setup' stage
myinstance.add_tag('Name', params['name'])
result = myinstance.__dict__
logging.info(self.getName() + ': created instance ' + params['iname'] + ', ' + result['id'] + ':' + result['public_dns_name'])
# packing creation results into params
params['id'] = result['id']
params['instance'] = result.copy()
mainq.put((0, 'setup', params))
return
elif instance_state == 'pending':
# maxwait seconds is enough to create an instance. If not -- EC2 failed.
logging.error('Error during instance creation: timeout in pending state')
result = myinstance.__dict__
if 'id' in result.keys():
# terminate stucked instance
params['id'] = result['id']
params['instance'] = result.copy()
mainq.put((0, 'terminate', params.copy()))
else:
# error occured
logging.error('Error during instance creation: ' + instance_state)
except boto.exception.EC2ResponseError, e:
# Boto errors should be handled according to their error Message - there are some well-known ones
logging.debug(self.getName() + ': got boto error during instance creation: %s' % e)
if str(e).find('<Code>InstanceLimitExceeded</Code>') != -1:
# InstanceLimit is temporary problem
logging.debug(self.getName() + ': got InstanceLimitExceeded - not increasing ntry')
ntry -= 1
elif str(e).find('<Code>InvalidParameterValue</Code>') != -1:
# InvalidParameterValue is really bad
logging.error(self.getName() + ': got boto error during instance creation: %s' % e)
# Try to speed up whole testing process, it's hardly possible to recover from this error
ntry += args.maxtries // 2 + 1
elif str(e).find('<Code>Unsupported</Code>') != -1:
# Unsupported
logging.error(self.getName() + ': got Unsupported - most likely the permanent error: %s' % e)
ntry += args.maxtries // 2 + 1
else:
logging.debug(self.getName() + ':' + traceback.format_exc())
def launch_instance(self, conn):
"""
Launch a cluster of the given name, by setting up its security groups,
and then starting new instances in them.
Returns a tuple of EC2 reservation objects for the master, slave
Fails if there already instances running in the cluster's groups.
"""
user_data = self.customise_cloudinit()
# Request a resource
mapping = BlockDeviceMapping()
sda1 = BlockDeviceType()
eph0 = BlockDeviceType()
eph1 = BlockDeviceType()
eph2 = BlockDeviceType()
eph3 = BlockDeviceType()
sda1.size = 10
eph0.ephemeral_name = 'ephemeral0'
eph1.ephemeral_name = 'ephemeral1'
eph2.ephemeral_name = 'ephemeral2'
eph3.ephemeral_name = 'ephemeral3'
mapping['/dev/sda1'] = sda1
mapping['/dev/sdb'] = eph0
mapping['/dev/sdc'] = eph1
mapping['/dev/sdd'] = eph2
mapping['/dev/sde'] = eph3
inst_req = conn.request_spot_instances(
price=self.price, image_id=self.ami,
subnet_id=self.subnet, count=1,
key_name=self.key_pair,
security_group_ids=[self.security_group],
instance_type=self.type,
user_data=user_data,
block_device_map=mapping)
# Reorder the list of id's
my_req_ids = [req.id for req in inst_req]
# Tag the request
for req in my_req_ids:
self.tag_requests(req, 'profiler', conn)
self.address = ""
# Wait for the resource to become active
while self.address == "":
while True:
# Only check every 10 seconds
time.sleep(10)
# Get all spot requests
reqs = conn.get_all_spot_instance_requests()
id_to_req = {}
for r in reqs:
id_to_req[r.id] = r
active = 0
instance_ids = []
for i in my_req_ids:
try:
# Once it is active add it to the list
if id_to_req[i].state == "active":
active += 1
instance_ids.append(id_to_req[i].instance_id)
except Exception, e:
self.logger.debug("Hmm, error, skipping for now.")
if active == 1:
# Once it is granted we can leave this loop
self.logger.debug("All %d slaves granted" % 1)
reservations = conn.get_all_instances(instance_ids)
slave_nodes = []
for r in reservations:
slave_nodes += r.instances
break
self.address = slave_nodes[0].private_dns_name
def launchInstance(region, rootsize, role, env, ami, key_name, instance_type,
placement, sec_group, iamrole, ebs, product, project, name,
chefversion):
'''Launch a single instance of the provided ami'''
conn = connectEC2(region)
mapping = BlockDeviceMapping()
root = BlockDeviceType()
root.size = rootsize
mapping['/dev/xvda'] = root
user_data = genUserData(role, env, chefversion)
sec_group_id = getSecurityGroupID(conn, sec_group)
subnet_id = getSubnetID(placement, env)
print Fore.YELLOW + 'Instance will be launched with following parameters.\n'
print Fore.GREEN + 'Region :\t\t' + region
print Fore.GREEN + 'Chef Role :\t\t' + role
print Fore.GREEN + 'Chef Env :\t\t' + env
print Fore.GREEN + 'AMI ID :\t\t' + ami
print Fore.GREEN + 'Keypair :\t\t' + key_name
print Fore.GREEN + 'Instance Type :\t\t' + instance_type
print Fore.GREEN + 'Placement :\t\t' + placement
print Fore.GREEN + 'Security Group :\t' + sec_group + ' (' + sec_group_id + ')'
print Fore.GREEN + 'Subnet ID :\t\t' + subnet_id
print Fore.GREEN + 'IAM Role :\t\t' + iamrole
if ebs:
print Fore.GREEN + 'EBS Volume Size: ' + str(ebs) + ' GB'
print Fore.YELLOW + '\nFollowing tags will be attached to the instance.\n'
if name:
print Fore.GREEN + 'Name :\t\t\t' + name.lower()
print Fore.GREEN + 'Product :\t\t' + product.lower()
print Fore.GREEN + 'Project :\t\t' + project.lower()
print Fore.GREEN + 'Environment :\t\t' + env.lower()
print Fore.YELLOW + '\nTrying to launch an EC2 instance ...'
reservation = conn.run_instances(
ami,
key_name=key_name,
user_data=user_data,
instance_type=instance_type,
placement=placement,
subnet_id=subnet_id,
block_device_map=mapping,
instance_initiated_shutdown_behavior='stop',
security_group_ids=['sg-********', sec_group_id],
instance_profile_name=iamrole,
dry_run=False)
print(Fore.YELLOW + '\nWaiting for instance to start ...')
instance = reservation.instances[0]
status = instance.update()
while status == 'pending':
time.sleep(5)
status = instance.update()
if status == 'running':
print(Fore.GREEN + 'New instance "' + instance.id +
'" accessible at ' + instance.private_ip_address)
print(Fore.YELLOW + 'Adding tags.')
if name:
instance.add_tag('Name', name.lower())
instance.add_tag('Product', product.lower())
instance.add_tag('Project', project.lower())
instance.add_tag('Environment', env.lower())
else:
print(Fore.YELLOW + 'Instance status: ' + status)
return
# If we got through the launching successfully, go ahead and create and attach a volume.
if ebs:
attachEBS(conn, instance, ebs)
print Fore.GREEN + '\nInstance ' + instance.id + ' launched successfuly.'
def launch_spot_request(conn, request, tenant, job):
try:
logger.debug("%s = %s. tenants vpc = %s" %
(request.zone, tenant.subnets[request.zone], tenant.vpc))
cost_aware_req = job.cost_aware
drafts_req = job.cost_aware
drafts_avg = job.cost_aware
mapping = BlockDeviceMapping()
sda1 = BlockDeviceType()
eph0 = BlockDeviceType()
eph1 = BlockDeviceType()
eph2 = BlockDeviceType()
eph3 = BlockDeviceType()
sda1.size = 10
eph0.ephemeral_name = 'ephemeral0'
eph1.ephemeral_name = 'ephemeral1'
eph2.ephemeral_name = 'ephemeral2'
eph3.ephemeral_name = 'ephemeral3'
mapping['/dev/sda1'] = sda1
mapping['/dev/sdb'] = eph0
mapping['/dev/sdc'] = eph1
mapping['/dev/sdd'] = eph2
mapping['/dev/sde'] = eph3
inst_req = None
inst_req = conn.request_spot_instances(
price=request.bid,
image_id=request.ami,
subnet_id=tenant.subnets[request.zone],
count=request.count,
key_name=tenant.key_pair,
security_group_ids=[tenant.security_group],
instance_type=request.instance_type,
user_data=customise_cloudinit(tenant, job),
block_device_map=mapping)
my_req_ids = [req.id for req in inst_req]
# address = ""
for req in my_req_ids:
insert_launch_stats(req, request, tenant)
# tag each request
tag_requests(req, tenant.name, conn)
ProvisionerConfig().dbconn.execute((
"insert into instance_request (tenant, instance_type, " +
"price, job_runner_id, request_type, request_id, " +
"subnet, cost_aware_ins, cost_aware_bid, cost_aware_subnet," +
" drafts_ins, drafts_bid, drafts_subnet, selected_avg_price,"
" cost_aware_avg_price, drafts_avg_price, drafts_avg_ins, " +
"drafts_avg_bid, drafts_avg_subnet, drafts_avg_avg_price) " +
"values ('%s', '%s', %s, %s, '%s', '%s', %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
) % (tenant.db_id, request.instance.db_id, request.price, job.id,
"spot", req, tenant.subnets_db_id[request.zone],
cost_aware_req.instance.db_id, cost_aware_req.bid,
tenant.subnets_db_id[cost_aware_req.zone],
drafts_req.instance.db_id, drafts_req.DrAFTS,
tenant.subnets_db_id[drafts_req.zone], request.AvgPrice,
cost_aware_req.AvgPrice, drafts_req.AvgPrice,
drafts_avg.instance.db_id, drafts_avg.DrAFTS,
tenant.subnets_db_id[drafts_avg.zone], drafts_avg.AvgPrice))
return my_req_ids
except boto.exception.EC2ResponseError:
logger.exception("There was an error communicating with EC2.")
def create_cluster(conn, args):
if args.identity_file is None:
print("ERROR: Must provide an identity file (-i) for ssh connections.",
file=stderr)
sys.exit(1)
if args.key_pair is None:
print("ERROR: Must provide a key pair name (-k) to use on instances.",
file=stderr)
sys.exit(1)
# make or get the security group.
security_group = get_or_make_group(conn, args.name, args.vpc_id)
# set the inbound permission rules
if len(security_group.rules) == 0:
if __name__ == '__main__':
if args.vpc_id is None:
security_group.authorize(src_group=security_group)
else:
security_group.authorize('tcp', 22, 22,
args.authorized_address)
security_group.authorize('tcp', 8888, 8888,
args.authorized_address)
security_group.authorize('tcp', 7000, 7000,
args.authorized_address)
security_group.authorize('tcp', 7001, 7001,
args.authorized_address)
security_group.authorize('tcp', 7199, 7199,
args.authorized_address)
security_group.authorize('tcp', 9042, 9042,
args.authorized_address)
security_group.authorize('tcp', 9160, 9160,
args.authorized_address)
else:
print("Security group already exists, skipping creation.")
instances = cluster_nodes(conn, args.name)
if any(instances):
additional_tags = {}
for i in instances:
i.add_tags(
dict(additional_tags,
Name="{cn}-node-{iid}".format(cn=args.name, iid=i.id)))
return instances
else:
print(
"Launching {m} instances for cluster...".format(m=args.node_count))
try:
image = conn.get_all_images(image_ids=args.ami)[0]
block_map = BlockDeviceMapping()
if args.ebs_vol_size > 0:
if args.instance_type.startswith('m3.'):
for i in range(get_num_disks(args.instance_type)):
device = BlockDeviceType()
device.ephemeral_name = "ephemeral%d" % i
name = "/dev/sd" + string.ascii_letters[i + 1]
block_map[name] = device
else:
device = EBSBlockDeviceType()
device.size = args.ebs_vol_size
device.volume_type = args.ebs_vol_type
device.delete_on_termination = True
key = "/dev/sd" + chr(ord('s') + 1)
block_map[key] = device
nodes = image.run(key_name=args.key_pair,
security_group_ids=[security_group.id],
instance_type="",
placement=args.zone,
min_count=args.node_count,
max_count=args.node_count,
block_device_map=block_map,
subnet_id=None,
placement_group=None,
user_data=None,
instance_initiated_shutdown_behavior="stop",
instance_profile_name=None)
print("Waiting for AWS to propagate instance metadata...")
time.sleep(15)
additional_tags = {}
for node in nodes.instances:
node.add_tags(
dict(additional_tags,
Name="{cn}-node-{iid}".format(cn=args.name,
iid=node.id)))
return nodes.instances
except Exception as e:
print("Caught exception: ", e)
print("ERROR: Could not find AMI " + args.ami, file=stderr)
sys.exit(1)
def handleRunInstances(self, action, clc, user_data_file, callback):
image_id = self.get_argument("ImageId")
min = self.get_argument("MinCount", "1")
max = self.get_argument("MaxCount", "1")
key = self.get_argument("KeyName", None)
groups = self.get_argument_list("SecurityGroup")
sec_group_ids = self.get_argument_list("SecurityGroupId")
if user_data_file:
user_data = user_data_file
else:
user_data = self.get_argument("UserData", "")
user_data = base64.b64decode(user_data)
addr_type = self.get_argument("AddressingType", None)
vm_type = self.get_argument("InstanceType", None)
placement = self.get_argument("Placement.AvailabilityZone", None)
placement_group = self.get_argument("Placement.GroupName", None)
tenancy = self.get_argument("Placement.Tenancy", None)
kernel = self.get_argument("KernelId", None)
ramdisk = self.get_argument("RamdiskId", None)
monitoring = False
if self.get_argument("Monitoring.Enabled", "") == "true":
monitoring = True
subnet = self.get_argument("SubnetId", None)
private_ip = self.get_argument("PrivateIpAddress", None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument("BlockDeviceMapping.1.DeviceName", None)
idx = 1
while mapping:
pre = "BlockDeviceMapping.%d" % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument("%s.VirtualName" % pre, None)
if not (block_dev_type.ephemeral_name):
block_dev_type.no_device = self.get_argument("%s.NoDevice" % pre, "") == "true"
block_dev_type.snapshot_id = self.get_argument("%s.Ebs.SnapshotId" % pre, None)
block_dev_type.size = self.get_argument("%s.Ebs.VolumeSize" % pre, None)
block_dev_type.delete_on_termination = (
self.get_argument("%s.Ebs.DeleteOnTermination" % pre, "") == "true"
)
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument("BlockDeviceMapping.%d.DeviceName" % idx, None)
if len(bdm) == 0:
bdm = None
api_termination = False
if self.get_argument("DisableApiTermination", "") == "true":
api_termination = True
instance_shutdown = False
if self.get_argument("InstanceInitiatedShutdownBehavior", "") == "true":
instance_shutdown = True
token = self.get_argument("ClientToken", None)
addition_info = self.get_argument("AdditionInfo", None)
instance_profile_name = self.get_argument("IamInstanceProfile.Name", None)
instance_profile_arn = self.get_argument("IamInstanceProfile.Arn", None)
return clc.run_instances(
image_id,
min_count=min,
max_count=max,
key_name=key,
security_groups=groups,
user_data=user_data,
addressing_type=addr_type,
instance_type=vm_type,
placement=placement,
kernel_id=kernel,
ramdisk_id=ramdisk,
monitoring_enabled=monitoring,
subnet_id=subnet,
block_device_map=bdm,
disable_api_termination=api_termination,
instance_initiated_shutdown_behavior=instance_shutdown,
private_ip_address=private_ip,
placement_group=placement_group,
client_token=token,
security_group_ids=sec_group_ids,
additional_info=addition_info,
instance_profile_name=instance_profile_name,
instance_profile_arn=instance_profile_arn,
tenancy=tenancy,
callback=callback,
)
def create_instance(name, config, region, key_name, ssh_key, instance_data,
deploypass, loaned_to, loan_bug, create_ami,
ignore_subnet_check, max_attempts):
"""Creates an AMI instance with the given name and config. The config must
specify things like ami id."""
conn = get_aws_connection(region)
# Make sure we don't request the same things twice
token = str(uuid.uuid4())[:16]
instance_data = instance_data.copy()
instance_data['name'] = name
instance_data['domain'] = config['domain']
instance_data['hostname'] = '{name}.{domain}'.format(
name=name, domain=config['domain'])
ami = conn.get_all_images(image_ids=[config["ami"]])[0]
bdm = None
if 'device_map' in config:
bdm = BlockDeviceMapping()
for device, device_info in config['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
# Overwrite root device size for HVM instances, since they cannot
# be resized online
if ami.virtualization_type == "hvm" and \
ami.root_device_name == device:
bd.size = ami.block_device_mapping[ami.root_device_name].size
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
bdm[device] = bd
interfaces = make_instance_interfaces(region, instance_data['hostname'],
ignore_subnet_check,
config.get('subnet_ids'),
config.get('security_group_ids', []),
config.get("use_public_ip"))
keep_going, attempt = True, 1
while keep_going:
try:
if 'user_data_file' in config:
user_data = open(config['user_data_file']).read()
else:
user_data = get_user_data_tmpl(config['type'])
if user_data:
user_data = user_data.format(
puppet_server=instance_data.get('default_puppet_server'),
fqdn=instance_data['hostname'],
hostname=instance_data['name'],
domain=instance_data['domain'],
dns_search_domain=config.get('dns_search_domain'),
password=deploypass,
moz_instance_type=config['type'],
region_dns_atom=get_region_dns_atom(region),
)
reservation = conn.run_instances(
image_id=config['ami'],
key_name=key_name,
instance_type=config['instance_type'],
block_device_map=bdm,
client_token=token,
disable_api_termination=config.get('disable_api_termination'),
user_data=user_data,
instance_profile_name=config.get('instance_profile_name'),
network_interfaces=interfaces,
)
break
except boto.exception.BotoServerError:
log.exception("Cannot start an instance")
time.sleep(10)
if max_attempts:
attempt += 1
keep_going = max_attempts >= attempt
instance = reservation.instances[0]
log.info("instance %s created, waiting to come up", instance)
# Wait for the instance to come up
wait_for_status(instance, "state", "running", "update")
instance.add_tag('Name', name)
instance.add_tag('FQDN', instance_data['hostname'])
instance.add_tag('created',
time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime()))
instance.add_tag('moz-type', config['type'])
if loaned_to:
instance.add_tag("moz-loaned-to", loaned_to)
if loan_bug:
instance.add_tag("moz-bug", loan_bug)
log.info("assimilating %s", instance)
instance.add_tag('moz-state', 'pending')
keep_going, attempt = True, 1
while keep_going:
try:
# Don't reboot if need to create ami
reboot = not create_ami
assimilate_instance(instance=instance,
config=config,
ssh_key=ssh_key,
instance_data=instance_data,
deploypass=deploypass,
reboot=reboot)
break
except NetworkError as e:
# it takes a while for the machine to start/reboot so the
# NetworkError exception is quite common, just log the error,
# without the full stack trace
log.warn(
"cannot connect; instance may still be starting %s (%s, %s) - %s,"
"retrying in %d sec ...", instance_data['hostname'],
instance.id, instance.private_ip_address, e, FAILURE_TIMEOUT)
time.sleep(FAILURE_TIMEOUT)
except:
# any other exception
log.warn(
"problem assimilating %s (%s, %s), retrying in "
"%d sec ...",
instance_data['hostname'],
instance.id,
instance.private_ip_address,
FAILURE_TIMEOUT,
exc_info=True)
time.sleep(FAILURE_TIMEOUT)
if max_attempts:
attempt += 1
keep_going = max_attempts >= attempt
instance.add_tag('moz-state', 'ready')
if create_ami:
ami_name = "spot-%s-%s" % (
config['type'], time.strftime("%Y-%m-%d-%H-%M", time.gmtime()))
log.info("Generating AMI %s", ami_name)
ami_cleanup(mount_point="/", distro=config["distro"])
root_bd = instance.block_device_mapping[instance.root_device_name]
volume = instance.connection.get_all_volumes(
volume_ids=[root_bd.volume_id])[0]
# The instance has to be stopped to flush EBS caches
instance.stop()
wait_for_status(instance, 'state', 'stopped', 'update')
ami = volume_to_ami(volume=volume,
ami_name=ami_name,
arch=instance.architecture,
virtualization_type=instance.virtualization_type,
kernel_id=instance.kernel,
root_device_name=instance.root_device_name,
tags=config["tags"])
log.info("AMI %s (%s) is ready", ami_name, ami.id)
log.warn("Terminating %s", instance)
instance.terminate()
def run_instances_for_role(role, count=1, owner=None):
"""Run instances for the role, returns the reservation which contains the
new instances
role (Role)
instance of a subclass of Role which contains details about the
ec2 instances to start
count (int)
owner (str)
"""
dev_sda1 = BlockDeviceType()
dev_sda1.size = role.instance_root_volume_size
bdm = BlockDeviceMapping()
bdm['/dev/sda1'] = dev_sda1
reservation = ec2_conn.run_instances(
default_ami[ec2_conn.region.name],
min_count=count,
max_count=count,
user_data=userdata.generate_userdata_base64(role.env, env.user),
key_name='ec2-key',
security_groups=role.ec2_security_groups,
instance_type=role.instance_type,
block_device_map=bdm,
placement=role.placement
)
# Give AWS a few seconds to get things synced on their side
sleep(10)
# Drop into a loop and wait for all the instances to listen for connections
for instance in reservation.instances:
while instance.update() != 'running':
sleep(20)
while instance.public_dns_name is None:
sleep(20)
instance.update()
while True:
s = socket()
try:
s.connect((instance.public_dns_name, 22))
except:
print '%s not yet accepting connections, waiting...' \
% instance.public_dns_name
sleep(10)
else:
break
dns = DynDNS()
for instance in reservation.instances:
shortname = '%s-%s-%s' % (role.env, role.role_for_name_tag, instance.id)
dns.add_cname(instance.public_dns_name, shortname)
sleep(5) # wait for dynect to catch up
instance.dns_name = shortname + '.domain.com'
instance.add_tag('Name', shortname)
instance.add_tag('role', role.role_name)
instance.add_tag('env', role.env)
if owner:
instance.add_tag('owner', owner)
if role.role_name == 'development':
print 'Create RDS instance for this development host...'
rds_instance = rds_conn.restore_dbinstance_from_dbsnapshot(
'development-rds-master',
shortname,
'db.m1.medium')
while rds_instance.update() != 'available':
print 'RDS instance not ready yet, sleeping 30 seconds.'
sleep(30)
with settings(host_string=instance.dns_name):
sudo('yum -y install mysql')
% rds_instance.endpoint[0])
def test_create_launch_configuration_with_block_device_mappings():
block_device_mapping = BlockDeviceMapping()
ephemeral_drive = BlockDeviceType()
ephemeral_drive.ephemeral_name = 'ephemeral0'
block_device_mapping['/dev/xvdb'] = ephemeral_drive
snapshot_drive = BlockDeviceType()
snapshot_drive.snapshot_id = "snap-1234abcd"
snapshot_drive.volume_type = "standard"
block_device_mapping['/dev/xvdp'] = snapshot_drive
ebs_drive = BlockDeviceType()
ebs_drive.volume_type = "io1"
ebs_drive.size = 100
ebs_drive.iops = 1000
ebs_drive.delete_on_termination = False
block_device_mapping['/dev/xvdh'] = ebs_drive
conn = boto.connect_autoscale(use_block_device_types=True)
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='m1.small',
key_name='the_keys',
security_groups=["default", "default2"],
user_data=b"This is some user_data",
instance_monitoring=True,
instance_profile_name=
'arn:aws:iam::123456789012:instance-profile/testing',
spot_price=0.1,
block_device_mappings=[block_device_mapping])
conn.create_launch_configuration(config)
launch_config = conn.get_all_launch_configurations()[0]
launch_config.name.should.equal('tester')
launch_config.image_id.should.equal('ami-abcd1234')
launch_config.instance_type.should.equal('m1.small')
launch_config.key_name.should.equal('the_keys')
set(launch_config.security_groups).should.equal(
set(['default', 'default2']))
launch_config.user_data.should.equal(b"This is some user_data")
launch_config.instance_monitoring.enabled.should.equal('true')
launch_config.instance_profile_name.should.equal(
'arn:aws:iam::123456789012:instance-profile/testing')
launch_config.spot_price.should.equal(0.1)
len(launch_config.block_device_mappings).should.equal(3)
returned_mapping = launch_config.block_device_mappings
set(returned_mapping.keys()).should.equal(
set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh']))
returned_mapping['/dev/xvdh'].iops.should.equal(1000)
returned_mapping['/dev/xvdh'].size.should.equal(100)
returned_mapping['/dev/xvdh'].volume_type.should.equal("io1")
returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false
returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd")
returned_mapping['/dev/xvdp'].volume_type.should.equal("standard")
returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')
def handleRunInstances(self, action, clc, user_data_file):
image_id = self.get_argument('ImageId')
min = self.get_argument('MinCount', '1')
max = self.get_argument('MaxCount', '1')
key = self.get_argument('KeyName', None)
groups = self.get_argument_list('SecurityGroup')
sec_group_ids = self.get_argument_list('SecurityGroupId')
if user_data_file:
user_data = user_data_file
else:
user_data = self.get_argument('UserData', None)
addr_type = self.get_argument('AddressingType', None)
vm_type = self.get_argument('InstanceType', None)
placement = self.get_argument('Placement.AvailabilityZone', None)
placement_group = self.get_argument('Placement.GroupName', None)
tenancy = self.get_argument('Placement.Tenancy', None)
kernel = self.get_argument('KernelId', None)
ramdisk = self.get_argument('RamdiskId', None)
monitoring = False
if self.get_argument('Monitoring.Enabled', '') == 'true':
monitoring = True
subnet = self.get_argument('SubnetId', None)
private_ip = self.get_argument('PrivateIpAddress', None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None)
idx = 1
while mapping:
pre = 'BlockDeviceMapping.%d' % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument(
'%s.VirtualName' % pre, None)
if not (block_dev_type.ephemeral_name):
block_dev_type.no_device = \
(self.get_argument('%s.NoDevice' % pre, '') == 'true')
block_dev_type.snapshot_id = \
self.get_argument('%s.Ebs.SnapshotId' % pre, None)
block_dev_type.size = \
self.get_argument('%s.Ebs.VolumeSize' % pre, None)
block_dev_type.delete_on_termination = \
(self.get_argument('%s.DeleteOnTermination' % pre, '') == 'true')
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument(
'BlockDeviceMapping.%d.DeviceName' % idx, None)
if len(bdm) == 0:
bdm = None
api_termination = False
if self.get_argument('DisableApiTermination', '') == 'true':
api_termination = True
instance_shutdown = False
if self.get_argument('InstanceInitiatedShutdownBehavior',
'') == 'true':
instance_shutdown = True
token = self.get_argument('ClientToken', None)
addition_info = self.get_argument('AdditionInfo', None)
instance_profile_name = self.get_argument('IamInstanceProfile.Name',
None)
instance_profile_arn = self.get_argument('IamInstanceProfile.Arn',
None)
return self.__normalize_instances__([
clc.run_instances(
image_id,
min_count=min,
max_count=max,
key_name=key,
security_groups=groups,
user_data=user_data,
addressing_type=addr_type,
instance_type=vm_type,
placement=placement,
kernel_id=kernel,
ramdisk_id=ramdisk,
monitoring_enabled=monitoring,
subnet_id=subnet,
block_device_map=bdm,
disable_api_termination=api_termination,
instance_initiated_shutdown_behavior=instance_shutdown,
private_ip_address=private_ip,
placement_group=placement_group,
client_token=token,
security_group_ids=sec_group_ids,
additional_info=addition_info,
instance_profile_name=instance_profile_name,
instance_profile_arn=instance_profile_arn,
tenancy=tenancy)
])
def handleImages(self, action, clc, callback=None):
if action == 'DescribeImages':
owner = self.get_argument('Owner', None);
if not owner:
owners = None
else:
owners = [owner]
filters = self.get_filter_args()
return clc.get_all_images(owners, filters, callback)
elif action == 'DescribeImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
return clc.get_image_attribute(imageid, attribute, callback)
elif action == 'ModifyImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
operation = self.get_argument('OperationType')
users = self.get_argument_list('UserId')
groups = self.get_argument_list('UserGroup')
return clc.modify_image_attribute(imageid, attribute, operation, users, groups, callback)
elif action == 'ResetImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
return clc.reset_image_attribute(imageid, attribute, callback)
elif action == 'DeregisterImage':
image_id = self.get_argument('ImageId')
return clc.deregister_image(image_id, callback)
elif action == 'RegisterImage':
image_location = self.get_argument('ImageLocation', None)
name = self.get_argument('Name')
description = self.get_argument('Description', None)
if description != None:
description = base64.b64decode(description);
architecture = self.get_argument('Architecture', None)
kernel_id = self.get_argument('KernelId', None)
ramdisk_id = self.get_argument('RamdiskId', None)
root_dev_name = self.get_argument('RootDeviceName', None)
snapshot_id = self.get_argument('SnapshotId', None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None)
idx = 1
while mapping:
pre = 'BlockDeviceMapping.%d' % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument('%s.VirtualName' % pre, None)
if not(block_dev_type.ephemeral_name):
block_dev_type.no_device = \
(self.get_argument('%s.NoDevice' % pre, '') == 'true')
block_dev_type.snapshot_id = \
self.get_argument('%s.Ebs.SnapshotId' % pre, None)
block_dev_type.size = \
self.get_argument('%s.Ebs.VolumeSize' % pre, None)
block_dev_type.delete_on_termination = \
(self.get_argument('%s.Ebs.DeleteOnTermination' % pre, '') == 'true')
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument('BlockDeviceMapping.%d.DeviceName' % idx, None)
if snapshot_id:
rootbdm = BlockDeviceType()
rootbdm.snapshot_id = snapshot_id
bdm['/dev/sda1'] = rootbdm
if len(bdm) == 0:
bdm = None
return clc.register_image(name, image_location, description, architecture, kernel_id, ramdisk_id, root_dev_name, bdm, callback)
def launchInstance(region, rootsize, role, env, ami, key_name, instance_type, placement, sec_group, iamrole, ebs, product, project, name, chefversion):
'''Launch a single instance of the provided ami'''
conn = connectEC2(region)
mapping = BlockDeviceMapping()
root = BlockDeviceType()
root.size = rootsize
mapping['/dev/xvda'] = root
user_data = genUserData(role, env, chefversion)
sec_group_id = getSecurityGroupID(conn, sec_group)
subnet_id = getSubnetID(placement, env)
print Fore.YELLOW + 'Instance will be launched with following parameters.\n'
print Fore.GREEN + 'Region :\t\t' + region
print Fore.GREEN + 'Chef Role :\t\t' + role
print Fore.GREEN + 'Chef Env :\t\t' + env
print Fore.GREEN + 'AMI ID :\t\t' + ami
print Fore.GREEN + 'Keypair :\t\t' + key_name
print Fore.GREEN + 'Instance Type :\t\t' + instance_type
print Fore.GREEN + 'Placement :\t\t' + placement
print Fore.GREEN + 'Security Group :\t' + sec_group + ' (' + sec_group_id + ')'
print Fore.GREEN + 'Subnet ID :\t\t' + subnet_id
print Fore.GREEN + 'IAM Role :\t\t' + iamrole
if ebs:
print Fore.GREEN + 'EBS Volume Size: ' + str(ebs) + ' GB'
print Fore.YELLOW + '\nFollowing tags will be attached to the instance.\n'
if name:
print Fore.GREEN + 'Name :\t\t\t' + name.lower()
print Fore.GREEN + 'Product :\t\t' + product.lower()
print Fore.GREEN + 'Project :\t\t' + project.lower()
print Fore.GREEN + 'Environment :\t\t' + env.lower()
print Fore.YELLOW + '\nTrying to launch an EC2 instance ...'
reservation = conn.run_instances(ami,
key_name=key_name,
user_data=user_data,
instance_type=instance_type,
placement=placement,
subnet_id=subnet_id,
block_device_map=mapping,
instance_initiated_shutdown_behavior='stop',
security_group_ids=['sg-********', sec_group_id],
instance_profile_name=iamrole,
dry_run=False)
print(Fore.YELLOW + '\nWaiting for instance to start ...')
instance = reservation.instances[0]
status = instance.update()
while status == 'pending':
time.sleep(5)
status = instance.update()
if status == 'running':
print(Fore.GREEN + 'New instance "' + instance.id + '" accessible at ' + instance.private_ip_address)
print(Fore.YELLOW + 'Adding tags.')
if name:
instance.add_tag('Name',name.lower())
instance.add_tag('Product', product.lower())
instance.add_tag('Project', project.lower())
instance.add_tag('Environment', env.lower())
else:
print(Fore.YELLOW + 'Instance status: ' + status)
return
# If we got through the launching successfully, go ahead and create and attach a volume.
if ebs:
attachEBS(conn, instance, ebs)
print Fore.GREEN + '\nInstance ' + instance.id + ' launched successfuly.'
def register(snapshot_id,
region,
arch,
size=None,
name=None,
desc=None,
pvm=False):
conn = utils.connect(region)
if None in (name, size):
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
virt = 'hvm'
kernel_id = None
device_base = '/dev/xvd'
ec2_arch = "x86_64" if arch == "amd64" else arch
if pvm:
kernel_id = utils.get_kernel(region, arch)
virt = 'paravirtual'
device_base = '/dev/sd'
name += '-pvm'
log.debug('creating block_device_map')
block_device_map = BlockDeviceMapping()
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
rootfs_device_name = device_base + 'a'
block_device_map[rootfs_device_name] = rootfs
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
ephemeral_device_name = device_base + 'b'
block_device_map[ephemeral_device_name] = ephemeral
log.debug('registering image - %s', name)
client3 = utils.connect_boto3(region)
response = client3.register_image(Name=name,
Architecture=ec2_arch,
RootDeviceName=rootfs_device_name,
BlockDeviceMappings=[{
'DeviceName': '/dev/xvda',
'Ebs': {
'DeleteOnTermination': True,
'VolumeSize': size,
'SnapshotId': snapshot_id,
},
}, {
'DeviceName':
'/dev/xvdb',
'VirtualName':
'ephemeral0',
}],
VirtualizationType=virt,
EnaSupport=True)
ami_id = response['ImageId']
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id, name
