def run_instance(conn, ami_id, key_name, instance_type,
sec_group, zone="us-east-1d", vol_size=None):
"""
@param connection: python boto connection
@param ami_id: AMI ID
@param key_name: SSH key name
@param instance_type: instance type, example 'm1.large'
@param sec_group: security group
@param zone: optional, defaults to 'us-east-1d'
@param vol_size: optional integer, if specified will change size of root volume to this size
@return boto.ec2.instance.Instance
"""
bdm = None
if vol_size:
# Create block device mapping info
dev_sda1 = BlockDeviceType()
dev_sda1.size = int(vol_size)
dev_sda1.delete_on_termination = True
bdm = BlockDeviceMapping()
bdm['/dev/sda1'] = dev_sda1
# Run instance
reservation = conn.run_instances(
ami_id,
key_name=key_name,
instance_type=instance_type,
placement=zone,
instance_initiated_shutdown_behavior="stop",
security_groups=[sec_group],
block_device_map=bdm)
return reservation.instances[0]
def create_block_device_mapping(ami, device_map):
bdm = BlockDeviceMapping()
for device, device_info in device_map.items():
if ami.root_device_type == "instance-store" and \
not device_info.get("ephemeral_name"):
# EBS is not supported by S3-backed AMIs at request time
# EBS volumes can be attached when an instance is running
continue
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
if ami.root_device_name == device:
ami_size = ami.block_device_mapping[device].size
if ami.virtualization_type == "hvm":
# Overwrite root device size for HVM instances, since they
# cannot be resized online
bd.size = ami_size
elif device_info.get('size'):
# make sure that size is enough for this AMI
assert ami_size <= device_info['size'], \
"Instance root device size cannot be smaller than AMI " \
"root device"
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
if device_info.get("volume_type"):
bd.volume_type = device_info["volume_type"]
if device_info["volume_type"] == "io1" \
and device_info.get("iops"):
bd.iops = device_info["iops"]
bdm[device] = bd
return bdm
def snapshot_register(self):
snapshot_id = self.snapshot.id
name = self.request.params.get('name')
description = self.request.params.get('description')
dot = self.request.params.get('dot')
reg_as_windows = self.request.params.get('reg_as_windows')
root_vol = BlockDeviceType(snapshot_id=snapshot_id)
root_vol.delete_on_termination = dot
bdm = BlockDeviceMapping()
root_device_name = '/dev/sda' if self.cloud_type == 'euca' else '/dev/sda1'
bdm[root_device_name] = root_vol
location = self.request.route_path('snapshot_view', id=snapshot_id)
if self.snapshot and self.register_form.validate():
with boto_error_handler(self.request, location):
self.log_request(_(u"Registering snapshot {0} as image {1}").format(snapshot_id, name))
self.snapshot.connection.register_image(
name=name, description=description,
root_device_name=root_device_name,
kernel_id=('windows' if reg_as_windows else None),
block_device_map=bdm)
prefix = _(u'Successfully registered snapshot')
msg = '{prefix} {id}'.format(prefix=prefix, id=snapshot_id)
# Clear images cache
#ImagesView.invalidate_images_cache()
self.request.session.flash(msg, queue=Notification.SUCCESS)
return HTTPFound(location=location)
return self.render_dict
def create_block_device_mapping(ami, device_map):
bdm = BlockDeviceMapping()
for device, device_info in device_map.items():
if ami.root_device_type == "instance-store" and \
not device_info.get("ephemeral_name"):
# EBS is not supported by S3-backed AMIs at request time
# EBS volumes can be attached when an instance is running
continue
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
if ami.root_device_name == device:
ami_size = ami.block_device_mapping[device].size
if ami.virtualization_type == "hvm":
# Overwrite root device size for HVM instances, since they
# cannot be resized online
bd.size = ami_size
elif device_info.get('size'):
# make sure that size is enough for this AMI
assert ami_size <= device_info['size'], \
"Instance root device size cannot be smaller than AMI " \
"root device"
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
if device_info.get("volume_type"):
bd.volume_type = device_info["volume_type"]
if device_info["volume_type"] == "io1" \
and device_info.get("iops"):
bd.iops = device_info["iops"]
bdm[device] = bd
return bdm
def get_block_device_map(bdmapping_json=None):
"""Parse block_device_mapping JSON and return a configured BlockDeviceMapping object
Mapping JSON structure...
{"/dev/sda":
{"snapshot_id": "snap-23E93E09", "volume_type": null, "delete_on_termination": true, "size": 1} }
"""
if bdmapping_json:
mapping = json.loads(bdmapping_json)
if mapping:
bdm = BlockDeviceMapping()
for key, val in mapping.items():
device = BlockDeviceType()
if val.get('virtual_name') is not None and val.get(
'virtual_name').startswith('ephemeral'):
device.ephemeral_name = val.get('virtual_name')
else:
device.volume_type = 'standard'
device.snapshot_id = val.get('snapshot_id') or None
device.size = val.get('size')
device.delete_on_termination = val.get(
'delete_on_termination', False)
bdm[key] = device
return bdm
return None
return None
def register_snapshot_by_id( self, snap_id, rdn="/dev/sda1", description="bfebs", windows=False, bdmdev=None, name=None, ramdisk=None, kernel=None, dot=True ):
'''
Register an image snapshot
snap_id (mandatory string) snapshot id
name (mandatory string) name of image to be registered
description (optional string) description of image to be registered
bdmdev (optional string) block-device-mapping device for image
rdn (optional string) root-device-name for image
dot (optional boolean) Delete On Terminate boolean
windows (optional boolean) Is windows image boolean
kernel (optional string) kernal (note for windows this name should be "windows"
'''
if (bdmdev is None):
bdmdev=rdn
if (name is None):
name="bfebs_"+ snap_id
if ( windows is True ) and ( kernel is not None):
kernel="windows"
bdmap = BlockDeviceMapping()
block_dev_type = BlockDeviceType()
block_dev_type.snapshot_id = snap_id
block_dev_type.delete_on_termination = dot
bdmap[bdmdev] = block_dev_type
self.debug("Register image with: snap_id:"+str(snap_id)+", rdn:"+str(rdn)+", desc:"+str(description)+", windows:"+str(windows)+", bdname:"+str(bdmdev)+", name:"+str(name)+", ramdisk:"+str(ramdisk)+", kernel:"+str(kernel))
image_id = self.ec2.register_image(name=name, description=description, kernel_id=kernel, ramdisk_id=ramdisk, block_device_map=bdmap, root_device_name=rdn)
self.debug("Image now registered as " + image_id)
return image_id
def snapshot_register(self):
snapshot_id = self.snapshot.id
name = self.request.params.get('name')
description = self.request.params.get('description')
dot = self.request.params.get('dot')
reg_as_windows = self.request.params.get('reg_as_windows')
root_vol = BlockDeviceType(snapshot_id=snapshot_id)
root_vol.delete_on_termination = dot
bdm = BlockDeviceMapping()
root_device_name = '/dev/sda' if self.cloud_type == 'euca' else '/dev/sda1'
bdm[root_device_name] = root_vol
location = self.request.route_path('snapshot_view', id=snapshot_id)
if self.snapshot and self.register_form.validate():
with boto_error_handler(self.request, location):
self.log_request(_(u"Registering snapshot {0} as image {1}").format(snapshot_id, name))
self.snapshot.connection.register_image(
name=name, description=description,
root_device_name=root_device_name,
kernel_id=('windows' if reg_as_windows else None),
block_device_map=bdm)
prefix = _(u'Successfully registered snapshot')
msg = u'{prefix} {id}'.format(prefix=prefix, id=snapshot_id)
# Clear images cache
self.invalidate_images_cache()
self.request.session.flash(msg, queue=Notification.SUCCESS)
return HTTPFound(location=location)
return self.render_dict
def _parse_block_device_mappings(user_input):
"""
Parse block device mappings per AWS CLI tools syntax (modified to add IOPS)
http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
Syntax:
/dev/xvd[a-z]=[snapshot-id|ephemeral]:[size in GB]:[Delete on Term]:[IOPS]
- Leave inapplicable fields blank
- Delete on Termination defaults to True
- IOPS limits are not validated
- EBS sizing is not validated
Mount an Ephemeral Drive:
/dev/xvdb1=ephemeral0
Mount multiple Ephemeral Drives:
/dev/xvdb1=ephemeral0,/dev/xvdb2=ephemeral1
Mount a Snapshot:
/dev/xvdp=snap-1234abcd
Mount a Snapshot to a 100GB drive:
/dev/xvdp=snap-1234abcd:100
Mount a Snapshot to a 100GB drive and do not delete on termination:
/dev/xvdp=snap-1234abcd:100:false
Mount a Fresh 100GB EBS device
/dev/xvdp=:100
Mount a Fresh 100GB EBS Device and do not delete on termination:
/dev/xvdp=:100:false
Mount a Fresh 100GB EBS Device with 1000 IOPS
/dev/xvdp=:100::1000
"""
block_device_map = BlockDeviceMapping()
mappings = user_input.split(",")
for mapping in mappings:
block_type = BlockDeviceType()
mount_point, drive_type, size, delete, iops = _parse_drive_mapping(
mapping)
if 'ephemeral' in drive_type:
block_type.ephemeral_name = drive_type
elif 'snap' in drive_type:
block_type.snapshot_id = drive_type
block_type.volume_type = "standard"
else:
block_type.volume_type = "standard"
block_type.size = size
block_type.delete_on_termination = delete
if iops:
block_type.iops = iops
block_type.volume_type = "io1"
block_device_map[mount_point] = block_type
return block_device_map
def test_create_launch_configuration_with_block_device_mappings():
block_device_mapping = BlockDeviceMapping()
ephemeral_drive = BlockDeviceType()
ephemeral_drive.ephemeral_name = 'ephemeral0'
block_device_mapping['/dev/xvdb'] = ephemeral_drive
snapshot_drive = BlockDeviceType()
snapshot_drive.snapshot_id = "snap-1234abcd"
snapshot_drive.volume_type = "standard"
block_device_mapping['/dev/xvdp'] = snapshot_drive
ebs_drive = BlockDeviceType()
ebs_drive.volume_type = "io1"
ebs_drive.size = 100
ebs_drive.iops = 1000
ebs_drive.delete_on_termination = False
block_device_mapping['/dev/xvdh'] = ebs_drive
conn = boto.connect_autoscale(use_block_device_types=True)
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='m1.small',
key_name='the_keys',
security_groups=["default", "default2"],
user_data="This is some user_data",
instance_monitoring=True,
instance_profile_name='arn:aws:iam::123456789012:instance-profile/testing',
spot_price=0.1,
block_device_mappings=[block_device_mapping]
)
conn.create_launch_configuration(config)
launch_config = conn.get_all_launch_configurations()[0]
launch_config.name.should.equal('tester')
launch_config.image_id.should.equal('ami-abcd1234')
launch_config.instance_type.should.equal('m1.small')
launch_config.key_name.should.equal('the_keys')
set(launch_config.security_groups).should.equal(set(['default', 'default2']))
launch_config.user_data.should.equal("This is some user_data")
launch_config.instance_monitoring.enabled.should.equal('true')
launch_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/testing')
launch_config.spot_price.should.equal(0.1)
len(launch_config.block_device_mappings).should.equal(3)
returned_mapping = launch_config.block_device_mappings
set(returned_mapping.keys()).should.equal(set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh']))
returned_mapping['/dev/xvdh'].iops.should.equal(1000)
returned_mapping['/dev/xvdh'].size.should.equal(100)
returned_mapping['/dev/xvdh'].volume_type.should.equal("io1")
returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false
returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd")
returned_mapping['/dev/xvdp'].volume_type.should.equal("standard")
returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')
def _parse_block_device_mappings(user_input):
"""
Parse block device mappings per AWS CLI tools syntax (modified to add IOPS)
http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
Syntax:
/dev/xvd[a-z]=[snapshot-id|ephemeral]:[size in GB]:[Delete on Term]:[IOPS]
- Leave inapplicable fields blank
- Delete on Termination defaults to True
- IOPS limits are not validated
- EBS sizing is not validated
Mount an Ephemeral Drive:
/dev/xvdb1=ephemeral0
Mount multiple Ephemeral Drives:
/dev/xvdb1=ephemeral0,/dev/xvdb2=ephemeral1
Mount a Snapshot:
/dev/xvdp=snap-1234abcd
Mount a Snapshot to a 100GB drive:
/dev/xvdp=snap-1234abcd:100
Mount a Snapshot to a 100GB drive and do not delete on termination:
/dev/xvdp=snap-1234abcd:100:false
Mount a Fresh 100GB EBS device
/dev/xvdp=:100
Mount a Fresh 100GB EBS Device and do not delete on termination:
/dev/xvdp=:100:false
Mount a Fresh 100GB EBS Device with 1000 IOPS
/dev/xvdp=:100::1000
"""
block_device_map = BlockDeviceMapping()
mappings = user_input.split(",")
for mapping in mappings:
block_type = BlockDeviceType()
mount_point, drive_type, size, delete, iops = _parse_drive_mapping(mapping)
if 'ephemeral' in drive_type:
block_type.ephemeral_name = drive_type
elif 'snap' in drive_type:
block_type.snapshot_id = drive_type
block_type.volume_type = "standard"
else:
block_type.volume_type = "standard"
block_type.size = size
block_type.delete_on_termination = delete
if iops:
block_type.iops = iops
block_type.volume_type = "io1"
block_device_map[mount_point] = block_type
return block_device_map
def register(snapshot_id,
region,
arch,
size=None,
name=None,
desc=None,
pvm=False):
conn = utils.connect(region)
if None in (name, size):
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
virt = 'hvm'
kernel_id = None
device_base = '/dev/xvd'
ec2_arch = "x86_64" if arch == "amd64" else arch
if pvm:
kernel_id = utils.get_kernel(region, arch)
virt = 'paravirtual'
device_base = '/dev/sd'
name += '-pvm'
log.debug('creating block_device_map')
block_device_map = BlockDeviceMapping()
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
rootfs_device_name = device_base + 'a'
block_device_map[rootfs_device_name] = rootfs
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
ephemeral_device_name = device_base + 'b'
block_device_map[ephemeral_device_name] = ephemeral
log.debug('registering image - %s', name)
ami_id = conn.register_image(name=name,
description=desc,
kernel_id=kernel_id,
architecture=ec2_arch,
root_device_name=rootfs_device_name,
block_device_map=block_device_map,
virtualization_type=virt)
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id, name
def __update_bdm(self, bdm, bd_spec):
"""Update the BlockDeviceMapping bdm with the block device
spec bd_spec.
"""
try:
dev_name, dev_value = bd_spec.split('=', 1)
except Exception:
raise CommandError(
"Block device spec missing '=' : %s" % (bd_spec,))
dot = None
bdt = BlockDeviceType()
if ':' in dev_value:
blockdev_origin, dot = dev_value.split(':', 1)
else:
blockdev_origin = dev_value
if blockdev_origin is None:
raise CommandError("No source specified for %s" % (dev_name,))
if blockdev_origin.startswith('ephemeral'):
bdt.ephemeral_name = blockdev_origin
elif blockdev_origin.startswith('snap-'):
bdt.snapshot_id = blockdev_origin
else:
raise CommandError("Bad source specified for %s: %s" % (dev_name,
blockdev_origin))
if dot is not None:
if dot == 'delete':
bdt.delete_on_termination = True
elif dot == 'nodelete':
bdt.delete_on_termination = False
else:
raise CommandError(
"Bad delete-on-termination specified for %s: %s" %
(dev_name, dot))
else:
bdt.delete_on_termination = False
dev_path = '/dev/' + dev_name
bdm[dev_path] = bdt
def _get_bmap(self, params):
bmap = BlockDeviceMapping()
for device in params['bmap']:
if not 'name' in device.keys():
self.logger.debug('bad device ' + str(device))
continue
dev = BlockDeviceType()
if 'size' in device.keys():
dev.size = device['size']
if 'delete_on_termination' in device.keys():
dev.delete_on_termination = device['delete_on_termination']
if 'ephemeral_name' in device.keys():
dev.ephemeral_name = device['ephemeral_name']
bmap[device['name']] = dev
return bmap
def _parse_block_device_mappings(self):
block_device_map = BlockDeviceMapping()
for mapping in self.block_device_mapping_dict:
block_type = BlockDeviceType()
mount_point = mapping.get('device_name')
if 'ephemeral' in mapping.get('virtual_name', ''):
block_type.ephemeral_name = mapping.get('virtual_name')
else:
block_type.volume_type = mapping.get('ebs._volume_type')
block_type.snapshot_id = mapping.get('ebs._snapshot_id')
block_type.delete_on_termination = mapping.get('ebs._delete_on_termination')
block_type.size = mapping.get('ebs._volume_size')
block_type.iops = mapping.get('ebs._iops')
block_device_map[mount_point] = block_type
return block_device_map
def parse_block_device_args(self, block_device_maps_args):
block_device_map = BlockDeviceMapping()
for block_device_map_arg in block_device_maps_args:
parts = block_device_map_arg.split('=')
if len(parts) > 1:
device_name = parts[0]
block_dev_type = BlockDeviceType()
block_dev_type.delete_on_termination = True
value_parts = parts[1].split(':')
if value_parts[0].startswith('snap'):
block_dev_type.snapshot_id = value_parts[0]
else:
if value_parts[0].startswith('ephemeral'):
block_dev_type.ephemeral_name = value_parts[0]
if len(value_parts) > 1:
try:
block_dev_type.size = int(value_parts[1])
except ValueError:
pass
if len(value_parts) > 2:
if value_parts[2] == 'false':
block_dev_type.delete_on_termination = False
block_device_map[device_name] = block_dev_type
return block_device_map
def parse_block_device_args(self, block_device_maps_args):
block_device_map = BlockDeviceMapping()
for block_device_map_arg in block_device_maps_args:
parts = block_device_map_arg.split('=')
if len(parts) > 1:
device_name = parts[0]
block_dev_type = BlockDeviceType()
block_dev_type.delete_on_termination = True
value_parts = parts[1].split(':')
if value_parts[0].startswith('snap'):
block_dev_type.snapshot_id = value_parts[0]
else:
if value_parts[0].startswith('ephemeral'):
block_dev_type.ephemeral_name = value_parts[0]
if len(value_parts) > 1:
try:
block_dev_type.size = int(value_parts[1])
except ValueError:
pass
if len(value_parts) > 2:
if value_parts[2] == 'false':
block_dev_type.delete_on_termination = False
block_device_map[device_name] = block_dev_type
return block_device_map
def _get_bmap(self, params):
bmap = BlockDeviceMapping()
for device in params['bmap']:
if not 'name' in device.keys():
self.logger.debug('bad device ' + str(device))
continue
dev = BlockDeviceType()
if 'size' in device.keys():
dev.size = device['size']
if 'delete_on_termination' in device.keys():
dev.delete_on_termination = device['delete_on_termination']
if 'ephemeral_name' in device.keys():
dev.ephemeral_name = device['ephemeral_name']
bmap[device['name']] = dev
return bmap
def _parse_block_device_mappings(self):
block_device_map = BlockDeviceMapping()
for mapping in self.block_device_mapping_dict:
block_type = BlockDeviceType()
mount_point = mapping.get('device_name')
if 'ephemeral' in mapping.get('virtual_name', ''):
block_type.ephemeral_name = mapping.get('virtual_name')
else:
block_type.volume_type = mapping.get('ebs._volume_type')
block_type.snapshot_id = mapping.get('ebs._snapshot_id')
block_type.delete_on_termination = mapping.get('ebs._delete_on_termination')
block_type.size = mapping.get('ebs._volume_size')
block_type.iops = mapping.get('ebs._iops')
block_device_map[mount_point] = block_type
return block_device_map
def startCluster(self, argv):
if len(argv) != 0:
print "ec2 startCluster"
sys.exit(-1)
regions = boto.ec2.regions()
regionInfo = '\n'.join(str(region).split(':')[1] for region in regions)
regionName = raw_input("select region:\n%s\n>>"%regionInfo)
region = boto.ec2.get_region(regionName)
conn = region.connect()
print "region connected successfully"
images = conn.get_all_images(owners='self')
imageInfo = '\n'.join(
str(image).split(':')[1] + ":" + image.name for image in images)
imageId = raw_input("enter imageId:\nself-created images:\n%s\n>>"%imageInfo)
instanceTypeInfo = ("m1.small, " "m1.large, " "m1.xlarge\n"
"c1.medium, " "c1.xlarge\n"
"m2.xlarge, " "m2.2xlarge, " "m2.4xlarge\n"
"cc1.4xlarge, " "t1.micro\n")
instanceType = raw_input("enter instanceType:\n%s\n>>"%instanceTypeInfo)
availZone = raw_input("enter placement[a,b,c]:\n>>")
availZone = regionName + availZone
diskSize = int(raw_input("enter disk size[G]:\n>>"))
rootDev = BlockDeviceType()
rootDev.name = 'root'
rootDev.size = diskSize
rootDev.delete_on_termination = True
instStorage = bool(raw_input("mount inst storage?\n>>"))
mapping = BlockDeviceMapping()
mapping['/dev/sda1'] = rootDev
if (instStorage == True):
eph0 = BlockDeviceType()
eph0.ephemeral_name = 'ephemeral0'
mapping['/dev/sdb'] = eph0
groups = conn.get_all_security_groups()
groupInfo = '\n'.join(str(group).split(':')[1] for group in groups)
group = raw_input("enter securityGroup:\n%s\n>>"%groupInfo)
keys = conn.get_all_key_pairs()
if len(keys) == 1:
key = keys[0].name
print 'using default key: ' + key
else:
keyInfo = '\n'.join(str(key).split(':')[1] for key in keys)
key = raw_input("enter key name:\n%s\n>>"%keyInfo)
numNodes = int(raw_input("number of nodes:\n>>"))
conn.run_instances(
imageId, min_count=numNodes, max_count=numNodes, placement=availZone,
security_groups = [group], instance_type=instanceType,
block_device_map=mapping, key_name=key)
def _parse_block_device_mappings(self):
block_device_map = BlockDeviceMapping()
for mapping in self.block_device_mapping_dict:
block_type = BlockDeviceType()
mount_point = mapping.get("device_name")
if "ephemeral" in mapping.get("virtual_name", ""):
block_type.ephemeral_name = mapping.get("virtual_name")
else:
block_type.volume_type = mapping.get("ebs._volume_type")
block_type.snapshot_id = mapping.get("ebs._snapshot_id")
block_type.delete_on_termination = mapping.get(
"ebs._delete_on_termination")
block_type.size = mapping.get("ebs._volume_size")
block_type.iops = mapping.get("ebs._iops")
block_device_map[mount_point] = block_type
return block_device_map
def register(snapshot_id, region, arch, size=None, name=None, desc=None, pvm=False):
conn = utils.connect(region)
if None in (name, size):
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
virt = 'hvm'
kernel_id = None
device_base = '/dev/xvd'
ec2_arch = "x86_64" if arch == "amd64" else arch
if pvm:
kernel_id = utils.get_kernel(region, arch)
virt = 'paravirtual'
device_base = '/dev/sd'
name += '-pvm'
log.debug('creating block_device_map')
block_device_map = BlockDeviceMapping()
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
rootfs_device_name = device_base + 'a'
block_device_map[rootfs_device_name] = rootfs
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
ephemeral_device_name = device_base + 'b'
block_device_map[ephemeral_device_name] = ephemeral
log.debug('registering image - %s', name)
ami_id = conn.register_image(
name=name,
description=desc,
kernel_id=kernel_id,
architecture=ec2_arch,
root_device_name=rootfs_device_name,
block_device_map=block_device_map,
virtualization_type=virt)
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id, name
def create_root_block_device(self):
# if the root volume size is not the same as the AMI default value:
if self.root_volume_size is not None:
# sda rather than xvda (for Windows)
dev_sda1 = BlockDeviceType()
dev_sda1.size = self.root_volume_size
dev_sda1.delete_on_termination = True
volume = BlockDeviceMapping()
# Check the OS type, if its windows we use sda, linux: xvda
images = self.ec2.get_all_images(image_ids=[self.node_obj.ami])
image = images[0]
if image.platform is None:
volume['/dev/xvda'] = dev_sda1
else:
volume['/dev/sda1'] = dev_sda1
self.volumes = volume
def start_remote(self):
self.get_bootstrap_kernel()
self.get_bootstrap_image()
# create EBS volume for /mnt/gentoo
device = BlockDeviceType()
device.size = self.settings["ec2/instance/device/size"]
device.delete_on_termination = True
mapping = BlockDeviceMapping()
self.root_device = "/dev/" + self.settings["ec2/instance/device/name"]
mapping[self.root_device] = device
# start bootstrapping instance
reservation = self.ec2.run_instances(self.bootstrap_image.id,
kernel_id=self.bootstrap_kernel.id,
instance_type=self.settings["ec2/instance/type"],
security_groups=[self.name],
key_name=self.name,
block_device_map=mapping)
self.instance = reservation.instances[0]
sys.stdout.write("waiting for instance to come up ..")
while self.instance.update() != 'running':
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(5)
sys.stdout.write("\n")
time.sleep(120)
self.ssh_uri = "ec2-user@" + self.instance.public_dns_name
self.ssh_port = "22"
self.remote_upload_path = "/tmp"
# enable sudo without a tty
cmd = "sudo sed -i -e '/requiretty/d' /etc/sudoers"
cmd = ["ssh", "-t"] + self.ssh_options() + [self.ssh_uri, cmd]
ssh = subprocess.Popen(cmd)
ssh.wait()
self.run_script_at_remote("steps/remote/postboot")
def start_remote(self):
self.get_bootstrap_kernel()
self.get_bootstrap_image()
# create EBS volume for /mnt/gentoo
device = BlockDeviceType()
device.size = self.settings["ec2/instance/device/size"]
device.delete_on_termination = True
mapping = BlockDeviceMapping()
self.root_device = "/dev/" + self.settings["ec2/instance/device/name"]
mapping[self.root_device] = device
# start bootstrapping instance
reservation = self.ec2.run_instances(
self.bootstrap_image.id,
kernel_id=self.bootstrap_kernel.id,
instance_type=self.settings["ec2/instance/type"],
security_groups=[self.name],
key_name=self.name,
block_device_map=mapping)
self.instance = reservation.instances[0]
sys.stdout.write("waiting for instance to come up ..")
while self.instance.update() != 'running':
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(5)
sys.stdout.write("\n")
time.sleep(120)
self.ssh_uri = "ec2-user@" + self.instance.public_dns_name
self.remote_upload_path = "/tmp"
# enable sudo without a tty
cmd = "sudo sed -i -e '/requiretty/d' /etc/sudoers"
cmd = ["ssh", "-t"] + self.ssh_options() + [self.ssh_uri, cmd]
ssh = subprocess.Popen(cmd)
ssh.wait()
self.run_script_at_remote("steps/remote/postboot")
def run_instance(conn, ami_id, key_name, instance_type, sec_group, zone, vol_size=None):
bdm = None
if vol_size:
# Create block device mapping info
dev_sda1 = BlockDeviceType()
dev_sda1.size = int(vol_size)
dev_sda1.delete_on_termination = True
bdm = BlockDeviceMapping()
bdm["/dev/sda1"] = dev_sda1
# Run instance
reservation = conn.run_instances(
ami_id,
key_name=key_name,
instance_type=instance_type,
placement=zone,
instance_initiated_shutdown_behavior="stop",
security_groups=[sec_group],
block_device_map=bdm,
)
return reservation.instances[0]
def register(snapshot_id, region, arch, size=None, name=None, desc=None):
conn = utils.connect(region)
if None in (name, size):
log.debug("getting snapshot - %s", snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
ec2_arch = "x86_64" if arch == "amd64" else arch
log.debug("creating block_device_map")
block_device_map = BlockDeviceMapping()
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
rootfs_device_name = "/dev/xvda"
block_device_map[rootfs_device_name] = rootfs
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = "ephemeral0"
ephemeral_device_name = "/dev/xvdb"
block_device_map[ephemeral_device_name] = ephemeral
log.debug("registering image - %s", name)
ami_id = conn.register_image(
name=name,
description=desc,
architecture=ec2_arch,
root_device_name=rootfs_device_name,
block_device_map=block_device_map,
virtualization_type="hvm",
)
log.info("registered image - %s %s %s", ami_id, name, region)
return ami_id, name
def register(snapshot_id, region, size=None, arch=None, name=None, desc=None):
conn = utils.connect(region)
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
desc = desc if desc else utils.parse_imagename(name)['url']
arch = arch if arch else utils.parse_imagename(name)['architecture']
kernel_id = utils.get_kernel(region, arch)
arch_ec2 = "x86_64" if arch == "amd64" else arch
log.debug('creating block_device_map')
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
block_device_map = BlockDeviceMapping()
block_device_map['/dev/sda1'] = rootfs
block_device_map['/dev/sda2'] = ephemeral
log.debug('registering image - %s', name)
ami_id = conn.register_image(
name=name,
description=desc,
architecture=arch_ec2,
kernel_id=kernel_id,
root_device_name="/dev/sda1",
block_device_map=block_device_map)
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id
def get_block_device_map(bdmapping_json=None):
"""Parse block_device_mapping JSON and return a configured BlockDeviceMapping object
Mapping JSON structure...
{"/dev/sda":
{"snapshot_id": "snap-23E93E09", "volume_type": null, "delete_on_termination": true, "size": 1} }
"""
if bdmapping_json:
mapping = json.loads(bdmapping_json)
if mapping:
bdm = BlockDeviceMapping()
for key, val in mapping.items():
device = BlockDeviceType()
if val.get('virtual_name') is not None and val.get('virtual_name').startswith('ephemeral'):
device.ephemeral_name = val.get('virtual_name')
else:
device.volume_type = 'standard'
device.snapshot_id = val.get('snapshot_id') or None
device.size = val.get('size')
device.delete_on_termination = val.get('delete_on_termination', False)
bdm[key] = device
return bdm
return None
return None
def register(snapshot_id, region, size=None, arch=None, name=None, desc=None):
conn = utils.connect(region)
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
desc = desc if desc else utils.parse_imagename(name)['url']
arch = arch if arch else utils.parse_imagename(name)['architecture']
kernel_id = utils.get_kernel(region, arch)
arch_ec2 = "x86_64" if arch == "amd64" else arch
log.debug('creating block_device_map')
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
block_device_map = BlockDeviceMapping()
block_device_map['/dev/sda1'] = rootfs
block_device_map['/dev/sda2'] = ephemeral
log.debug('registering image - %s', name)
ami_id = conn.register_image(name=name,
description=desc,
architecture=arch_ec2,
kernel_id=kernel_id,
root_device_name="/dev/sda1",
block_device_map=block_device_map)
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id
def run_instance(conn,
ami_id,
key_name,
instance_type,
sec_group,
zone="us-east-1d",
vol_size=None):
"""
@param connection: python boto connection
@param ami_id: AMI ID
@param key_name: SSH key name
@param instance_type: instance type, example 'm1.large'
@param sec_group: security group
@param zone: optional, defaults to 'us-east-1d'
@param vol_size: optional integer, if specified will change size of root volume to this size
@return boto.ec2.instance.Instance
"""
bdm = None
if vol_size:
# Create block device mapping info
dev_sda1 = BlockDeviceType()
dev_sda1.size = int(vol_size)
dev_sda1.delete_on_termination = True
bdm = BlockDeviceMapping()
bdm['/dev/sda1'] = dev_sda1
# Run instance
reservation = conn.run_instances(
ami_id,
key_name=key_name,
instance_type=instance_type,
placement=zone,
instance_initiated_shutdown_behavior="stop",
security_groups=[sec_group],
block_device_map=bdm)
return reservation.instances[0]
def do_request_spot_instance(region, secrets, moz_instance_type, price, ami,
instance_config, cached_cert_dir, instance_type,
availability_zone, dryrun):
conn = aws_connect_to_region(region, secrets)
interface = get_available_interface(conn=conn,
moz_instance_type=moz_instance_type,
availability_zone=availability_zone)
if not interface:
raise RuntimeError("No free network interfaces left in %s" % region)
# TODO: check DNS
fqdn = interface.tags.get("FQDN")
if not fqdn:
raise RuntimeError("Skipping %s without FQDN" % interface)
log.debug("Spot request for %s (%s)", fqdn, price)
if dryrun:
log.info("Dry run. skipping")
return
spec = NetworkInterfaceSpecification(network_interface_id=interface.id)
nc = NetworkInterfaceCollection(spec)
ip = interface.private_ip_address
certs = get_puppet_certs(ip, secrets, cached_cert_dir)
user_data = """
FQDN="%(fqdn)s"
cd /var/lib/puppet/ssl || exit 1
%(certs)s
cd -
""" % dict(fqdn=fqdn, certs=certs)
if instance_config[region].get("lvm"):
user_data += """
mkdir -p /etc/lvm-init/
cat <<EOF > /etc/lvm-init/lvm-init.json
%s
EOF
/sbin/lvm-init
""" % json.dumps(instance_config[region])
bdm = BlockDeviceMapping()
for device, device_info in instance_config[region]['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
if ami.root_device_name == device:
ami_size = ami.block_device_mapping[device].size
if ami.virtualization_type == "hvm":
# Overwrite root device size for HVM instances, since they
# cannot be resized online
bd.size = ami_size
elif device_info.get('size'):
# make sure that size is enough for this AMI
assert ami_size <= device_info['size'], \
"Instance root device size cannot be smaller than AMI " \
"root device"
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
bdm[device] = bd
sir = conn.request_spot_instances(
price=str(price),
image_id=ami.id,
count=1,
instance_type=instance_type,
key_name=instance_config[region]["ssh_key"],
user_data=user_data,
block_device_map=bdm,
network_interfaces=nc,
instance_profile_name=instance_config[region].get(
"instance_profile_name"),
)
sir[0].add_tag("moz-type", moz_instance_type)
def do_request_spot_instance(region, secrets, moz_instance_type, price, ami,
instance_config, cached_cert_dir, instance_type,
availability_zone, slaveset, active_network_ids, dryrun):
conn = get_aws_connection(region)
interface = get_available_interface(
conn=conn, moz_instance_type=moz_instance_type,
availability_zone=availability_zone,
slaveset=slaveset,
active_network_ids=active_network_ids)
if not interface:
log.warn("No free network interfaces left in %s" % region)
return False
# TODO: check DNS
fqdn = interface.tags.get("FQDN")
if not fqdn:
log.warn("interface %s has no FQDN", interface)
return False
log.debug("Spot request for %s (%s)", fqdn, price)
if dryrun:
log.info("Dry run. skipping")
return True
spec = NetworkInterfaceSpecification(
network_interface_id=interface.id)
nc = NetworkInterfaceCollection(spec)
ip = interface.private_ip_address
certs = get_puppet_certs(ip, secrets, cached_cert_dir)
user_data = """
FQDN="%(fqdn)s"
cd /var/lib/puppet/ssl || exit 1
%(certs)s
cd -
""" % dict(fqdn=fqdn, certs=certs)
if instance_config[region].get("lvm"):
user_data += """
mkdir -p /etc/lvm-init/
cat <<EOF > /etc/lvm-init/lvm-init.json
%s
EOF
/sbin/lvm-init
""" % json.dumps(instance_config[region])
bdm = BlockDeviceMapping()
for device, device_info in instance_config[region]['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
if ami.root_device_name == device:
ami_size = ami.block_device_mapping[device].size
if ami.virtualization_type == "hvm":
# Overwrite root device size for HVM instances, since they
# cannot be resized online
bd.size = ami_size
elif device_info.get('size'):
# make sure that size is enough for this AMI
assert ami_size <= device_info['size'], \
"Instance root device size cannot be smaller than AMI " \
"root device"
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
bdm[device] = bd
sir = conn.request_spot_instances(
price=str(price),
image_id=ami.id,
count=1,
instance_type=instance_type,
key_name=instance_config[region]["ssh_key"],
user_data=user_data,
block_device_map=bdm,
network_interfaces=nc,
instance_profile_name=instance_config[region].get("instance_profile_name"),
)
sir[0].add_tag("moz-type", moz_instance_type)
return True
def vm(install_list):
region = install_list["region"]
subnet_id = install_list["subnet_id"]
ami_id = install_list["ami_id"]
snap_id = install_list["snap_id"]
volume_capacity = install_list["volume_capacity"]
key_name = install_list["key_name"]
instance_type = install_list["instance_type"]
sg_id = install_list["sg_id"]
user_data = install_list["user_data"]
usage = install_list["usage"]
block_device_map = BlockDeviceMapping()
block_dev_type = BlockDeviceType()
block_dev_type.snapshot_id = snap_id
block_dev_type.delete_on_termination = False
block_dev_type.size = volume_capacity
block_device_map['/dev/sdb'] = block_dev_type
if region not in EXTRA_INFO:
aws_access_key_id = EXTRA_INFO["common"]["aws_access_key_id"]
aws_secret_access_key = EXTRA_INFO["common"]["aws_secret_access_key"]
name_key = EXTRA_INFO["common"]["name_key"]
else:
aws_access_key_id = EXTRA_INFO[region]["aws_access_key_id"]
aws_secret_access_key = EXTRA_INFO[region]["aws_secret_access_key"]
name_key = EXTRA_INFO[region]["name_key"]
ret = asset_hostname.get(region, usage)
hostname = ret["hostname"].split(".")[0]
user_data = user_data.replace("region=","region=%s" % region)
user_data = user_data.replace("hostname=","hostname=%s" % hostname)
user_data = user_data.replace("dns_vip=","dns_vip=%s" % \
DNS_INFO[region]["dns_vip"])
user_data = user_data.replace("ns_servers=","ns_servers='%s'" % \
" ".join(DNS_INFO[region]["ns_servers"]) )
network_interface = NetworkInterfaceSpecification(subnet_id=subnet_id, \
groups=[sg_id])
network_interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(\
network_interface)
conn = boto.ec2.connect_to_region(region, \
aws_access_key_id=aws_access_key_id, \
aws_secret_access_key=aws_secret_access_key)
reservation = conn.run_instances(ami_id,
key_name=key_name,
network_interfaces=network_interfaces,
instance_type=instance_type,
block_device_map=block_device_map,
# security_groups=[security_groups],
min_count=1,
max_count=1,
user_data=user_data,
)
instance = reservation.instances[0]
time_init = 0
time_total = 300
time_interval = 5
while time_init < time_total:
status = instance.update()
if status == 'running':
instance.add_tag(u"名称", hostname)
instance.add_tag("Name", hostname)
break
else:
time.sleep(time_interval)
time_init += time_interval
install_list["instance_id"] = str(instance).split(":")[-1]
install_list["placement"] = instance.placement
install_list["status"] = instance.update()
install_list["hostname"] = hostname
return install_list
def test_create_launch_configuration_with_block_device_mappings():
block_device_mapping = BlockDeviceMapping()
ephemeral_drive = BlockDeviceType()
ephemeral_drive.ephemeral_name = 'ephemeral0'
block_device_mapping['/dev/xvdb'] = ephemeral_drive
snapshot_drive = BlockDeviceType()
snapshot_drive.snapshot_id = "snap-1234abcd"
snapshot_drive.volume_type = "standard"
block_device_mapping['/dev/xvdp'] = snapshot_drive
ebs_drive = BlockDeviceType()
ebs_drive.volume_type = "io1"
ebs_drive.size = 100
ebs_drive.iops = 1000
ebs_drive.delete_on_termination = False
block_device_mapping['/dev/xvdh'] = ebs_drive
conn = boto.connect_autoscale(use_block_device_types=True)
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='m1.small',
key_name='the_keys',
security_groups=["default", "default2"],
user_data=b"This is some user_data",
instance_monitoring=True,
instance_profile_name=
'arn:aws:iam::123456789012:instance-profile/testing',
spot_price=0.1,
block_device_mappings=[block_device_mapping])
conn.create_launch_configuration(config)
launch_config = conn.get_all_launch_configurations()[0]
launch_config.name.should.equal('tester')
launch_config.image_id.should.equal('ami-abcd1234')
launch_config.instance_type.should.equal('m1.small')
launch_config.key_name.should.equal('the_keys')
set(launch_config.security_groups).should.equal(
set(['default', 'default2']))
launch_config.user_data.should.equal(b"This is some user_data")
launch_config.instance_monitoring.enabled.should.equal('true')
launch_config.instance_profile_name.should.equal(
'arn:aws:iam::123456789012:instance-profile/testing')
launch_config.spot_price.should.equal(0.1)
len(launch_config.block_device_mappings).should.equal(3)
returned_mapping = launch_config.block_device_mappings
set(returned_mapping.keys()).should.equal(
set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh']))
returned_mapping['/dev/xvdh'].iops.should.equal(1000)
returned_mapping['/dev/xvdh'].size.should.equal(100)
returned_mapping['/dev/xvdh'].volume_type.should.equal("io1")
returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false
returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd")
returned_mapping['/dev/xvdp'].volume_type.should.equal("standard")
returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')
import os
import boto.ec2
from pprint import pprint
def cloudconfig():
base_path = os.path.dirname(os.path.realpath(__file__))
cloud_config = open(os.path.join(base_path, 'cloud-config'))
return cloud_config.read()
close(os.path.join(base_path, 'cloud-config'))
conn = boto.ec2.connect_to_region("eu-west-1")
from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType
block_device_map = BlockDeviceMapping()
block_dev_type = BlockDeviceType()
block_dev_type.delete_on_termination = True
#block_dev_type.snapshot_id = snapshot_id
block_device_map['/dev/sda'] = block_dev_type
i = conn.run_instances(
# 'ami-1c2e8b6b',
'ami-0c10417b',
key_name = 'jpazdyga',
user_data = cloudconfig(),
instance_type = 't1.micro',
security_groups = ['automat'],
block_device_map = block_device_map)
print(i.instances)
def handleImages(self, action, clc, callback=None):
if action == "DescribeImages":
owner = self.get_argument("Owner", None)
if not owner:
owners = None
else:
owners = [owner]
filters = self.get_filter_args()
return clc.get_all_images(owners, filters, callback)
elif action == "DescribeImageAttribute":
imageid = self.get_argument("ImageId")
attribute = self.get_argument("Attribute")
return clc.get_image_attribute(imageid, attribute, callback)
elif action == "ModifyImageAttribute":
imageid = self.get_argument("ImageId")
attribute = self.get_argument("Attribute")
operation = self.get_argument("OperationType")
users = self.get_argument_list("UserId")
groups = self.get_argument_list("UserGroup")
return clc.modify_image_attribute(imageid, attribute, operation, users, groups, callback)
elif action == "ResetImageAttribute":
imageid = self.get_argument("ImageId")
attribute = self.get_argument("Attribute")
return clc.reset_image_attribute(imageid, attribute, callback)
elif action == "DeregisterImage":
image_id = self.get_argument("ImageId")
return clc.deregister_image(image_id, callback)
elif action == "RegisterImage":
image_location = self.get_argument("ImageLocation", None)
name = self.get_argument("Name")
description = self.get_argument("Description", None)
if description != None:
description = base64.b64decode(description)
architecture = self.get_argument("Architecture", None)
kernel_id = self.get_argument("KernelId", None)
ramdisk_id = self.get_argument("RamdiskId", None)
root_dev_name = self.get_argument("RootDeviceName", None)
snapshot_id = self.get_argument("SnapshotId", None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument("BlockDeviceMapping.1.DeviceName", None)
idx = 1
while mapping:
pre = "BlockDeviceMapping.%d" % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument("%s.VirtualName" % pre, None)
if not (block_dev_type.ephemeral_name):
block_dev_type.no_device = self.get_argument("%s.NoDevice" % pre, "") == "true"
block_dev_type.snapshot_id = self.get_argument("%s.Ebs.SnapshotId" % pre, None)
block_dev_type.size = self.get_argument("%s.Ebs.VolumeSize" % pre, None)
block_dev_type.delete_on_termination = (
self.get_argument("%s.Ebs.DeleteOnTermination" % pre, "") == "true"
)
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument("BlockDeviceMapping.%d.DeviceName" % idx, None)
if snapshot_id:
rootbdm = BlockDeviceType()
rootbdm.snapshot_id = snapshot_id
bdm["/dev/sda1"] = rootbdm
if len(bdm) == 0:
bdm = None
return clc.register_image(
name, image_location, description, architecture, kernel_id, ramdisk_id, root_dev_name, bdm, callback
)
def launch_cluster(conn, OPTS, cluster_name):
print "Setting up security groups..."
master_group = get_or_make_group(conn, cluster_name + "-master")
slave_group = get_or_make_group(conn, cluster_name + "-slaves")
ambari_group = get_or_make_group(conn, cluster_name + "-ambari")
if master_group.rules == []: # Group was just now created
master_group.authorize(src_group=master_group)
master_group.authorize(src_group=slave_group)
master_group.authorize(src_group=ambari_group)
# TODO: Currently Group is completely open
master_group.authorize('tcp', 0, 65535, '0.0.0.0/0')
if slave_group.rules == []: # Group was just now created
slave_group.authorize(src_group=master_group)
slave_group.authorize(src_group=slave_group)
slave_group.authorize(src_group=ambari_group)
# TODO: Currently Group is completely open
slave_group.authorize('tcp', 0, 65535, '0.0.0.0/0')
if ambari_group.rules == []: # Group was just now created
ambari_group.authorize(src_group=master_group)
ambari_group.authorize(src_group=slave_group)
ambari_group.authorize(src_group=ambari_group)
# TODO: Currently Group is completely open
ambari_group.authorize('tcp', 0, 65535, '0.0.0.0/0')
# Check if instances are already running in our groups
if OPTS.resume:
return get_existing_cluster(conn,
OPTS,
cluster_name,
die_on_error=False)
else:
active_nodes = get_existing_cluster(conn,
OPTS,
cluster_name,
die_on_error=False)
if any(active_nodes):
print >> stderr, (
"ERROR: There are already instances running in " +
"group %s or %s" % (master_group.name, slave_group.name))
sys.exit(1)
print "Launching instances..."
try:
image = conn.get_all_images(image_ids=[OPTS.ami])[0]
except:
print >> stderr, "Could not find AMI " + OPTS.ami
sys.exit(1)
# Create block device mapping so that we can add an EBS volume if asked to
block_map = BlockDeviceMapping()
device = BlockDeviceType()
device.ephemeral_name = 'ephemeral0'
device.delete_on_termination = True
block_map["/dev/sdv"] = device
# assume master and ambari hosts have the same instance type
master_type = OPTS.master_instance_type
if master_type == "":
master_type = OPTS.instance_type
# Launch slaves
if OPTS.spot_price != None:
# Launch spot instances with the requested price
num_spot_instances = OPTS.slaves + 2 # slaves, ambari host, master
print("Requesting %d slaves as spot instances with price $%.3f" %
(num_spot_instances, OPTS.spot_price))
zones = get_zones(conn, OPTS)
num_zones = len(zones)
i = 0
my_req_ids = []
ambari_req_ids = []
master_req_ids = []
for zone in zones:
num_slaves_this_zone = get_partition(OPTS.slaves, num_zones, i)
ambari_reqs = conn.request_spot_instances(
price=OPTS.spot_price,
image_id=OPTS.ami,
launch_group="launch-group-%s" % cluster_name,
placement=zone,
count=1,
key_name=OPTS.key_pair,
security_groups=[ambari_group],
instance_type=master_type,
block_device_map=block_map)
master_reqs = conn.request_spot_instances(
price=OPTS.spot_price,
image_id=OPTS.ami,
launch_group="launch-group-%s" % cluster_name,
placement=zone,
count=1,
key_name=OPTS.key_pair,
security_groups=[master_group],
instance_type=master_type,
block_device_map=block_map)
slave_reqs = conn.request_spot_instances(
price=OPTS.spot_price,
image_id=OPTS.ami,
launch_group="launch-group-%s" % cluster_name,
placement=zone,
count=num_slaves_this_zone,
key_name=OPTS.key_pair,
security_groups=[slave_group],
instance_type=OPTS.instance_type,
block_device_map=block_map)
my_req_ids += [req.id for req in slave_reqs]
ambari_req_ids += [req.id for req in ambari_reqs]
master_req_ids += [req.id for req in master_reqs]
i += 1
print "Waiting for spot instances to be granted..."
try:
while True:
time.sleep(10)
reqs = conn.get_all_spot_instance_requests()
id_to_req = {}
for r in reqs:
id_to_req[r.id] = r
active_instance_ids = []
ambari_instance_ids = []
master_instance_ids = []
for i in my_req_ids:
if i in id_to_req and id_to_req[i].state == "active":
active_instance_ids.append(
id_to_req[i].instance_id)
for i in master_req_ids:
if i in id_to_req and id_to_req[i].state == "active":
master_instance_ids.append(
id_to_req[i].instance_id)
for i in ambari_req_ids:
if i in id_to_req and id_to_req[i].state == "active":
ambari_instance_ids.append(
id_to_req[i].instance_id)
if len(active_instance_ids) == OPTS.slaves and len(
master_instance_ids) == 1 and len(
ambari_instance_ids) == 1:
print "All %d slaves, 1 master, 1 ambari host granted" % OPTS.slaves
slave_nodes = []
master_nodes = []
ambari_nodes = []
for r in conn.get_all_instances(active_instance_ids):
slave_nodes += r.instances
for r in conn.get_all_instances(master_instance_ids):
master_nodes += r.instances
for r in conn.get_all_instances(ambari_instance_ids):
ambari_nodes += r.instances
break
else:
print "%d of %d spot instance requests granted, waiting longer" % (
len(active_instance_ids), num_spot_instances)
except Exception as e:
print e
print "Canceling spot instance requests"
conn.cancel_spot_instance_requests(my_req_ids)
# Log a warning if any of these requests actually launched instances:
(master_nodes, slave_nodes,
ambari_nodes) = get_existing_cluster(conn,
OPTS,
cluster_name,
die_on_error=False)
running = len(master_nodes) + len(slave_nodes) + len(
ambari_nodes)
if running:
print >> stderr, (
"WARNING: %d instances are still running" % running)
sys.exit(0)
else:
# Launch non-spot instances
zones = get_zones(conn, OPTS)
num_zones = len(zones)
i = 0
slave_nodes = []
for zone in zones:
num_slaves_this_zone = get_partition(OPTS.slaves, num_zones, i)
if num_slaves_this_zone > 0:
slave_res = image.run(key_name=OPTS.key_pair,
security_groups=[slave_group],
instance_type=OPTS.instance_type,
placement=zone,
min_count=num_slaves_this_zone,
max_count=num_slaves_this_zone,
block_device_map=block_map)
slave_nodes += slave_res.instances
print "Launched %d slaves in %s, regid = %s" % (
num_slaves_this_zone, zone, slave_res.id)
i += 1
# Launch masters
if OPTS.zone == 'all':
OPTS.zone = random.choice(conn.get_all_zones()).name
master_res = image.run(key_name=OPTS.key_pair,
security_groups=[master_group],
instance_type=master_type,
placement=OPTS.zone,
min_count=1,
max_count=1,
block_device_map=block_map)
master_nodes = master_res.instances
print "Launched master in %s, regid = %s" % (zone, master_res.id)
ambari_type = master_type
if OPTS.zone == 'all':
OPTS.zone = random.choice(conn.get_all_zones()).name
ambari_res = image.run(key_name=OPTS.key_pair,
security_groups=[ambari_group],
instance_type=ambari_type,
placement=OPTS.zone,
min_count=1,
max_count=1,
block_device_map=block_map)
ambari_nodes = ambari_res.instances
print "Launched ambari in %s, regid = %s" % (zone, ambari_res.id)
# Return all the instances
return (master_nodes, slave_nodes, ambari_nodes)
def rebundle(reboot_if_needed=False):
"""
Rebundles the EC2 instance that is passed as the -H parameter
This script handles all aspects of the rebundling process and is (almost) fully automated.
Two things should be edited and provided before invoking it: AWS account information
and the desired size of the root volume for the new instance.
:rtype: bool
:return: If instance was successfully rebundled and an AMI ID was received,
return True.
False, otherwise.
"""
_check_fabric_version()
time_start = dt.datetime.utcnow()
print "Rebundling instance '%s'. Start time: %s" % (env.hosts[0], time_start)
_amazon_ec2_environment()
if boto:
# Select appropriate region:
availability_zone = run("curl --silent http://169.254.169.254/latest/meta-data/placement/availability-zone")
instance_region = availability_zone[:-1] # Truncate zone letter to get region name
ec2_conn = _get_ec2_conn(instance_region)
# hostname = env.hosts[0] # -H flag to fab command sets this variable so get only 1st hostname
instance_id = run("curl --silent http://169.254.169.254/latest/meta-data/instance-id")
# Get the size (in GB) of the root partition for the new image
vol_size = _get_root_vol_size(ec2_conn, instance_id)
# Handle reboot if required
if not _reboot(instance_id, reboot_if_needed):
return False # Indicates that rebundling was not completed and should be restarted
_clean() # Clean up the environment before rebundling
image_id = None
kernel_id = run("curl --silent http://169.254.169.254/latest/meta-data/kernel-id")
if ec2_conn and instance_id and availability_zone and kernel_id:
print "Rebundling instance with ID '%s' in region '%s'" % (instance_id, ec2_conn.region.name)
try:
# Need 2 volumes - one for image (rsync) and the other for the snapshot (see instance-to-ebs-ami.sh)
vol = ec2_conn.create_volume(vol_size, availability_zone)
vol2 = ec2_conn.create_volume(vol_size, availability_zone)
# TODO: wait until it becomes 'available'
print "Created 2 new volumes of size '%s' with IDs '%s' and '%s'" % (vol_size, vol.id, vol2.id)
except EC2ResponseError, e:
print(red("Error creating volume: %s" % e))
return False
if vol:
try:
# Attach newly created volumes to the instance
dev_id = '/dev/sdh'
if not _attach(ec2_conn, instance_id, vol.id, dev_id):
print(red("Error attaching volume '%s' to the instance. Aborting." % vol.id))
return False
dev_id = '/dev/sdj'
if not _attach(ec2_conn, instance_id, vol2.id, dev_id):
print(red("Error attaching volume '%s' to the instance. Aborting." % vol2.id))
return False
# Move the file system onto the new volume (with a help of a script)
url = os.path.join(REPO_ROOT_URL, "instance-to-ebs-ami.sh")
# with contextlib.nested(cd('/tmp'), settings(hide('stdout', 'stderr'))):
with cd('/tmp'):
if exists('/tmp/'+os.path.split(url)[1]):
sudo('rm /tmp/'+os.path.split(url)[1])
sudo('wget %s' % url)
sudo('chmod u+x /tmp/%s' % os.path.split(url)[1])
sudo('./%s' % os.path.split(url)[1])
# Detach the new volume
_detach(ec2_conn, instance_id, vol.id)
_detach(ec2_conn, instance_id, vol2.id)
answer = confirm("Would you like to terminate the instance used during rebundling?", default=False)
if answer:
ec2_conn.terminate_instances([instance_id])
# Create a snapshot of the new volume
commit_num = local('cd %s; hg tip | grep changeset | cut -d: -f2' % os.getcwd()).strip()
snap_id = _create_snapshot(ec2_conn, vol.id, "AMI: galaxy-cloudman (using mi-deployment at commit %s)" % commit_num)
# Register the snapshot of the new volume as a machine image (i.e., AMI)
arch = 'x86_64'
root_device_name = '/dev/sda1'
# Extra info on how EBS image registration is done: http://markmail.org/message/ofgkyecjktdhofgz
# http://www.elastician.com/2009/12/creating-ebs-backed-ami-from-s3-backed.html
# http://www.shlomoswidler.com/2010/01/creating-consistent-snapshots-of-live.html
ebs = BlockDeviceType()
ebs.snapshot_id = snap_id
ebs.delete_on_termination = True
ephemeral0_device_name = '/dev/sdb'
ephemeral0 = BlockDeviceType()
ephemeral0.ephemeral_name = 'ephemeral0'
ephemeral1_device_name = '/dev/sdc'
ephemeral1 = BlockDeviceType()
ephemeral1.ephemeral_name = 'ephemeral1'
# ephemeral2_device_name = '/dev/sdd' # Needed for instances w/ 3 ephemeral disks
# ephemeral2 = BlockDeviceType()
# ephemeral2.ephemeral_name = 'ephemeral2'
# ephemeral3_device_name = '/dev/sde' # Needed for instances w/ 4 ephemeral disks
# ephemeral3 = BlockDeviceType()
# ephemeral3.ephemeral_name = 'ephemeral3'
block_map = BlockDeviceMapping()
block_map[root_device_name] = ebs
block_map[ephemeral0_device_name] = ephemeral0
block_map[ephemeral1_device_name] = ephemeral1
print(yellow('galaxy-cloudman-%s' % time_start.strftime("%Y-%m-%d")))
name, desc = _get_image_name()
image_id = ec2_conn.register_image(name, description=desc, architecture=arch,
kernel_id=kernel_id, root_device_name=root_device_name, block_device_map=block_map)
answer = confirm("Volume with ID '%s' was created and used to make this AMI but is not longer needed. Would you like to delete it?" % vol.id)
if answer:
ec2_conn.delete_volume(vol.id)
print "Deleting the volume (%s) used for rsync only" % vol2.id
ec2_conn.delete_volume(vol2.id)
print(green("--------------------------"))
print(green("Finished creating new machine image. Image ID: '%s'" % (image_id)))
print(green("--------------------------"))
answer = confirm("Would you like to make this machine image public?", default=False)
if image_id and answer:
ec2_conn.modify_image_attribute(image_id, attribute='launchPermission', operation='add', groups=['all'])
except EC2ResponseError, e:
print(red("Error creating image: %s" % e))
return False
else:
print(red("Error creating new volume"))
return False
def launch_cluster(conn, OPTS, cluster_name):
print "Setting up security groups..."
master_group = get_or_make_group(conn, cluster_name + "-master")
slave_group = get_or_make_group(conn, cluster_name + "-slaves")
ambari_group = get_or_make_group(conn, cluster_name + "-ambari")
if master_group.rules == []: # Group was just now created
master_group.authorize(src_group=master_group)
master_group.authorize(src_group=slave_group)
master_group.authorize(src_group=ambari_group)
# TODO: Currently Group is completely open
master_group.authorize('tcp', 0, 65535, '0.0.0.0/0')
if slave_group.rules == []: # Group was just now created
slave_group.authorize(src_group=master_group)
slave_group.authorize(src_group=slave_group)
slave_group.authorize(src_group=ambari_group)
# TODO: Currently Group is completely open
slave_group.authorize('tcp', 0, 65535, '0.0.0.0/0')
if ambari_group.rules == []: # Group was just now created
ambari_group.authorize(src_group=master_group)
ambari_group.authorize(src_group=slave_group)
ambari_group.authorize(src_group=ambari_group)
# TODO: Currently Group is completely open
ambari_group.authorize('tcp', 0, 65535, '0.0.0.0/0')
# Check if instances are already running in our groups
if OPTS.resume:
return get_existing_cluster(conn, OPTS, cluster_name, die_on_error=False)
else:
active_nodes = get_existing_cluster(conn, OPTS, cluster_name, die_on_error=False)
if any(active_nodes):
print >> stderr, ("ERROR: There are already instances running in " +
"group %s or %s" % (master_group.name, slave_group.name))
sys.exit(1)
print "Launching instances..."
try:
image = conn.get_all_images(image_ids=[OPTS.ami])[0]
except:
print >> stderr, "Could not find AMI " + OPTS.ami
sys.exit(1)
# Create block device mapping so that we can add an EBS volume if asked to
block_map = BlockDeviceMapping()
device = BlockDeviceType()
device.ephemeral_name = 'ephemeral0'
device.delete_on_termination = True
block_map["/dev/sdv"] = device
# assume master and ambari hosts have the same instance type
master_type = OPTS.master_instance_type
if master_type == "":
master_type = OPTS.instance_type
# Launch slaves
if OPTS.spot_price != None:
# Launch spot instances with the requested price
num_spot_instances = OPTS.slaves + 2 # slaves, ambari host, master
print ("Requesting %d slaves as spot instances with price $%.3f" %
(num_spot_instances, OPTS.spot_price))
zones = get_zones(conn, OPTS)
num_zones = len(zones)
i = 0
my_req_ids = []
ambari_req_ids = []
master_req_ids = []
for zone in zones:
num_slaves_this_zone = get_partition(OPTS.slaves, num_zones, i)
ambari_reqs = conn.request_spot_instances(
price = OPTS.spot_price,
image_id = OPTS.ami,
launch_group = "launch-group-%s" % cluster_name,
placement = zone,
count = 1,
key_name = OPTS.key_pair,
security_groups = [ambari_group],
instance_type = master_type,
block_device_map = block_map)
master_reqs = conn.request_spot_instances(
price = OPTS.spot_price,
image_id = OPTS.ami,
launch_group = "launch-group-%s" % cluster_name,
placement = zone,
count = 1,
key_name = OPTS.key_pair,
security_groups = [master_group],
instance_type = master_type,
block_device_map = block_map)
slave_reqs = conn.request_spot_instances(
price = OPTS.spot_price,
image_id = OPTS.ami,
launch_group = "launch-group-%s" % cluster_name,
placement = zone,
count = num_slaves_this_zone,
key_name = OPTS.key_pair,
security_groups = [slave_group],
instance_type = OPTS.instance_type,
block_device_map = block_map)
my_req_ids += [req.id for req in slave_reqs]
ambari_req_ids += [req.id for req in ambari_reqs]
master_req_ids += [req.id for req in master_reqs]
i += 1
print "Waiting for spot instances to be granted..."
try:
while True:
time.sleep(10)
reqs = conn.get_all_spot_instance_requests()
id_to_req = {}
for r in reqs:
id_to_req[r.id] = r
active_instance_ids = []
ambari_instance_ids = []
master_instance_ids = []
for i in my_req_ids:
if i in id_to_req and id_to_req[i].state == "active":
active_instance_ids.append(id_to_req[i].instance_id)
for i in master_req_ids:
if i in id_to_req and id_to_req[i].state == "active":
master_instance_ids.append(id_to_req[i].instance_id)
for i in ambari_req_ids:
if i in id_to_req and id_to_req[i].state == "active":
ambari_instance_ids.append(id_to_req[i].instance_id)
if len(active_instance_ids) == OPTS.slaves and len(master_instance_ids) == 1 and len(ambari_instance_ids) == 1:
print "All %d slaves, 1 master, 1 ambari host granted" % OPTS.slaves
slave_nodes = []
master_nodes = []
ambari_nodes = []
for r in conn.get_all_instances(active_instance_ids):
slave_nodes += r.instances
for r in conn.get_all_instances(master_instance_ids):
master_nodes += r.instances
for r in conn.get_all_instances(ambari_instance_ids):
ambari_nodes += r.instances
break
else:
print "%d of %d spot instance requests granted, waiting longer" % (
len(active_instance_ids), num_spot_instances)
except Exception as e:
print e
print "Canceling spot instance requests"
conn.cancel_spot_instance_requests(my_req_ids)
# Log a warning if any of these requests actually launched instances:
(master_nodes, slave_nodes, ambari_nodes) = get_existing_cluster(
conn, OPTS, cluster_name, die_on_error=False)
running = len(master_nodes) + len(slave_nodes) + len(ambari_nodes)
if running:
print >> stderr, ("WARNING: %d instances are still running" % running)
sys.exit(0)
else:
# Launch non-spot instances
zones = get_zones(conn, OPTS)
num_zones = len(zones)
i = 0
slave_nodes = []
for zone in zones:
num_slaves_this_zone = get_partition(OPTS.slaves, num_zones, i)
if num_slaves_this_zone > 0:
slave_res = image.run(key_name = OPTS.key_pair,
security_groups = [slave_group],
instance_type = OPTS.instance_type,
placement = zone,
min_count = num_slaves_this_zone,
max_count = num_slaves_this_zone,
block_device_map = block_map)
slave_nodes += slave_res.instances
print "Launched %d slaves in %s, regid = %s" % (num_slaves_this_zone,
zone, slave_res.id)
i += 1
# Launch masters
if OPTS.zone == 'all':
OPTS.zone = random.choice(conn.get_all_zones()).name
master_res = image.run(key_name = OPTS.key_pair,
security_groups = [master_group],
instance_type = master_type,
placement = OPTS.zone,
min_count = 1,
max_count = 1,
block_device_map = block_map)
master_nodes = master_res.instances
print "Launched master in %s, regid = %s" % (zone, master_res.id)
ambari_type = master_type
if OPTS.zone == 'all':
OPTS.zone = random.choice(conn.get_all_zones()).name
ambari_res = image.run(key_name = OPTS.key_pair,
security_groups = [ambari_group],
instance_type = ambari_type,
placement = OPTS.zone,
min_count = 1,
max_count = 1,
block_device_map = block_map)
ambari_nodes = ambari_res.instances
print "Launched ambari in %s, regid = %s" % (zone, ambari_res.id)
# Return all the instances
return (master_nodes, slave_nodes, ambari_nodes)
def create_instance(name, config, region, key_name, ssh_key, instance_data,
deploypass, loaned_to, loan_bug, create_ami,
ignore_subnet_check, max_attempts):
"""Creates an AMI instance with the given name and config. The config must
specify things like ami id."""
conn = get_aws_connection(region)
# Make sure we don't request the same things twice
token = str(uuid.uuid4())[:16]
instance_data = instance_data.copy()
instance_data['name'] = name
instance_data['domain'] = config['domain']
instance_data['hostname'] = '{name}.{domain}'.format(
name=name, domain=config['domain'])
ami = conn.get_all_images(image_ids=[config["ami"]])[0]
bdm = None
if 'device_map' in config:
bdm = BlockDeviceMapping()
for device, device_info in config['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
# Overwrite root device size for HVM instances, since they cannot
# be resized online
if ami.virtualization_type == "hvm" and \
ami.root_device_name == device:
bd.size = ami.block_device_mapping[ami.root_device_name].size
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
if device_info.get("volume_type"):
bd.volume_type = device_info["volume_type"]
if device_info["volume_type"] == "io1" \
and device_info.get("iops"):
bd.iops = device_info["iops"]
bdm[device] = bd
interfaces = make_instance_interfaces(
region, instance_data['hostname'], ignore_subnet_check,
config.get('subnet_ids'), config.get('security_group_ids', []),
config.get("use_public_ip"))
keep_going, attempt = True, 1
while keep_going:
try:
puppet_master = pick_puppet_master(instance_data.get('puppet_masters'))
user_data = user_data_from_template(config['type'], {
"puppet_server": puppet_master,
"fqdn": instance_data['hostname'],
"hostname": instance_data['name'],
"domain": instance_data['domain'],
"dns_search_domain": config.get('dns_search_domain'),
"password": deploypass,
"moz_instance_type": config['type'],
"region_dns_atom": get_region_dns_atom(region)})
reservation = conn.run_instances(
image_id=config['ami'],
key_name=key_name,
instance_type=config['instance_type'],
block_device_map=bdm,
client_token=token,
disable_api_termination=config.get('disable_api_termination'),
user_data=user_data,
instance_profile_name=config.get('instance_profile_name'),
network_interfaces=interfaces,
)
break
except boto.exception.BotoServerError:
log.exception("Cannot start an instance")
time.sleep(10)
if max_attempts:
attempt += 1
keep_going = max_attempts >= attempt
instance = reservation.instances[0]
log.info("instance %s created, waiting to come up", instance)
# Wait for the instance to come up
wait_for_status(instance, "state", "running", "update")
instance.add_tag('Name', name)
instance.add_tag('FQDN', instance_data['hostname'])
instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z",
time.gmtime()))
instance.add_tag('moz-type', config['type'])
if loaned_to:
instance.add_tag("moz-loaned-to", loaned_to)
if loan_bug:
instance.add_tag("moz-bug", loan_bug)
log.info("assimilating %s", instance)
instance.add_tag('moz-state', 'pending')
keep_going, attempt = True, 1
while keep_going:
try:
# Don't reboot if need to create ami
reboot = not create_ami
assimilate_instance(instance=instance, config=config,
ssh_key=ssh_key, instance_data=instance_data,
deploypass=deploypass, reboot=reboot)
break
except NetworkError as e:
# it takes a while for the machine to start/reboot so the
# NetworkError exception is quite common, just log the error,
# without the full stack trace
log.warn("cannot connect; instance may still be starting %s (%s, %s) - %s,"
"retrying in %d sec ...", instance_data['hostname'], instance.id,
instance.private_ip_address, e, FAILURE_TIMEOUT)
time.sleep(FAILURE_TIMEOUT)
except:
# any other exception
log.warn("problem assimilating %s (%s, %s), retrying in "
"%d sec ...", instance_data['hostname'], instance.id,
instance.private_ip_address, FAILURE_TIMEOUT, exc_info=True)
time.sleep(FAILURE_TIMEOUT)
if max_attempts:
attempt += 1
keep_going = max_attempts >= attempt
instance.add_tag('moz-state', 'ready')
if create_ami:
ami_name = "spot-%s-%s" % (
config['type'], time.strftime("%Y-%m-%d-%H-%M", time.gmtime()))
log.info("Generating AMI %s", ami_name)
ami_cleanup(mount_point="/", distro=config["distro"])
root_bd = instance.block_device_mapping[instance.root_device_name]
volume = instance.connection.get_all_volumes(
volume_ids=[root_bd.volume_id])[0]
# The instance has to be stopped to flush EBS caches
instance.stop()
wait_for_status(instance, 'state', 'stopped', 'update')
ami = volume_to_ami(volume=volume, ami_name=ami_name,
arch=instance.architecture,
virtualization_type=instance.virtualization_type,
kernel_id=instance.kernel,
root_device_name=instance.root_device_name,
tags=config["tags"])
log.info("AMI %s (%s) is ready", ami_name, ami.id)
log.warn("Terminating %s", instance)
instance.terminate()
def test_create_launch_configuration_with_block_device_mappings():
block_device_mapping = BlockDeviceMapping()
ephemeral_drive = BlockDeviceType()
ephemeral_drive.ephemeral_name = "ephemeral0"
block_device_mapping["/dev/xvdb"] = ephemeral_drive
snapshot_drive = BlockDeviceType()
snapshot_drive.snapshot_id = "snap-1234abcd"
snapshot_drive.volume_type = "standard"
block_device_mapping["/dev/xvdp"] = snapshot_drive
ebs_drive = BlockDeviceType()
ebs_drive.volume_type = "io1"
ebs_drive.size = 100
ebs_drive.iops = 1000
ebs_drive.delete_on_termination = False
block_device_mapping["/dev/xvdh"] = ebs_drive
conn = boto.connect_autoscale(use_block_device_types=True)
config = LaunchConfiguration(
name="tester",
image_id="ami-abcd1234",
instance_type="m1.small",
key_name="the_keys",
security_groups=["default", "default2"],
user_data=b"This is some user_data",
instance_monitoring=True,
instance_profile_name="arn:aws:iam::{}:instance-profile/testing".format(
ACCOUNT_ID
),
spot_price=0.1,
block_device_mappings=[block_device_mapping],
)
conn.create_launch_configuration(config)
launch_config = conn.get_all_launch_configurations()[0]
launch_config.name.should.equal("tester")
launch_config.image_id.should.equal("ami-abcd1234")
launch_config.instance_type.should.equal("m1.small")
launch_config.key_name.should.equal("the_keys")
set(launch_config.security_groups).should.equal(set(["default", "default2"]))
launch_config.user_data.should.equal(b"This is some user_data")
launch_config.instance_monitoring.enabled.should.equal("true")
launch_config.instance_profile_name.should.equal(
"arn:aws:iam::{}:instance-profile/testing".format(ACCOUNT_ID)
)
launch_config.spot_price.should.equal(0.1)
len(launch_config.block_device_mappings).should.equal(3)
returned_mapping = launch_config.block_device_mappings
set(returned_mapping.keys()).should.equal(
set(["/dev/xvdb", "/dev/xvdp", "/dev/xvdh"])
)
returned_mapping["/dev/xvdh"].iops.should.equal(1000)
returned_mapping["/dev/xvdh"].size.should.equal(100)
returned_mapping["/dev/xvdh"].volume_type.should.equal("io1")
returned_mapping["/dev/xvdh"].delete_on_termination.should.be.false
returned_mapping["/dev/xvdp"].snapshot_id.should.equal("snap-1234abcd")
returned_mapping["/dev/xvdp"].volume_type.should.equal("standard")
returned_mapping["/dev/xvdb"].ephemeral_name.should.equal("ephemeral0")
import boto.ec2
from pprint import pprint
def cloudconfig():
base_path = os.path.dirname(os.path.realpath(__file__))
cloud_config = open(os.path.join(base_path, 'cloud-config'))
return cloud_config.read()
close(os.path.join(base_path, 'cloud-config'))
conn = boto.ec2.connect_to_region("eu-west-1")
from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType
block_device_map = BlockDeviceMapping()
block_dev_type = BlockDeviceType()
block_dev_type.delete_on_termination = True
#block_dev_type.snapshot_id = snapshot_id
block_device_map['/dev/sda'] = block_dev_type
i = conn.run_instances(
# 'ami-1c2e8b6b',
'ami-0c10417b',
key_name='jpazdyga',
user_data=cloudconfig(),
instance_type='t1.micro',
security_groups=['automat'],
block_device_map=block_device_map)
print(i.instances)
def create_instance(name, config, region, secrets, key_name, instance_data,
deploypass, loaned_to, loan_bug):
"""Creates an AMI instance with the given name and config. The config must
specify things like ami id."""
conn = get_connection(
region,
aws_access_key_id=secrets['aws_access_key_id'],
aws_secret_access_key=secrets['aws_secret_access_key'])
vpc = get_vpc(connection=conn,
aws_access_key_id=secrets['aws_access_key_id'],
aws_secret_access_key=secrets['aws_secret_access_key'])
# Make sure we don't request the same things twice
token = str(uuid.uuid4())[:16]
instance_data = instance_data.copy()
instance_data['name'] = name
instance_data['hostname'] = '{name}.{domain}'.format(
name=name, domain=config['domain'])
ami = conn.get_all_images(image_ids=[config["ami"]])[0]
bdm = None
if 'device_map' in config:
bdm = BlockDeviceMapping()
for device, device_info in config['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
# Overwrite root device size for HVM instances, since they cannot
# be resized online
if ami.virtualization_type == "hvm" and \
ami.root_device_name == device:
bd.size = ami.block_device_mapping[ami.root_device_name].size
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
bdm[device] = bd
ip_address = get_ip(instance_data['hostname'])
subnet_id = None
if ip_address:
s_id = get_subnet_id(vpc, ip_address)
if s_id in config['subnet_ids']:
if ip_available(conn, ip_address):
subnet_id = s_id
else:
log.warning("%s already assigned" % ip_address)
if not ip_address or not subnet_id:
ip_address = None
subnet_id = choice(config.get('subnet_ids'))
interface = NetworkInterfaceSpecification(
subnet_id=subnet_id,
private_ip_address=ip_address,
delete_on_termination=True,
groups=config.get('security_group_ids', []),
associate_public_ip_address=config.get("use_public_ip"))
interfaces = NetworkInterfaceCollection(interface)
while True:
try:
reservation = conn.run_instances(
image_id=config['ami'],
key_name=key_name,
instance_type=config['instance_type'],
block_device_map=bdm,
client_token=token,
disable_api_termination=bool(
config.get('disable_api_termination')),
network_interfaces=interfaces,
instance_profile_name=config.get("instance_profile_name"),
)
break
except boto.exception.BotoServerError:
log.exception("Cannot start an instance")
time.sleep(10)
instance = reservation.instances[0]
log.info("instance %s created, waiting to come up", instance)
# Wait for the instance to come up
while True:
try:
instance.update()
if instance.state == 'running':
break
except Exception:
log.warn("waiting for instance to come up, retrying in 10 sec...")
time.sleep(10)
instance.add_tag('Name', name)
instance.add_tag('FQDN', instance_data['hostname'])
instance.add_tag('created',
time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime()))
instance.add_tag('moz-type', config['type'])
if loaned_to:
instance.add_tag("moz-loaned-to", loaned_to)
if loan_bug:
instance.add_tag("moz-bug", loan_bug)
log.info("assimilating %s", instance)
instance.add_tag('moz-state', 'pending')
while True:
try:
assimilate(instance.private_ip_address, config, instance_data,
deploypass)
break
except:
log.warn("problem assimilating %s (%s), retrying in 10 sec ...",
instance_data['hostname'], instance.id)
time.sleep(10)
instance.add_tag('moz-state', 'ready')
def create_instance(name, config, region, key_name, ssh_key, instance_data,
deploypass, loaned_to, loan_bug, create_ami,
ignore_subnet_check, max_attempts):
"""Creates an AMI instance with the given name and config. The config must
specify things like ami id."""
conn = get_aws_connection(region)
# Make sure we don't request the same things twice
token = str(uuid.uuid4())[:16]
instance_data = instance_data.copy()
instance_data['name'] = name
instance_data['domain'] = config['domain']
instance_data['hostname'] = '{name}.{domain}'.format(
name=name, domain=config['domain'])
ami = conn.get_all_images(image_ids=[config["ami"]])[0]
bdm = None
if 'device_map' in config:
bdm = BlockDeviceMapping()
for device, device_info in config['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
# Overwrite root device size for HVM instances, since they cannot
# be resized online
if ami.virtualization_type == "hvm" and \
ami.root_device_name == device:
bd.size = ami.block_device_mapping[ami.root_device_name].size
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
bdm[device] = bd
interfaces = make_instance_interfaces(region, instance_data['hostname'],
ignore_subnet_check,
config.get('subnet_ids'),
config.get('security_group_ids', []),
config.get("use_public_ip"))
keep_going, attempt = True, 1
while keep_going:
try:
if 'user_data_file' in config:
user_data = open(config['user_data_file']).read()
else:
user_data = get_user_data_tmpl(config['type'])
if user_data:
user_data = user_data.format(
puppet_server=instance_data.get('default_puppet_server'),
fqdn=instance_data['hostname'],
hostname=instance_data['name'],
domain=instance_data['domain'],
dns_search_domain=config.get('dns_search_domain'),
password=deploypass,
moz_instance_type=config['type'],
region_dns_atom=get_region_dns_atom(region),
)
reservation = conn.run_instances(
image_id=config['ami'],
key_name=key_name,
instance_type=config['instance_type'],
block_device_map=bdm,
client_token=token,
disable_api_termination=config.get('disable_api_termination'),
user_data=user_data,
instance_profile_name=config.get('instance_profile_name'),
network_interfaces=interfaces,
)
break
except boto.exception.BotoServerError:
log.exception("Cannot start an instance")
time.sleep(10)
if max_attempts:
attempt += 1
keep_going = max_attempts >= attempt
instance = reservation.instances[0]
log.info("instance %s created, waiting to come up", instance)
# Wait for the instance to come up
wait_for_status(instance, "state", "running", "update")
instance.add_tag('Name', name)
instance.add_tag('FQDN', instance_data['hostname'])
instance.add_tag('created',
time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime()))
instance.add_tag('moz-type', config['type'])
if loaned_to:
instance.add_tag("moz-loaned-to", loaned_to)
if loan_bug:
instance.add_tag("moz-bug", loan_bug)
log.info("assimilating %s", instance)
instance.add_tag('moz-state', 'pending')
keep_going, attempt = True, 1
while keep_going:
try:
# Don't reboot if need to create ami
reboot = not create_ami
assimilate_instance(instance=instance,
config=config,
ssh_key=ssh_key,
instance_data=instance_data,
deploypass=deploypass,
reboot=reboot)
break
except NetworkError as e:
# it takes a while for the machine to start/reboot so the
# NetworkError exception is quite common, just log the error,
# without the full stack trace
log.warn(
"cannot connect; instance may still be starting %s (%s, %s) - %s,"
"retrying in %d sec ...", instance_data['hostname'],
instance.id, instance.private_ip_address, e, FAILURE_TIMEOUT)
time.sleep(FAILURE_TIMEOUT)
except:
# any other exception
log.warn(
"problem assimilating %s (%s, %s), retrying in "
"%d sec ...",
instance_data['hostname'],
instance.id,
instance.private_ip_address,
FAILURE_TIMEOUT,
exc_info=True)
time.sleep(FAILURE_TIMEOUT)
if max_attempts:
attempt += 1
keep_going = max_attempts >= attempt
instance.add_tag('moz-state', 'ready')
if create_ami:
ami_name = "spot-%s-%s" % (
config['type'], time.strftime("%Y-%m-%d-%H-%M", time.gmtime()))
log.info("Generating AMI %s", ami_name)
ami_cleanup(mount_point="/", distro=config["distro"])
root_bd = instance.block_device_mapping[instance.root_device_name]
volume = instance.connection.get_all_volumes(
volume_ids=[root_bd.volume_id])[0]
# The instance has to be stopped to flush EBS caches
instance.stop()
wait_for_status(instance, 'state', 'stopped', 'update')
ami = volume_to_ami(volume=volume,
ami_name=ami_name,
arch=instance.architecture,
virtualization_type=instance.virtualization_type,
kernel_id=instance.kernel,
root_device_name=instance.root_device_name,
tags=config["tags"])
log.info("AMI %s (%s) is ready", ami_name, ami.id)
log.warn("Terminating %s", instance)
instance.terminate()
def create_cluster(conn, args):
if args.identity_file is None:
print("ERROR: Must provide an identity file (-i) for ssh connections.",
file=stderr)
sys.exit(1)
if args.key_pair is None:
print("ERROR: Must provide a key pair name (-k) to use on instances.",
file=stderr)
sys.exit(1)
# make or get the security group.
security_group = get_or_make_group(conn, args.name, args.vpc_id)
# set the inbound permission rules
if len(security_group.rules) == 0:
if __name__ == '__main__':
if args.vpc_id is None:
security_group.authorize(src_group=security_group)
else:
security_group.authorize('tcp', 22, 22,
args.authorized_address)
security_group.authorize('tcp', 8888, 8888,
args.authorized_address)
security_group.authorize('tcp', 7000, 7000,
args.authorized_address)
security_group.authorize('tcp', 7001, 7001,
args.authorized_address)
security_group.authorize('tcp', 7199, 7199,
args.authorized_address)
security_group.authorize('tcp', 9042, 9042,
args.authorized_address)
security_group.authorize('tcp', 9160, 9160,
args.authorized_address)
else:
print("Security group already exists, skipping creation.")
instances = cluster_nodes(conn, args.name)
if any(instances):
additional_tags = {}
for i in instances:
i.add_tags(
dict(additional_tags,
Name="{cn}-node-{iid}".format(cn=args.name, iid=i.id)))
return instances
else:
print(
"Launching {m} instances for cluster...".format(m=args.node_count))
try:
image = conn.get_all_images(image_ids=args.ami)[0]
block_map = BlockDeviceMapping()
if args.ebs_vol_size > 0:
if args.instance_type.startswith('m3.'):
for i in range(get_num_disks(args.instance_type)):
device = BlockDeviceType()
device.ephemeral_name = "ephemeral%d" % i
name = "/dev/sd" + string.ascii_letters[i + 1]
block_map[name] = device
else:
device = EBSBlockDeviceType()
device.size = args.ebs_vol_size
device.volume_type = args.ebs_vol_type
device.delete_on_termination = True
key = "/dev/sd" + chr(ord('s') + 1)
block_map[key] = device
nodes = image.run(key_name=args.key_pair,
security_group_ids=[security_group.id],
instance_type="",
placement=args.zone,
min_count=args.node_count,
max_count=args.node_count,
block_device_map=block_map,
subnet_id=None,
placement_group=None,
user_data=None,
instance_initiated_shutdown_behavior="stop",
instance_profile_name=None)
print("Waiting for AWS to propagate instance metadata...")
time.sleep(15)
additional_tags = {}
for node in nodes.instances:
node.add_tags(
dict(additional_tags,
Name="{cn}-node-{iid}".format(cn=args.name,
iid=node.id)))
return nodes.instances
except Exception as e:
print("Caught exception: ", e)
print("ERROR: Could not find AMI " + args.ami, file=stderr)
sys.exit(1)
def handleRunInstances(self, action, clc, user_data_file, callback):
image_id = self.get_argument("ImageId")
min = self.get_argument("MinCount", "1")
max = self.get_argument("MaxCount", "1")
key = self.get_argument("KeyName", None)
groups = self.get_argument_list("SecurityGroup")
sec_group_ids = self.get_argument_list("SecurityGroupId")
if user_data_file:
user_data = user_data_file
else:
user_data = self.get_argument("UserData", "")
user_data = base64.b64decode(user_data)
addr_type = self.get_argument("AddressingType", None)
vm_type = self.get_argument("InstanceType", None)
placement = self.get_argument("Placement.AvailabilityZone", None)
placement_group = self.get_argument("Placement.GroupName", None)
tenancy = self.get_argument("Placement.Tenancy", None)
kernel = self.get_argument("KernelId", None)
ramdisk = self.get_argument("RamdiskId", None)
monitoring = False
if self.get_argument("Monitoring.Enabled", "") == "true":
monitoring = True
subnet = self.get_argument("SubnetId", None)
private_ip = self.get_argument("PrivateIpAddress", None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument("BlockDeviceMapping.1.DeviceName", None)
idx = 1
while mapping:
pre = "BlockDeviceMapping.%d" % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument("%s.VirtualName" % pre, None)
if not (block_dev_type.ephemeral_name):
block_dev_type.no_device = self.get_argument("%s.NoDevice" % pre, "") == "true"
block_dev_type.snapshot_id = self.get_argument("%s.Ebs.SnapshotId" % pre, None)
block_dev_type.size = self.get_argument("%s.Ebs.VolumeSize" % pre, None)
block_dev_type.delete_on_termination = (
self.get_argument("%s.Ebs.DeleteOnTermination" % pre, "") == "true"
)
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument("BlockDeviceMapping.%d.DeviceName" % idx, None)
if len(bdm) == 0:
bdm = None
api_termination = False
if self.get_argument("DisableApiTermination", "") == "true":
api_termination = True
instance_shutdown = False
if self.get_argument("InstanceInitiatedShutdownBehavior", "") == "true":
instance_shutdown = True
token = self.get_argument("ClientToken", None)
addition_info = self.get_argument("AdditionInfo", None)
instance_profile_name = self.get_argument("IamInstanceProfile.Name", None)
instance_profile_arn = self.get_argument("IamInstanceProfile.Arn", None)
return clc.run_instances(
image_id,
min_count=min,
max_count=max,
key_name=key,
security_groups=groups,
user_data=user_data,
addressing_type=addr_type,
instance_type=vm_type,
placement=placement,
kernel_id=kernel,
ramdisk_id=ramdisk,
monitoring_enabled=monitoring,
subnet_id=subnet,
block_device_map=bdm,
disable_api_termination=api_termination,
instance_initiated_shutdown_behavior=instance_shutdown,
private_ip_address=private_ip,
placement_group=placement_group,
client_token=token,
security_group_ids=sec_group_ids,
additional_info=addition_info,
instance_profile_name=instance_profile_name,
instance_profile_arn=instance_profile_arn,
tenancy=tenancy,
callback=callback,
)
def create_instance(name, config, region, key_name, instance_data,
deploypass, loaned_to, loan_bug):
"""Creates an AMI instance with the given name and config. The config must
specify things like ami id."""
conn = get_aws_connection(region)
vpc = get_vpc(region)
# Make sure we don't request the same things twice
token = str(uuid.uuid4())[:16]
instance_data = instance_data.copy()
instance_data['name'] = name
instance_data['hostname'] = '{name}.{domain}'.format(
name=name, domain=config['domain'])
ami = conn.get_all_images(image_ids=[config["ami"]])[0]
bdm = None
if 'device_map' in config:
bdm = BlockDeviceMapping()
for device, device_info in config['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
# Overwrite root device size for HVM instances, since they cannot
# be resized online
if ami.virtualization_type == "hvm" and \
ami.root_device_name == device:
bd.size = ami.block_device_mapping[ami.root_device_name].size
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
bdm[device] = bd
ip_address = get_ip(instance_data['hostname'])
subnet_id = None
if ip_address:
s_id = get_subnet_id(vpc, ip_address)
if s_id in config['subnet_ids']:
if ip_available(conn, ip_address):
subnet_id = s_id
else:
log.warning("%s already assigned" % ip_address)
if not ip_address or not subnet_id:
ip_address = None
subnet_id = choice(config.get('subnet_ids'))
interface = NetworkInterfaceSpecification(
subnet_id=subnet_id, private_ip_address=ip_address,
delete_on_termination=True,
groups=config.get('security_group_ids', []),
associate_public_ip_address=config.get("use_public_ip")
)
interfaces = NetworkInterfaceCollection(interface)
while True:
try:
reservation = conn.run_instances(
image_id=config['ami'],
key_name=key_name,
instance_type=config['instance_type'],
block_device_map=bdm,
client_token=token,
disable_api_termination=bool(config.get('disable_api_termination')),
network_interfaces=interfaces,
instance_profile_name=config.get("instance_profile_name"),
)
break
except boto.exception.BotoServerError:
log.exception("Cannot start an instance")
time.sleep(10)
instance = reservation.instances[0]
log.info("instance %s created, waiting to come up", instance)
# Wait for the instance to come up
wait_for_status(instance, "state", "running", "update")
instance.add_tag('Name', name)
instance.add_tag('FQDN', instance_data['hostname'])
instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z",
time.gmtime()))
instance.add_tag('moz-type', config['type'])
if loaned_to:
instance.add_tag("moz-loaned-to", loaned_to)
if loan_bug:
instance.add_tag("moz-bug", loan_bug)
log.info("assimilating %s", instance)
instance.add_tag('moz-state', 'pending')
while True:
try:
assimilate(instance.private_ip_address, config, instance_data,
deploypass)
break
except:
log.warn("problem assimilating %s (%s), retrying in 10 sec ...",
instance_data['hostname'], instance.id)
time.sleep(10)
instance.add_tag('moz-state', 'ready')
def do_create(self, ntry, params):
"""
Create stage of testing
@param ntry: number of try
@type ntry: int
@param params: list of testing parameters
@type params: list
"""
result = None
logging.debug(self.getName() + ': trying to create instance ' + params['iname'] + ', ntry ' + str(ntry))
ntry += 1
try:
bmap = BlockDeviceMapping()
for device in params['bmap']:
if not 'name' in device.keys():
logging.debug(self.getName() + ': bad device ' + str(device))
continue
d = BlockDeviceType()
if 'size' in device.keys():
d.size = device['size']
if 'delete_on_termination' in device.keys():
d.delete_on_termination = device['delete_on_termination']
if 'ephemeral_name' in device.keys():
d.ephemeral_name = device['ephemeral_name']
bmap[device['name']] = d
reg = boto.ec2.get_region(params['region'], aws_access_key_id=ec2_key, aws_secret_access_key=ec2_secret_key)
connection = reg.connect(aws_access_key_id=ec2_key, aws_secret_access_key=ec2_secret_key)
(ssh_key_name, ssh_key) = yamlconfig['ssh'][params['region']]
# all handled params to be put in here
boto_params = ['ami', 'subnet_id']
for param in boto_params:
params.setdefault(param)
reservation = connection.run_instances(
params['ami'],
instance_type=params['ec2name'],
key_name=ssh_key_name,
block_device_map=bmap,
subnet_id=params['subnet_id'],
user_data=params['userdata']
)
myinstance = reservation.instances[0]
count = 0
# Sometimes EC2 failes to return something meaningful without small timeout between run_instances() and update()
time.sleep(10)
while myinstance.update() == 'pending' and count < maxwait / 5:
# Waiting out instance to appear
logging.debug(params['iname'] + '... waiting...' + str(count))
time.sleep(5)
count += 1
connection.close()
instance_state = myinstance.update()
if instance_state == 'running':
# Instance appeared - scheduling 'setup' stage
myinstance.add_tag('Name', params['name'])
result = myinstance.__dict__
logging.info(self.getName() + ': created instance ' + params['iname'] + ', ' + result['id'] + ':' + result['public_dns_name'])
# packing creation results into params
params['id'] = result['id']
params['instance'] = result.copy()
mainq.put((0, 'setup', params))
return
elif instance_state == 'pending':
# maxwait seconds is enough to create an instance. If not -- EC2 failed.
logging.error('Error during instance creation: timeout in pending state')
result = myinstance.__dict__
if 'id' in result.keys():
# terminate stucked instance
params['id'] = result['id']
params['instance'] = result.copy()
mainq.put((0, 'terminate', params.copy()))
else:
# error occured
logging.error('Error during instance creation: ' + instance_state)
except boto.exception.EC2ResponseError, e:
# Boto errors should be handled according to their error Message - there are some well-known ones
logging.debug(self.getName() + ': got boto error during instance creation: %s' % e)
if str(e).find('<Code>InstanceLimitExceeded</Code>') != -1:
# InstanceLimit is temporary problem
logging.debug(self.getName() + ': got InstanceLimitExceeded - not increasing ntry')
ntry -= 1
elif str(e).find('<Code>InvalidParameterValue</Code>') != -1:
# InvalidParameterValue is really bad
logging.error(self.getName() + ': got boto error during instance creation: %s' % e)
# Try to speed up whole testing process, it's hardly possible to recover from this error
ntry += args.maxtries // 2 + 1
elif str(e).find('<Code>Unsupported</Code>') != -1:
# Unsupported
logging.error(self.getName() + ': got Unsupported - most likely the permanent error: %s' % e)
ntry += args.maxtries // 2 + 1
else:
logging.debug(self.getName() + ':' + traceback.format_exc())
def handleImages(self, action, clc):
if action == 'DescribeImages':
owner = self.get_argument('Owner', None)
if not owner:
owners = None
else:
owners = [owner]
return clc.get_all_images(owners)
elif action == 'DescribeImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
return clc.get_image_attribute(imageid, attribute)
elif action == 'ModifyImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
operation = self.get_argument('OperationType')
users = self.get_argument_list('UserId')
groups = self.get_argument_list('UserGroup')
return clc.modify_image_attribute(imageid, attribute, operation,
users, groups)
elif action == 'ResetImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
return clc.reset_image_attribute(imageid, attribute)
elif action == 'RegisterImage':
image_location = self.get_argument('ImageLocation', None)
name = self.get_argument('Name')
description = self.get_argument('Description', None)
description = base64.b64decode(description)
architecture = self.get_argument('Architecture', None)
kernel_id = self.get_argument('KernelId', None)
ramdisk_id = self.get_argument('RamdiskId', None)
root_dev_name = self.get_argument('RootDeviceName', None)
snapshot_id = self.get_argument('SnapshotId', None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument('BlockDeviceMapping.1.DeviceName',
None)
idx = 1
while mapping:
pre = 'BlockDeviceMapping.%d' % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument(
'%s.VirtualName' % pre, None)
if not (block_dev_type.ephemeral_name):
block_dev_type.no_device = \
(self.get_argument('%s.NoDevice' % pre, '') == 'true')
block_dev_type.snapshot_id = \
self.get_argument('%s.Ebs.SnapshotId' % pre, None)
block_dev_type.size = \
self.get_argument('%s.Ebs.VolumeSize' % pre, None)
block_dev_type.delete_on_termination = \
(self.get_argument('%s.DeleteOnTermination' % pre, '') == 'true')
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument(
'BlockDeviceMapping.%d.DeviceName' % idx, None)
if snapshot_id:
rootbdm = BlockDeviceType()
rootbdm.snapshot_id = snapshot_id
bdm['/dev/sda1'] = rootbdm
if len(bdm) == 0:
bdm = None
return clc.register_image(name, image_location, description,
architecture, kernel_id, ramdisk_id,
root_dev_name, bdm)
def handleRunInstances(self, action, clc, user_data_file):
image_id = self.get_argument('ImageId')
min = self.get_argument('MinCount', '1')
max = self.get_argument('MaxCount', '1')
key = self.get_argument('KeyName', None)
groups = self.get_argument_list('SecurityGroup')
sec_group_ids = self.get_argument_list('SecurityGroupId')
if user_data_file:
user_data = user_data_file
else:
user_data = self.get_argument('UserData', None)
addr_type = self.get_argument('AddressingType', None)
vm_type = self.get_argument('InstanceType', None)
placement = self.get_argument('Placement.AvailabilityZone', None)
placement_group = self.get_argument('Placement.GroupName', None)
tenancy = self.get_argument('Placement.Tenancy', None)
kernel = self.get_argument('KernelId', None)
ramdisk = self.get_argument('RamdiskId', None)
monitoring = False
if self.get_argument('Monitoring.Enabled', '') == 'true':
monitoring = True
subnet = self.get_argument('SubnetId', None)
private_ip = self.get_argument('PrivateIpAddress', None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None)
idx = 1
while mapping:
pre = 'BlockDeviceMapping.%d' % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument(
'%s.VirtualName' % pre, None)
if not (block_dev_type.ephemeral_name):
block_dev_type.no_device = \
(self.get_argument('%s.NoDevice' % pre, '') == 'true')
block_dev_type.snapshot_id = \
self.get_argument('%s.Ebs.SnapshotId' % pre, None)
block_dev_type.size = \
self.get_argument('%s.Ebs.VolumeSize' % pre, None)
block_dev_type.delete_on_termination = \
(self.get_argument('%s.DeleteOnTermination' % pre, '') == 'true')
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument(
'BlockDeviceMapping.%d.DeviceName' % idx, None)
if len(bdm) == 0:
bdm = None
api_termination = False
if self.get_argument('DisableApiTermination', '') == 'true':
api_termination = True
instance_shutdown = False
if self.get_argument('InstanceInitiatedShutdownBehavior',
'') == 'true':
instance_shutdown = True
token = self.get_argument('ClientToken', None)
addition_info = self.get_argument('AdditionInfo', None)
instance_profile_name = self.get_argument('IamInstanceProfile.Name',
None)
instance_profile_arn = self.get_argument('IamInstanceProfile.Arn',
None)
return self.__normalize_instances__([
clc.run_instances(
image_id,
min_count=min,
max_count=max,
key_name=key,
security_groups=groups,
user_data=user_data,
addressing_type=addr_type,
instance_type=vm_type,
placement=placement,
kernel_id=kernel,
ramdisk_id=ramdisk,
monitoring_enabled=monitoring,
subnet_id=subnet,
block_device_map=bdm,
disable_api_termination=api_termination,
instance_initiated_shutdown_behavior=instance_shutdown,
private_ip_address=private_ip,
placement_group=placement_group,
client_token=token,
security_group_ids=sec_group_ids,
additional_info=addition_info,
instance_profile_name=instance_profile_name,
instance_profile_arn=instance_profile_arn,
tenancy=tenancy)
])
def handleRunInstances(self, action, clc, user_data_file, callback):
image_id = self.get_argument('ImageId')
min = self.get_argument('MinCount', '1')
max = self.get_argument('MaxCount', '1')
key = self.get_argument('KeyName', None)
groups = self.get_argument_list('SecurityGroup')
sec_group_ids = self.get_argument_list('SecurityGroupId')
if user_data_file:
user_data = user_data_file
else:
user_data = self.get_argument('UserData', "")
user_data = base64.b64decode(user_data)
addr_type = self.get_argument('AddressingType', None)
vm_type = self.get_argument('InstanceType', None)
placement = self.get_argument('Placement.AvailabilityZone', None)
placement_group = self.get_argument('Placement.GroupName', None)
tenancy = self.get_argument('Placement.Tenancy', None)
kernel = self.get_argument('KernelId', None)
ramdisk = self.get_argument('RamdiskId', None)
monitoring=False
if self.get_argument('Monitoring.Enabled', '') == 'true':
monitoring=True
subnet = self.get_argument('SubnetId', None);
private_ip = self.get_argument('PrivateIpAddress', None);
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None)
idx = 1
while mapping:
pre = 'BlockDeviceMapping.%d' % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument('%s.VirtualName' % pre, None)
if not(block_dev_type.ephemeral_name):
block_dev_type.no_device = \
(self.get_argument('%s.NoDevice' % pre, '') == 'true')
block_dev_type.snapshot_id = \
self.get_argument('%s.Ebs.SnapshotId' % pre, None)
block_dev_type.size = \
self.get_argument('%s.Ebs.VolumeSize' % pre, None)
block_dev_type.delete_on_termination = \
(self.get_argument('%s.Ebs.DeleteOnTermination' % pre, '') == 'true')
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument('BlockDeviceMapping.%d.DeviceName' % idx, None)
if len(bdm) == 0:
bdm = None
api_termination=False
if self.get_argument('DisableApiTermination', '') == 'true':
api_termination=True
instance_shutdown=False
if self.get_argument('InstanceInitiatedShutdownBehavior', '') == 'true':
instance_shutdown=True
token = self.get_argument('ClientToken', None);
addition_info = self.get_argument('AdditionInfo', None);
instance_profile_name = self.get_argument('IamInstanceProfile.Name', None);
instance_profile_arn = self.get_argument('IamInstanceProfile.Arn', None);
return clc.run_instances(image_id, min_count=min, max_count=max,
key_name=key, security_groups=groups,
user_data=user_data, addressing_type=addr_type,
instance_type=vm_type, placement=placement,
kernel_id=kernel, ramdisk_id=ramdisk,
monitoring_enabled=monitoring, subnet_id=subnet,
block_device_map=bdm,
disable_api_termination=api_termination,
instance_initiated_shutdown_behavior=instance_shutdown,
private_ip_address=private_ip,
placement_group=placement_group, client_token=token,
security_group_ids=sec_group_ids,
additional_info=addition_info,
instance_profile_name=instance_profile_name,
instance_profile_arn=instance_profile_arn,
tenancy=tenancy, callback=callback)
def handleImages(self, action, clc, callback=None):
if action == 'DescribeImages':
owner = self.get_argument('Owner', None);
if not owner:
owners = None
else:
owners = [owner]
filters = self.get_filter_args()
return clc.get_all_images(owners, filters, callback)
elif action == 'DescribeImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
return clc.get_image_attribute(imageid, attribute, callback)
elif action == 'ModifyImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
operation = self.get_argument('OperationType')
users = self.get_argument_list('UserId')
groups = self.get_argument_list('UserGroup')
return clc.modify_image_attribute(imageid, attribute, operation, users, groups, callback)
elif action == 'ResetImageAttribute':
imageid = self.get_argument('ImageId')
attribute = self.get_argument('Attribute')
return clc.reset_image_attribute(imageid, attribute, callback)
elif action == 'DeregisterImage':
image_id = self.get_argument('ImageId')
return clc.deregister_image(image_id, callback)
elif action == 'RegisterImage':
image_location = self.get_argument('ImageLocation', None)
name = self.get_argument('Name')
description = self.get_argument('Description', None)
if description != None:
description = base64.b64decode(description);
architecture = self.get_argument('Architecture', None)
kernel_id = self.get_argument('KernelId', None)
ramdisk_id = self.get_argument('RamdiskId', None)
root_dev_name = self.get_argument('RootDeviceName', None)
snapshot_id = self.get_argument('SnapshotId', None)
# get block device mappings
bdm = BlockDeviceMapping()
mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None)
idx = 1
while mapping:
pre = 'BlockDeviceMapping.%d' % idx
dev_name = mapping
block_dev_type = BlockDeviceType()
block_dev_type.ephemeral_name = self.get_argument('%s.VirtualName' % pre, None)
if not(block_dev_type.ephemeral_name):
block_dev_type.no_device = \
(self.get_argument('%s.NoDevice' % pre, '') == 'true')
block_dev_type.snapshot_id = \
self.get_argument('%s.Ebs.SnapshotId' % pre, None)
block_dev_type.size = \
self.get_argument('%s.Ebs.VolumeSize' % pre, None)
block_dev_type.delete_on_termination = \
(self.get_argument('%s.Ebs.DeleteOnTermination' % pre, '') == 'true')
bdm[dev_name] = block_dev_type
idx += 1
mapping = self.get_argument('BlockDeviceMapping.%d.DeviceName' % idx, None)
if snapshot_id:
rootbdm = BlockDeviceType()
rootbdm.snapshot_id = snapshot_id
bdm['/dev/sda1'] = rootbdm
if len(bdm) == 0:
bdm = None
return clc.register_image(name, image_location, description, architecture, kernel_id, ramdisk_id, root_dev_name, bdm, callback)
def register(snapshot_id,
region,
arch,
size=None,
name=None,
desc=None,
pvm=False):
conn = utils.connect(region)
if None in (name, size):
log.debug('getting snapshot - %s', snapshot_id)
snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0]
size = size if size else snapshot.volume_size
name = name if name else snapshot.description
virt = 'hvm'
kernel_id = None
device_base = '/dev/xvd'
ec2_arch = "x86_64" if arch == "amd64" else arch
if pvm:
kernel_id = utils.get_kernel(region, arch)
virt = 'paravirtual'
device_base = '/dev/sd'
name += '-pvm'
log.debug('creating block_device_map')
block_device_map = BlockDeviceMapping()
rootfs = BlockDeviceType()
rootfs.delete_on_termination = True
rootfs.size = size
rootfs.snapshot_id = snapshot_id
rootfs_device_name = device_base + 'a'
block_device_map[rootfs_device_name] = rootfs
ephemeral = BlockDeviceType()
ephemeral.ephemeral_name = 'ephemeral0'
ephemeral_device_name = device_base + 'b'
block_device_map[ephemeral_device_name] = ephemeral
log.debug('registering image - %s', name)
client3 = utils.connect_boto3(region)
response = client3.register_image(Name=name,
Architecture=ec2_arch,
RootDeviceName=rootfs_device_name,
BlockDeviceMappings=[{
'DeviceName': '/dev/xvda',
'Ebs': {
'DeleteOnTermination': True,
'VolumeSize': size,
'SnapshotId': snapshot_id,
},
}, {
'DeviceName':
'/dev/xvdb',
'VirtualName':
'ephemeral0',
}],
VirtualizationType=virt,
EnaSupport=True)
ami_id = response['ImageId']
log.info('registered image - %s %s %s', ami_id, name, region)
return ami_id, name
