def run_instance(conn, ami_id, key_name, instance_type, sec_group, zone="us-east-1d", vol_size=None): """ @param connection: python boto connection @param ami_id: AMI ID @param key_name: SSH key name @param instance_type: instance type, example 'm1.large' @param sec_group: security group @param zone: optional, defaults to 'us-east-1d' @param vol_size: optional integer, if specified will change size of root volume to this size @return boto.ec2.instance.Instance """ bdm = None if vol_size: # Create block device mapping info dev_sda1 = BlockDeviceType() dev_sda1.size = int(vol_size) dev_sda1.delete_on_termination = True bdm = BlockDeviceMapping() bdm['/dev/sda1'] = dev_sda1 # Run instance reservation = conn.run_instances( ami_id, key_name=key_name, instance_type=instance_type, placement=zone, instance_initiated_shutdown_behavior="stop", security_groups=[sec_group], block_device_map=bdm) return reservation.instances[0]
def create_block_device_mapping(ami, device_map): bdm = BlockDeviceMapping() for device, device_info in device_map.items(): if ami.root_device_type == "instance-store" and \ not device_info.get("ephemeral_name"): # EBS is not supported by S3-backed AMIs at request time # EBS volumes can be attached when an instance is running continue bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] if ami.root_device_name == device: ami_size = ami.block_device_mapping[device].size if ami.virtualization_type == "hvm": # Overwrite root device size for HVM instances, since they # cannot be resized online bd.size = ami_size elif device_info.get('size'): # make sure that size is enough for this AMI assert ami_size <= device_info['size'], \ "Instance root device size cannot be smaller than AMI " \ "root device" if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] if device_info.get("volume_type"): bd.volume_type = device_info["volume_type"] if device_info["volume_type"] == "io1" \ and device_info.get("iops"): bd.iops = device_info["iops"] bdm[device] = bd return bdm
def snapshot_register(self): snapshot_id = self.snapshot.id name = self.request.params.get('name') description = self.request.params.get('description') dot = self.request.params.get('dot') reg_as_windows = self.request.params.get('reg_as_windows') root_vol = BlockDeviceType(snapshot_id=snapshot_id) root_vol.delete_on_termination = dot bdm = BlockDeviceMapping() root_device_name = '/dev/sda' if self.cloud_type == 'euca' else '/dev/sda1' bdm[root_device_name] = root_vol location = self.request.route_path('snapshot_view', id=snapshot_id) if self.snapshot and self.register_form.validate(): with boto_error_handler(self.request, location): self.log_request(_(u"Registering snapshot {0} as image {1}").format(snapshot_id, name)) self.snapshot.connection.register_image( name=name, description=description, root_device_name=root_device_name, kernel_id=('windows' if reg_as_windows else None), block_device_map=bdm) prefix = _(u'Successfully registered snapshot') msg = '{prefix} {id}'.format(prefix=prefix, id=snapshot_id) # Clear images cache #ImagesView.invalidate_images_cache() self.request.session.flash(msg, queue=Notification.SUCCESS) return HTTPFound(location=location) return self.render_dict
def get_block_device_map(bdmapping_json=None): """Parse block_device_mapping JSON and return a configured BlockDeviceMapping object Mapping JSON structure... {"/dev/sda": {"snapshot_id": "snap-23E93E09", "volume_type": null, "delete_on_termination": true, "size": 1} } """ if bdmapping_json: mapping = json.loads(bdmapping_json) if mapping: bdm = BlockDeviceMapping() for key, val in mapping.items(): device = BlockDeviceType() if val.get('virtual_name') is not None and val.get( 'virtual_name').startswith('ephemeral'): device.ephemeral_name = val.get('virtual_name') else: device.volume_type = 'standard' device.snapshot_id = val.get('snapshot_id') or None device.size = val.get('size') device.delete_on_termination = val.get( 'delete_on_termination', False) bdm[key] = device return bdm return None return None
def register_snapshot_by_id( self, snap_id, rdn="/dev/sda1", description="bfebs", windows=False, bdmdev=None, name=None, ramdisk=None, kernel=None, dot=True ): ''' Register an image snapshot snap_id (mandatory string) snapshot id name (mandatory string) name of image to be registered description (optional string) description of image to be registered bdmdev (optional string) block-device-mapping device for image rdn (optional string) root-device-name for image dot (optional boolean) Delete On Terminate boolean windows (optional boolean) Is windows image boolean kernel (optional string) kernal (note for windows this name should be "windows" ''' if (bdmdev is None): bdmdev=rdn if (name is None): name="bfebs_"+ snap_id if ( windows is True ) and ( kernel is not None): kernel="windows" bdmap = BlockDeviceMapping() block_dev_type = BlockDeviceType() block_dev_type.snapshot_id = snap_id block_dev_type.delete_on_termination = dot bdmap[bdmdev] = block_dev_type self.debug("Register image with: snap_id:"+str(snap_id)+", rdn:"+str(rdn)+", desc:"+str(description)+", windows:"+str(windows)+", bdname:"+str(bdmdev)+", name:"+str(name)+", ramdisk:"+str(ramdisk)+", kernel:"+str(kernel)) image_id = self.ec2.register_image(name=name, description=description, kernel_id=kernel, ramdisk_id=ramdisk, block_device_map=bdmap, root_device_name=rdn) self.debug("Image now registered as " + image_id) return image_id
def snapshot_register(self): snapshot_id = self.snapshot.id name = self.request.params.get('name') description = self.request.params.get('description') dot = self.request.params.get('dot') reg_as_windows = self.request.params.get('reg_as_windows') root_vol = BlockDeviceType(snapshot_id=snapshot_id) root_vol.delete_on_termination = dot bdm = BlockDeviceMapping() root_device_name = '/dev/sda' if self.cloud_type == 'euca' else '/dev/sda1' bdm[root_device_name] = root_vol location = self.request.route_path('snapshot_view', id=snapshot_id) if self.snapshot and self.register_form.validate(): with boto_error_handler(self.request, location): self.log_request(_(u"Registering snapshot {0} as image {1}").format(snapshot_id, name)) self.snapshot.connection.register_image( name=name, description=description, root_device_name=root_device_name, kernel_id=('windows' if reg_as_windows else None), block_device_map=bdm) prefix = _(u'Successfully registered snapshot') msg = u'{prefix} {id}'.format(prefix=prefix, id=snapshot_id) # Clear images cache self.invalidate_images_cache() self.request.session.flash(msg, queue=Notification.SUCCESS) return HTTPFound(location=location) return self.render_dict
def _parse_block_device_mappings(user_input): """ Parse block device mappings per AWS CLI tools syntax (modified to add IOPS) http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html Syntax: /dev/xvd[a-z]=[snapshot-id|ephemeral]:[size in GB]:[Delete on Term]:[IOPS] - Leave inapplicable fields blank - Delete on Termination defaults to True - IOPS limits are not validated - EBS sizing is not validated Mount an Ephemeral Drive: /dev/xvdb1=ephemeral0 Mount multiple Ephemeral Drives: /dev/xvdb1=ephemeral0,/dev/xvdb2=ephemeral1 Mount a Snapshot: /dev/xvdp=snap-1234abcd Mount a Snapshot to a 100GB drive: /dev/xvdp=snap-1234abcd:100 Mount a Snapshot to a 100GB drive and do not delete on termination: /dev/xvdp=snap-1234abcd:100:false Mount a Fresh 100GB EBS device /dev/xvdp=:100 Mount a Fresh 100GB EBS Device and do not delete on termination: /dev/xvdp=:100:false Mount a Fresh 100GB EBS Device with 1000 IOPS /dev/xvdp=:100::1000 """ block_device_map = BlockDeviceMapping() mappings = user_input.split(",") for mapping in mappings: block_type = BlockDeviceType() mount_point, drive_type, size, delete, iops = _parse_drive_mapping( mapping) if 'ephemeral' in drive_type: block_type.ephemeral_name = drive_type elif 'snap' in drive_type: block_type.snapshot_id = drive_type block_type.volume_type = "standard" else: block_type.volume_type = "standard" block_type.size = size block_type.delete_on_termination = delete if iops: block_type.iops = iops block_type.volume_type = "io1" block_device_map[mount_point] = block_type return block_device_map
def test_create_launch_configuration_with_block_device_mappings(): block_device_mapping = BlockDeviceMapping() ephemeral_drive = BlockDeviceType() ephemeral_drive.ephemeral_name = 'ephemeral0' block_device_mapping['/dev/xvdb'] = ephemeral_drive snapshot_drive = BlockDeviceType() snapshot_drive.snapshot_id = "snap-1234abcd" snapshot_drive.volume_type = "standard" block_device_mapping['/dev/xvdp'] = snapshot_drive ebs_drive = BlockDeviceType() ebs_drive.volume_type = "io1" ebs_drive.size = 100 ebs_drive.iops = 1000 ebs_drive.delete_on_termination = False block_device_mapping['/dev/xvdh'] = ebs_drive conn = boto.connect_autoscale(use_block_device_types=True) config = LaunchConfiguration( name='tester', image_id='ami-abcd1234', instance_type='m1.small', key_name='the_keys', security_groups=["default", "default2"], user_data="This is some user_data", instance_monitoring=True, instance_profile_name='arn:aws:iam::123456789012:instance-profile/testing', spot_price=0.1, block_device_mappings=[block_device_mapping] ) conn.create_launch_configuration(config) launch_config = conn.get_all_launch_configurations()[0] launch_config.name.should.equal('tester') launch_config.image_id.should.equal('ami-abcd1234') launch_config.instance_type.should.equal('m1.small') launch_config.key_name.should.equal('the_keys') set(launch_config.security_groups).should.equal(set(['default', 'default2'])) launch_config.user_data.should.equal("This is some user_data") launch_config.instance_monitoring.enabled.should.equal('true') launch_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/testing') launch_config.spot_price.should.equal(0.1) len(launch_config.block_device_mappings).should.equal(3) returned_mapping = launch_config.block_device_mappings set(returned_mapping.keys()).should.equal(set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh'])) returned_mapping['/dev/xvdh'].iops.should.equal(1000) returned_mapping['/dev/xvdh'].size.should.equal(100) returned_mapping['/dev/xvdh'].volume_type.should.equal("io1") returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd") returned_mapping['/dev/xvdp'].volume_type.should.equal("standard") returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')
def _parse_block_device_mappings(user_input): """ Parse block device mappings per AWS CLI tools syntax (modified to add IOPS) http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html Syntax: /dev/xvd[a-z]=[snapshot-id|ephemeral]:[size in GB]:[Delete on Term]:[IOPS] - Leave inapplicable fields blank - Delete on Termination defaults to True - IOPS limits are not validated - EBS sizing is not validated Mount an Ephemeral Drive: /dev/xvdb1=ephemeral0 Mount multiple Ephemeral Drives: /dev/xvdb1=ephemeral0,/dev/xvdb2=ephemeral1 Mount a Snapshot: /dev/xvdp=snap-1234abcd Mount a Snapshot to a 100GB drive: /dev/xvdp=snap-1234abcd:100 Mount a Snapshot to a 100GB drive and do not delete on termination: /dev/xvdp=snap-1234abcd:100:false Mount a Fresh 100GB EBS device /dev/xvdp=:100 Mount a Fresh 100GB EBS Device and do not delete on termination: /dev/xvdp=:100:false Mount a Fresh 100GB EBS Device with 1000 IOPS /dev/xvdp=:100::1000 """ block_device_map = BlockDeviceMapping() mappings = user_input.split(",") for mapping in mappings: block_type = BlockDeviceType() mount_point, drive_type, size, delete, iops = _parse_drive_mapping(mapping) if 'ephemeral' in drive_type: block_type.ephemeral_name = drive_type elif 'snap' in drive_type: block_type.snapshot_id = drive_type block_type.volume_type = "standard" else: block_type.volume_type = "standard" block_type.size = size block_type.delete_on_termination = delete if iops: block_type.iops = iops block_type.volume_type = "io1" block_device_map[mount_point] = block_type return block_device_map
def register(snapshot_id, region, arch, size=None, name=None, desc=None, pvm=False): conn = utils.connect(region) if None in (name, size): log.debug('getting snapshot - %s', snapshot_id) snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0] size = size if size else snapshot.volume_size name = name if name else snapshot.description virt = 'hvm' kernel_id = None device_base = '/dev/xvd' ec2_arch = "x86_64" if arch == "amd64" else arch if pvm: kernel_id = utils.get_kernel(region, arch) virt = 'paravirtual' device_base = '/dev/sd' name += '-pvm' log.debug('creating block_device_map') block_device_map = BlockDeviceMapping() rootfs = BlockDeviceType() rootfs.delete_on_termination = True rootfs.size = size rootfs.snapshot_id = snapshot_id rootfs_device_name = device_base + 'a' block_device_map[rootfs_device_name] = rootfs ephemeral = BlockDeviceType() ephemeral.ephemeral_name = 'ephemeral0' ephemeral_device_name = device_base + 'b' block_device_map[ephemeral_device_name] = ephemeral log.debug('registering image - %s', name) ami_id = conn.register_image(name=name, description=desc, kernel_id=kernel_id, architecture=ec2_arch, root_device_name=rootfs_device_name, block_device_map=block_device_map, virtualization_type=virt) log.info('registered image - %s %s %s', ami_id, name, region) return ami_id, name
def __update_bdm(self, bdm, bd_spec): """Update the BlockDeviceMapping bdm with the block device spec bd_spec. """ try: dev_name, dev_value = bd_spec.split('=', 1) except Exception: raise CommandError( "Block device spec missing '=' : %s" % (bd_spec,)) dot = None bdt = BlockDeviceType() if ':' in dev_value: blockdev_origin, dot = dev_value.split(':', 1) else: blockdev_origin = dev_value if blockdev_origin is None: raise CommandError("No source specified for %s" % (dev_name,)) if blockdev_origin.startswith('ephemeral'): bdt.ephemeral_name = blockdev_origin elif blockdev_origin.startswith('snap-'): bdt.snapshot_id = blockdev_origin else: raise CommandError("Bad source specified for %s: %s" % (dev_name, blockdev_origin)) if dot is not None: if dot == 'delete': bdt.delete_on_termination = True elif dot == 'nodelete': bdt.delete_on_termination = False else: raise CommandError( "Bad delete-on-termination specified for %s: %s" % (dev_name, dot)) else: bdt.delete_on_termination = False dev_path = '/dev/' + dev_name bdm[dev_path] = bdt
def _get_bmap(self, params): bmap = BlockDeviceMapping() for device in params['bmap']: if not 'name' in device.keys(): self.logger.debug('bad device ' + str(device)) continue dev = BlockDeviceType() if 'size' in device.keys(): dev.size = device['size'] if 'delete_on_termination' in device.keys(): dev.delete_on_termination = device['delete_on_termination'] if 'ephemeral_name' in device.keys(): dev.ephemeral_name = device['ephemeral_name'] bmap[device['name']] = dev return bmap
def _parse_block_device_mappings(self): block_device_map = BlockDeviceMapping() for mapping in self.block_device_mapping_dict: block_type = BlockDeviceType() mount_point = mapping.get('device_name') if 'ephemeral' in mapping.get('virtual_name', ''): block_type.ephemeral_name = mapping.get('virtual_name') else: block_type.volume_type = mapping.get('ebs._volume_type') block_type.snapshot_id = mapping.get('ebs._snapshot_id') block_type.delete_on_termination = mapping.get('ebs._delete_on_termination') block_type.size = mapping.get('ebs._volume_size') block_type.iops = mapping.get('ebs._iops') block_device_map[mount_point] = block_type return block_device_map
def parse_block_device_args(self, block_device_maps_args): block_device_map = BlockDeviceMapping() for block_device_map_arg in block_device_maps_args: parts = block_device_map_arg.split('=') if len(parts) > 1: device_name = parts[0] block_dev_type = BlockDeviceType() block_dev_type.delete_on_termination = True value_parts = parts[1].split(':') if value_parts[0].startswith('snap'): block_dev_type.snapshot_id = value_parts[0] else: if value_parts[0].startswith('ephemeral'): block_dev_type.ephemeral_name = value_parts[0] if len(value_parts) > 1: try: block_dev_type.size = int(value_parts[1]) except ValueError: pass if len(value_parts) > 2: if value_parts[2] == 'false': block_dev_type.delete_on_termination = False block_device_map[device_name] = block_dev_type return block_device_map
def startCluster(self, argv): if len(argv) != 0: print "ec2 startCluster" sys.exit(-1) regions = boto.ec2.regions() regionInfo = '\n'.join(str(region).split(':')[1] for region in regions) regionName = raw_input("select region:\n%s\n>>"%regionInfo) region = boto.ec2.get_region(regionName) conn = region.connect() print "region connected successfully" images = conn.get_all_images(owners='self') imageInfo = '\n'.join( str(image).split(':')[1] + ":" + image.name for image in images) imageId = raw_input("enter imageId:\nself-created images:\n%s\n>>"%imageInfo) instanceTypeInfo = ("m1.small, " "m1.large, " "m1.xlarge\n" "c1.medium, " "c1.xlarge\n" "m2.xlarge, " "m2.2xlarge, " "m2.4xlarge\n" "cc1.4xlarge, " "t1.micro\n") instanceType = raw_input("enter instanceType:\n%s\n>>"%instanceTypeInfo) availZone = raw_input("enter placement[a,b,c]:\n>>") availZone = regionName + availZone diskSize = int(raw_input("enter disk size[G]:\n>>")) rootDev = BlockDeviceType() rootDev.name = 'root' rootDev.size = diskSize rootDev.delete_on_termination = True instStorage = bool(raw_input("mount inst storage?\n>>")) mapping = BlockDeviceMapping() mapping['/dev/sda1'] = rootDev if (instStorage == True): eph0 = BlockDeviceType() eph0.ephemeral_name = 'ephemeral0' mapping['/dev/sdb'] = eph0 groups = conn.get_all_security_groups() groupInfo = '\n'.join(str(group).split(':')[1] for group in groups) group = raw_input("enter securityGroup:\n%s\n>>"%groupInfo) keys = conn.get_all_key_pairs() if len(keys) == 1: key = keys[0].name print 'using default key: ' + key else: keyInfo = '\n'.join(str(key).split(':')[1] for key in keys) key = raw_input("enter key name:\n%s\n>>"%keyInfo) numNodes = int(raw_input("number of nodes:\n>>")) conn.run_instances( imageId, min_count=numNodes, max_count=numNodes, placement=availZone, security_groups = [group], instance_type=instanceType, block_device_map=mapping, key_name=key)
def _parse_block_device_mappings(self): block_device_map = BlockDeviceMapping() for mapping in self.block_device_mapping_dict: block_type = BlockDeviceType() mount_point = mapping.get("device_name") if "ephemeral" in mapping.get("virtual_name", ""): block_type.ephemeral_name = mapping.get("virtual_name") else: block_type.volume_type = mapping.get("ebs._volume_type") block_type.snapshot_id = mapping.get("ebs._snapshot_id") block_type.delete_on_termination = mapping.get( "ebs._delete_on_termination") block_type.size = mapping.get("ebs._volume_size") block_type.iops = mapping.get("ebs._iops") block_device_map[mount_point] = block_type return block_device_map
def register(snapshot_id, region, arch, size=None, name=None, desc=None, pvm=False): conn = utils.connect(region) if None in (name, size): log.debug('getting snapshot - %s', snapshot_id) snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0] size = size if size else snapshot.volume_size name = name if name else snapshot.description virt = 'hvm' kernel_id = None device_base = '/dev/xvd' ec2_arch = "x86_64" if arch == "amd64" else arch if pvm: kernel_id = utils.get_kernel(region, arch) virt = 'paravirtual' device_base = '/dev/sd' name += '-pvm' log.debug('creating block_device_map') block_device_map = BlockDeviceMapping() rootfs = BlockDeviceType() rootfs.delete_on_termination = True rootfs.size = size rootfs.snapshot_id = snapshot_id rootfs_device_name = device_base + 'a' block_device_map[rootfs_device_name] = rootfs ephemeral = BlockDeviceType() ephemeral.ephemeral_name = 'ephemeral0' ephemeral_device_name = device_base + 'b' block_device_map[ephemeral_device_name] = ephemeral log.debug('registering image - %s', name) ami_id = conn.register_image( name=name, description=desc, kernel_id=kernel_id, architecture=ec2_arch, root_device_name=rootfs_device_name, block_device_map=block_device_map, virtualization_type=virt) log.info('registered image - %s %s %s', ami_id, name, region) return ami_id, name
def create_root_block_device(self): # if the root volume size is not the same as the AMI default value: if self.root_volume_size is not None: # sda rather than xvda (for Windows) dev_sda1 = BlockDeviceType() dev_sda1.size = self.root_volume_size dev_sda1.delete_on_termination = True volume = BlockDeviceMapping() # Check the OS type, if its windows we use sda, linux: xvda images = self.ec2.get_all_images(image_ids=[self.node_obj.ami]) image = images[0] if image.platform is None: volume['/dev/xvda'] = dev_sda1 else: volume['/dev/sda1'] = dev_sda1 self.volumes = volume
def start_remote(self): self.get_bootstrap_kernel() self.get_bootstrap_image() # create EBS volume for /mnt/gentoo device = BlockDeviceType() device.size = self.settings["ec2/instance/device/size"] device.delete_on_termination = True mapping = BlockDeviceMapping() self.root_device = "/dev/" + self.settings["ec2/instance/device/name"] mapping[self.root_device] = device # start bootstrapping instance reservation = self.ec2.run_instances(self.bootstrap_image.id, kernel_id=self.bootstrap_kernel.id, instance_type=self.settings["ec2/instance/type"], security_groups=[self.name], key_name=self.name, block_device_map=mapping) self.instance = reservation.instances[0] sys.stdout.write("waiting for instance to come up ..") while self.instance.update() != 'running': sys.stdout.write(".") sys.stdout.flush() time.sleep(5) sys.stdout.write("\n") time.sleep(120) self.ssh_uri = "ec2-user@" + self.instance.public_dns_name self.ssh_port = "22" self.remote_upload_path = "/tmp" # enable sudo without a tty cmd = "sudo sed -i -e '/requiretty/d' /etc/sudoers" cmd = ["ssh", "-t"] + self.ssh_options() + [self.ssh_uri, cmd] ssh = subprocess.Popen(cmd) ssh.wait() self.run_script_at_remote("steps/remote/postboot")
def start_remote(self): self.get_bootstrap_kernel() self.get_bootstrap_image() # create EBS volume for /mnt/gentoo device = BlockDeviceType() device.size = self.settings["ec2/instance/device/size"] device.delete_on_termination = True mapping = BlockDeviceMapping() self.root_device = "/dev/" + self.settings["ec2/instance/device/name"] mapping[self.root_device] = device # start bootstrapping instance reservation = self.ec2.run_instances( self.bootstrap_image.id, kernel_id=self.bootstrap_kernel.id, instance_type=self.settings["ec2/instance/type"], security_groups=[self.name], key_name=self.name, block_device_map=mapping) self.instance = reservation.instances[0] sys.stdout.write("waiting for instance to come up ..") while self.instance.update() != 'running': sys.stdout.write(".") sys.stdout.flush() time.sleep(5) sys.stdout.write("\n") time.sleep(120) self.ssh_uri = "ec2-user@" + self.instance.public_dns_name self.remote_upload_path = "/tmp" # enable sudo without a tty cmd = "sudo sed -i -e '/requiretty/d' /etc/sudoers" cmd = ["ssh", "-t"] + self.ssh_options() + [self.ssh_uri, cmd] ssh = subprocess.Popen(cmd) ssh.wait() self.run_script_at_remote("steps/remote/postboot")
def run_instance(conn, ami_id, key_name, instance_type, sec_group, zone, vol_size=None): bdm = None if vol_size: # Create block device mapping info dev_sda1 = BlockDeviceType() dev_sda1.size = int(vol_size) dev_sda1.delete_on_termination = True bdm = BlockDeviceMapping() bdm["/dev/sda1"] = dev_sda1 # Run instance reservation = conn.run_instances( ami_id, key_name=key_name, instance_type=instance_type, placement=zone, instance_initiated_shutdown_behavior="stop", security_groups=[sec_group], block_device_map=bdm, ) return reservation.instances[0]
def register(snapshot_id, region, arch, size=None, name=None, desc=None): conn = utils.connect(region) if None in (name, size): log.debug("getting snapshot - %s", snapshot_id) snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0] size = size if size else snapshot.volume_size name = name if name else snapshot.description ec2_arch = "x86_64" if arch == "amd64" else arch log.debug("creating block_device_map") block_device_map = BlockDeviceMapping() rootfs = BlockDeviceType() rootfs.delete_on_termination = True rootfs.size = size rootfs.snapshot_id = snapshot_id rootfs_device_name = "/dev/xvda" block_device_map[rootfs_device_name] = rootfs ephemeral = BlockDeviceType() ephemeral.ephemeral_name = "ephemeral0" ephemeral_device_name = "/dev/xvdb" block_device_map[ephemeral_device_name] = ephemeral log.debug("registering image - %s", name) ami_id = conn.register_image( name=name, description=desc, architecture=ec2_arch, root_device_name=rootfs_device_name, block_device_map=block_device_map, virtualization_type="hvm", ) log.info("registered image - %s %s %s", ami_id, name, region) return ami_id, name
def register(snapshot_id, region, size=None, arch=None, name=None, desc=None): conn = utils.connect(region) log.debug('getting snapshot - %s', snapshot_id) snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0] size = size if size else snapshot.volume_size name = name if name else snapshot.description desc = desc if desc else utils.parse_imagename(name)['url'] arch = arch if arch else utils.parse_imagename(name)['architecture'] kernel_id = utils.get_kernel(region, arch) arch_ec2 = "x86_64" if arch == "amd64" else arch log.debug('creating block_device_map') rootfs = BlockDeviceType() rootfs.delete_on_termination = True rootfs.size = size rootfs.snapshot_id = snapshot_id ephemeral = BlockDeviceType() ephemeral.ephemeral_name = 'ephemeral0' block_device_map = BlockDeviceMapping() block_device_map['/dev/sda1'] = rootfs block_device_map['/dev/sda2'] = ephemeral log.debug('registering image - %s', name) ami_id = conn.register_image( name=name, description=desc, architecture=arch_ec2, kernel_id=kernel_id, root_device_name="/dev/sda1", block_device_map=block_device_map) log.info('registered image - %s %s %s', ami_id, name, region) return ami_id
def get_block_device_map(bdmapping_json=None): """Parse block_device_mapping JSON and return a configured BlockDeviceMapping object Mapping JSON structure... {"/dev/sda": {"snapshot_id": "snap-23E93E09", "volume_type": null, "delete_on_termination": true, "size": 1} } """ if bdmapping_json: mapping = json.loads(bdmapping_json) if mapping: bdm = BlockDeviceMapping() for key, val in mapping.items(): device = BlockDeviceType() if val.get('virtual_name') is not None and val.get('virtual_name').startswith('ephemeral'): device.ephemeral_name = val.get('virtual_name') else: device.volume_type = 'standard' device.snapshot_id = val.get('snapshot_id') or None device.size = val.get('size') device.delete_on_termination = val.get('delete_on_termination', False) bdm[key] = device return bdm return None return None
def register(snapshot_id, region, size=None, arch=None, name=None, desc=None): conn = utils.connect(region) log.debug('getting snapshot - %s', snapshot_id) snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0] size = size if size else snapshot.volume_size name = name if name else snapshot.description desc = desc if desc else utils.parse_imagename(name)['url'] arch = arch if arch else utils.parse_imagename(name)['architecture'] kernel_id = utils.get_kernel(region, arch) arch_ec2 = "x86_64" if arch == "amd64" else arch log.debug('creating block_device_map') rootfs = BlockDeviceType() rootfs.delete_on_termination = True rootfs.size = size rootfs.snapshot_id = snapshot_id ephemeral = BlockDeviceType() ephemeral.ephemeral_name = 'ephemeral0' block_device_map = BlockDeviceMapping() block_device_map['/dev/sda1'] = rootfs block_device_map['/dev/sda2'] = ephemeral log.debug('registering image - %s', name) ami_id = conn.register_image(name=name, description=desc, architecture=arch_ec2, kernel_id=kernel_id, root_device_name="/dev/sda1", block_device_map=block_device_map) log.info('registered image - %s %s %s', ami_id, name, region) return ami_id
def do_request_spot_instance(region, secrets, moz_instance_type, price, ami, instance_config, cached_cert_dir, instance_type, availability_zone, dryrun): conn = aws_connect_to_region(region, secrets) interface = get_available_interface(conn=conn, moz_instance_type=moz_instance_type, availability_zone=availability_zone) if not interface: raise RuntimeError("No free network interfaces left in %s" % region) # TODO: check DNS fqdn = interface.tags.get("FQDN") if not fqdn: raise RuntimeError("Skipping %s without FQDN" % interface) log.debug("Spot request for %s (%s)", fqdn, price) if dryrun: log.info("Dry run. skipping") return spec = NetworkInterfaceSpecification(network_interface_id=interface.id) nc = NetworkInterfaceCollection(spec) ip = interface.private_ip_address certs = get_puppet_certs(ip, secrets, cached_cert_dir) user_data = """ FQDN="%(fqdn)s" cd /var/lib/puppet/ssl || exit 1 %(certs)s cd - """ % dict(fqdn=fqdn, certs=certs) if instance_config[region].get("lvm"): user_data += """ mkdir -p /etc/lvm-init/ cat <<EOF > /etc/lvm-init/lvm-init.json %s EOF /sbin/lvm-init """ % json.dumps(instance_config[region]) bdm = BlockDeviceMapping() for device, device_info in instance_config[region]['device_map'].items(): bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] if ami.root_device_name == device: ami_size = ami.block_device_mapping[device].size if ami.virtualization_type == "hvm": # Overwrite root device size for HVM instances, since they # cannot be resized online bd.size = ami_size elif device_info.get('size'): # make sure that size is enough for this AMI assert ami_size <= device_info['size'], \ "Instance root device size cannot be smaller than AMI " \ "root device" if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] bdm[device] = bd sir = conn.request_spot_instances( price=str(price), image_id=ami.id, count=1, instance_type=instance_type, key_name=instance_config[region]["ssh_key"], user_data=user_data, block_device_map=bdm, network_interfaces=nc, instance_profile_name=instance_config[region].get( "instance_profile_name"), ) sir[0].add_tag("moz-type", moz_instance_type)
def do_request_spot_instance(region, secrets, moz_instance_type, price, ami, instance_config, cached_cert_dir, instance_type, availability_zone, slaveset, active_network_ids, dryrun): conn = get_aws_connection(region) interface = get_available_interface( conn=conn, moz_instance_type=moz_instance_type, availability_zone=availability_zone, slaveset=slaveset, active_network_ids=active_network_ids) if not interface: log.warn("No free network interfaces left in %s" % region) return False # TODO: check DNS fqdn = interface.tags.get("FQDN") if not fqdn: log.warn("interface %s has no FQDN", interface) return False log.debug("Spot request for %s (%s)", fqdn, price) if dryrun: log.info("Dry run. skipping") return True spec = NetworkInterfaceSpecification( network_interface_id=interface.id) nc = NetworkInterfaceCollection(spec) ip = interface.private_ip_address certs = get_puppet_certs(ip, secrets, cached_cert_dir) user_data = """ FQDN="%(fqdn)s" cd /var/lib/puppet/ssl || exit 1 %(certs)s cd - """ % dict(fqdn=fqdn, certs=certs) if instance_config[region].get("lvm"): user_data += """ mkdir -p /etc/lvm-init/ cat <<EOF > /etc/lvm-init/lvm-init.json %s EOF /sbin/lvm-init """ % json.dumps(instance_config[region]) bdm = BlockDeviceMapping() for device, device_info in instance_config[region]['device_map'].items(): bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] if ami.root_device_name == device: ami_size = ami.block_device_mapping[device].size if ami.virtualization_type == "hvm": # Overwrite root device size for HVM instances, since they # cannot be resized online bd.size = ami_size elif device_info.get('size'): # make sure that size is enough for this AMI assert ami_size <= device_info['size'], \ "Instance root device size cannot be smaller than AMI " \ "root device" if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] bdm[device] = bd sir = conn.request_spot_instances( price=str(price), image_id=ami.id, count=1, instance_type=instance_type, key_name=instance_config[region]["ssh_key"], user_data=user_data, block_device_map=bdm, network_interfaces=nc, instance_profile_name=instance_config[region].get("instance_profile_name"), ) sir[0].add_tag("moz-type", moz_instance_type) return True
def vm(install_list): region = install_list["region"] subnet_id = install_list["subnet_id"] ami_id = install_list["ami_id"] snap_id = install_list["snap_id"] volume_capacity = install_list["volume_capacity"] key_name = install_list["key_name"] instance_type = install_list["instance_type"] sg_id = install_list["sg_id"] user_data = install_list["user_data"] usage = install_list["usage"] block_device_map = BlockDeviceMapping() block_dev_type = BlockDeviceType() block_dev_type.snapshot_id = snap_id block_dev_type.delete_on_termination = False block_dev_type.size = volume_capacity block_device_map['/dev/sdb'] = block_dev_type if region not in EXTRA_INFO: aws_access_key_id = EXTRA_INFO["common"]["aws_access_key_id"] aws_secret_access_key = EXTRA_INFO["common"]["aws_secret_access_key"] name_key = EXTRA_INFO["common"]["name_key"] else: aws_access_key_id = EXTRA_INFO[region]["aws_access_key_id"] aws_secret_access_key = EXTRA_INFO[region]["aws_secret_access_key"] name_key = EXTRA_INFO[region]["name_key"] ret = asset_hostname.get(region, usage) hostname = ret["hostname"].split(".")[0] user_data = user_data.replace("region=","region=%s" % region) user_data = user_data.replace("hostname=","hostname=%s" % hostname) user_data = user_data.replace("dns_vip=","dns_vip=%s" % \ DNS_INFO[region]["dns_vip"]) user_data = user_data.replace("ns_servers=","ns_servers='%s'" % \ " ".join(DNS_INFO[region]["ns_servers"]) ) network_interface = NetworkInterfaceSpecification(subnet_id=subnet_id, \ groups=[sg_id]) network_interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(\ network_interface) conn = boto.ec2.connect_to_region(region, \ aws_access_key_id=aws_access_key_id, \ aws_secret_access_key=aws_secret_access_key) reservation = conn.run_instances(ami_id, key_name=key_name, network_interfaces=network_interfaces, instance_type=instance_type, block_device_map=block_device_map, # security_groups=[security_groups], min_count=1, max_count=1, user_data=user_data, ) instance = reservation.instances[0] time_init = 0 time_total = 300 time_interval = 5 while time_init < time_total: status = instance.update() if status == 'running': instance.add_tag(u"名称", hostname) instance.add_tag("Name", hostname) break else: time.sleep(time_interval) time_init += time_interval install_list["instance_id"] = str(instance).split(":")[-1] install_list["placement"] = instance.placement install_list["status"] = instance.update() install_list["hostname"] = hostname return install_list
def test_create_launch_configuration_with_block_device_mappings(): block_device_mapping = BlockDeviceMapping() ephemeral_drive = BlockDeviceType() ephemeral_drive.ephemeral_name = 'ephemeral0' block_device_mapping['/dev/xvdb'] = ephemeral_drive snapshot_drive = BlockDeviceType() snapshot_drive.snapshot_id = "snap-1234abcd" snapshot_drive.volume_type = "standard" block_device_mapping['/dev/xvdp'] = snapshot_drive ebs_drive = BlockDeviceType() ebs_drive.volume_type = "io1" ebs_drive.size = 100 ebs_drive.iops = 1000 ebs_drive.delete_on_termination = False block_device_mapping['/dev/xvdh'] = ebs_drive conn = boto.connect_autoscale(use_block_device_types=True) config = LaunchConfiguration( name='tester', image_id='ami-abcd1234', instance_type='m1.small', key_name='the_keys', security_groups=["default", "default2"], user_data=b"This is some user_data", instance_monitoring=True, instance_profile_name= 'arn:aws:iam::123456789012:instance-profile/testing', spot_price=0.1, block_device_mappings=[block_device_mapping]) conn.create_launch_configuration(config) launch_config = conn.get_all_launch_configurations()[0] launch_config.name.should.equal('tester') launch_config.image_id.should.equal('ami-abcd1234') launch_config.instance_type.should.equal('m1.small') launch_config.key_name.should.equal('the_keys') set(launch_config.security_groups).should.equal( set(['default', 'default2'])) launch_config.user_data.should.equal(b"This is some user_data") launch_config.instance_monitoring.enabled.should.equal('true') launch_config.instance_profile_name.should.equal( 'arn:aws:iam::123456789012:instance-profile/testing') launch_config.spot_price.should.equal(0.1) len(launch_config.block_device_mappings).should.equal(3) returned_mapping = launch_config.block_device_mappings set(returned_mapping.keys()).should.equal( set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh'])) returned_mapping['/dev/xvdh'].iops.should.equal(1000) returned_mapping['/dev/xvdh'].size.should.equal(100) returned_mapping['/dev/xvdh'].volume_type.should.equal("io1") returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd") returned_mapping['/dev/xvdp'].volume_type.should.equal("standard") returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')
import os import boto.ec2 from pprint import pprint def cloudconfig(): base_path = os.path.dirname(os.path.realpath(__file__)) cloud_config = open(os.path.join(base_path, 'cloud-config')) return cloud_config.read() close(os.path.join(base_path, 'cloud-config')) conn = boto.ec2.connect_to_region("eu-west-1") from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType block_device_map = BlockDeviceMapping() block_dev_type = BlockDeviceType() block_dev_type.delete_on_termination = True #block_dev_type.snapshot_id = snapshot_id block_device_map['/dev/sda'] = block_dev_type i = conn.run_instances( # 'ami-1c2e8b6b', 'ami-0c10417b', key_name = 'jpazdyga', user_data = cloudconfig(), instance_type = 't1.micro', security_groups = ['automat'], block_device_map = block_device_map) print(i.instances)
def handleImages(self, action, clc, callback=None): if action == "DescribeImages": owner = self.get_argument("Owner", None) if not owner: owners = None else: owners = [owner] filters = self.get_filter_args() return clc.get_all_images(owners, filters, callback) elif action == "DescribeImageAttribute": imageid = self.get_argument("ImageId") attribute = self.get_argument("Attribute") return clc.get_image_attribute(imageid, attribute, callback) elif action == "ModifyImageAttribute": imageid = self.get_argument("ImageId") attribute = self.get_argument("Attribute") operation = self.get_argument("OperationType") users = self.get_argument_list("UserId") groups = self.get_argument_list("UserGroup") return clc.modify_image_attribute(imageid, attribute, operation, users, groups, callback) elif action == "ResetImageAttribute": imageid = self.get_argument("ImageId") attribute = self.get_argument("Attribute") return clc.reset_image_attribute(imageid, attribute, callback) elif action == "DeregisterImage": image_id = self.get_argument("ImageId") return clc.deregister_image(image_id, callback) elif action == "RegisterImage": image_location = self.get_argument("ImageLocation", None) name = self.get_argument("Name") description = self.get_argument("Description", None) if description != None: description = base64.b64decode(description) architecture = self.get_argument("Architecture", None) kernel_id = self.get_argument("KernelId", None) ramdisk_id = self.get_argument("RamdiskId", None) root_dev_name = self.get_argument("RootDeviceName", None) snapshot_id = self.get_argument("SnapshotId", None) # get block device mappings bdm = BlockDeviceMapping() mapping = self.get_argument("BlockDeviceMapping.1.DeviceName", None) idx = 1 while mapping: pre = "BlockDeviceMapping.%d" % idx dev_name = mapping block_dev_type = BlockDeviceType() block_dev_type.ephemeral_name = self.get_argument("%s.VirtualName" % pre, None) if not (block_dev_type.ephemeral_name): block_dev_type.no_device = self.get_argument("%s.NoDevice" % pre, "") == "true" block_dev_type.snapshot_id = self.get_argument("%s.Ebs.SnapshotId" % pre, None) block_dev_type.size = self.get_argument("%s.Ebs.VolumeSize" % pre, None) block_dev_type.delete_on_termination = ( self.get_argument("%s.Ebs.DeleteOnTermination" % pre, "") == "true" ) bdm[dev_name] = block_dev_type idx += 1 mapping = self.get_argument("BlockDeviceMapping.%d.DeviceName" % idx, None) if snapshot_id: rootbdm = BlockDeviceType() rootbdm.snapshot_id = snapshot_id bdm["/dev/sda1"] = rootbdm if len(bdm) == 0: bdm = None return clc.register_image( name, image_location, description, architecture, kernel_id, ramdisk_id, root_dev_name, bdm, callback )
def launch_cluster(conn, OPTS, cluster_name): print "Setting up security groups..." master_group = get_or_make_group(conn, cluster_name + "-master") slave_group = get_or_make_group(conn, cluster_name + "-slaves") ambari_group = get_or_make_group(conn, cluster_name + "-ambari") if master_group.rules == []: # Group was just now created master_group.authorize(src_group=master_group) master_group.authorize(src_group=slave_group) master_group.authorize(src_group=ambari_group) # TODO: Currently Group is completely open master_group.authorize('tcp', 0, 65535, '0.0.0.0/0') if slave_group.rules == []: # Group was just now created slave_group.authorize(src_group=master_group) slave_group.authorize(src_group=slave_group) slave_group.authorize(src_group=ambari_group) # TODO: Currently Group is completely open slave_group.authorize('tcp', 0, 65535, '0.0.0.0/0') if ambari_group.rules == []: # Group was just now created ambari_group.authorize(src_group=master_group) ambari_group.authorize(src_group=slave_group) ambari_group.authorize(src_group=ambari_group) # TODO: Currently Group is completely open ambari_group.authorize('tcp', 0, 65535, '0.0.0.0/0') # Check if instances are already running in our groups if OPTS.resume: return get_existing_cluster(conn, OPTS, cluster_name, die_on_error=False) else: active_nodes = get_existing_cluster(conn, OPTS, cluster_name, die_on_error=False) if any(active_nodes): print >> stderr, ( "ERROR: There are already instances running in " + "group %s or %s" % (master_group.name, slave_group.name)) sys.exit(1) print "Launching instances..." try: image = conn.get_all_images(image_ids=[OPTS.ami])[0] except: print >> stderr, "Could not find AMI " + OPTS.ami sys.exit(1) # Create block device mapping so that we can add an EBS volume if asked to block_map = BlockDeviceMapping() device = BlockDeviceType() device.ephemeral_name = 'ephemeral0' device.delete_on_termination = True block_map["/dev/sdv"] = device # assume master and ambari hosts have the same instance type master_type = OPTS.master_instance_type if master_type == "": master_type = OPTS.instance_type # Launch slaves if OPTS.spot_price != None: # Launch spot instances with the requested price num_spot_instances = OPTS.slaves + 2 # slaves, ambari host, master print("Requesting %d slaves as spot instances with price $%.3f" % (num_spot_instances, OPTS.spot_price)) zones = get_zones(conn, OPTS) num_zones = len(zones) i = 0 my_req_ids = [] ambari_req_ids = [] master_req_ids = [] for zone in zones: num_slaves_this_zone = get_partition(OPTS.slaves, num_zones, i) ambari_reqs = conn.request_spot_instances( price=OPTS.spot_price, image_id=OPTS.ami, launch_group="launch-group-%s" % cluster_name, placement=zone, count=1, key_name=OPTS.key_pair, security_groups=[ambari_group], instance_type=master_type, block_device_map=block_map) master_reqs = conn.request_spot_instances( price=OPTS.spot_price, image_id=OPTS.ami, launch_group="launch-group-%s" % cluster_name, placement=zone, count=1, key_name=OPTS.key_pair, security_groups=[master_group], instance_type=master_type, block_device_map=block_map) slave_reqs = conn.request_spot_instances( price=OPTS.spot_price, image_id=OPTS.ami, launch_group="launch-group-%s" % cluster_name, placement=zone, count=num_slaves_this_zone, key_name=OPTS.key_pair, security_groups=[slave_group], instance_type=OPTS.instance_type, block_device_map=block_map) my_req_ids += [req.id for req in slave_reqs] ambari_req_ids += [req.id for req in ambari_reqs] master_req_ids += [req.id for req in master_reqs] i += 1 print "Waiting for spot instances to be granted..." try: while True: time.sleep(10) reqs = conn.get_all_spot_instance_requests() id_to_req = {} for r in reqs: id_to_req[r.id] = r active_instance_ids = [] ambari_instance_ids = [] master_instance_ids = [] for i in my_req_ids: if i in id_to_req and id_to_req[i].state == "active": active_instance_ids.append( id_to_req[i].instance_id) for i in master_req_ids: if i in id_to_req and id_to_req[i].state == "active": master_instance_ids.append( id_to_req[i].instance_id) for i in ambari_req_ids: if i in id_to_req and id_to_req[i].state == "active": ambari_instance_ids.append( id_to_req[i].instance_id) if len(active_instance_ids) == OPTS.slaves and len( master_instance_ids) == 1 and len( ambari_instance_ids) == 1: print "All %d slaves, 1 master, 1 ambari host granted" % OPTS.slaves slave_nodes = [] master_nodes = [] ambari_nodes = [] for r in conn.get_all_instances(active_instance_ids): slave_nodes += r.instances for r in conn.get_all_instances(master_instance_ids): master_nodes += r.instances for r in conn.get_all_instances(ambari_instance_ids): ambari_nodes += r.instances break else: print "%d of %d spot instance requests granted, waiting longer" % ( len(active_instance_ids), num_spot_instances) except Exception as e: print e print "Canceling spot instance requests" conn.cancel_spot_instance_requests(my_req_ids) # Log a warning if any of these requests actually launched instances: (master_nodes, slave_nodes, ambari_nodes) = get_existing_cluster(conn, OPTS, cluster_name, die_on_error=False) running = len(master_nodes) + len(slave_nodes) + len( ambari_nodes) if running: print >> stderr, ( "WARNING: %d instances are still running" % running) sys.exit(0) else: # Launch non-spot instances zones = get_zones(conn, OPTS) num_zones = len(zones) i = 0 slave_nodes = [] for zone in zones: num_slaves_this_zone = get_partition(OPTS.slaves, num_zones, i) if num_slaves_this_zone > 0: slave_res = image.run(key_name=OPTS.key_pair, security_groups=[slave_group], instance_type=OPTS.instance_type, placement=zone, min_count=num_slaves_this_zone, max_count=num_slaves_this_zone, block_device_map=block_map) slave_nodes += slave_res.instances print "Launched %d slaves in %s, regid = %s" % ( num_slaves_this_zone, zone, slave_res.id) i += 1 # Launch masters if OPTS.zone == 'all': OPTS.zone = random.choice(conn.get_all_zones()).name master_res = image.run(key_name=OPTS.key_pair, security_groups=[master_group], instance_type=master_type, placement=OPTS.zone, min_count=1, max_count=1, block_device_map=block_map) master_nodes = master_res.instances print "Launched master in %s, regid = %s" % (zone, master_res.id) ambari_type = master_type if OPTS.zone == 'all': OPTS.zone = random.choice(conn.get_all_zones()).name ambari_res = image.run(key_name=OPTS.key_pair, security_groups=[ambari_group], instance_type=ambari_type, placement=OPTS.zone, min_count=1, max_count=1, block_device_map=block_map) ambari_nodes = ambari_res.instances print "Launched ambari in %s, regid = %s" % (zone, ambari_res.id) # Return all the instances return (master_nodes, slave_nodes, ambari_nodes)
def rebundle(reboot_if_needed=False): """ Rebundles the EC2 instance that is passed as the -H parameter This script handles all aspects of the rebundling process and is (almost) fully automated. Two things should be edited and provided before invoking it: AWS account information and the desired size of the root volume for the new instance. :rtype: bool :return: If instance was successfully rebundled and an AMI ID was received, return True. False, otherwise. """ _check_fabric_version() time_start = dt.datetime.utcnow() print "Rebundling instance '%s'. Start time: %s" % (env.hosts[0], time_start) _amazon_ec2_environment() if boto: # Select appropriate region: availability_zone = run("curl --silent http://169.254.169.254/latest/meta-data/placement/availability-zone") instance_region = availability_zone[:-1] # Truncate zone letter to get region name ec2_conn = _get_ec2_conn(instance_region) # hostname = env.hosts[0] # -H flag to fab command sets this variable so get only 1st hostname instance_id = run("curl --silent http://169.254.169.254/latest/meta-data/instance-id") # Get the size (in GB) of the root partition for the new image vol_size = _get_root_vol_size(ec2_conn, instance_id) # Handle reboot if required if not _reboot(instance_id, reboot_if_needed): return False # Indicates that rebundling was not completed and should be restarted _clean() # Clean up the environment before rebundling image_id = None kernel_id = run("curl --silent http://169.254.169.254/latest/meta-data/kernel-id") if ec2_conn and instance_id and availability_zone and kernel_id: print "Rebundling instance with ID '%s' in region '%s'" % (instance_id, ec2_conn.region.name) try: # Need 2 volumes - one for image (rsync) and the other for the snapshot (see instance-to-ebs-ami.sh) vol = ec2_conn.create_volume(vol_size, availability_zone) vol2 = ec2_conn.create_volume(vol_size, availability_zone) # TODO: wait until it becomes 'available' print "Created 2 new volumes of size '%s' with IDs '%s' and '%s'" % (vol_size, vol.id, vol2.id) except EC2ResponseError, e: print(red("Error creating volume: %s" % e)) return False if vol: try: # Attach newly created volumes to the instance dev_id = '/dev/sdh' if not _attach(ec2_conn, instance_id, vol.id, dev_id): print(red("Error attaching volume '%s' to the instance. Aborting." % vol.id)) return False dev_id = '/dev/sdj' if not _attach(ec2_conn, instance_id, vol2.id, dev_id): print(red("Error attaching volume '%s' to the instance. Aborting." % vol2.id)) return False # Move the file system onto the new volume (with a help of a script) url = os.path.join(REPO_ROOT_URL, "instance-to-ebs-ami.sh") # with contextlib.nested(cd('/tmp'), settings(hide('stdout', 'stderr'))): with cd('/tmp'): if exists('/tmp/'+os.path.split(url)[1]): sudo('rm /tmp/'+os.path.split(url)[1]) sudo('wget %s' % url) sudo('chmod u+x /tmp/%s' % os.path.split(url)[1]) sudo('./%s' % os.path.split(url)[1]) # Detach the new volume _detach(ec2_conn, instance_id, vol.id) _detach(ec2_conn, instance_id, vol2.id) answer = confirm("Would you like to terminate the instance used during rebundling?", default=False) if answer: ec2_conn.terminate_instances([instance_id]) # Create a snapshot of the new volume commit_num = local('cd %s; hg tip | grep changeset | cut -d: -f2' % os.getcwd()).strip() snap_id = _create_snapshot(ec2_conn, vol.id, "AMI: galaxy-cloudman (using mi-deployment at commit %s)" % commit_num) # Register the snapshot of the new volume as a machine image (i.e., AMI) arch = 'x86_64' root_device_name = '/dev/sda1' # Extra info on how EBS image registration is done: http://markmail.org/message/ofgkyecjktdhofgz # http://www.elastician.com/2009/12/creating-ebs-backed-ami-from-s3-backed.html # http://www.shlomoswidler.com/2010/01/creating-consistent-snapshots-of-live.html ebs = BlockDeviceType() ebs.snapshot_id = snap_id ebs.delete_on_termination = True ephemeral0_device_name = '/dev/sdb' ephemeral0 = BlockDeviceType() ephemeral0.ephemeral_name = 'ephemeral0' ephemeral1_device_name = '/dev/sdc' ephemeral1 = BlockDeviceType() ephemeral1.ephemeral_name = 'ephemeral1' # ephemeral2_device_name = '/dev/sdd' # Needed for instances w/ 3 ephemeral disks # ephemeral2 = BlockDeviceType() # ephemeral2.ephemeral_name = 'ephemeral2' # ephemeral3_device_name = '/dev/sde' # Needed for instances w/ 4 ephemeral disks # ephemeral3 = BlockDeviceType() # ephemeral3.ephemeral_name = 'ephemeral3' block_map = BlockDeviceMapping() block_map[root_device_name] = ebs block_map[ephemeral0_device_name] = ephemeral0 block_map[ephemeral1_device_name] = ephemeral1 print(yellow('galaxy-cloudman-%s' % time_start.strftime("%Y-%m-%d"))) name, desc = _get_image_name() image_id = ec2_conn.register_image(name, description=desc, architecture=arch, kernel_id=kernel_id, root_device_name=root_device_name, block_device_map=block_map) answer = confirm("Volume with ID '%s' was created and used to make this AMI but is not longer needed. Would you like to delete it?" % vol.id) if answer: ec2_conn.delete_volume(vol.id) print "Deleting the volume (%s) used for rsync only" % vol2.id ec2_conn.delete_volume(vol2.id) print(green("--------------------------")) print(green("Finished creating new machine image. Image ID: '%s'" % (image_id))) print(green("--------------------------")) answer = confirm("Would you like to make this machine image public?", default=False) if image_id and answer: ec2_conn.modify_image_attribute(image_id, attribute='launchPermission', operation='add', groups=['all']) except EC2ResponseError, e: print(red("Error creating image: %s" % e)) return False else: print(red("Error creating new volume")) return False
def launch_cluster(conn, OPTS, cluster_name): print "Setting up security groups..." master_group = get_or_make_group(conn, cluster_name + "-master") slave_group = get_or_make_group(conn, cluster_name + "-slaves") ambari_group = get_or_make_group(conn, cluster_name + "-ambari") if master_group.rules == []: # Group was just now created master_group.authorize(src_group=master_group) master_group.authorize(src_group=slave_group) master_group.authorize(src_group=ambari_group) # TODO: Currently Group is completely open master_group.authorize('tcp', 0, 65535, '0.0.0.0/0') if slave_group.rules == []: # Group was just now created slave_group.authorize(src_group=master_group) slave_group.authorize(src_group=slave_group) slave_group.authorize(src_group=ambari_group) # TODO: Currently Group is completely open slave_group.authorize('tcp', 0, 65535, '0.0.0.0/0') if ambari_group.rules == []: # Group was just now created ambari_group.authorize(src_group=master_group) ambari_group.authorize(src_group=slave_group) ambari_group.authorize(src_group=ambari_group) # TODO: Currently Group is completely open ambari_group.authorize('tcp', 0, 65535, '0.0.0.0/0') # Check if instances are already running in our groups if OPTS.resume: return get_existing_cluster(conn, OPTS, cluster_name, die_on_error=False) else: active_nodes = get_existing_cluster(conn, OPTS, cluster_name, die_on_error=False) if any(active_nodes): print >> stderr, ("ERROR: There are already instances running in " + "group %s or %s" % (master_group.name, slave_group.name)) sys.exit(1) print "Launching instances..." try: image = conn.get_all_images(image_ids=[OPTS.ami])[0] except: print >> stderr, "Could not find AMI " + OPTS.ami sys.exit(1) # Create block device mapping so that we can add an EBS volume if asked to block_map = BlockDeviceMapping() device = BlockDeviceType() device.ephemeral_name = 'ephemeral0' device.delete_on_termination = True block_map["/dev/sdv"] = device # assume master and ambari hosts have the same instance type master_type = OPTS.master_instance_type if master_type == "": master_type = OPTS.instance_type # Launch slaves if OPTS.spot_price != None: # Launch spot instances with the requested price num_spot_instances = OPTS.slaves + 2 # slaves, ambari host, master print ("Requesting %d slaves as spot instances with price $%.3f" % (num_spot_instances, OPTS.spot_price)) zones = get_zones(conn, OPTS) num_zones = len(zones) i = 0 my_req_ids = [] ambari_req_ids = [] master_req_ids = [] for zone in zones: num_slaves_this_zone = get_partition(OPTS.slaves, num_zones, i) ambari_reqs = conn.request_spot_instances( price = OPTS.spot_price, image_id = OPTS.ami, launch_group = "launch-group-%s" % cluster_name, placement = zone, count = 1, key_name = OPTS.key_pair, security_groups = [ambari_group], instance_type = master_type, block_device_map = block_map) master_reqs = conn.request_spot_instances( price = OPTS.spot_price, image_id = OPTS.ami, launch_group = "launch-group-%s" % cluster_name, placement = zone, count = 1, key_name = OPTS.key_pair, security_groups = [master_group], instance_type = master_type, block_device_map = block_map) slave_reqs = conn.request_spot_instances( price = OPTS.spot_price, image_id = OPTS.ami, launch_group = "launch-group-%s" % cluster_name, placement = zone, count = num_slaves_this_zone, key_name = OPTS.key_pair, security_groups = [slave_group], instance_type = OPTS.instance_type, block_device_map = block_map) my_req_ids += [req.id for req in slave_reqs] ambari_req_ids += [req.id for req in ambari_reqs] master_req_ids += [req.id for req in master_reqs] i += 1 print "Waiting for spot instances to be granted..." try: while True: time.sleep(10) reqs = conn.get_all_spot_instance_requests() id_to_req = {} for r in reqs: id_to_req[r.id] = r active_instance_ids = [] ambari_instance_ids = [] master_instance_ids = [] for i in my_req_ids: if i in id_to_req and id_to_req[i].state == "active": active_instance_ids.append(id_to_req[i].instance_id) for i in master_req_ids: if i in id_to_req and id_to_req[i].state == "active": master_instance_ids.append(id_to_req[i].instance_id) for i in ambari_req_ids: if i in id_to_req and id_to_req[i].state == "active": ambari_instance_ids.append(id_to_req[i].instance_id) if len(active_instance_ids) == OPTS.slaves and len(master_instance_ids) == 1 and len(ambari_instance_ids) == 1: print "All %d slaves, 1 master, 1 ambari host granted" % OPTS.slaves slave_nodes = [] master_nodes = [] ambari_nodes = [] for r in conn.get_all_instances(active_instance_ids): slave_nodes += r.instances for r in conn.get_all_instances(master_instance_ids): master_nodes += r.instances for r in conn.get_all_instances(ambari_instance_ids): ambari_nodes += r.instances break else: print "%d of %d spot instance requests granted, waiting longer" % ( len(active_instance_ids), num_spot_instances) except Exception as e: print e print "Canceling spot instance requests" conn.cancel_spot_instance_requests(my_req_ids) # Log a warning if any of these requests actually launched instances: (master_nodes, slave_nodes, ambari_nodes) = get_existing_cluster( conn, OPTS, cluster_name, die_on_error=False) running = len(master_nodes) + len(slave_nodes) + len(ambari_nodes) if running: print >> stderr, ("WARNING: %d instances are still running" % running) sys.exit(0) else: # Launch non-spot instances zones = get_zones(conn, OPTS) num_zones = len(zones) i = 0 slave_nodes = [] for zone in zones: num_slaves_this_zone = get_partition(OPTS.slaves, num_zones, i) if num_slaves_this_zone > 0: slave_res = image.run(key_name = OPTS.key_pair, security_groups = [slave_group], instance_type = OPTS.instance_type, placement = zone, min_count = num_slaves_this_zone, max_count = num_slaves_this_zone, block_device_map = block_map) slave_nodes += slave_res.instances print "Launched %d slaves in %s, regid = %s" % (num_slaves_this_zone, zone, slave_res.id) i += 1 # Launch masters if OPTS.zone == 'all': OPTS.zone = random.choice(conn.get_all_zones()).name master_res = image.run(key_name = OPTS.key_pair, security_groups = [master_group], instance_type = master_type, placement = OPTS.zone, min_count = 1, max_count = 1, block_device_map = block_map) master_nodes = master_res.instances print "Launched master in %s, regid = %s" % (zone, master_res.id) ambari_type = master_type if OPTS.zone == 'all': OPTS.zone = random.choice(conn.get_all_zones()).name ambari_res = image.run(key_name = OPTS.key_pair, security_groups = [ambari_group], instance_type = ambari_type, placement = OPTS.zone, min_count = 1, max_count = 1, block_device_map = block_map) ambari_nodes = ambari_res.instances print "Launched ambari in %s, regid = %s" % (zone, ambari_res.id) # Return all the instances return (master_nodes, slave_nodes, ambari_nodes)
def create_instance(name, config, region, key_name, ssh_key, instance_data, deploypass, loaned_to, loan_bug, create_ami, ignore_subnet_check, max_attempts): """Creates an AMI instance with the given name and config. The config must specify things like ami id.""" conn = get_aws_connection(region) # Make sure we don't request the same things twice token = str(uuid.uuid4())[:16] instance_data = instance_data.copy() instance_data['name'] = name instance_data['domain'] = config['domain'] instance_data['hostname'] = '{name}.{domain}'.format( name=name, domain=config['domain']) ami = conn.get_all_images(image_ids=[config["ami"]])[0] bdm = None if 'device_map' in config: bdm = BlockDeviceMapping() for device, device_info in config['device_map'].items(): bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] # Overwrite root device size for HVM instances, since they cannot # be resized online if ami.virtualization_type == "hvm" and \ ami.root_device_name == device: bd.size = ami.block_device_mapping[ami.root_device_name].size if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] if device_info.get("volume_type"): bd.volume_type = device_info["volume_type"] if device_info["volume_type"] == "io1" \ and device_info.get("iops"): bd.iops = device_info["iops"] bdm[device] = bd interfaces = make_instance_interfaces( region, instance_data['hostname'], ignore_subnet_check, config.get('subnet_ids'), config.get('security_group_ids', []), config.get("use_public_ip")) keep_going, attempt = True, 1 while keep_going: try: puppet_master = pick_puppet_master(instance_data.get('puppet_masters')) user_data = user_data_from_template(config['type'], { "puppet_server": puppet_master, "fqdn": instance_data['hostname'], "hostname": instance_data['name'], "domain": instance_data['domain'], "dns_search_domain": config.get('dns_search_domain'), "password": deploypass, "moz_instance_type": config['type'], "region_dns_atom": get_region_dns_atom(region)}) reservation = conn.run_instances( image_id=config['ami'], key_name=key_name, instance_type=config['instance_type'], block_device_map=bdm, client_token=token, disable_api_termination=config.get('disable_api_termination'), user_data=user_data, instance_profile_name=config.get('instance_profile_name'), network_interfaces=interfaces, ) break except boto.exception.BotoServerError: log.exception("Cannot start an instance") time.sleep(10) if max_attempts: attempt += 1 keep_going = max_attempts >= attempt instance = reservation.instances[0] log.info("instance %s created, waiting to come up", instance) # Wait for the instance to come up wait_for_status(instance, "state", "running", "update") instance.add_tag('Name', name) instance.add_tag('FQDN', instance_data['hostname']) instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime())) instance.add_tag('moz-type', config['type']) if loaned_to: instance.add_tag("moz-loaned-to", loaned_to) if loan_bug: instance.add_tag("moz-bug", loan_bug) log.info("assimilating %s", instance) instance.add_tag('moz-state', 'pending') keep_going, attempt = True, 1 while keep_going: try: # Don't reboot if need to create ami reboot = not create_ami assimilate_instance(instance=instance, config=config, ssh_key=ssh_key, instance_data=instance_data, deploypass=deploypass, reboot=reboot) break except NetworkError as e: # it takes a while for the machine to start/reboot so the # NetworkError exception is quite common, just log the error, # without the full stack trace log.warn("cannot connect; instance may still be starting %s (%s, %s) - %s," "retrying in %d sec ...", instance_data['hostname'], instance.id, instance.private_ip_address, e, FAILURE_TIMEOUT) time.sleep(FAILURE_TIMEOUT) except: # any other exception log.warn("problem assimilating %s (%s, %s), retrying in " "%d sec ...", instance_data['hostname'], instance.id, instance.private_ip_address, FAILURE_TIMEOUT, exc_info=True) time.sleep(FAILURE_TIMEOUT) if max_attempts: attempt += 1 keep_going = max_attempts >= attempt instance.add_tag('moz-state', 'ready') if create_ami: ami_name = "spot-%s-%s" % ( config['type'], time.strftime("%Y-%m-%d-%H-%M", time.gmtime())) log.info("Generating AMI %s", ami_name) ami_cleanup(mount_point="/", distro=config["distro"]) root_bd = instance.block_device_mapping[instance.root_device_name] volume = instance.connection.get_all_volumes( volume_ids=[root_bd.volume_id])[0] # The instance has to be stopped to flush EBS caches instance.stop() wait_for_status(instance, 'state', 'stopped', 'update') ami = volume_to_ami(volume=volume, ami_name=ami_name, arch=instance.architecture, virtualization_type=instance.virtualization_type, kernel_id=instance.kernel, root_device_name=instance.root_device_name, tags=config["tags"]) log.info("AMI %s (%s) is ready", ami_name, ami.id) log.warn("Terminating %s", instance) instance.terminate()
def test_create_launch_configuration_with_block_device_mappings(): block_device_mapping = BlockDeviceMapping() ephemeral_drive = BlockDeviceType() ephemeral_drive.ephemeral_name = "ephemeral0" block_device_mapping["/dev/xvdb"] = ephemeral_drive snapshot_drive = BlockDeviceType() snapshot_drive.snapshot_id = "snap-1234abcd" snapshot_drive.volume_type = "standard" block_device_mapping["/dev/xvdp"] = snapshot_drive ebs_drive = BlockDeviceType() ebs_drive.volume_type = "io1" ebs_drive.size = 100 ebs_drive.iops = 1000 ebs_drive.delete_on_termination = False block_device_mapping["/dev/xvdh"] = ebs_drive conn = boto.connect_autoscale(use_block_device_types=True) config = LaunchConfiguration( name="tester", image_id="ami-abcd1234", instance_type="m1.small", key_name="the_keys", security_groups=["default", "default2"], user_data=b"This is some user_data", instance_monitoring=True, instance_profile_name="arn:aws:iam::{}:instance-profile/testing".format( ACCOUNT_ID ), spot_price=0.1, block_device_mappings=[block_device_mapping], ) conn.create_launch_configuration(config) launch_config = conn.get_all_launch_configurations()[0] launch_config.name.should.equal("tester") launch_config.image_id.should.equal("ami-abcd1234") launch_config.instance_type.should.equal("m1.small") launch_config.key_name.should.equal("the_keys") set(launch_config.security_groups).should.equal(set(["default", "default2"])) launch_config.user_data.should.equal(b"This is some user_data") launch_config.instance_monitoring.enabled.should.equal("true") launch_config.instance_profile_name.should.equal( "arn:aws:iam::{}:instance-profile/testing".format(ACCOUNT_ID) ) launch_config.spot_price.should.equal(0.1) len(launch_config.block_device_mappings).should.equal(3) returned_mapping = launch_config.block_device_mappings set(returned_mapping.keys()).should.equal( set(["/dev/xvdb", "/dev/xvdp", "/dev/xvdh"]) ) returned_mapping["/dev/xvdh"].iops.should.equal(1000) returned_mapping["/dev/xvdh"].size.should.equal(100) returned_mapping["/dev/xvdh"].volume_type.should.equal("io1") returned_mapping["/dev/xvdh"].delete_on_termination.should.be.false returned_mapping["/dev/xvdp"].snapshot_id.should.equal("snap-1234abcd") returned_mapping["/dev/xvdp"].volume_type.should.equal("standard") returned_mapping["/dev/xvdb"].ephemeral_name.should.equal("ephemeral0")
import boto.ec2 from pprint import pprint def cloudconfig(): base_path = os.path.dirname(os.path.realpath(__file__)) cloud_config = open(os.path.join(base_path, 'cloud-config')) return cloud_config.read() close(os.path.join(base_path, 'cloud-config')) conn = boto.ec2.connect_to_region("eu-west-1") from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType block_device_map = BlockDeviceMapping() block_dev_type = BlockDeviceType() block_dev_type.delete_on_termination = True #block_dev_type.snapshot_id = snapshot_id block_device_map['/dev/sda'] = block_dev_type i = conn.run_instances( # 'ami-1c2e8b6b', 'ami-0c10417b', key_name='jpazdyga', user_data=cloudconfig(), instance_type='t1.micro', security_groups=['automat'], block_device_map=block_device_map) print(i.instances)
def create_instance(name, config, region, secrets, key_name, instance_data, deploypass, loaned_to, loan_bug): """Creates an AMI instance with the given name and config. The config must specify things like ami id.""" conn = get_connection( region, aws_access_key_id=secrets['aws_access_key_id'], aws_secret_access_key=secrets['aws_secret_access_key']) vpc = get_vpc(connection=conn, aws_access_key_id=secrets['aws_access_key_id'], aws_secret_access_key=secrets['aws_secret_access_key']) # Make sure we don't request the same things twice token = str(uuid.uuid4())[:16] instance_data = instance_data.copy() instance_data['name'] = name instance_data['hostname'] = '{name}.{domain}'.format( name=name, domain=config['domain']) ami = conn.get_all_images(image_ids=[config["ami"]])[0] bdm = None if 'device_map' in config: bdm = BlockDeviceMapping() for device, device_info in config['device_map'].items(): bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] # Overwrite root device size for HVM instances, since they cannot # be resized online if ami.virtualization_type == "hvm" and \ ami.root_device_name == device: bd.size = ami.block_device_mapping[ami.root_device_name].size if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] bdm[device] = bd ip_address = get_ip(instance_data['hostname']) subnet_id = None if ip_address: s_id = get_subnet_id(vpc, ip_address) if s_id in config['subnet_ids']: if ip_available(conn, ip_address): subnet_id = s_id else: log.warning("%s already assigned" % ip_address) if not ip_address or not subnet_id: ip_address = None subnet_id = choice(config.get('subnet_ids')) interface = NetworkInterfaceSpecification( subnet_id=subnet_id, private_ip_address=ip_address, delete_on_termination=True, groups=config.get('security_group_ids', []), associate_public_ip_address=config.get("use_public_ip")) interfaces = NetworkInterfaceCollection(interface) while True: try: reservation = conn.run_instances( image_id=config['ami'], key_name=key_name, instance_type=config['instance_type'], block_device_map=bdm, client_token=token, disable_api_termination=bool( config.get('disable_api_termination')), network_interfaces=interfaces, instance_profile_name=config.get("instance_profile_name"), ) break except boto.exception.BotoServerError: log.exception("Cannot start an instance") time.sleep(10) instance = reservation.instances[0] log.info("instance %s created, waiting to come up", instance) # Wait for the instance to come up while True: try: instance.update() if instance.state == 'running': break except Exception: log.warn("waiting for instance to come up, retrying in 10 sec...") time.sleep(10) instance.add_tag('Name', name) instance.add_tag('FQDN', instance_data['hostname']) instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime())) instance.add_tag('moz-type', config['type']) if loaned_to: instance.add_tag("moz-loaned-to", loaned_to) if loan_bug: instance.add_tag("moz-bug", loan_bug) log.info("assimilating %s", instance) instance.add_tag('moz-state', 'pending') while True: try: assimilate(instance.private_ip_address, config, instance_data, deploypass) break except: log.warn("problem assimilating %s (%s), retrying in 10 sec ...", instance_data['hostname'], instance.id) time.sleep(10) instance.add_tag('moz-state', 'ready')
def create_instance(name, config, region, key_name, ssh_key, instance_data, deploypass, loaned_to, loan_bug, create_ami, ignore_subnet_check, max_attempts): """Creates an AMI instance with the given name and config. The config must specify things like ami id.""" conn = get_aws_connection(region) # Make sure we don't request the same things twice token = str(uuid.uuid4())[:16] instance_data = instance_data.copy() instance_data['name'] = name instance_data['domain'] = config['domain'] instance_data['hostname'] = '{name}.{domain}'.format( name=name, domain=config['domain']) ami = conn.get_all_images(image_ids=[config["ami"]])[0] bdm = None if 'device_map' in config: bdm = BlockDeviceMapping() for device, device_info in config['device_map'].items(): bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] # Overwrite root device size for HVM instances, since they cannot # be resized online if ami.virtualization_type == "hvm" and \ ami.root_device_name == device: bd.size = ami.block_device_mapping[ami.root_device_name].size if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] bdm[device] = bd interfaces = make_instance_interfaces(region, instance_data['hostname'], ignore_subnet_check, config.get('subnet_ids'), config.get('security_group_ids', []), config.get("use_public_ip")) keep_going, attempt = True, 1 while keep_going: try: if 'user_data_file' in config: user_data = open(config['user_data_file']).read() else: user_data = get_user_data_tmpl(config['type']) if user_data: user_data = user_data.format( puppet_server=instance_data.get('default_puppet_server'), fqdn=instance_data['hostname'], hostname=instance_data['name'], domain=instance_data['domain'], dns_search_domain=config.get('dns_search_domain'), password=deploypass, moz_instance_type=config['type'], region_dns_atom=get_region_dns_atom(region), ) reservation = conn.run_instances( image_id=config['ami'], key_name=key_name, instance_type=config['instance_type'], block_device_map=bdm, client_token=token, disable_api_termination=config.get('disable_api_termination'), user_data=user_data, instance_profile_name=config.get('instance_profile_name'), network_interfaces=interfaces, ) break except boto.exception.BotoServerError: log.exception("Cannot start an instance") time.sleep(10) if max_attempts: attempt += 1 keep_going = max_attempts >= attempt instance = reservation.instances[0] log.info("instance %s created, waiting to come up", instance) # Wait for the instance to come up wait_for_status(instance, "state", "running", "update") instance.add_tag('Name', name) instance.add_tag('FQDN', instance_data['hostname']) instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime())) instance.add_tag('moz-type', config['type']) if loaned_to: instance.add_tag("moz-loaned-to", loaned_to) if loan_bug: instance.add_tag("moz-bug", loan_bug) log.info("assimilating %s", instance) instance.add_tag('moz-state', 'pending') keep_going, attempt = True, 1 while keep_going: try: # Don't reboot if need to create ami reboot = not create_ami assimilate_instance(instance=instance, config=config, ssh_key=ssh_key, instance_data=instance_data, deploypass=deploypass, reboot=reboot) break except NetworkError as e: # it takes a while for the machine to start/reboot so the # NetworkError exception is quite common, just log the error, # without the full stack trace log.warn( "cannot connect; instance may still be starting %s (%s, %s) - %s," "retrying in %d sec ...", instance_data['hostname'], instance.id, instance.private_ip_address, e, FAILURE_TIMEOUT) time.sleep(FAILURE_TIMEOUT) except: # any other exception log.warn( "problem assimilating %s (%s, %s), retrying in " "%d sec ...", instance_data['hostname'], instance.id, instance.private_ip_address, FAILURE_TIMEOUT, exc_info=True) time.sleep(FAILURE_TIMEOUT) if max_attempts: attempt += 1 keep_going = max_attempts >= attempt instance.add_tag('moz-state', 'ready') if create_ami: ami_name = "spot-%s-%s" % ( config['type'], time.strftime("%Y-%m-%d-%H-%M", time.gmtime())) log.info("Generating AMI %s", ami_name) ami_cleanup(mount_point="/", distro=config["distro"]) root_bd = instance.block_device_mapping[instance.root_device_name] volume = instance.connection.get_all_volumes( volume_ids=[root_bd.volume_id])[0] # The instance has to be stopped to flush EBS caches instance.stop() wait_for_status(instance, 'state', 'stopped', 'update') ami = volume_to_ami(volume=volume, ami_name=ami_name, arch=instance.architecture, virtualization_type=instance.virtualization_type, kernel_id=instance.kernel, root_device_name=instance.root_device_name, tags=config["tags"]) log.info("AMI %s (%s) is ready", ami_name, ami.id) log.warn("Terminating %s", instance) instance.terminate()
def create_cluster(conn, args): if args.identity_file is None: print("ERROR: Must provide an identity file (-i) for ssh connections.", file=stderr) sys.exit(1) if args.key_pair is None: print("ERROR: Must provide a key pair name (-k) to use on instances.", file=stderr) sys.exit(1) # make or get the security group. security_group = get_or_make_group(conn, args.name, args.vpc_id) # set the inbound permission rules if len(security_group.rules) == 0: if __name__ == '__main__': if args.vpc_id is None: security_group.authorize(src_group=security_group) else: security_group.authorize('tcp', 22, 22, args.authorized_address) security_group.authorize('tcp', 8888, 8888, args.authorized_address) security_group.authorize('tcp', 7000, 7000, args.authorized_address) security_group.authorize('tcp', 7001, 7001, args.authorized_address) security_group.authorize('tcp', 7199, 7199, args.authorized_address) security_group.authorize('tcp', 9042, 9042, args.authorized_address) security_group.authorize('tcp', 9160, 9160, args.authorized_address) else: print("Security group already exists, skipping creation.") instances = cluster_nodes(conn, args.name) if any(instances): additional_tags = {} for i in instances: i.add_tags( dict(additional_tags, Name="{cn}-node-{iid}".format(cn=args.name, iid=i.id))) return instances else: print( "Launching {m} instances for cluster...".format(m=args.node_count)) try: image = conn.get_all_images(image_ids=args.ami)[0] block_map = BlockDeviceMapping() if args.ebs_vol_size > 0: if args.instance_type.startswith('m3.'): for i in range(get_num_disks(args.instance_type)): device = BlockDeviceType() device.ephemeral_name = "ephemeral%d" % i name = "/dev/sd" + string.ascii_letters[i + 1] block_map[name] = device else: device = EBSBlockDeviceType() device.size = args.ebs_vol_size device.volume_type = args.ebs_vol_type device.delete_on_termination = True key = "/dev/sd" + chr(ord('s') + 1) block_map[key] = device nodes = image.run(key_name=args.key_pair, security_group_ids=[security_group.id], instance_type="", placement=args.zone, min_count=args.node_count, max_count=args.node_count, block_device_map=block_map, subnet_id=None, placement_group=None, user_data=None, instance_initiated_shutdown_behavior="stop", instance_profile_name=None) print("Waiting for AWS to propagate instance metadata...") time.sleep(15) additional_tags = {} for node in nodes.instances: node.add_tags( dict(additional_tags, Name="{cn}-node-{iid}".format(cn=args.name, iid=node.id))) return nodes.instances except Exception as e: print("Caught exception: ", e) print("ERROR: Could not find AMI " + args.ami, file=stderr) sys.exit(1)
def handleRunInstances(self, action, clc, user_data_file, callback): image_id = self.get_argument("ImageId") min = self.get_argument("MinCount", "1") max = self.get_argument("MaxCount", "1") key = self.get_argument("KeyName", None) groups = self.get_argument_list("SecurityGroup") sec_group_ids = self.get_argument_list("SecurityGroupId") if user_data_file: user_data = user_data_file else: user_data = self.get_argument("UserData", "") user_data = base64.b64decode(user_data) addr_type = self.get_argument("AddressingType", None) vm_type = self.get_argument("InstanceType", None) placement = self.get_argument("Placement.AvailabilityZone", None) placement_group = self.get_argument("Placement.GroupName", None) tenancy = self.get_argument("Placement.Tenancy", None) kernel = self.get_argument("KernelId", None) ramdisk = self.get_argument("RamdiskId", None) monitoring = False if self.get_argument("Monitoring.Enabled", "") == "true": monitoring = True subnet = self.get_argument("SubnetId", None) private_ip = self.get_argument("PrivateIpAddress", None) # get block device mappings bdm = BlockDeviceMapping() mapping = self.get_argument("BlockDeviceMapping.1.DeviceName", None) idx = 1 while mapping: pre = "BlockDeviceMapping.%d" % idx dev_name = mapping block_dev_type = BlockDeviceType() block_dev_type.ephemeral_name = self.get_argument("%s.VirtualName" % pre, None) if not (block_dev_type.ephemeral_name): block_dev_type.no_device = self.get_argument("%s.NoDevice" % pre, "") == "true" block_dev_type.snapshot_id = self.get_argument("%s.Ebs.SnapshotId" % pre, None) block_dev_type.size = self.get_argument("%s.Ebs.VolumeSize" % pre, None) block_dev_type.delete_on_termination = ( self.get_argument("%s.Ebs.DeleteOnTermination" % pre, "") == "true" ) bdm[dev_name] = block_dev_type idx += 1 mapping = self.get_argument("BlockDeviceMapping.%d.DeviceName" % idx, None) if len(bdm) == 0: bdm = None api_termination = False if self.get_argument("DisableApiTermination", "") == "true": api_termination = True instance_shutdown = False if self.get_argument("InstanceInitiatedShutdownBehavior", "") == "true": instance_shutdown = True token = self.get_argument("ClientToken", None) addition_info = self.get_argument("AdditionInfo", None) instance_profile_name = self.get_argument("IamInstanceProfile.Name", None) instance_profile_arn = self.get_argument("IamInstanceProfile.Arn", None) return clc.run_instances( image_id, min_count=min, max_count=max, key_name=key, security_groups=groups, user_data=user_data, addressing_type=addr_type, instance_type=vm_type, placement=placement, kernel_id=kernel, ramdisk_id=ramdisk, monitoring_enabled=monitoring, subnet_id=subnet, block_device_map=bdm, disable_api_termination=api_termination, instance_initiated_shutdown_behavior=instance_shutdown, private_ip_address=private_ip, placement_group=placement_group, client_token=token, security_group_ids=sec_group_ids, additional_info=addition_info, instance_profile_name=instance_profile_name, instance_profile_arn=instance_profile_arn, tenancy=tenancy, callback=callback, )
def create_instance(name, config, region, key_name, instance_data, deploypass, loaned_to, loan_bug): """Creates an AMI instance with the given name and config. The config must specify things like ami id.""" conn = get_aws_connection(region) vpc = get_vpc(region) # Make sure we don't request the same things twice token = str(uuid.uuid4())[:16] instance_data = instance_data.copy() instance_data['name'] = name instance_data['hostname'] = '{name}.{domain}'.format( name=name, domain=config['domain']) ami = conn.get_all_images(image_ids=[config["ami"]])[0] bdm = None if 'device_map' in config: bdm = BlockDeviceMapping() for device, device_info in config['device_map'].items(): bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] # Overwrite root device size for HVM instances, since they cannot # be resized online if ami.virtualization_type == "hvm" and \ ami.root_device_name == device: bd.size = ami.block_device_mapping[ami.root_device_name].size if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] bdm[device] = bd ip_address = get_ip(instance_data['hostname']) subnet_id = None if ip_address: s_id = get_subnet_id(vpc, ip_address) if s_id in config['subnet_ids']: if ip_available(conn, ip_address): subnet_id = s_id else: log.warning("%s already assigned" % ip_address) if not ip_address or not subnet_id: ip_address = None subnet_id = choice(config.get('subnet_ids')) interface = NetworkInterfaceSpecification( subnet_id=subnet_id, private_ip_address=ip_address, delete_on_termination=True, groups=config.get('security_group_ids', []), associate_public_ip_address=config.get("use_public_ip") ) interfaces = NetworkInterfaceCollection(interface) while True: try: reservation = conn.run_instances( image_id=config['ami'], key_name=key_name, instance_type=config['instance_type'], block_device_map=bdm, client_token=token, disable_api_termination=bool(config.get('disable_api_termination')), network_interfaces=interfaces, instance_profile_name=config.get("instance_profile_name"), ) break except boto.exception.BotoServerError: log.exception("Cannot start an instance") time.sleep(10) instance = reservation.instances[0] log.info("instance %s created, waiting to come up", instance) # Wait for the instance to come up wait_for_status(instance, "state", "running", "update") instance.add_tag('Name', name) instance.add_tag('FQDN', instance_data['hostname']) instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime())) instance.add_tag('moz-type', config['type']) if loaned_to: instance.add_tag("moz-loaned-to", loaned_to) if loan_bug: instance.add_tag("moz-bug", loan_bug) log.info("assimilating %s", instance) instance.add_tag('moz-state', 'pending') while True: try: assimilate(instance.private_ip_address, config, instance_data, deploypass) break except: log.warn("problem assimilating %s (%s), retrying in 10 sec ...", instance_data['hostname'], instance.id) time.sleep(10) instance.add_tag('moz-state', 'ready')
def do_create(self, ntry, params): """ Create stage of testing @param ntry: number of try @type ntry: int @param params: list of testing parameters @type params: list """ result = None logging.debug(self.getName() + ': trying to create instance ' + params['iname'] + ', ntry ' + str(ntry)) ntry += 1 try: bmap = BlockDeviceMapping() for device in params['bmap']: if not 'name' in device.keys(): logging.debug(self.getName() + ': bad device ' + str(device)) continue d = BlockDeviceType() if 'size' in device.keys(): d.size = device['size'] if 'delete_on_termination' in device.keys(): d.delete_on_termination = device['delete_on_termination'] if 'ephemeral_name' in device.keys(): d.ephemeral_name = device['ephemeral_name'] bmap[device['name']] = d reg = boto.ec2.get_region(params['region'], aws_access_key_id=ec2_key, aws_secret_access_key=ec2_secret_key) connection = reg.connect(aws_access_key_id=ec2_key, aws_secret_access_key=ec2_secret_key) (ssh_key_name, ssh_key) = yamlconfig['ssh'][params['region']] # all handled params to be put in here boto_params = ['ami', 'subnet_id'] for param in boto_params: params.setdefault(param) reservation = connection.run_instances( params['ami'], instance_type=params['ec2name'], key_name=ssh_key_name, block_device_map=bmap, subnet_id=params['subnet_id'], user_data=params['userdata'] ) myinstance = reservation.instances[0] count = 0 # Sometimes EC2 failes to return something meaningful without small timeout between run_instances() and update() time.sleep(10) while myinstance.update() == 'pending' and count < maxwait / 5: # Waiting out instance to appear logging.debug(params['iname'] + '... waiting...' + str(count)) time.sleep(5) count += 1 connection.close() instance_state = myinstance.update() if instance_state == 'running': # Instance appeared - scheduling 'setup' stage myinstance.add_tag('Name', params['name']) result = myinstance.__dict__ logging.info(self.getName() + ': created instance ' + params['iname'] + ', ' + result['id'] + ':' + result['public_dns_name']) # packing creation results into params params['id'] = result['id'] params['instance'] = result.copy() mainq.put((0, 'setup', params)) return elif instance_state == 'pending': # maxwait seconds is enough to create an instance. If not -- EC2 failed. logging.error('Error during instance creation: timeout in pending state') result = myinstance.__dict__ if 'id' in result.keys(): # terminate stucked instance params['id'] = result['id'] params['instance'] = result.copy() mainq.put((0, 'terminate', params.copy())) else: # error occured logging.error('Error during instance creation: ' + instance_state) except boto.exception.EC2ResponseError, e: # Boto errors should be handled according to their error Message - there are some well-known ones logging.debug(self.getName() + ': got boto error during instance creation: %s' % e) if str(e).find('<Code>InstanceLimitExceeded</Code>') != -1: # InstanceLimit is temporary problem logging.debug(self.getName() + ': got InstanceLimitExceeded - not increasing ntry') ntry -= 1 elif str(e).find('<Code>InvalidParameterValue</Code>') != -1: # InvalidParameterValue is really bad logging.error(self.getName() + ': got boto error during instance creation: %s' % e) # Try to speed up whole testing process, it's hardly possible to recover from this error ntry += args.maxtries // 2 + 1 elif str(e).find('<Code>Unsupported</Code>') != -1: # Unsupported logging.error(self.getName() + ': got Unsupported - most likely the permanent error: %s' % e) ntry += args.maxtries // 2 + 1 else: logging.debug(self.getName() + ':' + traceback.format_exc())
def handleImages(self, action, clc): if action == 'DescribeImages': owner = self.get_argument('Owner', None) if not owner: owners = None else: owners = [owner] return clc.get_all_images(owners) elif action == 'DescribeImageAttribute': imageid = self.get_argument('ImageId') attribute = self.get_argument('Attribute') return clc.get_image_attribute(imageid, attribute) elif action == 'ModifyImageAttribute': imageid = self.get_argument('ImageId') attribute = self.get_argument('Attribute') operation = self.get_argument('OperationType') users = self.get_argument_list('UserId') groups = self.get_argument_list('UserGroup') return clc.modify_image_attribute(imageid, attribute, operation, users, groups) elif action == 'ResetImageAttribute': imageid = self.get_argument('ImageId') attribute = self.get_argument('Attribute') return clc.reset_image_attribute(imageid, attribute) elif action == 'RegisterImage': image_location = self.get_argument('ImageLocation', None) name = self.get_argument('Name') description = self.get_argument('Description', None) description = base64.b64decode(description) architecture = self.get_argument('Architecture', None) kernel_id = self.get_argument('KernelId', None) ramdisk_id = self.get_argument('RamdiskId', None) root_dev_name = self.get_argument('RootDeviceName', None) snapshot_id = self.get_argument('SnapshotId', None) # get block device mappings bdm = BlockDeviceMapping() mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None) idx = 1 while mapping: pre = 'BlockDeviceMapping.%d' % idx dev_name = mapping block_dev_type = BlockDeviceType() block_dev_type.ephemeral_name = self.get_argument( '%s.VirtualName' % pre, None) if not (block_dev_type.ephemeral_name): block_dev_type.no_device = \ (self.get_argument('%s.NoDevice' % pre, '') == 'true') block_dev_type.snapshot_id = \ self.get_argument('%s.Ebs.SnapshotId' % pre, None) block_dev_type.size = \ self.get_argument('%s.Ebs.VolumeSize' % pre, None) block_dev_type.delete_on_termination = \ (self.get_argument('%s.DeleteOnTermination' % pre, '') == 'true') bdm[dev_name] = block_dev_type idx += 1 mapping = self.get_argument( 'BlockDeviceMapping.%d.DeviceName' % idx, None) if snapshot_id: rootbdm = BlockDeviceType() rootbdm.snapshot_id = snapshot_id bdm['/dev/sda1'] = rootbdm if len(bdm) == 0: bdm = None return clc.register_image(name, image_location, description, architecture, kernel_id, ramdisk_id, root_dev_name, bdm)
def handleRunInstances(self, action, clc, user_data_file): image_id = self.get_argument('ImageId') min = self.get_argument('MinCount', '1') max = self.get_argument('MaxCount', '1') key = self.get_argument('KeyName', None) groups = self.get_argument_list('SecurityGroup') sec_group_ids = self.get_argument_list('SecurityGroupId') if user_data_file: user_data = user_data_file else: user_data = self.get_argument('UserData', None) addr_type = self.get_argument('AddressingType', None) vm_type = self.get_argument('InstanceType', None) placement = self.get_argument('Placement.AvailabilityZone', None) placement_group = self.get_argument('Placement.GroupName', None) tenancy = self.get_argument('Placement.Tenancy', None) kernel = self.get_argument('KernelId', None) ramdisk = self.get_argument('RamdiskId', None) monitoring = False if self.get_argument('Monitoring.Enabled', '') == 'true': monitoring = True subnet = self.get_argument('SubnetId', None) private_ip = self.get_argument('PrivateIpAddress', None) # get block device mappings bdm = BlockDeviceMapping() mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None) idx = 1 while mapping: pre = 'BlockDeviceMapping.%d' % idx dev_name = mapping block_dev_type = BlockDeviceType() block_dev_type.ephemeral_name = self.get_argument( '%s.VirtualName' % pre, None) if not (block_dev_type.ephemeral_name): block_dev_type.no_device = \ (self.get_argument('%s.NoDevice' % pre, '') == 'true') block_dev_type.snapshot_id = \ self.get_argument('%s.Ebs.SnapshotId' % pre, None) block_dev_type.size = \ self.get_argument('%s.Ebs.VolumeSize' % pre, None) block_dev_type.delete_on_termination = \ (self.get_argument('%s.DeleteOnTermination' % pre, '') == 'true') bdm[dev_name] = block_dev_type idx += 1 mapping = self.get_argument( 'BlockDeviceMapping.%d.DeviceName' % idx, None) if len(bdm) == 0: bdm = None api_termination = False if self.get_argument('DisableApiTermination', '') == 'true': api_termination = True instance_shutdown = False if self.get_argument('InstanceInitiatedShutdownBehavior', '') == 'true': instance_shutdown = True token = self.get_argument('ClientToken', None) addition_info = self.get_argument('AdditionInfo', None) instance_profile_name = self.get_argument('IamInstanceProfile.Name', None) instance_profile_arn = self.get_argument('IamInstanceProfile.Arn', None) return self.__normalize_instances__([ clc.run_instances( image_id, min_count=min, max_count=max, key_name=key, security_groups=groups, user_data=user_data, addressing_type=addr_type, instance_type=vm_type, placement=placement, kernel_id=kernel, ramdisk_id=ramdisk, monitoring_enabled=monitoring, subnet_id=subnet, block_device_map=bdm, disable_api_termination=api_termination, instance_initiated_shutdown_behavior=instance_shutdown, private_ip_address=private_ip, placement_group=placement_group, client_token=token, security_group_ids=sec_group_ids, additional_info=addition_info, instance_profile_name=instance_profile_name, instance_profile_arn=instance_profile_arn, tenancy=tenancy) ])
def handleRunInstances(self, action, clc, user_data_file, callback): image_id = self.get_argument('ImageId') min = self.get_argument('MinCount', '1') max = self.get_argument('MaxCount', '1') key = self.get_argument('KeyName', None) groups = self.get_argument_list('SecurityGroup') sec_group_ids = self.get_argument_list('SecurityGroupId') if user_data_file: user_data = user_data_file else: user_data = self.get_argument('UserData', "") user_data = base64.b64decode(user_data) addr_type = self.get_argument('AddressingType', None) vm_type = self.get_argument('InstanceType', None) placement = self.get_argument('Placement.AvailabilityZone', None) placement_group = self.get_argument('Placement.GroupName', None) tenancy = self.get_argument('Placement.Tenancy', None) kernel = self.get_argument('KernelId', None) ramdisk = self.get_argument('RamdiskId', None) monitoring=False if self.get_argument('Monitoring.Enabled', '') == 'true': monitoring=True subnet = self.get_argument('SubnetId', None); private_ip = self.get_argument('PrivateIpAddress', None); # get block device mappings bdm = BlockDeviceMapping() mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None) idx = 1 while mapping: pre = 'BlockDeviceMapping.%d' % idx dev_name = mapping block_dev_type = BlockDeviceType() block_dev_type.ephemeral_name = self.get_argument('%s.VirtualName' % pre, None) if not(block_dev_type.ephemeral_name): block_dev_type.no_device = \ (self.get_argument('%s.NoDevice' % pre, '') == 'true') block_dev_type.snapshot_id = \ self.get_argument('%s.Ebs.SnapshotId' % pre, None) block_dev_type.size = \ self.get_argument('%s.Ebs.VolumeSize' % pre, None) block_dev_type.delete_on_termination = \ (self.get_argument('%s.Ebs.DeleteOnTermination' % pre, '') == 'true') bdm[dev_name] = block_dev_type idx += 1 mapping = self.get_argument('BlockDeviceMapping.%d.DeviceName' % idx, None) if len(bdm) == 0: bdm = None api_termination=False if self.get_argument('DisableApiTermination', '') == 'true': api_termination=True instance_shutdown=False if self.get_argument('InstanceInitiatedShutdownBehavior', '') == 'true': instance_shutdown=True token = self.get_argument('ClientToken', None); addition_info = self.get_argument('AdditionInfo', None); instance_profile_name = self.get_argument('IamInstanceProfile.Name', None); instance_profile_arn = self.get_argument('IamInstanceProfile.Arn', None); return clc.run_instances(image_id, min_count=min, max_count=max, key_name=key, security_groups=groups, user_data=user_data, addressing_type=addr_type, instance_type=vm_type, placement=placement, kernel_id=kernel, ramdisk_id=ramdisk, monitoring_enabled=monitoring, subnet_id=subnet, block_device_map=bdm, disable_api_termination=api_termination, instance_initiated_shutdown_behavior=instance_shutdown, private_ip_address=private_ip, placement_group=placement_group, client_token=token, security_group_ids=sec_group_ids, additional_info=addition_info, instance_profile_name=instance_profile_name, instance_profile_arn=instance_profile_arn, tenancy=tenancy, callback=callback)
def handleImages(self, action, clc, callback=None): if action == 'DescribeImages': owner = self.get_argument('Owner', None); if not owner: owners = None else: owners = [owner] filters = self.get_filter_args() return clc.get_all_images(owners, filters, callback) elif action == 'DescribeImageAttribute': imageid = self.get_argument('ImageId') attribute = self.get_argument('Attribute') return clc.get_image_attribute(imageid, attribute, callback) elif action == 'ModifyImageAttribute': imageid = self.get_argument('ImageId') attribute = self.get_argument('Attribute') operation = self.get_argument('OperationType') users = self.get_argument_list('UserId') groups = self.get_argument_list('UserGroup') return clc.modify_image_attribute(imageid, attribute, operation, users, groups, callback) elif action == 'ResetImageAttribute': imageid = self.get_argument('ImageId') attribute = self.get_argument('Attribute') return clc.reset_image_attribute(imageid, attribute, callback) elif action == 'DeregisterImage': image_id = self.get_argument('ImageId') return clc.deregister_image(image_id, callback) elif action == 'RegisterImage': image_location = self.get_argument('ImageLocation', None) name = self.get_argument('Name') description = self.get_argument('Description', None) if description != None: description = base64.b64decode(description); architecture = self.get_argument('Architecture', None) kernel_id = self.get_argument('KernelId', None) ramdisk_id = self.get_argument('RamdiskId', None) root_dev_name = self.get_argument('RootDeviceName', None) snapshot_id = self.get_argument('SnapshotId', None) # get block device mappings bdm = BlockDeviceMapping() mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None) idx = 1 while mapping: pre = 'BlockDeviceMapping.%d' % idx dev_name = mapping block_dev_type = BlockDeviceType() block_dev_type.ephemeral_name = self.get_argument('%s.VirtualName' % pre, None) if not(block_dev_type.ephemeral_name): block_dev_type.no_device = \ (self.get_argument('%s.NoDevice' % pre, '') == 'true') block_dev_type.snapshot_id = \ self.get_argument('%s.Ebs.SnapshotId' % pre, None) block_dev_type.size = \ self.get_argument('%s.Ebs.VolumeSize' % pre, None) block_dev_type.delete_on_termination = \ (self.get_argument('%s.Ebs.DeleteOnTermination' % pre, '') == 'true') bdm[dev_name] = block_dev_type idx += 1 mapping = self.get_argument('BlockDeviceMapping.%d.DeviceName' % idx, None) if snapshot_id: rootbdm = BlockDeviceType() rootbdm.snapshot_id = snapshot_id bdm['/dev/sda1'] = rootbdm if len(bdm) == 0: bdm = None return clc.register_image(name, image_location, description, architecture, kernel_id, ramdisk_id, root_dev_name, bdm, callback)
def register(snapshot_id, region, arch, size=None, name=None, desc=None, pvm=False): conn = utils.connect(region) if None in (name, size): log.debug('getting snapshot - %s', snapshot_id) snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0] size = size if size else snapshot.volume_size name = name if name else snapshot.description virt = 'hvm' kernel_id = None device_base = '/dev/xvd' ec2_arch = "x86_64" if arch == "amd64" else arch if pvm: kernel_id = utils.get_kernel(region, arch) virt = 'paravirtual' device_base = '/dev/sd' name += '-pvm' log.debug('creating block_device_map') block_device_map = BlockDeviceMapping() rootfs = BlockDeviceType() rootfs.delete_on_termination = True rootfs.size = size rootfs.snapshot_id = snapshot_id rootfs_device_name = device_base + 'a' block_device_map[rootfs_device_name] = rootfs ephemeral = BlockDeviceType() ephemeral.ephemeral_name = 'ephemeral0' ephemeral_device_name = device_base + 'b' block_device_map[ephemeral_device_name] = ephemeral log.debug('registering image - %s', name) client3 = utils.connect_boto3(region) response = client3.register_image(Name=name, Architecture=ec2_arch, RootDeviceName=rootfs_device_name, BlockDeviceMappings=[{ 'DeviceName': '/dev/xvda', 'Ebs': { 'DeleteOnTermination': True, 'VolumeSize': size, 'SnapshotId': snapshot_id, }, }, { 'DeviceName': '/dev/xvdb', 'VirtualName': 'ephemeral0', }], VirtualizationType=virt, EnaSupport=True) ami_id = response['ImageId'] log.info('registered image - %s %s %s', ami_id, name, region) return ami_id, name