def create_block_device_mapping(ami, device_map): bdm = BlockDeviceMapping() for device, device_info in device_map.items(): if ami.root_device_type == "instance-store" and \ not device_info.get("ephemeral_name"): # EBS is not supported by S3-backed AMIs at request time # EBS volumes can be attached when an instance is running continue bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] if ami.root_device_name == device: ami_size = ami.block_device_mapping[device].size if ami.virtualization_type == "hvm": # Overwrite root device size for HVM instances, since they # cannot be resized online bd.size = ami_size elif device_info.get('size'): # make sure that size is enough for this AMI assert ami_size <= device_info['size'], \ "Instance root device size cannot be smaller than AMI " \ "root device" if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] if device_info.get("volume_type"): bd.volume_type = device_info["volume_type"] if device_info["volume_type"] == "io1" \ and device_info.get("iops"): bd.iops = device_info["iops"] bdm[device] = bd return bdm
def run_instance(conn, ami_id, key_name, instance_type, sec_group, zone="us-east-1d", vol_size=None): """ @param connection: python boto connection @param ami_id: AMI ID @param key_name: SSH key name @param instance_type: instance type, example 'm1.large' @param sec_group: security group @param zone: optional, defaults to 'us-east-1d' @param vol_size: optional integer, if specified will change size of root volume to this size @return boto.ec2.instance.Instance """ bdm = None if vol_size: # Create block device mapping info dev_sda1 = BlockDeviceType() dev_sda1.size = int(vol_size) dev_sda1.delete_on_termination = True bdm = BlockDeviceMapping() bdm['/dev/sda1'] = dev_sda1 # Run instance reservation = conn.run_instances( ami_id, key_name=key_name, instance_type=instance_type, placement=zone, instance_initiated_shutdown_behavior="stop", security_groups=[sec_group], block_device_map=bdm) return reservation.instances[0]
def get_block_device_map(bdmapping_json=None): """Parse block_device_mapping JSON and return a configured BlockDeviceMapping object Mapping JSON structure... {"/dev/sda": {"snapshot_id": "snap-23E93E09", "volume_type": null, "delete_on_termination": true, "size": 1} } """ if bdmapping_json: mapping = json.loads(bdmapping_json) if mapping: bdm = BlockDeviceMapping() for key, val in mapping.items(): device = BlockDeviceType() if val.get('virtual_name') is not None and val.get( 'virtual_name').startswith('ephemeral'): device.ephemeral_name = val.get('virtual_name') else: device.volume_type = 'standard' device.snapshot_id = val.get('snapshot_id') or None device.size = val.get('size') device.delete_on_termination = val.get( 'delete_on_termination', False) bdm[key] = device return bdm return None return None
def test_create_launch_configuration_with_block_device_mappings(): block_device_mapping = BlockDeviceMapping() ephemeral_drive = BlockDeviceType() ephemeral_drive.ephemeral_name = 'ephemeral0' block_device_mapping['/dev/xvdb'] = ephemeral_drive snapshot_drive = BlockDeviceType() snapshot_drive.snapshot_id = "snap-1234abcd" snapshot_drive.volume_type = "standard" block_device_mapping['/dev/xvdp'] = snapshot_drive ebs_drive = BlockDeviceType() ebs_drive.volume_type = "io1" ebs_drive.size = 100 ebs_drive.iops = 1000 ebs_drive.delete_on_termination = False block_device_mapping['/dev/xvdh'] = ebs_drive conn = boto.connect_autoscale(use_block_device_types=True) config = LaunchConfiguration( name='tester', image_id='ami-abcd1234', instance_type='m1.small', key_name='the_keys', security_groups=["default", "default2"], user_data="This is some user_data", instance_monitoring=True, instance_profile_name='arn:aws:iam::123456789012:instance-profile/testing', spot_price=0.1, block_device_mappings=[block_device_mapping] ) conn.create_launch_configuration(config) launch_config = conn.get_all_launch_configurations()[0] launch_config.name.should.equal('tester') launch_config.image_id.should.equal('ami-abcd1234') launch_config.instance_type.should.equal('m1.small') launch_config.key_name.should.equal('the_keys') set(launch_config.security_groups).should.equal(set(['default', 'default2'])) launch_config.user_data.should.equal("This is some user_data") launch_config.instance_monitoring.enabled.should.equal('true') launch_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/testing') launch_config.spot_price.should.equal(0.1) len(launch_config.block_device_mappings).should.equal(3) returned_mapping = launch_config.block_device_mappings set(returned_mapping.keys()).should.equal(set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh'])) returned_mapping['/dev/xvdh'].iops.should.equal(1000) returned_mapping['/dev/xvdh'].size.should.equal(100) returned_mapping['/dev/xvdh'].volume_type.should.equal("io1") returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd") returned_mapping['/dev/xvdp'].volume_type.should.equal("standard") returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')
def _parse_block_device_mappings(user_input): """ Parse block device mappings per AWS CLI tools syntax (modified to add IOPS) http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html Syntax: /dev/xvd[a-z]=[snapshot-id|ephemeral]:[size in GB]:[Delete on Term]:[IOPS] - Leave inapplicable fields blank - Delete on Termination defaults to True - IOPS limits are not validated - EBS sizing is not validated Mount an Ephemeral Drive: /dev/xvdb1=ephemeral0 Mount multiple Ephemeral Drives: /dev/xvdb1=ephemeral0,/dev/xvdb2=ephemeral1 Mount a Snapshot: /dev/xvdp=snap-1234abcd Mount a Snapshot to a 100GB drive: /dev/xvdp=snap-1234abcd:100 Mount a Snapshot to a 100GB drive and do not delete on termination: /dev/xvdp=snap-1234abcd:100:false Mount a Fresh 100GB EBS device /dev/xvdp=:100 Mount a Fresh 100GB EBS Device and do not delete on termination: /dev/xvdp=:100:false Mount a Fresh 100GB EBS Device with 1000 IOPS /dev/xvdp=:100::1000 """ block_device_map = BlockDeviceMapping() mappings = user_input.split(",") for mapping in mappings: block_type = BlockDeviceType() mount_point, drive_type, size, delete, iops = _parse_drive_mapping( mapping) if 'ephemeral' in drive_type: block_type.ephemeral_name = drive_type elif 'snap' in drive_type: block_type.snapshot_id = drive_type block_type.volume_type = "standard" else: block_type.volume_type = "standard" block_type.size = size block_type.delete_on_termination = delete if iops: block_type.iops = iops block_type.volume_type = "io1" block_device_map[mount_point] = block_type return block_device_map
def _parse_block_device_mappings(user_input): """ Parse block device mappings per AWS CLI tools syntax (modified to add IOPS) http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html Syntax: /dev/xvd[a-z]=[snapshot-id|ephemeral]:[size in GB]:[Delete on Term]:[IOPS] - Leave inapplicable fields blank - Delete on Termination defaults to True - IOPS limits are not validated - EBS sizing is not validated Mount an Ephemeral Drive: /dev/xvdb1=ephemeral0 Mount multiple Ephemeral Drives: /dev/xvdb1=ephemeral0,/dev/xvdb2=ephemeral1 Mount a Snapshot: /dev/xvdp=snap-1234abcd Mount a Snapshot to a 100GB drive: /dev/xvdp=snap-1234abcd:100 Mount a Snapshot to a 100GB drive and do not delete on termination: /dev/xvdp=snap-1234abcd:100:false Mount a Fresh 100GB EBS device /dev/xvdp=:100 Mount a Fresh 100GB EBS Device and do not delete on termination: /dev/xvdp=:100:false Mount a Fresh 100GB EBS Device with 1000 IOPS /dev/xvdp=:100::1000 """ block_device_map = BlockDeviceMapping() mappings = user_input.split(",") for mapping in mappings: block_type = BlockDeviceType() mount_point, drive_type, size, delete, iops = _parse_drive_mapping(mapping) if 'ephemeral' in drive_type: block_type.ephemeral_name = drive_type elif 'snap' in drive_type: block_type.snapshot_id = drive_type block_type.volume_type = "standard" else: block_type.volume_type = "standard" block_type.size = size block_type.delete_on_termination = delete if iops: block_type.iops = iops block_type.volume_type = "io1" block_device_map[mount_point] = block_type return block_device_map
def register(snapshot_id, region, arch, size=None, name=None, desc=None, pvm=False): conn = utils.connect(region) if None in (name, size): log.debug('getting snapshot - %s', snapshot_id) snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0] size = size if size else snapshot.volume_size name = name if name else snapshot.description virt = 'hvm' kernel_id = None device_base = '/dev/xvd' ec2_arch = "x86_64" if arch == "amd64" else arch if pvm: kernel_id = utils.get_kernel(region, arch) virt = 'paravirtual' device_base = '/dev/sd' name += '-pvm' log.debug('creating block_device_map') block_device_map = BlockDeviceMapping() rootfs = BlockDeviceType() rootfs.delete_on_termination = True rootfs.size = size rootfs.snapshot_id = snapshot_id rootfs_device_name = device_base + 'a' block_device_map[rootfs_device_name] = rootfs ephemeral = BlockDeviceType() ephemeral.ephemeral_name = 'ephemeral0' ephemeral_device_name = device_base + 'b' block_device_map[ephemeral_device_name] = ephemeral log.debug('registering image - %s', name) ami_id = conn.register_image(name=name, description=desc, kernel_id=kernel_id, architecture=ec2_arch, root_device_name=rootfs_device_name, block_device_map=block_device_map, virtualization_type=virt) log.info('registered image - %s %s %s', ami_id, name, region) return ami_id, name
def _process_block_device_mappings(self, launch_config, zone=None): """ Processes block device mapping information and returns a Boto BlockDeviceMapping object. If new volumes are requested (source is None and destination is VOLUME), they will be created and the relevant volume ids included in the mapping. """ bdm = BlockDeviceMapping() # Assign letters from f onwards # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html next_letter = iter(list(string.ascii_lowercase[6:])) # assign ephemeral devices from 0 onwards ephemeral_counter = 0 for device in launch_config.block_devices: bd_type = BlockDeviceType() if device.is_volume: if device.is_root: bdm['/dev/sda1'] = bd_type else: bdm['sd' + next(next_letter)] = bd_type if isinstance(device.source, Snapshot): bd_type.snapshot_id = device.source.id elif isinstance(device.source, Volume): bd_type.volume_id = device.source.id elif isinstance(device.source, MachineImage): # Not supported pass else: # source is None, but destination is volume, therefore # create a blank volume. If the Zone is None, this # could fail since the volume and instance may be created # in two different zones. if not zone: raise InvalidConfigurationException( "A zone must be specified when launching with a" " new blank volume block device mapping.") new_vol = self.provider.block_store.volumes.create( '', device.size, zone) bd_type.volume_id = new_vol.id bd_type.delete_on_terminate = device.delete_on_terminate if device.size: bd_type.size = device.size else: # device is ephemeral bd_type.ephemeral_name = 'ephemeral%s' % ephemeral_counter return bdm
def _parse_block_device_mappings(self): block_device_map = BlockDeviceMapping() for mapping in self.block_device_mapping_dict: block_type = BlockDeviceType() mount_point = mapping.get('device_name') if 'ephemeral' in mapping.get('virtual_name', ''): block_type.ephemeral_name = mapping.get('virtual_name') else: block_type.volume_type = mapping.get('ebs._volume_type') block_type.snapshot_id = mapping.get('ebs._snapshot_id') block_type.delete_on_termination = mapping.get('ebs._delete_on_termination') block_type.size = mapping.get('ebs._volume_size') block_type.iops = mapping.get('ebs._iops') block_device_map[mount_point] = block_type return block_device_map
def _getBlockDeviceMapping(cls, instanceType, rootVolSize=50): # determine number of ephemeral drives via cgcloud-lib bdtKeys = ['', '/dev/xvdb', '/dev/xvdc', '/dev/xvdd'] bdm = BlockDeviceMapping() # Change root volume size to allow for bigger Docker instances root_vol = BlockDeviceType(delete_on_termination=True) root_vol.size = rootVolSize bdm["/dev/xvda"] = root_vol # the first disk is already attached for us so start with 2nd. for disk in xrange(1, instanceType.disks + 1): bdm[bdtKeys[disk]] = BlockDeviceType( ephemeral_name='ephemeral{}'.format(disk - 1)) # ephemeral counts start at 0 logger.debug('Device mapping: %s', bdm) return bdm
def _getBlockDeviceMapping(cls, instanceType, rootVolSize=50): # determine number of ephemeral drives via cgcloud-lib bdtKeys = [''] + ['/dev/xvd{}'.format(c) for c in string.lowercase[1:]] bdm = BlockDeviceMapping() # Change root volume size to allow for bigger Docker instances root_vol = BlockDeviceType(delete_on_termination=True) root_vol.size = rootVolSize bdm["/dev/xvda"] = root_vol # the first disk is already attached for us so start with 2nd. for disk in range(1, instanceType.disks + 1): bdm[bdtKeys[disk]] = BlockDeviceType( ephemeral_name='ephemeral{}'.format(disk - 1)) # ephemeral counts start at 0 logger.debug('Device mapping: %s', bdm) return bdm
def _get_bmap(self, params): bmap = BlockDeviceMapping() for device in params['bmap']: if not 'name' in device.keys(): self.logger.debug('bad device ' + str(device)) continue dev = BlockDeviceType() if 'size' in device.keys(): dev.size = device['size'] if 'delete_on_termination' in device.keys(): dev.delete_on_termination = device['delete_on_termination'] if 'ephemeral_name' in device.keys(): dev.ephemeral_name = device['ephemeral_name'] bmap[device['name']] = dev return bmap
def _getBlockDeviceMapping(cls, instanceType): # determine number of ephemeral drives via cgcloud-lib bdtKeys = ['', '/dev/xvdb', '/dev/xvdc', '/dev/xvdd'] bdm = BlockDeviceMapping() # Change root volume size to allow for bigger Docker instances root_vol = BlockDeviceType() root_vol.size = 50 bdm["/dev/xvda"] = root_vol # the first disk is already attached for us so start with 2nd. for disk in xrange(1, instanceType.disks + 1): bdm[bdtKeys[disk]] = BlockDeviceType( ephemeral_name='ephemeral{}'.format(disk - 1)) # ephemeral counts start at 0 logger.debug('Device mapping: %s', bdm) return bdm
def startCluster(self, argv): if len(argv) != 0: print "ec2 startCluster" sys.exit(-1) regions = boto.ec2.regions() regionInfo = '\n'.join(str(region).split(':')[1] for region in regions) regionName = raw_input("select region:\n%s\n>>"%regionInfo) region = boto.ec2.get_region(regionName) conn = region.connect() print "region connected successfully" images = conn.get_all_images(owners='self') imageInfo = '\n'.join( str(image).split(':')[1] + ":" + image.name for image in images) imageId = raw_input("enter imageId:\nself-created images:\n%s\n>>"%imageInfo) instanceTypeInfo = ("m1.small, " "m1.large, " "m1.xlarge\n" "c1.medium, " "c1.xlarge\n" "m2.xlarge, " "m2.2xlarge, " "m2.4xlarge\n" "cc1.4xlarge, " "t1.micro\n") instanceType = raw_input("enter instanceType:\n%s\n>>"%instanceTypeInfo) availZone = raw_input("enter placement[a,b,c]:\n>>") availZone = regionName + availZone diskSize = int(raw_input("enter disk size[G]:\n>>")) rootDev = BlockDeviceType() rootDev.name = 'root' rootDev.size = diskSize rootDev.delete_on_termination = True instStorage = bool(raw_input("mount inst storage?\n>>")) mapping = BlockDeviceMapping() mapping['/dev/sda1'] = rootDev if (instStorage == True): eph0 = BlockDeviceType() eph0.ephemeral_name = 'ephemeral0' mapping['/dev/sdb'] = eph0 groups = conn.get_all_security_groups() groupInfo = '\n'.join(str(group).split(':')[1] for group in groups) group = raw_input("enter securityGroup:\n%s\n>>"%groupInfo) keys = conn.get_all_key_pairs() if len(keys) == 1: key = keys[0].name print 'using default key: ' + key else: keyInfo = '\n'.join(str(key).split(':')[1] for key in keys) key = raw_input("enter key name:\n%s\n>>"%keyInfo) numNodes = int(raw_input("number of nodes:\n>>")) conn.run_instances( imageId, min_count=numNodes, max_count=numNodes, placement=availZone, security_groups = [group], instance_type=instanceType, block_device_map=mapping, key_name=key)
def _parse_block_device_mappings(self): block_device_map = BlockDeviceMapping() for mapping in self.block_device_mapping_dict: block_type = BlockDeviceType() mount_point = mapping.get("device_name") if "ephemeral" in mapping.get("virtual_name", ""): block_type.ephemeral_name = mapping.get("virtual_name") else: block_type.volume_type = mapping.get("ebs._volume_type") block_type.snapshot_id = mapping.get("ebs._snapshot_id") block_type.delete_on_termination = mapping.get( "ebs._delete_on_termination") block_type.size = mapping.get("ebs._volume_size") block_type.iops = mapping.get("ebs._iops") block_device_map[mount_point] = block_type return block_device_map
def _getBlockDeviceMapping(cls, instanceType, rootVolSize=50): # determine number of ephemeral drives via cgcloud-lib (actually this is moved into toil's lib bdtKeys = [''] + ['/dev/xvd{}'.format(c) for c in string.ascii_lowercase[1:]] bdm = BlockDeviceMapping() # Change root volume size to allow for bigger Docker instances root_vol = BlockDeviceType(delete_on_termination=True) root_vol.size = rootVolSize bdm["/dev/xvda"] = root_vol # The first disk is already attached for us so start with 2nd. # Disk count is weirdly a float in our instance database, so make it an int here. for disk in range(1, int(instanceType.disks) + 1): bdm[bdtKeys[disk]] = BlockDeviceType( ephemeral_name='ephemeral{}'.format(disk - 1)) # ephemeral counts start at 0 logger.debug('Device mapping: %s', bdm) return bdm
def _get_block_device_mapping(device_name, size): """ Returns a block device mapping object for the specified device and size. Block Device Mapping is used to associate a device on the VM with an EBS Volume. parameters: device_name -- The name of the device in the VM, such as /dev/sda1, /dev/sdb1. etc size -- The amount of space to allocate for the EBS drive. """ block_device = BlockDeviceType() block_device.size = size bdm = BlockDeviceMapping() bdm[device_name] = block_device return bdm
def register(snapshot_id, region, arch, size=None, name=None, desc=None, pvm=False): conn = utils.connect(region) if None in (name, size): log.debug('getting snapshot - %s', snapshot_id) snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0] size = size if size else snapshot.volume_size name = name if name else snapshot.description virt = 'hvm' kernel_id = None device_base = '/dev/xvd' ec2_arch = "x86_64" if arch == "amd64" else arch if pvm: kernel_id = utils.get_kernel(region, arch) virt = 'paravirtual' device_base = '/dev/sd' name += '-pvm' log.debug('creating block_device_map') block_device_map = BlockDeviceMapping() rootfs = BlockDeviceType() rootfs.delete_on_termination = True rootfs.size = size rootfs.snapshot_id = snapshot_id rootfs_device_name = device_base + 'a' block_device_map[rootfs_device_name] = rootfs ephemeral = BlockDeviceType() ephemeral.ephemeral_name = 'ephemeral0' ephemeral_device_name = device_base + 'b' block_device_map[ephemeral_device_name] = ephemeral log.debug('registering image - %s', name) ami_id = conn.register_image( name=name, description=desc, kernel_id=kernel_id, architecture=ec2_arch, root_device_name=rootfs_device_name, block_device_map=block_device_map, virtualization_type=virt) log.info('registered image - %s %s %s', ami_id, name, region) return ami_id, name
def launch_ondemand_request(conn, request, tenant, job): try: mapping = BlockDeviceMapping() sda1 = BlockDeviceType() eph0 = BlockDeviceType() eph1 = BlockDeviceType() eph2 = BlockDeviceType() eph3 = BlockDeviceType() sda1.size = 10 eph0.ephemeral_name = 'ephemeral0' eph1.ephemeral_name = 'ephemeral1' eph2.ephemeral_name = 'ephemeral2' eph3.ephemeral_name = 'ephemeral3' mapping['/dev/sda1'] = sda1 mapping['/dev/sdb'] = eph0 mapping['/dev/sdc'] = eph1 mapping['/dev/sdd'] = eph2 mapping['/dev/sde'] = eph3 # issue a run_instances command for this request res = conn.run_instances(min_count=request.count, max_count=request.count, key_name=tenant.key_pair, image_id=request.ami, security_group_ids=[tenant.security_group], user_data=customise_cloudinit(tenant, job), instance_type=request.instance_type, subnet_id=tenant.subnet, block_device_map=mapping) my_req_ids = [req.id for req in res.instances] # address = "" for req in my_req_ids: # tag each request tag_requests(req, tenant.name, conn) # update the database to include the new request ProvisionerConfig().dbconn.execute( ("insert into instance_request (tenant, instance_type, " + "price, job_runner_id, request_type, request_id, " + "subnet) values ('%s', '%s', %s, %s, '%s', '%s', %s)") % (tenant.db_id, request.instance.db_id, request.instance.ondemand, job.id, "ondemand", req, tenant.subnet_id)) # ProvisionerConfig().dbconn.commit() return except boto.exception.EC2ResponseError: logger.exception("There was an error communicating with EC2.")
def _get_block_device_mapping(device_name, size, delete_on_terminate = False): """ Returns a block device mapping object for the specified device and size. Block Device Mapping is used to associate a device on the VM with an EBS Volume. parameters: device_name -- The name of the device in the VM, such as /dev/sda1, /dev/sdb1. etc size -- The amount of space to allocate for the EBS drive. delete_on_terminate -- Whether the volume should be deleted when the instance is terminated """ block_device = BlockDeviceType(delete_on_termination=delete_on_terminate) block_device.size = size bdm = BlockDeviceMapping() bdm[device_name] = block_device return bdm
def create_root_block_device(self): # if the root volume size is not the same as the AMI default value: if self.root_volume_size is not None: # sda rather than xvda (for Windows) dev_sda1 = BlockDeviceType() dev_sda1.size = self.root_volume_size dev_sda1.delete_on_termination = True volume = BlockDeviceMapping() # Check the OS type, if its windows we use sda, linux: xvda images = self.ec2.get_all_images(image_ids=[self.node_obj.ami]) image = images[0] if image.platform is None: volume['/dev/xvda'] = dev_sda1 else: volume['/dev/sda1'] = dev_sda1 self.volumes = volume
def start_remote(self): self.get_bootstrap_kernel() self.get_bootstrap_image() # create EBS volume for /mnt/gentoo device = BlockDeviceType() device.size = self.settings["ec2/instance/device/size"] device.delete_on_termination = True mapping = BlockDeviceMapping() self.root_device = "/dev/" + self.settings["ec2/instance/device/name"] mapping[self.root_device] = device # start bootstrapping instance reservation = self.ec2.run_instances( self.bootstrap_image.id, kernel_id=self.bootstrap_kernel.id, instance_type=self.settings["ec2/instance/type"], security_groups=[self.name], key_name=self.name, block_device_map=mapping) self.instance = reservation.instances[0] sys.stdout.write("waiting for instance to come up ..") while self.instance.update() != 'running': sys.stdout.write(".") sys.stdout.flush() time.sleep(5) sys.stdout.write("\n") time.sleep(120) self.ssh_uri = "ec2-user@" + self.instance.public_dns_name self.remote_upload_path = "/tmp" # enable sudo without a tty cmd = "sudo sed -i -e '/requiretty/d' /etc/sudoers" cmd = ["ssh", "-t"] + self.ssh_options() + [self.ssh_uri, cmd] ssh = subprocess.Popen(cmd) ssh.wait() self.run_script_at_remote("steps/remote/postboot")
def start_remote(self): self.get_bootstrap_kernel() self.get_bootstrap_image() # create EBS volume for /mnt/gentoo device = BlockDeviceType() device.size = self.settings["ec2/instance/device/size"] device.delete_on_termination = True mapping = BlockDeviceMapping() self.root_device = "/dev/" + self.settings["ec2/instance/device/name"] mapping[self.root_device] = device # start bootstrapping instance reservation = self.ec2.run_instances(self.bootstrap_image.id, kernel_id=self.bootstrap_kernel.id, instance_type=self.settings["ec2/instance/type"], security_groups=[self.name], key_name=self.name, block_device_map=mapping) self.instance = reservation.instances[0] sys.stdout.write("waiting for instance to come up ..") while self.instance.update() != 'running': sys.stdout.write(".") sys.stdout.flush() time.sleep(5) sys.stdout.write("\n") time.sleep(120) self.ssh_uri = "ec2-user@" + self.instance.public_dns_name self.ssh_port = "22" self.remote_upload_path = "/tmp" # enable sudo without a tty cmd = "sudo sed -i -e '/requiretty/d' /etc/sudoers" cmd = ["ssh", "-t"] + self.ssh_options() + [self.ssh_uri, cmd] ssh = subprocess.Popen(cmd) ssh.wait() self.run_script_at_remote("steps/remote/postboot")
def run_instance(conn, ami_id, key_name, instance_type, sec_group, zone, vol_size=None): bdm = None if vol_size: # Create block device mapping info dev_sda1 = BlockDeviceType() dev_sda1.size = int(vol_size) dev_sda1.delete_on_termination = True bdm = BlockDeviceMapping() bdm["/dev/sda1"] = dev_sda1 # Run instance reservation = conn.run_instances( ami_id, key_name=key_name, instance_type=instance_type, placement=zone, instance_initiated_shutdown_behavior="stop", security_groups=[sec_group], block_device_map=bdm, ) return reservation.instances[0]
def register(snapshot_id, region, arch, size=None, name=None, desc=None): conn = utils.connect(region) if None in (name, size): log.debug("getting snapshot - %s", snapshot_id) snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0] size = size if size else snapshot.volume_size name = name if name else snapshot.description ec2_arch = "x86_64" if arch == "amd64" else arch log.debug("creating block_device_map") block_device_map = BlockDeviceMapping() rootfs = BlockDeviceType() rootfs.delete_on_termination = True rootfs.size = size rootfs.snapshot_id = snapshot_id rootfs_device_name = "/dev/xvda" block_device_map[rootfs_device_name] = rootfs ephemeral = BlockDeviceType() ephemeral.ephemeral_name = "ephemeral0" ephemeral_device_name = "/dev/xvdb" block_device_map[ephemeral_device_name] = ephemeral log.debug("registering image - %s", name) ami_id = conn.register_image( name=name, description=desc, architecture=ec2_arch, root_device_name=rootfs_device_name, block_device_map=block_device_map, virtualization_type="hvm", ) log.info("registered image - %s %s %s", ami_id, name, region) return ami_id, name
def register(snapshot_id, region, size=None, arch=None, name=None, desc=None): conn = utils.connect(region) log.debug('getting snapshot - %s', snapshot_id) snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0] size = size if size else snapshot.volume_size name = name if name else snapshot.description desc = desc if desc else utils.parse_imagename(name)['url'] arch = arch if arch else utils.parse_imagename(name)['architecture'] kernel_id = utils.get_kernel(region, arch) arch_ec2 = "x86_64" if arch == "amd64" else arch log.debug('creating block_device_map') rootfs = BlockDeviceType() rootfs.delete_on_termination = True rootfs.size = size rootfs.snapshot_id = snapshot_id ephemeral = BlockDeviceType() ephemeral.ephemeral_name = 'ephemeral0' block_device_map = BlockDeviceMapping() block_device_map['/dev/sda1'] = rootfs block_device_map['/dev/sda2'] = ephemeral log.debug('registering image - %s', name) ami_id = conn.register_image( name=name, description=desc, architecture=arch_ec2, kernel_id=kernel_id, root_device_name="/dev/sda1", block_device_map=block_device_map) log.info('registered image - %s %s %s', ami_id, name, region) return ami_id
def parse_block_device_args(self, block_device_maps_args): block_device_map = BlockDeviceMapping() for block_device_map_arg in block_device_maps_args: parts = block_device_map_arg.split('=') if len(parts) > 1: device_name = parts[0] block_dev_type = BlockDeviceType() value_parts = parts[1].split(':') if value_parts[0].startswith('snap'): block_dev_type.snapshot_id = value_parts[0] else: if value_parts[0].startswith('ephemeral'): block_dev_type.ephemeral_name = value_parts[0] if len(value_parts) > 1: try: block_dev_type.size = int(value_parts[1]) except ValueError: pass if len(value_parts) > 2: if value_parts[2] == 'true': block_dev_type.delete_on_termination = True block_device_map[device_name] = block_dev_type return block_device_map
def get_block_device_map(bdmapping_json=None): """Parse block_device_mapping JSON and return a configured BlockDeviceMapping object Mapping JSON structure... {"/dev/sda": {"snapshot_id": "snap-23E93E09", "volume_type": null, "delete_on_termination": true, "size": 1} } """ if bdmapping_json: mapping = json.loads(bdmapping_json) if mapping: bdm = BlockDeviceMapping() for key, val in mapping.items(): device = BlockDeviceType() if val.get('virtual_name') is not None and val.get('virtual_name').startswith('ephemeral'): device.ephemeral_name = val.get('virtual_name') else: device.volume_type = 'standard' device.snapshot_id = val.get('snapshot_id') or None device.size = val.get('size') device.delete_on_termination = val.get('delete_on_termination', False) bdm[key] = device return bdm return None return None
def register(snapshot_id, region, size=None, arch=None, name=None, desc=None): conn = utils.connect(region) log.debug('getting snapshot - %s', snapshot_id) snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0] size = size if size else snapshot.volume_size name = name if name else snapshot.description desc = desc if desc else utils.parse_imagename(name)['url'] arch = arch if arch else utils.parse_imagename(name)['architecture'] kernel_id = utils.get_kernel(region, arch) arch_ec2 = "x86_64" if arch == "amd64" else arch log.debug('creating block_device_map') rootfs = BlockDeviceType() rootfs.delete_on_termination = True rootfs.size = size rootfs.snapshot_id = snapshot_id ephemeral = BlockDeviceType() ephemeral.ephemeral_name = 'ephemeral0' block_device_map = BlockDeviceMapping() block_device_map['/dev/sda1'] = rootfs block_device_map['/dev/sda2'] = ephemeral log.debug('registering image - %s', name) ami_id = conn.register_image(name=name, description=desc, architecture=arch_ec2, kernel_id=kernel_id, root_device_name="/dev/sda1", block_device_map=block_device_map) log.info('registered image - %s %s %s', ami_id, name, region) return ami_id
def create_instance(name, config, region, secrets, key_name, instance_data, deploypass, loaned_to, loan_bug): """Creates an AMI instance with the given name and config. The config must specify things like ami id.""" conn = get_connection( region, aws_access_key_id=secrets['aws_access_key_id'], aws_secret_access_key=secrets['aws_secret_access_key']) vpc = get_vpc(connection=conn, aws_access_key_id=secrets['aws_access_key_id'], aws_secret_access_key=secrets['aws_secret_access_key']) # Make sure we don't request the same things twice token = str(uuid.uuid4())[:16] instance_data = instance_data.copy() instance_data['name'] = name instance_data['hostname'] = '{name}.{domain}'.format( name=name, domain=config['domain']) ami = conn.get_all_images(image_ids=[config["ami"]])[0] bdm = None if 'device_map' in config: bdm = BlockDeviceMapping() for device, device_info in config['device_map'].items(): bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] # Overwrite root device size for HVM instances, since they cannot # be resized online if ami.virtualization_type == "hvm" and \ ami.root_device_name == device: bd.size = ami.block_device_mapping[ami.root_device_name].size if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] bdm[device] = bd ip_address = get_ip(instance_data['hostname']) subnet_id = None if ip_address: s_id = get_subnet_id(vpc, ip_address) if s_id in config['subnet_ids']: if ip_available(conn, ip_address): subnet_id = s_id else: log.warning("%s already assigned" % ip_address) if not ip_address or not subnet_id: ip_address = None subnet_id = choice(config.get('subnet_ids')) interface = NetworkInterfaceSpecification( subnet_id=subnet_id, private_ip_address=ip_address, delete_on_termination=True, groups=config.get('security_group_ids', []), associate_public_ip_address=config.get("use_public_ip")) interfaces = NetworkInterfaceCollection(interface) while True: try: reservation = conn.run_instances( image_id=config['ami'], key_name=key_name, instance_type=config['instance_type'], block_device_map=bdm, client_token=token, disable_api_termination=bool( config.get('disable_api_termination')), network_interfaces=interfaces, instance_profile_name=config.get("instance_profile_name"), ) break except boto.exception.BotoServerError: log.exception("Cannot start an instance") time.sleep(10) instance = reservation.instances[0] log.info("instance %s created, waiting to come up", instance) # Wait for the instance to come up while True: try: instance.update() if instance.state == 'running': break except Exception: log.warn("waiting for instance to come up, retrying in 10 sec...") time.sleep(10) instance.add_tag('Name', name) instance.add_tag('FQDN', instance_data['hostname']) instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime())) instance.add_tag('moz-type', config['type']) if loaned_to: instance.add_tag("moz-loaned-to", loaned_to) if loan_bug: instance.add_tag("moz-bug", loan_bug) log.info("assimilating %s", instance) instance.add_tag('moz-state', 'pending') while True: try: assimilate(instance.private_ip_address, config, instance_data, deploypass) break except: log.warn("problem assimilating %s (%s), retrying in 10 sec ...", instance_data['hostname'], instance.id) time.sleep(10) instance.add_tag('moz-state', 'ready')
def vm(install_list): region = install_list["region"] subnet_id = install_list["subnet_id"] ami_id = install_list["ami_id"] snap_id = install_list["snap_id"] volume_capacity = install_list["volume_capacity"] key_name = install_list["key_name"] instance_type = install_list["instance_type"] sg_id = install_list["sg_id"] user_data = install_list["user_data"] usage = install_list["usage"] block_device_map = BlockDeviceMapping() block_dev_type = BlockDeviceType() block_dev_type.snapshot_id = snap_id block_dev_type.delete_on_termination = False block_dev_type.size = volume_capacity block_device_map['/dev/sdb'] = block_dev_type if region not in EXTRA_INFO: aws_access_key_id = EXTRA_INFO["common"]["aws_access_key_id"] aws_secret_access_key = EXTRA_INFO["common"]["aws_secret_access_key"] name_key = EXTRA_INFO["common"]["name_key"] else: aws_access_key_id = EXTRA_INFO[region]["aws_access_key_id"] aws_secret_access_key = EXTRA_INFO[region]["aws_secret_access_key"] name_key = EXTRA_INFO[region]["name_key"] ret = asset_hostname.get(region, usage) hostname = ret["hostname"].split(".")[0] user_data = user_data.replace("region=","region=%s" % region) user_data = user_data.replace("hostname=","hostname=%s" % hostname) user_data = user_data.replace("dns_vip=","dns_vip=%s" % \ DNS_INFO[region]["dns_vip"]) user_data = user_data.replace("ns_servers=","ns_servers='%s'" % \ " ".join(DNS_INFO[region]["ns_servers"]) ) network_interface = NetworkInterfaceSpecification(subnet_id=subnet_id, \ groups=[sg_id]) network_interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(\ network_interface) conn = boto.ec2.connect_to_region(region, \ aws_access_key_id=aws_access_key_id, \ aws_secret_access_key=aws_secret_access_key) reservation = conn.run_instances(ami_id, key_name=key_name, network_interfaces=network_interfaces, instance_type=instance_type, block_device_map=block_device_map, # security_groups=[security_groups], min_count=1, max_count=1, user_data=user_data, ) instance = reservation.instances[0] time_init = 0 time_total = 300 time_interval = 5 while time_init < time_total: status = instance.update() if status == 'running': instance.add_tag(u"名称", hostname) instance.add_tag("Name", hostname) break else: time.sleep(time_interval) time_init += time_interval install_list["instance_id"] = str(instance).split(":")[-1] install_list["placement"] = instance.placement install_list["status"] = instance.update() install_list["hostname"] = hostname return install_list
def do_request_spot_instance(region, secrets, moz_instance_type, price, ami, instance_config, cached_cert_dir, instance_type, availability_zone, dryrun): conn = aws_connect_to_region(region, secrets) interface = get_available_interface(conn=conn, moz_instance_type=moz_instance_type, availability_zone=availability_zone) if not interface: raise RuntimeError("No free network interfaces left in %s" % region) # TODO: check DNS fqdn = interface.tags.get("FQDN") if not fqdn: raise RuntimeError("Skipping %s without FQDN" % interface) log.debug("Spot request for %s (%s)", fqdn, price) if dryrun: log.info("Dry run. skipping") return spec = NetworkInterfaceSpecification(network_interface_id=interface.id) nc = NetworkInterfaceCollection(spec) ip = interface.private_ip_address certs = get_puppet_certs(ip, secrets, cached_cert_dir) user_data = """ FQDN="%(fqdn)s" cd /var/lib/puppet/ssl || exit 1 %(certs)s cd - """ % dict(fqdn=fqdn, certs=certs) if instance_config[region].get("lvm"): user_data += """ mkdir -p /etc/lvm-init/ cat <<EOF > /etc/lvm-init/lvm-init.json %s EOF /sbin/lvm-init """ % json.dumps(instance_config[region]) bdm = BlockDeviceMapping() for device, device_info in instance_config[region]['device_map'].items(): bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] if ami.root_device_name == device: ami_size = ami.block_device_mapping[device].size if ami.virtualization_type == "hvm": # Overwrite root device size for HVM instances, since they # cannot be resized online bd.size = ami_size elif device_info.get('size'): # make sure that size is enough for this AMI assert ami_size <= device_info['size'], \ "Instance root device size cannot be smaller than AMI " \ "root device" if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] bdm[device] = bd sir = conn.request_spot_instances( price=str(price), image_id=ami.id, count=1, instance_type=instance_type, key_name=instance_config[region]["ssh_key"], user_data=user_data, block_device_map=bdm, network_interfaces=nc, instance_profile_name=instance_config[region].get( "instance_profile_name"), ) sir[0].add_tag("moz-type", moz_instance_type)
def handleRunInstances(self, action, clc, user_data_file, callback): image_id = self.get_argument('ImageId') min = self.get_argument('MinCount', '1') max = self.get_argument('MaxCount', '1') key = self.get_argument('KeyName', None) groups = self.get_argument_list('SecurityGroup') sec_group_ids = self.get_argument_list('SecurityGroupId') if user_data_file: user_data = user_data_file else: user_data = self.get_argument('UserData', "") user_data = base64.b64decode(user_data) addr_type = self.get_argument('AddressingType', None) vm_type = self.get_argument('InstanceType', None) placement = self.get_argument('Placement.AvailabilityZone', None) placement_group = self.get_argument('Placement.GroupName', None) tenancy = self.get_argument('Placement.Tenancy', None) kernel = self.get_argument('KernelId', None) ramdisk = self.get_argument('RamdiskId', None) monitoring=False if self.get_argument('Monitoring.Enabled', '') == 'true': monitoring=True subnet = self.get_argument('SubnetId', None); private_ip = self.get_argument('PrivateIpAddress', None); # get block device mappings bdm = BlockDeviceMapping() mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None) idx = 1 while mapping: pre = 'BlockDeviceMapping.%d' % idx dev_name = mapping block_dev_type = BlockDeviceType() block_dev_type.ephemeral_name = self.get_argument('%s.VirtualName' % pre, None) if not(block_dev_type.ephemeral_name): block_dev_type.no_device = \ (self.get_argument('%s.NoDevice' % pre, '') == 'true') block_dev_type.snapshot_id = \ self.get_argument('%s.Ebs.SnapshotId' % pre, None) block_dev_type.size = \ self.get_argument('%s.Ebs.VolumeSize' % pre, None) block_dev_type.delete_on_termination = \ (self.get_argument('%s.Ebs.DeleteOnTermination' % pre, '') == 'true') bdm[dev_name] = block_dev_type idx += 1 mapping = self.get_argument('BlockDeviceMapping.%d.DeviceName' % idx, None) if len(bdm) == 0: bdm = None api_termination=False if self.get_argument('DisableApiTermination', '') == 'true': api_termination=True instance_shutdown=False if self.get_argument('InstanceInitiatedShutdownBehavior', '') == 'true': instance_shutdown=True token = self.get_argument('ClientToken', None); addition_info = self.get_argument('AdditionInfo', None); instance_profile_name = self.get_argument('IamInstanceProfile.Name', None); instance_profile_arn = self.get_argument('IamInstanceProfile.Arn', None); return clc.run_instances(image_id, min_count=min, max_count=max, key_name=key, security_groups=groups, user_data=user_data, addressing_type=addr_type, instance_type=vm_type, placement=placement, kernel_id=kernel, ramdisk_id=ramdisk, monitoring_enabled=monitoring, subnet_id=subnet, block_device_map=bdm, disable_api_termination=api_termination, instance_initiated_shutdown_behavior=instance_shutdown, private_ip_address=private_ip, placement_group=placement_group, client_token=token, security_group_ids=sec_group_ids, additional_info=addition_info, instance_profile_name=instance_profile_name, instance_profile_arn=instance_profile_arn, tenancy=tenancy, callback=callback)
def handleImages(self, action, clc): if action == 'DescribeImages': owner = self.get_argument('Owner', None) if not owner: owners = None else: owners = [owner] return clc.get_all_images(owners) elif action == 'DescribeImageAttribute': imageid = self.get_argument('ImageId') attribute = self.get_argument('Attribute') return clc.get_image_attribute(imageid, attribute) elif action == 'ModifyImageAttribute': imageid = self.get_argument('ImageId') attribute = self.get_argument('Attribute') operation = self.get_argument('OperationType') users = self.get_argument_list('UserId') groups = self.get_argument_list('UserGroup') return clc.modify_image_attribute(imageid, attribute, operation, users, groups) elif action == 'ResetImageAttribute': imageid = self.get_argument('ImageId') attribute = self.get_argument('Attribute') return clc.reset_image_attribute(imageid, attribute) elif action == 'RegisterImage': image_location = self.get_argument('ImageLocation', None) name = self.get_argument('Name') description = self.get_argument('Description', None) description = base64.b64decode(description) architecture = self.get_argument('Architecture', None) kernel_id = self.get_argument('KernelId', None) ramdisk_id = self.get_argument('RamdiskId', None) root_dev_name = self.get_argument('RootDeviceName', None) snapshot_id = self.get_argument('SnapshotId', None) # get block device mappings bdm = BlockDeviceMapping() mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None) idx = 1 while mapping: pre = 'BlockDeviceMapping.%d' % idx dev_name = mapping block_dev_type = BlockDeviceType() block_dev_type.ephemeral_name = self.get_argument( '%s.VirtualName' % pre, None) if not (block_dev_type.ephemeral_name): block_dev_type.no_device = \ (self.get_argument('%s.NoDevice' % pre, '') == 'true') block_dev_type.snapshot_id = \ self.get_argument('%s.Ebs.SnapshotId' % pre, None) block_dev_type.size = \ self.get_argument('%s.Ebs.VolumeSize' % pre, None) block_dev_type.delete_on_termination = \ (self.get_argument('%s.DeleteOnTermination' % pre, '') == 'true') bdm[dev_name] = block_dev_type idx += 1 mapping = self.get_argument( 'BlockDeviceMapping.%d.DeviceName' % idx, None) if snapshot_id: rootbdm = BlockDeviceType() rootbdm.snapshot_id = snapshot_id bdm['/dev/sda1'] = rootbdm if len(bdm) == 0: bdm = None return clc.register_image(name, image_location, description, architecture, kernel_id, ramdisk_id, root_dev_name, bdm)
def handleImages(self, action, clc, callback=None): if action == "DescribeImages": owner = self.get_argument("Owner", None) if not owner: owners = None else: owners = [owner] filters = self.get_filter_args() return clc.get_all_images(owners, filters, callback) elif action == "DescribeImageAttribute": imageid = self.get_argument("ImageId") attribute = self.get_argument("Attribute") return clc.get_image_attribute(imageid, attribute, callback) elif action == "ModifyImageAttribute": imageid = self.get_argument("ImageId") attribute = self.get_argument("Attribute") operation = self.get_argument("OperationType") users = self.get_argument_list("UserId") groups = self.get_argument_list("UserGroup") return clc.modify_image_attribute(imageid, attribute, operation, users, groups, callback) elif action == "ResetImageAttribute": imageid = self.get_argument("ImageId") attribute = self.get_argument("Attribute") return clc.reset_image_attribute(imageid, attribute, callback) elif action == "DeregisterImage": image_id = self.get_argument("ImageId") return clc.deregister_image(image_id, callback) elif action == "RegisterImage": image_location = self.get_argument("ImageLocation", None) name = self.get_argument("Name") description = self.get_argument("Description", None) if description != None: description = base64.b64decode(description) architecture = self.get_argument("Architecture", None) kernel_id = self.get_argument("KernelId", None) ramdisk_id = self.get_argument("RamdiskId", None) root_dev_name = self.get_argument("RootDeviceName", None) snapshot_id = self.get_argument("SnapshotId", None) # get block device mappings bdm = BlockDeviceMapping() mapping = self.get_argument("BlockDeviceMapping.1.DeviceName", None) idx = 1 while mapping: pre = "BlockDeviceMapping.%d" % idx dev_name = mapping block_dev_type = BlockDeviceType() block_dev_type.ephemeral_name = self.get_argument("%s.VirtualName" % pre, None) if not (block_dev_type.ephemeral_name): block_dev_type.no_device = self.get_argument("%s.NoDevice" % pre, "") == "true" block_dev_type.snapshot_id = self.get_argument("%s.Ebs.SnapshotId" % pre, None) block_dev_type.size = self.get_argument("%s.Ebs.VolumeSize" % pre, None) block_dev_type.delete_on_termination = ( self.get_argument("%s.Ebs.DeleteOnTermination" % pre, "") == "true" ) bdm[dev_name] = block_dev_type idx += 1 mapping = self.get_argument("BlockDeviceMapping.%d.DeviceName" % idx, None) if snapshot_id: rootbdm = BlockDeviceType() rootbdm.snapshot_id = snapshot_id bdm["/dev/sda1"] = rootbdm if len(bdm) == 0: bdm = None return clc.register_image( name, image_location, description, architecture, kernel_id, ramdisk_id, root_dev_name, bdm, callback )
def create_instance(name, config, region, key_name, ssh_key, instance_data, deploypass, loaned_to, loan_bug, create_ami, ignore_subnet_check, max_attempts): """Creates an AMI instance with the given name and config. The config must specify things like ami id.""" conn = get_aws_connection(region) # Make sure we don't request the same things twice token = str(uuid.uuid4())[:16] instance_data = instance_data.copy() instance_data['name'] = name instance_data['domain'] = config['domain'] instance_data['hostname'] = '{name}.{domain}'.format( name=name, domain=config['domain']) ami = conn.get_all_images(image_ids=[config["ami"]])[0] bdm = None if 'device_map' in config: bdm = BlockDeviceMapping() for device, device_info in config['device_map'].items(): bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] # Overwrite root device size for HVM instances, since they cannot # be resized online if ami.virtualization_type == "hvm" and \ ami.root_device_name == device: bd.size = ami.block_device_mapping[ami.root_device_name].size if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] if device_info.get("volume_type"): bd.volume_type = device_info["volume_type"] if device_info["volume_type"] == "io1" \ and device_info.get("iops"): bd.iops = device_info["iops"] bdm[device] = bd interfaces = make_instance_interfaces( region, instance_data['hostname'], ignore_subnet_check, config.get('subnet_ids'), config.get('security_group_ids', []), config.get("use_public_ip")) keep_going, attempt = True, 1 while keep_going: try: puppet_master = pick_puppet_master(instance_data.get('puppet_masters')) user_data = user_data_from_template(config['type'], { "puppet_server": puppet_master, "fqdn": instance_data['hostname'], "hostname": instance_data['name'], "domain": instance_data['domain'], "dns_search_domain": config.get('dns_search_domain'), "password": deploypass, "moz_instance_type": config['type'], "region_dns_atom": get_region_dns_atom(region)}) reservation = conn.run_instances( image_id=config['ami'], key_name=key_name, instance_type=config['instance_type'], block_device_map=bdm, client_token=token, disable_api_termination=config.get('disable_api_termination'), user_data=user_data, instance_profile_name=config.get('instance_profile_name'), network_interfaces=interfaces, ) break except boto.exception.BotoServerError: log.exception("Cannot start an instance") time.sleep(10) if max_attempts: attempt += 1 keep_going = max_attempts >= attempt instance = reservation.instances[0] log.info("instance %s created, waiting to come up", instance) # Wait for the instance to come up wait_for_status(instance, "state", "running", "update") instance.add_tag('Name', name) instance.add_tag('FQDN', instance_data['hostname']) instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime())) instance.add_tag('moz-type', config['type']) if loaned_to: instance.add_tag("moz-loaned-to", loaned_to) if loan_bug: instance.add_tag("moz-bug", loan_bug) log.info("assimilating %s", instance) instance.add_tag('moz-state', 'pending') keep_going, attempt = True, 1 while keep_going: try: # Don't reboot if need to create ami reboot = not create_ami assimilate_instance(instance=instance, config=config, ssh_key=ssh_key, instance_data=instance_data, deploypass=deploypass, reboot=reboot) break except NetworkError as e: # it takes a while for the machine to start/reboot so the # NetworkError exception is quite common, just log the error, # without the full stack trace log.warn("cannot connect; instance may still be starting %s (%s, %s) - %s," "retrying in %d sec ...", instance_data['hostname'], instance.id, instance.private_ip_address, e, FAILURE_TIMEOUT) time.sleep(FAILURE_TIMEOUT) except: # any other exception log.warn("problem assimilating %s (%s, %s), retrying in " "%d sec ...", instance_data['hostname'], instance.id, instance.private_ip_address, FAILURE_TIMEOUT, exc_info=True) time.sleep(FAILURE_TIMEOUT) if max_attempts: attempt += 1 keep_going = max_attempts >= attempt instance.add_tag('moz-state', 'ready') if create_ami: ami_name = "spot-%s-%s" % ( config['type'], time.strftime("%Y-%m-%d-%H-%M", time.gmtime())) log.info("Generating AMI %s", ami_name) ami_cleanup(mount_point="/", distro=config["distro"]) root_bd = instance.block_device_mapping[instance.root_device_name] volume = instance.connection.get_all_volumes( volume_ids=[root_bd.volume_id])[0] # The instance has to be stopped to flush EBS caches instance.stop() wait_for_status(instance, 'state', 'stopped', 'update') ami = volume_to_ami(volume=volume, ami_name=ami_name, arch=instance.architecture, virtualization_type=instance.virtualization_type, kernel_id=instance.kernel, root_device_name=instance.root_device_name, tags=config["tags"]) log.info("AMI %s (%s) is ready", ami_name, ami.id) log.warn("Terminating %s", instance) instance.terminate()
def launch_instance(self): """ Launch an instance for the profiler to run on. """ # Read in the cloudinit file cfgfile = open('deploy.cfg') user_data = cfgfile.read() # Create a block mapping to get instance storage for the machine mapping = BlockDeviceMapping() sda1 = BlockDeviceType() eph0 = BlockDeviceType() eph1 = BlockDeviceType() eph2 = BlockDeviceType() eph3 = BlockDeviceType() sda1.size = 10 eph0.ephemeral_name = 'ephemeral0' eph1.ephemeral_name = 'ephemeral1' eph2.ephemeral_name = 'ephemeral2' eph3.ephemeral_name = 'ephemeral3' mapping['/dev/sda1'] = sda1 mapping['/dev/sdb'] = eph0 mapping['/dev/sdc'] = eph1 mapping['/dev/sdd'] = eph2 mapping['/dev/sde'] = eph3 conn = boto.connect_ec2(self.access_key, self.secret_key) inst_req = None if self.ondemand: inst_req= conn.run_instances( min_count=1, max_count=1, key_name=self.key_pair, image_id=self.ami, security_group_ids=[self.security_group], user_data=user_data, instance_type=self.instance_type, block_device_map=mapping) else: inst_req = conn.request_spot_instances( price=self.bid, image_id=self.ami, subnet_id=self.subnet, count=1, key_name=self.key_pair, security_group_ids=[self.security_group], instance_type=self.type, user_data=user_data, block_device_map=mapping) # Reorder the list of id's my_req_ids = [req.id for req in inst_req] # Tag the request for req in my_req_ids: self.tag_requests(req, 'profiler', conn) self.address = "" # Wait for the resource to become active while self.address == "": while True: # Only check every 10 seconds time.sleep(10) # Get all spot requests reqs = conn.get_all_spot_instance_requests() id_to_req = {} for r in reqs: id_to_req[r.id] = r active = 0 instance_ids = [] for i in my_req_ids: try: # Once it is active add it to the list if id_to_req[i].state == "active": active += 1 instance_ids.append(id_to_req[i].instance_id) except Exception, e: self.logger.debug("Hmm, error, skipping for now.") if active == 1: # Once it is granted we can leave this loop self.logger.debug("All %d slaves granted" % 1) reservations = conn.get_all_instances(instance_ids) slave_nodes = [] for r in reservations: slave_nodes += r.instances break self.address = slave_nodes[0].private_dns_name
def test_create_launch_configuration_with_block_device_mappings(): block_device_mapping = BlockDeviceMapping() ephemeral_drive = BlockDeviceType() ephemeral_drive.ephemeral_name = "ephemeral0" block_device_mapping["/dev/xvdb"] = ephemeral_drive snapshot_drive = BlockDeviceType() snapshot_drive.snapshot_id = "snap-1234abcd" snapshot_drive.volume_type = "standard" block_device_mapping["/dev/xvdp"] = snapshot_drive ebs_drive = BlockDeviceType() ebs_drive.volume_type = "io1" ebs_drive.size = 100 ebs_drive.iops = 1000 ebs_drive.delete_on_termination = False block_device_mapping["/dev/xvdh"] = ebs_drive conn = boto.connect_autoscale(use_block_device_types=True) config = LaunchConfiguration( name="tester", image_id="ami-abcd1234", instance_type="m1.small", key_name="the_keys", security_groups=["default", "default2"], user_data=b"This is some user_data", instance_monitoring=True, instance_profile_name="arn:aws:iam::{}:instance-profile/testing".format( ACCOUNT_ID ), spot_price=0.1, block_device_mappings=[block_device_mapping], ) conn.create_launch_configuration(config) launch_config = conn.get_all_launch_configurations()[0] launch_config.name.should.equal("tester") launch_config.image_id.should.equal("ami-abcd1234") launch_config.instance_type.should.equal("m1.small") launch_config.key_name.should.equal("the_keys") set(launch_config.security_groups).should.equal(set(["default", "default2"])) launch_config.user_data.should.equal(b"This is some user_data") launch_config.instance_monitoring.enabled.should.equal("true") launch_config.instance_profile_name.should.equal( "arn:aws:iam::{}:instance-profile/testing".format(ACCOUNT_ID) ) launch_config.spot_price.should.equal(0.1) len(launch_config.block_device_mappings).should.equal(3) returned_mapping = launch_config.block_device_mappings set(returned_mapping.keys()).should.equal( set(["/dev/xvdb", "/dev/xvdp", "/dev/xvdh"]) ) returned_mapping["/dev/xvdh"].iops.should.equal(1000) returned_mapping["/dev/xvdh"].size.should.equal(100) returned_mapping["/dev/xvdh"].volume_type.should.equal("io1") returned_mapping["/dev/xvdh"].delete_on_termination.should.be.false returned_mapping["/dev/xvdp"].snapshot_id.should.equal("snap-1234abcd") returned_mapping["/dev/xvdp"].volume_type.should.equal("standard") returned_mapping["/dev/xvdb"].ephemeral_name.should.equal("ephemeral0")
def main(): n_arg = len(sys.argv) if 1 < n_arg: if sys.argv[1] == 'unixbench': u_data_model = 'unixbench/unixbench_ec2_userscript_model.dat' elif sys.argv[1] == 'x264': u_data_model = 'x264/x264_userscript_model.dat' else: print "usage: %s [unixbench|x264]" % sys.argv[0] sys.exit(0) # Block device storage size if n_arg == 3: dev_sda1 = BlockDeviceType() dev_sda1.size = int(sys.argv[2]) bdm = BlockDeviceMapping() bdm['/dev/sda1'] = dev_sda1 else: bdm = None # Lists of instance types to be benchmarked and already completed instances = [] completed = [] try: instances_dict = json.load(open("web/data/instances.json", "r")) except IOError: print "*** web/data/instances.json not found! Try ./update_instances.py ***" sys.exit(1) for k, v in instances_dict.iteritems(): if v['cloud'] == 'EC2': if u_data_model == 'unixbench/unixbench_userscript_model.dat': try: instance_logs = Table(k) instance_logs.describe() completed.append(k) except JSONResponseError: instances.append(k) elif u_data_model == 'x264/x264_userscript_model.dat': instances.append(k) else: print 'Nothing to do' sys.exit(0) # Start all the benchmark at once will most likely exceeds the quota limit per user # Better to execute the benchmark on a category to category basis conn = boto.ec2.connect_to_region(region) #instances = [] #completed = ['t1.micro_paravirtual'] num_instances = len(instances) while 0 < len(instances): for i in instances: print '%s is waiting for launch' % i for i in completed: if i in instances: instances.remove(i) for i in instances: # Generate an user-script per instance userscript = '' if u_data_model == 'unixbench/unixbench_userscript_model.dat': userscript = "#!/bin/sh\nTRIAL=%d\nINSTANCE_NAME=%s\n"%(trial,i) + open(u_data_model,'r').read() elif u_data_model == 'x264/x264_userscript_model.dat': userscript = "#!/bin/sh\nTRIAL=%d\necho %s > /var/local/instance_name\n"%(trial,i) + open(u_data_model,'r').read() u_data = base64.b64encode(userscript) res = start_benchmark_instance(conn, i, u_data, bdm) if res is not None and not res in completed: completed.append(res) sleep(5) if len(completed) == num_instances: break else: print '*** Cooling down...' # 30 mins interval sleep(60*30)
def do_create(self, ntry, params): """ Create stage of testing @param ntry: number of try @type ntry: int @param params: list of testing parameters @type params: list """ result = None logging.debug(self.getName() + ': trying to create instance ' + params['iname'] + ', ntry ' + str(ntry)) ntry += 1 try: bmap = BlockDeviceMapping() for device in params['bmap']: if not 'name' in device.keys(): logging.debug(self.getName() + ': bad device ' + str(device)) continue d = BlockDeviceType() if 'size' in device.keys(): d.size = device['size'] if 'delete_on_termination' in device.keys(): d.delete_on_termination = device['delete_on_termination'] if 'ephemeral_name' in device.keys(): d.ephemeral_name = device['ephemeral_name'] bmap[device['name']] = d reg = boto.ec2.get_region(params['region'], aws_access_key_id=ec2_key, aws_secret_access_key=ec2_secret_key) connection = reg.connect(aws_access_key_id=ec2_key, aws_secret_access_key=ec2_secret_key) (ssh_key_name, ssh_key) = yamlconfig['ssh'][params['region']] # all handled params to be put in here boto_params = ['ami', 'subnet_id'] for param in boto_params: params.setdefault(param) reservation = connection.run_instances( params['ami'], instance_type=params['ec2name'], key_name=ssh_key_name, block_device_map=bmap, subnet_id=params['subnet_id'], user_data=params['userdata'] ) myinstance = reservation.instances[0] count = 0 # Sometimes EC2 failes to return something meaningful without small timeout between run_instances() and update() time.sleep(10) while myinstance.update() == 'pending' and count < maxwait / 5: # Waiting out instance to appear logging.debug(params['iname'] + '... waiting...' + str(count)) time.sleep(5) count += 1 connection.close() instance_state = myinstance.update() if instance_state == 'running': # Instance appeared - scheduling 'setup' stage myinstance.add_tag('Name', params['name']) result = myinstance.__dict__ logging.info(self.getName() + ': created instance ' + params['iname'] + ', ' + result['id'] + ':' + result['public_dns_name']) # packing creation results into params params['id'] = result['id'] params['instance'] = result.copy() mainq.put((0, 'setup', params)) return elif instance_state == 'pending': # maxwait seconds is enough to create an instance. If not -- EC2 failed. logging.error('Error during instance creation: timeout in pending state') result = myinstance.__dict__ if 'id' in result.keys(): # terminate stucked instance params['id'] = result['id'] params['instance'] = result.copy() mainq.put((0, 'terminate', params.copy())) else: # error occured logging.error('Error during instance creation: ' + instance_state) except boto.exception.EC2ResponseError, e: # Boto errors should be handled according to their error Message - there are some well-known ones logging.debug(self.getName() + ': got boto error during instance creation: %s' % e) if str(e).find('<Code>InstanceLimitExceeded</Code>') != -1: # InstanceLimit is temporary problem logging.debug(self.getName() + ': got InstanceLimitExceeded - not increasing ntry') ntry -= 1 elif str(e).find('<Code>InvalidParameterValue</Code>') != -1: # InvalidParameterValue is really bad logging.error(self.getName() + ': got boto error during instance creation: %s' % e) # Try to speed up whole testing process, it's hardly possible to recover from this error ntry += args.maxtries // 2 + 1 elif str(e).find('<Code>Unsupported</Code>') != -1: # Unsupported logging.error(self.getName() + ': got Unsupported - most likely the permanent error: %s' % e) ntry += args.maxtries // 2 + 1 else: logging.debug(self.getName() + ':' + traceback.format_exc())
def launch_instance(self, conn): """ Launch a cluster of the given name, by setting up its security groups, and then starting new instances in them. Returns a tuple of EC2 reservation objects for the master, slave Fails if there already instances running in the cluster's groups. """ user_data = self.customise_cloudinit() # Request a resource mapping = BlockDeviceMapping() sda1 = BlockDeviceType() eph0 = BlockDeviceType() eph1 = BlockDeviceType() eph2 = BlockDeviceType() eph3 = BlockDeviceType() sda1.size = 10 eph0.ephemeral_name = 'ephemeral0' eph1.ephemeral_name = 'ephemeral1' eph2.ephemeral_name = 'ephemeral2' eph3.ephemeral_name = 'ephemeral3' mapping['/dev/sda1'] = sda1 mapping['/dev/sdb'] = eph0 mapping['/dev/sdc'] = eph1 mapping['/dev/sdd'] = eph2 mapping['/dev/sde'] = eph3 inst_req = conn.request_spot_instances( price=self.price, image_id=self.ami, subnet_id=self.subnet, count=1, key_name=self.key_pair, security_group_ids=[self.security_group], instance_type=self.type, user_data=user_data, block_device_map=mapping) # Reorder the list of id's my_req_ids = [req.id for req in inst_req] # Tag the request for req in my_req_ids: self.tag_requests(req, 'profiler', conn) self.address = "" # Wait for the resource to become active while self.address == "": while True: # Only check every 10 seconds time.sleep(10) # Get all spot requests reqs = conn.get_all_spot_instance_requests() id_to_req = {} for r in reqs: id_to_req[r.id] = r active = 0 instance_ids = [] for i in my_req_ids: try: # Once it is active add it to the list if id_to_req[i].state == "active": active += 1 instance_ids.append(id_to_req[i].instance_id) except Exception, e: self.logger.debug("Hmm, error, skipping for now.") if active == 1: # Once it is granted we can leave this loop self.logger.debug("All %d slaves granted" % 1) reservations = conn.get_all_instances(instance_ids) slave_nodes = [] for r in reservations: slave_nodes += r.instances break self.address = slave_nodes[0].private_dns_name
def launchInstance(region, rootsize, role, env, ami, key_name, instance_type, placement, sec_group, iamrole, ebs, product, project, name, chefversion): '''Launch a single instance of the provided ami''' conn = connectEC2(region) mapping = BlockDeviceMapping() root = BlockDeviceType() root.size = rootsize mapping['/dev/xvda'] = root user_data = genUserData(role, env, chefversion) sec_group_id = getSecurityGroupID(conn, sec_group) subnet_id = getSubnetID(placement, env) print Fore.YELLOW + 'Instance will be launched with following parameters.\n' print Fore.GREEN + 'Region :\t\t' + region print Fore.GREEN + 'Chef Role :\t\t' + role print Fore.GREEN + 'Chef Env :\t\t' + env print Fore.GREEN + 'AMI ID :\t\t' + ami print Fore.GREEN + 'Keypair :\t\t' + key_name print Fore.GREEN + 'Instance Type :\t\t' + instance_type print Fore.GREEN + 'Placement :\t\t' + placement print Fore.GREEN + 'Security Group :\t' + sec_group + ' (' + sec_group_id + ')' print Fore.GREEN + 'Subnet ID :\t\t' + subnet_id print Fore.GREEN + 'IAM Role :\t\t' + iamrole if ebs: print Fore.GREEN + 'EBS Volume Size: ' + str(ebs) + ' GB' print Fore.YELLOW + '\nFollowing tags will be attached to the instance.\n' if name: print Fore.GREEN + 'Name :\t\t\t' + name.lower() print Fore.GREEN + 'Product :\t\t' + product.lower() print Fore.GREEN + 'Project :\t\t' + project.lower() print Fore.GREEN + 'Environment :\t\t' + env.lower() print Fore.YELLOW + '\nTrying to launch an EC2 instance ...' reservation = conn.run_instances( ami, key_name=key_name, user_data=user_data, instance_type=instance_type, placement=placement, subnet_id=subnet_id, block_device_map=mapping, instance_initiated_shutdown_behavior='stop', security_group_ids=['sg-********', sec_group_id], instance_profile_name=iamrole, dry_run=False) print(Fore.YELLOW + '\nWaiting for instance to start ...') instance = reservation.instances[0] status = instance.update() while status == 'pending': time.sleep(5) status = instance.update() if status == 'running': print(Fore.GREEN + 'New instance "' + instance.id + '" accessible at ' + instance.private_ip_address) print(Fore.YELLOW + 'Adding tags.') if name: instance.add_tag('Name', name.lower()) instance.add_tag('Product', product.lower()) instance.add_tag('Project', project.lower()) instance.add_tag('Environment', env.lower()) else: print(Fore.YELLOW + 'Instance status: ' + status) return # If we got through the launching successfully, go ahead and create and attach a volume. if ebs: attachEBS(conn, instance, ebs) print Fore.GREEN + '\nInstance ' + instance.id + ' launched successfuly.'
def launch_spot_request(conn, request, tenant, job): try: logger.debug("%s = %s. tenants vpc = %s" % (request.zone, tenant.subnets[request.zone], tenant.vpc)) cost_aware_req = job.cost_aware drafts_req = job.cost_aware drafts_avg = job.cost_aware mapping = BlockDeviceMapping() sda1 = BlockDeviceType() eph0 = BlockDeviceType() eph1 = BlockDeviceType() eph2 = BlockDeviceType() eph3 = BlockDeviceType() sda1.size = 10 eph0.ephemeral_name = 'ephemeral0' eph1.ephemeral_name = 'ephemeral1' eph2.ephemeral_name = 'ephemeral2' eph3.ephemeral_name = 'ephemeral3' mapping['/dev/sda1'] = sda1 mapping['/dev/sdb'] = eph0 mapping['/dev/sdc'] = eph1 mapping['/dev/sdd'] = eph2 mapping['/dev/sde'] = eph3 inst_req = None inst_req = conn.request_spot_instances( price=request.bid, image_id=request.ami, subnet_id=tenant.subnets[request.zone], count=request.count, key_name=tenant.key_pair, security_group_ids=[tenant.security_group], instance_type=request.instance_type, user_data=customise_cloudinit(tenant, job), block_device_map=mapping) my_req_ids = [req.id for req in inst_req] # address = "" for req in my_req_ids: insert_launch_stats(req, request, tenant) # tag each request tag_requests(req, tenant.name, conn) ProvisionerConfig().dbconn.execute(( "insert into instance_request (tenant, instance_type, " + "price, job_runner_id, request_type, request_id, " + "subnet, cost_aware_ins, cost_aware_bid, cost_aware_subnet," + " drafts_ins, drafts_bid, drafts_subnet, selected_avg_price," " cost_aware_avg_price, drafts_avg_price, drafts_avg_ins, " + "drafts_avg_bid, drafts_avg_subnet, drafts_avg_avg_price) " + "values ('%s', '%s', %s, %s, '%s', '%s', %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" ) % (tenant.db_id, request.instance.db_id, request.price, job.id, "spot", req, tenant.subnets_db_id[request.zone], cost_aware_req.instance.db_id, cost_aware_req.bid, tenant.subnets_db_id[cost_aware_req.zone], drafts_req.instance.db_id, drafts_req.DrAFTS, tenant.subnets_db_id[drafts_req.zone], request.AvgPrice, cost_aware_req.AvgPrice, drafts_req.AvgPrice, drafts_avg.instance.db_id, drafts_avg.DrAFTS, tenant.subnets_db_id[drafts_avg.zone], drafts_avg.AvgPrice)) return my_req_ids except boto.exception.EC2ResponseError: logger.exception("There was an error communicating with EC2.")
def create_cluster(conn, args): if args.identity_file is None: print("ERROR: Must provide an identity file (-i) for ssh connections.", file=stderr) sys.exit(1) if args.key_pair is None: print("ERROR: Must provide a key pair name (-k) to use on instances.", file=stderr) sys.exit(1) # make or get the security group. security_group = get_or_make_group(conn, args.name, args.vpc_id) # set the inbound permission rules if len(security_group.rules) == 0: if __name__ == '__main__': if args.vpc_id is None: security_group.authorize(src_group=security_group) else: security_group.authorize('tcp', 22, 22, args.authorized_address) security_group.authorize('tcp', 8888, 8888, args.authorized_address) security_group.authorize('tcp', 7000, 7000, args.authorized_address) security_group.authorize('tcp', 7001, 7001, args.authorized_address) security_group.authorize('tcp', 7199, 7199, args.authorized_address) security_group.authorize('tcp', 9042, 9042, args.authorized_address) security_group.authorize('tcp', 9160, 9160, args.authorized_address) else: print("Security group already exists, skipping creation.") instances = cluster_nodes(conn, args.name) if any(instances): additional_tags = {} for i in instances: i.add_tags( dict(additional_tags, Name="{cn}-node-{iid}".format(cn=args.name, iid=i.id))) return instances else: print( "Launching {m} instances for cluster...".format(m=args.node_count)) try: image = conn.get_all_images(image_ids=args.ami)[0] block_map = BlockDeviceMapping() if args.ebs_vol_size > 0: if args.instance_type.startswith('m3.'): for i in range(get_num_disks(args.instance_type)): device = BlockDeviceType() device.ephemeral_name = "ephemeral%d" % i name = "/dev/sd" + string.ascii_letters[i + 1] block_map[name] = device else: device = EBSBlockDeviceType() device.size = args.ebs_vol_size device.volume_type = args.ebs_vol_type device.delete_on_termination = True key = "/dev/sd" + chr(ord('s') + 1) block_map[key] = device nodes = image.run(key_name=args.key_pair, security_group_ids=[security_group.id], instance_type="", placement=args.zone, min_count=args.node_count, max_count=args.node_count, block_device_map=block_map, subnet_id=None, placement_group=None, user_data=None, instance_initiated_shutdown_behavior="stop", instance_profile_name=None) print("Waiting for AWS to propagate instance metadata...") time.sleep(15) additional_tags = {} for node in nodes.instances: node.add_tags( dict(additional_tags, Name="{cn}-node-{iid}".format(cn=args.name, iid=node.id))) return nodes.instances except Exception as e: print("Caught exception: ", e) print("ERROR: Could not find AMI " + args.ami, file=stderr) sys.exit(1)
def handleRunInstances(self, action, clc, user_data_file, callback): image_id = self.get_argument("ImageId") min = self.get_argument("MinCount", "1") max = self.get_argument("MaxCount", "1") key = self.get_argument("KeyName", None) groups = self.get_argument_list("SecurityGroup") sec_group_ids = self.get_argument_list("SecurityGroupId") if user_data_file: user_data = user_data_file else: user_data = self.get_argument("UserData", "") user_data = base64.b64decode(user_data) addr_type = self.get_argument("AddressingType", None) vm_type = self.get_argument("InstanceType", None) placement = self.get_argument("Placement.AvailabilityZone", None) placement_group = self.get_argument("Placement.GroupName", None) tenancy = self.get_argument("Placement.Tenancy", None) kernel = self.get_argument("KernelId", None) ramdisk = self.get_argument("RamdiskId", None) monitoring = False if self.get_argument("Monitoring.Enabled", "") == "true": monitoring = True subnet = self.get_argument("SubnetId", None) private_ip = self.get_argument("PrivateIpAddress", None) # get block device mappings bdm = BlockDeviceMapping() mapping = self.get_argument("BlockDeviceMapping.1.DeviceName", None) idx = 1 while mapping: pre = "BlockDeviceMapping.%d" % idx dev_name = mapping block_dev_type = BlockDeviceType() block_dev_type.ephemeral_name = self.get_argument("%s.VirtualName" % pre, None) if not (block_dev_type.ephemeral_name): block_dev_type.no_device = self.get_argument("%s.NoDevice" % pre, "") == "true" block_dev_type.snapshot_id = self.get_argument("%s.Ebs.SnapshotId" % pre, None) block_dev_type.size = self.get_argument("%s.Ebs.VolumeSize" % pre, None) block_dev_type.delete_on_termination = ( self.get_argument("%s.Ebs.DeleteOnTermination" % pre, "") == "true" ) bdm[dev_name] = block_dev_type idx += 1 mapping = self.get_argument("BlockDeviceMapping.%d.DeviceName" % idx, None) if len(bdm) == 0: bdm = None api_termination = False if self.get_argument("DisableApiTermination", "") == "true": api_termination = True instance_shutdown = False if self.get_argument("InstanceInitiatedShutdownBehavior", "") == "true": instance_shutdown = True token = self.get_argument("ClientToken", None) addition_info = self.get_argument("AdditionInfo", None) instance_profile_name = self.get_argument("IamInstanceProfile.Name", None) instance_profile_arn = self.get_argument("IamInstanceProfile.Arn", None) return clc.run_instances( image_id, min_count=min, max_count=max, key_name=key, security_groups=groups, user_data=user_data, addressing_type=addr_type, instance_type=vm_type, placement=placement, kernel_id=kernel, ramdisk_id=ramdisk, monitoring_enabled=monitoring, subnet_id=subnet, block_device_map=bdm, disable_api_termination=api_termination, instance_initiated_shutdown_behavior=instance_shutdown, private_ip_address=private_ip, placement_group=placement_group, client_token=token, security_group_ids=sec_group_ids, additional_info=addition_info, instance_profile_name=instance_profile_name, instance_profile_arn=instance_profile_arn, tenancy=tenancy, callback=callback, )
def create_instance(name, config, region, key_name, ssh_key, instance_data, deploypass, loaned_to, loan_bug, create_ami, ignore_subnet_check, max_attempts): """Creates an AMI instance with the given name and config. The config must specify things like ami id.""" conn = get_aws_connection(region) # Make sure we don't request the same things twice token = str(uuid.uuid4())[:16] instance_data = instance_data.copy() instance_data['name'] = name instance_data['domain'] = config['domain'] instance_data['hostname'] = '{name}.{domain}'.format( name=name, domain=config['domain']) ami = conn.get_all_images(image_ids=[config["ami"]])[0] bdm = None if 'device_map' in config: bdm = BlockDeviceMapping() for device, device_info in config['device_map'].items(): bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] # Overwrite root device size for HVM instances, since they cannot # be resized online if ami.virtualization_type == "hvm" and \ ami.root_device_name == device: bd.size = ami.block_device_mapping[ami.root_device_name].size if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] bdm[device] = bd interfaces = make_instance_interfaces(region, instance_data['hostname'], ignore_subnet_check, config.get('subnet_ids'), config.get('security_group_ids', []), config.get("use_public_ip")) keep_going, attempt = True, 1 while keep_going: try: if 'user_data_file' in config: user_data = open(config['user_data_file']).read() else: user_data = get_user_data_tmpl(config['type']) if user_data: user_data = user_data.format( puppet_server=instance_data.get('default_puppet_server'), fqdn=instance_data['hostname'], hostname=instance_data['name'], domain=instance_data['domain'], dns_search_domain=config.get('dns_search_domain'), password=deploypass, moz_instance_type=config['type'], region_dns_atom=get_region_dns_atom(region), ) reservation = conn.run_instances( image_id=config['ami'], key_name=key_name, instance_type=config['instance_type'], block_device_map=bdm, client_token=token, disable_api_termination=config.get('disable_api_termination'), user_data=user_data, instance_profile_name=config.get('instance_profile_name'), network_interfaces=interfaces, ) break except boto.exception.BotoServerError: log.exception("Cannot start an instance") time.sleep(10) if max_attempts: attempt += 1 keep_going = max_attempts >= attempt instance = reservation.instances[0] log.info("instance %s created, waiting to come up", instance) # Wait for the instance to come up wait_for_status(instance, "state", "running", "update") instance.add_tag('Name', name) instance.add_tag('FQDN', instance_data['hostname']) instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime())) instance.add_tag('moz-type', config['type']) if loaned_to: instance.add_tag("moz-loaned-to", loaned_to) if loan_bug: instance.add_tag("moz-bug", loan_bug) log.info("assimilating %s", instance) instance.add_tag('moz-state', 'pending') keep_going, attempt = True, 1 while keep_going: try: # Don't reboot if need to create ami reboot = not create_ami assimilate_instance(instance=instance, config=config, ssh_key=ssh_key, instance_data=instance_data, deploypass=deploypass, reboot=reboot) break except NetworkError as e: # it takes a while for the machine to start/reboot so the # NetworkError exception is quite common, just log the error, # without the full stack trace log.warn( "cannot connect; instance may still be starting %s (%s, %s) - %s," "retrying in %d sec ...", instance_data['hostname'], instance.id, instance.private_ip_address, e, FAILURE_TIMEOUT) time.sleep(FAILURE_TIMEOUT) except: # any other exception log.warn( "problem assimilating %s (%s, %s), retrying in " "%d sec ...", instance_data['hostname'], instance.id, instance.private_ip_address, FAILURE_TIMEOUT, exc_info=True) time.sleep(FAILURE_TIMEOUT) if max_attempts: attempt += 1 keep_going = max_attempts >= attempt instance.add_tag('moz-state', 'ready') if create_ami: ami_name = "spot-%s-%s" % ( config['type'], time.strftime("%Y-%m-%d-%H-%M", time.gmtime())) log.info("Generating AMI %s", ami_name) ami_cleanup(mount_point="/", distro=config["distro"]) root_bd = instance.block_device_mapping[instance.root_device_name] volume = instance.connection.get_all_volumes( volume_ids=[root_bd.volume_id])[0] # The instance has to be stopped to flush EBS caches instance.stop() wait_for_status(instance, 'state', 'stopped', 'update') ami = volume_to_ami(volume=volume, ami_name=ami_name, arch=instance.architecture, virtualization_type=instance.virtualization_type, kernel_id=instance.kernel, root_device_name=instance.root_device_name, tags=config["tags"]) log.info("AMI %s (%s) is ready", ami_name, ami.id) log.warn("Terminating %s", instance) instance.terminate()
def run_instances_for_role(role, count=1, owner=None): """Run instances for the role, returns the reservation which contains the new instances role (Role) instance of a subclass of Role which contains details about the ec2 instances to start count (int) owner (str) """ dev_sda1 = BlockDeviceType() dev_sda1.size = role.instance_root_volume_size bdm = BlockDeviceMapping() bdm['/dev/sda1'] = dev_sda1 reservation = ec2_conn.run_instances( default_ami[ec2_conn.region.name], min_count=count, max_count=count, user_data=userdata.generate_userdata_base64(role.env, env.user), key_name='ec2-key', security_groups=role.ec2_security_groups, instance_type=role.instance_type, block_device_map=bdm, placement=role.placement ) # Give AWS a few seconds to get things synced on their side sleep(10) # Drop into a loop and wait for all the instances to listen for connections for instance in reservation.instances: while instance.update() != 'running': sleep(20) while instance.public_dns_name is None: sleep(20) instance.update() while True: s = socket() try: s.connect((instance.public_dns_name, 22)) except: print '%s not yet accepting connections, waiting...' \ % instance.public_dns_name sleep(10) else: break dns = DynDNS() for instance in reservation.instances: shortname = '%s-%s-%s' % (role.env, role.role_for_name_tag, instance.id) dns.add_cname(instance.public_dns_name, shortname) sleep(5) # wait for dynect to catch up instance.dns_name = shortname + '.domain.com' instance.add_tag('Name', shortname) instance.add_tag('role', role.role_name) instance.add_tag('env', role.env) if owner: instance.add_tag('owner', owner) if role.role_name == 'development': print 'Create RDS instance for this development host...' rds_instance = rds_conn.restore_dbinstance_from_dbsnapshot( 'development-rds-master', shortname, 'db.m1.medium') while rds_instance.update() != 'available': print 'RDS instance not ready yet, sleeping 30 seconds.' sleep(30) with settings(host_string=instance.dns_name): sudo('yum -y install mysql') % rds_instance.endpoint[0])
def test_create_launch_configuration_with_block_device_mappings(): block_device_mapping = BlockDeviceMapping() ephemeral_drive = BlockDeviceType() ephemeral_drive.ephemeral_name = 'ephemeral0' block_device_mapping['/dev/xvdb'] = ephemeral_drive snapshot_drive = BlockDeviceType() snapshot_drive.snapshot_id = "snap-1234abcd" snapshot_drive.volume_type = "standard" block_device_mapping['/dev/xvdp'] = snapshot_drive ebs_drive = BlockDeviceType() ebs_drive.volume_type = "io1" ebs_drive.size = 100 ebs_drive.iops = 1000 ebs_drive.delete_on_termination = False block_device_mapping['/dev/xvdh'] = ebs_drive conn = boto.connect_autoscale(use_block_device_types=True) config = LaunchConfiguration( name='tester', image_id='ami-abcd1234', instance_type='m1.small', key_name='the_keys', security_groups=["default", "default2"], user_data=b"This is some user_data", instance_monitoring=True, instance_profile_name= 'arn:aws:iam::123456789012:instance-profile/testing', spot_price=0.1, block_device_mappings=[block_device_mapping]) conn.create_launch_configuration(config) launch_config = conn.get_all_launch_configurations()[0] launch_config.name.should.equal('tester') launch_config.image_id.should.equal('ami-abcd1234') launch_config.instance_type.should.equal('m1.small') launch_config.key_name.should.equal('the_keys') set(launch_config.security_groups).should.equal( set(['default', 'default2'])) launch_config.user_data.should.equal(b"This is some user_data") launch_config.instance_monitoring.enabled.should.equal('true') launch_config.instance_profile_name.should.equal( 'arn:aws:iam::123456789012:instance-profile/testing') launch_config.spot_price.should.equal(0.1) len(launch_config.block_device_mappings).should.equal(3) returned_mapping = launch_config.block_device_mappings set(returned_mapping.keys()).should.equal( set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh'])) returned_mapping['/dev/xvdh'].iops.should.equal(1000) returned_mapping['/dev/xvdh'].size.should.equal(100) returned_mapping['/dev/xvdh'].volume_type.should.equal("io1") returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd") returned_mapping['/dev/xvdp'].volume_type.should.equal("standard") returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')
def handleRunInstances(self, action, clc, user_data_file): image_id = self.get_argument('ImageId') min = self.get_argument('MinCount', '1') max = self.get_argument('MaxCount', '1') key = self.get_argument('KeyName', None) groups = self.get_argument_list('SecurityGroup') sec_group_ids = self.get_argument_list('SecurityGroupId') if user_data_file: user_data = user_data_file else: user_data = self.get_argument('UserData', None) addr_type = self.get_argument('AddressingType', None) vm_type = self.get_argument('InstanceType', None) placement = self.get_argument('Placement.AvailabilityZone', None) placement_group = self.get_argument('Placement.GroupName', None) tenancy = self.get_argument('Placement.Tenancy', None) kernel = self.get_argument('KernelId', None) ramdisk = self.get_argument('RamdiskId', None) monitoring = False if self.get_argument('Monitoring.Enabled', '') == 'true': monitoring = True subnet = self.get_argument('SubnetId', None) private_ip = self.get_argument('PrivateIpAddress', None) # get block device mappings bdm = BlockDeviceMapping() mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None) idx = 1 while mapping: pre = 'BlockDeviceMapping.%d' % idx dev_name = mapping block_dev_type = BlockDeviceType() block_dev_type.ephemeral_name = self.get_argument( '%s.VirtualName' % pre, None) if not (block_dev_type.ephemeral_name): block_dev_type.no_device = \ (self.get_argument('%s.NoDevice' % pre, '') == 'true') block_dev_type.snapshot_id = \ self.get_argument('%s.Ebs.SnapshotId' % pre, None) block_dev_type.size = \ self.get_argument('%s.Ebs.VolumeSize' % pre, None) block_dev_type.delete_on_termination = \ (self.get_argument('%s.DeleteOnTermination' % pre, '') == 'true') bdm[dev_name] = block_dev_type idx += 1 mapping = self.get_argument( 'BlockDeviceMapping.%d.DeviceName' % idx, None) if len(bdm) == 0: bdm = None api_termination = False if self.get_argument('DisableApiTermination', '') == 'true': api_termination = True instance_shutdown = False if self.get_argument('InstanceInitiatedShutdownBehavior', '') == 'true': instance_shutdown = True token = self.get_argument('ClientToken', None) addition_info = self.get_argument('AdditionInfo', None) instance_profile_name = self.get_argument('IamInstanceProfile.Name', None) instance_profile_arn = self.get_argument('IamInstanceProfile.Arn', None) return self.__normalize_instances__([ clc.run_instances( image_id, min_count=min, max_count=max, key_name=key, security_groups=groups, user_data=user_data, addressing_type=addr_type, instance_type=vm_type, placement=placement, kernel_id=kernel, ramdisk_id=ramdisk, monitoring_enabled=monitoring, subnet_id=subnet, block_device_map=bdm, disable_api_termination=api_termination, instance_initiated_shutdown_behavior=instance_shutdown, private_ip_address=private_ip, placement_group=placement_group, client_token=token, security_group_ids=sec_group_ids, additional_info=addition_info, instance_profile_name=instance_profile_name, instance_profile_arn=instance_profile_arn, tenancy=tenancy) ])
def handleImages(self, action, clc, callback=None): if action == 'DescribeImages': owner = self.get_argument('Owner', None); if not owner: owners = None else: owners = [owner] filters = self.get_filter_args() return clc.get_all_images(owners, filters, callback) elif action == 'DescribeImageAttribute': imageid = self.get_argument('ImageId') attribute = self.get_argument('Attribute') return clc.get_image_attribute(imageid, attribute, callback) elif action == 'ModifyImageAttribute': imageid = self.get_argument('ImageId') attribute = self.get_argument('Attribute') operation = self.get_argument('OperationType') users = self.get_argument_list('UserId') groups = self.get_argument_list('UserGroup') return clc.modify_image_attribute(imageid, attribute, operation, users, groups, callback) elif action == 'ResetImageAttribute': imageid = self.get_argument('ImageId') attribute = self.get_argument('Attribute') return clc.reset_image_attribute(imageid, attribute, callback) elif action == 'DeregisterImage': image_id = self.get_argument('ImageId') return clc.deregister_image(image_id, callback) elif action == 'RegisterImage': image_location = self.get_argument('ImageLocation', None) name = self.get_argument('Name') description = self.get_argument('Description', None) if description != None: description = base64.b64decode(description); architecture = self.get_argument('Architecture', None) kernel_id = self.get_argument('KernelId', None) ramdisk_id = self.get_argument('RamdiskId', None) root_dev_name = self.get_argument('RootDeviceName', None) snapshot_id = self.get_argument('SnapshotId', None) # get block device mappings bdm = BlockDeviceMapping() mapping = self.get_argument('BlockDeviceMapping.1.DeviceName', None) idx = 1 while mapping: pre = 'BlockDeviceMapping.%d' % idx dev_name = mapping block_dev_type = BlockDeviceType() block_dev_type.ephemeral_name = self.get_argument('%s.VirtualName' % pre, None) if not(block_dev_type.ephemeral_name): block_dev_type.no_device = \ (self.get_argument('%s.NoDevice' % pre, '') == 'true') block_dev_type.snapshot_id = \ self.get_argument('%s.Ebs.SnapshotId' % pre, None) block_dev_type.size = \ self.get_argument('%s.Ebs.VolumeSize' % pre, None) block_dev_type.delete_on_termination = \ (self.get_argument('%s.Ebs.DeleteOnTermination' % pre, '') == 'true') bdm[dev_name] = block_dev_type idx += 1 mapping = self.get_argument('BlockDeviceMapping.%d.DeviceName' % idx, None) if snapshot_id: rootbdm = BlockDeviceType() rootbdm.snapshot_id = snapshot_id bdm['/dev/sda1'] = rootbdm if len(bdm) == 0: bdm = None return clc.register_image(name, image_location, description, architecture, kernel_id, ramdisk_id, root_dev_name, bdm, callback)
def launchInstance(region, rootsize, role, env, ami, key_name, instance_type, placement, sec_group, iamrole, ebs, product, project, name, chefversion): '''Launch a single instance of the provided ami''' conn = connectEC2(region) mapping = BlockDeviceMapping() root = BlockDeviceType() root.size = rootsize mapping['/dev/xvda'] = root user_data = genUserData(role, env, chefversion) sec_group_id = getSecurityGroupID(conn, sec_group) subnet_id = getSubnetID(placement, env) print Fore.YELLOW + 'Instance will be launched with following parameters.\n' print Fore.GREEN + 'Region :\t\t' + region print Fore.GREEN + 'Chef Role :\t\t' + role print Fore.GREEN + 'Chef Env :\t\t' + env print Fore.GREEN + 'AMI ID :\t\t' + ami print Fore.GREEN + 'Keypair :\t\t' + key_name print Fore.GREEN + 'Instance Type :\t\t' + instance_type print Fore.GREEN + 'Placement :\t\t' + placement print Fore.GREEN + 'Security Group :\t' + sec_group + ' (' + sec_group_id + ')' print Fore.GREEN + 'Subnet ID :\t\t' + subnet_id print Fore.GREEN + 'IAM Role :\t\t' + iamrole if ebs: print Fore.GREEN + 'EBS Volume Size: ' + str(ebs) + ' GB' print Fore.YELLOW + '\nFollowing tags will be attached to the instance.\n' if name: print Fore.GREEN + 'Name :\t\t\t' + name.lower() print Fore.GREEN + 'Product :\t\t' + product.lower() print Fore.GREEN + 'Project :\t\t' + project.lower() print Fore.GREEN + 'Environment :\t\t' + env.lower() print Fore.YELLOW + '\nTrying to launch an EC2 instance ...' reservation = conn.run_instances(ami, key_name=key_name, user_data=user_data, instance_type=instance_type, placement=placement, subnet_id=subnet_id, block_device_map=mapping, instance_initiated_shutdown_behavior='stop', security_group_ids=['sg-********', sec_group_id], instance_profile_name=iamrole, dry_run=False) print(Fore.YELLOW + '\nWaiting for instance to start ...') instance = reservation.instances[0] status = instance.update() while status == 'pending': time.sleep(5) status = instance.update() if status == 'running': print(Fore.GREEN + 'New instance "' + instance.id + '" accessible at ' + instance.private_ip_address) print(Fore.YELLOW + 'Adding tags.') if name: instance.add_tag('Name',name.lower()) instance.add_tag('Product', product.lower()) instance.add_tag('Project', project.lower()) instance.add_tag('Environment', env.lower()) else: print(Fore.YELLOW + 'Instance status: ' + status) return # If we got through the launching successfully, go ahead and create and attach a volume. if ebs: attachEBS(conn, instance, ebs) print Fore.GREEN + '\nInstance ' + instance.id + ' launched successfuly.'
def register(snapshot_id, region, arch, size=None, name=None, desc=None, pvm=False): conn = utils.connect(region) if None in (name, size): log.debug('getting snapshot - %s', snapshot_id) snapshot = conn.get_all_snapshots(snapshot_ids=[snapshot_id])[0] size = size if size else snapshot.volume_size name = name if name else snapshot.description virt = 'hvm' kernel_id = None device_base = '/dev/xvd' ec2_arch = "x86_64" if arch == "amd64" else arch if pvm: kernel_id = utils.get_kernel(region, arch) virt = 'paravirtual' device_base = '/dev/sd' name += '-pvm' log.debug('creating block_device_map') block_device_map = BlockDeviceMapping() rootfs = BlockDeviceType() rootfs.delete_on_termination = True rootfs.size = size rootfs.snapshot_id = snapshot_id rootfs_device_name = device_base + 'a' block_device_map[rootfs_device_name] = rootfs ephemeral = BlockDeviceType() ephemeral.ephemeral_name = 'ephemeral0' ephemeral_device_name = device_base + 'b' block_device_map[ephemeral_device_name] = ephemeral log.debug('registering image - %s', name) client3 = utils.connect_boto3(region) response = client3.register_image(Name=name, Architecture=ec2_arch, RootDeviceName=rootfs_device_name, BlockDeviceMappings=[{ 'DeviceName': '/dev/xvda', 'Ebs': { 'DeleteOnTermination': True, 'VolumeSize': size, 'SnapshotId': snapshot_id, }, }, { 'DeviceName': '/dev/xvdb', 'VirtualName': 'ephemeral0', }], VirtualizationType=virt, EnaSupport=True) ami_id = response['ImageId'] log.info('registered image - %s %s %s', ami_id, name, region) return ami_id, name