def test_create_launch_configuration_with_block_device_mappings(): block_device_mapping = BlockDeviceMapping() ephemeral_drive = BlockDeviceType() ephemeral_drive.ephemeral_name = 'ephemeral0' block_device_mapping['/dev/xvdb'] = ephemeral_drive snapshot_drive = BlockDeviceType() snapshot_drive.snapshot_id = "snap-1234abcd" snapshot_drive.volume_type = "standard" block_device_mapping['/dev/xvdp'] = snapshot_drive ebs_drive = BlockDeviceType() ebs_drive.volume_type = "io1" ebs_drive.size = 100 ebs_drive.iops = 1000 ebs_drive.delete_on_termination = False block_device_mapping['/dev/xvdh'] = ebs_drive conn = boto.connect_autoscale(use_block_device_types=True) config = LaunchConfiguration( name='tester', image_id='ami-abcd1234', instance_type='m1.small', key_name='the_keys', security_groups=["default", "default2"], user_data="This is some user_data", instance_monitoring=True, instance_profile_name='arn:aws:iam::123456789012:instance-profile/testing', spot_price=0.1, block_device_mappings=[block_device_mapping] ) conn.create_launch_configuration(config) launch_config = conn.get_all_launch_configurations()[0] launch_config.name.should.equal('tester') launch_config.image_id.should.equal('ami-abcd1234') launch_config.instance_type.should.equal('m1.small') launch_config.key_name.should.equal('the_keys') set(launch_config.security_groups).should.equal(set(['default', 'default2'])) launch_config.user_data.should.equal("This is some user_data") launch_config.instance_monitoring.enabled.should.equal('true') launch_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/testing') launch_config.spot_price.should.equal(0.1) len(launch_config.block_device_mappings).should.equal(3) returned_mapping = launch_config.block_device_mappings set(returned_mapping.keys()).should.equal(set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh'])) returned_mapping['/dev/xvdh'].iops.should.equal(1000) returned_mapping['/dev/xvdh'].size.should.equal(100) returned_mapping['/dev/xvdh'].volume_type.should.equal("io1") returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd") returned_mapping['/dev/xvdp'].volume_type.should.equal("standard") returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')
def _parse_block_device_mappings(user_input): """ Parse block device mappings per AWS CLI tools syntax (modified to add IOPS) http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html Syntax: /dev/xvd[a-z]=[snapshot-id|ephemeral]:[size in GB]:[Delete on Term]:[IOPS] - Leave inapplicable fields blank - Delete on Termination defaults to True - IOPS limits are not validated - EBS sizing is not validated Mount an Ephemeral Drive: /dev/xvdb1=ephemeral0 Mount multiple Ephemeral Drives: /dev/xvdb1=ephemeral0,/dev/xvdb2=ephemeral1 Mount a Snapshot: /dev/xvdp=snap-1234abcd Mount a Snapshot to a 100GB drive: /dev/xvdp=snap-1234abcd:100 Mount a Snapshot to a 100GB drive and do not delete on termination: /dev/xvdp=snap-1234abcd:100:false Mount a Fresh 100GB EBS device /dev/xvdp=:100 Mount a Fresh 100GB EBS Device and do not delete on termination: /dev/xvdp=:100:false Mount a Fresh 100GB EBS Device with 1000 IOPS /dev/xvdp=:100::1000 """ block_device_map = BlockDeviceMapping() mappings = user_input.split(",") for mapping in mappings: block_type = BlockDeviceType() mount_point, drive_type, size, delete, iops = _parse_drive_mapping( mapping) if 'ephemeral' in drive_type: block_type.ephemeral_name = drive_type elif 'snap' in drive_type: block_type.snapshot_id = drive_type block_type.volume_type = "standard" else: block_type.volume_type = "standard" block_type.size = size block_type.delete_on_termination = delete if iops: block_type.iops = iops block_type.volume_type = "io1" block_device_map[mount_point] = block_type return block_device_map
def _parse_block_device_mappings(user_input): """ Parse block device mappings per AWS CLI tools syntax (modified to add IOPS) http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html Syntax: /dev/xvd[a-z]=[snapshot-id|ephemeral]:[size in GB]:[Delete on Term]:[IOPS] - Leave inapplicable fields blank - Delete on Termination defaults to True - IOPS limits are not validated - EBS sizing is not validated Mount an Ephemeral Drive: /dev/xvdb1=ephemeral0 Mount multiple Ephemeral Drives: /dev/xvdb1=ephemeral0,/dev/xvdb2=ephemeral1 Mount a Snapshot: /dev/xvdp=snap-1234abcd Mount a Snapshot to a 100GB drive: /dev/xvdp=snap-1234abcd:100 Mount a Snapshot to a 100GB drive and do not delete on termination: /dev/xvdp=snap-1234abcd:100:false Mount a Fresh 100GB EBS device /dev/xvdp=:100 Mount a Fresh 100GB EBS Device and do not delete on termination: /dev/xvdp=:100:false Mount a Fresh 100GB EBS Device with 1000 IOPS /dev/xvdp=:100::1000 """ block_device_map = BlockDeviceMapping() mappings = user_input.split(",") for mapping in mappings: block_type = BlockDeviceType() mount_point, drive_type, size, delete, iops = _parse_drive_mapping(mapping) if 'ephemeral' in drive_type: block_type.ephemeral_name = drive_type elif 'snap' in drive_type: block_type.snapshot_id = drive_type block_type.volume_type = "standard" else: block_type.volume_type = "standard" block_type.size = size block_type.delete_on_termination = delete if iops: block_type.iops = iops block_type.volume_type = "io1" block_device_map[mount_point] = block_type return block_device_map
def get_block_device_map(bdmapping_json=None): """Parse block_device_mapping JSON and return a configured BlockDeviceMapping object Mapping JSON structure... {"/dev/sda": {"snapshot_id": "snap-23E93E09", "volume_type": null, "delete_on_termination": true, "size": 1} } """ if bdmapping_json: mapping = json.loads(bdmapping_json) if mapping: bdm = BlockDeviceMapping() for key, val in mapping.items(): device = BlockDeviceType() if val.get('virtual_name') is not None and val.get( 'virtual_name').startswith('ephemeral'): device.ephemeral_name = val.get('virtual_name') else: device.volume_type = 'standard' device.snapshot_id = val.get('snapshot_id') or None device.size = val.get('size') device.delete_on_termination = val.get( 'delete_on_termination', False) bdm[key] = device return bdm return None return None
def create_block_device_mapping(ami, device_map): bdm = BlockDeviceMapping() for device, device_info in device_map.items(): if ami.root_device_type == "instance-store" and \ not device_info.get("ephemeral_name"): # EBS is not supported by S3-backed AMIs at request time # EBS volumes can be attached when an instance is running continue bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] if ami.root_device_name == device: ami_size = ami.block_device_mapping[device].size if ami.virtualization_type == "hvm": # Overwrite root device size for HVM instances, since they # cannot be resized online bd.size = ami_size elif device_info.get('size'): # make sure that size is enough for this AMI assert ami_size <= device_info['size'], \ "Instance root device size cannot be smaller than AMI " \ "root device" if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] if device_info.get("volume_type"): bd.volume_type = device_info["volume_type"] if device_info["volume_type"] == "io1" \ and device_info.get("iops"): bd.iops = device_info["iops"] bdm[device] = bd return bdm
def create_block_device_mapping(ami, device_map): bdm = BlockDeviceMapping() for device, device_info in device_map.items(): if ami.root_device_type == "instance-store" and \ not device_info.get("ephemeral_name"): # EBS is not supported by S3-backed AMIs at request time # EBS volumes can be attached when an instance is running continue bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] if ami.root_device_name == device: ami_size = ami.block_device_mapping[device].size if ami.virtualization_type == "hvm": # Overwrite root device size for HVM instances, since they # cannot be resized online bd.size = ami_size elif device_info.get('size'): # make sure that size is enough for this AMI assert ami_size <= device_info['size'], \ "Instance root device size cannot be smaller than AMI " \ "root device" if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] if device_info.get("volume_type"): bd.volume_type = device_info["volume_type"] if device_info["volume_type"] == "io1" \ and device_info.get("iops"): bd.iops = device_info["iops"] bdm[device] = bd return bdm
def _parse_block_device_mappings(self): block_device_map = BlockDeviceMapping() for mapping in self.block_device_mapping_dict: block_type = BlockDeviceType() mount_point = mapping.get('device_name') if 'ephemeral' in mapping.get('virtual_name', ''): block_type.ephemeral_name = mapping.get('virtual_name') else: block_type.volume_type = mapping.get('ebs._volume_type') block_type.snapshot_id = mapping.get('ebs._snapshot_id') block_type.delete_on_termination = mapping.get('ebs._delete_on_termination') block_type.size = mapping.get('ebs._volume_size') block_type.iops = mapping.get('ebs._iops') block_device_map[mount_point] = block_type return block_device_map
def _parse_block_device_mappings(self): block_device_map = BlockDeviceMapping() for mapping in self.block_device_mapping_dict: block_type = BlockDeviceType() mount_point = mapping.get('device_name') if 'ephemeral' in mapping.get('virtual_name', ''): block_type.ephemeral_name = mapping.get('virtual_name') else: block_type.volume_type = mapping.get('ebs._volume_type') block_type.snapshot_id = mapping.get('ebs._snapshot_id') block_type.delete_on_termination = mapping.get('ebs._delete_on_termination') block_type.size = mapping.get('ebs._volume_size') block_type.iops = mapping.get('ebs._iops') block_device_map[mount_point] = block_type return block_device_map
def _parse_block_device_mappings(self): block_device_map = BlockDeviceMapping() for mapping in self.block_device_mapping_dict: block_type = BlockDeviceType() mount_point = mapping.get("device_name") if "ephemeral" in mapping.get("virtual_name", ""): block_type.ephemeral_name = mapping.get("virtual_name") else: block_type.volume_type = mapping.get("ebs._volume_type") block_type.snapshot_id = mapping.get("ebs._snapshot_id") block_type.delete_on_termination = mapping.get( "ebs._delete_on_termination") block_type.size = mapping.get("ebs._volume_size") block_type.iops = mapping.get("ebs._iops") block_device_map[mount_point] = block_type return block_device_map
def get_block_device_map(bdmapping_json=None): """Parse block_device_mapping JSON and return a configured BlockDeviceMapping object Mapping JSON structure... {"/dev/sda": {"snapshot_id": "snap-23E93E09", "volume_type": null, "delete_on_termination": true, "size": 1} } """ if bdmapping_json: mapping = json.loads(bdmapping_json) if mapping: bdm = BlockDeviceMapping() for key, val in mapping.items(): device = BlockDeviceType() if val.get('virtual_name') is not None and val.get('virtual_name').startswith('ephemeral'): device.ephemeral_name = val.get('virtual_name') else: device.volume_type = 'standard' device.snapshot_id = val.get('snapshot_id') or None device.size = val.get('size') device.delete_on_termination = val.get('delete_on_termination', False) bdm[key] = device return bdm return None return None
def create_instance(name, config, region, key_name, ssh_key, instance_data, deploypass, loaned_to, loan_bug, create_ami, ignore_subnet_check, max_attempts): """Creates an AMI instance with the given name and config. The config must specify things like ami id.""" conn = get_aws_connection(region) # Make sure we don't request the same things twice token = str(uuid.uuid4())[:16] instance_data = instance_data.copy() instance_data['name'] = name instance_data['domain'] = config['domain'] instance_data['hostname'] = '{name}.{domain}'.format( name=name, domain=config['domain']) ami = conn.get_all_images(image_ids=[config["ami"]])[0] bdm = None if 'device_map' in config: bdm = BlockDeviceMapping() for device, device_info in config['device_map'].items(): bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] # Overwrite root device size for HVM instances, since they cannot # be resized online if ami.virtualization_type == "hvm" and \ ami.root_device_name == device: bd.size = ami.block_device_mapping[ami.root_device_name].size if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] if device_info.get("volume_type"): bd.volume_type = device_info["volume_type"] if device_info["volume_type"] == "io1" \ and device_info.get("iops"): bd.iops = device_info["iops"] bdm[device] = bd interfaces = make_instance_interfaces( region, instance_data['hostname'], ignore_subnet_check, config.get('subnet_ids'), config.get('security_group_ids', []), config.get("use_public_ip")) keep_going, attempt = True, 1 while keep_going: try: puppet_master = pick_puppet_master(instance_data.get('puppet_masters')) user_data = user_data_from_template(config['type'], { "puppet_server": puppet_master, "fqdn": instance_data['hostname'], "hostname": instance_data['name'], "domain": instance_data['domain'], "dns_search_domain": config.get('dns_search_domain'), "password": deploypass, "moz_instance_type": config['type'], "region_dns_atom": get_region_dns_atom(region)}) reservation = conn.run_instances( image_id=config['ami'], key_name=key_name, instance_type=config['instance_type'], block_device_map=bdm, client_token=token, disable_api_termination=config.get('disable_api_termination'), user_data=user_data, instance_profile_name=config.get('instance_profile_name'), network_interfaces=interfaces, ) break except boto.exception.BotoServerError: log.exception("Cannot start an instance") time.sleep(10) if max_attempts: attempt += 1 keep_going = max_attempts >= attempt instance = reservation.instances[0] log.info("instance %s created, waiting to come up", instance) # Wait for the instance to come up wait_for_status(instance, "state", "running", "update") instance.add_tag('Name', name) instance.add_tag('FQDN', instance_data['hostname']) instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime())) instance.add_tag('moz-type', config['type']) if loaned_to: instance.add_tag("moz-loaned-to", loaned_to) if loan_bug: instance.add_tag("moz-bug", loan_bug) log.info("assimilating %s", instance) instance.add_tag('moz-state', 'pending') keep_going, attempt = True, 1 while keep_going: try: # Don't reboot if need to create ami reboot = not create_ami assimilate_instance(instance=instance, config=config, ssh_key=ssh_key, instance_data=instance_data, deploypass=deploypass, reboot=reboot) break except NetworkError as e: # it takes a while for the machine to start/reboot so the # NetworkError exception is quite common, just log the error, # without the full stack trace log.warn("cannot connect; instance may still be starting %s (%s, %s) - %s," "retrying in %d sec ...", instance_data['hostname'], instance.id, instance.private_ip_address, e, FAILURE_TIMEOUT) time.sleep(FAILURE_TIMEOUT) except: # any other exception log.warn("problem assimilating %s (%s, %s), retrying in " "%d sec ...", instance_data['hostname'], instance.id, instance.private_ip_address, FAILURE_TIMEOUT, exc_info=True) time.sleep(FAILURE_TIMEOUT) if max_attempts: attempt += 1 keep_going = max_attempts >= attempt instance.add_tag('moz-state', 'ready') if create_ami: ami_name = "spot-%s-%s" % ( config['type'], time.strftime("%Y-%m-%d-%H-%M", time.gmtime())) log.info("Generating AMI %s", ami_name) ami_cleanup(mount_point="/", distro=config["distro"]) root_bd = instance.block_device_mapping[instance.root_device_name] volume = instance.connection.get_all_volumes( volume_ids=[root_bd.volume_id])[0] # The instance has to be stopped to flush EBS caches instance.stop() wait_for_status(instance, 'state', 'stopped', 'update') ami = volume_to_ami(volume=volume, ami_name=ami_name, arch=instance.architecture, virtualization_type=instance.virtualization_type, kernel_id=instance.kernel, root_device_name=instance.root_device_name, tags=config["tags"]) log.info("AMI %s (%s) is ready", ami_name, ami.id) log.warn("Terminating %s", instance) instance.terminate()
def create_instance(name, config, region, key_name, ssh_key, instance_data, deploypass, loaned_to, loan_bug, create_ami, ignore_subnet_check, max_attempts): """Creates an AMI instance with the given name and config. The config must specify things like ami id.""" conn = get_aws_connection(region) # Make sure we don't request the same things twice token = str(uuid.uuid4())[:16] instance_data = instance_data.copy() instance_data['name'] = name instance_data['domain'] = config['domain'] instance_data['hostname'] = '{name}.{domain}'.format( name=name, domain=config['domain']) ami = conn.get_all_images(image_ids=[config["ami"]])[0] bdm = None if 'device_map' in config: bdm = BlockDeviceMapping() for device, device_info in config['device_map'].items(): bd = BlockDeviceType() if device_info.get('size'): bd.size = device_info['size'] # Overwrite root device size for HVM instances, since they cannot # be resized online if ami.virtualization_type == "hvm" and \ ami.root_device_name == device: bd.size = ami.block_device_mapping[ami.root_device_name].size if device_info.get("delete_on_termination") is not False: bd.delete_on_termination = True if device_info.get("ephemeral_name"): bd.ephemeral_name = device_info["ephemeral_name"] if device_info.get("volume_type"): bd.volume_type = device_info["volume_type"] if device_info["volume_type"] == "io1" \ and device_info.get("iops"): bd.iops = device_info["iops"] bdm[device] = bd interfaces = make_instance_interfaces(region, instance_data['hostname'], ignore_subnet_check, config.get('subnet_ids'), config.get('security_group_ids', []), config.get("use_public_ip")) keep_going, attempt = True, 1 while keep_going: try: puppet_master = pick_puppet_master( instance_data.get('puppet_masters')) user_data = user_data_from_template( config['type'], { "puppet_server": puppet_master, "fqdn": instance_data['hostname'], "hostname": instance_data['name'], "domain": instance_data['domain'], "dns_search_domain": config.get('dns_search_domain'), "password": deploypass, "moz_instance_type": config['type'], "region_dns_atom": get_region_dns_atom(region) }) reservation = conn.run_instances( image_id=config['ami'], key_name=key_name, instance_type=config['instance_type'], block_device_map=bdm, client_token=token, disable_api_termination=config.get('disable_api_termination'), user_data=user_data, instance_profile_name=config.get('instance_profile_name'), network_interfaces=interfaces, ) break except boto.exception.BotoServerError: log.exception("Cannot start an instance") time.sleep(10) if max_attempts: attempt += 1 keep_going = max_attempts >= attempt instance = reservation.instances[0] log.info("instance %s created, waiting to come up", instance) # Wait for the instance to come up wait_for_status(instance, "state", "running", "update") instance.add_tag('Name', name) instance.add_tag('FQDN', instance_data['hostname']) instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime())) instance.add_tag('moz-type', config['type']) if loaned_to: instance.add_tag("moz-loaned-to", loaned_to) if loan_bug: instance.add_tag("moz-bug", loan_bug) log.info("assimilating %s", instance) instance.add_tag('moz-state', 'pending') keep_going, attempt = True, 1 while keep_going: try: # Don't reboot if need to create ami reboot = not create_ami assimilate_instance(instance=instance, config=config, ssh_key=ssh_key, instance_data=instance_data, deploypass=deploypass, reboot=reboot) break except NetworkError as e: # it takes a while for the machine to start/reboot so the # NetworkError exception is quite common, just log the error, # without the full stack trace log.warn( "cannot connect; instance may still be starting %s (%s, %s) - %s," "retrying in %d sec ...", instance_data['hostname'], instance.id, instance.private_ip_address, e, FAILURE_TIMEOUT) time.sleep(FAILURE_TIMEOUT) except: # any other exception log.warn( "problem assimilating %s (%s, %s), retrying in " "%d sec ...", instance_data['hostname'], instance.id, instance.private_ip_address, FAILURE_TIMEOUT, exc_info=True) time.sleep(FAILURE_TIMEOUT) if max_attempts: attempt += 1 keep_going = max_attempts >= attempt instance.add_tag('moz-state', 'ready') if create_ami: ami_name = "spot-%s-%s" % ( config['type'], time.strftime("%Y-%m-%d-%H-%M", time.gmtime())) log.info("Generating AMI %s", ami_name) ami_cleanup(mount_point="/", distro=config["distro"]) root_bd = instance.block_device_mapping[instance.root_device_name] volume = instance.connection.get_all_volumes( volume_ids=[root_bd.volume_id])[0] # The instance has to be stopped to flush EBS caches # The sleep is to prevent the occasional interference of the shutdown with the capture of Windows AMIs time.sleep(15) instance.stop() wait_for_status(instance, 'state', 'stopped', 'update') ami = volume_to_ami(volume=volume, ami_name=ami_name, arch=instance.architecture, virtualization_type=instance.virtualization_type, kernel_id=instance.kernel, root_device_name=instance.root_device_name, tags=config["tags"]) log.info("AMI %s (%s) is ready", ami_name, ami.id) log.warn("Terminating %s", instance) instance.terminate()
def test_create_launch_configuration_with_block_device_mappings(): block_device_mapping = BlockDeviceMapping() ephemeral_drive = BlockDeviceType() ephemeral_drive.ephemeral_name = "ephemeral0" block_device_mapping["/dev/xvdb"] = ephemeral_drive snapshot_drive = BlockDeviceType() snapshot_drive.snapshot_id = "snap-1234abcd" snapshot_drive.volume_type = "standard" block_device_mapping["/dev/xvdp"] = snapshot_drive ebs_drive = BlockDeviceType() ebs_drive.volume_type = "io1" ebs_drive.size = 100 ebs_drive.iops = 1000 ebs_drive.delete_on_termination = False block_device_mapping["/dev/xvdh"] = ebs_drive conn = boto.connect_autoscale(use_block_device_types=True) config = LaunchConfiguration( name="tester", image_id="ami-abcd1234", instance_type="m1.small", key_name="the_keys", security_groups=["default", "default2"], user_data=b"This is some user_data", instance_monitoring=True, instance_profile_name="arn:aws:iam::{}:instance-profile/testing".format( ACCOUNT_ID ), spot_price=0.1, block_device_mappings=[block_device_mapping], ) conn.create_launch_configuration(config) launch_config = conn.get_all_launch_configurations()[0] launch_config.name.should.equal("tester") launch_config.image_id.should.equal("ami-abcd1234") launch_config.instance_type.should.equal("m1.small") launch_config.key_name.should.equal("the_keys") set(launch_config.security_groups).should.equal(set(["default", "default2"])) launch_config.user_data.should.equal(b"This is some user_data") launch_config.instance_monitoring.enabled.should.equal("true") launch_config.instance_profile_name.should.equal( "arn:aws:iam::{}:instance-profile/testing".format(ACCOUNT_ID) ) launch_config.spot_price.should.equal(0.1) len(launch_config.block_device_mappings).should.equal(3) returned_mapping = launch_config.block_device_mappings set(returned_mapping.keys()).should.equal( set(["/dev/xvdb", "/dev/xvdp", "/dev/xvdh"]) ) returned_mapping["/dev/xvdh"].iops.should.equal(1000) returned_mapping["/dev/xvdh"].size.should.equal(100) returned_mapping["/dev/xvdh"].volume_type.should.equal("io1") returned_mapping["/dev/xvdh"].delete_on_termination.should.be.false returned_mapping["/dev/xvdp"].snapshot_id.should.equal("snap-1234abcd") returned_mapping["/dev/xvdp"].volume_type.should.equal("standard") returned_mapping["/dev/xvdb"].ephemeral_name.should.equal("ephemeral0")
def create_cluster(conn, args): if args.identity_file is None: print("ERROR: Must provide an identity file (-i) for ssh connections.", file=stderr) sys.exit(1) if args.key_pair is None: print("ERROR: Must provide a key pair name (-k) to use on instances.", file=stderr) sys.exit(1) # make or get the security group. security_group = get_or_make_group(conn, args.name, args.vpc_id) # set the inbound permission rules if len(security_group.rules) == 0: if __name__ == '__main__': if args.vpc_id is None: security_group.authorize(src_group=security_group) else: security_group.authorize('tcp', 22, 22, args.authorized_address) security_group.authorize('tcp', 8888, 8888, args.authorized_address) security_group.authorize('tcp', 7000, 7000, args.authorized_address) security_group.authorize('tcp', 7001, 7001, args.authorized_address) security_group.authorize('tcp', 7199, 7199, args.authorized_address) security_group.authorize('tcp', 9042, 9042, args.authorized_address) security_group.authorize('tcp', 9160, 9160, args.authorized_address) else: print("Security group already exists, skipping creation.") instances = cluster_nodes(conn, args.name) if any(instances): additional_tags = {} for i in instances: i.add_tags( dict(additional_tags, Name="{cn}-node-{iid}".format(cn=args.name, iid=i.id))) return instances else: print( "Launching {m} instances for cluster...".format(m=args.node_count)) try: image = conn.get_all_images(image_ids=args.ami)[0] block_map = BlockDeviceMapping() if args.ebs_vol_size > 0: if args.instance_type.startswith('m3.'): for i in range(get_num_disks(args.instance_type)): device = BlockDeviceType() device.ephemeral_name = "ephemeral%d" % i name = "/dev/sd" + string.ascii_letters[i + 1] block_map[name] = device else: device = EBSBlockDeviceType() device.size = args.ebs_vol_size device.volume_type = args.ebs_vol_type device.delete_on_termination = True key = "/dev/sd" + chr(ord('s') + 1) block_map[key] = device nodes = image.run(key_name=args.key_pair, security_group_ids=[security_group.id], instance_type="", placement=args.zone, min_count=args.node_count, max_count=args.node_count, block_device_map=block_map, subnet_id=None, placement_group=None, user_data=None, instance_initiated_shutdown_behavior="stop", instance_profile_name=None) print("Waiting for AWS to propagate instance metadata...") time.sleep(15) additional_tags = {} for node in nodes.instances: node.add_tags( dict(additional_tags, Name="{cn}-node-{iid}".format(cn=args.name, iid=node.id))) return nodes.instances except Exception as e: print("Caught exception: ", e) print("ERROR: Could not find AMI " + args.ami, file=stderr) sys.exit(1)
def test_create_launch_configuration_with_block_device_mappings(): block_device_mapping = BlockDeviceMapping() ephemeral_drive = BlockDeviceType() ephemeral_drive.ephemeral_name = 'ephemeral0' block_device_mapping['/dev/xvdb'] = ephemeral_drive snapshot_drive = BlockDeviceType() snapshot_drive.snapshot_id = "snap-1234abcd" snapshot_drive.volume_type = "standard" block_device_mapping['/dev/xvdp'] = snapshot_drive ebs_drive = BlockDeviceType() ebs_drive.volume_type = "io1" ebs_drive.size = 100 ebs_drive.iops = 1000 ebs_drive.delete_on_termination = False block_device_mapping['/dev/xvdh'] = ebs_drive conn = boto.connect_autoscale(use_block_device_types=True) config = LaunchConfiguration( name='tester', image_id='ami-abcd1234', instance_type='m1.small', key_name='the_keys', security_groups=["default", "default2"], user_data=b"This is some user_data", instance_monitoring=True, instance_profile_name= 'arn:aws:iam::123456789012:instance-profile/testing', spot_price=0.1, block_device_mappings=[block_device_mapping]) conn.create_launch_configuration(config) launch_config = conn.get_all_launch_configurations()[0] launch_config.name.should.equal('tester') launch_config.image_id.should.equal('ami-abcd1234') launch_config.instance_type.should.equal('m1.small') launch_config.key_name.should.equal('the_keys') set(launch_config.security_groups).should.equal( set(['default', 'default2'])) launch_config.user_data.should.equal(b"This is some user_data") launch_config.instance_monitoring.enabled.should.equal('true') launch_config.instance_profile_name.should.equal( 'arn:aws:iam::123456789012:instance-profile/testing') launch_config.spot_price.should.equal(0.1) len(launch_config.block_device_mappings).should.equal(3) returned_mapping = launch_config.block_device_mappings set(returned_mapping.keys()).should.equal( set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh'])) returned_mapping['/dev/xvdh'].iops.should.equal(1000) returned_mapping['/dev/xvdh'].size.should.equal(100) returned_mapping['/dev/xvdh'].volume_type.should.equal("io1") returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd") returned_mapping['/dev/xvdp'].volume_type.should.equal("standard") returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')