def component_stups_auto_configuration(definition, configuration, args, info, force):
vpc_conn = boto.vpc.connect_to_region(args.region)
availability_zones = configuration.get('AvailabilityZones')
server_subnets = []
lb_subnets = []
for subnet in vpc_conn.get_all_subnets():
name = subnet.tags.get('Name', '')
if availability_zones and subnet.availability_zone not in availability_zones:
# skip subnet as it's not in one of the given AZs
continue
if 'dmz' in name:
lb_subnets.append(subnet.id)
else:
server_subnets.append(subnet.id)
if not lb_subnets:
# no DMZ subnets were found, just use the same set for both LB and instances
lb_subnets = server_subnets
configuration = ensure_keys(configuration, "ServerSubnets", args.region)
configuration["ServerSubnets"][args.region] = server_subnets
configuration = ensure_keys(configuration, "LoadBalancerSubnets", args.region)
configuration["LoadBalancerSubnets"][args.region] = lb_subnets
most_recent_image = find_taupage_image(args.region)
configuration = ensure_keys(configuration, "Images", 'LatestTaupageImage', args.region)
configuration["Images"]['LatestTaupageImage'][args.region] = most_recent_image.id
component_configuration(definition, configuration, args, info, force)
return definition
def component_taupage_auto_scaling_group(definition, configuration, args, info, force, account_info):
# inherit from the normal auto scaling group but discourage user info and replace with a Taupage config
if 'Image' not in configuration:
configuration['Image'] = 'LatestTaupageImage'
definition = component_auto_scaling_group(definition, configuration, args, info, force, account_info)
taupage_config = configuration['TaupageConfig']
if 'notify_cfn' not in taupage_config:
taupage_config['notify_cfn'] = {'stack': '{}-{}'.format(info["StackName"], info["StackVersion"]),
'resource': configuration['Name']}
if 'application_id' not in taupage_config:
taupage_config['application_id'] = info['StackName']
if 'application_version' not in taupage_config:
taupage_config['application_version'] = info['StackVersion']
check_application_id(taupage_config['application_id'])
check_application_version(taupage_config['application_version'])
runtime = taupage_config.get('runtime')
if runtime != 'Docker':
raise click.UsageError('Taupage only supports the "Docker" runtime currently')
source = taupage_config.get('source')
if not source:
raise click.UsageError('The "source" property of TaupageConfig must be specified')
docker_image = pierone.api.DockerImage.parse(source)
if not force and docker_image.registry:
check_docker_image_exists(docker_image)
config_name = configuration["Name"] + "Config"
ensure_keys(definition, "Resources", config_name, "Properties")
properties = definition["Resources"][config_name]["Properties"]
mappings = definition.get('Mappings', {})
server_subnets = set(mappings.get('ServerSubnets', {}).get(args.region, {}).get('Subnets', []))
# in dmz or public subnet but without public ip
if server_subnets and not properties.get('AssociatePublicIpAddress') and server_subnets ==\
set(mappings.get('LoadBalancerInternalSubnets', {}).get(args.region, {}).get('Subnets', [])):
# we need to extend taupage_config with the mapping subnet-id => net ip
nat_gateways = {}
ec2 = boto3.client('ec2', args.region)
for nat_gateway in ec2.describe_nat_gateways()['NatGateways']:
if nat_gateway['SubnetId'] in server_subnets:
for address in nat_gateway['NatGatewayAddresses']:
nat_gateways[nat_gateway['SubnetId']] = address['PrivateIp']
break
if nat_gateways:
taupage_config['nat_gateways'] = nat_gateways
properties["UserData"] = {"Fn::Base64": generate_user_data(taupage_config, args.region)}
return definition
def component_subnet_auto_configuration(definition, configuration, args, info, force, account_info):
ec2 = boto3.resource('ec2', args.region)
vpc_id = configuration.get('VpcId', account_info.VpcID)
availability_zones = configuration.get('AvailabilityZones')
public_only = configuration.get('PublicOnly')
server_subnets = []
lb_subnets = []
lb_internal_subnets = []
all_subnets = []
for subnet in ec2.subnets.filter(Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]):
name = get_tag(subnet.tags, 'Name', '')
if availability_zones and subnet.availability_zone not in availability_zones:
# skip subnet as it's not in one of the given AZs
continue
all_subnets.append(subnet.id)
if public_only:
if 'dmz' in name:
lb_subnets.append(subnet.id)
lb_internal_subnets.append(subnet.id)
server_subnets.append(subnet.id)
else:
if 'dmz' in name:
lb_subnets.append(subnet.id)
elif 'internal' in name:
lb_internal_subnets.append(subnet.id)
server_subnets.append(subnet.id)
elif 'nat' in name:
# ignore creating listeners in NAT gateway subnets
pass
else:
server_subnets.append(subnet.id)
if not lb_subnets:
if public_only:
# assume default AWS VPC setup with all subnets being public
lb_subnets = all_subnets
lb_internal_subnets = all_subnets
server_subnets = all_subnets
else:
# no DMZ subnets were found, just use the same set for both LB and instances
lb_subnets = server_subnets
configuration = ensure_keys(configuration, "ServerSubnets", args.region)
configuration["ServerSubnets"][args.region] = server_subnets
configuration = ensure_keys(configuration, "LoadBalancerSubnets", args.region)
configuration["LoadBalancerSubnets"][args.region] = lb_subnets
configuration = ensure_keys(configuration, "LoadBalancerInternalSubnets", args.region)
configuration["LoadBalancerInternalSubnets"][args.region] = lb_internal_subnets
component_configuration(definition, configuration, args, info, force, account_info)
return definition
def component_redis_cluster(definition, configuration, args, info, force, account_info):
name = configuration["Name"]
definition = ensure_keys(definition, "Resources")
number_of_nodes = int(configuration.get('NumberOfNodes', '2'))
definition["Resources"]["RedisReplicationGroup"] = {
"Type": "AWS::ElastiCache::ReplicationGroup",
"Properties": {
"AutomaticFailoverEnabled": True,
"CacheNodeType": configuration.get('CacheNodeType', 'cache.t2.small'),
"CacheSubnetGroupName": {
"Ref": "RedisSubnetGroup"
},
"Engine": "redis",
"EngineVersion": configuration.get('EngineVersion', '2.8.19'),
"CacheParameterGroupName": configuration.get('CacheParameterGroupName', 'default.redis2.8'),
"NumCacheClusters": number_of_nodes,
"CacheNodeType": configuration.get('CacheNodeType', 'cache.t2.small'),
"SecurityGroupIds": resolve_security_groups(configuration["SecurityGroups"], args.region),
"ReplicationGroupDescription": "Redis replicated cache cluster: " + name,
}
}
definition["Resources"]["RedisSubnetGroup"] = {
"Type": "AWS::ElastiCache::SubnetGroup",
"Properties": {
"Description": "Redis cluster subnet group",
"SubnetIds": {"Fn::FindInMap": ["ServerSubnets", {"Ref": "AWS::Region"}, "Subnets"]}
}
}
return definition
def component_redis_node(definition, configuration, args, info, force, account_info):
name = configuration["Name"]
definition = ensure_keys(definition, "Resources")
definition["Resources"]["RedisCacheCluster"] = {
"Type": "AWS::ElastiCache::CacheCluster",
"Properties": {
"ClusterName": name,
"Engine": "redis",
"EngineVersion": configuration.get('EngineVersion', '2.8.19'),
"CacheParameterGroupName": configuration.get('CacheParameterGroupName', 'default.redis2.8'),
"NumCacheNodes": 1,
"CacheNodeType": configuration.get('CacheNodeType', 'cache.t2.small'),
"CacheSubnetGroupName": {
"Ref": "RedisSubnetGroup"
},
"VpcSecurityGroupIds": resolve_security_groups(configuration["SecurityGroups"], args.region)
}
}
definition["Resources"]["RedisSubnetGroup"] = {
"Type": "AWS::ElastiCache::SubnetGroup",
"Properties": {
"Description": "Redis cluster subnet group",
"SubnetIds": {"Fn::FindInMap": ["ServerSubnets", {"Ref": "AWS::Region"}, "Subnets"]}
}
}
return definition
def component_iam_role(definition, configuration, args, info, force, account_info):
definition = ensure_keys(definition, "Resources")
role_name = configuration["Name"]
definition["Resources"][role_name] = {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": configuration.get(
"AssumeRolePolicyDocument",
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {"Service": ["ec2.amazonaws.com"]},
"Action": ["sts:AssumeRole"],
}
],
},
),
"Path": configuration.get("Path", "/"),
"Policies": configuration.get("Policies", [])
+ get_merged_policies(configuration.get("MergePoliciesFromIamRoles", [])),
},
}
return definition
def component_stups_auto_configuration(definition, configuration, args, info,
force):
vpc_conn = boto.vpc.connect_to_region(args.region)
availability_zones = configuration.get('AvailabilityZones')
server_subnets = []
lb_subnets = []
lb_internal_subnets = []
for subnet in vpc_conn.get_all_subnets():
name = subnet.tags.get('Name', '')
if availability_zones and subnet.availability_zone not in availability_zones:
# skip subnet as it's not in one of the given AZs
continue
if 'dmz' in name:
lb_subnets.append(subnet.id)
elif 'internal' in name:
lb_internal_subnets.append(subnet.id)
server_subnets.append(subnet.id)
else:
server_subnets.append(subnet.id)
if not lb_subnets:
# no DMZ subnets were found, just use the same set for both LB and instances
lb_subnets = server_subnets
configuration = ensure_keys(configuration, "ServerSubnets", args.region)
configuration["ServerSubnets"][args.region] = server_subnets
configuration = ensure_keys(configuration, "LoadBalancerSubnets",
args.region)
configuration["LoadBalancerSubnets"][args.region] = lb_subnets
configuration = ensure_keys(configuration, "LoadBalancerInternalSubnets",
args.region)
configuration["LoadBalancerInternalSubnets"][
args.region] = lb_internal_subnets
most_recent_image = find_taupage_image(args.region)
configuration = ensure_keys(configuration, "Images", 'LatestTaupageImage',
args.region)
configuration["Images"]['LatestTaupageImage'][
args.region] = most_recent_image.id
component_configuration(definition, configuration, args, info, force)
return definition
def component_configuration(definition, configuration, args, info, force,
account_info):
# define parameters
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/parameters-section-structure.html
if "Parameters" in info:
definition = ensure_keys(definition, "Parameters")
default_parameter = {"Type": "String"}
for parameter in info["Parameters"]:
name, value = named_value(parameter)
value_default = default_parameter.copy()
value_default.update(value)
definition["Parameters"][name] = value_default
if 'Description' not in definition:
# set some sane default stack description
definition['Description'] = get_default_description(info, args)
# ServerSubnets
for region, subnets in configuration.get('ServerSubnets', {}).items():
definition = ensure_keys(definition, "Mappings", "ServerSubnets",
region)
definition["Mappings"]["ServerSubnets"][region]["Subnets"] = subnets
# LoadBalancerSubnets
for region, subnets in configuration.get('LoadBalancerSubnets',
{}).items():
definition = ensure_keys(definition, "Mappings", "LoadBalancerSubnets",
region)
definition["Mappings"]["LoadBalancerSubnets"][region][
"Subnets"] = subnets
# LoadBalancerInternalSubnets
for region, subnets in configuration.get('LoadBalancerInternalSubnets',
{}).items():
definition = ensure_keys(definition, "Mappings",
"LoadBalancerInternalSubnets", region)
definition["Mappings"]["LoadBalancerInternalSubnets"][region][
"Subnets"] = subnets
# Images
for name, image in configuration.get('Images', {}).items():
for region, ami in image.items():
definition = ensure_keys(definition, "Mappings", "Images", region,
name)
definition["Mappings"]["Images"][region][name] = ami
return definition
def component_coreos_auto_configuration(definition, configuration, args, info, force, account_info):
ami_id = find_coreos_image(configuration.get('ReleaseChannel') or 'stable', args.region)
configuration = ensure_keys(configuration, "Images", 'LatestCoreOSImage', args.region)
configuration["Images"]['LatestCoreOSImage'][args.region] = ami_id
component_subnet_auto_configuration(definition, configuration, args, info, force, account_info)
return definition
def component_stups_auto_configuration(definition, configuration, args, info, force, account_info):
most_recent_image = find_taupage_image(args.region)
configuration = ensure_keys(configuration, "Images", 'LatestTaupageImage', args.region)
configuration["Images"]['LatestTaupageImage'][args.region] = most_recent_image.id
component_subnet_auto_configuration(definition, configuration, args, info, force, account_info)
return definition
def component_taupage_auto_scaling_group(definition, configuration, args, info,
force, account_info):
# inherit from the normal auto scaling group but discourage user info and replace with a Taupage config
if 'Image' not in configuration:
configuration['Image'] = 'LatestTaupageImage'
definition = component_auto_scaling_group(definition, configuration, args,
info, force, account_info)
taupage_config = configuration['TaupageConfig']
if 'notify_cfn' not in taupage_config:
taupage_config['notify_cfn'] = {
'stack': '{}-{}'.format(info["StackName"], info["StackVersion"]),
'resource': configuration['Name']
}
if 'application_id' not in taupage_config:
taupage_config['application_id'] = info['StackName']
if 'application_version' not in taupage_config:
taupage_config['application_version'] = info['StackVersion']
runtime = taupage_config.get('runtime')
if runtime != 'Docker':
raise click.UsageError(
'Taupage only supports the "Docker" runtime currently')
source = taupage_config.get('source')
if not source:
raise click.UsageError(
'The "source" property of TaupageConfig must be specified')
docker_image = pierone.api.DockerImage.parse(source)
if not force and docker_image.registry:
check_docker_image_exists(docker_image)
userdata = generate_user_data(taupage_config, args.region)
config_name = configuration["Name"] + "Config"
ensure_keys(definition, "Resources", config_name, "Properties", "UserData")
definition["Resources"][config_name]["Properties"]["UserData"][
"Fn::Base64"] = userdata
return definition
def ensure_instance_monitoring(elastigroup_config):
"""
This functions will set the monitoring property to True if not set already in the compute.launchSpecification
section. This enables EC2 enhanced monitoring, which is also the general STUPS behavior
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
if "monitoring" in elastigroup_config["compute"]["launchSpecification"]:
return
elastigroup_config["compute"]["launchSpecification"]["monitoring"] = True
def ensure_default_product(elastigroup_config):
"""
This function ensures that the compute.product attribute for the Elastigroup is defined with a default value.
See ELASTIGROUP_DEFAULT_PRODUCT
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute")
if "product" in elastigroup_config["compute"]:
return
elastigroup_config["compute"]["product"] = ELASTIGROUP_DEFAULT_PRODUCT
def ensure_default_product(elastigroup_config):
"""
This function ensures that the compute.product attribute for the Elastigroup is defined with a default value.
See ELASTIGROUP_DEFAULT_PRODUCT
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute")
if "product" in elastigroup_config["compute"]:
return
elastigroup_config["compute"]["product"] = ELASTIGROUP_DEFAULT_PRODUCT
def ensure_instance_monitoring(elastigroup_config):
"""
This functions will set the monitoring property to True if not set already in the compute.launchSpecification
section. This enables EC2 enhanced monitoring, which is also the general STUPS behavior
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
if "monitoring" in elastigroup_config["compute"]["launchSpecification"]:
return
elastigroup_config["compute"]["launchSpecification"]["monitoring"] = True
def extract_image_id(elastigroup_config: dict):
"""
This function identifies whether a senza formatted AMI mapping is configured,
if so it transforms it into a Spotinst Elastigroup AMI API configuration
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
launch_spec_config = elastigroup_config["compute"]["launchSpecification"]
if "imageId" not in launch_spec_config.keys():
launch_spec_config["imageId"] = {"Fn::FindInMap": ["Images", {"Ref": "AWS::Region"}, "LatestTaupageImage"]}
def extract_image_id(elastigroup_config: dict):
"""
This function identifies whether a senza formatted AMI mapping is configured,
if so it transforms it into a Spotinst Elastigroup AMI API configuration
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
launch_spec_config = elastigroup_config["compute"]["launchSpecification"]
if "imageId" not in launch_spec_config.keys():
launch_spec_config["imageId"] = {"Fn::FindInMap": ["Images", {"Ref": "AWS::Region"}, "LatestTaupageImage"]}
def component_stups_auto_configuration(definition, configuration, args, info, force, account_info):
ec2 = boto3.resource('ec2', args.region)
vpc_id = configuration.get('VpcId', account_info.VpcID)
availability_zones = configuration.get('AvailabilityZones')
server_subnets = []
lb_subnets = []
lb_internal_subnets = []
for subnet in ec2.subnets.filter(Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]):
name = get_tag(subnet.tags, 'Name', '')
if availability_zones and subnet.availability_zone not in availability_zones:
# skip subnet as it's not in one of the given AZs
continue
if 'dmz' in name:
lb_subnets.append(subnet.id)
elif 'internal' in name:
lb_internal_subnets.append(subnet.id)
server_subnets.append(subnet.id)
else:
server_subnets.append(subnet.id)
if not lb_subnets:
# no DMZ subnets were found, just use the same set for both LB and instances
lb_subnets = server_subnets
configuration = ensure_keys(configuration, "ServerSubnets", args.region)
configuration["ServerSubnets"][args.region] = server_subnets
configuration = ensure_keys(configuration, "LoadBalancerSubnets", args.region)
configuration["LoadBalancerSubnets"][args.region] = lb_subnets
configuration = ensure_keys(configuration, "LoadBalancerInternalSubnets", args.region)
configuration["LoadBalancerInternalSubnets"][args.region] = lb_internal_subnets
most_recent_image = find_taupage_image(args.region)
configuration = ensure_keys(configuration, "Images", 'LatestTaupageImage', args.region)
configuration["Images"]['LatestTaupageImage'][args.region] = most_recent_image.id
component_configuration(definition, configuration, args, info, force, account_info)
return definition
def component_configuration(definition, configuration, args, info, force, account_info):
# define parameters
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/parameters-section-structure.html
if "Parameters" in info and configuration.get('DefineParameters', True):
definition = ensure_keys(definition, "Parameters")
default_parameter = {
"Type": "String"
}
for parameter in info["Parameters"]:
name, value = named_value(parameter)
value_default = default_parameter.copy()
value_default.update(value)
definition["Parameters"][name] = value_default
if 'Description' not in definition:
# set some sane default stack description
# we need to truncate at 1024 chars (should be Bytes actually)
# see http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-description-structure.html
definition['Description'] = get_default_description(info, args)[:1024]
# ServerSubnets
for region, subnets in configuration.get('ServerSubnets', {}).items():
definition = ensure_keys(definition, "Mappings", "ServerSubnets", region)
definition["Mappings"]["ServerSubnets"][region]["Subnets"] = subnets
# LoadBalancerSubnets
for region, subnets in configuration.get('LoadBalancerSubnets', {}).items():
definition = ensure_keys(definition, "Mappings", "LoadBalancerSubnets", region)
definition["Mappings"]["LoadBalancerSubnets"][region]["Subnets"] = subnets
# LoadBalancerInternalSubnets
for region, subnets in configuration.get('LoadBalancerInternalSubnets', {}).items():
definition = ensure_keys(definition, "Mappings", "LoadBalancerInternalSubnets", region)
definition["Mappings"]["LoadBalancerInternalSubnets"][region]["Subnets"] = subnets
# Images
for name, image in configuration.get('Images', {}).items():
for region, ami in image.items():
definition = ensure_keys(definition, "Mappings", "Images", region, name)
definition["Mappings"]["Images"][region][name] = ami
return definition
def component_taupage_auto_scaling_group(definition, configuration, args, info, force, account_info):
# inherit from the normal auto scaling group but discourage user info and replace with a Taupage config
if 'Image' not in configuration:
configuration['Image'] = 'LatestTaupageImage'
definition = component_auto_scaling_group(definition, configuration, args, info, force, account_info)
taupage_config = configuration['TaupageConfig']
if 'notify_cfn' not in taupage_config:
taupage_config['notify_cfn'] = {'stack': '{}-{}'.format(info["StackName"], info["StackVersion"]),
'resource': configuration['Name']}
if 'application_id' not in taupage_config:
taupage_config['application_id'] = info['StackName']
if 'application_version' not in taupage_config:
taupage_config['application_version'] = info['StackVersion']
check_application_id(taupage_config['application_id'])
check_application_version(taupage_config['application_version'])
runtime = taupage_config.get('runtime')
if runtime != 'Docker':
raise click.UsageError('Taupage only supports the "Docker" runtime currently')
source = taupage_config.get('source')
if not source:
raise click.UsageError('The "source" property of TaupageConfig must be specified')
docker_image = pierone.api.DockerImage.parse(source)
if not force and docker_image.registry:
check_docker_image_exists(docker_image)
userdata = generate_user_data(taupage_config, args.region)
config_name = configuration["Name"] + "Config"
ensure_keys(definition, "Resources", config_name, "Properties", "UserData")
definition["Resources"][config_name]["Properties"]["UserData"]["Fn::Base64"] = userdata
return definition
def extract_subnets(configuration, elastigroup_config, account_info):
"""
This fills in the subnetIds and region attributes of the Spotinst elastigroup, in case they're not defined already
The subnetIds are discovered by Senza::StupsAutoConfiguration and the region is provided by the AccountInfo object
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute")
subnet_ids = elastigroup_config["compute"].get("subnetIds", [])
target_region = elastigroup_config.get("region", account_info.Region)
if not subnet_ids:
subnet_set = "LoadBalancerSubnets" if configuration.get("AssociatePublicIpAddress", False) else "ServerSubnets"
elastigroup_config["compute"]["subnetIds"] = {"Fn::FindInMap": [subnet_set, {"Ref": "AWS::Region"}, "Subnets"]}
elastigroup_config["region"] = target_region
def component_stups_auto_configuration(definition, configuration, args, info,
force, account_info):
most_recent_image = find_taupage_image(args.region)
configuration = ensure_keys(configuration, "Images", 'LatestTaupageImage',
args.region)
configuration["Images"]['LatestTaupageImage'][
args.region] = most_recent_image.id
component_subnet_auto_configuration(definition, configuration, args, info,
force, account_info)
return definition
def component_coreos_auto_configuration(definition, configuration, args, info,
force, account_info):
ami_id = find_coreos_image(
configuration.get('ReleaseChannel') or 'stable', args.region)
configuration = ensure_keys(configuration, "Images", 'LatestCoreOSImage',
args.region)
configuration["Images"]['LatestCoreOSImage'][args.region] = ami_id
component_subnet_auto_configuration(definition, configuration, args, info,
force, account_info)
return definition
def extract_subnets(definition, elastigroup_config, account_info):
"""
This fills in the subnetIds and region attributes of the Spotinst elastigroup, in case they're not defined already
The subnetIds are discovered by Senza::StupsAutoConfiguration and the region is provided by the AccountInfo object
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute")
subnet_ids = elastigroup_config["compute"].get("subnetIds", [])
target_region = elastigroup_config.get("region", account_info.Region)
if not subnet_ids:
subnet_ids = definition["Mappings"]["ServerSubnets"].get(target_region, {}).get("Subnets", [])
elastigroup_config["region"] = target_region
elastigroup_config["compute"]["subnetIds"] = subnet_ids
def component_configuration(definition, configuration, args, info, force):
# add info as mappings
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/mappings-section-structure.html
definition = ensure_keys(definition, "Mappings", "Senza", "Info")
definition["Mappings"]["Senza"]["Info"] = info
# define parameters
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/parameters-section-structure.html
if "Parameters" in info:
definition = ensure_keys(definition, "Parameters")
default_parameter = {
"Type": "String"
}
for parameter in info["Parameters"]:
name, value = named_value(parameter)
value_default = default_parameter.copy()
value_default.update(value)
definition["Parameters"][name] = value_default
if 'Description' not in definition:
# set some sane default stack description
definition['Description'] = get_default_description(info, args)
# ServerSubnets
for region, subnets in configuration.get('ServerSubnets', {}).items():
definition = ensure_keys(definition, "Mappings", "ServerSubnets", region)
definition["Mappings"]["ServerSubnets"][region]["Subnets"] = subnets
# LoadBalancerSubnets
for region, subnets in configuration.get('LoadBalancerSubnets', {}).items():
definition = ensure_keys(definition, "Mappings", "LoadBalancerSubnets", region)
definition["Mappings"]["LoadBalancerSubnets"][region]["Subnets"] = subnets
# Images
for name, image in configuration.get('Images', {}).items():
for region, ami in image.items():
definition = ensure_keys(definition, "Mappings", "Images", region, name)
definition["Mappings"]["Images"][region][name] = ami
return definition
def component_taupage_auto_scaling_group(definition, configuration, args, info, force, account_info):
# inherit from the normal auto scaling group but discourage user info and replace with a Taupage config
if "Image" not in configuration:
configuration["Image"] = "LatestTaupageImage"
definition = component_auto_scaling_group(definition, configuration, args, info, force, account_info)
taupage_config = configuration["TaupageConfig"]
if "notify_cfn" not in taupage_config:
taupage_config["notify_cfn"] = {
"stack": "{}-{}".format(info["StackName"], info["StackVersion"]),
"resource": configuration["Name"],
}
if "application_id" not in taupage_config:
taupage_config["application_id"] = info["StackName"]
if "application_version" not in taupage_config:
taupage_config["application_version"] = info["StackVersion"]
runtime = taupage_config.get("runtime")
if runtime != "Docker":
raise click.UsageError('Taupage only supports the "Docker" runtime currently')
source = taupage_config.get("source")
if not source:
raise click.UsageError('The "source" property of TaupageConfig must be specified')
docker_image = pierone.api.DockerImage.parse(source)
if not force and docker_image.registry:
check_docker_image_exists(docker_image)
userdata = generate_user_data(taupage_config)
config_name = configuration["Name"] + "Config"
ensure_keys(definition, "Resources", config_name, "Properties", "UserData")
definition["Resources"][config_name]["Properties"]["UserData"]["Fn::Base64"] = userdata
return definition
def component_stups_auto_configuration(definition, configuration, args, info, force, account_info):
for channel in taupage.CHANNELS.values():
most_recent_image = taupage.find_image(args.region, channel)
if most_recent_image:
configuration = ensure_keys(configuration, "Images", channel.image_mapping, args.region)
configuration["Images"][channel.image_mapping][args.region] = most_recent_image.id
elif channel == taupage.DEFAULT_CHANNEL:
# Require at least one image from the stable channel
raise Exception('No Taupage AMI found')
component_subnet_auto_configuration(definition, configuration, args, info, force, account_info)
return definition
def component_stups_auto_configuration(definition, configuration, args, info,
force, account_info):
for mapping, suffix in _TAUPAGE_CHANNELS.items():
most_recent_image = find_taupage_image(args.region, suffix)
if most_recent_image:
configuration = ensure_keys(configuration, "Images", mapping,
args.region)
configuration["Images"][mapping][
args.region] = most_recent_image.id
component_subnet_auto_configuration(definition, configuration, args, info,
force, account_info)
return definition
def component_redis_cluster(definition, configuration, args, info, force,
account_info):
name = configuration["Name"]
definition = ensure_keys(definition, "Resources")
number_of_nodes = int(configuration.get('NumberOfNodes', '2'))
definition["Resources"]["RedisReplicationGroup"] = {
"Type": "AWS::ElastiCache::ReplicationGroup",
"Properties": {
"AutomaticFailoverEnabled":
True,
"CacheNodeType":
configuration.get('CacheNodeType', 'cache.t2.small'),
"CacheSubnetGroupName": {
"Ref": "RedisSubnetGroup"
},
"Engine":
"redis",
"EngineVersion":
configuration.get('EngineVersion', '2.8.19'),
"CacheParameterGroupName":
configuration.get('CacheParameterGroupName', 'default.redis2.8'),
"NumCacheClusters":
number_of_nodes,
"CacheNodeType":
configuration.get('CacheNodeType', 'cache.t2.small'),
"SecurityGroupIds":
resolve_security_groups(configuration["SecurityGroups"],
args.region),
"ReplicationGroupDescription":
"Redis replicated cache cluster: " + name,
}
}
definition["Resources"]["RedisSubnetGroup"] = {
"Type": "AWS::ElastiCache::SubnetGroup",
"Properties": {
"Description": "Redis cluster subnet group",
"SubnetIds": {
"Fn::FindInMap":
["ServerSubnets", {
"Ref": "AWS::Region"
}, "Subnets"]
}
}
}
return definition
def component_elastigroup(definition, configuration, args, info, force, account_info):
"""
This component creates a Spotinst Elastigroup CloudFormation custom resource template.
- For a high level overview see; https://spotinst.com/workload-management/elastigroup/
- For the API reference see; http://api.spotinst.com/elastigroup/amazon-web-services/create/
- For the CloudFormation integration see; http://blog.spotinst.com/2016/04/05/elastigroup-cloudformation/
"""
definition = ensure_keys(ensure_keys(definition, "Resources"), "Mappings", "Senza", "Info")
if "SpotinstAccessToken" not in definition["Mappings"]["Senza"]["Info"]:
raise click.UsageError("You have to specify your SpotinstAccessToken attribute inside the SenzaInfo "
"to be able to use Elastigroups")
configuration = ensure_keys(configuration, "Elastigroup")
# launch configuration
elastigroup_config = configuration["Elastigroup"]
ensure_keys(elastigroup_config, "scheduling")
ensure_keys(elastigroup_config, "thirdPartiesIntegration")
fill_standard_tags(definition, elastigroup_config)
ensure_default_strategy(elastigroup_config)
ensure_default_product(elastigroup_config)
ensure_instance_monitoring(elastigroup_config)
extract_subnets(configuration, elastigroup_config, account_info)
extract_user_data(configuration, elastigroup_config, info, force, account_info)
extract_load_balancer_name(configuration, elastigroup_config)
extract_public_ips(configuration, elastigroup_config)
extract_image_id(elastigroup_config)
extract_security_group_ids(configuration, elastigroup_config, args)
extract_instance_types(configuration, elastigroup_config)
extract_autoscaling_capacity(configuration, elastigroup_config)
extract_auto_scaling_rules(configuration, elastigroup_config)
extract_block_mappings(configuration, elastigroup_config)
extract_instance_profile(args, definition, configuration, elastigroup_config)
patch_cross_stack_policy(definition, elastigroup_config)
# cfn definition
access_token = _extract_spotinst_access_token(definition)
config_name = configuration["Name"]
definition["Resources"][config_name] = {
"Type": ELASTIGROUP_RESOURCE_TYPE,
"Properties": {
"ServiceToken": create_service_token(args.region),
"accessToken": access_token,
"accountId": extract_spotinst_account_id(access_token, definition, account_info),
"group": elastigroup_config
}
}
if "SpotPrice" in configuration:
print("warning: SpotPrice is ignored when using Senza::Elastigroup", file=sys.stderr)
return definition
def extract_user_data(configuration, elastigroup_config, info: dict, force,
account_info):
"""
This function converts a classic TaupageConfig into a base64 encoded value for the
compute.launchSpecification.userData
See https://api.spotinst.com/elastigroup/amazon-web-services/create/#compute.launchSpecification.userData
Any existing TaupageConfig will _always_ overwrite the userData for the Elastigroup
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute",
"launchSpecification")
taupage_config = configuration.get("TaupageConfig", None)
if taupage_config:
if 'notify_cfn' not in taupage_config:
taupage_config['notify_cfn'] = {
'stack': '{}-{}'.format(info["StackName"],
info["StackVersion"]),
'resource': configuration['Name']
}
if 'application_id' not in taupage_config:
taupage_config['application_id'] = info['StackName']
if 'application_version' not in taupage_config:
taupage_config['application_version'] = info['StackVersion']
check_application_id(taupage_config['application_id'])
check_application_version(taupage_config['application_version'])
runtime = taupage_config.get('runtime')
if runtime != 'Docker':
raise click.UsageError(
'Taupage only supports the "Docker" runtime currently')
source = taupage_config.get('source')
if not source:
raise click.UsageError(
'The "source" property of TaupageConfig must be specified')
docker_image = pierone.api.DockerImage.parse(source)
if not force and docker_image.registry:
check_docker_image_exists(docker_image)
user_data = base64.urlsafe_b64encode(
generate_user_data(taupage_config,
account_info.Region).encode('utf-8'))
elastigroup_config["compute"]["launchSpecification"][
"userData"] = user_data.decode('utf-8')
def component_elastigroup(definition, configuration, args, info, force, account_info):
"""
This component creates a Spotinst Elastigroup CloudFormation custom resource template.
- For a high level overview see; https://spotinst.com/workload-management/elastigroup/
- For the API reference see; http://api.spotinst.com/elastigroup/amazon-web-services/create/
- For the CloudFormation integration see; http://blog.spotinst.com/2016/04/05/elastigroup-cloudformation/
"""
definition = ensure_keys(ensure_keys(definition, "Resources"), "Mappings", "Senza", "Info")
if "SpotinstAccessToken" not in definition["Mappings"]["Senza"]["Info"]:
raise click.UsageError("You have to specify your SpotinstAccessToken attribute inside the SenzaInfo "
"to be able to use Elastigroups")
configuration = ensure_keys(configuration, "Elastigroup")
# launch configuration
elastigroup_config = configuration["Elastigroup"]
ensure_keys(elastigroup_config, "scheduling")
ensure_keys(elastigroup_config, "thirdPartiesIntegration")
fill_standard_tags(definition, elastigroup_config)
ensure_default_strategy(elastigroup_config)
ensure_default_product(elastigroup_config)
ensure_instance_monitoring(elastigroup_config)
extract_subnets(definition, elastigroup_config, account_info)
extract_user_data(configuration, elastigroup_config, info, force, account_info)
extract_load_balancer_name(configuration, elastigroup_config)
extract_public_ips(configuration, elastigroup_config)
extract_image_id(elastigroup_config)
extract_security_group_ids(configuration, elastigroup_config, args)
extract_instance_types(configuration, elastigroup_config)
extract_autoscaling_capacity(configuration, elastigroup_config)
extract_auto_scaling_rules(configuration, elastigroup_config)
extract_block_mappings(configuration, elastigroup_config)
extract_instance_profile(args, definition, configuration, elastigroup_config)
# cfn definition
access_token = _extract_spotinst_access_token(definition)
config_name = configuration["Name"]
definition["Resources"][config_name] = {
"Type": ELASTIGROUP_RESOURCE_TYPE,
"Properties": {
"ServiceToken": create_service_token(args.region),
"accessToken": access_token,
"accountId": extract_spotinst_account_id(access_token, definition, account_info),
"group": elastigroup_config
}
}
if "SpotPrice" in configuration:
print("warning: SpotPrice is ignored when using Senza::Elastigroup", file=sys.stderr)
return definition
def extract_public_ips(configuration, elastigroup_config):
"""
This function will setup the Spotinst Elastigroup to use Public IPs if the
Senza AssociatePublicIpAddress is set to True.
If there's already a compute.launchSpecification.networkInterfaces config it is left untouched
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
if configuration.pop("AssociatePublicIpAddress", False):
launch_spec_config = elastigroup_config["compute"]["launchSpecification"]
if "networkInterfaces" not in launch_spec_config.keys():
launch_spec_config["networkInterfaces"] = [
{
"deleteOnTermination": True,
"deviceIndex": 0,
"associatePublicIpAddress": True
}
]
def extract_public_ips(configuration, elastigroup_config):
"""
This function will setup the Spotinst Elastigroup to use Public IPs if the
Senza AssociatePublicIpAddress is set to True.
If there's already a compute.launchSpecification.networkInterfaces config it is left untouched
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
if configuration.pop("AssociatePublicIpAddress", False):
launch_spec_config = elastigroup_config["compute"]["launchSpecification"]
if "networkInterfaces" not in launch_spec_config.keys():
launch_spec_config["networkInterfaces"] = [
{
"deleteOnTermination": True,
"deviceIndex": 0,
"associatePublicIpAddress": True
}
]
def fill_standard_tags(definition, elastigroup_config):
"""
This function adds the default STUPS EC2 Tags when none are defined in the Elastigroup. It also sets the
Elastigroup name attribute to the same value as the EC2 Name tag if found empty.
The default STUPS EC2 Tags are Name, StackName and StackVersion
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
name = definition["Mappings"]["Senza"]["Info"]["StackName"]
version = definition["Mappings"]["Senza"]["Info"]["StackVersion"]
full_name = "{}-{}".format(name, version)
if "tags" not in elastigroup_config["compute"]["launchSpecification"]:
elastigroup_config["compute"]["launchSpecification"]["tags"] = [
{"tagKey": "Name", "tagValue": full_name},
{"tagKey": "StackName", "tagValue": name},
{"tagKey": "StackVersion", "tagValue": version}
]
if elastigroup_config.get("name", "") == "":
elastigroup_config["name"] = full_name
def fill_standard_tags(definition, elastigroup_config):
"""
This function adds the default STUPS EC2 Tags when none are defined in the Elastigroup. It also sets the
Elastigroup name attribute to the same value as the EC2 Name tag if found empty.
The default STUPS EC2 Tags are Name, StackName and StackVersion
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
name = definition["Mappings"]["Senza"]["Info"]["StackName"]
version = definition["Mappings"]["Senza"]["Info"]["StackVersion"]
full_name = "{}-{}".format(name, version)
if "tags" not in elastigroup_config["compute"]["launchSpecification"]:
elastigroup_config["compute"]["launchSpecification"]["tags"] = [
{"tagKey": "Name", "tagValue": full_name},
{"tagKey": "StackName", "tagValue": name},
{"tagKey": "StackVersion", "tagValue": version}
]
if elastigroup_config.get("name", "") == "":
elastigroup_config["name"] = full_name
def component_redis_node(definition, configuration, args, info, force):
name = configuration["Name"]
definition = ensure_keys(definition, "Resources")
definition["Resources"]["RedisCacheCluster"] = {
"Type": "AWS::ElastiCache::CacheCluster",
"Properties": {
"ClusterName":
name,
"Engine":
"redis",
"EngineVersion":
configuration.get('EngineVersion', '2.8.19'),
"CacheParameterGroupName":
configuration.get('CacheParameterGroupName', 'default.redis2.8'),
"NumCacheNodes":
1,
"CacheNodeType":
configuration.get('CacheNodeType', 'cache.t2.small'),
"CacheSubnetGroupName": {
"Ref": "RedisSubnetGroup"
},
"VpcSecurityGroupIds":
resolve_security_groups(configuration["SecurityGroups"],
args.region)
}
}
definition["Resources"]["RedisSubnetGroup"] = {
"Type": "AWS::ElastiCache::SubnetGroup",
"Properties": {
"Description": "Redis cluster subnet group",
"SubnetIds": {
"Fn::FindInMap":
["ServerSubnets", {
"Ref": "AWS::Region"
}, "Subnets"]
}
}
}
return definition
def extract_instance_types(configuration, elastigroup_config):
"""
This function will set up the Elastigroup instance type, both for on-demand and spot. If there
are no SpotAlternatives the Elastigroup will have the same ondemand type as spot alternative
If there's already a compute.instanceTypes config it will be left untouched
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute")
compute_config = elastigroup_config["compute"]
if "InstanceType" not in configuration:
raise click.UsageError("You need to specify the InstanceType attribute to be able to use Elastigroups")
instance_type = configuration.pop("InstanceType")
spot_alternatives = configuration.pop("SpotAlternatives", None)
if "instanceTypes" not in compute_config:
instance_types = {}
instance_types.update({"ondemand": instance_type})
if spot_alternatives:
instance_types.update({"spot": spot_alternatives})
else:
instance_types.update({"spot": [instance_type]})
compute_config["instanceTypes"] = instance_types
def extract_instance_types(configuration, elastigroup_config):
"""
This function will set up the Elastigroup instance type, both for on-demand and spot. If there
are no SpotAlternatives the Elastigroup will have the same ondemand type as spot alternative
If there's already a compute.instanceTypes config it will be left untouched
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute")
compute_config = elastigroup_config["compute"]
if "InstanceType" not in configuration:
raise click.UsageError("You need to specify the InstanceType attribute to be able to use Elastigroups")
instance_type = configuration.pop("InstanceType")
spot_alternatives = configuration.pop("SpotAlternatives", None)
if "instanceTypes" not in compute_config:
instance_types = {}
instance_types.update({"ondemand": instance_type})
if spot_alternatives:
instance_types.update({"spot": spot_alternatives})
else:
instance_types.update({"spot": [instance_type]})
compute_config["instanceTypes"] = instance_types
def extract_security_group_ids(configuration, elastigroup_config: dict, args):
"""
This function identifies whether a senza formatted EC2-sg (by name) is configured,
if so it transforms it into a Spotinst Elastigroup EC2-sq (by id) API configuration
If there's already a compute.launchSpecification.securityGroupIds config it's left unchanged
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
launch_spec_config = elastigroup_config["compute"]["launchSpecification"]
security_group_ids = []
if "securityGroupIds" not in launch_spec_config.keys():
if "SecurityGroups" in configuration.keys():
security_groups_ref = configuration.pop("SecurityGroups")
if isinstance(security_groups_ref, str):
security_group_ids = resolve_security_groups([security_groups_ref], args.region)
elif isinstance(security_groups_ref, list):
security_group_ids = resolve_security_groups(security_groups_ref, args.region)
if len(security_group_ids) > 0:
launch_spec_config["securityGroupIds"] = security_group_ids
def extract_instance_profile(args, definition, configuration, elastigroup_config):
"""
Resolves the Senza IAM role or instance profile into the appropriate Spotinst launchSpecification.iamRole
settings.
If only IAM roles are specified, a new instance profile is created and the Elastigroup definition will have a
reference to the newly created instance profile.
If the launchSpecification already has the iamRole defined it is left untouched
When the Senza manifest includes both the IAMRoles and the IamInstanceProfile attributes the IAMRoles takes
precedence.
The IamInstanceProfile can specify either the ARN or just the instance profile name. This function will accept both
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
launch_spec = elastigroup_config["compute"]["launchSpecification"]
if "iamRole" in launch_spec:
return
if "IamRoles" in configuration:
logical_id = senza.components.auto_scaling_group.handle_iam_roles(definition, configuration, args)
launch_spec["iamRole"] = {"name": {"Ref": logical_id}}
elif "IamInstanceProfile" in configuration:
logical_id = configuration["IamInstanceProfile"]
attribute = "arn" if logical_id.startswith("arn:aws:iam::") else "name"
launch_spec["iamRole"] = {attribute: logical_id}
def extract_security_group_ids(configuration, elastigroup_config: dict, args):
"""
This function identifies whether a senza formatted EC2-sg (by name) is configured,
if so it transforms it into a Spotinst Elastigroup EC2-sq (by id) API configuration
If there's already a compute.launchSpecification.securityGroupIds config it's left unchanged
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
launch_spec_config = elastigroup_config["compute"]["launchSpecification"]
security_group_ids = []
if "securityGroupIds" not in launch_spec_config.keys():
if "SecurityGroups" in configuration.keys():
security_groups_ref = configuration.pop("SecurityGroups")
if isinstance(security_groups_ref, str):
security_group_ids = resolve_security_groups([security_groups_ref], args.region)
elif isinstance(security_groups_ref, list):
security_group_ids = resolve_security_groups(security_groups_ref, args.region)
if len(security_group_ids) > 0:
launch_spec_config["securityGroupIds"] = security_group_ids
def extract_instance_profile(args, definition, configuration, elastigroup_config):
"""
Resolves the Senza IAM role or instance profile into the appropriate Spotinst launchSpecification.iamRole
settings.
If only IAM roles are specified, a new instance profile is created and the Elastigroup definition will have a
reference to the newly created instance profile.
If the launchSpecification already has the iamRole defined it is left untouched
When the Senza manifest includes both the IAMRoles and the IamInstanceProfile attributes the IAMRoles takes
precedence.
The IamInstanceProfile can specify either the ARN or just the instance profile name. This function will accept both
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
launch_spec = elastigroup_config["compute"]["launchSpecification"]
if "iamRole" in launch_spec:
return
if "IamRoles" in configuration:
logical_id = senza.components.auto_scaling_group.handle_iam_roles(definition, configuration, args)
launch_spec["iamRole"] = {"name": {"Ref": logical_id}}
elif "IamInstanceProfile" in configuration:
logical_id = configuration["IamInstanceProfile"]
attribute = "arn" if logical_id.startswith("arn:aws:iam::") else "name"
launch_spec["iamRole"] = {attribute: logical_id}
def extract_user_data(configuration, elastigroup_config, info: dict, force, account_info):
"""
This function converts a classic TaupageConfig into a base64 encoded value for the
compute.launchSpecification.userData
See https://api.spotinst.com/elastigroup/amazon-web-services/create/#compute.launchSpecification.userData
Any existing TaupageConfig will _always_ overwrite the userData for the Elastigroup
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
taupage_config = configuration.get("TaupageConfig", None)
if taupage_config:
if 'notify_cfn' not in taupage_config:
taupage_config['notify_cfn'] = {'stack': '{}-{}'.format(info["StackName"], info["StackVersion"]),
'resource': configuration['Name']}
if 'application_id' not in taupage_config:
taupage_config['application_id'] = info['StackName']
if 'application_version' not in taupage_config:
taupage_config['application_version'] = info['StackVersion']
check_application_id(taupage_config['application_id'])
check_application_version(taupage_config['application_version'])
runtime = taupage_config.get('runtime')
if runtime != 'Docker':
raise click.UsageError('Taupage only supports the "Docker" runtime currently')
source = taupage_config.get('source')
if not source:
raise click.UsageError('The "source" property of TaupageConfig must be specified')
docker_image = pierone.api.DockerImage.parse(source)
if not force and docker_image.registry:
check_docker_image_exists(docker_image)
user_data = base64.urlsafe_b64encode(generate_user_data(taupage_config, account_info.Region).encode('utf-8'))
elastigroup_config["compute"]["launchSpecification"]["userData"] = user_data.decode('utf-8')
def fill_standard_tags(definition, elastigroup_config):
"""
This function adds the default STUPS EC2 Tags when none are defined in the Elastigroup. It also sets the
Elastigroup name attribute to the same value as the EC2 Name tag if found empty.
The default STUPS EC2 Tags are Name, StackName and StackVersion
"""
# Tag keys are case-sensitive: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html
standard_tags = {"Name", "StackName", "StackVersion"}
elastigroup_config = ensure_keys(elastigroup_config, "compute",
"launchSpecification")
name = definition["Mappings"]["Senza"]["Info"]["StackName"]
version = definition["Mappings"]["Senza"]["Info"]["StackVersion"]
full_name = "{}-{}".format(name, version)
tags = []
if "tags" in elastigroup_config["compute"]["launchSpecification"]:
tags = elastigroup_config["compute"]["launchSpecification"]["tags"]
# Remove any standard tags specified in ElastiGroup configuration
tags = list(filter(lambda tag: tag["tagKey"] not in standard_tags, tags))
# Add standard tags from Senza definition
tags.extend([{
"tagKey": "Name",
"tagValue": full_name
}, {
"tagKey": "StackName",
"tagValue": name
}, {
"tagKey": "StackVersion",
"tagValue": version
}])
elastigroup_config["compute"]["launchSpecification"]["tags"] = tags
if elastigroup_config.get("name", "") == "":
elastigroup_config["name"] = full_name
def extract_block_mappings(configuration, elastigroup_config):
"""
This function converts a Senza BlockDeviceMappings section into the matching section of the Elastigroup
If there's a launchSpecification.blockDeviceMappings section already it's left untouched
"""
if "BlockDeviceMappings" not in configuration:
return
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
launch_spec = elastigroup_config["compute"]["launchSpecification"]
if "blockDeviceMappings" in launch_spec:
return
block_device_mappings = configuration.pop("BlockDeviceMappings")
elastigroup_mappings = []
for mapping in block_device_mappings:
elastigroup_mappings.append({
"deviceName": mapping["DeviceName"],
"ebs": {
"deleteOnTermination": True,
"volumeType": "gp2",
"volumeSize": mapping["Ebs"]["VolumeSize"]
}
})
if elastigroup_mappings:
launch_spec["blockDeviceMappings"] = elastigroup_mappings
def component_iam_role(definition, configuration, args, info, force):
definition = ensure_keys(definition, "Resources")
role_name = configuration['Name']
definition['Resources'][role_name] = {
'Type': 'AWS::IAM::Role',
'Properties': {
"AssumeRolePolicyDocument": configuration.get('AssumeRolePolicyDocument', {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": ["ec2.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}
]
}),
'Path': configuration.get('Path', '/'),
'Policies': configuration.get('Policies', []) + get_merged_policies(
configuration.get('MergePoliciesFromIamRoles', []), args.region)
}
}
return definition
def extract_block_mappings(configuration, elastigroup_config):
"""
This function converts a Senza BlockDeviceMappings section into the matching section of the Elastigroup
If there's a launchSpecification.blockDeviceMappings section already it's left untouched
"""
if "BlockDeviceMappings" not in configuration:
return
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
launch_spec = elastigroup_config["compute"]["launchSpecification"]
if "blockDeviceMappings" in launch_spec:
return
block_device_mappings = configuration.pop("BlockDeviceMappings")
elastigroup_mappings = []
for mapping in block_device_mappings:
elastigroup_mappings.append({
"deviceName": mapping["DeviceName"],
"ebs": {
"deleteOnTermination": True,
"volumeType": "gp2",
"volumeSize": mapping["Ebs"]["VolumeSize"]
}
})
if elastigroup_mappings:
launch_spec["blockDeviceMappings"] = elastigroup_mappings
def component_auto_scaling_group(definition, configuration, args, info, force, account_info):
definition = ensure_keys(definition, "Resources")
# launch configuration
config_name = configuration["Name"] + "Config"
definition["Resources"][config_name] = {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"InstanceType": configuration["InstanceType"],
"ImageId": {"Fn::FindInMap": ["Images", {"Ref": "AWS::Region"}, configuration["Image"]]},
"AssociatePublicIpAddress": configuration.get('AssociatePublicIpAddress', False),
"EbsOptimized": configuration.get('EbsOptimized', False)
}
}
if 'BlockDeviceMappings' in configuration:
definition['Resources'][config_name]['Properties']['BlockDeviceMappings'] = configuration['BlockDeviceMappings']
if "IamInstanceProfile" in configuration:
definition["Resources"][config_name]["Properties"]["IamInstanceProfile"] = configuration["IamInstanceProfile"]
if 'IamRoles' in configuration:
logical_id = configuration['Name'] + 'InstanceProfile'
roles = configuration['IamRoles']
if len(roles) > 1:
for role in roles:
if isinstance(role, dict):
raise click.UsageError('Cannot merge policies of Cloud Formation references ({"Ref": ".."}): ' +
'You can use at most one IAM role with "Ref".')
logical_role_id = configuration['Name'] + 'Role'
definition['Resources'][logical_role_id] = {
'Type': 'AWS::IAM::Role',
'Properties': {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": ["ec2.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}
]
},
'Path': '/',
'Policies': get_merged_policies(roles)
}
}
instance_profile_roles = [{'Ref': logical_role_id}]
elif isinstance(roles[0], dict):
instance_profile_roles = [resolve_referenced_resource(roles[0], args.region)]
else:
instance_profile_roles = roles
definition['Resources'][logical_id] = {
'Type': 'AWS::IAM::InstanceProfile',
'Properties': {
'Path': '/',
'Roles': instance_profile_roles
}
}
definition["Resources"][config_name]["Properties"]["IamInstanceProfile"] = {'Ref': logical_id}
if "SecurityGroups" in configuration:
definition["Resources"][config_name]["Properties"]["SecurityGroups"] = \
resolve_security_groups(configuration["SecurityGroups"], args.region)
if "UserData" in configuration:
definition["Resources"][config_name]["Properties"]["UserData"] = {
"Fn::Base64": configuration["UserData"]
}
# auto scaling group
asg_name = configuration["Name"]
asg_success = ["1", "PT15M"]
if "AutoScaling" in configuration:
if "SuccessRequires" in configuration["AutoScaling"]:
asg_success = normalize_asg_success(configuration["AutoScaling"]["SuccessRequires"])
definition["Resources"][asg_name] = {
"Type": "AWS::AutoScaling::AutoScalingGroup",
# wait to get a signal from an amount of servers to signal that it booted
"CreationPolicy": {
"ResourceSignal": {
"Count": asg_success[0],
"Timeout": asg_success[1]
}
},
"Properties": {
# for our operator some notifications
"LaunchConfigurationName": {"Ref": config_name},
"VPCZoneIdentifier": {"Fn::FindInMap": ["ServerSubnets", {"Ref": "AWS::Region"}, "Subnets"]},
"Tags": [
# Tag "Name"
{
"Key": "Name",
"PropagateAtLaunch": True,
"Value": "{0}-{1}".format(info["StackName"], info["StackVersion"])
},
# Tag "StackName"
{
"Key": "StackName",
"PropagateAtLaunch": True,
"Value": info["StackName"],
},
# Tag "StackVersion"
{
"Key": "StackVersion",
"PropagateAtLaunch": True,
"Value": info["StackVersion"]
}
]
}
}
asg_properties = definition["Resources"][asg_name]["Properties"]
if "OperatorTopicId" in info:
asg_properties["NotificationConfiguration"] = {
"NotificationTypes": [
"autoscaling:EC2_INSTANCE_LAUNCH",
"autoscaling:EC2_INSTANCE_LAUNCH_ERROR",
"autoscaling:EC2_INSTANCE_TERMINATE",
"autoscaling:EC2_INSTANCE_TERMINATE_ERROR"
],
"TopicARN": resolve_topic_arn(args.region, info["OperatorTopicId"])
}
default_health_check_type = 'EC2'
if "ElasticLoadBalancer" in configuration:
if isinstance(configuration["ElasticLoadBalancer"], str):
asg_properties["LoadBalancerNames"] = [{"Ref": configuration["ElasticLoadBalancer"]}]
elif isinstance(configuration["ElasticLoadBalancer"], list):
asg_properties["LoadBalancerNames"] = [{'Ref': ref} for ref in configuration["ElasticLoadBalancer"]]
# use ELB health check by default
default_health_check_type = 'ELB'
asg_properties['HealthCheckType'] = configuration.get('HealthCheckType', default_health_check_type)
asg_properties['HealthCheckGracePeriod'] = configuration.get('HealthCheckGracePeriod', 300)
if "AutoScaling" in configuration:
as_conf = configuration["AutoScaling"]
asg_properties["MaxSize"] = as_conf["Maximum"]
asg_properties["MinSize"] = as_conf["Minimum"]
asg_properties["DesiredCapacity"] = max(int(as_conf["Minimum"]), int(as_conf.get('DesiredCapacity', 1)))
scaling_adjustment = int(as_conf.get("ScalingAdjustment", 1))
# ScaleUp policy
definition["Resources"][asg_name + "ScaleUp"] = {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"ScalingAdjustment": str(scaling_adjustment),
"Cooldown": str(as_conf.get("Cooldown", "60")),
"AutoScalingGroupName": {
"Ref": asg_name
}
}
}
# ScaleDown policy
definition["Resources"][asg_name + "ScaleDown"] = {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"ScalingAdjustment": str((-1) * scaling_adjustment),
"Cooldown": str(as_conf.get("Cooldown", "60")),
"AutoScalingGroupName": {
"Ref": asg_name
}
}
}
if "MetricType" in as_conf:
metric_type = as_conf["MetricType"]
metricfns = {
"CPU": metric_cpu,
"NetworkIn": metric_network,
"NetworkOut": metric_network
}
# lowercase cpu is an acceptable metric, be compatible
if metric_type.lower() not in map(lambda t: t.lower(), metricfns.keys()):
raise click.UsageError('Auto scaling MetricType "{}" not supported.'.format(metric_type))
metricfn = metricfns[metric_type]
definition = metricfn(asg_name, definition, as_conf, args, info, force)
else:
asg_properties["MaxSize"] = 1
asg_properties["MinSize"] = 1
return definition
def component_auto_scaling_group(definition, configuration, args, info, force,
account_info):
definition = ensure_keys(definition, "Resources")
# launch configuration
config_name = configuration["Name"] + "Config"
definition["Resources"][config_name] = {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"InstanceType":
configuration["InstanceType"],
"ImageId": {
"Fn::FindInMap":
["Images", {
"Ref": "AWS::Region"
}, configuration["Image"]]
},
"AssociatePublicIpAddress":
configuration.get('AssociatePublicIpAddress', False),
"EbsOptimized":
configuration.get('EbsOptimized', False)
}
}
if 'IamRoles' in configuration:
logical_id = configuration['Name'] + 'InstanceProfile'
roles = configuration['IamRoles']
if len(roles) > 1:
for role in roles:
if isinstance(role, dict):
raise click.UsageError(
'Cannot merge policies of Cloud Formation references ({"Ref": ".."}): '
+ 'You can use at most one IAM role with "Ref".')
logical_role_id = configuration['Name'] + 'Role'
definition['Resources'][logical_role_id] = {
'Type': 'AWS::IAM::Role',
'Properties': {
"AssumeRolePolicyDocument": {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": ["ec2.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}]
},
'Path': '/',
'Policies': get_merged_policies(roles)
}
}
instance_profile_roles = [{'Ref': logical_role_id}]
elif isinstance(roles[0], dict):
instance_profile_roles = [
resolve_referenced_resource(roles[0], args.region)
]
else:
instance_profile_roles = roles
definition['Resources'][logical_id] = {
'Type': 'AWS::IAM::InstanceProfile',
'Properties': {
'Path': '/',
'Roles': instance_profile_roles
}
}
definition["Resources"][config_name]["Properties"][
"IamInstanceProfile"] = {
'Ref': logical_id
}
if "SecurityGroups" in configuration:
definition["Resources"][config_name]["Properties"]["SecurityGroups"] = \
resolve_security_groups(configuration["SecurityGroups"], args.region)
if "UserData" in configuration:
definition["Resources"][config_name]["Properties"]["UserData"] = {
"Fn::Base64": configuration["UserData"]
}
# auto scaling group
asg_name = configuration["Name"]
asg_success = ["1", "PT15M"]
if "AutoScaling" in configuration:
if "SuccessRequires" in configuration["AutoScaling"]:
asg_success = normalize_asg_success(
configuration["AutoScaling"]["SuccessRequires"])
tags = [
# Tag "Name"
{
"Key": "Name",
"PropagateAtLaunch": True,
"Value": "{0}-{1}".format(info["StackName"], info["StackVersion"])
},
# Tag "StackName"
{
"Key": "StackName",
"PropagateAtLaunch": True,
"Value": info["StackName"],
},
# Tag "StackVersion"
{
"Key": "StackVersion",
"PropagateAtLaunch": True,
"Value": info["StackVersion"]
}
]
if "Tags" in configuration:
for tag in configuration["Tags"]:
tags.append({
"Key": tag["Key"],
"PropagateAtLaunch": True,
"Value": tag["Value"]
})
definition["Resources"][asg_name] = {
"Type": "AWS::AutoScaling::AutoScalingGroup",
# wait to get a signal from an amount of servers to signal that it booted
"CreationPolicy": {
"ResourceSignal": {
"Count": asg_success[0],
"Timeout": asg_success[1]
}
},
"Properties": {
# for our operator some notifications
"LaunchConfigurationName": {
"Ref": config_name
},
"VPCZoneIdentifier": {
"Fn::FindInMap":
["ServerSubnets", {
"Ref": "AWS::Region"
}, "Subnets"]
},
"Tags": tags
}
}
asg_properties = definition["Resources"][asg_name]["Properties"]
if "OperatorTopicId" in info:
asg_properties["NotificationConfiguration"] = {
"NotificationTypes": [
"autoscaling:EC2_INSTANCE_LAUNCH",
"autoscaling:EC2_INSTANCE_LAUNCH_ERROR",
"autoscaling:EC2_INSTANCE_TERMINATE",
"autoscaling:EC2_INSTANCE_TERMINATE_ERROR"
],
"TopicARN":
resolve_topic_arn(args.region, info["OperatorTopicId"])
}
default_health_check_type = 'EC2'
if "ElasticLoadBalancer" in configuration:
if isinstance(configuration["ElasticLoadBalancer"], str):
asg_properties["LoadBalancerNames"] = [{
"Ref":
configuration["ElasticLoadBalancer"]
}]
elif isinstance(configuration["ElasticLoadBalancer"], list):
asg_properties["LoadBalancerNames"] = [{
'Ref': ref
} for ref in configuration["ElasticLoadBalancer"]]
# use ELB health check by default
default_health_check_type = 'ELB'
if "ElasticLoadBalancerV2" in configuration:
if isinstance(configuration["ElasticLoadBalancerV2"], str):
asg_properties["TargetGroupARNs"] = [{
"Ref":
configuration["ElasticLoadBalancerV2"] + 'TargetGroup'
}]
elif isinstance(configuration["ElasticLoadBalancerV2"], list):
asg_properties["TargetGroupARNs"] = [{
'Ref': ref + 'TargetGroup'
} for ref in configuration["ElasticLoadBalancerV2"]]
# use ELB health check by default
default_health_check_type = 'ELB'
asg_properties['HealthCheckType'] = configuration.get(
'HealthCheckType', default_health_check_type)
asg_properties['HealthCheckGracePeriod'] = configuration.get(
'HealthCheckGracePeriod', 300)
if "AutoScaling" in configuration:
as_conf = configuration["AutoScaling"]
asg_properties["MaxSize"] = as_conf["Maximum"]
asg_properties["MinSize"] = as_conf["Minimum"]
asg_properties["DesiredCapacity"] = max(
int(as_conf["Minimum"]), int(as_conf.get('DesiredCapacity', 1)))
default_scaling_adjustment = as_conf.get("ScalingAdjustment", 1)
default_cooldown = as_conf.get("Cooldown", "60")
# ScaleUp policy
scale_up_name = asg_name + "ScaleUp"
scale_up_adjustment = int(
as_conf.get("ScaleUpAdjustment", default_scaling_adjustment))
scale_up_cooldown = as_conf.get("ScaleUpCooldown", default_cooldown)
definition["Resources"][scale_up_name] = create_autoscaling_policy(
asg_name, scale_up_name, scale_up_adjustment, scale_up_cooldown,
definition)
# ScaleDown policy
scale_down_name = asg_name + "ScaleDown"
scale_down_adjustment = (-1) * int(
as_conf.get("ScaleDownAdjustment", default_scaling_adjustment))
scale_down_cooldown = as_conf.get("ScaleDownCooldown",
default_cooldown)
definition["Resources"][scale_down_name] = create_autoscaling_policy(
asg_name, scale_down_name, scale_down_adjustment,
scale_down_cooldown, definition)
if "MetricType" in as_conf:
metric_type = as_conf["MetricType"]
metricfns = {
"CPU": metric_cpu,
"NetworkIn": metric_network,
"NetworkOut": metric_network
}
# lowercase cpu is an acceptable metric, be compatible
if metric_type.lower() not in map(lambda t: t.lower(),
metricfns.keys()):
raise click.UsageError(
'Auto scaling MetricType "{}" not supported.'.format(
metric_type))
metricfn = metricfns[metric_type]
definition = metricfn(asg_name, definition, as_conf, args, info,
force)
else:
asg_properties["MaxSize"] = 1
asg_properties["MinSize"] = 1
for res in (config_name, asg_name):
props = definition['Resources'][res]['Properties']
additional_cf_properties = ADDITIONAL_PROPERTIES.get(
definition['Resources'][res]['Type'])
properties_allowed_to_overwrite = (
set(props.keys()) - SENZA_PROPERTIES) | additional_cf_properties
for key in properties_allowed_to_overwrite:
if key in configuration:
props[key] = configuration[key]
return definition
def component_taupage_auto_scaling_group(definition, configuration, args, info,
force, account_info):
# inherit from the normal auto scaling group but discourage user info and replace with a Taupage config
if 'Image' not in configuration:
configuration['Image'] = 'LatestTaupageImage'
definition = component_auto_scaling_group(definition, configuration, args,
info, force, account_info)
taupage_config = configuration['TaupageConfig']
if 'notify_cfn' not in taupage_config:
taupage_config['notify_cfn'] = {
'stack': '{}-{}'.format(info["StackName"], info["StackVersion"]),
'resource': configuration['Name']
}
if 'application_id' not in taupage_config:
taupage_config['application_id'] = info['StackName']
if 'application_version' not in taupage_config:
taupage_config['application_version'] = info['StackVersion']
check_application_id(taupage_config['application_id'])
check_application_version(taupage_config['application_version'])
runtime = taupage_config.get('runtime')
if runtime != 'Docker':
raise click.UsageError(
'Taupage only supports the "Docker" runtime currently')
source = taupage_config.get('source')
if not source:
raise click.UsageError(
'The "source" property of TaupageConfig must be specified')
docker_image = pierone.api.DockerImage.parse(source)
if not force and docker_image.registry:
check_docker_image_exists(docker_image)
config_name = configuration["Name"] + "Config"
ensure_keys(definition, "Resources", config_name, "Properties")
properties = definition["Resources"][config_name]["Properties"]
mappings = definition.get('Mappings', {})
server_subnets = set(
mappings.get('ServerSubnets', {}).get(args.region,
{}).get('Subnets', []))
# in dmz or public subnet but without public ip
if server_subnets and not properties.get('AssociatePublicIpAddress') and server_subnets ==\
set(mappings.get('LoadBalancerInternalSubnets', {}).get(args.region, {}).get('Subnets', [])):
# we need to extend taupage_config with the mapping subnet-id => net ip
nat_gateways = {}
ec2 = boto3.client('ec2', args.region)
for nat_gateway in ec2.describe_nat_gateways()['NatGateways']:
if nat_gateway['SubnetId'] in server_subnets:
for address in nat_gateway['NatGatewayAddresses']:
nat_gateways[
nat_gateway['SubnetId']] = address['PrivateIp']
break
if nat_gateways:
taupage_config['nat_gateways'] = nat_gateways
properties["UserData"] = {
"Fn::Base64": generate_user_data(taupage_config, args.region)
}
return definition
def component_auto_scaling_group(definition, configuration, args, info, force, account_info):
definition = ensure_keys(definition, "Resources")
# launch configuration
config_name = configuration["Name"] + "Config"
definition["Resources"][config_name] = {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"InstanceType": configuration["InstanceType"],
"ImageId": {"Fn::FindInMap": ["Images", {"Ref": "AWS::Region"}, configuration["Image"]]},
"AssociatePublicIpAddress": configuration.get('AssociatePublicIpAddress', False),
"EbsOptimized": configuration.get('EbsOptimized', False)
}
}
if 'BlockDeviceMappings' in configuration:
definition['Resources'][config_name]['Properties']['BlockDeviceMappings'] = configuration['BlockDeviceMappings']
if "IamInstanceProfile" in configuration:
definition["Resources"][config_name]["Properties"]["IamInstanceProfile"] = configuration["IamInstanceProfile"]
if 'IamRoles' in configuration:
logical_id = configuration['Name'] + 'InstanceProfile'
roles = configuration['IamRoles']
if len(roles) > 1:
for role in roles:
if isinstance(role, dict):
raise click.UsageError('Cannot merge policies of Cloud Formation references ({"Ref": ".."}): ' +
'You can use at most one IAM role with "Ref".')
logical_role_id = configuration['Name'] + 'Role'
definition['Resources'][logical_role_id] = {
'Type': 'AWS::IAM::Role',
'Properties': {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": ["ec2.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}
]
},
'Path': '/',
'Policies': get_merged_policies(roles)
}
}
instance_profile_roles = [{'Ref': logical_role_id}]
else:
instance_profile_roles = roles
definition['Resources'][logical_id] = {
'Type': 'AWS::IAM::InstanceProfile',
'Properties': {
'Path': '/',
'Roles': instance_profile_roles
}
}
definition["Resources"][config_name]["Properties"]["IamInstanceProfile"] = {'Ref': logical_id}
if "SecurityGroups" in configuration:
definition["Resources"][config_name]["Properties"]["SecurityGroups"] = \
resolve_security_groups(configuration["SecurityGroups"], args.region)
if "UserData" in configuration:
definition["Resources"][config_name]["Properties"]["UserData"] = {
"Fn::Base64": configuration["UserData"]
}
# auto scaling group
asg_name = configuration["Name"]
definition["Resources"][asg_name] = {
"Type": "AWS::AutoScaling::AutoScalingGroup",
# wait up to 15 minutes to get a signal from at least one server that it booted
"CreationPolicy": {
"ResourceSignal": {
"Count": "1",
"Timeout": "PT15M"
}
},
"Properties": {
# for our operator some notifications
"LaunchConfigurationName": {"Ref": config_name},
"VPCZoneIdentifier": {"Fn::FindInMap": ["ServerSubnets", {"Ref": "AWS::Region"}, "Subnets"]},
"Tags": [
# Tag "Name"
{
"Key": "Name",
"PropagateAtLaunch": True,
"Value": "{0}-{1}".format(info["StackName"], info["StackVersion"])
},
# Tag "StackName"
{
"Key": "StackName",
"PropagateAtLaunch": True,
"Value": info["StackName"],
},
# Tag "StackVersion"
{
"Key": "StackVersion",
"PropagateAtLaunch": True,
"Value": info["StackVersion"]
}
]
}
}
if "OperatorTopicId" in info:
definition["Resources"][asg_name]["Properties"]["NotificationConfiguration"] = {
"NotificationTypes": [
"autoscaling:EC2_INSTANCE_LAUNCH",
"autoscaling:EC2_INSTANCE_LAUNCH_ERROR",
"autoscaling:EC2_INSTANCE_TERMINATE",
"autoscaling:EC2_INSTANCE_TERMINATE_ERROR"
],
"TopicARN": resolve_topic_arn(args.region, info["OperatorTopicId"])
}
default_health_check_type = 'EC2'
if "ElasticLoadBalancer" in configuration:
if isinstance(configuration["ElasticLoadBalancer"], str):
definition["Resources"][asg_name]["Properties"]["LoadBalancerNames"] = [
{"Ref": configuration["ElasticLoadBalancer"]}]
elif isinstance(configuration["ElasticLoadBalancer"], list):
definition["Resources"][asg_name]["Properties"]["LoadBalancerNames"] = []
for ref in configuration["ElasticLoadBalancer"]:
definition["Resources"][asg_name]["Properties"]["LoadBalancerNames"].append({'Ref': ref})
# use ELB health check by default
default_health_check_type = 'ELB'
definition["Resources"][asg_name]['Properties']['HealthCheckType'] = \
configuration.get('HealthCheckType', default_health_check_type)
definition["Resources"][asg_name]['Properties']['HealthCheckGracePeriod'] = \
configuration.get('HealthCheckGracePeriod', 300)
if "AutoScaling" in configuration:
definition["Resources"][asg_name]["Properties"]["MaxSize"] = configuration["AutoScaling"]["Maximum"]
definition["Resources"][asg_name]["Properties"]["MinSize"] = configuration["AutoScaling"]["Minimum"]
# ScaleUp policy
definition["Resources"][asg_name + "ScaleUp"] = {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"ScalingAdjustment": "1",
"Cooldown": "60",
"AutoScalingGroupName": {
"Ref": asg_name
}
}
}
# ScaleDown policy
definition["Resources"][asg_name + "ScaleDown"] = {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"ScalingAdjustment": "-1",
"Cooldown": "60",
"AutoScalingGroupName": {
"Ref": asg_name
}
}
}
metric_type = configuration["AutoScaling"]["MetricType"]
metricfn = globals().get('metric_{}'.format(metric_type.lower()))
if not metricfn:
raise click.UsageError('Auto scaling MetricType "{}" not supported.'.format(metric_type))
definition = metricfn(asg_name, definition, configuration["AutoScaling"], args, info, force)
else:
definition["Resources"][asg_name]["Properties"]["MaxSize"] = 1
definition["Resources"][asg_name]["Properties"]["MinSize"] = 1
return definition
def extract_load_balancer_name(configuration, elastigroup_config: dict):
"""
This function identifies whether a senza ELB is configured,
if so it transforms it into a Spotinst Elastigroup balancer API configuration
If there's already a Spotinst launchSpecification present it is left untouched
It also handles the health check definitions (type and grace period) giving precedence to any existing Elastigroup
defintions.
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute",
"launchSpecification")
launch_spec_config = elastigroup_config["compute"]["launchSpecification"]
health_check_type = "EC2"
if "loadBalancersConfig" not in launch_spec_config.keys():
load_balancers = []
if "ElasticLoadBalancer" in configuration:
load_balancer_refs = configuration.pop("ElasticLoadBalancer")
health_check_type = "ELB"
if isinstance(load_balancer_refs, str):
load_balancers.append({
"name": {
"Ref": load_balancer_refs
},
"type": "CLASSIC"
})
elif isinstance(load_balancer_refs, list):
for load_balancer_ref in load_balancer_refs:
load_balancers.append({
"name": {
"Ref": load_balancer_ref
},
"type": "CLASSIC"
})
if "ElasticLoadBalancerV2" in configuration:
health_check_type = "TARGET_GROUP"
load_balancer_refs = configuration.pop("ElasticLoadBalancerV2")
custom_target_groups = configuration.pop("TargetGroupARNs", None)
if custom_target_groups:
for custom_target_group in custom_target_groups:
load_balancers.append({
"arn": custom_target_group,
"type": "TARGET_GROUP"
})
else:
if isinstance(load_balancer_refs, str):
load_balancers.append({
"arn": {
"Ref": load_balancer_refs + 'TargetGroup'
},
"type": "TARGET_GROUP"
})
elif isinstance(load_balancer_refs, list):
for load_balancer_ref in load_balancer_refs:
load_balancers.append({
"arn": {
"Ref": load_balancer_ref + "TargetGroup"
},
"type": "TARGET_GROUP"
})
if len(load_balancers) > 0:
launch_spec_config["loadBalancersConfig"] = {
"loadBalancers": load_balancers
}
health_check_type = launch_spec_config.get(
"healthCheckType",
configuration.get("HealthCheckType", health_check_type))
grace_period = launch_spec_config.get(
"healthCheckGracePeriod",
configuration.get('HealthCheckGracePeriod', 300))
launch_spec_config["healthCheckType"] = health_check_type
launch_spec_config["healthCheckGracePeriod"] = grace_period
def component_subnet_auto_configuration(definition, configuration, args, info,
force, account_info):
ec2 = boto3.resource('ec2', args.region)
vpc_id = configuration.get('VpcId', account_info.VpcID)
availability_zones = configuration.get('AvailabilityZones')
public_only = configuration.get('PublicOnly')
server_subnets = []
lb_subnets = []
lb_internal_subnets = []
all_subnets = []
for subnet in ec2.subnets.filter(Filters=[{
'Name': 'vpc-id',
'Values': [vpc_id]
}]):
name = get_tag(subnet.tags, 'Name', '')
if availability_zones and subnet.availability_zone not in availability_zones:
# skip subnet as it's not in one of the given AZs
continue
all_subnets.append(subnet.id)
if public_only:
if 'dmz' in name:
lb_subnets.append(subnet.id)
lb_internal_subnets.append(subnet.id)
server_subnets.append(subnet.id)
else:
if 'dmz' in name:
lb_subnets.append(subnet.id)
elif 'internal' in name:
lb_internal_subnets.append(subnet.id)
server_subnets.append(subnet.id)
elif 'nat' in name:
# ignore creating listeners in NAT gateway subnets
pass
else:
server_subnets.append(subnet.id)
if not lb_subnets:
if public_only:
# assume default AWS VPC setup with all subnets being public
lb_subnets = all_subnets
lb_internal_subnets = all_subnets
server_subnets = all_subnets
else:
# no DMZ subnets were found, just use the same set for both LB and instances
lb_subnets = server_subnets
configuration = ensure_keys(configuration, "ServerSubnets", args.region)
configuration["ServerSubnets"][args.region] = server_subnets
configuration = ensure_keys(configuration, "LoadBalancerSubnets",
args.region)
configuration["LoadBalancerSubnets"][args.region] = lb_subnets
configuration = ensure_keys(configuration, "LoadBalancerInternalSubnets",
args.region)
configuration["LoadBalancerInternalSubnets"][
args.region] = lb_internal_subnets
component_configuration(definition, configuration, args, info, force,
account_info)
return definition
def component_auto_scaling_group(definition, configuration, args, info, force,
account_info):
definition = ensure_keys(definition, "Resources")
# launch configuration
config_name = configuration["Name"] + "Config"
definition["Resources"][config_name] = {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"InstanceType":
configuration["InstanceType"],
"ImageId": {
"Fn::FindInMap": [
"Images",
{
"Ref": "AWS::Region"
},
configuration["Image"],
]
},
"AssociatePublicIpAddress":
configuration.get("AssociatePublicIpAddress", False),
"EbsOptimized":
configuration.get("EbsOptimized", False),
},
}
if "IamRoles" in configuration:
logical_id = handle_iam_roles(definition, configuration, args)
definition["Resources"][config_name]["Properties"][
"IamInstanceProfile"] = {
"Ref": logical_id
}
if "SecurityGroups" in configuration:
definition["Resources"][config_name]["Properties"][
"SecurityGroups"] = resolve_security_groups(
configuration["SecurityGroups"], args.region)
if "UserData" in configuration:
definition["Resources"][config_name]["Properties"]["UserData"] = {
"Fn::Base64": configuration["UserData"]
}
# auto scaling group
asg_name = configuration["Name"]
asg_success = ["1", "PT15M"]
if "AutoScaling" in configuration:
if "SuccessRequires" in configuration["AutoScaling"]:
asg_success = normalize_asg_success(
configuration["AutoScaling"]["SuccessRequires"])
tags = [
# Tag "Name"
{
"Key": "Name",
"PropagateAtLaunch": True,
"Value": "{0}-{1}".format(info["StackName"], info["StackVersion"]),
},
# Tag "StackName"
{
"Key": "StackName",
"PropagateAtLaunch": True,
"Value": info["StackName"]
},
# Tag "StackVersion"
{
"Key": "StackVersion",
"PropagateAtLaunch": True,
"Value": info["StackVersion"],
},
]
if "Tags" in configuration:
for tag in configuration["Tags"]:
tags.append({
"Key": tag["Key"],
"PropagateAtLaunch": True,
"Value": tag["Value"]
})
definition["Resources"][asg_name] = {
"Type": "AWS::AutoScaling::AutoScalingGroup",
# wait to get a signal from an amount of servers to signal that it booted
"CreationPolicy": {
"ResourceSignal": {
"Count": asg_success[0],
"Timeout": asg_success[1]
}
},
"Properties": {
# for our operator some notifications
"LaunchConfigurationName": {
"Ref": config_name
},
"VPCZoneIdentifier": {
"Fn::FindInMap":
["ServerSubnets", {
"Ref": "AWS::Region"
}, "Subnets"]
},
"Tags": tags,
},
}
asg_properties = definition["Resources"][asg_name]["Properties"]
if "OperatorTopicId" in info:
asg_properties["NotificationConfiguration"] = {
"NotificationTypes": [
"autoscaling:EC2_INSTANCE_LAUNCH",
"autoscaling:EC2_INSTANCE_LAUNCH_ERROR",
"autoscaling:EC2_INSTANCE_TERMINATE",
"autoscaling:EC2_INSTANCE_TERMINATE_ERROR",
],
"TopicARN":
resolve_topic_arn(args.region, info["OperatorTopicId"]),
}
default_health_check_type = "EC2"
if "ElasticLoadBalancer" in configuration:
if isinstance(configuration["ElasticLoadBalancer"], str):
asg_properties["LoadBalancerNames"] = [{
"Ref":
configuration["ElasticLoadBalancer"]
}]
elif isinstance(configuration["ElasticLoadBalancer"], list):
asg_properties["LoadBalancerNames"] = [{
"Ref": ref
} for ref in configuration["ElasticLoadBalancer"]]
# use ELB health check by default
default_health_check_type = "ELB"
if "ElasticLoadBalancerV2" in configuration:
if isinstance(configuration["ElasticLoadBalancerV2"], str):
asg_properties["TargetGroupARNs"] = [{
"Ref":
configuration["ElasticLoadBalancerV2"] + "TargetGroup"
}]
elif isinstance(configuration["ElasticLoadBalancerV2"], list):
asg_properties["TargetGroupARNs"] = [{
"Ref": ref + "TargetGroup"
} for ref in configuration["ElasticLoadBalancerV2"]]
# use ELB health check by default
default_health_check_type = "ELB"
asg_properties["HealthCheckType"] = configuration.get(
"HealthCheckType", default_health_check_type)
asg_properties["HealthCheckGracePeriod"] = configuration.get(
"HealthCheckGracePeriod", 300)
if "AutoScaling" in configuration:
as_conf = configuration["AutoScaling"]
asg_properties["MaxSize"] = as_conf["Maximum"]
asg_properties["MinSize"] = as_conf["Minimum"]
asg_properties["DesiredCapacity"] = max(
int(as_conf["Minimum"]), int(as_conf.get("DesiredCapacity", 1)))
default_scaling_adjustment = as_conf.get("ScalingAdjustment", 1)
default_cooldown = as_conf.get("Cooldown", "60")
# ScaleUp policy
scale_up_name = asg_name + "ScaleUp"
scale_up_adjustment = int(
as_conf.get("ScaleUpAdjustment", default_scaling_adjustment))
scale_up_cooldown = as_conf.get("ScaleUpCooldown", default_cooldown)
definition["Resources"][scale_up_name] = create_autoscaling_policy(
asg_name, scale_up_name, scale_up_adjustment, scale_up_cooldown,
definition)
# ScaleDown policy
scale_down_name = asg_name + "ScaleDown"
scale_down_adjustment = (-1) * int(
as_conf.get("ScaleDownAdjustment", default_scaling_adjustment))
scale_down_cooldown = as_conf.get("ScaleDownCooldown",
default_cooldown)
definition["Resources"][scale_down_name] = create_autoscaling_policy(
asg_name,
scale_down_name,
scale_down_adjustment,
scale_down_cooldown,
definition,
)
if "MetricType" in as_conf:
metric_type = as_conf["MetricType"]
metricfns = {
"CPU": metric_cpu,
"NetworkIn": metric_network,
"NetworkOut": metric_network,
}
# lowercase cpu is an acceptable metric, be compatible
if metric_type.lower() not in map(lambda t: t.lower(),
metricfns.keys()):
raise click.UsageError(
'Auto scaling MetricType "{}" not supported.'.format(
metric_type))
metricfn = metricfns[metric_type]
definition = metricfn(asg_name, definition, as_conf, args, info,
force)
else:
asg_properties["MaxSize"] = 1
asg_properties["MinSize"] = 1
for res in (config_name, asg_name):
props = definition["Resources"][res]["Properties"]
additional_cf_properties = ADDITIONAL_PROPERTIES.get(
definition["Resources"][res]["Type"])
properties_allowed_to_overwrite = (
set(props.keys()) - SENZA_PROPERTIES) | additional_cf_properties
for key in properties_allowed_to_overwrite:
if key in configuration:
props[key] = configuration[key]
return definition
def component_auto_scaling_group(definition, configuration, args, info, force):
definition = ensure_keys(definition, "Resources")
# launch configuration
config_name = configuration["Name"] + "Config"
definition["Resources"][config_name] = {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"InstanceType": configuration["InstanceType"],
"ImageId": {"Fn::FindInMap": ["Images", {"Ref": "AWS::Region"}, configuration["Image"]]},
"AssociatePublicIpAddress": configuration.get('AssociatePublicIpAddress', False),
"EbsOptimized": configuration.get('EbsOptimized', False)
}
}
if 'BlockDeviceMappings' in configuration:
definition['Resources'][config_name]['Properties']['BlockDeviceMappings'] = configuration['BlockDeviceMappings']
if "IamInstanceProfile" in configuration:
definition["Resources"][config_name]["Properties"]["IamInstanceProfile"] = configuration["IamInstanceProfile"]
if 'IamRoles' in configuration:
logical_id = configuration['Name'] + 'InstanceProfile'
roles = configuration['IamRoles']
if len(roles) > 1:
for role in roles:
if isinstance(role, dict):
raise click.UsageError('Cannot merge policies of Cloud Formation references ({"Ref": ".."}): ' +
'You can use at most one IAM role with "Ref".')
logical_role_id = configuration['Name'] + 'Role'
definition['Resources'][logical_role_id] = {
'Type': 'AWS::IAM::Role',
'Properties': {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": ["ec2.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}
]
},
'Path': '/',
'Policies': get_merged_policies(roles, args.region)
}
}
instance_profile_roles = [{'Ref': logical_role_id}]
else:
instance_profile_roles = roles
definition['Resources'][logical_id] = {
'Type': 'AWS::IAM::InstanceProfile',
'Properties': {
'Path': '/',
'Roles': instance_profile_roles
}
}
definition["Resources"][config_name]["Properties"]["IamInstanceProfile"] = {'Ref': logical_id}
if "SecurityGroups" in configuration:
definition["Resources"][config_name]["Properties"]["SecurityGroups"] = \
resolve_security_groups(configuration["SecurityGroups"], args.region)
if "UserData" in configuration:
definition["Resources"][config_name]["Properties"]["UserData"] = {
"Fn::Base64": configuration["UserData"]
}
# auto scaling group
asg_name = configuration["Name"]
definition["Resources"][asg_name] = {
"Type": "AWS::AutoScaling::AutoScalingGroup",
# wait up to 15 minutes to get a signal from at least one server that it booted
"CreationPolicy": {
"ResourceSignal": {
"Count": "1",
"Timeout": "PT15M"
}
},
"Properties": {
# for our operator some notifications
"LaunchConfigurationName": {"Ref": config_name},
"VPCZoneIdentifier": {"Fn::FindInMap": ["ServerSubnets", {"Ref": "AWS::Region"}, "Subnets"]},
"Tags": [
# Tag "Name"
{
"Key": "Name",
"PropagateAtLaunch": True,
"Value": "{0}-{1}".format(info["StackName"], info["StackVersion"])
},
# Tag "StackName"
{
"Key": "StackName",
"PropagateAtLaunch": True,
"Value": info["StackName"],
},
# Tag "StackVersion"
{
"Key": "StackVersion",
"PropagateAtLaunch": True,
"Value": info["StackVersion"]
}
]
}
}
if "OperatorTopicId" in info:
definition["Resources"][asg_name]["Properties"]["NotificationConfiguration"] = {
"NotificationTypes": [
"autoscaling:EC2_INSTANCE_LAUNCH",
"autoscaling:EC2_INSTANCE_LAUNCH_ERROR",
"autoscaling:EC2_INSTANCE_TERMINATE",
"autoscaling:EC2_INSTANCE_TERMINATE_ERROR"
],
"TopicARN": resolve_topic_arn(args.region, info["OperatorTopicId"])
}
default_health_check_type = 'EC2'
if "ElasticLoadBalancer" in configuration:
definition["Resources"][asg_name]["Properties"]["LoadBalancerNames"] = [
{"Ref": configuration["ElasticLoadBalancer"]}]
# use ELB health check by default
default_health_check_type = 'ELB'
definition["Resources"][asg_name]['Properties']['HealthCheckType'] = \
configuration.get('HealthCheckType', default_health_check_type)
definition["Resources"][asg_name]['Properties']['HealthCheckGracePeriod'] = \
configuration.get('HealthCheckGracePeriod', 300)
if "AutoScaling" in configuration:
definition["Resources"][asg_name]["Properties"]["MaxSize"] = configuration["AutoScaling"]["Maximum"]
definition["Resources"][asg_name]["Properties"]["MinSize"] = configuration["AutoScaling"]["Minimum"]
# ScaleUp policy
definition["Resources"][asg_name + "ScaleUp"] = {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"ScalingAdjustment": "1",
"Cooldown": "60",
"AutoScalingGroupName": {
"Ref": asg_name
}
}
}
# ScaleDown policy
definition["Resources"][asg_name + "ScaleDown"] = {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"ScalingAdjustment": "-1",
"Cooldown": "60",
"AutoScalingGroupName": {
"Ref": asg_name
}
}
}
metric_type = configuration["AutoScaling"]["MetricType"]
metricfn = globals().get('metric_{}'.format(metric_type.lower()))
if not metricfn:
raise click.UsageError('Auto scaling MetricType "{}" not supported.'.format(metric_type))
definition = metricfn(asg_name, definition, configuration["AutoScaling"], args, info, force)
else:
definition["Resources"][asg_name]["Properties"]["MaxSize"] = 1
definition["Resources"][asg_name]["Properties"]["MinSize"] = 1
return definition
def extract_load_balancer_name(configuration, elastigroup_config: dict):
"""
This function identifies whether a senza ELB is configured,
if so it transforms it into a Spotinst Elastigroup balancer API configuration
If there's already a Spotinst launchSpecification present it is left untouched
It also handles the health check definitions (type and grace period) giving precedence to any existing Elastigroup
defintions.
"""
elastigroup_config = ensure_keys(elastigroup_config, "compute", "launchSpecification")
launch_spec_config = elastigroup_config["compute"]["launchSpecification"]
health_check_type = "EC2"
if "loadBalancersConfig" not in launch_spec_config.keys():
load_balancers = []
if "ElasticLoadBalancer" in configuration:
load_balancer_refs = configuration.pop("ElasticLoadBalancer")
health_check_type = "ELB"
if isinstance(load_balancer_refs, str):
load_balancers.append({
"name": {"Ref": load_balancer_refs},
"type": "CLASSIC"
})
elif isinstance(load_balancer_refs, list):
for load_balancer_ref in load_balancer_refs:
load_balancers.append({
"name": {"Ref": load_balancer_ref},
"type": "CLASSIC"
})
if "ElasticLoadBalancerV2" in configuration:
health_check_type = "TARGET_GROUP"
load_balancer_refs = configuration.pop("ElasticLoadBalancerV2")
custom_target_groups = configuration.pop("TargetGroupARNs", None)
if custom_target_groups:
for custom_target_group in custom_target_groups:
load_balancers.append({
"arn": custom_target_group,
"type": "TARGET_GROUP"
})
else:
if isinstance(load_balancer_refs, str):
load_balancers.append({
"arn": {"Ref": load_balancer_refs + 'TargetGroup'},
"type": "TARGET_GROUP"
})
elif isinstance(load_balancer_refs, list):
for load_balancer_ref in load_balancer_refs:
load_balancers.append({
"arn": {"Ref": load_balancer_ref + "TargetGroup"},
"type": "TARGET_GROUP"
})
if len(load_balancers) > 0:
launch_spec_config["loadBalancersConfig"] = {"loadBalancers": load_balancers}
health_check_type = launch_spec_config.get("healthCheckType",
configuration.get("HealthCheckType", health_check_type))
grace_period = launch_spec_config.get("healthCheckGracePeriod",
configuration.get('HealthCheckGracePeriod', 300))
launch_spec_config["healthCheckType"] = health_check_type
launch_spec_config["healthCheckGracePeriod"] = grace_period
