def define_cluster(root_stack, cluster_def):
"""
Function to create the cluster from provided properties.
:param dict cluster_def:
:param ecs_composex.common.stacks.ComposeXStack root_stack:
:return: cluster
:rtype: troposphere.ecs.Cluster
"""
cluster_params = {}
if not keyisset("Properties", cluster_def):
return get_default_cluster_config()
props = cluster_def["Properties"]
if keyisset("ClusterName", props):
root_stack.Parameters.update({CLUSTER_NAME_T: props["ClusterName"]})
if not keyisset("CapacityProviders", props):
LOG.warning("No capacity providers defined. Setting it to default.")
cluster_params["CapacityProviders"] = DEFAULT_PROVIDERS
else:
cluster_params["CapacityProviders"] = props["CapacityProviders"]
if not keyisset("DefaultCapacityProviderStrategy", props):
LOG.warning("No Default Strategy set. Setting to default.")
cluster_params["DefaultCapacityProviderStrategy"] = DEFAULT_STRATEGY
else:
cluster_params[
"DefaultCapacityProviderStrategy"] = import_capacity_strategy(
props["DefaultCapacityProviderStrategy"])
cluster_params["Metadata"] = metadata
cluster_params["ClusterName"] = If(GENERATED_CLUSTER_NAME_CON_T,
Ref(AWS_STACK_NAME),
Ref(CLUSTER_NAME_T))
cluster = Cluster(CLUSTER_T, **cluster_params)
return cluster
def set_default_cluster_config(self, root_stack):
"""
Function to get the default defined ECS Cluster configuration
:return: cluster
:rtype: troposphere.ecs.Cluster
"""
self.log_group = LogGroup(
"EcsExecLogGroup",
LogGroupName=Sub(f"ecs/execute-logs/${{{AWS_STACK_NAME}}}"),
RetentionInDays=120,
)
self.cfn_resource = Cluster(
CLUSTER_T,
ClusterName=Ref(AWS_STACK_NAME),
CapacityProviders=FARGATE_PROVIDERS,
DefaultCapacityProviderStrategy=DEFAULT_STRATEGY,
Configuration=ClusterConfiguration(
ExecuteCommandConfiguration=ExecuteCommandConfiguration(
Logging="OVERRIDE",
LogConfiguration=ExecuteCommandLogConfiguration(
CloudWatchLogGroupName=Ref(self.log_group), ),
)),
Metadata=metadata,
)
root_stack.stack_template.add_resource(self.log_group)
root_stack.stack_template.add_resource(self.cfn_resource)
self.capacity_providers = FARGATE_PROVIDERS
self.default_strategy_providers = [
cap.CapacityProvider for cap in DEFAULT_STRATEGY
]
self.cluster_identifier = Ref(self.cfn_resource)
def add_ecs_cluster(self):
'''
Add ECS cluster
'''
self.cfn_template.add_resource(
Cluster(title=constants.CLUSTER)
)
return self.cfn_template
def _add_cluster(self):
cluster = Cluster(
'Cluster',
ClusterName=Ref('AWS::StackName'),
Tags=Tags(environment=self.env)
)
self.template.add_resource(cluster)
self._add_ec2_auto_scaling()
self._add_cluster_alarms(cluster)
return cluster
def get_default_cluster_config():
"""
Function to get the default defined ECS Cluster configuration
:return: cluster
:rtype: troposphere.ecs.Cluster
"""
return Cluster(
CLUSTER_T,
ClusterName=If(GENERATED_CLUSTER_NAME_CON_T, Ref(AWS_STACK_NAME),
Ref(CLUSTER_NAME_T)),
CapacityProviders=DEFAULT_PROVIDERS,
DefaultCapacityProviderStrategy=DEFAULT_STRATEGY,
Metadata=metadata,
)
def define_cluster(self, root_stack, settings: ComposeXSettings):
"""
Function to create the cluster from provided properties.
"""
props = import_record_properties(self.properties, Cluster)
props["Metadata"] = metadata
if not keyisset("ClusterName", props):
props["ClusterName"] = Ref(AWS_STACK_NAME)
if keyisset("DefaultCapacityProviderStrategy",
props) and not keyisset("CapacityProviders", props):
raise KeyError("When specifying DefaultCapacityProviderStrategy"
" you must specify CapacityProviders")
cluster_name = props["ClusterName"]
if self.parameters:
configuration = self.update_props_from_parameters(
cluster_name, root_stack, settings)
props["Configuration"] = configuration
self.cfn_resource = Cluster(CLUSTER_T, **props)
root_stack.stack_template.add_resource(self.cfn_resource)
self.cluster_identifier = Ref(self.cfn_resource)
def main():
"""Generates the CloudFormation template"""
template = Template()
template.add_version('2010-09-09')
template.add_description(
'This template deploys an ECS cluster to the ' +
'provided VPC and subnets using an Auto Scaling Group')
# Parameters
# EnvironmentName
env_name_param = template.add_parameter(
Parameter(
'EnvironmentName',
Type='String',
Description=
'An environment name that will be prefixed to resource names',
))
# InstanceType
instance_type_param = template.add_parameter(
Parameter(
'InstanceType',
Type='String',
Default='t2.nano',
Description=
'Which instance type should we use to build the ECS cluster?',
AllowedValues=[
't2.nano',
't2.micro',
't2.small',
't2.medium',
't2.large',
't2.xlarge',
't2.2xlarge',
],
))
# ClusterSize
cluster_size_param = template.add_parameter(
Parameter(
'ClusterSize',
Type='Number',
Description='How many ECS hosts do you want to initially deploy?',
Default='1',
))
# VPC
template.add_parameter(
Parameter(
'VPC',
Type='AWS::EC2::VPC::Id',
Description=
'Choose which VPC this ECS cluster should be deployed to',
))
# Subnets
subnets_param = template.add_parameter(
Parameter(
'Subnets',
Type='List<AWS::EC2::Subnet::Id>',
Description=
'Choose which subnets this ECS cluster should be deployed to',
))
# SecurityGroup
sg_param = template.add_parameter(
Parameter(
'SecurityGroup',
Type='AWS::EC2::SecurityGroup::Id',
Description=
'Select the Security Group to use for the ECS cluster hosts',
))
# Mappings
# AWSRegionToAMI
template.add_mapping(
'AWSRegionToAMI',
{
'us-east-1': {
'AMI': 'ami-a58760b3'
},
'us-east-2': {
'AMI': 'ami-a6e4bec3'
},
'us-west-1': {
'AMI': 'ami-74cb9b14'
},
'us-west-2': {
'AMI': 'ami-5b6dde3b'
},
'eu-west-1': {
'AMI': 'ami-e3fbd290'
},
'eu-west-2': {
'AMI': 'ami-77f6fc13'
},
'eu-central-1': {
'AMI': 'ami-38dc1157'
},
'ap-northeast-1': {
'AMI': 'ami-30bdce57'
},
'ap-southeast-1': {
'AMI': 'ami-9f75ddfc'
},
'ap-southeast-2': {
'AMI': 'ami-cf393cac'
},
'ca-central-1': {
'AMI': 'ami-1b01b37f'
},
},
)
# Resources
ecs_role = template.add_resource(
Role(
'ECSRole',
Path='/',
RoleName=Sub('${EnvironmentName}-ECSRole-${AWS::Region}'),
AssumeRolePolicyDocument=awacs.aws.Policy(Statement=[
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[awacs.aws.Action('sts', 'AssumeRole')],
Principal=awacs.aws.Principal('Service',
['ec2.amazonaws.com']),
),
]),
Policies=[
Policy(
PolicyName='ecs-service',
PolicyDocument=awacs.aws.Policy(Statement=[
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[
awacs.aws.Action('ecs', 'CreateCluster'),
awacs.aws.Action(
'ecs', 'DeregisterContainerInstance'),
awacs.aws.Action('ecs',
'DiscoverPollEndpoint'),
awacs.aws.Action('ecs', 'Poll'),
awacs.aws.Action('ecs',
'RegisterContainerInstance'),
awacs.aws.Action('ecs',
'StartTelemetrySession'),
awacs.aws.Action('ecs', 'Submit*'),
awacs.aws.Action('logs', 'CreateLogStream'),
awacs.aws.Action(
'ecr', 'BatchCheckLayerAvailability'),
awacs.aws.Action('ecr', 'BatchGetImage'),
awacs.aws.Action('ecr',
'GetDownloadUrlForLayer'),
awacs.aws.Action('ecr',
'GetAuthorizationToken'),
],
Resource=['*'],
),
], ),
),
],
))
ecs_instance_profile = template.add_resource(
InstanceProfile(
'ECSInstanceProfile',
Path='/',
Roles=[Ref(ecs_role)],
))
# ECSCluster
ecs_cluster = template.add_resource(
Cluster(
'ECSCluster',
ClusterName=Ref(env_name_param),
))
instance_metadata = Metadata(
Init({
'config':
InitConfig(
commands={
'01_add_instance_to_cluster': {
'command':
Join('', [
'#!/bin/bash\n', 'echo ECS_CLUSTER=',
Ref(ecs_cluster), ' >> /etc/ecs/ecs.config'
])
},
},
files=InitFiles({
'/etc/cfn/cfn-hup.conf':
InitFile(
mode='000400',
owner='root',
group='root',
content=Join('', [
'[main]\n', 'stack=',
Ref('AWS::StackId'), '\n', 'region=',
Ref('AWS::Region'), '\n'
]),
),
'/etc/cfn/hooks.d/cfn-auto-reloader.conf':
InitFile(
mode='000400',
owner='root',
group='root',
content=Join('', [
'[cfn-auto-reloader-hook]\n',
'triggers=post.update\n',
'path=Resources.ContainerInstances.Metadata.AWS::CloudFormation::Init\n'
'action=/opt/aws/bin/cfn-init -v --region ',
Ref('AWS::Region'), ' --stack ',
Ref('AWS::StackId'),
' --resource ECSLaunchConfiguration\n'
]),
)
}),
services=InitServices({
'cfn-hup':
InitService(enabled='true',
ensureRunning='true',
files=[
'/etc/cfn/cfn-hup.conf',
'/etc/cfn/hooks.d/cfn-auto-reloader.conf'
])
}),
)
}))
ecs_launch_config = template.add_resource(
LaunchConfiguration(
'ECSLaunchConfiguration',
ImageId=FindInMap('AWSRegionToAMI', Ref('AWS::Region'), 'AMI'),
InstanceType=Ref(instance_type_param),
SecurityGroups=[Ref(sg_param)],
IamInstanceProfile=Ref(ecs_instance_profile),
UserData=Base64(
Join('', [
'#!/bin/bash\n',
'yum install -y aws-cfn-bootstrap\n',
'/opt/aws/bin/cfn-init -v --region ',
Ref('AWS::Region'),
' --stack ',
Ref('AWS::StackName'),
' --resource ECSLaunchConfiguration\n',
'/opt/aws/bin/cfn-signal -e $? --region ',
Ref('AWS::Region'),
' --stack ',
Ref('AWS::StackName'),
' --resource ECSAutoScalingGroup\n',
])),
Metadata=instance_metadata,
))
# ECSAutoScalingGroup:
template.add_resource(
AutoScalingGroup(
'ECSAutoScalingGroup',
VPCZoneIdentifier=Ref(subnets_param),
LaunchConfigurationName=Ref(ecs_launch_config),
MinSize=Ref(cluster_size_param),
MaxSize=Ref(cluster_size_param),
DesiredCapacity=Ref(cluster_size_param),
Tags=ASTags(Name=(Sub('${EnvironmentName} ECS host'), True)),
CreationPolicy=CreationPolicy(
ResourceSignal=ResourceSignal(Timeout='PT15M'), ),
UpdatePolicy=UpdatePolicy(
AutoScalingRollingUpdate=AutoScalingRollingUpdate(
MinInstancesInService='1',
MaxBatchSize='1',
PauseTime='PT15M',
WaitOnResourceSignals=True,
)),
))
# Output
template.add_output(
Output(
'Cluster',
Description='A reference to the ECS cluster',
Value=Ref(ecs_cluster),
))
print(template.to_json())
Type="AWS::EC2::KeyPair::KeyName",
ConstraintDescription="must be the name of an existing EC2 KeyPair.",
))
t.add_parameter(Parameter("VpcId", Type="AWS::EC2::VPC::Id",
Description="VPC"))
t.add_parameter(
Parameter("PublicSubnet",
Type="List<AWS::EC2::Subnet::Id>",
Description="PublicSubnet",
ConstraintDescription="PublicSubnet"))
# Cluster Config
t.add_resource(Cluster(
"ECSCluster",
ClusterName="CodingTestCluster",
))
# Networking Config, VPC, Subnet, SecuritGroup
t.add_resource(
ec2.SecurityGroup("SecurityGroup",
GroupDescription="Allow SSH and private network access",
GroupName="CodingTestCluster-SG",
SecurityGroupIngress=[
ec2.SecurityGroupRule(IpProtocol="tcp",
FromPort=0,
ToPort=65535,
CidrIp="10.0.64.0/18"),
ec2.SecurityGroupRule(IpProtocol="tcp",
FromPort=0,
ToPort=65535,
def main():
t = Template()
t.add_version('2010-09-09')
t.set_description("AWS CloudFormation ECS example")
# Add the Parameters
AMI = t.add_parameter(Parameter(
"AMI",
Type="String",
))
ClusterSize = t.add_parameter(Parameter(
"ClusterSize",
Type="String",
))
ClusterType = t.add_parameter(Parameter(
"ClusterType",
Type="String",
))
InstanceType = t.add_parameter(Parameter(
"InstanceType",
Type="String",
))
IamInstanceProfile = t.add_parameter(
Parameter(
"IamInstanceProfile",
Type="String",
))
KeyName = t.add_parameter(
Parameter(
"KeyName",
Type="AWS::EC2::KeyPair::KeyName",
))
MaxClusterSize = t.add_parameter(
Parameter(
"MaxClusterSize",
Type="String",
))
RollingUpdate = t.add_parameter(Parameter(
"RollingUpdate",
Type="String",
))
Stage = t.add_parameter(Parameter(
"Stage",
Type="String",
))
Subnets = t.add_parameter(
Parameter(
"Subnets",
Type="List<AWS::EC2::Subnet::Id>",
))
VpcCidr = t.add_parameter(Parameter(
"VpcCidr",
Type="String",
))
VpcId = t.add_parameter(Parameter(
"VpcId",
Type="AWS::EC2::VPC::Id",
))
ContainerInstances = t.add_resource(
LaunchConfiguration(
'ContainerInstances',
UserData=Base64(
Join('', [
'#!/bin/bash -xe\n', 'echo ECS_CLUSTER=',
Ref('AWS::StackName'), '>> /etc/ecs/ecs.config\n',
'systemctl enable [email protected]\n',
'systemctl start [email protected]\n',
'/usr/bin/cfn-signal -e $? ', ' --stack ',
Ref('AWS::StackName'),
' --resource ECSAutoScalingGroup ',
' --region ',
Ref('AWS::Region'), '\n'
])),
ImageId=Ref(AMI),
KeyName=Ref(KeyName),
SecurityGroups=[Ref('EcsSecurityGroup')],
IamInstanceProfile=Ref(IamInstanceProfile),
InstanceType=Ref(InstanceType)))
ECSCluster = t.add_resource(
Cluster('EcsCluster', ClusterName=Ref('AWS::StackName')))
ECSAutoScalingGroup = t.add_resource(
AutoScalingGroup(
'ECSAutoScalingGroup',
DesiredCapacity=Ref(ClusterSize),
MinSize=Ref(ClusterSize),
MaxSize=Ref(MaxClusterSize),
VPCZoneIdentifier=Ref(Subnets),
LaunchConfigurationName=Ref('ContainerInstances'),
HealthCheckType="EC2",
UpdatePolicy=UpdatePolicy(
AutoScalingReplacingUpdate=AutoScalingReplacingUpdate(
WillReplace=True, ),
AutoScalingRollingUpdate=AutoScalingRollingUpdate(
PauseTime='PT5M',
MinInstancesInService=Ref(ClusterSize),
MaxBatchSize='1',
WaitOnResourceSignals=True)),
Tags=[
Tag("Project", "demo", True),
Tag("Stage", Ref(Stage), True),
Tag("Name", "home-ecs", True),
]))
t.add_resource(
ScalingPolicy("EcsAsgScaleDown",
AdjustmentType="PercentChangeInCapacity",
AutoScalingGroupName=Ref("ECSAutoScalingGroup"),
MetricAggregationType="Average",
MinAdjustmentMagnitude="1",
PolicyType="StepScaling",
StepAdjustments=[
StepAdjustments(MetricIntervalLowerBound="-10",
MetricIntervalUpperBound="0",
ScalingAdjustment="-10"),
StepAdjustments(MetricIntervalUpperBound="-10",
ScalingAdjustment="-20")
]))
t.add_resource(
ScalingPolicy('EcsScaleUp',
AdjustmentType="PercentChangeInCapacity",
AutoScalingGroupName=Ref("ECSAutoScalingGroup"),
EstimatedInstanceWarmup="300",
MetricAggregationType="Average",
MinAdjustmentMagnitude="1",
PolicyType="StepScaling",
StepAdjustments=[
StepAdjustments(MetricIntervalLowerBound="0",
MetricIntervalUpperBound="10",
ScalingAdjustment="10"),
StepAdjustments(MetricIntervalLowerBound="10",
ScalingAdjustment="20")
]))
t.add_resource(
cloudwatch.Alarm("EcsScaleDownAlarm",
ActionsEnabled="True",
MetricName="CPUUtilization",
AlarmActions=[Ref("EcsAsgScaleDown")],
AlarmDescription="Scale down ECS Instances",
Namespace="AWS/EC2",
Statistic="Average",
Period="60",
EvaluationPeriods="6",
Threshold="25",
ComparisonOperator="LessThanThreshold",
Dimensions=[
cloudwatch.MetricDimension(
Name="AutoScalingGroupName",
Value=Ref("ECSAutoScalingGroup"))
]))
t.add_resource(
cloudwatch.Alarm("EcsAsgScaleUpAlarm",
ActionsEnabled="True",
MetricName="CPUUtilization",
AlarmActions=[Ref("EcsScaleUp")],
AlarmDescription="Scale up ECS Instances",
Namespace="AWS/EC2",
Statistic="Average",
Period="60",
EvaluationPeriods="3",
Threshold="65",
ComparisonOperator="GreaterThanThreshold",
Dimensions=[
cloudwatch.MetricDimension(
Name="AutoScalingGroupName",
Value=Ref("ECSAutoScalingGroup"))
]))
EC2SecurityGroup = t.add_resource(
ec2.SecurityGroup('EcsSecurityGroup',
GroupDescription='ECS InstanceSecurityGroup',
SecurityGroupIngress=[
ec2.SecurityGroupRule(IpProtocol='tcp',
FromPort='22',
ToPort='22',
CidrIp='0.0.0.0/0'),
ec2.SecurityGroupRule(IpProtocol='tcp',
FromPort='31000',
ToPort='61000',
CidrIp='0.0.0.0/0')
],
VpcId=Ref(VpcId)))
with open("ecs-ec2-cluster-cf.yaml", "w") as yamlout:
yamlout.write(t.to_yaml())
def init_template(self):
self.template.add_description(self.TEMPLATE_DESCRIPTION)
ecs_cluster = self.template.add_resource(Cluster(self.CLUSTER_NAME))
ecs_instance_role = self.template.add_resource(
Role('sitInstanceRole',
Path='/',
AssumeRolePolicyDocument={
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": ["ec2.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}]
}))
ecs_instance_profile = self.template.add_resource(
InstanceProfile('sitInstanceProfile',
Path='/',
Roles=[Ref(ecs_instance_role)]))
ecs_instance_policy = self.template.add_resource(
PolicyType('sitInstancePolicy',
PolicyName='ecs-policy',
Roles=[Ref(ecs_instance_role)],
PolicyDocument={
"Statement": [{
"Effect":
"Allow",
"Action": [
"ecs:CreateCluster",
"ecs:RegisterContainerInstance",
"ecs:DeregisterContainerInstance",
"ecs:DiscoverPollEndpoint", "ecs:Submit*",
"ecs:Poll", "ecs:StartTelemetrySession",
"ecr:GetAuthorizationToken",
"ecr:BatchCheckLayerAvailability",
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage", "logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource":
"*"
}],
}))
commands = {
'01_add_instance_to_cluster': {
'command':
Join('', [
'#!/bin/bash\n', 'echo ECS_CLUSTER=',
Ref(ecs_cluster),
'$"\n"ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION=',
self.ECS_TASK_CLEANUP_WAIT, ' >> /etc/ecs/ecs.config'
])
}
}
files = {
"/etc/cfn/cfn-hup.conf": {
"content":
Join("", [
"[main]\n", "stack=",
Ref("AWS::StackId"), "\n", "region=",
Ref("AWS::Region"), "\n"
]),
"mode":
"000400",
"owner":
"root",
"group":
"root"
},
"/etc/cfn/hooks.d/cfn-auto-reloader.conf": {
"content":
Join("", [
"[cfn-auto-reloader-hook]\n", "triggers=post.update\n",
"path=Resources.{0}.Metadata.AWS::CloudFormation::Init\n".
format(self.LAUNCH_CONFIGURATION_NAME),
"action=/opt/aws/bin/cfn-init -v ", " --stack ",
Ref("AWS::StackName"), " --resource {0}".format(
self.LAUNCH_CONFIGURATION_NAME), " --region ",
Ref("AWS::Region"), "\n", "runas=root\n"
])
}
}
services = {
"sysvinit": {
"cfn-hup": {
"enabled":
"true",
"ensureRunning":
"true",
"files": [
"/etc/cfn/cfn-hup.conf",
"/etc/cfn/hooks.d/cfn-auto-reloader.conf"
]
}
}
}
launch_configuration = self.template.add_resource(
LaunchConfiguration(self.LAUNCH_CONFIGURATION_NAME,
ImageId=self.AMI_ID,
IamInstanceProfile=Ref(ecs_instance_profile),
InstanceType=self.INSTANCE_TYPE,
UserData=self.user_data.get_base64_data(),
AssociatePublicIpAddress=False,
SecurityGroups=self.SECURITY_GROUPS,
KeyName=self.KEY_NAME,
Metadata=autoscaling.Metadata(
cloudformation.Init({
"config":
cloudformation.InitConfig(
commands=commands,
files=files,
services=services)
})),
BlockDeviceMappings=[
autoscaling.BlockDeviceMapping(
DeviceName=self.EBS_DEVICE_NAME,
Ebs=autoscaling.EBSBlockDevice(
DeleteOnTermination=True,
VolumeSize=self.EBS_VOLUME_SIZE,
VolumeType='gp2'))
]))
auto_scaling_group = self.template.add_resource(
AutoScalingGroup(self.AUTOSCALING_GROUP_NAME,
MaxSize=self.MAX_SIZE,
MinSize=self.MIN_SIZE,
Cooldown=60,
LaunchConfigurationName=Ref(launch_configuration),
VPCZoneIdentifier=[self.SUBNET]))
""" Scale UP Policy """
scaling_up_policy = self.template.add_resource(
ScalingPolicy('{0}ScaleUpPolicy'.format(
self.AUTOSCALING_GROUP_NAME),
AdjustmentType='ChangeInCapacity',
AutoScalingGroupName=Ref(auto_scaling_group),
Cooldown=60,
ScalingAdjustment='1'))
for alarm_name, alarm in self.AUTOSCALE_UP_ALARMS.iteritems():
""" Cloud Watch Alarm """
self.template.add_resource(
Alarm('{0}ScaleUp{1}'.format(self.AUTOSCALING_GROUP_NAME,
alarm_name),
ActionsEnabled=True,
Namespace='AWS/ECS',
MetricName=alarm['scaling_metric'],
ComparisonOperator='GreaterThanOrEqualToThreshold',
Threshold=alarm['scale_up_threshold'],
EvaluationPeriods=1,
Statistic=alarm['statistic'],
Period=alarm['period'],
AlarmActions=[Ref(scaling_up_policy)],
Dimensions=[
MetricDimension(Name='ClusterName',
Value=Ref(ecs_cluster))
]))
""" Scale DOWN Policy """
scaling_down_policy = self.template.add_resource(
ScalingPolicy('{0}ScaleDownPolicy'.format(
self.AUTOSCALING_GROUP_NAME),
AdjustmentType='ChangeInCapacity',
AutoScalingGroupName=Ref(auto_scaling_group),
Cooldown=60,
ScalingAdjustment='-1'))
for alarm_name, alarm in self.AUTOSCALE_DOWN_ALARMS.iteritems():
""" Cloud Watch Alarm """
self.template.add_resource(
Alarm('{0}ScaleDown{1}'.format(self.AUTOSCALING_GROUP_NAME,
alarm_name),
ActionsEnabled=True,
Namespace='AWS/ECS',
MetricName=alarm['scaling_metric'],
ComparisonOperator='LessThanOrEqualToThreshold',
Threshold=alarm['scale_down_threshold'],
EvaluationPeriods=1,
Statistic=alarm['statistic'],
Period=alarm['period'],
AlarmActions=[Ref(scaling_down_policy)],
Dimensions=[
MetricDimension(Name='ClusterName',
Value=Ref(ecs_cluster))
]))
"Action": [
"ecr:BatchCheckLayerAvailability",
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage"
],
"Resource": "*",
}
]
}
),
]
))
hugged_cluster = t.add_resource(Cluster(
"HuggedCluster",
ClusterName="AllTheHugs",
))
cfn_hup = InitFile(
content=Sub(
"[main]\nstack=${AWS::StackId}\nregion=${AWS::Region}\n"
),
mode='000400',
owner='root',
group='root'
)
reloader = InitFile(
content=Sub("""
[cfn-auto-reloader-hook]
triggers=post.add, post.update
path=Resources.NetKANCompute.Metadata.AWS::CloudFormation::Init
def generate_template(d):
# Set template metadata
t = Template()
t.add_version("2010-09-09")
t.set_description(d["cf_template_description"])
# aws_account_id = Ref("AWS::AccountId")
# aws_region = Ref("AWS::Region")
# ALB SG
ALBSG = t.add_resource(
SecurityGroup(
"ALBSG",
GroupDescription="Enable HTTP access.",
SecurityGroupIngress=[
SecurityGroupRule(IpProtocol="tcp",
FromPort="80",
ToPort="80",
CidrIp="0.0.0.0/0")
],
VpcId=ImportValue(d["network_stack_name"] + "-VPCId"),
))
# ALB
ALB = t.add_resource(
elb.LoadBalancer(
"ALB",
Name=d["project_name"],
Scheme="internet-facing",
SecurityGroups=[Ref("ALBSG")],
Subnets=[
ImportValue(d["network_stack_name"] + "-PublicSubnetId1"),
ImportValue(d["network_stack_name"] + "-PublicSubnetId2"),
],
Tags=Tags(d["tags"]),
))
# ECS cluster
ECSCluster = t.add_resource(
Cluster("ECSCluster",
ClusterName=d["project_name"],
Tags=Tags(d["tags"])))
# ECS cluster SG
ClusterSG = t.add_resource(
SecurityGroup(
"ClusterSG",
GroupDescription="Enable HTTP access.",
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol="tcp",
FromPort="0",
ToPort="65535",
SourceSecurityGroupId=Ref(ALBSG),
)
],
VpcId=ImportValue(d["network_stack_name"] + "-VPCId"),
))
# ALB listener
listener80 = t.add_resource(
elb.Listener(
"Listener80",
Port="80",
Protocol="HTTP",
LoadBalancerArn=Ref("ALB"),
DefaultActions=[
elb.Action(
Type="fixed-response",
FixedResponseConfig=elb.FixedResponseConfig(
StatusCode="200",
MessageBody=(
"This is a fixed response for the default "
"ALB action"),
ContentType="text/plain",
),
)
],
))
# ECS service role
ECSClusterRole = t.add_resource(
Role("ECSClusterRole",
RoleName="ECSClusterRole",
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": "ecs-tasks.amazonaws.com"
},
"Action": "sts:AssumeRole",
}],
}))
# ECS Cluster Role Policy
t.add_resource(
PolicyType(
'ECSClusterRolePolicy',
PolicyName="ECSCLusterRolePolicy",
Roles=[Ref(ECSClusterRole)],
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Sid":
"ECSTaskManagement",
"Effect":
"Allow",
"Action": [
"ec2:AttachNetworkInterface",
"ec2:CreateNetworkInterface",
"ec2:CreateNetworkInterfacePermission",
"ec2:DeleteNetworkInterface",
"ec2:DeleteNetworkInterfacePermission",
"ec2:Describe*",
"ec2:DetachNetworkInterface",
"elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
"elasticloadbalancing:DeregisterTargets",
"elasticloadbalancing:Describe*",
"elasticloadbalancing:RegisterInstancesWithLoadBalancer",
"elasticloadbalancing:RegisterTargets",
"route53:ChangeResourceRecordSets",
"route53:CreateHealthCheck",
"route53:DeleteHealthCheck",
"route53:Get*",
"route53:List*",
"route53:UpdateHealthCheck",
"servicediscovery:DeregisterInstance",
"servicediscovery:Get*",
"servicediscovery:List*",
"servicediscovery:RegisterInstance",
"servicediscovery:UpdateInstanceCustomHealthStatus",
"ecr:*",
"cloudwatch:*",
"logs:*",
"iam:*",
],
"Resource":
"*"
}, {
"Sid": "AutoScaling",
"Effect": "Allow",
"Action": ["autoscaling:Describe*"],
"Resource": "*"
}, {
"Sid":
"AutoScalingManagement",
"Effect":
"Allow",
"Action": [
"autoscaling:DeletePolicy",
"autoscaling:PutScalingPolicy",
"autoscaling:SetInstanceProtection",
"autoscaling:UpdateAutoScalingGroup"
],
"Resource":
"*",
"Condition": {
"Null": {
"autoscaling:ResourceTag/AmazonECSManaged": "false"
}
}
}, {
"Sid":
"AutoScalingPlanManagement",
"Effect":
"Allow",
"Action": [
"autoscaling-plans:CreateScalingPlan",
"autoscaling-plans:DeleteScalingPlan",
"autoscaling-plans:DescribeScalingPlans"
],
"Resource":
"*"
}, {
"Sid":
"CWAlarmManagement",
"Effect":
"Allow",
"Action": [
"cloudwatch:DeleteAlarms", "cloudwatch:DescribeAlarms",
"cloudwatch:PutMetricAlarm"
],
"Resource":
"arn:aws:cloudwatch:*:*:alarm:*"
}, {
"Sid": "ECSTagging",
"Effect": "Allow",
"Action": ["ec2:CreateTags"],
"Resource": "arn:aws:ec2:*:*:network-interface/*"
}, {
"Sid":
"CWLogGroupManagement",
"Effect":
"Allow",
"Action": [
"logs:CreateLogGroup", "logs:DescribeLogGroups",
"logs:PutRetentionPolicy"
],
"Resource":
"arn:aws:logs:*:*:log-group:/aws/ecs/*"
}, {
"Sid":
"CWLogStreamManagement",
"Effect":
"Allow",
"Action": [
"logs:CreateLogStream", "logs:DescribeLogStreams",
"logs:PutLogEvents"
],
"Resource":
"arn:aws:logs:*:*:log-group:/aws/ecs/*:log-stream:*"
}]
}))
# Codebuild role
CodebuildDeveloperRole = t.add_resource(
Role("CodebuildDeveloperRole",
RoleName="CodebuilDevelopRole",
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": "codebuild.amazonaws.com"
},
"Action": "sts:AssumeRole",
}],
}))
# Codebuild developer role policy
t.add_resource(
PolicyType(
'CodebuildDeveloperRolePolicy',
PolicyName="CodebuildDeveloperRolePolicy",
Roles=[Ref(CodebuildDeveloperRole)],
PolicyDocument={
"Statement": [{
"Action": [
"codebuild:StartBuild", "codebuild:StopBuild",
"codebuild:BatchGet*", "codebuild:GetResourcePolicy",
"codebuild:DescribeTestCases", "codebuild:List*",
"codecommit:GetBranch", "codecommit:GetCommit",
"codecommit:GetRepository", "codecommit:ListBranches",
"cloudwatch:GetMetricStatistics",
"events:DescribeRule", "events:ListTargetsByRule",
"events:ListRuleNamesByTarget", "logs:GetLogEvents",
"s3:*", "logs:*", "ecr:*"
],
"Effect":
"Allow",
"Resource":
"*"
}, {
"Effect":
"Allow",
"Action": ["ssm:PutParameter"],
"Resource":
"arn:aws:ssm:*:*:parameter/CodeBuild/*"
}, {
"Sid":
"CodeStarNotificationsReadWriteAccess",
"Effect":
"Allow",
"Action": [
"codestar-notifications:CreateNotificationRule",
"codestar-notifications:DescribeNotificationRule",
"codestar-notifications:UpdateNotificationRule",
"codestar-notifications:Subscribe",
"codestar-notifications:Unsubscribe"
],
"Resource":
"*",
"Condition": {
"StringLike": {
"codestar-notifications:NotificationsForResource":
"arn:aws:codebuild:*"
}
}
}, {
"Sid":
"CodeStarNotificationsListAccess",
"Effect":
"Allow",
"Action": [
"codestar-notifications:ListNotificationRules",
"codestar-notifications:ListEventTypes",
"codestar-notifications:ListTargets",
"codestar-notifications:ListTagsforResource"
],
"Resource":
"*"
}, {
"Sid":
"SNSTopicListAccess",
"Effect":
"Allow",
"Action": ["sns:ListTopics", "sns:GetTopicAttributes"],
"Resource":
"*"
}],
"Version":
"2012-10-17"
}))
# Codepipeline role
CodePipelineRole = t.add_resource(
Role("CodePipelineRole",
RoleName="CodePipelineRole",
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": "codepipeline.amazonaws.com"
},
"Action": "sts:AssumeRole",
}],
}))
t.add_resource(
PolicyType(
'CodePipelineRolePolicy',
PolicyName="CodePipelineRolePolicy",
Roles=[Ref(CodePipelineRole)],
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": [
"cloudformation:CreateStack",
"cloudformation:DeleteStack",
"cloudformation:DescribeStacks",
"cloudformation:UpdateStack",
"cloudformation:CreateChangeSet",
"cloudformation:DeleteChangeSet",
"cloudformation:DescribeChangeSet",
"cloudformation:ExecuteChangeSet",
"cloudformation:SetStackPolicy",
"cloudformation:ValidateTemplate",
"iam:PassRole",
"s3:*",
],
"Resource":
"*",
"Effect":
"Allow"
}, {
"Action":
["codebuild:BatchGetBuilds", "codebuild:StartBuild"],
"Resource":
"*",
"Effect":
"Allow"
}, {
"Action": ["ecr:*"],
"Resource": "*",
"Effect": "Allow"
}, {
"Action": [
"ecs:DescribeServices", "ecs:DescribeTaskDefinition",
"ecs:DescribeTasks", "ecs:ListTasks",
"ecs:RegisterTaskDefinition", "ecs:UpdateService"
],
"Resource":
"*",
"Effect":
"Allow"
}, {
"Action": [
"codedeploy:CreateDeployment",
"codedeploy:GetDeployment",
"codedeploy:GetApplication",
"codedeploy:GetApplicationRevision",
"codedeploy:RegisterApplicationRevision",
"codedeploy:GetDeploymentConfig",
"ecs:RegisterTaskDefinition", "iam:PassRole"
],
"Resource":
"*",
"Effect":
"Allow"
}]
}))
# Outputs
# ListenerArnHTTP
t.add_output(
Output(
"ListenerArnHTTP",
Description="Listener Arn (HTTP) of the newly created Listener",
Export=Export(Sub("${AWS::StackName}-ListenerArnHTTP")),
Value=Ref(listener80),
))
t.add_output(
Output(
"ECSClusterRole",
Description="ECS Cluster role with managed policy",
Export=Export(Sub("${AWS::StackName}-ECSClusterRole")),
Value=GetAtt(ECSClusterRole, "Arn"),
))
t.add_output(
Output(
"CodePipelineRole",
Description="CodePipeline role with managed policy",
Export=Export(Sub("${AWS::StackName}-CodePipelineRole")),
Value=GetAtt(CodePipelineRole, "Arn"),
))
t.add_output(
Output(
"CodebuildDeveloperRole",
Description="Codebuild role with managed policy",
Export=Export(Sub("${AWS::StackName}-CodebuildDeveloperRole")),
Value=GetAtt(CodebuildDeveloperRole, "Arn"),
))
t.add_output(
Output(
"ECSClusterName",
Description="ECS cluster name.",
Export=Export(Sub("${AWS::StackName}-ECSClusterName")),
# Value=GetAtt(ECSCluster, "Arn"),
Value=Ref(ECSCluster),
))
t.add_output(
Output(
"ALB",
Description="ALB name.",
Export=Export(Sub("${AWS::StackName}-ALBName")),
Value=GetAtt(ALB, "LoadBalancerName"),
))
t.add_output(
Output(
"ECSClusterSG",
Description="ECS Cluster SG name.",
Export=Export(Sub("${AWS::StackName}-ECSClusterSG")),
Value=Ref(ClusterSG),
))
return t
def build_template(sierrafile):
template = Template()
template.add_version('2010-09-09')
template.add_metadata(build_interface(sierrafile.extra_params))
parameters = AttrDict(
# Network Parameters
vpc_cidr=template.add_parameter(Parameter(
'VpcCidr',
Type='String',
Default='192.172.0.0/16',
)),
subnet1_cidr=template.add_parameter(Parameter(
'Subnet1Cidr',
Type='String',
Default='192.172.1.0/24',
)),
subnet2_cidr=template.add_parameter(Parameter(
'Subnet2Cidr',
Type='String',
Default='192.172.2.0/24',
)),
# ECS Parameters
cluster_size=template.add_parameter(Parameter(
'ClusterSize',
Type='Number',
Default=2,
)),
instance_type=template.add_parameter(Parameter(
'InstanceType',
Type='String',
Default='t2.medium'
)),
key_name=template.add_parameter(Parameter(
'KeyName',
Type='AWS::EC2::KeyPair::KeyName',
)),
image_id=template.add_parameter(Parameter(
'ImageId',
Type='AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>',
Default=(
'/aws/service/ecs/optimized-ami'
'/amazon-linux/recommended/image_id'
),
Description=(
'An SSM parameter that resolves to a valid AMI ID.'
' This is the AMI that will be used to create ECS hosts.'
' The default is the current recommended ECS-optimized AMI.'
)
)),
# Other Parameters
github_token=template.add_parameter(Parameter(
'GitHubToken',
Type='String',
NoEcho=True,
)),
)
# Environment Variable Parameters
for env_var_param, env_var_name in sierrafile.extra_params:
template.add_parameter(Parameter(
env_var_param,
Type='String',
NoEcho=True,
))
# Resource Declarations
# # Network
network_vpc = template.add_resource(VPC(
'NetworkVpc',
CidrBlock=Ref(parameters.vpc_cidr),
Tags=Tags(Name=Ref('AWS::StackName')),
))
network_ig = template.add_resource(InternetGateway(
'NetworkInternetGateway',
Tags=Tags(Name=Ref('AWS::StackName')),
))
vpc_attach = template.add_resource(VPCGatewayAttachment(
'NetworkInternetGatewayAttachment',
InternetGatewayId=Ref(network_ig),
VpcId=Ref(network_vpc),
))
route_table = template.add_resource(RouteTable(
'NetworkRouteTable',
VpcId=Ref(network_vpc),
Tags=Tags(Name=Ref('AWS::StackName')),
))
template.add_resource(Route(
'NetworkDefaultRoute',
DependsOn=[vpc_attach.title],
RouteTableId=Ref(route_table),
DestinationCidrBlock='0.0.0.0/0',
GatewayId=Ref(network_ig),
))
subnet1 = template.add_resource(Subnet(
'NetworkSubnet1',
VpcId=Ref(network_vpc),
AvailabilityZone=Select(0, GetAZs()),
MapPublicIpOnLaunch=True,
CidrBlock=Ref(parameters.subnet1_cidr),
Tags=Tags(Name=Sub('${AWS::StackName} (Public)')),
))
subnet2 = template.add_resource(Subnet(
'NetworkSubnet2',
VpcId=Ref(network_vpc),
AvailabilityZone=Select(1, GetAZs()),
MapPublicIpOnLaunch=True,
CidrBlock=Ref(parameters.subnet2_cidr),
Tags=Tags(Name=Sub('${AWS::StackName} (Public)')),
))
template.add_resource(SubnetRouteTableAssociation(
'NetworkSubnet1RouteTableAssociation',
RouteTableId=Ref(route_table),
SubnetId=Ref(subnet1),
))
template.add_resource(SubnetRouteTableAssociation(
'NetworkSubnet2RouteTableAssociation',
RouteTableId=Ref(route_table),
SubnetId=Ref(subnet2),
))
elb = template.add_resource(LoadBalancer(
ELB_NAME,
Name=Sub('${AWS::StackName}-elb'),
Type='network',
Subnets=[Ref(subnet1), Ref(subnet2)],
))
# # Cluster
ecs_host_role = template.add_resource(Role(
'EcsHostRole',
AssumeRolePolicyDocument=PolicyDocument(
Statement=[Statement(
Effect=Allow,
Principal=Principal('Service', 'ec2.amazonaws.com'),
Action=[awacs.sts.AssumeRole]
)],
),
ManagedPolicyArns=[
'arn:aws:iam::aws:policy/'
'service-role/AmazonEC2ContainerServiceforEC2Role'
]
))
ecs_host_profile = template.add_resource(InstanceProfile(
'EcsHostInstanceProfile',
Roles=[Ref(ecs_host_role)]
))
ecs_host_sg = template.add_resource(SecurityGroup(
'EcsHostSecurityGroup',
GroupDescription=Sub('${AWS::StackName}-hosts'),
VpcId=Ref(network_vpc),
SecurityGroupIngress=[SecurityGroupRule(
CidrIp='0.0.0.0/0',
IpProtocol='-1'
)]
))
cluster = template.add_resource(Cluster(
'EcsCluster',
ClusterName=Ref('AWS::StackName')
))
autoscaling_name = 'EcsHostAutoScalingGroup'
launch_conf_name = 'EcsHostLaunchConfiguration'
launch_conf = template.add_resource(LaunchConfiguration(
launch_conf_name,
ImageId=Ref(parameters.image_id),
InstanceType=Ref(parameters.instance_type),
IamInstanceProfile=Ref(ecs_host_profile),
KeyName=Ref(parameters.key_name),
SecurityGroups=[Ref(ecs_host_sg)],
UserData=Base64(Sub(
'#!/bin/bash\n'
'yum install -y aws-cfn-bootstrap\n'
'/opt/aws/bin/cfn-init -v'
' --region ${AWS::Region}'
' --stack ${AWS::StackName}'
f' --resource {launch_conf_name}\n'
'/opt/aws/bin/cfn-signal -e $?'
' --region ${AWS::Region}'
' --stack ${AWS::StackName}'
f' --resource {autoscaling_name}\n'
)),
Metadata={
'AWS::CloudFormation::Init': {
'config': {
'commands': {
'01_add_instance_to_cluster': {
'command': Sub(
f'echo ECS_CLUSTER=${{{cluster.title}}}'
f' > /etc/ecs/ecs.config'
),
}
},
'files': {
'/etc/cfn/cfn-hup.conf': {
'mode': 0o400,
'owner': 'root',
'group': 'root',
'content': Sub(
'[main]\n'
'stack=${AWS::StackId}\n'
'region=${AWS::Region}\n'
),
},
'/etc/cfn/hooks.d/cfn-auto-reloader.conf': {
'content': Sub(
'[cfn-auto-reloader-hook]\n'
'triggers=post.update\n'
'path=Resources.ContainerInstances.Metadata'
'.AWS::CloudFormation::Init\n'
'action=/opt/aws/bin/cfn-init -v'
' --region ${AWS::Region}'
' --stack ${AWS::StackName}'
f' --resource {launch_conf_name}\n'
),
},
},
'services': {
'sysvinit': {
'cfn-hup': {
'enabled': True,
'ensureRunning': True,
'files': [
'/etc/cfn/cfn-hup.conf',
'/etc/cfn/hooks.d/cfn-auto-reloader.conf'
]
}
}
}
}
}
}
))
autoscaling_group = template.add_resource(AutoScalingGroup(
autoscaling_name,
VPCZoneIdentifier=[Ref(subnet1), Ref(subnet2)],
LaunchConfigurationName=Ref(launch_conf),
DesiredCapacity=Ref(parameters.cluster_size),
MinSize=Ref(parameters.cluster_size),
MaxSize=Ref(parameters.cluster_size),
Tags=[{
'Key': 'Name',
'Value': Sub('${AWS::StackName} - ECS Host'),
'PropagateAtLaunch': True,
}],
CreationPolicy=CreationPolicy(
ResourceSignal=ResourceSignal(Timeout='PT15M'),
),
UpdatePolicy=UpdatePolicy(
AutoScalingRollingUpdate=AutoScalingRollingUpdate(
MinInstancesInService=1,
MaxBatchSize=1,
PauseTime='PT5M',
WaitOnResourceSignals=True,
),
),
))
# # Services
task_role = template.add_resource(Role(
'TaskExecutionRole',
AssumeRolePolicyDocument=PolicyDocument(
Statement=[Statement(
Effect=Allow,
Principal=Principal('Service', 'ecs-tasks.amazonaws.com'),
Action=[awacs.sts.AssumeRole],
)],
),
ManagedPolicyArns=[
'arn:aws:iam::aws:policy/'
'service-role/AmazonECSTaskExecutionRolePolicy'
],
))
artifact_bucket = template.add_resource(Bucket(
'ArtifactBucket',
DeletionPolicy='Retain',
))
codebuild_role = template.add_resource(Role(
'CodeBuildServiceRole',
Path='/',
AssumeRolePolicyDocument=PolicyDocument(
Version='2012-10-17',
Statement=[
Statement(
Effect=Allow,
Principal=Principal(
'Service', 'codebuild.amazonaws.com'
),
Action=[
awacs.sts.AssumeRole,
],
),
],
),
Policies=[Policy(
PolicyName='root',
PolicyDocument=PolicyDocument(
Version='2012-10-17',
Statement=[
Statement(
Resource=['*'],
Effect=Allow,
Action=[
awacs.ssm.GetParameters,
],
),
Statement(
Resource=['*'],
Effect=Allow,
Action=[
awacs.s3.GetObject,
awacs.s3.PutObject,
awacs.s3.GetObjectVersion,
],
),
Statement(
Resource=['*'],
Effect=Allow,
Action=[
awacs.logs.CreateLogGroup,
awacs.logs.CreateLogStream,
awacs.logs.PutLogEvents,
],
),
],
),
)],
))
codepipeline_role = template.add_resource(Role(
'CodePipelineServiceRole',
Path='/',
AssumeRolePolicyDocument=PolicyDocument(
Version='2012-10-17',
Statement=[
Statement(
Effect=Allow,
Principal=Principal(
'Service', 'codepipeline.amazonaws.com'
),
Action=[
awacs.sts.AssumeRole,
],
),
],
),
Policies=[Policy(
PolicyName='root',
PolicyDocument=PolicyDocument(
Version='2012-10-17',
Statement=[
Statement(
Resource=[
Sub(f'${{{artifact_bucket.title}.Arn}}/*')
],
Effect=Allow,
Action=[
awacs.s3.GetBucketVersioning,
awacs.s3.GetObject,
awacs.s3.GetObjectVersion,
awacs.s3.PutObject,
],
),
Statement(
Resource=['*'],
Effect=Allow,
Action=[
awacs.ecs.DescribeServices,
awacs.ecs.DescribeTaskDefinition,
awacs.ecs.DescribeTasks,
awacs.ecs.ListTasks,
awacs.ecs.RegisterTaskDefinition,
awacs.ecs.UpdateService,
awacs.codebuild.StartBuild,
awacs.codebuild.BatchGetBuilds,
awacs.iam.PassRole,
],
),
],
),
)],
))
log_group = template.add_resource(LogGroup(
'LogGroup',
LogGroupName=Sub('/ecs/${AWS::StackName}'),
))
if any(conf.pipeline.enable for conf in sierrafile.services.values()):
project = template.add_resource(Project(
'CodeBuildProject',
Name=Sub('${AWS::StackName}-build'),
ServiceRole=Ref(codebuild_role),
Artifacts=Artifacts(Type='CODEPIPELINE'),
Source=Source(Type='CODEPIPELINE'),
Environment=Environment(
ComputeType='BUILD_GENERAL1_SMALL',
Image='aws/codebuild/docker:17.09.0',
Type='LINUX_CONTAINER',
),
))
for name, settings in sierrafile.services.items():
task_definition = template.add_resource(TaskDefinition(
f'{name}TaskDefinition',
RequiresCompatibilities=['EC2'],
Cpu=str(settings.container.cpu),
Memory=str(settings.container.memory),
NetworkMode='bridge',
ExecutionRoleArn=Ref(task_role.title),
ContainerDefinitions=[
ContainerDefinition(
Name=f'{name}',
Image=settings.container.image,
Memory=str(settings.container.memory),
Essential=True,
PortMappings=[
PortMapping(
ContainerPort=settings.container.port,
Protocol='tcp',
),
],
Environment=[
troposphere.ecs.Environment(Name=k, Value=v)
for k, v in sierrafile.env_vars.items()
if k in settings.get('environment', [])
],
LogConfiguration=LogConfiguration(
LogDriver='awslogs',
Options={
'awslogs-region': Ref('AWS::Region'),
'awslogs-group': Ref(log_group.title),
'awslogs-stream-prefix': Ref('AWS::StackName'),
},
),
),
],
))
target_group = template.add_resource(TargetGroup(
f'{name}TargetGroup',
Port=settings.container.port,
Protocol='TCP',
VpcId=Ref(network_vpc),
Tags=Tags(Name=Sub(f'${{AWS::StackName}}-{name}')),
))
listener = template.add_resource(Listener(
f'{name}ElbListener',
LoadBalancerArn=Ref(elb),
Port=settings.container.port,
Protocol='TCP',
DefaultActions=[
Action(TargetGroupArn=Ref(target_group), Type='forward')
],
))
service = template.add_resource(Service(
f'{name}Service',
Cluster=Ref(cluster),
ServiceName=f'{name}-service',
DependsOn=[autoscaling_group.title, listener.title],
DesiredCount=settings.container.count,
TaskDefinition=Ref(task_definition),
LaunchType='EC2',
LoadBalancers=[
troposphere.ecs.LoadBalancer(
ContainerName=f'{name}',
ContainerPort=settings.container.port,
TargetGroupArn=Ref(target_group),
),
],
))
if settings.pipeline.enable:
pipeline = template.add_resource(Pipeline(
f'{name}Pipeline',
RoleArn=GetAtt(codepipeline_role, 'Arn'),
ArtifactStore=ArtifactStore(
Type='S3',
Location=Ref(artifact_bucket),
),
Stages=[
Stages(
Name='Source',
Actions=[Actions(
Name='Source',
ActionTypeId=ActionTypeId(
Category='Source',
Owner='ThirdParty',
Version='1',
Provider='GitHub',
),
OutputArtifacts=[
OutputArtifacts(Name=f'{name}Source'),
],
RunOrder='1',
Configuration={
'Owner': settings.pipeline.user,
'Repo': settings.pipeline.repo,
'Branch': settings.pipeline.branch,
'OAuthToken': Ref(parameters.github_token),
},
)],
),
Stages(
Name='Build',
Actions=[Actions(
Name='Build',
ActionTypeId=ActionTypeId(
Category='Build',
Owner='AWS',
Version='1',
Provider='CodeBuild',
),
InputArtifacts=[
InputArtifacts(Name=f'{name}Source'),
],
OutputArtifacts=[
OutputArtifacts(Name=f'{name}Build'),
],
RunOrder='1',
Configuration={
'ProjectName': Ref(project),
},
)],
),
Stages(
Name='Deploy',
Actions=[Actions(
Name='Deploy',
ActionTypeId=ActionTypeId(
Category='Deploy',
Owner='AWS',
Version='1',
Provider='ECS',
),
InputArtifacts=[
InputArtifacts(Name=f'{name}Build')
],
RunOrder='1',
Configuration={
'ClusterName': Ref(cluster),
'ServiceName': Ref(service),
'FileName': 'image.json',
},
)],
),
],
))
template.add_resource(Webhook(
f'{name}CodePipelineWebhook',
Name=Sub(f'${{AWS::StackName}}-{name}-webhook'),
Authentication='GITHUB_HMAC',
AuthenticationConfiguration=AuthenticationConfiguration(
SecretToken=Ref(parameters.github_token),
),
Filters=[FilterRule(
JsonPath='$.ref',
MatchEquals=f'refs/heads/{settings.pipeline.branch}'
)],
TargetAction='Source',
TargetPipeline=Ref(pipeline),
TargetPipelineVersion=1,
RegisterWithThirdParty=True,
))
return template
def add_ecs_fargate_cluster(self):
ecs_cluster = Cluster("Cluster")
self.template.add_resource(ecs_cluster)
'ApplicationListener',
Certificates=[elasticloadbalancingv2.Certificate(
CertificateArn=acm_cluster_certificate_arn,
)],
LoadBalancerArn=Ref(application_load_balancer),
Protocol='HTTPS',
Port=443,
DefaultActions=[Action(
TargetGroupArn=Ref(application_target_group),
Type='forward',
)]
))
# ECS cluster
cluster = template.add_resource(Cluster(
"Cluster",
))
# ECS container role
container_instance_role = template.add_resource(iam.Role(
"ContainerInstanceRole",
AssumeRolePolicyDocument=dict(Statement=[dict(
Effect="Allow",
Principal=dict(Service=["ec2.amazonaws.com"]),
Action=["sts:AssumeRole"],
)]),
Path="/",
Policies=[
iam.Policy(
PolicyName="AssetsManagementPolicy",
PolicyDocument=dict(
"Value": Ref(parameters["Project"])
}],
TargetGroupAttributes=[
TargetGroupAttribute(Key="deregistration_delay.timeout_seconds",
Value="90"),
TargetGroupAttribute(Key="stickiness.enabled", Value="true"),
TargetGroupAttribute(Key="stickiness.type", Value="lb_cookie")
],
TargetType="ip",
UnhealthyThresholdCount=3,
VpcId=ImportValue("VPC")))
### ECS Cluster ###
resources["HelloWorldECSCluster"] = template.add_resource(
Cluster("HelloWorldECSCluster",
ClusterName=Join("-", [Ref(parameters["Project"]), "cluster"])))
### ECS Task Definition IAM Role ###
resources["HelloWorldTaskExecutionRole"] = template.add_resource(
Role(
"HelloWorldTaskExecutionRole",
AssumeRolePolicyDocument={
"Version":
"2008-10-17",
"Statement": [{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "ecs-tasks.amazonaws.com"
},
GroupId = GetAtt(security_group, "GroupId"),
CidrIp = "0.0.0.0/0",
IpProtocol = "tcp",
FromPort = "443",
ToPort = "443"
)
template.add_resource(alb_security_group)
template.add_resource(alb_security_group_ingress_http)
template.add_resource(alb_security_group_ingress_https)
cluster = Cluster(
region.replace("-", "") + "ecslive",
ClusterName = "live",
ClusterSettings = [
ClusterSetting(
Name = "containerInsights",
Value = "enabled"
)
]
)
ecs_role = Role(
region.replace("-", "") + "ecsrole",
AssumeRolePolicyDocument = {
"Statement": [{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
}
}]
"Action": [
"s3:ListBucket",
],
"Resource": ["arn:aws:s3:::status.ksp-ckan.space"],
},
],
}))
# Indexer Compute
# We could utilise an autoscaling group, but that is way
# more complicated for our use case. If at some point we'd
# to scale the service beyond a single instance (due to some
# infrastructure sponsorship) it wouldn't take more than
# adding an AutoScalingGroup + LoadBalancer to scale this.
netkan_ecs = t.add_resource(
Cluster('NetKANCluster', ClusterName='NetKANCluster'))
netkan_userdata = Sub("""
#!/bin/bash -xe
echo ECS_CLUSTER=NetKANCluster > /etc/ecs/ecs.config
yum install -y aws-cfn-bootstrap
# Install the files and packages from the metadata
/opt/aws/bin/cfn-init -v --stack ${AWS::StackName} \
--resource NetKANCompute --region ${AWS::Region}
# ECS Volumes are a pain and I don't want to shave any more yaks
mkdir /mnt/letsencrypt
mkfs.ext4 -L CKANCACHE /dev/xvdh
mkdir -p /mnt/ckan_cache
echo "LABEL=CKANCACHE /mnt/ckan_cache ext4 defaults 0 2" >> /etc/fstab
mount -a
UnhealthyThreshold="2",
Interval="100",
Timeout="10",
),
Scheme="internal")
template.add_output(
Output("LoadBalancerDNSName",
Description="Loadbalancer DNS",
Value=Join("",
["http://", GetAtt(load_balancer, "DNSName")])))
# ECS cluster
main_cluster = Cluster(
"MainCluster",
ClusterName=Join("", ["MainCluster-", Ref(AWS_STACK_NAME)]),
template=template,
)
# ECS container role
container_instance_role = iam.Role(
"ContainerInstanceRole",
template=template,
AssumeRolePolicyDocument=dict(Statement=[
dict(
Effect="Allow",
Principal=dict(Service=["ec2.amazonaws.com"]),
Action=["sts:AssumeRole"],
)
]),
Path="/",
def scaffold(self):
""" Create long lived stack resources for the cluster """
self.t.add_resource(
Cluster("Cluster", ClusterName=self.cluster_vars['name']))
OUTPUT_SG = ["ALB", "DB", "Cache", "Aux"]
for sg in OUTPUT_SG:
tmpsg = SecurityGroup(
"{}BadgeSg".format(sg),
GroupDescription=
"SG for {} to wear in order to talk to ecs instances".format(
sg),
VpcId=self.cluster_vars.get('vpc'))
self.t.add_resource(tmpsg)
self.t.add_output(
Output("{}BadgeSg".format(sg),
Description="{} Security Group Badge".format(sg),
Export=Export(Sub("${AWS::StackName}:%sBadgeSg" % sg)),
Value=GetAtt(tmpsg, "GroupId")))
# Refactor like this
### removing this because it's in the agent now
add_asg_cleanup(self.t,
sanitize_cfn_resource_name(self.cluster_vars['name']))
# add metric lambda
self.t.add_resource(
Function("ECSMetricLambda",
Code=Code(S3Bucket=Sub("${S3Bucket}"),
S3Key=Sub("${S3Prefix}/deployment.zip")),
Handler="metrics.cluster_metrics.lambda_handler",
Role=GetAtt("CronLambdaRole", "Arn"),
Runtime="python3.7",
MemorySize=128,
Timeout=300,
Environment=Environment(
Variables={
"CLUSTER": Sub("${ClusterName}"),
"ASGPREFIX": Sub("${ClusterName}-asg-"),
"REGION": Ref("AWS::Region")
})))
self.t.add_resource(
Role("CronLambdaRole",
AssumeRolePolicyDocument={
"Statement": [{
"Effect": "Allow",
"Action": "sts:AssumeRole",
"Principal": {
"Service": "lambda.amazonaws.com"
},
}]
},
Policies=[
Policy(PolicyName="logs-and-stuff",
PolicyDocument={
"Statement": [{
"Effect": "Allow",
"Action": ["logs:*"],
"Resource": "arn:aws:logs:*:*:*"
}, {
"Effect":
"Allow",
"Action": [
"ec2:DescribeAutoScalingGroups",
"ec2:UpdateAutoScalingGroup", "ecs:*",
"cloudwatch:PutMetricData"
],
"Resource":
"*"
}]
})
]))
# run metrics every minute
self.t.add_resource(
Rule(
"CronStats",
ScheduleExpression="rate(1 minute)",
Description="Cron for cluster stats",
Targets=[Target(Id="1", Arn=GetAtt("ECSMetricLambda",
"Arn"))]))
self.t.add_resource(
Permission("StatPerm",
Action="lambda:InvokeFunction",
FunctionName=GetAtt("ECSMetricLambda", "Arn"),
Principal="events.amazonaws.com",
SourceArn=GetAtt("CronStats", "Arn")))
t.add_description("""\
AWS CloudFormation ECS Cluster\
""")
# Parameters
ecsClusterName_param = t.add_parameter(Parameter(
"ECSClusterName",
Description="ECS Cluster Name",
Default=data['ClusterInfo']['Name'],
Type="String"
))
# Creating the ECS Cluster (using the name provided when running CloudFormation.
ECSCluster = t.add_resource(Cluster(
'ECSCluster',
ClusterName=Ref(ecsClusterName_param),
))
# Policy Amazon EC2 Container Registry - Enable our ECS Cluster to work with the ECR
PolicyEcr = t.add_resource(PolicyType(
'PolicyEcr',
PolicyName='EcrPolicy',
PolicyDocument={'Version': '2012-10-17',
'Statement': [{'Action': ['ecr:GetAuthorizationToken'],
'Resource': ['*'],
'Effect': 'Allow'},
{'Action': ['ecr:GetDownloadUrlForLayer',
'ecr:BatchGetImage',
'ecr:BatchCheckLayerAvailability'
],
application_listener = Listener(
'ApplicationListener',
template=template,
LoadBalancerArn=Ref(application_load_balancer),
Protocol='HTTP',
Port=8080,
DefaultActions=[Action(
TargetGroupArn=Ref(application_target_group),
Type='forward',
)]
)
# ECS cluster
cluster = Cluster(
"Cluster",
template=template,
)
# ECS container role
container_instance_role = iam.Role(
"ContainerInstanceRole",
template=template,
AssumeRolePolicyDocument=dict(Statement=[dict(
Effect="Allow",
Principal=dict(Service=["ec2.amazonaws.com"]),
Action=["sts:AssumeRole"],
)]),
Path="/",
Policies=[
iam.Policy(
NetworkConfiguration,
PortMapping,
Service,
TaskDefinition,
)
t = Template()
t.set_version("2010-09-09")
t.add_parameter(
Parameter(
"Subnet",
Type="AWS::EC2::Subnet::Id",
Description="A VPC subnet ID for the container.",
))
cluster = t.add_resource(Cluster("Cluster"))
task_definition = t.add_resource(
TaskDefinition(
"TaskDefinition",
RequiresCompatibilities=["FARGATE"],
Cpu="256",
Memory="512",
NetworkMode="awsvpc",
ContainerDefinitions=[
ContainerDefinition(
Name="nginx",
Image="nginx",
Essential=True,
PortMappings=[PortMapping(ContainerPort=80)],
)
from troposphere import Parameter, Ref, Template
from troposphere.ecs import (Cluster, Service, TaskDefinition,
ContainerDefinition, NetworkConfiguration,
AwsvpcConfiguration, PortMapping)
t = Template()
t.add_version('2010-09-09')
t.add_parameter(
Parameter(
'Subnet',
Type='AWS::EC2::Subnet::Id',
Description='A VPC subnet ID for the container.',
))
cluster = t.add_resource(Cluster('Cluster'))
task_definition = t.add_resource(
TaskDefinition('TaskDefinition',
RequiresCompatibilities=['FARGATE'],
Cpu='256',
Memory='512',
NetworkMode='awsvpc',
ContainerDefinitions=[
ContainerDefinition(
Name='nginx',
Image='nginx',
Essential=True,
PortMappings=[PortMapping(ContainerPort=80)])
]))
service = t.add_resource(
],
)
),
ImageId="ami-13f84d60",
KeyName="yourkey",
SecurityGroups=["sg-96114ef2"],
IamInstanceProfile=Ref("EC2InstanceProfile"),
InstanceType="t2.micro",
AssociatePublicIpAddress="true",
)
)
ECSCluster = t.add_resource(
Cluster(
"ECSCluster",
)
)
ECSAutoScalingGroup = t.add_resource(
AutoScalingGroup(
"ECSAutoScalingGroup",
DesiredCapacity="1",
MinSize="1",
MaxSize="1",
VPCZoneIdentifier=["subnet-72849a0a", "subnet-72849a08"],
AvailabilityZones=["eu-west-1a", "eu-west-1b"],
LaunchConfigurationName=Ref("ContainerInstances"),
)
)
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=0,
ToPort=65535,
CidrIp="172.16.0.0/12",
),
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort="22",
ToPort="22",
CidrIp=PublicCidrIp,
),
],
VpcId=Ref("VpcId")))
t.add_resource(Cluster('ECSCluster', ))
t.add_resource(
Role(
'EcsClusterRole',
ManagedPolicyArns=[
'arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM',
'arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly',
'arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role',
'arn:aws:iam::aws:policy/CloudWatchFullAccess'
],
AssumeRolePolicyDocument={
'Version':
'2012-10-17',
'Statement': [{
'Action': 'sts:AssumeRole',
def __create_ecs():
template = Template()
desired_count = template.add_parameter(
parameter=Parameter(title='DesiredCount', Default=1, Type='Number'))
cpu = template.add_parameter(
parameter=Parameter(title='Cpu', Default=256, Type='Number'))
memory = template.add_parameter(
parameter=Parameter(title='Memory', Default=512, Type='Number'))
cluster = template.add_resource(resource=Cluster(title='SampleCluster', ))
log_group = template.add_resource(resource=LogGroup(
title='SampleLogGroup', LogGroupName='/aws/ecs/sample'))
container_name = 'sample-nginx'
task_definition = template.add_resource(resource=TaskDefinition(
title='SampleTaskDefinition',
Cpu=Ref(cpu),
Family='sample-fargate-task',
RequiresCompatibilities=['FARGATE'],
Memory=Ref(memory),
NetworkMode='awsvpc',
ExecutionRoleArn=Sub(
'arn:aws:iam::${AWS::AccountId}:role/ecsTaskExecutionRole'),
ContainerDefinitions=[
ContainerDefinition(
Image='nginx:latest',
Name=container_name,
PortMappings=[
PortMapping(ContainerPort=80, HostPort=80, Protocol='tcp')
],
LogConfiguration=LogConfiguration(
LogDriver='awslogs',
Options={
'awslogs-region': Ref('AWS::Region'),
'awslogs-group': Ref(log_group),
'awslogs-stream-prefix': 'nginx'
}))
]))
template.add_resource(resource=Service(
title='SampleService',
ServiceName='sample-fargate',
Cluster=Ref(cluster),
DesiredCount=Ref(desired_count),
TaskDefinition=Ref(task_definition),
LaunchType='FARGATE',
NetworkConfiguration=NetworkConfiguration(
AwsvpcConfiguration=AwsvpcConfiguration(
AssignPublicIp='ENABLED',
SecurityGroups=[
ImportValue(ExportName.TASK_SECURITY_GROUP.value)
],
Subnets=[
ImportValue(
CommonResource.ExportName.PUBLIC_SUBNET_A_ID.value),
ImportValue(
CommonResource.ExportName.PUBLIC_SUBNET_B_ID.value),
])),
LoadBalancers=[
EcsLoadBalancer(ContainerName=container_name,
ContainerPort=80,
TargetGroupArn=ImportValue(
ExportName.TARGET_GROUP.value))
]))
output_template_file(template, 'ecs.yml')