def create_load_balancer(self):
t = self.template
elb_name = ELB_NAME % self.name
elb_sg = ELB_SG_NAME % self.name
t.add_resource(elb.LoadBalancer(
elb_name,
HealthCheck=elb.HealthCheck(
Target='HTTP:80/',
HealthyThreshold=3,
UnhealthyThreshold=3,
Interval=5,
Timeout=3),
Listeners=self.setup_listeners(),
SecurityGroups=[Ref(elb_sg), ],
Subnets=Ref("PublicSubnets"),
Condition="CreateELB"))
# Setup ELB DNS
t.add_resource(
RecordSetType(
'%sDnsRecord' % elb_name,
# Appends a '.' to the end of the domain
HostedZoneName=Join("", [Ref("BaseDomain"), "."]),
Comment='Router ELB DNS',
Name=Join('.', [Ref("ELBHostName"), Ref("BaseDomain")]),
Type='CNAME',
TTL='120',
ResourceRecords=[
GetAtt(elb_name, 'DNSName')],
Condition="SetupELBDNS"))
def create_load_balancer(self, worker_lb_security_group):
worker_lb_name = 'elbWorker'
return self.add_resource(elb.LoadBalancer(
worker_lb_name,
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=300,
),
CrossZone=True,
SecurityGroups=[Ref(worker_lb_security_group)],
Listeners=[
elb.Listener(
LoadBalancerPort='80',
InstancePort='80',
Protocol='HTTP',
)
],
HealthCheck=elb.HealthCheck(
Target='HTTP:80/',
HealthyThreshold='3',
UnhealthyThreshold='2',
Interval='60',
Timeout='10',
),
Subnets=Ref(self.public_subnets),
Tags=self.get_tags(Name=worker_lb_name)
))
def create_load_balancer(self):
t = self.template
t.add_resource(
elb.LoadBalancer('EmpireControllerLoadBalancer',
HealthCheck=elb.HealthCheck(
Target='HTTP:8080/health',
HealthyThreshold=3,
UnhealthyThreshold=3,
Interval=5,
Timeout=3),
Listeners=self.setup_listeners(),
SecurityGroups=[
Ref(ELB_SG_NAME),
],
Subnets=Ref("PublicSubnets")))
# Setup ELB DNS
t.add_resource(
RecordSetType('EmpireControllerElbDnsRecord',
Condition="UseDNS",
HostedZoneName=Join("",
[Ref("ExternalDomain"), "."]),
Comment='Router ELB DNS',
Name=Join('.',
["empire", Ref("ExternalDomain")]),
Type='CNAME',
TTL='120',
ResourceRecords=[
GetAtt("EmpireControllerLoadBalancer", 'DNSName')
]))
def create_load_balancer(self):
t = self.template
t.add_resource(
elb.LoadBalancer(
"LoadBalancer",
HealthCheck=elb.HealthCheck(Target="HTTP:8081/health",
HealthyThreshold=3,
UnhealthyThreshold=3,
Interval=5,
Timeout=3),
ConnectionSettings=elb.ConnectionSettings(
IdleTimeout=3600), # 1 hour
Listeners=self.setup_listeners(),
SecurityGroups=[
Ref(ELB_SG_NAME),
],
Subnets=Ref("PublicSubnets")))
# Setup ELB DNS
t.add_resource(
RecordSetType("ElbDnsRecord",
HostedZoneName=Join("",
[Ref("ExternalDomain"), "."]),
Comment="Router ELB DNS",
Name=Join(".",
["empire", Ref("ExternalDomain")]),
Type="CNAME",
TTL="120",
ResourceRecords=[GetAtt("LoadBalancer", "DNSName")]))
def getTemplate(self):
#elbHealthCheckJson = simplejson.loads(simplejson.dumps(self.elbHealthCheckConfig))
self.template.add_resource(
LoadBalancer(
self.elbname,
ConnectionDrainingPolicy=self.getConnectionDrainingPolicy(),
Subnets=[self.subnet],
HealthCheck=elb.HealthCheck(
Target=self.elbHealthCheckConfig['Target'],
HealthyThreshold=self.
elbHealthCheckConfig['HealthyThreshold'],
UnhealthyThreshold=self.
elbHealthCheckConfig['UnhealthyThreshold'],
Interval=self.elbHealthCheckConfig['Interval'],
Timeout=self.elbHealthCheckConfig['Timeout'],
),
Listeners=[
self.getELBListener(),
],
Scheme="internal",
SecurityGroups=[self.externalSecurityGroup],
LoadBalancerName=Join("_", [self.friendlyName, self.branch]),
CrossZone=True,
))
return self.template
def create_load_balancer(self, tile_server_lb_security_group):
tile_server_lb_name = 'elbTileServer'
return self.add_resource(elb.LoadBalancer(
tile_server_lb_name,
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=300,
),
CrossZone=True,
SecurityGroups=[Ref(tile_server_lb_security_group)],
Listeners=[
elb.Listener(
LoadBalancerPort='80',
InstancePort='80',
Protocol='HTTP',
),
elb.Listener(
LoadBalancerPort='443',
InstancePort='80',
Protocol='HTTPS',
SSLCertificateId=Ref(self.ssl_certificate_arn)
)
],
HealthCheck=elb.HealthCheck(
Target='HTTP:80/health-check/',
HealthyThreshold='3',
UnhealthyThreshold='2',
Interval='30',
Timeout='5',
),
Subnets=Ref(self.public_subnets),
Tags=self.get_tags(Name=tile_server_lb_name)
))
def _load_balancer(self):
return LoadBalancer(
"LoadBalancer",
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=120,
),
AccessLoggingPolicy=elb.AccessLoggingPolicy(
EmitInterval=5,
Enabled=True,
S3BucketName=self.config['elb_logging_bucket'],
S3BucketPrefix="ELB",
),
Subnets=self.config['public_subnet'],
HealthCheck=elb.HealthCheck(
Target="HTTP:80/",
HealthyThreshold="5",
UnhealthyThreshold="2",
Interval="20",
Timeout="15",
),
Listeners=[
elb.Listener(
LoadBalancerPort=self.config['elb_port'],
InstancePort=self.config['instance_port'],
Protocol="HTTP",
InstanceProtocol="HTTP",
),
],
CrossZone=True,
SecurityGroups=self.config['elb_sg'],
LoadBalancerName="duy%sELB" % self.config['env'],
Scheme="internet-facing",
)
def __init__(self, title, **kwargs):
super().__init__(title, **kwargs)
self.AccessLoggingPolicy = If(
'LoadBalancerLog',
elb.AccessLoggingPolicy(
EmitInterval=get_endvalue('LoadBalancerLog'),
Enabled=True,
S3BucketName=Sub(cfg.BucketLogs),
S3BucketPrefix=''), Ref('AWS::NoValue'))
self.ConnectionDrainingPolicy = elb.ConnectionDrainingPolicy(
Enabled=True, Timeout=5)
self.ConnectionSettings = elb.ConnectionSettings(
IdleTimeout=get_endvalue('LoadBalancerIdleTimeout'))
self.CrossZone = True
self.HealthCheck = elb.HealthCheck(
HealthyThreshold=get_endvalue('HealthyThresholdCount'),
Interval=get_endvalue('HealthCheckIntervalSeconds'),
Target=get_endvalue('HealthCheckTarget'),
Timeout=get_endvalue('HealthCheckTimeoutSeconds'),
UnhealthyThreshold=get_endvalue('UnhealthyThresholdCount'))
self.LBCookieStickinessPolicy = If('LoadBalancerCookieSticky', [
elb.LBCookieStickinessPolicy(PolicyName='LBCookieStickinessPolicy',
CookieExpirationPeriod=get_endvalue(
'LoadBalancerCookieSticky'))
], Ref('AWS::NoValue'))
self.SecurityGroups = [GetAtt('SecurityGroupLoadBalancer', 'GroupId')]
def add_loadbalancer(self):
self.LoadBalancer = self.template.add_resource(
elasticloadbalancing.LoadBalancer(
"LoadBalancer",
ConnectionDrainingPolicy=elasticloadbalancing.
ConnectionDrainingPolicy(
Enabled=True,
Timeout=120,
),
Subnets=self.subnets,
HealthCheck=elasticloadbalancing.HealthCheck(
Target="HTTP:80/",
HealthyThreshold="2",
UnhealthyThreshold="2",
Interval="5",
Timeout="4",
),
Listeners=[
elasticloadbalancing.Listener(LoadBalancerPort="80",
InstancePort="80",
Protocol="TCP",
InstanceProtocol="TCP"),
],
CrossZone=True,
SecurityGroups=[Ref(self.securityGroupLoadBalancer)],
LoadBalancerName="demo-elb",
Scheme="internet-facing",
))
def create_frontend_elb(stack, cert=None):
"""Add EC2 ELB Resource."""
return stack.stack.add_resource(
elb.LoadBalancer(
'ElasticLoadBalancer',
Subnets=[Ref(stack.frontend1_subnet),
Ref(stack.frontend2_subnet)],
Listeners=[
elb.Listener(
LoadBalancerPort='80',
InstancePort='80',
Protocol='HTTP',
InstanceProtocol='HTTP',
),
elb.Listener(
LoadBalancerPort='443',
InstancePort='443',
Protocol='HTTPS',
InstanceProtocol='HTTPS',
SSLCertificateId=cert,
),
],
HealthCheck=elb.HealthCheck(
Target='SSL:443',
HealthyThreshold='5',
UnhealthyThreshold='2',
Interval='15',
Timeout='5',
),
CrossZone=True,
SecurityGroups=[Ref(stack.frontend_security_group)],
Scheme='internet-facing',
))
class PypiElb(skies.resources.EXTERNAL_HTTPS_ELB):
EXPOSED_PORT = '443' # default
ALLOW_IN = [ ] # same as before
# extracts attribute 'ssl_cert_id' from the atlas that binds to this
# template
SSL_CERT = skies.fields.Bound('ssl_cert_id')
# TS health check
HEALTH_CHECK = elb.HealthCheck(Target='TCP:443',
HealthyThreshold='3',
UnhealthyThreshold='5',
Interval='30',
Timeout='5')
def create_load_balancer(self, elb_args):
'''
Method creates a elastic load balancer and adds it to the resources list
@param elb_args [dict] collection of keyword arguments for the elastic load balancer
'''
health_check_args = elb_args['health_check']
health_check = elb.HealthCheck(
Target=health_check_args['target'],
HealthyThreshold=health_check_args['healthy_threshold'],
UnhealthyThreshold=health_check_args['unhealthy_threshold'],
Interval=health_check_args['interval'],
Timeout=health_check_args['timeout'],
)
listener_args = elb_args['listener']
listener = elb.Listener(
LoadBalancerPort=listener_args['load_balancer_port'],
InstancePort=listener_args['instance_port'],
Protocol=listener_args['protocol'],
InstanceProtocol=listener_args['instance_protocol'],
)
self.load_balancer = LoadBalancer(
"LoadBalancer",
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=120,
),
Subnets=self.subnets,
HealthCheck=health_check,
Listeners=[
listener,
],
CrossZone=True,
SecurityGroups=[Ref(self.load_balancer_security_group)],
LoadBalancerName="Cloudformation-Loadbalancer",
Scheme="internet-facing",
)
self.add_resource(self.load_balancer)
if elb_args.has_key('output_dns'):
self.add_output(
Output("ELBDNS",
Description="DNS Name of ELB",
Value=Join(
"",
["http://",
GetAtt(self.load_balancer, "DNSName")])))
def add_elb(self):
self.ElasticLoadBalancer = self.template.add_resource(
elb.LoadBalancer(
"ElbWeb",
Subnets=[Ref(self.Subnet1),
Ref(self.Subnet2)],
Listeners=[{
"InstancePort": "80",
"LoadBalancerPort": "80",
"Protocol": "HTTP"
}],
CrossZone="true",
LoadBalancerName=Join("-", ["elb", Ref(self.Project)]),
SecurityGroups=[Ref(self.ElbSecurityGroup)],
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=300,
),
HealthCheck=elb.HealthCheck(
HealthyThreshold="3",
Interval="30",
Target="HTTP:80/",
Timeout="5",
UnhealthyThreshold="5",
),
Tags=Tags(
Name=Join("-", ["ELB", Ref(self.Project)]),
Environment=Ref(self.Environment),
),
))
self.ELBcname = self.template.add_resource(
route53.RecordSetType(
"ELBcname",
HostedZoneName=Join("", [Ref(self.Domain), "."]),
Comment="CNAME to Web ELB",
Name=Join(
".",
[Ref(self.Hostname), Ref(self.Domain)]),
Type="CNAME",
TTL="60",
ResourceRecords=[GetAtt(self.ElasticLoadBalancer, "DNSName")]))
DependsOn="AttachGateway",
LoadBalancerName=Join("", [Ref("CellName"), "-" + name.lower()]),
CrossZone="true",
Scheme="internal" if internal else "internet-facing",
SecurityGroups=security_groups,
# Note that if multiple subnets defined, they must be in separate AZs.
Subnets=[Ref(private_subnet) if internal else Ref(public_subnet)],
Listeners=[{
"InstancePort": str(instance_port),
"LoadBalancerPort": "80",
"Protocol": "HTTP"
}],
HealthCheck=elb.HealthCheck(
HealthyThreshold="3",
Interval="30",
Target="HTTP:%d%s" % (instance_port, target),
Timeout="5",
UnhealthyThreshold="5",
),
Tags=Tags(
cell=Ref(CellName),
),
))
MarathonElb = create_load_balancer(t,
"Marathon",
8080,
"/status"
)
def set_up_stack(self):
"""Sets up the stack"""
if not self.INPUTS or not self.STACK_NAME_PREFIX or not self.HEALTH_ENDPOINT:
raise MKInputError(
'Must define INPUTS, STACK_NAME_PREFIX, and HEALTH_ENDPOINT')
super(AppServerStack, self).set_up_stack()
tags = self.get_input('Tags').copy()
self.add_description('{} App Server Stack for Cac'.format(
self.STACK_NAME_PREFIX))
assert isinstance(tags, dict), 'tags must be a dictionary'
self.availability_zones = get_availability_zones()
tags.update({'StackType': 'AppServer'})
self.default_tags = tags
self.app_server_instance_type_parameter = self.add_parameter(
Parameter(
'AppServerInstanceType',
Type='String',
Default='t2.medium',
Description='NAT EC2 instance type',
AllowedValues=EC2_INSTANCE_TYPES,
ConstraintDescription='must be a valid EC2 instance type.'),
source='AppServerInstanceType')
self.param_app_server_iam_profile = self.add_parameter(
Parameter('AppServerIAMProfile',
Type='String',
Description='IAM Profile for instances'),
source='AppServerIAMProfile')
self.app_server_ami = self.add_parameter(Parameter(
'AppServerAMI',
Type='String',
Description='{} Server EC2 AMI'.format(self.STACK_NAME_PREFIX)),
source='AppServerAMI')
self.keyname_parameter = self.add_parameter(Parameter(
'KeyName',
Type='String',
Default='cac',
Description='Name of an existing EC2 key pair'),
source='KeyName')
self.param_color = self.add_parameter(Parameter(
'StackColor',
Type='String',
Description='Stack color',
AllowedValues=['Blue', 'Green', 'Orange']),
source='StackColor')
self.param_stacktype = self.add_parameter(Parameter(
'StackType',
Type='String',
Description='Stack type',
AllowedValues=['Development', 'Staging', 'Production']),
source='StackType')
self.param_public_hosted_zone_name = self.add_parameter(
Parameter('PublicHostedZoneName',
Type='String',
Description='Public hosted zone name'),
source='PublicHostedZoneName')
self.param_vpc = self.add_parameter(Parameter(
'VpcId', Type='String', Description='Name of an existing VPC'),
source='VpcId')
self.param_notification_arn = self.add_parameter(
Parameter(
'GlobalNotificationsARN',
Type='String',
Description='Physical resource ID on an AWS::SNS::Topic for '
'notifications'),
source='GlobalNotificationsARN')
self.param_ssl_certificate_arn = self.add_parameter(
Parameter('SSLCertificateARN',
Type='String',
Description=
'Physical resource ID on an AWS::IAM::ServerCertificate '
'for the application server load balancer'),
source='SSLCertificateARN')
self.param_public_subnets = self.add_parameter(
Parameter('PublicSubnets',
Type='CommaDelimitedList',
Description='A list of public subnets'),
source='AppServerPublicSubnets')
self.param_private_subnets = self.add_parameter(
Parameter('PrivateSubnets',
Type='CommaDelimitedList',
Description='A list of private subnets'),
source='AppServerPrivateSubnets')
self.param_bastion_security_group = self.add_parameter(
Parameter('BastionSecurityGroup',
Type='String',
Description='The ID of the bastion security group'),
source='BastionSecurityGroup')
self.param_database_security_group = self.add_parameter(
Parameter('DatabaseSecurityGroup',
Type='String',
Description='The ID of the database security group'),
source='DatabaseSecurityGroup')
self.param_nat_security_group = self.add_parameter(
Parameter('NATSecurityGroup',
Type='String',
Description='The ID of the NAT security group'),
source='NATSecurityGroup')
self.param_min_size = self.add_parameter(Parameter(
'ASGMinSize',
Type='Number',
Default='1',
Description='Min size of ASG'),
source='ASGMinSize')
self.param_max_size = self.add_parameter(Parameter(
'ASGMaxSize',
Type='Number',
Default='1',
Description='Max size of ASG'),
source='ASGMaxSize')
self.param_desired_capacity = self.add_parameter(
Parameter('ASGDesiredCapacity',
Type='Number',
Default='1',
Description='Desired capacity of ASG'),
source='ASGDesiredCapacity')
#
# Security Group
#
app_server_load_balancer_security_group = self.add_resource(
ec2.SecurityGroup(
'sgAppServerLoadBalancer',
GroupDescription=
'Enables access to app servers via a load balancer',
VpcId=Ref(self.param_vpc),
SecurityGroupIngress=[
ec2.SecurityGroupRule(IpProtocol='tcp',
CidrIp=ALLOW_ALL_CIDR,
FromPort=p,
ToPort=p) for p in [80, 443]
],
Tags=Tags(Name='sgAppServerLoadBalancer',
Color=Ref(self.param_color))))
app_server_security_group = self.add_resource(
ec2.SecurityGroup(
'sgAppServer',
GroupDescription='Enables access to App Servers',
VpcId=Ref(self.param_vpc),
SecurityGroupIngress=[
ec2.SecurityGroupRule(IpProtocol='tcp',
CidrIp=VPC_CIDR,
FromPort=p,
ToPort=p) for p in [22, 80, 443]
] + [
ec2.SecurityGroupRule(IpProtocol='tcp',
SourceSecurityGroupId=Ref(sg),
FromPort=80,
ToPort=80)
for sg in [app_server_load_balancer_security_group]
] + [
ec2.SecurityGroupRule(IpProtocol='tcp',
SourceSecurityGroupId=Ref(sg),
FromPort=443,
ToPort=443)
for sg in [app_server_load_balancer_security_group]
],
SecurityGroupEgress=[
ec2.SecurityGroupRule(IpProtocol='tcp',
CidrIp=ALLOW_ALL_CIDR,
FromPort=p,
ToPort=p)
for p in [80, 443, PAPERTRAIL_PORT]
],
Tags=Tags(Name='sgAppServer', Color=Ref(self.param_color))))
# ELB to App Server
self.add_resource(
ec2.SecurityGroupEgress(
'sgEgressELBtoAppHTTP',
GroupId=Ref(app_server_load_balancer_security_group),
DestinationSecurityGroupId=Ref(app_server_security_group),
IpProtocol='tcp',
FromPort=80,
ToPort=80))
self.add_resource(
ec2.SecurityGroupEgress(
'sgEgressELBtoAppHTTPS',
GroupId=Ref(app_server_load_balancer_security_group),
DestinationSecurityGroupId=Ref(app_server_security_group),
IpProtocol='tcp',
FromPort=443,
ToPort=443))
# Bastion to App Server, app server to db, app server to inet
rules = [(self.param_bastion_security_group, app_server_security_group,
[80, 443, 22]),
(app_server_security_group,
self.param_database_security_group, [POSTGRES]),
(app_server_security_group, self.param_nat_security_group,
[80, 443, 22, 587, PAPERTRAIL_PORT])]
for num, (srcsg, destsg, ports) in enumerate(rules):
for port in ports:
self.add_resource(
ec2.SecurityGroupEgress(
'sgEgress{}p{}'.format(num, port),
GroupId=Ref(srcsg),
DestinationSecurityGroupId=Ref(destsg),
IpProtocol='tcp',
FromPort=port,
ToPort=port))
self.add_resource(
ec2.SecurityGroupIngress('sgIngress{}p{}'.format(
num, port),
GroupId=Ref(destsg),
SourceSecurityGroupId=Ref(srcsg),
IpProtocol='tcp',
FromPort=port,
ToPort=port))
#
# ELB
#
app_server_load_balancer = self.add_resource(
elb.LoadBalancer(
'elbAppServer',
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True, Timeout=300),
CrossZone=True,
SecurityGroups=[Ref(app_server_load_balancer_security_group)],
Listeners=[
elb.Listener(LoadBalancerPort='80',
Protocol='HTTP',
InstancePort='80',
InstanceProtocol='HTTP'),
elb.Listener(LoadBalancerPort='443',
Protocol='HTTPS',
InstancePort='443',
InstanceProtocol='HTTP',
SSLCertificateId=Ref(
self.param_ssl_certificate_arn))
],
HealthCheck=elb.HealthCheck(
Target=self.HEALTH_ENDPOINT,
HealthyThreshold='3',
UnhealthyThreshold='2',
Interval='30',
Timeout='5',
),
Subnets=Ref(self.param_public_subnets),
Tags=Tags(Name='elbAppServer', Color=Ref(self.param_color))))
self.add_resource(
cw.Alarm('alarmAppServerBackend4xx',
AlarmActions=[Ref(self.param_notification_arn)],
Statistic='Sum',
Period=300,
Threshold='5',
EvaluationPeriods=1,
ComparisonOperator='GreaterThanThreshold',
MetricName='HTTPCode_Backend_4XX',
Namespace='AWS/ELB',
Dimensions=[
cw.MetricDimension(
'metricLoadBalancerName',
Name='LoadBalancerName',
Value=Ref(app_server_load_balancer))
]))
self.add_resource(
cw.Alarm('alarmAppServerBackend5xx',
AlarmActions=[Ref(self.param_notification_arn)],
Statistic='Sum',
Period=60,
Threshold='0',
EvaluationPeriods=1,
ComparisonOperator='GreaterThanThreshold',
MetricName='HTTPCode_Backend_5XX',
Namespace='AWS/ELB',
Dimensions=[
cw.MetricDimension(
'metricLoadBalancerName',
Name='LoadBalancerName',
Value=Ref(app_server_load_balancer))
]))
#
# ASG
#
app_server_launch_config = self.add_resource(
asg.LaunchConfiguration(
'lcAppServer',
ImageId=Ref(self.app_server_ami),
IamInstanceProfile=Ref(self.param_app_server_iam_profile),
InstanceType=Ref(self.app_server_instance_type_parameter),
KeyName=Ref(self.keyname_parameter),
SecurityGroups=[Ref(app_server_security_group)]))
autoscaling_group = self.add_resource(
asg.AutoScalingGroup(
'asgAppServer',
AvailabilityZones=self.get_input(
'AppServerAvailabilityZones').split(','),
Cooldown=300,
DesiredCapacity=Ref(self.param_desired_capacity),
HealthCheckGracePeriod=600,
HealthCheckType='ELB',
LaunchConfigurationName=Ref(app_server_launch_config),
LoadBalancerNames=[Ref(app_server_load_balancer)],
MaxSize=Ref(self.param_max_size),
MinSize=Ref(self.param_min_size),
NotificationConfiguration=asg.NotificationConfiguration(
TopicARN=Ref(self.param_notification_arn),
NotificationTypes=[
asg.EC2_INSTANCE_LAUNCH, asg.EC2_INSTANCE_LAUNCH_ERROR,
asg.EC2_INSTANCE_TERMINATE,
asg.EC2_INSTANCE_TERMINATE_ERROR
]),
VPCZoneIdentifier=Ref(self.param_private_subnets),
Tags=[
asg.Tag('Name', '{}Server'.format(self.STACK_NAME_PREFIX),
True),
asg.Tag('Color', Ref(self.param_color), True)
]))
# autoscaling policies
autoscaling_policy_add = self.add_resource(
asg.ScalingPolicy('scalingPolicyAddAppServer',
AdjustmentType='ChangeInCapacity',
AutoScalingGroupName=Ref(autoscaling_group),
Cooldown=600,
ScalingAdjustment='1'))
autoscaling_policy_remove = self.add_resource(
asg.ScalingPolicy('scalingPolicyRemoveAppServer',
AdjustmentType='ChangeInCapacity',
AutoScalingGroupName=Ref(autoscaling_group),
Cooldown=600,
ScalingAdjustment='-1'))
if self.STACK_NAME_PREFIX == 'Otp':
# trigger scale down if CPU avg usage < 10% for 3 consecutive 5 min periods
self.add_resource(
cw.Alarm('alarmAppServerLowCPU',
AlarmActions=[Ref(autoscaling_policy_remove)],
Statistic='Average',
Period=300,
Threshold='10',
EvaluationPeriods=3,
ComparisonOperator='LessThanThreshold',
MetricName='CPUUtilization',
Namespace='AWS/EC2',
Dimensions=[
cw.MetricDimension('metricAutoScalingGroupName',
Name='AutoScalingGroupName',
Value=Ref(autoscaling_group))
]))
# trigger scale up if CPU avg usage >= 30% for a 5 min period
self.add_resource(
cw.Alarm('alarmAppServerHighCPU',
AlarmActions=[
Ref(self.param_notification_arn),
Ref(autoscaling_policy_add)
],
Statistic='Average',
Period=300,
Threshold='30',
EvaluationPeriods=1,
ComparisonOperator='GreaterThanOrEqualToThreshold',
MetricName='CPUUtilization',
Namespace='AWS/EC2',
Dimensions=[
cw.MetricDimension('metricAutoScalingGroupName',
Name='AutoScalingGroupName',
Value=Ref(autoscaling_group))
]))
else:
# scale web servers based on network usage
self.add_resource(
cw.Alarm('alarmAppServerLowNetworkUsage',
AlarmActions=[Ref(autoscaling_policy_remove)],
Statistic='Average',
Period=300,
Threshold='500000',
EvaluationPeriods=3,
ComparisonOperator='LessThanThreshold',
MetricName='NetworkOut',
Namespace='AWS/EC2',
Dimensions=[
cw.MetricDimension('metricAutoScalingGroupName',
Name='AutoScalingGroupName',
Value=Ref(autoscaling_group))
]))
self.add_resource(
cw.Alarm('alarmAppServerHighNetworkUsage',
AlarmActions=[
Ref(self.param_notification_arn),
Ref(autoscaling_policy_add)
],
Statistic='Average',
Period=300,
Threshold='10000000',
EvaluationPeriods=1,
ComparisonOperator='GreaterThanOrEqualToThreshold',
MetricName='NetworkOut',
Namespace='AWS/EC2',
Dimensions=[
cw.MetricDimension('metricAutoScalingGroupName',
Name='AutoScalingGroupName',
Value=Ref(autoscaling_group))
]))
#
# DNS name
#
self.create_resource(
route53.RecordSetType(
'dnsName',
Name=Join('.', [
Ref(self.param_color),
Ref(self.param_stacktype), self.STACK_NAME_PREFIX,
Ref(self.param_public_hosted_zone_name)
]),
Type='A',
AliasTarget=route53.AliasTarget(
GetAtt(app_server_load_balancer,
'CanonicalHostedZoneNameID'),
GetAtt(app_server_load_balancer, 'DNSName')),
HostedZoneName=Ref(self.param_public_hosted_zone_name)))
self.add_output([
Output('{}ServerLoadBalancerEndpoint'.format(
self.STACK_NAME_PREFIX),
Description='Application server endpoint',
Value=GetAtt(app_server_load_balancer, 'DNSName')),
Output('{}ServerLoadBalancerHostedZoneNameID'.format(
self.STACK_NAME_PREFIX),
Description='ID of canonical hosted zone name for ELB',
Value=GetAtt(app_server_load_balancer,
'CanonicalHostedZoneNameID'))
])
def generate_cloudformation_template():
enable_elb = sys.argv[1]
input_scaling_policies = ast.literal_eval(sys.argv[2])
input_alarms = ast.literal_eval(sys.argv[3])
enable_elb = enable_elb == 'True'
elb_listeners = ast.literal_eval(sys.argv[4])
template = Template()
template.add_description("""\
Configures Auto Scaling Group for the app""")
project_name = template.add_parameter(
Parameter(
"Name",
Type="String",
Description="Instances will be tagged with this name",
))
scalecapacity = template.add_parameter(
Parameter(
"ScaleCapacity",
Default="1",
Type="String",
Description="Number of api servers to run",
))
minsize = template.add_parameter(
Parameter(
"MinScale",
Type="String",
Description="Minimum number of servers to keep in the ASG",
))
maxsize = template.add_parameter(
Parameter(
"MaxScale",
Type="String",
Description="Maximum number of servers to keep in the ASG",
))
signalcount = template.add_parameter(
Parameter(
"SignalCount",
Default="1",
Type="String",
Description=
"No. of signals CF must receive before it sets the status as CREATE_COMPLETE",
))
signaltimeout = template.add_parameter(
Parameter(
"SignalTimeout",
Default="PT5M",
Type="String",
Description=
"Time that CF waits for the number of signals that was specified in Count ",
))
minsuccessfulinstancespercent = template.add_parameter(
Parameter(
"MinSuccessfulInstancesPercent",
Default="100",
Type="String",
Description=
"% instances in a rolling update that must signal success for CF to succeed",
))
environment = template.add_parameter(
Parameter(
"Environment",
Type="String",
Description="The environment being deployed into",
))
subnet = template.add_parameter(
Parameter(
"Subnets",
Type="CommaDelimitedList",
))
launchconfigurationname = template.add_parameter(
Parameter(
"LaunchConfigurationName",
Type="String",
))
health_check_grace_period = template.add_parameter(
Parameter(
"HealthCheckGracePeriod",
Type="String",
Default="300",
))
if enable_elb:
elb_subnets = template.add_parameter(
Parameter(
"LoadBalancerSubnets",
Type="CommaDelimitedList",
))
elb_bucket_name = template.add_parameter(
Parameter("LoadBalancerBucketName",
Type="String",
Description="S3 Bucket for the ELB access logs"))
template.add_condition("ElbLoggingCondition",
Not(Equals(Ref(elb_bucket_name), "")))
elb_schema = template.add_parameter(
Parameter(
"LoadBalancerSchema",
Type="String",
))
health_check_interval = template.add_parameter(
Parameter(
"LoadBalancerHealthCheckInterval",
Type="String",
))
health_check_timeout = template.add_parameter(
Parameter(
"LoadBalancerHealthCheckTimeout",
Type="String",
))
healthy_threshold = template.add_parameter(
Parameter(
"LoadBalancerHealthyThreshold",
Type="String",
))
unhealthy_threshold = template.add_parameter(
Parameter(
"LoadBalancerUnHealthyThreshold",
Type="String",
))
enable_connection_draining = template.add_parameter(
Parameter(
"LoadBalancerEnableConnectionDraining",
Type="String",
Default="True",
))
connection_draining_timeout = template.add_parameter(
Parameter(
"LoadBalancerConnectionDrainingTimeout",
Type="String",
Default="30",
))
loadbalancersecuritygroup = template.add_parameter(
Parameter(
"LoadBalancerSecurityGroup",
Type="CommaDelimitedList",
Description="Security group for api app load balancer.",
))
hostedzone = template.add_parameter(
Parameter(
"HostedZoneName",
Description=
"The DNS name of an existing Amazon Route 53 hosted zone",
Type="String",
))
dns_record = template.add_parameter(
Parameter(
"DNSRecord",
Type="String",
))
dns_ttl = template.add_parameter(
Parameter(
"DNSTTL",
Default="300",
Type="String",
))
new_weight = template.add_parameter(
Parameter(
"NewDnsWeight",
Type="String",
Default="100",
))
health_check_protocol = template.add_parameter(
Parameter(
"LoadBalancerHealthCheckProtocol",
Type="String",
))
template.add_condition("ElbTCPProtocolCondition",
Equals(Ref(health_check_protocol), "TCP"))
health_check_port = template.add_parameter(
Parameter(
"LoadBalancerHealthCheckPort",
Type="String",
))
health_check_path = template.add_parameter(
Parameter(
"LoadBalancerHealthCheckPath",
Type="String",
))
load_balancer_listeners = []
for listener in elb_listeners:
load_balancer_listeners.append(
elb.Listener(
LoadBalancerPort=listener['load_balancer_port'],
InstancePort=listener['instance_port'],
Protocol=listener['protocol'],
InstanceProtocol=Ref(health_check_protocol),
))
loadbalancer = template.add_resource(
elb.LoadBalancer(
"LoadBalancer",
AccessLoggingPolicy=If(
"ElbLoggingCondition",
elb.AccessLoggingPolicy(EmitInterval=60,
Enabled=True,
S3BucketName=Ref(elb_bucket_name),
S3BucketPrefix="ELBLogs"),
Ref("AWS::NoValue")),
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=Ref(enable_connection_draining),
Timeout=Ref(connection_draining_timeout),
),
Subnets=Ref(elb_subnets),
HealthCheck=elb.HealthCheck(
Target=Join("", [
Ref(health_check_protocol), ":",
Ref(health_check_port),
If("ElbTCPProtocolCondition", Ref("AWS::NoValue"),
Ref(health_check_path))
]),
HealthyThreshold=Ref(healthy_threshold),
UnhealthyThreshold=Ref(unhealthy_threshold),
Interval=Ref(health_check_interval),
Timeout=Ref(health_check_timeout),
),
Listeners=load_balancer_listeners,
CrossZone=True,
SecurityGroups=Ref(loadbalancersecuritygroup),
Scheme=Ref(elb_schema)))
route53record = template.add_resource(
RecordSetType(
"DNS",
HostedZoneName=Join("", [Ref(hostedzone), "."]),
Name=Join("", [Ref(dns_record), ".",
Ref(hostedzone), "."]),
ResourceRecords=[GetAtt(loadbalancer, "DNSName")],
SetIdentifier=Ref(project_name),
TTL=Ref(dns_ttl),
Type="CNAME",
Weight=Ref(new_weight),
))
autoscalinggroup = template.add_resource(
AutoScalingGroup(
"AutoscalingGroup",
Tags=[
Tag("Name", Ref(project_name), True),
Tag("Environment", Ref(environment), True)
],
LaunchConfigurationName=Ref(launchconfigurationname),
MinSize=Ref(minsize),
MaxSize=Ref(maxsize),
DesiredCapacity=Ref(scalecapacity),
VPCZoneIdentifier=Ref(subnet),
HealthCheckGracePeriod=Ref(health_check_grace_period),
CreationPolicy=CreationPolicy(
ResourceSignal=ResourceSignal(Count=Ref(signalcount),
Timeout=Ref(signaltimeout)),
AutoScalingCreationPolicy=AutoScalingCreationPolicy(
MinSuccessfulInstancesPercent=Ref(
minsuccessfulinstancespercent))),
UpdatePolicy=UpdatePolicy(
AutoScalingRollingUpdate=AutoScalingRollingUpdate(
MaxBatchSize='1',
MinInstancesInService='1',
MinSuccessfulInstancesPercent=Ref(
minsuccessfulinstancespercent),
PauseTime=Ref(signaltimeout),
WaitOnResourceSignals=True))))
autoscalinggroup.HealthCheckType = 'EC2'
if enable_elb:
autoscalinggroup.LoadBalancerNames = [Ref(loadbalancer)]
autoscalinggroup.HealthCheckType = 'ELB'
created_scaling_policies = dict()
for scaling_policy in input_scaling_policies:
policy_properties = {
'AdjustmentType': scaling_policy['adjustment_type'],
'AutoScalingGroupName': Ref(autoscalinggroup),
'Cooldown': scaling_policy['cooldown'],
'PolicyType': scaling_policy['policy_type'],
'ScalingAdjustment': scaling_policy['scaling_adjustment'],
}
if scaling_policy['policy_type'] != "SimpleScaling" \
and 'estimated_instance_warmup' in scaling_policy:
policy_properties['EstimatedInstanceWarmup'] = \
scaling_policy['estimated_instance_warmup']
if scaling_policy['policy_type'] != "SimpleScaling" \
and 'metric_aggregation_type' in scaling_policy:
policy_properties['MetricAggregationType'] = scaling_policy[
'metric_aggregation_type']
if scaling_policy['adjustment_type'] == "PercentChangeInCapacity" \
and 'min_adjustment_magnitude' in scaling_policy:
policy_properties['MinAdjustmentMagnitude'] = scaling_policy[
'min_adjustment_magnitude']
if 'step_adjustments' in scaling_policy:
policy_properties['StepAdjustments'] = scaling_policy[
'step_adjustments']
created_scaling_policies[
scaling_policy['name']] = template.add_resource(
ScalingPolicy(scaling_policy['name'], **policy_properties))
for alarm in input_alarms:
template.add_resource(
Alarm(
alarm['name'],
ActionsEnabled=True,
AlarmActions=[
Ref(created_scaling_policies[alarm['scaling_policy_name']])
],
AlarmDescription=alarm['description'],
ComparisonOperator=alarm['comparison'],
Dimensions=[
MetricDimension(Name="AutoScalingGroupName",
Value=Ref(autoscalinggroup)),
],
EvaluationPeriods=alarm['evaluation_periods'],
InsufficientDataActions=[],
MetricName=alarm['metric'],
Namespace=alarm['namespace'],
OKActions=[],
Period=alarm['period'],
Statistic=alarm['statistics'],
Threshold=str(alarm['threshold']),
Unit=alarm['unit'],
))
template.add_output(
Output("StackName", Value=Ref(project_name), Description="Stack Name"))
if enable_elb:
template.add_output(
Output("DomainName",
Value=Ref(route53record),
Description="DNS to access the service"))
template.add_output(
Output("LoadBalancer",
Value=GetAtt(loadbalancer, "DNSName"),
Description="ELB dns"))
template.add_output(
Output("AutoScalingGroup",
Value=Ref(autoscalinggroup),
Description="Auto Scaling Group"))
template.add_output(
Output("LaunchConfiguration",
Value=Ref(launchconfigurationname),
Description="LaunchConfiguration for this deploy"))
return template
def __init__(self, title, template, network_config, elb_config):
"""
Public Class to create an Elastic Loadbalancer in the unit stack environment
AWS Cloud Formation: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-elb.html
Troposphere: https://github.com/cloudtools/troposphere/blob/master/troposphere/elasticloadbalancing.py
:param title: Name of the Cloud formation stack object
:param template: The troposphere template to add the Elastic Loadbalancer to.
:param network_config: object containing network related variables
:param elb_config: object containing elb related variables, including list of listeners (elb_listener_config)
"""
self.title = title + 'Elb'
self.elb_r53 = None
self.elb_config = elb_config
self.network_config = network_config
super(Elb, self).__init__(vpc=network_config.vpc, title=title, template=template)
elb_listeners = elb_config.elb_listeners_config
subnets = network_config.public_subnets if elb_config.public_unit is True else network_config.private_subnets
# Create ELB
self.trop_elb = self.template.add_resource(
elb.LoadBalancer(self.title,
CrossZone=True,
HealthCheck=elb.HealthCheck(
Target=elb_config.elb_health_check,
HealthyThreshold=elb_config.healthy_threshold,
UnhealthyThreshold=elb_config.unhealthy_threshold,
Interval=elb_config.interval,
Timeout=elb_config.timeout),
Listeners=[elb.Listener(LoadBalancerPort=elb_listener.loadbalancer_port,
Protocol=elb_listener.loadbalancer_protocol,
InstancePort=elb_listener.instance_port,
InstanceProtocol=elb_listener.instance_protocol) for elb_listener
in elb_listeners],
Scheme='internet-facing' if elb_config.public_unit is True else 'internal',
SecurityGroups=[self.security_group],
Subnets=subnets,
Tags=Tags(Name=self.title)))
if network_config.get_depends_on():
self.trop_elb.DependsOn = network_config.get_depends_on()
# App sticky session cookies
sticky_app_cookie_policies = []
# sticky_app_cookie defaults to None, gather listeners that have cookies
listeners_with_cookies = [listener for listener in elb_listeners if listener.sticky_app_cookie]
for listener_num, listener in enumerate(listeners_with_cookies):
policy_name = self.title + 'AppCookiePolicy' + listener.sticky_app_cookie \
+ str(listener.instance_port) + str(listener.loadbalancer_port) \
+ str(listener.instance_protocol)
sticky_app_cookie_policies.append(elb.AppCookieStickinessPolicy(
CookieName=listener.sticky_app_cookie,
PolicyName=policy_name
))
# Even though ELB.Listeners.PolicyNames is a List in the cloudformation documentation,
# it only accepts a single list element, not multiple...
self.trop_elb.Listeners[listener_num].PolicyNames = [policy_name]
if sticky_app_cookie_policies:
self.trop_elb.AppCookieStickinessPolicy = sticky_app_cookie_policies
# Create SSL for Listeners
for listener in self.trop_elb.Listeners:
if elb_config.ssl_certificate_id:
listener.SSLCertificateId = elb_config.ssl_certificate_id
# Create ELB Log Bucket
if elb_config.elb_log_bucket:
self.trop_elb.AccessLoggingPolicy = elb.AccessLoggingPolicy(
EmitInterval='60',
Enabled=True,
S3BucketName=elb_config.elb_log_bucket,
S3BucketPrefix=Join('', [Ref('AWS::StackName'),
'-',
self.title])
)
if not elb_config.public_unit:
self.create_r53_record(network_config.private_hosted_zone_domain)
elif network_config.public_hosted_zone_name:
self.create_r53_record(network_config.public_hosted_zone_name)
else:
self.template.add_output(Output(
self.trop_elb.title,
Description='URL of the {0} ELB'.format(self.title),
Value=Join('', ['http://', GetAtt(self.trop_elb, 'DNSName')])
))
self.network_config.endpoints[title] = GetAtt(self.trop_elb, 'DNSName')
def elb_adder(self, name, instance_list, dns):
ports = self.config['apps'].values()[0].get("ports", None)
elb_type = self.config['apps'].values()[0].get("type",
'internet-facing')
elb_check_type = self.config['apps'].values()[0].get(
"elb_check_type", 'TCP')
elb_check_path = self.config['apps'].values()[0].get(
"elb_check_path", "")
if 'HTTP' in elb_check_type.upper() and not elb_check_path:
elb_check_path = "/"
if instance_list is None:
instance_list = []
elb_tags = self.__tag_elb_role_generator()
elb_id = ''.join(ch for ch in name if ch.isalnum())
elb_sec_group = name + "_group"
elb_sec_group = elb_sec_group.translate(None, '_')
elasticlb = self.template.add_resource(
elb.LoadBalancer(
elb_id,
Subnets=self.config['public_subnets'],
Scheme=elb_type,
LoadBalancerName=Join(
"", [Ref("AWS::StackName"), '-',
random.randint(1, 999)]),
SecurityGroups=[Ref(elb_sec_group)],
LBCookieStickinessPolicy=[
elb.LBCookieStickinessPolicy(
PolicyName='LBCookeStickinessPolicy', )
],
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=300,
),
CrossZone=True,
Instances=[Ref(r) for r in instance_list],
Tags=elb_tags))
elasticlb.Listeners = [
elb.Listener(LoadBalancerPort="80",
InstancePort=Ref("WebServerPort"),
Protocol="HTTP",
PolicyNames=['LBCookeStickinessPolicy']),
elb.Listener(
LoadBalancerPort="443",
InstancePort=Ref("WebServerPort"),
Protocol="HTTPS",
SSLCertificateId=Ref("CertName"),
PolicyNames=['LBCookeStickinessPolicy'],
)
]
elasticlb.HealthCheck = elb.HealthCheck(Target=Join(
"", [elb_check_type.upper(), ":", ports, elb_check_path]),
HealthyThreshold="3",
UnhealthyThreshold="5",
Interval="30",
Timeout="5")
if dns:
self.dns_adder(dns, elb_id)
return elasticlb
IpProtocol='tcp',
FromPort='443',
ToPort='443',
CidrIp='0.0.0.0/0',
),
],
VpcId=Ref(VPC),
))
API_SERVER_LOAD_BALANCER = TEMPLATE.add_resource(
elasticloadbalancing.LoadBalancer(
'APIServerLoadBalancer',
HealthCheck=elasticloadbalancing.HealthCheck(
Target='TCP:443',
HealthyThreshold='3',
UnhealthyThreshold='5',
Interval='30',
Timeout='5',
),
Listeners=[
elasticloadbalancing.Listener(
LoadBalancerPort='443',
InstancePort='443',
Protocol='TCP',
),
],
Scheme='internal',
SecurityGroups=[Ref(API_SERVER_SECURITY_GROUP)],
Subnets=[Ref(SUBNET)],
))
def main():
template = Template()
vpc = template.add_resource(
ec2.VPC('MyVPC',
CidrBlock='10.0.0.0/16',
EnableDnsSupport='true',
EnableDnsHostnames='true'))
internet_gateway = template.add_resource(
ec2.InternetGateway('MyInternetGateway',
Tags=Tags(Name='MyInternetGateway')))
template.add_resource(
ec2.VPCGatewayAttachment('MyVPCGatewayAttachment',
InternetGatewayId=Ref(internet_gateway),
VpcId=Ref(vpc),
DependsOn=internet_gateway.title))
security_group = template.add_resource(
ec2.SecurityGroup(
'mySecGroup',
GroupDescription='Security group',
VpcId=Ref(vpc),
))
template.add_resource(
ec2.SecurityGroupEgress('mySecGroupOut',
CidrIp='0.0.0.0/0',
IpProtocol='tcp',
FromPort='0',
ToPort='0',
GroupId=Ref(security_group)))
template.add_resource(
ec2.SecurityGroupIngress('mySecGroupIn',
CidrIp='0.0.0.0/0',
IpProtocol='tcp',
FromPort='0',
ToPort='0',
GroupId=Ref(security_group)))
public_subnets = [
template.add_resource(
ec2.Subnet('MyPubSub1',
AvailabilityZone='ap-southeast-2a',
VpcId=Ref(vpc),
CidrBlock='10.0.1.0/24')),
template.add_resource(
ec2.Subnet('MyPubSub2',
AvailabilityZone='ap-southeast-2b',
VpcId=Ref(vpc),
CidrBlock='10.0.2.0/24')),
template.add_resource(
ec2.Subnet('MyPubSub3',
AvailabilityZone='ap-southeast-2c',
VpcId=Ref(vpc),
CidrBlock='10.0.3.0/24'))
]
template.add_resource(
elb.LoadBalancer('myLoadBalancer',
CrossZone=True,
HealthCheck=elb.HealthCheck(
Target='HTTP:80/index.html',
HealthyThreshold='10',
UnhealthyThreshold='2',
Interval='300',
Timeout='60'),
Listeners=[
elb.Listener(LoadBalancerPort='80',
Protocol='HTTP',
InstancePort='80',
InstanceProtocol='HTTP')
],
Scheme='internet-facing',
SecurityGroups=[Ref(security_group)],
Subnets=[Ref(subnet) for subnet in public_subnets]))
template.add_resource(
ec2.Instance('myinstance',
KeyName='INSERT_YOUR_KEYPAIR_HERE',
ImageId='ami-dc361ebf',
InstanceType='t2.nano',
NetworkInterfaces=[
ec2.NetworkInterfaceProperty(
GroupSet=[Ref(security_group)],
AssociatePublicIpAddress=True,
DeviceIndex='0',
DeleteOnTermination=True,
SubnetId=Ref(public_subnets[0]))
],
SourceDestCheck=True,
DependsOn=internet_gateway.title))
HostedZone(template=template, domain='mypublic.hz', vpcs=None)
HostedZone(template=template, domain='myprivate.hz', vpcs=[Ref(vpc)])
print(template.to_json(indent=2, separators=(',', ': ')))
def render_elb(context, template, ec2_instances):
elb_is_public = True if context['full_hostname'] else False
listeners_policy_names = []
app_cookie_stickiness_policy = []
lb_cookie_stickiness_policy = []
if context['elb']['stickiness']:
if context['elb']['stickiness']['type'] == 'cookie':
app_cookie_stickiness_policy = [
elb.AppCookieStickinessPolicy(
CookieName=context['elb']['stickiness']['cookie-name'],
PolicyName='AppCookieStickinessPolicy')
]
listeners_policy_names.append('AppCookieStickinessPolicy')
elif context['elb']['stickiness']['type'] == 'browser':
lb_cookie_stickiness_policy = [
elb.LBCookieStickinessPolicy(
PolicyName='BrowserSessionLongCookieStickinessPolicy')
]
listeners_policy_names.append(
'BrowserSessionLongCookieStickinessPolicy')
else:
raise ValueError('Unsupported stickiness: %s' %
context['elb']['stickiness'])
protocols = _elb_protocols(context)
listeners = []
elb_ports = []
for protocol in protocols:
if protocol == 'http':
listeners.append(
elb.Listener(
InstanceProtocol='HTTP',
InstancePort='80',
LoadBalancerPort='80',
PolicyNames=listeners_policy_names,
Protocol='HTTP',
))
elb_ports.append(80)
elif protocol == 'https':
listeners.append(
elb.Listener(InstanceProtocol='HTTP',
InstancePort='80',
LoadBalancerPort='443',
PolicyNames=listeners_policy_names,
Protocol='HTTPS',
SSLCertificateId=context['elb']['certificate']))
elb_ports.append(443)
elif isinstance(protocol, int):
port = protocol
listeners.append(
elb.Listener(InstanceProtocol='TCP',
InstancePort=str(port),
LoadBalancerPort=str(port),
PolicyNames=listeners_policy_names,
Protocol='TCP'))
elb_ports.append(port)
else:
raise RuntimeError("Unknown procotol `%s`" %
context['elb']['protocol'])
for _, listener in context['elb']['additional_listeners'].items():
listeners.append(
elb.Listener(InstanceProtocol='HTTP',
InstancePort=str(listener['port']),
LoadBalancerPort=str(listener['port']),
PolicyNames=listeners_policy_names,
Protocol=listener['protocol'].upper(),
SSLCertificateId=context['elb']['certificate']))
elb_ports.append(listener['port'])
template.add_resource(
elb.LoadBalancer(
ELB_TITLE,
AppCookieStickinessPolicy=app_cookie_stickiness_policy,
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=60,
),
ConnectionSettings=elb.ConnectionSettings(
IdleTimeout=context['elb']['idle_timeout']),
CrossZone=True,
Instances=lmap(Ref, ec2_instances.values()),
# TODO: from configuration
Listeners=listeners,
LBCookieStickinessPolicy=lb_cookie_stickiness_policy,
HealthCheck=elb.HealthCheck(
Target=_elb_healthcheck_target(context),
HealthyThreshold=str(context['elb']['healthcheck'].get(
'healthy_threshold', 10)),
UnhealthyThreshold=str(context['elb']['healthcheck'].get(
'unhealthy_threshold', 2)),
Interval=str(context['elb']['healthcheck'].get('interval',
30)),
Timeout=str(context['elb']['healthcheck'].get('timeout', 30)),
),
SecurityGroups=[Ref(SECURITY_GROUP_ELB_TITLE)],
Scheme='internet-facing' if elb_is_public else 'internal',
Subnets=context['elb']['subnets'],
Tags=elb_tags(context)))
template.add_output(
mkoutput("ElasticLoadBalancer", "Generated name of the ELB",
Ref(ELB_TITLE)))
template.add_resource(
security_group(SECURITY_GROUP_ELB_TITLE, context['aws']['vpc-id'],
_convert_ports_to_dictionary(
elb_ports))) # list of strings or dicts
if any([context['full_hostname'], context['int_full_hostname']]):
dns = external_dns_elb if elb_is_public else internal_dns_elb
template.add_resource(dns(context))
if context['full_hostname']:
[template.add_resource(cname) for cname in cnames(context)]
InstanceProtocol='tcp',
InstancePort=3000,
Protocol='tcp'),
elb.Listener(LoadBalancerPort=3001,
InstanceProtocol='tcp',
InstancePort=3001,
Protocol='tcp'),
elb.Listener(LoadBalancerPort=3002,
InstanceProtocol='tcp',
InstancePort=3002,
Protocol='tcp'),
],
HealthCheck=elb.HealthCheck(
Target=Join("", ["HTTP:", 8080, "/"]),
HealthyThreshold="2",
UnhealthyThreshold="2",
Interval="100",
Timeout="10",
),
Scheme="internal")
template.add_output(
Output("LoadBalancerDNSName",
Description="Loadbalancer DNS",
Value=Join("",
["http://", GetAtt(load_balancer, "DNSName")])))
# ECS cluster
main_cluster = Cluster(
"MainCluster",
ClusterName=Join("", ["MainCluster-", Ref(AWS_STACK_NAME)]),
def main():
network_config, template = get_network_config()
load_balancer = template.add_resource(
elb.LoadBalancer('MyELB',
CrossZone=True,
HealthCheck=elb.HealthCheck(
Target='HTTP:8080/error/noindex.html',
HealthyThreshold='2',
UnhealthyThreshold='5',
Interval='15',
Timeout='5'),
Listeners=[
elb.Listener(LoadBalancerPort='80',
Protocol='HTTP',
InstancePort='80',
InstanceProtocol='HTTP')
],
Scheme='internet-facing',
Subnets=network_config.public_subnets))
block_devices_config = [
BlockDevicesConfig(device_name='/dev/xvda',
ebs_volume_size='15',
ebs_volume_type='gp2',
ebs_encrypted=False,
ebs_snapshot_id=None,
virtual_name=False),
BlockDevicesConfig(device_name='/dev/xvda2',
ebs_volume_size='15',
ebs_volume_type='gp2',
ebs_encrypted=False,
ebs_snapshot_id='ami-ce0531ad',
virtual_name=False),
BlockDevicesConfig(device_name='/dev/sda1',
ebs_volume_size=None,
ebs_volume_type=None,
ebs_encrypted=False,
ebs_snapshot_id=None,
virtual_name=True),
BlockDevicesConfig(device_name='/dev/sda2',
ebs_volume_size='15',
ebs_volume_type='gp2',
ebs_encrypted=True,
ebs_snapshot_id=None,
virtual_name=False)
]
asg_config = AsgConfig(
image_id='ami-dc361ebf',
instance_type='t2.nano',
minsize=1,
maxsize=2,
userdata='',
health_check_grace_period=300,
health_check_type='ELB',
iam_instance_profile_arn=
'arn:aws:iam::12345678987654321:role/InstanceProfileRole',
block_devices_config=block_devices_config,
simple_scaling_policy_config=None,
ec2_scheduled_shutdown=None,
pausetime='10',
owner='*****@*****.**')
Asg(title='simple',
network_config=network_config,
load_balancers=[load_balancer],
template=template,
asg_config=asg_config)
print(template.to_json(indent=2, separators=(',', ': ')))
def add_resources(self):
self.LoyaltyNavigatorPublicLBSG = self.template.add_resource(ec2.SecurityGroup(
"LoyaltyNavigatorPublicLBSG",
GroupDescription="Loadbalancer Security Group For Loyalty Navigator Public LB",
VpcId=Ref(self.VpcId),
SecurityGroupIngress=[
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=3389,
ToPort=3389,
CidrIp=Ref(self.AdminCidrBlock),
),
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=80,
ToPort=80,
CidrIp=Ref(self.AdminCidrBlock),
),
],
Tags=self.base_tags + Tags(Name=self.environment_parameters["ClientEnvironmentKey"] + "-LoyaltyNavigatorPublicLBSG"),
))
self.LoyaltyNavigatorSG = self.template.add_resource(ec2.SecurityGroup(
"LoyaltyNavigatorSG",
GroupDescription="Allow communication between Loyalty Navigator Load Balancer and Loyalty NavigatorEc2s",
VpcId=Ref(self.VpcId),
SecurityGroupIngress=[
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=3389,
ToPort=3389,
SourceSecurityGroupId=Ref(self.LoyaltyNavigatorPublicLBSG),
),
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=80,
ToPort=80,
SourceSecurityGroupId=Ref(self.LoyaltyNavigatorPublicLBSG),
),
],
Tags=self.base_tags + Tags(Name=self.environment_parameters["ClientEnvironmentKey"] + "-LoyaltyNavigatorEc2SG"),
))
self.LoyaltyNavigatorPublicLoadBalancer = self.template.add_resource(elasticloadbalancing.LoadBalancer(
"LoyaltyNavigatorPublicLoadBalancer",
LoadBalancerName=self.environment_parameters["ClientEnvironmentKey"] + "-LoyaltyNavigatorPubLB",
Scheme="internet-facing",
Listeners=[
elasticloadbalancing.Listener(
LoadBalancerPort="3389",
InstancePort="3389",
Protocol="TCP",
InstanceProtocol="TCP",
),
elasticloadbalancing.Listener(
LoadBalancerPort="80",
InstancePort="80",
Protocol="HTTP",
InstanceProtocol="HTTP",
)
],
Instances=[],
SecurityGroups=[Ref(self.LoyaltyNavigatorPublicLBSG)],
Subnets=[Ref(self.RESTPubSubnet1), Ref(self.RESTPubSubnet2)],
ConnectionDrainingPolicy=elasticloadbalancing.ConnectionDrainingPolicy(
Enabled=True,
Timeout=300,
),
CrossZone=True,
HealthCheck=elasticloadbalancing.HealthCheck(
Target=Join("", ["TCP:", "3389"]),
HealthyThreshold="3",
UnhealthyThreshold="5",
Interval="10",
Timeout="5", ),
Tags=self.base_tags + Tags(Name=self.environment_parameters["ClientEnvironmentKey"] + "-LoyaltyNavigatorPubLB"),
))
self.LoyaltyNavigatorLaunchConfiguration = self.template.add_resource(LaunchConfiguration(
"LoyaltyNavigatorLaunchConfiguration",
ImageId=Ref(self.LoyaltyNavigatorImageId),
InstanceType=Ref(self.LoyaltyNavigatorInstanceType),
IamInstanceProfile=Ref(self.LoyaltyNavigatorIAMInstanceProfile),
KeyName=Ref(self.LoyaltyNavigatorKeyName),
SecurityGroups=[Ref(self.LoyaltyNavigatorSG)],
))
self.LoyaltyNavigatorAutoScalingGroup = self.template.add_resource(AutoScalingGroup(
"LoyaltyNavigatorAutoscalingGroup",
AutoScalingGroupName=self.environment_parameters["ClientEnvironmentKey"] + "-LoyaltyNavigatorAutoScalingGroup",
LaunchConfigurationName=Ref(self.LoyaltyNavigatorLaunchConfiguration),
LoadBalancerNames=[Ref(self.LoyaltyNavigatorPublicLoadBalancer), Ref(self.LoyaltyNavigatorPublicLoadBalancer)],
MaxSize=Ref(self.LoyaltyNavigatorASGGroupMaxSize),
MinSize=Ref(self.LoyaltyNavigatorASGGroupMinSize),
DesiredCapacity=Ref(self.LoyaltyNavigatorASGGroupDesiredSize),
HealthCheckType=Ref(self.LoyaltyNavigatorASGHealthCheckType),
HealthCheckGracePeriod=Ref(self.LoyaltyNavigatorASGHealthCheckGracePeriod),
Cooldown=Ref(self.LoyaltyNavigatorASGCoolDown),
VPCZoneIdentifier=[Ref(self.RESTPrivSubnet1), Ref(self.RESTPrivSubnet2)],
Tags=[
AutoScalingTag("Name", self.environment_parameters["ClientEnvironmentKey"] + "-LoyaltyNavigator-Ec2", True),
AutoScalingTag("Environment", self.environment_parameters["EnvironmentName"], True),
AutoScalingTag("ResourceOwner", self.environment_parameters["ResourceOwner"], True),
AutoScalingTag("ClientCode", self.environment_parameters["ClientEnvironmentKey"], True),
],
))
def elb_asg_lc_template(app, env, nameSGRDS, rdsPort, instanceType, ami,
subnets, elbPort, elbCidrBlock, ec2Port,
desiredCapacity, minSize, maxSize, region, nameBucket,
officeIP):
template = Template()
sgELB = template.add_resource(
SecurityGroup(
"SecurityGroupELB" + app + env,
GroupDescription="Security group for " + app + "-" + env,
VpcId=ImportValue("VPC" + env),
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol="tcp",
FromPort=elbPort,
ToPort=elbPort,
CidrIp=elbCidrBlock,
)
],
SecurityGroupEgress=[
SecurityGroupRule(IpProtocol="-1",
ToPort=0,
FromPort=65535,
CidrIp="0.0.0.0/0")
],
Tags=Tags(
env=env,
Name="sg-ELB" + app + "-" + env,
app=app,
),
))
sgEC2 = template.add_resource(
SecurityGroup(
"SecurityGroupEC2" + app + env,
GroupDescription="Security group for EC2 " + app + "-" + env,
VpcId=ImportValue("VPC" + env),
DependsOn="SecurityGroupELB" + app + env,
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol="tcp",
FromPort=ec2Port,
ToPort=ec2Port,
SourceSecurityGroupId=Ref(sgELB),
),
SecurityGroupRule(
IpProtocol="tcp",
FromPort=22,
ToPort=22,
CidrIp=officeIP,
),
],
SecurityGroupEgress=[
SecurityGroupRule(IpProtocol="-1",
ToPort=0,
FromPort=65535,
CidrIp="0.0.0.0/0")
],
Tags=Tags(
env=env,
Name="sg-EC2-" + app + "-" + env,
app=app,
),
))
addIngressRDS = template.add_resource(
SecurityGroupIngress(
"ingressSGRDS" + app + env,
SourceSecurityGroupId=Ref(sgEC2),
Description="From EC2 instances",
GroupId=ImportValue("SG-" + nameSGRDS + "-" + app + "-" + env),
IpProtocol="tcp",
FromPort=rdsPort,
ToPort=rdsPort,
DependsOn="SecurityGroupEC2" + app + env,
))
launchConfig = template.add_resource(
LaunchConfiguration("LaunchConfiguration" + app + env,
InstanceType=instanceType,
ImageId=ami,
SecurityGroups=[Ref(sgEC2)],
IamInstanceProfile=ImportValue("Role-" + app +
"-" + env)))
bucketPolicy = template.add_resource(
BucketPolicy("BucketPolicy" + nameBucket + app + env,
Bucket=ImportValue("Bucket" + nameBucket + app + env),
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["s3:PutObject"],
"Effect":
"Allow",
"Resource":
Join("", [
"arn:aws:s3:::",
ImportValue("Bucket" + nameBucket + app +
env), "/AWSLogs/",
Ref("AWS::AccountId"), "/*"
]),
"Principal": {
"AWS": ["156460612806"]
}
}]
}))
lb = template.add_resource(
LoadBalancer("LoadBalancer" + app + env,
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=120,
),
Subnets=subnets,
HealthCheck=elb.HealthCheck(
"HealthCheck",
Target="TCP:" + str(ec2Port),
HealthyThreshold="5",
UnhealthyThreshold="5",
Interval="30",
Timeout="15",
),
Listeners=[
elb.Listener(
LoadBalancerPort=elbPort,
InstancePort=ec2Port,
Protocol="HTTP",
InstanceProtocol="HTTP",
),
],
CrossZone=True,
SecurityGroups=[Ref(sgELB)],
LoadBalancerName="lb-" + app + "-" + env,
Scheme="internet-facing",
AccessLoggingPolicy=AccessLoggingPolicy(
"LoggingELB" + app + env,
EmitInterval=5,
Enabled=True,
S3BucketName=ImportValue("Bucket" + nameBucket + app +
env),
)))
asg = template.add_resource(
AutoScalingGroup(
"AutoscalingGroup" + app + env,
DesiredCapacity=desiredCapacity,
Tags=[Tag("Environment", env, True)],
LaunchConfigurationName=Ref(launchConfig),
MinSize=minSize,
MaxSize=maxSize,
LoadBalancerNames=[Ref(lb)],
AvailabilityZones=GetAZs(region),
VPCZoneIdentifier=subnets,
HealthCheckType="ELB",
HealthCheckGracePeriod=300,
UpdatePolicy=UpdatePolicy(
AutoScalingReplacingUpdate=AutoScalingReplacingUpdate(
WillReplace=True, ),
AutoScalingRollingUpdate=AutoScalingRollingUpdate(
PauseTime='PT5M',
MinInstancesInService="1",
MaxBatchSize='1',
WaitOnResourceSignals=True,
))))
return (template.to_json())
def generate_cloudformation_template():
template = Template()
template.add_description("""\
Configures Auto Scaling Group for the app""")
hostedzone = template.add_parameter(
Parameter(
"HostedZoneName",
Description=
"The DNS name of an existing Amazon Route 53 hosted zone",
Type="String",
))
dnsRecord = template.add_parameter(Parameter(
"DNSRecord",
Type="String",
))
loadbalancersecuritygroup = template.add_parameter(
Parameter(
"LoadBalancerSecurityGroup",
Type="String",
Description="Security group for api app load balancer.",
))
scalecapacity = template.add_parameter(
Parameter(
"ScaleCapacity",
Default="1",
Type="String",
Description="Number of api servers to run",
))
minsize = template.add_parameter(
Parameter(
"MinScale",
Type="String",
Description="Minimum number of servers to keep in the ASG",
))
maxsize = template.add_parameter(
Parameter(
"MaxScale",
Type="String",
Description="Maximum number of servers to keep in the ASG",
))
environment = template.add_parameter(
Parameter(
"Environment",
Type="String",
Description="The environment being deployed into",
))
subnet = template.add_parameter(
Parameter(
"Subnets",
Type="CommaDelimitedList",
))
loadbalancername = template.add_parameter(
Parameter(
"LoadBalancerName",
Type="String",
))
elbSchema = template.add_parameter(
Parameter(
"LoadBalancerSchema",
Type="String",
))
healthCheckTarget = template.add_parameter(
Parameter(
"LoadBalancerHealthCheckTarget",
Type="String",
))
healthCheckInterval = template.add_parameter(
Parameter(
"LoadBalancerHealthCheckInterval",
Type="String",
))
healthCheckTimeout = template.add_parameter(
Parameter(
"LoadBalancerHealthCheckTimeout",
Type="String",
))
healthyThreshold = template.add_parameter(
Parameter(
"LoadBalancerHealthyThreshold",
Type="String",
))
unhealthyThreshold = template.add_parameter(
Parameter(
"LoadBalancerUnHealthyThreshold",
Type="String",
))
launchconfigurationname = template.add_parameter(
Parameter(
"LaunchConfigurationName",
Type="String",
))
loadbalancer = template.add_resource(
elb.LoadBalancer(
"LoadBalancer",
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=30,
),
Subnets=Ref(subnet),
HealthCheck=elb.HealthCheck(
Target=Ref(healthCheckTarget),
HealthyThreshold=Ref(healthyThreshold),
UnhealthyThreshold=Ref(unhealthyThreshold),
Interval=Ref(healthCheckInterval),
Timeout=Ref(healthCheckTimeout),
),
Listeners=[
elb.Listener(
LoadBalancerPort="80",
InstancePort="80",
Protocol="HTTP",
InstanceProtocol="HTTP",
),
],
CrossZone=True,
SecurityGroups=[Ref(loadbalancersecuritygroup)],
LoadBalancerName=Ref(loadbalancername),
Scheme=Ref(elbSchema),
))
autoscalinggroup = template.add_resource(
AutoScalingGroup("AutoscalingGroup",
Tags=[Tag("Environment", Ref(environment), True)],
LaunchConfigurationName=Ref(launchconfigurationname),
MinSize=Ref(minsize),
MaxSize=Ref(maxsize),
DesiredCapacity=Ref(scalecapacity),
LoadBalancerNames=[Ref(loadbalancer)],
VPCZoneIdentifier=Ref(subnet),
HealthCheckType='ELB',
HealthCheckGracePeriod=30,
UpdatePolicy=UpdatePolicy(
AutoScalingRollingUpdate=AutoScalingRollingUpdate(
PauseTime='PT1M',
MinInstancesInService="1",
MaxBatchSize='1'))))
route53record = template.add_resource(
RecordSetType(
"myDNSRecord",
HostedZoneName=Join("", [Ref(hostedzone), "."]),
Name=Join("", [Ref(dnsRecord), ".",
Ref(hostedzone), "."]),
Type="CNAME",
TTL="300",
ResourceRecords=[GetAtt(loadbalancer, "DNSName")],
))
template.add_output(
Output("DomainName",
Value=Ref(route53record),
Description="DNS to access the service"))
template.add_output(
Output("LoadBalancer",
Value=GetAtt(loadbalancer, "DNSName"),
Description="ELB dns"))
template.add_output(
Output("AutoScalingGroup",
Value=Ref(autoscalinggroup),
Description="Created Auto Scaling Group"))
template.add_output(
Output("LaunchConfiguration",
Value=Ref(launchconfigurationname),
Description="LaunchConfiguration for this deploy"))
return template
LoadBalancerPort='443',
InstancePort='443',
Protocol='HTTPS',
InstanceProtocol='HTTPS',
SSLCertificateId=tsp.Join('', [
'arn:aws:iam::',
tsp.Ref('AWS::AccountId'),
':server-certificate/',
atlas.ssl_cert_id,
]),
),
],
HealthCheck=elb.HealthCheck(
Target='TCP:443',
HealthyThreshold='3',
UnhealthyThreshold='5',
Interval='30',
Timeout='5',
)
)
# scale group
scaling_group = atlas.instance_scalegrp(
template,
name='Pypi',
LaunchConfigurationName=tsp.Ref(i_launchconf),
MinSize=tsp.Ref('MinSize'),
MaxSize=tsp.Ref('MaxSize'),
DesiredCapacity=tsp.Ref('DesiredCapacity'),
)
def create_template(num_masters, num_agents, num_publicAgents):
#outfilename = "test.json"
outfilename = "cf_" + str(num_masters) + "." + str(num_agents) + "." + str(
num_publicAgents) + ".json"
# Create the Template
t = Template()
t.add_version('2010-09-09')
t.add_description('Creates a set of Servers for DC/OS using CentOS 7.3 AMI. Creates a boot server to host the DC/OS installer and a NAT Instance for outbound connections from private agents. Creates ' + str(num_masters) + ' Master(s), ' \
+ str(num_agents) + ' Private Agent(s), and ' + str(num_publicAgents) + ' Public Agent(s). After creating the Stack; Log into the boot server and run the DCOS Bash Script installer for AWS')
# Amazon Linux AMI 2016.09.1.20170119 x86_64 VPC NAT HVM EBS
# amzn-ami-vpc-nat-hvm-2016.09.1.20170119-x86_64-ebs -
# ami-dd3dd7cb us-east-1 (N. Virginia)
# ami-564b6e33 us-east-2 (Ohio)
# ami-7d54061d us-west-1 (N. Cal)
# ami-3b6fd05b us-west-2 (Oregon)
t.add_mapping(
'NATAmi', {
'us-east-1': {
'default': 'ami-dd3dd7cb'
},
'us-east-2': {
'default': 'ami-564b6e33'
},
'us-west-1': {
'default': 'ami-7d54061d'
},
'us-west-2': {
'default': 'ami-3b6fd05b'
},
})
# The c73 AMI pre created and deployed on each region
t.add_mapping(
'c73Ami', {
'us-east-1': {
'default': 'ami-46c1b650'
},
'us-east-2': {
'default': 'ami-18f8df7d'
},
'us-west-1': {
'default': 'ami-f5d7f195'
},
'us-west-2': {
'default': 'ami-f4533694'
},
})
# CloudFormation Parameters
# Sometimes when I deployed stack on us-east-1; it would fail on av zone us-east-1c with error messages instance type not support on this AZ. I added this parameter to fix all of the components in on AZ for now
avzone_param = t.add_parameter(
Parameter(
"AVZoneName",
ConstraintDescription='Must be the name of an an Availability Zone',
Description='Name of an Availability Zone',
Type='AWS::EC2::AvailabilityZone::Name',
))
# Every agent will get a data drive of this size
dataDriveSizeGB_param = t.add_parameter(
Parameter(
"dataDriveSizeGB",
Default="100",
MinValue=20,
MaxValue=1000,
Description=
'Size of data drive to add to private agents from 20 to 1000GB',
Type='Number'))
# The key will be added to the centos user so you can login to centos using the key
keyname_param = t.add_parameter(
Parameter(
"KeyName",
ConstraintDescription=
'Must be the name of an existing EC2 KeyPair.',
Description=
'Name of an existing EC2 KeyPair to enable SSH access to the instance',
Type='AWS::EC2::KeyPair::KeyName',
))
# While you can allow everyone it's more secure to just allow a single machine or subnet of machines; web port will also be opened to this CIDR
sshlocation_param = t.add_parameter(
Parameter(
"sshlocation",
Type="String",
Description=
"Subnet allowed to ssh to these servers. 0.0.0.0/0 to allow all."))
# Instance type for Master
instanceTypeMaster_param = t.add_parameter(
Parameter(
'InstanceTypeMaster',
Type='String',
Description='EC2 instance type for ' + str(num_masters) +
' Masters(s)',
Default='m4.xlarge',
AllowedValues=[
't2.xlarge',
't2.2xlarge',
'm4.xlarge',
'm4.2xlarge',
'm4.4xlarge',
'm4.10xlarge',
'c4.xlarge',
'c4.2xlarge',
'c4.4xlarge',
'c4.8xlarge',
],
ConstraintDescription='Must be a valid EC2 instance type.',
))
# Instance type for Agents
instanceTypeAgent_param = t.add_parameter(
Parameter(
'InstanceTypeAgent',
Type='String',
Description='EC2 instance type for ' + str(num_agents) +
' Private Agent(s)',
Default='m4.2xlarge',
AllowedValues=[
't2.xlarge',
't2.2xlarge',
'm4.xlarge',
'm4.2xlarge',
'm4.4xlarge',
'm4.10xlarge',
'c4.xlarge',
'c4.2xlarge',
'c4.4xlarge',
'c4.8xlarge',
],
ConstraintDescription='Must be a valid EC2 instance type.',
))
# Instance type for Public Agents
instanceTypePublicAgent_param = t.add_parameter(
Parameter(
'InstanceTypePublicAgent',
Type='String',
Description='EC2 instance type for ' + str(num_publicAgents) +
' Public Agent(s)',
Default='m4.xlarge',
AllowedValues=[
't2.xlarge',
't2.2xlarge',
'm4.xlarge',
'm4.2xlarge',
'm4.4xlarge',
'm4.10xlarge',
'c4.xlarge',
'c4.2xlarge',
'c4.4xlarge',
'c4.8xlarge',
],
ConstraintDescription='Must be a valid EC2 instance type.',
))
# Adding Resources
ref_stack_id = Ref('AWS::StackId')
ref_region = Ref('AWS::Region')
ref_stack_name = Ref('AWS::StackName')
# Create VPC
nm = 'vpc'
vpc = t.add_resource(
VPC(nm,
CidrBlock='10.10.0.0/16',
EnableDnsSupport=True,
EnableDnsHostnames=True,
Tags=Tags(Application=ref_stack_id,
Name=Join("", [Ref('AWS::StackName'), "-", nm]))))
# Create Subnet for Masters
nm = 'mastersSubnet'
subnetMasters = t.add_resource(
Subnet(nm,
AvailabilityZone=Ref(avzone_param),
CidrBlock='10.10.0.0/24',
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,
Name=Join("", [Ref('AWS::StackName'), "-", nm]))))
# Create Subnet for Agents
nm = 'agentsSubnet'
subnetAgents = t.add_resource(
Subnet(nm,
AvailabilityZone=Ref(avzone_param),
CidrBlock='10.10.16.0/24',
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,
Name=Join("", [Ref('AWS::StackName'), "-", nm]))))
# Create Subnet for Public Agents
nm = 'publicAgentsSubnet'
subnetPublicAgents = t.add_resource(
Subnet(nm,
AvailabilityZone=Ref(avzone_param),
CidrBlock='10.10.32.0/24',
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,
Name=Join("", [Ref('AWS::StackName'), "-", nm]))))
# Create Gateway; route to the outside world (Internet)
nm = 'ig'
internetGateway = t.add_resource(
InternetGateway(nm,
Tags=Tags(Application=ref_stack_id,
Name=Join(
"", [Ref('AWS::StackName'), "-", nm]))))
# Attach Gateway to VPC
nm = 'gatewayAttachment'
gatewayAttachment = t.add_resource(
VPCGatewayAttachment(nm,
VpcId=Ref(vpc),
InternetGatewayId=Ref(internetGateway)))
# Create Route Table
nm = 'routeTable'
routeTable = t.add_resource(
RouteTable(nm,
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,
Name=Join("", [Ref('AWS::StackName'), "-", nm]))))
# Add Routes
# Allow all outbound traffic
nm = 'route'
route = t.add_resource(
Route(
nm,
DependsOn=gatewayAttachment.title,
GatewayId=Ref(internetGateway),
DestinationCidrBlock='0.0.0.0/0',
RouteTableId=Ref(routeTable),
))
# Associate RouteTable to Master and Public Subnets
nm = 'subnetRTAMasters'
subnetRouteTableAssociation = t.add_resource(
SubnetRouteTableAssociation(
nm,
SubnetId=Ref(subnetMasters),
RouteTableId=Ref(routeTable),
))
nm = 'subnetRTAPublicAgents'
subnetRouteTableAssociation = t.add_resource(
SubnetRouteTableAssociation(
nm,
SubnetId=Ref(subnetPublicAgents),
RouteTableId=Ref(routeTable),
))
# Create Security Group (General access to ssh and internal connectionsn between masters, agents, and public agents)
nm = 'securityGroup'
securityGroup = t.add_resource(
SecurityGroup(nm,
GroupDescription='Security Group',
SecurityGroupIngress=[
SecurityGroupRule(IpProtocol='tcp',
FromPort='22',
ToPort='22',
CidrIp=Ref(sshlocation_param)),
SecurityGroupRule(IpProtocol='-1',
CidrIp='10.10.0.0/16')
],
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,
Name=Join("",
[Ref('AWS::StackName'), "-", nm]))))
# Create Security Group Public Agents
nm = 'securityGroupPublicAgents'
publicAgentsSG = t.add_resource(
SecurityGroup(nm,
GroupDescription='Security Group Public Agents',
SecurityGroupIngress=[
SecurityGroupRule(IpProtocol='tcp',
FromPort='80',
ToPort='80',
CidrIp='0.0.0.0/0'),
SecurityGroupRule(IpProtocol='tcp',
FromPort='443',
ToPort='443',
CidrIp='0.0.0.0/0'),
SecurityGroupRule(IpProtocol='tcp',
FromPort='10000',
ToPort='10010',
CidrIp='0.0.0.0/0'),
SecurityGroupRule(IpProtocol='tcp',
FromPort='9090',
ToPort='9090',
CidrIp='0.0.0.0/0')
],
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,
Name=Join("",
[Ref('AWS::StackName'), "-", nm]))))
# Create Security Group Masters Allow Access from sshlocation param as test
nm = 'securityGroupMasters'
mastersSG = t.add_resource(
SecurityGroup(nm,
GroupDescription='Security Group Masters',
SecurityGroupIngress=[
SecurityGroupRule(IpProtocol='tcp',
FromPort='80',
ToPort='80',
CidrIp=Ref(sshlocation_param)),
SecurityGroupRule(IpProtocol='tcp',
FromPort='443',
ToPort='443',
CidrIp=Ref(sshlocation_param))
],
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,
Name=Join("",
[Ref('AWS::StackName'), "-", nm]))))
if useNatInstance:
# **** Also change in natRoute ****
# Create NAT instance; This allows private agents to get out to the Internet
nm = 'nat'
nat = t.add_resource(
Instance(
nm,
SourceDestCheck="false",
ImageId=FindInMap("NATAmi", Ref("AWS::Region"), "default"),
InstanceType="m4.large",
AvailabilityZone=Ref(avzone_param),
KeyName=Ref(keyname_param),
DependsOn=internetGateway.title,
NetworkInterfaces=[
NetworkInterfaceProperty(GroupSet=[Ref(securityGroup)],
AssociatePublicIpAddress='true',
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(subnetMasters),
PrivateIpAddress='10.10.0.9')
],
BlockDeviceMappings=[
BlockDeviceMapping(DeviceName="/dev/xvda",
Ebs=EBSBlockDevice(
DeleteOnTermination='true', ))
],
Tags=Tags(Application=ref_stack_id,
Name=Join("", [Ref('AWS::StackName'), "-", nm]))))
else:
# Create Elastic IP for NatGateay
nm = 'natIP'
nat_eip = t.add_resource(EIP(
nm,
Domain="vpc",
))
# Create NAT Gateway
nm = 'natGateway'
nat = t.add_resource(
NatGateway(
nm,
AllocationId=GetAtt(nat_eip, 'AllocationId'),
SubnetId=Ref(subnetMasters),
))
# Create Route Table for NAT
nm = 'natRouteTable'
routeTableNAT = t.add_resource(
RouteTable(nm,
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,
Name=Join("", [Ref('AWS::StackName'), "-", nm]))))
# Associate Agent Subnet to NAT
nm = 'subnetRTAAgents'
subnetRouteTableAssociation = t.add_resource(
SubnetRouteTableAssociation(
nm,
SubnetId=Ref(subnetAgents),
RouteTableId=Ref(routeTableNAT),
))
# Add Routes (Agents can reach out anywhere)
nm = 'natRoute'
if useNatInstance:
route = t.add_resource(
Route(
nm,
RouteTableId=Ref(routeTableNAT),
DestinationCidrBlock='0.0.0.0/0',
InstanceId=Ref(nat),
))
else:
route = t.add_resource(
Route(
nm,
RouteTableId=Ref(routeTableNAT),
DestinationCidrBlock='0.0.0.0/0',
NatGatewayId=Ref(nat),
))
# ****************************************
# NOTE: I am using static PrivateIPAddresses; this may not be a good choice; however, it simplified the install script. The range of IP's for the master and agents are limited to 24 subnet and I start at 11
# With this configuration the max number of agents is around 240.
# ****************************************
# Create boot instance
# Installs on AWS so far have taken longer than on Azure. Takes about 10 minutes for the boot server to configure.
# Tried several InstanceType from t2.micro to m4.large; all take about 10 minutes for boot to load. The docker start of mesosphere/dcos-genconf seems to be taking longer than it did on azure.
nm = 'boot'
boot = t.add_resource(
Instance(nm,
ImageId=FindInMap("c73Ami", Ref("AWS::Region"), "default"),
InstanceType="m4.xlarge",
AvailabilityZone=Ref(avzone_param),
KeyName=Ref(keyname_param),
NetworkInterfaces=[
NetworkInterfaceProperty(GroupSet=[Ref(securityGroup)],
AssociatePublicIpAddress='true',
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(subnetMasters),
PrivateIpAddress='10.10.0.10')
],
BlockDeviceMappings=[
BlockDeviceMapping(DeviceName="/dev/sda1",
Ebs=EBSBlockDevice(
VolumeSize="100",
DeleteOnTermination='true',
))
],
Tags=Tags(Application=ref_stack_id,
Name=Join("", [Ref('AWS::StackName'), "-", nm]))))
# Create master instance(s)
masters = []
i = 1
while i <= num_masters:
nm = 'm' + str(i)
private_ip = "10.10.0." + str(i + 10)
instance = t.add_resource(
Instance(nm,
ImageId=FindInMap("c73Ami", Ref("AWS::Region"),
"default"),
InstanceType=Ref(instanceTypeMaster_param),
AvailabilityZone=Ref(avzone_param),
KeyName=Ref(keyname_param),
NetworkInterfaces=[
NetworkInterfaceProperty(
GroupSet=[Ref(securityGroup),
Ref(mastersSG)],
AssociatePublicIpAddress='true',
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(subnetMasters),
PrivateIpAddress=private_ip)
],
BlockDeviceMappings=[
BlockDeviceMapping(DeviceName="/dev/sda1",
Ebs=EBSBlockDevice(
VolumeSize="100",
DeleteOnTermination='true',
))
],
Tags=Tags(Application=ref_stack_id,
Name=Join("",
[Ref('AWS::StackName'), "-", nm]))))
masters.append(instance)
i += 1
# Create agent instance(s)
i = 1
while i <= num_agents:
nm = 'a' + str(i)
private_ip = "10.10.16." + str(i + 10)
instance = t.add_resource(
Instance(
nm,
ImageId=FindInMap("c73Ami", Ref("AWS::Region"), "default"),
InstanceType=Ref(instanceTypeAgent_param),
AvailabilityZone=Ref(avzone_param),
KeyName=Ref(keyname_param),
NetworkInterfaces=[
NetworkInterfaceProperty(GroupSet=[Ref(securityGroup)],
AssociatePublicIpAddress='false',
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(subnetAgents),
PrivateIpAddress=private_ip)
],
BlockDeviceMappings=[
BlockDeviceMapping(DeviceName="/dev/sda1",
Ebs=EBSBlockDevice(
VolumeSize="100",
DeleteOnTermination='true',
))
],
Tags=Tags(Application=ref_stack_id,
Name=Join("", [Ref('AWS::StackName'), "-", nm]))))
volume = t.add_resource(
Volume(nm + "data",
AvailabilityZone=Ref(avzone_param),
Size=Ref(dataDriveSizeGB_param),
Tags=Tags(
Application=ref_stack_id,
Name=Join("",
[Ref('AWS::StackName'), "-", nm + "data"]))))
volattach = t.add_resource(
VolumeAttachment(nm + "dataattach",
InstanceId=Ref(instance),
VolumeId=Ref(volume),
Device="/dev/sdc"))
i += 1
# Create public agent instance(s)
publicAgents = []
i = 1
nm = "p1"
while i <= num_publicAgents:
nm = 'p' + str(i)
private_ip = "10.10.32." + str(i + 10)
instance = t.add_resource(
Instance(
nm,
ImageId=FindInMap("c73Ami", Ref("AWS::Region"), "default"),
InstanceType=Ref(instanceTypePublicAgent_param),
AvailabilityZone=Ref(avzone_param),
KeyName=Ref(keyname_param),
NetworkInterfaces=[
NetworkInterfaceProperty(
GroupSet=[Ref(securityGroup),
Ref(publicAgentsSG)],
AssociatePublicIpAddress='true',
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(subnetPublicAgents),
PrivateIpAddress=private_ip)
],
BlockDeviceMappings=[
BlockDeviceMapping(DeviceName="/dev/sda1",
Ebs=EBSBlockDevice(
VolumeSize="100",
DeleteOnTermination='true',
))
],
Tags=Tags(Application=ref_stack_id,
Name=Join("", [Ref('AWS::StackName'), "-", nm]))))
publicAgents.append(instance)
i += 1
# Load Balancer Masters
nm = "masters"
elasticLBMasters = t.add_resource(
elb.LoadBalancer(
nm,
Instances=[Ref(r) for r in masters],
Subnets=[Ref(subnetMasters)],
SecurityGroups=[Ref(mastersSG)],
CrossZone=False,
Listeners=[
elb.Listener(
LoadBalancerPort="80",
InstancePort="80",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="443",
InstancePort="443",
Protocol="TCP",
),
],
# Health Checking on port 80 which should be there after DCOS has been installed.
HealthCheck=elb.HealthCheck(
Target="TCP:80",
HealthyThreshold="2",
UnhealthyThreshold="2",
Interval="30",
Timeout="5",
),
Tags=Tags(Application=ref_stack_id,
Name=Join("", [Ref('AWS::StackName'), "-", nm]))))
# Load Balancer Public Agents
nm = "publicagents"
elasticLBPublicAgents = t.add_resource(
elb.LoadBalancer(
nm,
#AvailabilityZones=GetAZs(""),
Instances=[Ref(r) for r in publicAgents],
Subnets=[Ref(subnetPublicAgents)],
SecurityGroups=[Ref(publicAgentsSG)],
CrossZone=False,
Listeners=[
elb.Listener(
LoadBalancerPort="10000",
InstancePort="10000",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10001",
InstancePort="10001",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10002",
InstancePort="10002",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10003",
InstancePort="10003",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10004",
InstancePort="10004",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10005",
InstancePort="10005",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10006",
InstancePort="10006",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10007",
InstancePort="10007",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10008",
InstancePort="10008",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10009",
InstancePort="10009",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10010",
InstancePort="10010",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="9090",
InstancePort="9090",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="80",
InstancePort="80",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="443",
InstancePort="443",
Protocol="TCP",
)
],
# I've added health check for port 9090; becomes healthy after Marathon-LB is installed.
HealthCheck=elb.HealthCheck(
Target="TCP:9090",
HealthyThreshold="2",
UnhealthyThreshold="2",
Interval="30",
Timeout="5",
),
Tags=Tags(Application=ref_stack_id,
Name=Join("", [Ref('AWS::StackName'), "-", nm]))))
# Outputs
t.add_output(
Output("BootServer",
Description="Name/IP of Boot Server",
Value=Join(
"/",
[GetAtt(boot, "PublicDnsName"),
GetAtt(boot, "PublicIp")])))
t.add_output(
Output("MastersURL",
Description="URL of the Masters",
Value=Join(
"",
["http://", GetAtt(elasticLBMasters, "DNSName")])))
t.add_output(
Output(
"PublicAgentsURL",
Description="URL of the Public Agents haproxy stats.",
Value=Join("", [
"http://",
GetAtt(elasticLBPublicAgents, "DNSName"), ":9090/haproxy?stats"
])))
# Write json to file
jsonStr = t.to_json()
fout = open(outfilename, "w")
fout.write(jsonStr)
fout.close()
# Print the json to screen
print(jsonStr)
))
t.add_resource(
elb.LoadBalancer(
"LoadBalancer",
Scheme="internet-facing",
Listeners=[
elb.Listener(LoadBalancerPort="3000",
InstancePort="3000",
Protocol="HTTP",
InstanceProtocol="HTTP"),
],
HealthCheck=elb.HealthCheck(
Target="HTTP:3000/",
HealthyThreshold="5",
UnhealthyThreshold="2",
Interval="20",
Timeout="15",
),
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=10,
),
CrossZone=True,
Subnets=Ref("PublicSubnet"),
SecurityGroups=[Ref("LoadBalancerSecurityGroup")],
))
ud = Base64(
Join('\n', [
"#!/bin/bash", "yum install --enablerepo=epel -y git",
t.add_resource(elb.LoadBalancer(
"LoadBalancer",
Scheme="internet-facing",
Listeners=[
elb.Listener(
LoadBalancerPort="80",
InstancePort=ApplicationPort,
Protocol="HTTP",
InstanceProtocol="HTTP"
),
],
HealthCheck=elb.HealthCheck(
Target=Join("", [
"TCP:",
ApplicationPort
]),
HealthyThreshold="5",
UnhealthyThreshold="2",
Interval="20",
Timeout="15",
),
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout="10",
),
CrossZone=True,
Subnets=Ref("PublicSubnet"),
SecurityGroups=[Ref("LoadBalancerSecurityGroup")],
))
ud = Base64(Join('', [
"#!/bin/bash\n",