def gen_sg():
sg = SecurityGroup("DBSecurityGroup",
GroupDescription="Enable Postgres on the inbound port",
VpcId=ImportValue(
Sub("${NetworkName}-network-vpc-VpcId")))
self_sg_rule = SecurityGroupIngress(
"IngressFromSelfRule",
GroupId=Ref("DBSecurityGroup"),
IpProtocol="tcp",
FromPort="5432",
ToPort="5432",
SourceSecurityGroupId=Ref("DBSecurityGroup"),
)
lambda_sg_rule = SecurityGroupIngress(
"IngressFromLambdaRule",
Condition="InstallPostgis",
GroupId=Ref("DBSecurityGroup"),
IpProtocol="tcp",
FromPort="5432",
ToPort="5432",
SourceSecurityGroupId=ImportValue(
Sub("${ProvisionerStackName}-SecurityGroup")),
)
return [sg, self_sg_rule, lambda_sg_rule]
def __init__(self, title, template, *args, **kwargs):
self.props['PeerCidrBlock'] = (str, True)
super().__init__(title, template, *args, **kwargs)
for index in list(template.resources):
resource = template.resources[index]
if isinstance(resource, RouteTable):
Route(index + 'to' + title + 'route',
template,
DestinationCidrBlock=kwargs['PeerCidrBlock'],
RouteTableId=Ref(resource),
VpcPeeringConnectionId=Ref(self))
SecurityGroup(title + "securitygroup",
template,
GroupDescription="Allow SSH & ICMP from " + title,
VpcId=kwargs['VpcId'])
SecurityGroupIngress(title + "sshingress",
template,
GroupId=Ref(title + "securitygroup"),
CidrIp=kwargs['PeerCidrBlock'],
IpProtocol="tcp",
FromPort="22",
ToPort="22")
SecurityGroupIngress(title + "icmpingress",
template,
GroupId=Ref(title + "securitygroup"),
CidrIp=kwargs['PeerCidrBlock'],
IpProtocol="icmp",
FromPort="-1",
ToPort="-1")
self.props.pop('PeerCidrBlock')
self.properties.pop('PeerCidrBlock')
def set_aws_sources_ingress(self, settings, destination_title,
sg_ref) -> None:
"""
Method to define AWS Sources ingresses
:param settings:
:param destination_title:
:param sg_ref:
"""
for source in self.aws_sources:
for port in self.ports:
if (keyisset("Ports", source)
and port["published"] not in source["Ports"]):
continue
target_port = set_else_none("published",
port,
alt_value=set_else_none(
"target", port, None))
if target_port is None:
raise ValueError(
"Wrong port definition value for security group ingress",
port)
common_args = {
"FromPort": target_port,
"ToPort": target_port,
"IpProtocol": port["protocol"],
"GroupId": sg_ref,
}
if source["Type"] == "SecurityGroup":
if keyisset("Id", source):
sg_id = source["Id"]
elif keyisset("Lookup", source):
sg_id = lookup_security_group(settings,
source["Lookup"])
else:
raise KeyError(
"Information missing to identify the SecurityGroup. Requires either Id or Lookup"
)
common_args.update({
"Description":
Sub(f"From {sg_id} to {destination_title} on port {target_port}"
)
})
self.aws_ingress_rules.append(
SecurityGroupIngress(
f"From{NONALPHANUM.sub('', sg_id)}ToServiceOn{target_port}",
SourceSecurityGroupId=sg_id,
SourceSecurityGroupOwnerId=Ref(AWS_ACCOUNT_ID),
**common_args,
))
elif source["Type"] == "PrefixList":
self.aws_ingress_rules.append(
SecurityGroupIngress(
f"From{NONALPHANUM.sub('', source['Id'])}ToServiceOn{target_port}",
SourcePrefixListId=source["Id"],
**common_args,
))
def attach_security_group(template, vpc):
secgroup = SecurityGroup("sgdemosecuritygroup", GroupDescription = "sgdemosecuritygroup", VpcId = Ref(vpc))
template.add_resource(secgroup)
ingress = SecurityGroupIngress("sgdemoingressssh", GroupId = Ref(secgroup), IpProtocol = "tcp", FromPort = "22", ToPort = "22", CidrIp = "0.0.0.0/0")
template.add_resource(ingress)
ingress = SecurityGroupIngress("sgdemoingresses", GroupId = Ref(secgroup), IpProtocol = "tcp", FromPort = "9200", ToPort = "9399", CidrIp = "0.0.0.0/0")
template.add_resource(ingress)
ingress = SecurityGroupIngress("sgdemoingresskibana", GroupId = Ref(secgroup), IpProtocol = "tcp", FromPort = "5601", ToPort = "5601", CidrIp = "0.0.0.0/0")
template.add_resource(ingress)
egress = SecurityGroupEgress("sgdemoegress", GroupId = Ref(secgroup), IpProtocol = "-1", FromPort = "-1", ToPort = "-1", CidrIp = "0.0.0.0/0")
template.add_resource(egress)
return secgroup
def add_aws_sources(self, sg):
"""
Method to add ingress rules from other AWS Sources
:param sg:
:return:
"""
allowed_keys = ["type", "id"]
allowed_types = ["SecurityGroup", "PrefixList"]
for source in self.config.aws_sources:
if not all(key in allowed_keys for key in source.keys()):
raise KeyError(
"Missing ingress properties. Got",
source.keys,
"Expected",
allowed_keys,
)
if not source["type"] in allowed_types:
raise ValueError(
"Invalid type specified. Got",
source["type"],
"Allowed one of ",
allowed_types,
)
if source["type"] == "SecurityGroup":
for port in self.config.ports:
SecurityGroupIngress(
f"From{NONALPHANUM.sub('', source['id'])}ToServiceOn{port['published']}",
template=self.template,
FromPort=port["published"],
ToPort=port["published"],
IpProtocol=port["protocol"],
GroupId=GetAtt(sg, "GroupId"),
SourceSecurityGroupOwnerId=Ref(AWS_ACCOUNT_ID),
SourceSecurityGroupId=source["id"],
Description=Sub(
f"From {source['id']} to ${{{SERVICE_NAME_T}}} on port {port['published']}"
),
)
elif source["type"] == "PrefixList":
for port in self.config.ports:
SecurityGroupIngress(
f"From{NONALPHANUM.sub('', source['id'])}ToServiceOn{port['published']}",
template=self.template,
FromPort=port["published"],
ToPort=port["published"],
IpProtocol=port["protocol"],
GroupId=GetAtt(sg, "GroupId"),
SourcePrefixListId=source["id"],
Description=Sub(
f"From {source['id']} to ${{{SERVICE_NAME_T}}} on port {port['published']}"
),
)
def add_self_ingress(self) -> None:
"""
Method to allow communications internally to the group on set ports
"""
if (not self.family.template or not self.family.ecs_service
or not self.ingress_from_self):
return
for port in self.ports:
target_port = set_else_none("published",
port,
alt_value=set_else_none(
"target", port, None))
if target_port is None:
raise ValueError(
"Wrong port definition value for security group ingress",
port)
self.ingress.to_self_rules.append(
SecurityGroupIngress(
f"AllowingInterCommunicationPort{target_port}{port['protocol']}",
template=self.family.template,
FromPort=target_port,
ToPort=target_port,
IpProtocol=port["protocol"],
GroupId=GetAtt(
self.family.service_networking.security_group,
"GroupId"),
SourceSecurityGroupId=GetAtt(
self.family.service_networking.security_group,
"GroupId"),
SourceSecurityGroupOwnerId=Ref(AWS_ACCOUNT_ID),
Description=Sub(
f"Allowing traffic internally on port {target_port}"),
))
def add_security_group_ingress(service_stack: ComposeXStack, db_name: str,
sg_id, port):
"""
Function to add a SecurityGroupIngress rule into the ECS Service template
:param ecs_composex.ecs.ServicesStack service_stack: The root stack for the services
:param str db_name: the name of the database to use for imports
:param sg_id: The security group Id to use for ingress. DB Security group, not service's
:param port: The port for Ingress to the DB.
"""
service_template = service_stack.stack_template
add_resource(
service_template,
SecurityGroupIngress(
f"AllowFrom{service_stack.title}to{db_name}",
template=service_template,
GroupId=sg_id,
FromPort=port,
ToPort=port,
Description=Sub(f"Allow FROM {service_stack.title} TO {db_name}"),
SourceSecurityGroupId=GetAtt(service_template.resources[SG_T],
"GroupId"),
SourceSecurityGroupOwnerId=Ref("AWS::AccountId"),
IpProtocol="6",
),
)
def custom_security_group(vpc_stack, region, request, cfn_stacks_factory):
template = Template()
template.set_version("2010-09-09")
template.set_description("custom security group stack for testing additional_sg and vpc_security_group_id")
security_group = template.add_resource(
SecurityGroup(
"SecurityGroupResource",
GroupDescription="custom security group for testing additional_sg and vpc_security_group_id",
VpcId=vpc_stack.cfn_outputs["VpcId"],
)
)
template.add_resource(
SecurityGroupIngress(
"SecurityGroupIngressResource",
IpProtocol="-1",
FromPort=0,
ToPort=65535,
SourceSecurityGroupId=Ref(security_group),
GroupId=Ref(security_group),
)
)
stack = CfnStack(
name=generate_stack_name("integ-tests-custom-sg", request.config.getoption("stackname_suffix")),
region=region,
template=template.to_json(),
)
cfn_stacks_factory.create_stack(stack)
yield stack
if not request.config.getoption("no_delete"):
cfn_stacks_factory.delete_stack(stack.name, region)
def add_dependant_ingress_rules(dst_family: ComposeFamily,
dst_family_sg_param: Parameter,
src_family: ComposeFamily) -> None:
for port in dst_family.service_networking.ports:
target_port = set_else_none("published",
port,
alt_value=set_else_none(
"target", port, None))
if target_port is None:
raise ValueError(
"Wrong port definition value for security group ingress", port)
common_args = {
"FromPort":
target_port,
"ToPort":
target_port,
"IpProtocol":
port["protocol"],
"SourceSecurityGroupOwnerId":
Ref(AWS_ACCOUNT_ID),
"Description":
Sub(f"From ${{{SERVICE_NAME_T}}} to {dst_family.stack.title} on port {target_port}"
),
}
src_family.template.add_resource(
SecurityGroupIngress(
f"From{src_family.logical_name}To{dst_family.stack.title}On{target_port}",
SourceSecurityGroupId=GetAtt(
src_family.service_networking.security_group, "GroupId"),
GroupId=Ref(dst_family_sg_param),
**common_args,
))
def add_ingress_rule_to_db_security_group(template, db_sg, db_port, stack_sg):
sg_ingress = SecurityGroupIngress(stack_name_strict + "DbIngressRule",
IpProtocol="tcp",
FromPort=Ref(db_port),
ToPort=Ref(db_port),
SourceSecurityGroupId=Ref(stack_sg),
GroupId=Ref(db_sg))
template.add_resource(sg_ingress)
def addIngress(self, name, app, env, cidr, from_port, to_port, protocol,
resource):
ingress = SecurityGroupIngress(
"ingress" + name + app + env,
CidrIp=cidr,
Description="Ingress number ",
GroupId=Ref(resource),
IpProtocol=protocol,
DependsOn="SecurityGroup" + name + app + env,
FromPort=from_port,
ToPort=to_port,
)
return (ingress)
def security_group_ingress(name, desc, ip_protocol, from_port, to_port, group_id, cidr):
"""
Returns a security group ingress rule resource
"""
resource_name = '{}SGIngress'.format(sanitize_resource_name(name))
return SecurityGroupIngress(
resource_name,
Description=desc,
IpProtocol=ip_protocol,
FromPort=from_port,
ToPort=to_port,
GroupId=group_id,
CidrIp=cidr,
)
def create_ext_sources_ingress_rule(self, destination_title,
allowed_source, security_group,
**props) -> None:
"""
Creates the Security Ingress rule for a CIDR based rule
:param str destination_title:
:param dict allowed_source:
:param security_group:
:param dict props:
"""
for port in self.ports:
target_port = set_else_none("published",
port,
alt_value=set_else_none(
"target", port, None))
if target_port is None:
raise ValueError(
"Wrong port definition value for security group ingress",
port)
if (keyisset("Ports", allowed_source)
and target_port not in allowed_source["Ports"]):
continue
if keyisset("Name", allowed_source):
name = NONALPHANUM.sub("", allowed_source["Name"])
title = f"From{name.title()}To{target_port}{port['protocol']}"
description = Sub(
f"From {name.title()} "
f"To {target_port}{port['protocol']} for {destination_title}"
)
else:
title = (f"From{flatten_ip(allowed_source[self.ipv4_key])}"
f"To{target_port}{port['protocol']}")
description = Sub(f"Public {target_port}{port['protocol']}"
f" for {destination_title}")
self.ext_ingress_rules.append(
SecurityGroupIngress(
title,
Description=description
if not keyisset("Description", allowed_source) else
allowed_source["Description"],
GroupId=security_group,
IpProtocol=port["protocol"],
FromPort=target_port,
ToPort=target_port,
**props,
))
def create_ingress_rule(self, root_stack, nodes):
"""
Creates EC2 ingress rules to allow all traffic from node to backends nodes SG.
:param ecs_composex.common.stacks.ComposeXStack root_stack:
:param list<ecs_composex.appmesh.appmesh_nodes.MeshNodes> nodes: list of nodes
"""
for node in nodes:
rule_name = f"AllowAllFrom{self.node.title}To{node.node.title}ForMesh"
LOG.debug(rule_name)
SecurityGroupIngress(
rule_name,
template=root_stack.stack_template,
FromPort="-1",
ToPort="-1",
GroupId=node.get_sg_param,
SourceSecurityGroupId=self.get_sg_param,
IpProtocol="-1",
)
def add_nat_sg_rules(self):
'''
Add the security group rules necessary for the NAT to operate
For now, this is opening all ingress from the VPC and all egress to the internet
'''
self.add_resource(
SecurityGroupIngress("Nat%sIngress" % str(self.subnet_index),
ToPort="-1",
FromPort="-1",
IpProtocol="-1",
GroupId=Ref(self.sg),
CidrIp=self.vpc_cidr))
self.add_resource(
SecurityGroupEgress("Nat%sEgress" % str(self.subnet_index),
ToPort="-1",
FromPort="-1",
IpProtocol="-1",
GroupId=Ref(self.sg),
CidrIp='0.0.0.0/0'))
def add_self_ingress(self, sg):
"""
Method to allow communications internally to the group on set ports
:param sg:
:return:
"""
for port in self.config.ports:
SecurityGroupIngress(
f"AllowingMyselfToMyselfOnPort{port['published']}",
template=self.template,
FromPort=port["published"],
ToPort=port["published"],
IpProtocol=port["protocol"],
GroupId=GetAtt(sg, "GroupId"),
SourceSecurityGroupId=GetAtt(sg, "GroupId"),
SourceSecurityGroupOwnerId=Ref(AWS_ACCOUNT_ID),
Description=Sub(
f"Allowing traffic internally on port {port['published']}"
),
)
def add_independent_rules(dst_family: ComposeFamily, service_name: str,
root_stack: ComposeXStack) -> None:
"""
Adds security groups rules in the root stack as both services need to be created (with their SG)
before the ingress rule can be defined.
:param dst_family:
:param service_name:
:param root_stack:
:return:
"""
src_service_stack = root_stack.stack_template.resources[service_name]
for port in dst_family.service_networking.ports:
target_port = set_else_none("published",
port,
alt_value=set_else_none(
"target", port, None))
if target_port is None:
raise ValueError(
"Wrong port definition value for security group ingress", port)
ingress_rule = SecurityGroupIngress(
f"From{src_service_stack.title}To{dst_family.logical_name}On{target_port}",
FromPort=target_port,
ToPort=target_port,
IpProtocol=port["protocol"],
Description=Sub(
f"From {src_service_stack.title} to {dst_family.logical_name}"
f" on port {target_port}/{port['protocol']}"),
GroupId=GetAtt(
dst_family.stack.title,
f"Outputs.{dst_family.logical_name}GroupId",
),
SourceSecurityGroupId=GetAtt(
src_service_stack.title,
f"Outputs.{src_service_stack.title}GroupId",
),
SourceSecurityGroupOwnerId=Ref(AWS_ACCOUNT_ID),
)
if ingress_rule.title not in root_stack.stack_template.resources:
root_stack.stack_template.add_resource(ingress_rule)
def add_alb_sg(self, ports):
"""
Method to add a security group for an AWS ALB (ELBv2, application type).
:param ports: list of ports to add ingress from the ALB to ecs_service to
:type ports: list of ports
:return: The ALB's SG
:rtype: troposphere.ec2.SecurityGroup
"""
suffix = "Private"
if self.config.is_public:
suffix = "Public"
self.alb_sg = SecurityGroup(
f"AlbSecurityGroup{suffix}",
template=self.template,
GroupDescription=Sub(
f"ALB SG for ${{{SERVICE_NAME_T}}} in ${{{ROOT_STACK_NAME_T}}}"
),
VpcId=Ref(VPC_ID),
Tags=Tags({
"Name":
Sub(f"alb-sg-${{{SERVICE_NAME_T}}}-${{{ROOT_STACK_NAME_T}}}"),
"StackName":
Ref(AWS_STACK_NAME),
"MicroserviceName":
Ref(SERVICE_NAME),
}),
)
for port in ports:
SecurityGroupIngress(
f"FromAlbToServiceOnPort{port}",
template=self.template,
FromPort=port,
ToPort=port,
GroupId=GetAtt(SG_T, "GroupId"),
SourceSecurityGroupId=GetAtt(self.alb_sg, "GroupId"),
SourceSecurityGroupOwnerId=Ref(AWS_ACCOUNT_ID),
IpProtocol="tcp",
)
def add_security_group_ingress(service_stack, db_name):
"""
Function to add a SecurityGroupIngress rule into the ECS Service template
:param ecs_composex.ecs.ServicesStack service_stack: The root stack for the services
:param str db_name: the name of the database to use for imports
"""
service_template = service_stack.stack_template
sg_id = get_import_value(db_name, DB_EXPORT_SG_ID_T)
port = get_import_value(db_name, DB_EXPORT_PORT_T)
SecurityGroupIngress(
f"AllowRdsFrom{db_name}to{service_stack.title}",
template=service_template,
GroupId=sg_id,
FromPort=port,
ToPort=port,
Description=Sub(f"Allow FROM {db_name} TO {service_stack.title}"),
SourceSecurityGroupId=GetAtt(service_template.resources[SG_T],
"GroupId"),
SourceSecurityGroupOwnerId=Ref("AWS::AccountId"),
IpProtocol="6",
)
def add_lb_ingress(self, lb_name, lb_sg_ref) -> None:
"""
Method to add ingress rules from other AWS Sources
:param str lb_name:
:param lb_sg_ref:
:return:
"""
if not self.family.template or not self.family.ecs_service:
return
for port in self.ports:
title = f"FromLB{lb_name}To{self.family.stack.title}On{port['target']}"
common_args = {
"FromPort":
port["target"],
"ToPort":
port["target"],
"IpProtocol":
port["protocol"],
"GroupId":
GetAtt(self.family.service_networking.security_group,
"GroupId"),
"SourceSecurityGroupOwnerId":
Ref(AWS_ACCOUNT_ID),
"Description":
Sub(f"From ELB {lb_name} to ${{{SERVICE_NAME_T}}} on port {port['target']}"
),
}
if title in self.family.template.resources:
return
SecurityGroupIngress(
title,
template=self.family.template,
SourceSecurityGroupId=lb_sg_ref,
**common_args,
)
def create_lb_ingress_rule(self, allowed_source, security_group, **props):
for port in self.config.ports:
if keyisset("source_name", allowed_source):
title = f"From{allowed_source['source_name'].title()}Onto{port['published']}{port['protocol']}"
description = Sub(
f"From {allowed_source['source_name'].title()} "
f"To {port['published']}{port['protocol']} for ${{{SERVICE_NAME_T}}}"
)
else:
title = (f"From{flatten_ip(allowed_source['ipv4'])}"
"To{port['published']}{port['protocol']}")
description = Sub(
f"Public {port['published']}{port['protocol']}"
f" for ${{{SERVICE_NAME_T}}}")
SecurityGroupIngress(
title,
template=self.template,
Description=description,
GroupId=GetAtt(security_group, "GroupId"),
IpProtocol=port["protocol"],
FromPort=port["published"],
ToPort=port["published"],
**props,
)
))
# VPC with Public and Private Subnets
DefaultPrivateRoute = t.add_resource(
Route(
"DefaultPrivateRoute",
InstanceId=Ref("Nat"),
DestinationCidrBlock="0.0.0.0/0",
RouteTableId=Ref("PrivateRouteTable"),
))
NatHTTPIngress = t.add_resource(
SecurityGroupIngress(
"NatHTTPIngress",
ToPort="80",
FromPort="80",
IpProtocol="tcp",
GroupId=Ref("NatSG"),
CidrIp=Ref(privatesubnetcidr_param),
))
PublicRouteTable = t.add_resource(
RouteTable("PublicRouteTable",
VpcId=Ref("VPC"),
Tags=Tags(Name=Join("", [Ref("AWS::StackName"), "-public"]), )))
NatEIP = t.add_resource(EIP(
"NatEIP",
InstanceId=Ref("Nat"),
Domain="vpc",
))
# IpProtocol='tcp',
# FromPort='8081',
# ToPort='8081',
# CidrIp=quad_zero_ip),
],
))
'''
Necessary to create separate Security Group Ingress Rule objects to avoid
circular references errors ( which are not caught during template validation )
Allow all TCP, ICMP and UDP traffic within Hadoop Cluster SG
'''
# TODO: Develop specific Port list and refine security rules based on protocol port
SecurityGroupIngress('hadoopClusterICPM1',
template=t,
IpProtocol='icmp',
FromPort='-1',
ToPort='-1',
GroupId=Ref(hadoop_cluser_sg),
SourceSecurityGroupId=Ref(hadoop_cluser_sg))
SecurityGroupIngress('hadoopClusterTCP1',
template=t,
IpProtocol='tcp',
ToPort='65535',
FromPort='0',
GroupId=Ref(hadoop_cluser_sg),
SourceSecurityGroupId=Ref(hadoop_cluser_sg))
SecurityGroupIngress('hadoopClusterUDP1',
template=t,
IpProtocol='udp',
def _create_cfn_template(self):
self.tpl = Template()
self.tpl.add_version('2010-09-09')
self.tpl.add_description('CFN template to create an EKS node group and affiliated resources.')
eks_tag = 'kubernetes.io/cluster/{}'.format(self.cluster.name)
r = self.resources.get(self.RESOURCE_NG_ROLE.name)
if self.role:
profile = InstanceProfile(
self.RESOURCE_NG_PROFILE.name, InstanceProfileName=self.tag_name, Path='/', Roles=[self.role])
account_id = boto3.session.Session().client('sts').get_caller_identity().get('Account')
role_arn = 'arn:aws:iam::{}:role/{}'.format(account_id, self.role)
self.tpl.add_output(
Output(self.RESOURCE_NG_ROLE.name, Value=role_arn, Description='Node group role'))
r.status = Status.provided
r.resource_id = role_arn
else:
role = Role(
self.RESOURCE_NG_ROLE.name, RoleName=self.tag_name,
AssumeRolePolicyDocument=Policy(Statement=[
Statement(Effect=Allow, Action=[AssumeRole],
Principal=Principal('Service', ['ec2.amazonaws.com'])), ], ),
ManagedPolicyArns=['arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy',
'arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy',
'arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly']
)
self.tpl.add_resource(role)
profile = InstanceProfile(
self.RESOURCE_NG_PROFILE.name, InstanceProfileName=self.tag_name, Path='/', Roles=[Ref(role)])
self.tpl.add_output(
Output(self.RESOURCE_NG_ROLE.name, Value=GetAtt(role, 'Arn'), Description='Node group role'))
self.tpl.add_resource(profile)
if self.sg_igresses:
sg = SecurityGroup(
self.RESOURCE_NG_SG.name, VpcId=self.cluster.vpc, Tags=Tags({'Name': self.tag_name, eks_tag: 'owned'}),
GroupDescription='Security Group applied to the EKS node group',
SecurityGroupIngress=[SecurityGroupRule(IpProtocol=r.protocol, FromPort=r.from_port, ToPort=r.to_port,
CidrIp=r.cidr) for r in self.sg_igresses]
)
else:
sg = SecurityGroup(
self.RESOURCE_NG_SG.name, VpcId=self.cluster.vpc, Tags=Tags({'Name': self.tag_name, eks_tag: 'owned'}),
GroupDescription='Security Group applied to the EKS node group',
)
self.tpl.add_resource(sg)
self.tpl.add_resource(SecurityGroupIngress(
self.RESOURCE_NG_SG_INGRESS.name, DependsOn=sg, Description='Allow node to communicate with each other',
GroupId=Ref(sg), SourceSecurityGroupId=Ref(sg), IpProtocol='-1', FromPort=0, ToPort=65535
))
self.tpl.add_resource(SecurityGroupIngress(
self.RESOURCE_NG_SG_CP_INGRESS.name, DependsOn=sg,
Description='Allow kubelet and pods on the nodes to receive communication from the cluster control plane',
GroupId=Ref(sg), SourceSecurityGroupId=self.cluster.sg, IpProtocol='tcp', FromPort=1025, ToPort=65535
))
self.tpl.add_resource(SecurityGroupEgress(
self.RESOURCE_CP_EGRESS_TO_NG.name, DependsOn=sg,
Description='Allow the cluster control plane to communicate with nodes kubelet and pods',
GroupId=self.cluster.sg, DestinationSecurityGroupId=Ref(sg), IpProtocol='tcp', FromPort=1025, ToPort=65535
))
self.tpl.add_resource(SecurityGroupIngress(
self.RESOURCE_CP_SG_INGRESS.name, DependsOn=sg,
Description='Allow pods to communicate with the cluster API Server',
GroupId=self.cluster.sg, SourceSecurityGroupId=Ref(sg), IpProtocol='tcp', FromPort=443, ToPort=443
))
# keypair
ec2 = boto3.session.Session().resource('ec2')
r = self.resources.get(self.RESOURCE_NG_KEYPAIR.name)
if not self.keypair:
keyname = 'eks{}'.format(''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(5)))
with open(self.ssh_public_key, 'rb') as f:
ec2.import_key_pair(KeyName=keyname, PublicKeyMaterial=f.read())
self.keypair = keyname
self.keypair_imported = True
self.tpl.add_output(Output(self.OUTPUT_KEYNAME, Value=self.keypair, Description='Imported kaypair name'))
r.status = Status.created
else:
r.status = Status.provided
r.resource_id = self.keypair
# auto-scaling group and launch configuration
if self.no_user_data:
lc = LaunchConfiguration(
self.RESOURCE_NG_ASG_LC.name, AssociatePublicIpAddress=self.use_public_ip,
IamInstanceProfile=Ref(profile),
ImageId=self.ami, InstanceType=self.instance, KeyName=self.keypair, SecurityGroups=[Ref(sg)])
else:
user_data = Base64(
Join('', [line + '\n' for line in
Environment().from_string(self.USER_DATA).render(
ci=self.cluster, ng_asg=self.RESOURCE_NG_ASG.name, stack_name=self.stack_name,
max_pods=self.MAX_PODS.get(self.instance), region=self.region).split('\n')]))
lc = LaunchConfiguration(
self.RESOURCE_NG_ASG_LC.name, AssociatePublicIpAddress=self.use_public_ip,
IamInstanceProfile=Ref(profile),
ImageId=self.ami, InstanceType=self.instance, KeyName=self.keypair, SecurityGroups=[Ref(sg)],
UserData=user_data)
self.tpl.add_resource(lc)
self.tpl.add_resource(AutoScalingGroup(
self.RESOURCE_NG_ASG.name, DesiredCapacity=self.desired, MinSize=self.min, MaxSize=self.max,
LaunchConfigurationName=Ref(lc), VPCZoneIdentifier=self.subnets,
Tags=[Tag('Name', self.tag_name, True), Tag(eks_tag, 'owned', True)],
UpdatePolicy=UpdatePolicy(
AutoScalingRollingUpdate=AutoScalingRollingUpdate(MinInstancesInService=1, MaxBatchSize=1))))
consul_sg.VpcId = Ref(vpc)
consul_sg.Tags = Tags(Name = config['name'] + '-consul')
t.add_resource(consul_sg)
# consul ports
consul_ports = [
8300,
8301,
8302,
8400,
8500,
8600
]
for port in consul_ports:
rule = SecurityGroupIngress('consul' + str(port))
rule.SourceSecurityGroupId = Ref(consul_sg)
rule.FromPort = port
rule.ToPort = port
rule.GroupId = Ref(consul_sg)
rule.IpProtocol = 'tcp'
t.add_resource(rule)
# access web ui from home
consul_ui_rule = SecurityGroupIngress('consulUi')
consul_ui_rule.CidrIp = home_egress_ips[0][0]
consul_ui_rule.FromPort = 8500
consul_ui_rule.ToPort = 8500
consul_ui_rule.GroupId = Ref(consul_sg)
consul_ui_rule.IpProtocol = 'tcp'
t.add_resource(consul_ui_rule)
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol='tcp',
FromPort='22',
ToPort='22',
SourceSecurityGroupId=Ref(zk_ssh_source))
],
GroupDescription="Security group for access to Zookeeper",
VpcId=Ref(vpcid),
Tags=Tags(Name='zookeeper', Environment=Ref(environment))))
zk_ingress_rule = template.add_resource(
SecurityGroupIngress("ZkIngressRule",
FromPort='-1',
ToPort='-1',
IpProtocol='-1',
GroupId=Ref(security_group),
SourceSecurityGroupId=Ref(security_group),
DependsOn='ZkSecurityGroup'))
# Launch Configuration for Zookeeper
launch_config = template.add_resource(
LaunchConfiguration(
"ZkLaunchConfig",
KeyName=Ref(keyname),
InstanceType=Ref(instance_type),
SecurityGroups=[Ref(security_group)],
ImageId=Ref(ami_id),
IamInstanceProfile=Ref(instance_role),
UserData=Base64(
Join('', [
def _add_ec2_auto_scaling(self):
instance_profile = self._add_instance_profile()
self.sg_alb = SecurityGroup(
"SecurityGroupAlb",
VpcId=Ref(self.vpc),
GroupDescription=Sub("${AWS::StackName}-alb"))
self.template.add_resource(self.sg_alb)
self.sg_hosts = SecurityGroup(
"SecurityGroupEc2Hosts",
SecurityGroupIngress=[{
'SourceSecurityGroupId': Ref(self.sg_alb),
'IpProtocol': -1
}],
VpcId=Ref(self.vpc),
GroupDescription=Sub("${AWS::StackName}-hosts"))
self.template.add_resource(self.sg_hosts)
sg_host_ingress = SecurityGroupIngress("SecurityEc2HostsIngress",
SourceSecurityGroupId=Ref(
self.sg_hosts),
IpProtocol="-1",
GroupId=Ref(self.sg_hosts),
FromPort="-1",
ToPort="-1")
self.template.add_resource(sg_host_ingress)
database_security_group = SecurityGroup(
"SecurityGroupDatabases",
SecurityGroupIngress=[{
'SourceSecurityGroupId': Ref(self.sg_hosts),
'IpProtocol': -1
}],
VpcId=Ref(self.vpc),
GroupDescription=Sub("${AWS::StackName}-databases"))
self.template.add_resource(database_security_group)
user_data = Base64(
Sub('\n'.join([
"#!/bin/bash", "yum update -y",
"yum install -y aws-cfn-bootstrap",
"/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource LaunchConfiguration",
"/opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource AutoScalingGroup",
"yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm",
"systemctl enable amazon-ssm-agent",
"systemctl start amazon-ssm-agent", ""
])))
lc_metadata = cloudformation.Init({
"config":
cloudformation.InitConfig(
files=cloudformation.InitFiles({
"/etc/cfn/cfn-hup.conf":
cloudformation.InitFile(
content=Sub('\n'.join([
'[main]', 'stack=${AWS::StackId}',
'region=${AWS::Region}', ''
])),
mode='256', # TODO: Why 256
owner="root",
group="root"),
"/etc/cfn/hooks.d/cfn-auto-reloader.conf":
cloudformation.InitFile(content=Sub('\n'.join([
'[cfn-auto-reloader-hook]', 'triggers=post.update',
'path=Resources.ContainerInstances.Metadata.AWS::CloudFormation::Init',
'action=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource LaunchConfiguration',
''
])), )
}),
services={
"sysvinit":
cloudformation.InitServices({
"cfn-hup":
cloudformation.InitService(
enabled=True,
ensureRunning=True,
files=[
'/etc/cfn/cfn-hup.conf',
'/etc/cfn/hooks.d/cfn-auto-reloader.conf'
])
})
},
commands={
'01_add_instance_to_cluster': {
'command':
Sub('echo "ECS_CLUSTER=${Cluster}\nECS_RESERVED_MEMORY=256" > /etc/ecs/ecs.config'
)
}
})
})
launch_configuration = LaunchConfiguration(
'LaunchConfiguration',
UserData=user_data,
IamInstanceProfile=Ref(instance_profile),
SecurityGroups=[Ref(self.sg_hosts)],
InstanceType=Ref('InstanceType'),
ImageId=FindInMap("AWSRegionToAMI", Ref("AWS::Region"), "AMI"),
Metadata=lc_metadata,
KeyName=Ref(self.key_pair))
self.template.add_resource(launch_configuration)
# , PauseTime='PT15M', WaitOnResourceSignals=True, MaxBatchSize=1, MinInstancesInService=1)
up = AutoScalingRollingUpdate('AutoScalingRollingUpdate')
# TODO: clean up
subnets = list(self.private_subnets)
self.auto_scaling_group = AutoScalingGroup(
"AutoScalingGroup",
UpdatePolicy=up,
DesiredCapacity=self.desired_instances,
Tags=[{
'PropagateAtLaunch': True,
'Value': Sub('${AWS::StackName} - ECS Host'),
'Key': 'Name'
}],
MinSize=Ref('MinSize'),
MaxSize=Ref('MaxSize'),
VPCZoneIdentifier=[Ref(subnets.pop()),
Ref(subnets.pop())],
LaunchConfigurationName=Ref(launch_configuration),
CreationPolicy=CreationPolicy(ResourceSignal=ResourceSignal(
Timeout='PT15M')))
self.template.add_resource(self.auto_scaling_group)
self.cluster_scaling_policy = ScalingPolicy(
'AutoScalingPolicy',
AdjustmentType='ChangeInCapacity',
AutoScalingGroupName=Ref(self.auto_scaling_group),
Cooldown=300,
PolicyType='SimpleScaling',
ScalingAdjustment=1)
self.template.add_resource(self.cluster_scaling_policy)
))
internetgateway = t.add_resource(InternetGateway(
"InternetGateway",
))
gatewayattachment = t.add_resource(VPCGatewayAttachment(
"GatewayAttachment",
VpcId=Ref("VPC"),
InternetGatewayId=Ref("InternetGateway"),
))
securitygroupingress1 = SecurityGroupIngress(
"SecurityGroupIngress1",
CidrIp="10.0.0.0/16",
FromPort="80",
ToPort="80",
IpProtocol="tcp",
)
securitygroup = t.add_resource(SecurityGroup(
"SecurityGroup",
GroupDescription="Security Group",
SecurityGroupIngress=[securitygroupingress1],
VpcId=Ref("VPC"),
))
t.add_output(Output(
"ClusterEndpoint",
Value=Join(":", [GetAtt(redshiftcluster, "Endpoint.Address"),
GetAtt(redshiftcluster, "Endpoint.Port")]),
SecurityGroup(
"InstanceSecurityGroup",
VpcId=Ref(vpcId),
GroupDescription=Join(
"-",
[Ref(environment), Ref(appName), "instances"]),
Tags=Tags(Name=Join(
"-",
[Ref(environment), Ref(appName), "instances"]), ),
))
IngressForELB = t.add_resource(
SecurityGroupIngress(
"IngressForELB",
IpProtocol=Ref(appProtocol),
FromPort=Ref(appPort),
ToPort=Ref(appPort),
SourceSecurityGroupId=Ref("LoadBalancerSecurityGroup"),
GroupId=Ref("InstanceSecurityGroup")))
IngressForInstances = t.add_resource(
SecurityGroupIngress("IngressForInstances",
IpProtocol=Ref(appProtocol),
FromPort=Ref(appPort),
ToPort=Ref(appPort),
SourceSecurityGroupId=Ref("InstanceSecurityGroup"),
GroupId=Ref("InstanceSecurityGroup")))
IngressForSSH = t.add_resource(
SecurityGroupIngress("IngressForSSH",
IpProtocol="tcp",
FromPort=22,
ToPort=22,
def elb_asg_lc_template(app, env, nameSGRDS, rdsPort, instanceType, ami,
subnets, elbPort, elbCidrBlock, ec2Port,
desiredCapacity, minSize, maxSize, region, nameBucket,
officeIP):
template = Template()
sgELB = template.add_resource(
SecurityGroup(
"SecurityGroupELB" + app + env,
GroupDescription="Security group for " + app + "-" + env,
VpcId=ImportValue("VPC" + env),
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol="tcp",
FromPort=elbPort,
ToPort=elbPort,
CidrIp=elbCidrBlock,
)
],
SecurityGroupEgress=[
SecurityGroupRule(IpProtocol="-1",
ToPort=0,
FromPort=65535,
CidrIp="0.0.0.0/0")
],
Tags=Tags(
env=env,
Name="sg-ELB" + app + "-" + env,
app=app,
),
))
sgEC2 = template.add_resource(
SecurityGroup(
"SecurityGroupEC2" + app + env,
GroupDescription="Security group for EC2 " + app + "-" + env,
VpcId=ImportValue("VPC" + env),
DependsOn="SecurityGroupELB" + app + env,
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol="tcp",
FromPort=ec2Port,
ToPort=ec2Port,
SourceSecurityGroupId=Ref(sgELB),
),
SecurityGroupRule(
IpProtocol="tcp",
FromPort=22,
ToPort=22,
CidrIp=officeIP,
),
],
SecurityGroupEgress=[
SecurityGroupRule(IpProtocol="-1",
ToPort=0,
FromPort=65535,
CidrIp="0.0.0.0/0")
],
Tags=Tags(
env=env,
Name="sg-EC2-" + app + "-" + env,
app=app,
),
))
addIngressRDS = template.add_resource(
SecurityGroupIngress(
"ingressSGRDS" + app + env,
SourceSecurityGroupId=Ref(sgEC2),
Description="From EC2 instances",
GroupId=ImportValue("SG-" + nameSGRDS + "-" + app + "-" + env),
IpProtocol="tcp",
FromPort=rdsPort,
ToPort=rdsPort,
DependsOn="SecurityGroupEC2" + app + env,
))
launchConfig = template.add_resource(
LaunchConfiguration("LaunchConfiguration" + app + env,
InstanceType=instanceType,
ImageId=ami,
SecurityGroups=[Ref(sgEC2)],
IamInstanceProfile=ImportValue("Role-" + app +
"-" + env)))
bucketPolicy = template.add_resource(
BucketPolicy("BucketPolicy" + nameBucket + app + env,
Bucket=ImportValue("Bucket" + nameBucket + app + env),
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["s3:PutObject"],
"Effect":
"Allow",
"Resource":
Join("", [
"arn:aws:s3:::",
ImportValue("Bucket" + nameBucket + app +
env), "/AWSLogs/",
Ref("AWS::AccountId"), "/*"
]),
"Principal": {
"AWS": ["156460612806"]
}
}]
}))
lb = template.add_resource(
LoadBalancer("LoadBalancer" + app + env,
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=120,
),
Subnets=subnets,
HealthCheck=elb.HealthCheck(
"HealthCheck",
Target="TCP:" + str(ec2Port),
HealthyThreshold="5",
UnhealthyThreshold="5",
Interval="30",
Timeout="15",
),
Listeners=[
elb.Listener(
LoadBalancerPort=elbPort,
InstancePort=ec2Port,
Protocol="HTTP",
InstanceProtocol="HTTP",
),
],
CrossZone=True,
SecurityGroups=[Ref(sgELB)],
LoadBalancerName="lb-" + app + "-" + env,
Scheme="internet-facing",
AccessLoggingPolicy=AccessLoggingPolicy(
"LoggingELB" + app + env,
EmitInterval=5,
Enabled=True,
S3BucketName=ImportValue("Bucket" + nameBucket + app +
env),
)))
asg = template.add_resource(
AutoScalingGroup(
"AutoscalingGroup" + app + env,
DesiredCapacity=desiredCapacity,
Tags=[Tag("Environment", env, True)],
LaunchConfigurationName=Ref(launchConfig),
MinSize=minSize,
MaxSize=maxSize,
LoadBalancerNames=[Ref(lb)],
AvailabilityZones=GetAZs(region),
VPCZoneIdentifier=subnets,
HealthCheckType="ELB",
HealthCheckGracePeriod=300,
UpdatePolicy=UpdatePolicy(
AutoScalingReplacingUpdate=AutoScalingReplacingUpdate(
WillReplace=True, ),
AutoScalingRollingUpdate=AutoScalingRollingUpdate(
PauseTime='PT5M',
MinInstancesInService="1",
MaxBatchSize='1',
WaitOnResourceSignals=True,
))))
return (template.to_json())
###########################################
ec2_security_group = t.add_resource(SecurityGroup(
'Ec2SecurityGroup',
VpcId=Ref(vpc),
GroupDescription='Security group for Minecraft EC2 instance.',
Tags=Tags(
Name=Sub('$minecraft-sg'),
)
))
ec2_minecraft_tcp_ingress = t.add_resource(SecurityGroupIngress(
'Ec2MinecraftTcpIngress',
DependsOn='Ec2SecurityGroup',
ToPort='25565',
FromPort='25565',
IpProtocol='tcp',
GroupId=Ref(ec2_security_group),
CidrIp=c.QUAD_ZERO,
))
ec2_minecraft_udp_ingress = t.add_resource(SecurityGroupIngress(
'Ec2MinecraftUdpIngress',
DependsOn='Ec2SecurityGroup',
ToPort='25565',
FromPort='25565',
IpProtocol='udp',
GroupId=Ref(ec2_security_group),
CidrIp=c.QUAD_ZERO,
))