from troposphere import (Join, Ref, Template)
from troposphere.codebuild import (Artifacts, Environment, Project, Source)
from troposphere.iam import Role
t = Template()
t.add_description("Effective DevOps in AWS: CodeBuild - Helloworld container")
t.add_resource(
Role("ServiceRole",
AssumeRolePolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service",
["codebuild.amazonaws.com"]))
]),
Path="/",
ManagedPolicyArns=[
'arn:aws:iam::aws:policy/AWSCodePipelineReadOnlyAccess',
'arn:aws:iam::aws:policy/AWSCodeBuildDeveloperAccess',
'arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser',
'arn:aws:iam::aws:policy/AmazonS3FullAccess',
'arn:aws:iam::aws:policy/CloudWatchLogsFullAccess'
]))
environment = Environment(
ComputeType='BUILD_GENERAL1_SMALL',
Image='aws/codebuild/docker:1.12.1',
Type='LINUX_CONTAINER',
EnvironmentVariables=[
))
ud = Base64(
Join('\n', [
"#!/bin/bash", "yum install --enablerepo=epel -y git",
"pip install ansible", AnsiblePullCmd,
"echo '*/10 * * * * {}' > /etc/cron.d/ansible-pull".format(
AnsiblePullCmd)
]))
t.add_resource(
Role("Role",
AssumeRolePolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", ["ec2.amazonaws.com"]))
])))
t.add_resource(
IAMPolicy("Policy",
PolicyName="AllowCodePipeline",
PolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[Action("codepipeline", "*")],
Resource=["*"])
]),
Roles=[Ref("Role")]))
t.add_resource(
InstanceProfile("InstanceProfile", Path="/", Roles=[Ref("Role")]))
'OpsWorksPrincipal': 'opsworks.amazonaws.com'
},
'us-west-2': {
'EC2Principal': 'ec2.amazonaws.com',
'OpsWorksPrincipal': 'opsworks.amazonaws.com'
}
})
web_server_role = Role(
"WebServerRole",
template=template,
AssumeRolePolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", [
FindInMap("Region2Principal", Ref("AWS::Region"),
"EC2Principal")
]))
]),
Path="/",
Policies=[
assets_management_policy,
logging_policy,
iam.Policy(
PolicyName="EBBucketAccess",
PolicyDocument=dict(Statement=[
dict(
Effect="Allow",
Action=[
"s3:Get*",
"s3:List*",
"s3:PutObject",
def add_resources(self):
self.EKSControlPlaneSG = self.template.add_resource(
ec2.SecurityGroup(
"EKSControlPlaneSG",
GroupDescription=
"Allow communication between WorkerNodes and EKS",
VpcId=Ref(self.SharedServicesVpcId),
Tags=self.base_tags +
Tags(Name=self.environment_parameters["ClientEnvironmentKey"] +
"-SS-EksControlPlane-SG"),
))
self.EKSClusterRole = self.template.add_resource(
Role(
"EKSClusterRole",
AssumeRolePolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service",
["eks.amazonaws.com"]))
]),
ManagedPolicyArns=[
"arn:aws:iam::aws:policy/AmazonEKSClusterPolicy",
"arn:aws:iam::aws:policy/AmazonEKSServicePolicy"
],
))
self.EKSCluster = self.template.add_resource(
eks.Cluster(
"EKSCluster",
DependsOn=["EKSControlPlaneSG", "EKSClusterRole"],
Name=self.environment_parameters["ClientEnvironmentKey"] +
"-SS-EKS",
RoleArn=GetAtt(self.EKSClusterRole, "Arn"),
Version=Ref(self.EksClusterVersion),
ResourcesVpcConfig=eks.ResourcesVpcConfig(
SecurityGroupIds=[Ref(self.EKSControlPlaneSG)],
SubnetIds=[
Ref(self.SharedServicesPubSubnet1),
Ref(self.SharedServicesPubSubnet2)
],
),
))
self.WorkerNodeEc2SG = self.template.add_resource(
ec2.SecurityGroup(
"WorkerNodeEc2SG",
DependsOn=["EKSCluster"],
GroupDescription=
"Allow communication between WorkerNodes and EKS",
VpcId=Ref(self.SharedServicesVpcId),
Tags=self.base_tags +
Tags(Name=self.environment_parameters["ClientEnvironmentKey"] +
"-SS-EksWorkerNodes-Ec2SG") +
Tags({
"kubernetes.io/cluster/" + self.environment_parameters["ClientEnvironmentKey"] + "-SS-EKS":
"owned"
}),
))
self.WorkerNodeInstanceRole = self.template.add_resource(
Role(
"WorkerNodeInstanceRole",
AssumeRolePolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service",
["ec2.amazonaws.com"]))
]),
ManagedPolicyArns=[
"arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
],
))
self.WorkerNodeInstanceProfile = self.template.add_resource(
InstanceProfile(
"WorkerNodeInstanceProfile",
Path="/",
Roles=[Ref(self.WorkerNodeInstanceRole)],
))
self.WorkerNodeEc2SGIngress = self.template.add_resource(
ec2.SecurityGroupIngress(
"WorkerNodeEc2SGIngress",
DependsOn=["WorkerNodeEc2SG"],
GroupId=Ref(self.WorkerNodeEc2SG),
IpProtocol="-1",
FromPort=0,
ToPort=65535,
SourceSecurityGroupId=Ref(self.WorkerNodeEc2SG),
))
self.WorkerNodeEc2SGIngressFromEKSControlPlane = self.template.add_resource(
ec2.SecurityGroupIngress(
"WorkerNodeEc2SGIngressFromEKSControlPlane",
DependsOn=["WorkerNodeEc2SG"],
GroupId=Ref(self.WorkerNodeEc2SG),
IpProtocol="tcp",
FromPort=1025,
ToPort=65535,
SourceSecurityGroupId=Ref(self.EKSControlPlaneSG),
))
self.EksControlPlaneEgressToWorkerNodes = self.template.add_resource(
ec2.SecurityGroupEgress(
"EksControlPlaneEgressToWorkerNodes",
DependsOn=["WorkerNodeEc2SG"],
GroupId=Ref(self.EKSControlPlaneSG),
IpProtocol="tcp",
FromPort=1025,
ToPort=65535,
DestinationSecurityGroupId=Ref(self.WorkerNodeEc2SG),
))
self.WorkerNodeEc2SG443IngressFromEKSControlPlane = self.template.add_resource(
ec2.SecurityGroupIngress(
"WorkerNodeEc2SG443IngressFromEKSControlPlane",
DependsOn=["WorkerNodeEc2SG"],
GroupId=Ref(self.WorkerNodeEc2SG),
IpProtocol="tcp",
FromPort=443,
ToPort=443,
SourceSecurityGroupId=Ref(self.EKSControlPlaneSG),
))
self.EKSControlPlaneSG443IngressFromWorkerNode = self.template.add_resource(
ec2.SecurityGroupIngress(
"EKSControlPlaneSG443IngressFromWorkerNode",
DependsOn=["WorkerNodeEc2SG"],
GroupId=Ref(self.EKSControlPlaneSG),
IpProtocol="tcp",
FromPort=443,
ToPort=443,
SourceSecurityGroupId=Ref(self.WorkerNodeEc2SG),
))
self.EksControlPlane443EgressToWorkerNodes = self.template.add_resource(
ec2.SecurityGroupEgress(
"EksControlPlane443EgressToWorkerNodes",
DependsOn=["WorkerNodeEc2SG"],
GroupId=Ref(self.EKSControlPlaneSG),
IpProtocol="tcp",
FromPort=443,
ToPort=443,
DestinationSecurityGroupId=Ref(self.WorkerNodeEc2SG),
))
self.WorkerNodeLaunchConfiguration = self.template.add_resource(
LaunchConfiguration(
"WorkerNodeLaunchConfiguration",
ImageId=Ref(self.WorkerNodeImageId),
InstanceType=Ref(self.WorkerNodeInstanceType),
IamInstanceProfile=Ref(self.WorkerNodeInstanceProfile),
KeyName=Ref(self.WorkerNodeKeyName),
SecurityGroups=[Ref(self.WorkerNodeEc2SG)],
UserData=Base64(
Join('', [
"#!/bin/bash \n", "set -o xtrace \n"
"ClusterName=\"" +
self.environment_parameters["ClientEnvironmentKey"] +
"-SS-EKS" + "\" \n", "BootstrapArguments=\""
"\" \n",
"/etc/eks/bootstrap.sh ${ClusterName} ${BootstrapArguments} \n"
]))))
self.WorkerNodeAutoScalingGroup = self.template.add_resource(
AutoScalingGroup(
"WorkerNodeAutoscalingGroup",
AutoScalingGroupName=self.
environment_parameters["ClientEnvironmentKey"] +
"-SS-EksWorkerNodeAutoScalingGroup",
LaunchConfigurationName=Ref(
self.WorkerNodeLaunchConfiguration),
MaxSize=Ref(self.WorkerNodeASGGroupMaxSize),
MinSize=Ref(self.WorkerNodeASGGroupMinSize),
DesiredCapacity=Ref(self.WorkerNodeASGGroupDesiredSize),
HealthCheckType=Ref(self.WorkerNodeASGHealthCheckType),
HealthCheckGracePeriod=Ref(
self.WorkerNodeASGHealthCheckGracePeriod),
Cooldown=Ref(self.WorkerNodeASGCoolDown),
VPCZoneIdentifier=[
Ref(self.SharedServicesPrivSubnet1),
Ref(self.SharedServicesPrivSubnet2)
],
Tags=[
AutoScalingTag(
"Name",
self.environment_parameters["ClientEnvironmentKey"] +
"-SS-EKS" + "-WorkerNodeGroup-Node", True),
AutoScalingTag(
"kubernetes.io/cluster/" +
self.environment_parameters["ClientEnvironmentKey"] +
"-SS-EKS", "owned", True),
AutoScalingTag(
"Environment",
self.environment_parameters["EnvironmentName"], True),
AutoScalingTag(
"ResourceOwner",
self.environment_parameters["ResourceOwner"], True),
AutoScalingTag(
"ClientCode",
self.environment_parameters["ClientEnvironmentKey"],
True),
],
))
KeySchema=[
KeySchema(AttributeName="uuid", KeyType="HASH"),
KeySchema(AttributeName="industry", KeyType="RANGE")
],
ProvisionedThroughput=ProvisionedThroughput(ReadCapacityUnits=1,
WriteCapacityUnits=1)))
companyRole = t.add_resource(
Role(
"Role",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[Action("sts", "AssumeRole")],
Principal=Principal("Service",
["lambda.amazonaws.com"]))
]),
Policies=[
iamPolicy(
PolicyName="Policy",
PolicyDocument=Policy(Statement=[
Statement(
Effect=Allow,
Action=[Action("s3", "Get*"),
Action("s3", "List*")],
Resource=[
Ref(shared_resources_bucket_arn),
]),
Statement(Effect=Allow,
Action=[
Action("logs", "CreateLogGroup"),
def make_simple_assume_statement(principal):
return Statement(Principal=Principal('Service', [principal]),
Effect=Allow,
Action=[sts.AssumeRole])
def _add_push_messages_api(self):
self.MessagesServerRole = self.add_resource(
Role(
"MessagesServerRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service",
"ec2.amazonaws.com"))
]),
Path="/",
))
self.MessagesServerPolicy = self.add_resource(
PolicyType("MessagesServerPolicy",
PolicyName="MessagesServerRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=self.PushMessageStatements,
),
Roles=[Ref(self.MessagesServerRole)],
DependsOn="MessagesServerRole"))
self.MessagesServerInstanceProfile = self.add_resource(
InstanceProfile("MessagesServerInstanceProfile",
Path="/",
Roles=[Ref(self.MessagesServerRole)]))
self.MessagesServerSG = self.add_resource(
SecurityGroup(
"MessagesServerSG",
SecurityGroupIngress=[
allow_tcp(22),
allow_tcp(80),
],
VpcId=Ref(self.ProcessorVPCId),
GroupDescription="Allow HTTP traffic to Message Server node",
))
self.MessagesServerInstance = self.add_resource(
Instance(
"MessagesServerInstance",
ImageId="ami-2c393546",
InstanceType="t2.micro",
SecurityGroupIds=[
GetAtt(self.LambdaProcessorSG, "GroupId"),
GetAtt(self.MessagesServerSG, "GroupId"),
],
KeyName=Ref(self.KeyPair),
IamInstanceProfile=Ref(self.MessagesServerInstanceProfile),
CreationPolicy=CreationPolicy(ResourceSignal=ResourceSignal(
Timeout='PT15M')),
UserData=Base64(
Join("", [
"#cloud-config\n\n",
"coreos:\n",
" units:\n",
] + self._aws_cfn_signal_service(
"pushmessages", "MessagesServerInstance"
) + [
" - name: 'pushmessages.service'\n",
" command: 'start'\n",
" content: |\n",
" [Unit]\n",
" Description=Push Messages container\n",
" Author=Mozilla Services\n",
" After=docker.service\n",
" \n",
" [Service]\n",
" Restart=always\n",
" ExecStartPre=-/usr/bin/docker kill pushmessages\n",
" ExecStartPre=-/usr/bin/docker rm pushmessages\n",
" ExecStartPre=/usr/bin/docker pull ",
"bbangert/push-messages:",
Ref(self.PushMessagesVersion),
"\n",
" ExecStart=/usr/bin/docker run ",
"--name pushmessages ",
"-p 80:8000 ",
"-e 'AWS_DEFAULT_REGION=us-east-1' ",
"-e 'REDIS_ELASTICACHE=",
Ref(self.RedisCluster),
"' ",
"bbangert/push-messages:",
Ref(self.PushMessagesVersion),
"\n",
])),
SubnetId=Ref(self.MessageAPISubnetId),
DependsOn="MessagesServerPolicy",
Tags=self._instance_tags("push-messages", "push-messages"),
))
self._template.add_output([
Output("MessagesAPI",
Description="Push Messages API URL",
Value=Join("", [
"http://",
GetAtt(self.MessagesServerInstance, "PublicIp"), "/"
]))
])
def _add_processor(self):
self.ProcessorExecRole = self.add_resource(
Role(
"ProcessorExecRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service",
"lambda.amazonaws.com"))
]),
Path="/",
))
# Common statements for accessing Redis
self.PushMessageStatements = [
Statement(Effect=Allow,
Action=[
elasticache.DescribeCacheClusters,
],
Resource=["*"]),
Statement(Effect=Allow,
Action=[
ddb.ListTables,
ddb.DescribeTable,
],
Resource=["*"]),
]
self.ProcessorLambdaPolicy = self.add_resource(
PolicyType(
"ProcessorLambdaPolicy",
PolicyName="ProcessorLambdaRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[
Action("logs", "CreateLogGroup"),
Action("logs", "CreateLogStream"),
Action("logs", "PutLogEvents"),
],
Resource=["arn:aws:logs:*:*:*"]),
Statement(Effect=Allow,
Action=[
s3.GetBucketLocation,
s3.GetObject,
s3.ListBucket,
s3.ListBucketMultipartUploads,
],
Resource=[
Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
]),
Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
"/*",
])
]),
Statement(Effect=Allow,
Action=[
ec2.CreateNetworkInterface,
ec2.DescribeNetworkInterfaces,
ec2.DeleteNetworkInterface,
],
Resource=["*"]),
] + self.PushMessageStatements),
Roles=[Ref(self.ProcessorExecRole)],
DependsOn="ProcessorExecRole"))
self.ProcessorS3Settings = self.add_resource(
CustomResource("ProcessorS3Settings",
ServiceToken=GetAtt(self.S3WriterCFCustomResource,
"Arn"),
Bucket=Ref(self.FirehoseLoggingBucket),
Key="processor_settings.json",
Content=dict(
redis_name=Ref(self.RedisCluster),
file_type="json",
),
DependsOn=["S3WriterCustomResource"]))
self.ProcessorLambda = self.add_resource(
Function("ProcessorLambda",
Description=("Processes logfiles when they hit S3"),
Runtime="python2.7",
Timeout=300,
Handler="lambda.handler",
Role=GetAtt(self.ProcessorExecRole, "Arn"),
Code=Code(
S3Bucket=Ref(self.ProcessorLambdaBucket),
S3Key=Ref(self.ProcessorLambdaKey),
),
VpcConfig=VPCConfig(
SecurityGroupIds=[
Ref(self.LambdaProcessorSG),
],
SubnetIds=Ref(self.ProcessorSubnetIds),
),
DependsOn=[
"ProcessorExecRole",
"ProcessorS3Settings",
]))
def _add_firehose(self):
self.FirehoseLoggingBucket = self.add_resource(
Bucket(
"FirehoseLoggingBucket",
DeletionPolicy="Retain",
))
self.FirehoseLoggingRole = self.add_resource(
Role(
"FirehoseRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal(
"Service", "firehose.amazonaws.com"))
]),
Path="/",
))
self.FirehosePolicy = self.add_resource(
PolicyType("FirehosePolicy",
PolicyName="FirehoseRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[
s3.AbortMultipartUpload,
s3.GetBucketLocation, s3.GetObject,
s3.ListBucket,
s3.ListBucketMultipartUploads,
s3.PutObject
],
Resource=[
Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
]),
Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
"/*",
])
]),
]),
Roles=[Ref(self.FirehoseLoggingRole)]))
self.FirehoseLogstream = self.add_resource(
CustomResource("FirehoseLogStream",
ServiceToken=GetAtt(self.FirehoseCFCustomResource,
"Arn"),
S3DestinationConfiguration=dict(
RoleARN=GetAtt(self.FirehoseLoggingRole, "Arn"),
BucketARN=Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
]),
BufferingHints=dict(
SizeInMBs=5,
IntervalInSeconds=60,
)),
DependsOn=["FirehosePolicy"]))
self._template.add_output([
Output(
"FirehoseLoggingBucket",
Description="Firehose Logging Bucket",
Value=Ref(self.FirehoseLoggingBucket),
)
])
