}, "eu-central-1": { "AMI": "ami-034fffcc6a0063961" }, "sa-east-1": { "AMI": "ami-0112d42866980b373" } }) ec2_instance = template.add_resource( ec2.Instance("Ec2Instance", ImageId=FindInMap("RegionMap", Ref("AWS::Region"), "AMI"), InstanceType="t2.micro", KeyName=Ref(keyname_param), SecurityGroups=["default"], UserData=Base64("80"))) template.add_output([ Output( "InstanceId", Description="InstanceId of the newly created EC2 instance", Value=Ref(ec2_instance), ), Output( "AZ", Description="Availability Zone of the newly created EC2 instance", Value=GetAtt(ec2_instance, "AvailabilityZone"), ), Output( "PublicIP", Description="Public IP address of the newly created EC2 instance",
), ec2.SecurityGroupRule( IpProtocol="tcp", FromPort=ApplicationPort, ToPort=ApplicationPort, CidrIp="0.0.0.0/0", ), ], )) ud = Base64( Join('\n', [ "#!/bin/bash", "sudo yum install --enablerepo=epel -y git", "sudo yum -y install python34", "sudo wget https://bootstrap.pypa.io/get-pip.py -O /home/ec2-user/get-pip.py", "sudo python3 /home/ec2-user/get-pip.py --user", "sudo pip install --upgrade pip", "sudo /usr/local/bin/pip install --upgrade setuptools", "sudo /usr/local/bin/pip install ansible", AnsiblePullCmd, "sudo echo '*/10 * * * * ec2-user {}' > /etc/cron.d/ansible-pull". format(AnsiblePullCmd) ])) t.add_resource( Role("Role", AssumeRolePolicyDocument=Policy(Statement=[ Statement(Effect=Allow, Action=[AssumeRole], Principal=Principal("Service", ["ec2.amazonaws.com"])) ]))) t.add_resource(
ec2.SecurityGroupRule( IpProtocol="tcp", FromPort=ApplicationPort, ToPort=ApplicationPort, CidrIp="0.0.0.0/0", ), ], )) ud = Base64( Join( '\n', [ "#!/bin/bash", "sudo apt update", "sudo apt install nodejs", "wget http://bit.ly/2vESNuc -O /home/ubuntu/helloworld.js", "sudo wget https://raw.githubusercontent.com/Ohizzy21/EffectiveDevOpsTemplates/master/Helloworld.service -O /etc/systemd/system/helloworld.service", "sudo systemctl daemon-reload", "sudo systemctl enable helloworld", "sudo systemctl start helloworld" # "cat helloworld.js", # "wget http://bit.ly/2vVvT18 -O /home/ubuntu/helloworld.conf", # "sudo systemctl start helloworld" ])) t.add_resource( ec2.Instance( "instance", ImageId="ami-085925f297f89fce1", InstanceType="t2.micro", SecurityGroups=[Ref("SecurityGroup")], KeyName=Ref("KeyPair"), UserData=ud,
def create_answer(self, command_list, delimiter=""): return Base64(Join(delimiter, command_list)).to_dict()
def _add_ec2_auto_scaling(self): instance_profile = self._add_instance_profile() self.sg_alb = SecurityGroup( "SecurityGroupAlb", VpcId=Ref(self.vpc), GroupDescription=Sub("${AWS::StackName}-alb")) self.template.add_resource(self.sg_alb) self.sg_hosts = SecurityGroup( "SecurityGroupEc2Hosts", SecurityGroupIngress=[{ 'SourceSecurityGroupId': Ref(self.sg_alb), 'IpProtocol': -1 }], VpcId=Ref(self.vpc), GroupDescription=Sub("${AWS::StackName}-hosts")) self.template.add_resource(self.sg_hosts) sg_host_ingress = SecurityGroupIngress("SecurityEc2HostsIngress", SourceSecurityGroupId=Ref( self.sg_hosts), IpProtocol="-1", GroupId=Ref(self.sg_hosts), FromPort="-1", ToPort="-1") self.template.add_resource(sg_host_ingress) database_security_group = SecurityGroup( "SecurityGroupDatabases", SecurityGroupIngress=[{ 'SourceSecurityGroupId': Ref(self.sg_hosts), 'IpProtocol': -1 }], VpcId=Ref(self.vpc), GroupDescription=Sub("${AWS::StackName}-databases")) self.template.add_resource(database_security_group) user_data = Base64( Sub('\n'.join([ "#!/bin/bash", "yum update -y", "yum install -y aws-cfn-bootstrap", "/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource LaunchConfiguration", "/opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource AutoScalingGroup", "yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm", "systemctl enable amazon-ssm-agent", "systemctl start amazon-ssm-agent", "" ]))) lc_metadata = cloudformation.Init({ "config": cloudformation.InitConfig( files=cloudformation.InitFiles({ "/etc/cfn/cfn-hup.conf": cloudformation.InitFile( content=Sub('\n'.join([ '[main]', 'stack=${AWS::StackId}', 'region=${AWS::Region}', '' ])), mode='256', # TODO: Why 256 owner="root", group="root"), "/etc/cfn/hooks.d/cfn-auto-reloader.conf": cloudformation.InitFile(content=Sub('\n'.join([ '[cfn-auto-reloader-hook]', 'triggers=post.update', 'path=Resources.ContainerInstances.Metadata.AWS::CloudFormation::Init', 'action=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource LaunchConfiguration', '' ])), ) }), services={ "sysvinit": cloudformation.InitServices({ "cfn-hup": cloudformation.InitService( enabled=True, ensureRunning=True, files=[ '/etc/cfn/cfn-hup.conf', '/etc/cfn/hooks.d/cfn-auto-reloader.conf' ]) }) }, commands={ '01_add_instance_to_cluster': { 'command': Sub('echo "ECS_CLUSTER=${Cluster}\nECS_RESERVED_MEMORY=256" > /etc/ecs/ecs.config' ) } }) }) launch_configuration = LaunchConfiguration( 'LaunchConfiguration', UserData=user_data, IamInstanceProfile=Ref(instance_profile), SecurityGroups=[Ref(self.sg_hosts)], InstanceType=Ref('InstanceType'), ImageId=FindInMap("AWSRegionToAMI", Ref("AWS::Region"), "AMI"), Metadata=lc_metadata, KeyName=Ref(self.key_pair)) self.template.add_resource(launch_configuration) # , PauseTime='PT15M', WaitOnResourceSignals=True, MaxBatchSize=1, MinInstancesInService=1) up = AutoScalingRollingUpdate('AutoScalingRollingUpdate') # TODO: clean up subnets = list(self.private_subnets) self.auto_scaling_group = AutoScalingGroup( "AutoScalingGroup", UpdatePolicy=up, DesiredCapacity=self.desired_instances, Tags=[{ 'PropagateAtLaunch': True, 'Value': Sub('${AWS::StackName} - ECS Host'), 'Key': 'Name' }], MinSize=Ref('MinSize'), MaxSize=Ref('MaxSize'), VPCZoneIdentifier=[Ref(subnets.pop()), Ref(subnets.pop())], LaunchConfigurationName=Ref(launch_configuration), CreationPolicy=CreationPolicy(ResourceSignal=ResourceSignal( Timeout='PT15M'))) self.template.add_resource(self.auto_scaling_group) self.cluster_scaling_policy = ScalingPolicy( 'AutoScalingPolicy', AdjustmentType='ChangeInCapacity', AutoScalingGroupName=Ref(self.auto_scaling_group), Cooldown=300, PolicyType='SimpleScaling', ScalingAdjustment=1) self.template.add_resource(self.cluster_scaling_policy)
)) t.add_resource(InstanceProfile( 'EC2InstanceProfile', Roles=[Ref('EcsClusterRole')], )) t.add_resource(LaunchConfiguration( 'ContainerInstances', UserData=Base64(Join('', [ "#!/bin/bash -xe\n", "echo ECS_CLUSTER=", Ref('ECSCluster'), " >> /etc/ecs/ecs.config\n", "yum install -y aws-cfn-bootstrap\n", "/opt/aws/bin/cfn-signal -e $? ", " --stack ", Ref('AWS::StackName'), " --resource ECSAutoScalingGroup ", " --region ", Ref('AWS::Region'), "\n"])), ImageId='ami-045f1b3f87ed83659', KeyName=Ref("KeyPair"), SecurityGroups=[Ref("SecurityGroup")], IamInstanceProfile=Ref('EC2InstanceProfile'), InstanceType='t2.micro', AssociatePublicIpAddress='true', )) t.add_resource(AutoScalingGroup(
FromPort="22", ToPort="22", CidrIp=PublicIpCidr, ), ec2.SecurityGroupRule( IpProtocol="tcp", FromPort=ApplicationPort, ToPort=ApplicationPort, CidrIp="0.0.0.0/0", ), ], )) ud = Base64(Join('\n', [ "#!/bin/bash", "yum install --enablerepo=epel -y git", "pip install ansible", AnsiblePullCmd, "echo '*/10 * * * * {}' > /etc/cron.d/ansible- pull".format(AnsiblePullCmd) ])) t.add_resource(ec2.Instance( "instance", ImageId="ami-cfe4b2b0", InstanceType="t2.micro", SecurityGroups=[Ref("SecurityGroup")], KeyName=Ref("KeyPair"), UserData=ud, )) t.add_output(Output( "InstancePublicIp", Description="Public IP of our instance.",
)) ud = Base64(Join('', [ "#!/bin/bash\n", "yum -y update\n", "curl http://10.0.1.20:8081/artifactory/thirdparty/jdk-8u161-linux-x64.rpm -u admin:password --output jdk-8u161-linux-x64.rpm\n", "yum -y localinstall jdk-8u161-linux-x64.rpm\n", "/usr/sbin/alternatives --install /usr/bin/java java /usr/java/jdk1.8.0_11/bin/java 20000\n", "/usr/sbin/alternatives --set java /usr/java/jdk1.8.0_161/jre/bin/java\n", "curl --output /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat/jenkins.repo\n", "rpm --import https://pkg.jenkins.io/redhat/jenkins.io.key\n", "yum -y install jenkins python-simplejson\n", "service jenkins stop\n", "yum -y remove java-1.7.0-openjdk.x86_64\n", "curl https://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo --output /etc/yum.repos.d/epel-apache-maven.repo\n", "yum install -y apache-maven\n", "service jenkins start\n", "chkconfig jenkins on\n", "yum -y install ansible\n", "yum -y install git python-lxml\n", "pip install lxml\n", "chmod 0600 /var/lib/jenkins/.ssh/id_rsa*\n", "yum -y install docker\n", "service docker start\n", "chkconfig docker on\n", "usermod -a -G root jenkins\n", "curl --output /usr/local/bin/dcos https://downloads.dcos.io/binaries/cli/linux/x86-64/dcos-1.8/dcos\n", "chmod +x /usr/local/bin/dcos\n", "dcos config set core.dcos_url http://10.10.0.6\n" ])) t.add_resource(Role(
def create_instance(): return ec2.Instance( 'devserver', BlockDeviceMappings=[ ec2.BlockDeviceMapping( DeviceName='/dev/xvda', Ebs=ec2.EBSBlockDevice( VolumeSize=100, VolumeType='gp2', DeleteOnTermination=True, ), ), ], ImageId=Ref('amiId'), InstanceType='t2.medium', KeyName=Ref('keypair'), SecurityGroupIds=[Ref('secgrpDevServer')], SubnetId=Ref('subnetA'), Tags=_tags(), UserData=Base64( textwrap.dedent(r''' #!/bin/bash -ex exec > >(tee ~/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 echo BEGIN date '+%Y-%m-%d %H:%M:%S' # --- System Config hostnamectl set-hostname gwa-dev yum install -y git jq tree vim # --- UX cat <<-EOT > /etc/profile.d/ux.sh alias vi='vim' alias tree='tree -C' EOT cat <<-EOT >> /etc/vimrc set autoindent set modeline set tabstop=4 set listchars=tab:—— EOT # --- Docker yum install -y docker systemctl enable docker systemctl start docker usermod -aG docker ec2-user docker network create geowave-admin # --- Jenkins sudo wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo sudo rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key yum install -y jenkins java-1.8.0-openjdk usermod -aG docker jenkins systemctl enable jenkins systemctl start jenkins echo END date '+%Y-%m-%d %H:%M:%S' ''').lstrip()), )
def main(**params): try: # Metadata t = Template() t.set_version("2010-09-09") t.set_description("(SOCA) - Base template to deploy compute nodes.") allow_anonymous_data_collection = params["MetricCollectionAnonymous"] debug = False mip_usage = False instances_list = params["InstanceType"].split("+") asg_lt = asg_LaunchTemplate() ltd = LaunchTemplateData("NodeLaunchTemplateData") mip = MixedInstancesPolicy() stack_name = Ref("AWS::StackName") # Begin LaunchTemplateData UserData = '''#!/bin/bash -xe export PATH=$PATH:/usr/local/bin if [[ "''' + params['BaseOS'] + '''" == "centos7" ]] || [[ "''' + params['BaseOS'] + '''" == "rhel7" ]]; then EASY_INSTALL=$(which easy_install-2.7) $EASY_INSTALL pip PIP=$(which pip2.7) $PIP install awscli yum install -y nfs-utils # enforce install of nfs-utils else # Upgrade awscli on ALI (do not use yum) EASY_INSTALL=$(which easy_install-2.7) $EASY_INSTALL pip PIP=$(which pip) $PIP install awscli --upgrade fi if [[ "''' + params['BaseOS'] + '''" == "amazonlinux2" ]]; then /usr/sbin/update-motd --disable fi GET_INSTANCE_TYPE=$(curl http://169.254.169.254/latest/meta-data/instance-type) echo export "SOCA_CONFIGURATION="''' + str(params['ClusterId']) + '''"" >> /etc/environment echo export "SOCA_BASE_OS="''' + str(params['BaseOS']) + '''"" >> /etc/environment echo export "SOCA_JOB_QUEUE="''' + str(params['JobQueue']) + '''"" >> /etc/environment echo export "SOCA_JOB_OWNER="''' + str(params['JobOwner']) + '''"" >> /etc/environment echo export "SOCA_JOB_NAME="''' + str(params['JobName']) + '''"" >> /etc/environment echo export "SOCA_JOB_PROJECT="''' + str(params['JobProject']) + '''"" >> /etc/environment echo export "SOCA_VERSION="''' + str(params['Version']) + '''"" >> /etc/environment echo export "SOCA_JOB_EFA="''' + str(params['Efa']).lower() + '''"" >> /etc/environment echo export "SOCA_JOB_ID="''' + str(params['JobId']) + '''"" >> /etc/environment echo export "SOCA_SCRATCH_SIZE=''' + str(params['ScratchSize']) + '''" >> /etc/environment echo export "SOCA_INSTALL_BUCKET="''' + str(params['S3Bucket']) + '''"" >> /etc/environment echo export "SOCA_INSTALL_BUCKET_FOLDER="''' + str(params['S3InstallFolder']) + '''"" >> /etc/environment echo export "SOCA_FSX_LUSTRE_BUCKET="''' + str(params['FSxLustreConfiguration']['fsx_lustre']).lower() + '''"" >> /etc/environment echo export "SOCA_FSX_LUSTRE_DNS="''' + str(params['FSxLustreConfiguration']['existing_fsx']).lower() + '''"" >> /etc/environment echo export "SOCA_INSTANCE_TYPE=$GET_INSTANCE_TYPE" >> /etc/environment echo export "SOCA_INSTANCE_HYPERTHREADING="''' + str(params['ThreadsPerCore']).lower() + '''"" >> /etc/environment echo export "SOCA_HOST_SYSTEM_LOG="/apps/soca/''' + str(params['ClusterId']) + '''/cluster_node_bootstrap/logs/''' + str(params['JobId']) + '''/$(hostname -s)"" >> /etc/environment echo export "AWS_STACK_ID=${AWS::StackName}" >> /etc/environment echo export "AWS_DEFAULT_REGION=${AWS::Region}" >> /etc/environment source /etc/environment AWS=$(which aws) # Give yum permission to the user on this specific machine echo "''' + params['JobOwner'] + ''' ALL=(ALL) /bin/yum" >> /etc/sudoers mkdir -p /apps mkdir -p /data # Mount EFS echo "''' + params['EFSDataDns'] + ''':/ /data nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0" >> /etc/fstab echo "''' + params['EFSAppsDns'] + ''':/ /apps nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0" >> /etc/fstab mount -a # Configure NTP yum remove -y ntp yum install -y chrony mv /etc/chrony.conf /etc/chrony.conf.original echo -e """ # use the local instance NTP service, if available server 169.254.169.123 prefer iburst minpoll 4 maxpoll 4 # Use public servers from the pool.ntp.org project. # Please consider joining the pool (http://www.pool.ntp.org/join.html). # !!! [BEGIN] SOCA REQUIREMENT # You will need to open UDP egress traffic on your security group if you want to enable public pool #pool 2.amazon.pool.ntp.org iburst # !!! [END] SOCA REQUIREMENT # Record the rate at which the system clock gains/losses time. driftfile /var/lib/chrony/drift # Allow the system clock to be stepped in the first three updates # if its offset is larger than 1 second. makestep 1.0 3 # Specify file containing keys for NTP authentication. keyfile /etc/chrony.keys # Specify directory for log files. logdir /var/log/chrony # save data between restarts for fast re-load dumponexit dumpdir /var/run/chrony """ > /etc/chrony.conf systemctl enable chronyd # Prepare Log folder mkdir -p $SOCA_HOST_SYSTEM_LOG echo "@reboot /bin/bash /apps/soca/$SOCA_CONFIGURATION/cluster_node_bootstrap/ComputeNodePostReboot.sh >> $SOCA_HOST_SYSTEM_LOG/ComputeNodePostInstall.log 2>&1" | crontab - $AWS s3 cp s3://$SOCA_INSTALL_BUCKET/$SOCA_INSTALL_BUCKET_FOLDER/scripts/config.cfg /root/ /bin/bash /apps/soca/$SOCA_CONFIGURATION/cluster_node_bootstrap/ComputeNode.sh ''' + params['SchedulerHostname'] + ''' >> $SOCA_HOST_SYSTEM_LOG/ComputeNode.sh.log 2>&1''' ltd.EbsOptimized = True for instance in instances_list: if "t2." in instance: ltd.EbsOptimized = False else: # metal + t2 does not support CpuOptions if "metal" not in instance: ltd.CpuOptions = CpuOptions( CoreCount=int(params["CoreCount"]), ThreadsPerCore=1 if params["ThreadsPerCore"] is False else 2) ltd.IamInstanceProfile = IamInstanceProfile(Arn=params["ComputeNodeInstanceProfileArn"]) ltd.KeyName = params["SSHKeyPair"] ltd.ImageId = params["ImageId"] if params["SpotPrice"] is not False and params["SpotAllocationCount"] is False: ltd.InstanceMarketOptions = InstanceMarketOptions( MarketType="spot", SpotOptions=SpotOptions( MaxPrice=Ref("AWS::NoValue") if params["SpotPrice"] == "auto" else str(params["SpotPrice"]) # auto -> cap at OD price ) ) ltd.InstanceType = instances_list[0] ltd.NetworkInterfaces = [NetworkInterfaces( InterfaceType="efa" if params["Efa"] is not False else Ref("AWS::NoValue"), DeleteOnTermination=True, DeviceIndex=0, Groups=[params["SecurityGroupId"]] )] ltd.UserData = Base64(Sub(UserData)) ltd.BlockDeviceMappings = [ BlockDeviceMapping( DeviceName="/dev/xvda" if params["BaseOS"] == "amazonlinux2" else "/dev/sda1", Ebs=EBSBlockDevice( VolumeSize=params["RootSize"], VolumeType="gp2", DeleteOnTermination="false" if params["KeepEbs"] is True else "true", Encrypted=True)) ] if int(params["ScratchSize"]) > 0: ltd.BlockDeviceMappings.append( BlockDeviceMapping( DeviceName="/dev/xvdbx", Ebs=EBSBlockDevice( VolumeSize=params["ScratchSize"], VolumeType="io1" if int(params["VolumeTypeIops"]) > 0 else "gp2", Iops=params["VolumeTypeIops"] if int(params["VolumeTypeIops"]) > 0 else Ref("AWS::NoValue"), DeleteOnTermination="false" if params["KeepEbs"] is True else "true", Encrypted=True)) ) # End LaunchTemplateData # Begin Launch Template Resource lt = LaunchTemplate("NodeLaunchTemplate") lt.LaunchTemplateName = params["ClusterId"] + "-" + str(params["JobId"]) lt.LaunchTemplateData = ltd t.add_resource(lt) # End Launch Template Resource asg_lt.LaunchTemplateSpecification = LaunchTemplateSpecification( LaunchTemplateId=Ref(lt), Version=GetAtt(lt, "LatestVersionNumber") ) asg_lt.Overrides = [] for instance in instances_list: asg_lt.Overrides.append(LaunchTemplateOverrides( InstanceType=instance)) # Begin InstancesDistribution if params["SpotPrice"] is not False and \ params["SpotAllocationCount"] is not False and \ (params["DesiredCapacity"] - params["SpotAllocationCount"]) > 0: mip_usage = True idistribution = InstancesDistribution() idistribution.OnDemandAllocationStrategy = "prioritized" # only supported value idistribution.OnDemandBaseCapacity = params["DesiredCapacity"] - params["SpotAllocationCount"] idistribution.OnDemandPercentageAboveBaseCapacity = "0" # force the other instances to be SPOT idistribution.SpotMaxPrice = Ref("AWS::NoValue") if params["SpotPrice"] == "auto" else str( params["SpotPrice"]) idistribution.SpotAllocationStrategy = params['SpotAllocationStrategy'] mip.InstancesDistribution = idistribution # End MixedPolicyInstance # Begin FSx for Lustre if params["FSxLustreConfiguration"]["fsx_lustre"] is not False: if params["FSxLustreConfiguration"]["existing_fsx"] is False: fsx_lustre = FileSystem("FSxForLustre") fsx_lustre.FileSystemType = "LUSTRE" fsx_lustre.StorageCapacity = params["FSxLustreConfiguration"]["capacity"] fsx_lustre.SecurityGroupIds = [params["SecurityGroupId"]] fsx_lustre.SubnetIds = params["SubnetId"] if params["FSxLustreConfiguration"]["s3_backend"] is not False: fsx_lustre_configuration = LustreConfiguration() fsx_lustre_configuration.ImportPath = params["FSxLustreConfiguration"]["import_path"] if params["FSxLustreConfiguration"]["import_path"] is not False else params["FSxLustreConfiguration"]["s3_backend"] fsx_lustre_configuration.ExportPath = params["FSxLustreConfiguration"]["import_path"] if params["FSxLustreConfiguration"]["import_path"] is not False else params["FSxLustreConfiguration"]["s3_backend"] + "/" + params["ClusterId"] + "-fsxoutput/job-" + params["JobId"] + "/" fsx_lustre.LustreConfiguration = fsx_lustre_configuration fsx_lustre.Tags = base_Tags( # False disable PropagateAtLaunch Name=str(params["ClusterId"] + "-compute-job-" + params["JobId"]), _soca_JobId=str(params["JobId"]), _soca_JobName=str(params["JobName"]), _soca_JobQueue=str(params["JobQueue"]), _soca_StackId=stack_name, _soca_JobOwner=str(params["JobOwner"]), _soca_JobProject=str(params["JobProject"]), _soca_KeepForever=str(params["KeepForever"]).lower(), _soca_FSx="true", _soca_ClusterId=str(params["ClusterId"]), ) t.add_resource(fsx_lustre) # End FSx For Lustre # Begin AutoScalingGroup Resource asg = AutoScalingGroup("AutoScalingComputeGroup") asg.DependsOn = "NodeLaunchTemplate" if mip_usage is True or instances_list.__len__() > 1: mip.LaunchTemplate = asg_lt asg.MixedInstancesPolicy = mip else: asg.LaunchTemplate = LaunchTemplateSpecification( LaunchTemplateId=Ref(lt), Version=GetAtt(lt, "LatestVersionNumber")) asg.MinSize = int(params["DesiredCapacity"]) asg.MaxSize = int(params["DesiredCapacity"]) asg.VPCZoneIdentifier = params["SubnetId"] if params["PlacementGroup"] is True: pg = PlacementGroup("ComputeNodePlacementGroup") pg.Strategy = "cluster" t.add_resource(pg) asg.PlacementGroup = Ref(pg) asg.Tags = Tags( Name=str(params["ClusterId"]) + "-compute-job-" + str(params["JobId"]), _soca_JobId=str(params["JobId"]), _soca_JobName=str(params["JobName"]), _soca_JobQueue=str(params["JobQueue"]), _soca_StackId=stack_name, _soca_JobOwner=str(params["JobOwner"]), _soca_JobProject=str(params["JobProject"]), _soca_KeepForever=str(params["KeepForever"]).lower(), _soca_ClusterId=str(params["ClusterId"]), _soca_NodeType="soca-compute-node") t.add_resource(asg) # End AutoScalingGroup Resource # Begin Custom Resource # Change Mapping to No if you want to disable this if allow_anonymous_data_collection is True: metrics = CustomResourceSendAnonymousMetrics("SendAnonymousData") metrics.ServiceToken = params["SolutionMetricLambda"] metrics.DesiredCapacity = str(params["DesiredCapacity"]) metrics.InstanceType = str(params["InstanceType"]) metrics.Efa = str(params["Efa"]) metrics.ScratchSize = str(params["ScratchSize"]) metrics.RootSize = str(params["RootSize"]) metrics.SpotPrice = str(params["SpotPrice"]) metrics.BaseOS = str(params["BaseOS"]) metrics.StackUUID = str(params["StackUUID"]) metrics.KeepForever = str(params["KeepForever"]) metrics.FsxLustre = str(params["FSxLustreConfiguration"]) t.add_resource(metrics) # End Custom Resource if debug is True: print(t.to_json()) # Tags must use "soca:<Key>" syntax template_output = t.to_yaml().replace("_soca_", "soca:") return {'success': True, 'output': template_output} except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] return {'success': False, 'output': 'cloudformation_builder.py: ' + ( str(e) + ': error :' + str(exc_type) + ' ' + str(fname) + ' ' + str(exc_tb.tb_lineno))}
def create_template(self): """Create template (main function called by Stacker).""" template = self.template variables = self.get_variables() template.add_version('2010-09-09') template.add_description('Kubernetes workers via EKS - V1.0.0 ' '- compatible with amazon-eks-node-v23+') # Metadata template.add_metadata({ 'AWS::CloudFormation::Interface': { 'ParameterGroups': [ {'Label': {'default': 'EKS Cluster'}, 'Parameters': [variables[i].name for i in ['ClusterName', 'ClusterControlPlaneSecurityGroup']]}, {'Label': {'default': 'Worker Node Configuration'}, 'Parameters': [variables[i].name for i in ['NodeGroupName', 'NodeAutoScalingGroupMinSize', 'NodeAutoScalingGroupMaxSize', 'UseDesiredInstanceCount', 'NodeInstanceType', 'NodeInstanceProfile', 'NodeImageId', 'NodeVolumeSize', 'KeyName', 'UseSpotInstances', 'SpotBidPrice', 'BootstrapArguments']]}, {'Label': {'default': 'Worker Network Configuration'}, 'Parameters': [variables[i].name for i in ['VpcId', 'Subnets']]} ] } }) # Conditions template.add_condition( 'SetSpotPrice', Equals(variables['UseSpotInstances'].ref, 'yes') ) template.add_condition( 'DesiredInstanceCountSpecified', Equals(variables['UseDesiredInstanceCount'].ref, 'true') ) template.add_condition( 'KeyNameSpecified', Not(Equals(variables['KeyName'].ref, '')) ) # Resources nodesecuritygroup = template.add_resource( ec2.SecurityGroup( 'NodeSecurityGroup', GroupDescription='Security group for all nodes in the cluster', Tags=[ {'Key': Sub('kubernetes.io/cluster/${ClusterName}'), 'Value': 'owned'}, ], VpcId=variables['VpcId'].ref ) ) template.add_output( Output( 'NodeSecurityGroup', Description='Security group for all nodes in the cluster', Value=nodesecuritygroup.ref() ) ) template.add_resource( ec2.SecurityGroupIngress( 'NodeSecurityGroupIngress', Description='Allow node to communicate with each other', GroupId=nodesecuritygroup.ref(), SourceSecurityGroupId=nodesecuritygroup.ref(), IpProtocol='-1', FromPort=0, ToPort=65535 ) ) template.add_resource( ec2.SecurityGroupIngress( 'NodeSecurityGroupFromControlPlaneIngress', Description='Allow worker Kubelets and pods to receive ' 'communication from the cluster control plane', GroupId=nodesecuritygroup.ref(), SourceSecurityGroupId=variables['ClusterControlPlaneSecurityGroup'].ref, # noqa IpProtocol='tcp', FromPort=1025, ToPort=65535 ) ) template.add_resource( ec2.SecurityGroupEgress( 'ControlPlaneEgressToNodeSecurityGroup', Description='Allow the cluster control plane to communicate ' 'with worker Kubelet and pods', GroupId=variables['ClusterControlPlaneSecurityGroup'].ref, DestinationSecurityGroupId=nodesecuritygroup.ref(), IpProtocol='tcp', FromPort=1025, ToPort=65535 ) ) template.add_resource( ec2.SecurityGroupIngress( 'NodeSecurityGroupFromControlPlaneOn443Ingress', Description='Allow pods running extension API servers on port ' '443 to receive communication from cluster ' 'control plane', GroupId=nodesecuritygroup.ref(), SourceSecurityGroupId=variables['ClusterControlPlaneSecurityGroup'].ref, # noqa IpProtocol='tcp', FromPort=443, ToPort=443 ) ) template.add_resource( ec2.SecurityGroupEgress( 'ControlPlaneEgressToNodeSecurityGroupOn443', Description='Allow the cluster control plane to communicate ' 'with pods running extension API servers on port ' '443', GroupId=variables['ClusterControlPlaneSecurityGroup'].ref, DestinationSecurityGroupId=nodesecuritygroup.ref(), IpProtocol='tcp', FromPort=443, ToPort=443 ) ) template.add_resource( ec2.SecurityGroupIngress( 'ClusterControlPlaneSecurityGroupIngress', Description='Allow pods to communicate with the cluster API ' 'Server', GroupId=variables['ClusterControlPlaneSecurityGroup'].ref, SourceSecurityGroupId=nodesecuritygroup.ref(), IpProtocol='tcp', FromPort=443, ToPort=443 ) ) nodelaunchconfig = template.add_resource( autoscaling.LaunchConfiguration( 'NodeLaunchConfig', AssociatePublicIpAddress=True, IamInstanceProfile=variables['NodeInstanceProfile'].ref, ImageId=variables['NodeImageId'].ref, InstanceType=variables['NodeInstanceType'].ref, KeyName=If( 'KeyNameSpecified', variables['KeyName'].ref, NoValue ), SecurityGroups=[nodesecuritygroup.ref()], SpotPrice=If('SetSpotPrice', variables['SpotBidPrice'].ref, NoValue), BlockDeviceMappings=[autoscaling.BlockDeviceMapping( DeviceName='/dev/xvda', Ebs=autoscaling.EBSBlockDevice( VolumeSize=variables['NodeVolumeSize'].ref, VolumeType='gp2', DeleteOnTermination=True ) )], UserData=Base64( Sub('\n'.join([ '#!/bin/bash', 'set -o xtrace', '/etc/eks/bootstrap.sh ${ClusterName} ${BootstrapArguments}', # noqa '/opt/aws/bin/cfn-signal --exit-code $? \\', '--stack ${AWS::StackName} \\', '--resource NodeGroup \\', '--region ${AWS::Region}' ])) ) ) ) template.add_resource( autoscaling.AutoScalingGroup( 'NodeGroup', DesiredCapacity=If( 'DesiredInstanceCountSpecified', variables['NodeAutoScalingGroupMaxSize'].ref, NoValue ), LaunchConfigurationName=nodelaunchconfig.ref(), MinSize=variables['NodeAutoScalingGroupMinSize'].ref, MaxSize=variables['NodeAutoScalingGroupMaxSize'].ref, VPCZoneIdentifier=variables['Subnets'].ref, Tags=[ autoscaling.Tag( 'Name', Sub('${ClusterName}-${NodeGroupName}-Node'), True), autoscaling.Tag( Sub('kubernetes.io/cluster/${ClusterName}'), 'owned', True) ], UpdatePolicy=UpdatePolicy( AutoScalingRollingUpdate=AutoScalingRollingUpdate( MinInstancesInService='1', MaxBatchSize='1' ) ) ) )
FromPort="22", ToPort="22", CidrIp=PublicCidrIP, ), ec2.SecurityGroupRule( IpProtocol="tcp", FromPort=ApplicationPort, ToPort=ApplicationPort, CidrIp="0.0.0.0/0", ), ], )) ud = Base64( Join('\n', [ "#!/bin/bash", "sudo apt update && sudo apt install -y git ansible", AnsiblePullCmd, "echo '*/10 * * * * {}' > /etc/cron.d/ansiblepull". format(AnsiblePullCmd) ])) t.add_resource( Role("Role", AssumeRolePolicyDocument=Policy(Statement=[ Statement(Effect=Allow, Action=[AssumeRole], Principal=Principal("Service", ["ec2.amazonaws.com"])) ]))) t.add_resource( InstanceProfile("InstanceProfile", Path="/", Roles=[Ref("Role")])) t.add_resource(
def add_resources(self): self.CassandraPublicLBSG = self.template.add_resource( ec2.SecurityGroup( "CassandraPublicLBSG", GroupDescription= "Loadbalancer Security Group For Cassandra Public LB", VpcId=Ref(self.VpcId), SecurityGroupIngress=[ ec2.SecurityGroupRule( IpProtocol="tcp", FromPort=22, ToPort=22, CidrIp=Ref(self.AdminCidrBlock), ), ], Tags=self.base_tags + Tags(Name=self.environment_parameters["ClientEnvironmentKey"] + "-CassandraPublicLBSG"), )) self.CassandraSG = self.template.add_resource( ec2.SecurityGroup( "CassandraSG", GroupDescription= "Allow communication between Cassandra Seed and Non-Seed Nodes", VpcId=Ref(self.VpcId), SecurityGroupIngress=[ ec2.SecurityGroupRule( IpProtocol="tcp", FromPort=22, ToPort=22, SourceSecurityGroupId=Ref(self.CassandraPublicLBSG), ), ], Tags=self.base_tags + Tags(Name=self.environment_parameters["ClientEnvironmentKey"] + "-CassandraEc2SG"), )) self.CassandraSGInterNodeCommunicationIngress = self.template.add_resource( ec2.SecurityGroupIngress( "CassandraSGInterNodeCommunicationIngress", DependsOn=self.CassandraSG, GroupId=Ref(self.CassandraSG), IpProtocol="tcp", FromPort=7000, ToPort=7001, SourceSecurityGroupId=Ref(self.CassandraSG), )) self.CassandraSeedNetworkInterface = self.template.add_resource( ec2.NetworkInterface( "Eth0", Description="eth0", GroupSet=[Ref(self.CassandraSG)], SubnetId=Ref(self.RESTPrivSubnet1), PrivateIpAddress="10.0.1.132", Tags=self.base_tags + Tags(Name=self.environment_parameters["ClientEnvironmentKey"] + "-CassandraSeedNetworkInterface"), )) self.CassandraSeed1 = self.template.add_resource( ec2.Instance( "CassandraSeed1", ImageId=Ref(self.CassandraImageId), KeyName=Ref(self.CassandraServerKeyName), InstanceType=Ref(self.CassandraServerInstanceType), IamInstanceProfile=Ref(self.CassandraServerIAMInstanceProfile), NetworkInterfaces=[ ec2.NetworkInterfaceProperty( NetworkInterfaceId=Ref( self.CassandraSeedNetworkInterface), DeviceIndex="0", ), ], UserData=Base64( Join('', [ "#!/bin/bash -x\n", "export NODE_IP=`hostname -I`\n", "export SEED_LIST=\"10.0.1.132\"\n", "export CASSANDRA_YML=\"/etc/cassandra/conf/cassandra.yaml\"\n", "export CLUSTER_NAME=\"devops_cluster\"\n", "export SNITCH_TYPE=\"Ec2Snitch\"\n", "sed -i \"/cluster_name:/c\\cluster_name: \\'${CLUSTER_NAME}\\'\" ${CASSANDRA_YML}\n", "sed -i \"/- seeds:/c\\ - seeds: \\\"${SEED_LIST}\\\"\" ${CASSANDRA_YML}\n", "sed -i \"/listen_address:/c\\listen_address: ${NODE_IP}\" ${CASSANDRA_YML}\n", "sed -i \"/rpc_address:/c\\rpc_address: ${NODE_IP}\" ${CASSANDRA_YML}\n", "sed -i \"/endpoint_snitch:/c\\endpoint_snitch: ${SNITCH_TYPE}\" ${CASSANDRA_YML}\n", "sed -i \"/authenticator: AllowAllAuthenticator/c\\authenticator: PasswordAuthenticator\" ${CASSANDRA_YML}\n" "echo 'auto_bootstrap: false' >> ${CASSANDRA_YML}\n", "service cassandra start\n" ])), Tags=self.base_tags + Tags(Name=self.environment_parameters["ClientEnvironmentKey"] + "-CassandraSeed-1-Ec2"), )) self.CassandraPublicLoadBalancer = self.template.add_resource( elb.LoadBalancer( "CassandraPublicLoadBalancer", LoadBalancerName=self. environment_parameters["ClientEnvironmentKey"] + "-CassandraNonSeedPubLB", Scheme="internet-facing", Listeners=[ elb.Listener( LoadBalancerPort="22", InstancePort="22", Protocol="TCP", InstanceProtocol="TCP", ) ], Instances=[], SecurityGroups=[Ref(self.CassandraPublicLBSG)], Subnets=[Ref(self.RESTPubSubnet1)], Tags=self.base_tags + Tags(Name=self.environment_parameters["ClientEnvironmentKey"] + "-CassandraNonSeedPubLB"), )) self.CassandraNonSeedLaunchConfiguration = self.template.add_resource( LaunchConfiguration( "CassandraNonSeedLaunchConfiguration", ImageId=Ref(self.CassandraImageId), InstanceType=Ref(self.CassandraServerInstanceType), IamInstanceProfile=Ref(self.CassandraServerIAMInstanceProfile), KeyName=Ref(self.CassandraServerKeyName), SecurityGroups=[Ref(self.CassandraSG)], UserData=Base64( Join('', [ "#!/bin/bash -x\n", "export NODE_IP=`hostname -I`\n", "export SEED_LIST=\"10.0.1.132\"\n", "export CASSANDRA_YML=\"/etc/cassandra/conf/cassandra.yaml\"\n", "export CLUSTER_NAME=\"devoops_cluster\"\n", "export SNITCH_TYPE=\"Ec2Snitch\"\n", "sed -i \"/cluster_name:/c\\cluster_name: \\'${CLUSTER_NAME}\\'\" ${CASSANDRA_YML}\n", "sed -i \"/- seeds:/c\\ - seeds: \\\"${SEED_LIST}\\\"\" ${CASSANDRA_YML}\n", "sed -i \"/listen_address:/c\\listen_address: ${NODE_IP}\" ${CASSANDRA_YML}\n", "sed -i \"/rpc_address:/c\\rpc_address: ${NODE_IP}\" ${CASSANDRA_YML}\n", "sed -i \"/endpoint_snitch:/c\\endpoint_snitch: ${SNITCH_TYPE}\" ${CASSANDRA_YML}\n", "sed -i \"/authenticator: AllowAllAuthenticator/c\\authenticator: PasswordAuthenticator\" ${CASSANDRA_YML}\n", "echo 'auto_bootstrap: false' >> ${CASSANDRA_YML}\n", "service cassandra start\n" ])), )) self.CassandraNonSeedAutoScalingGroup = self.template.add_resource( AutoScalingGroup( "CassandraNonSeedAutoscalingGroup", AutoScalingGroupName=self. environment_parameters["ClientEnvironmentKey"] + "-CassandraNonSeedAutoScalingGroup", LaunchConfigurationName=Ref( self.CassandraNonSeedLaunchConfiguration), LoadBalancerNames=[Ref(self.CassandraPublicLoadBalancer)], MaxSize="1", MinSize="1", DesiredCapacity="1", VPCZoneIdentifier=[Ref(self.RESTPrivSubnet1)], Tags=[ AutoScalingTag( "Name", self.environment_parameters["ClientEnvironmentKey"] + "-CassandraNonSeedEc2", True), AutoScalingTag( "Environment", self.environment_parameters["EnvironmentName"], True), AutoScalingTag( "ResourceOwner", self.environment_parameters["ResourceOwner"], True), AutoScalingTag( "ClientCode", self.environment_parameters["ClientEnvironmentKey"], True), ], ))
remote_ip_prefix="0.0.0.0/0", ), ] ) ) openstack_instance = template.add_resource(nova.Server( "OpenStackInstance", image="MyImage", flavor="t1.micro", key_name=Ref(keyname_param), networks=[neutron.Port( "OpenStackPort", fixed_ips=[neutron.FixedIP( ip_address="192.168.1.20" )], network_id="3e47c369-7007-472e-9e96-7dadb51e3e99", security_groups=[Ref(security_group)], )], user_data=Base64(Join('\n', [ "#!/bin/bash", "echo \"Upgrade started at $(date)\"", "apt-get update", "apt-get -y upgrade", "echo \"Upgrade complete at $(date)\"", ])) )) print(template.to_json())
), ec2.SecurityGroupRule( IpProtocol="tcp", FromPort=ApplicationPort, ToPort=ApplicationPort, CidrIp="0.0.0.0/0", ), ], )) ud = Base64( Join('\n', [ "#!/bin/bash", "sudo yum -y update", "sudo yum install ruby", "sudo yum -y install java-1.8.0", "sudo yum -y remove java-1.7.0-openjdk", "curl --silent --location https://rpm.nodesource.com/setup_10.x | sudo bash -", "sudo yum -y install nodejs", "yum install --enablerepo=epel -y git", "pip install ansible", AnsiblePullCmd, "echo '*/5 * * * * root {}' > /etc/cron.d/ansible-pull".format( AnsiblePullCmd) ])) t.add_resource( Role("Role", AssumeRolePolicyDocument=Policy(Statement=[ Statement(Effect=Allow, Action=[AssumeRole], Principal=Principal("Service", ["ec2.amazonaws.com"])) ]))) t.add_resource(
def buildInstance(t, args): t.add_resource( ec2.SecurityGroup('WebserverIngressSG', GroupDescription='Global Webserver Access', VpcId=Ref('VPC'), Tags=Tags(Name='Global Webserver Access'))) t.add_resource( ec2.SecurityGroupIngress('WebserverIngressSG80', GroupId=Ref('WebserverIngressSG'), IpProtocol='tcp', CidrIp='0.0.0.0/0', FromPort='80', ToPort='80')) t.add_resource( ec2.SecurityGroupIngress('WebserverIngress443', GroupId=Ref('WebserverIngressSG'), IpProtocol='tcp', CidrIp='0.0.0.0/0', FromPort='443', ToPort='443')) t.add_resource( ec2.SecurityGroup('SysAdminAccessSG', GroupDescription='System Administrator Access', VpcId=Ref('VPC'), Tags=Tags(Name='System Administrator Access'))) if (args.dev): t.add_resource( ec2.SecurityGroupIngress('DevSysadminIngress22', GroupId=Ref('SysAdminAccessSG'), IpProtocol='tcp', CidrIp='0.0.0.0/0', FromPort='22', ToPort='22')) rolePolicyStatements = [{ "Sid": "Stmt1500699052003", "Effect": "Allow", "Action": ["s3:ListBucket"], "Resource": [Join("", ["arn:aws:s3:::", Ref('S3Bucket')])] }, { "Sid": "Stmt1500699052000", "Effect": "Allow", "Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject"], "Resource": [Join("", ["arn:aws:s3:::", Ref('S3Bucket'), '/Backup/*'])] }, { "Sid": "Stmt1500612724002", "Effect": "Allow", "Action": ["kms:Encrypt", "kms:Decrypt", "kms:GenerateDataKey*"], "Resource": [OpenEMRKeyARN] }] if (args.recovery): rolePolicyStatements.extend([ { "Sid": "Stmt1500699052004", "Effect": "Allow", "Action": ["s3:ListBucket"], "Resource": [Join( "", ["arn:aws:s3:::", Ref('RecoveryS3Bucket')])] }, { "Sid": "Stmt1500699052005", "Effect": "Allow", "Action": [ "s3:GetObject", ], "Resource": [ Join("", [ "arn:aws:s3:::", Ref('RecoveryS3Bucket'), '/Backup/*' ]) ] }, ]) t.add_resource( iam.ManagedPolicy('WebserverPolicy', Description='Policy for webserver instance', PolicyDocument={ "Version": "2012-10-17", "Statement": rolePolicyStatements })) t.add_resource( iam.Role('WebserverRole', AssumeRolePolicyDocument={ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "Service": ["ec2.amazonaws.com"] }, "Action": ["sts:AssumeRole"] }] }, Path='/', ManagedPolicyArns=[Ref('WebserverPolicy')])) t.add_resource( iam.InstanceProfile('WebserverInstanceProfile', Path='/', Roles=[Ref('WebserverRole')])) t.add_resource( ec2.Volume('DockerVolume', DeletionPolicy='Delete' if args.dev else 'Snapshot', Size=Ref('PracticeStorage'), AvailabilityZone=Select("0", GetAZs("")), VolumeType='gp2', Encrypted=True, KmsKeyId=OpenEMRKeyID, Tags=Tags(Name="OpenEMR Practice"))) bootstrapScript = [ "#!/bin/bash -x\n", "exec > /var/log/openemr-cfn-bootstrap 2>&1\n", "cfn-init -v ", " --stack ", ref_stack_name, " --resource WebserverInstance ", " --configsets Setup ", " --region ", ref_region, "\n", "cfn-signal -e $? ", " --stack ", ref_stack_name, " --resource WebserverInstance ", " --region ", ref_region, "\n" ] setupScript = [ "#!/bin/bash -xe\n", "exec > /tmp/cloud-setup.log 2>&1\n", "/root/openemr-devops/packages/standard/ami/ami-configure.sh\n" ] stackPassthroughFile = [ "S3=", Ref('S3Bucket'), "\n", "KMS=", OpenEMRKeyID, "\n" ] if (args.recovery): stackPassthroughFile.extend([ "RECOVERYS3=", Ref('RecoveryS3Bucket'), "\n", "RECOVERY_NEWRDS=", GetAtt('RDSInstance', 'Endpoint.Address'), "\n", ]) if (args.recovery): dockerComposeFile = [ "version: '3.1'\n", "services:\n", " openemr:\n", " restart: always\n", " image: openemr/openemr", docker_version, "\n", " ports:\n", " - 80:80\n", " - 443:443\n", " volumes:\n", " - logvolume01:/var/log\n", " - sitevolume:/var/www/localhost/htdocs/openemr/sites\n", " environment:\n", " MANUAL_SETUP: 1\n", "volumes:\n", " logvolume01: {}\n", " sitevolume: {}\n" ] else: dockerComposeFile = [ "version: '3.1'\n", "services:\n", " openemr:\n", " restart: always\n", " image: openemr/openemr", docker_version, "\n", " ports:\n", " - 80:80\n", " - 443:443\n", " volumes:\n", " - logvolume01:/var/log\n", " - sitevolume:/var/www/localhost/htdocs/openemr/sites\n", " environment:\n", " MYSQL_HOST: '", GetAtt('RDSInstance', 'Endpoint.Address'), "'\n", " MYSQL_ROOT_USER: openemr\n", " MYSQL_ROOT_PASS: '******'RDSPassword'), "'\n", " MYSQL_USER: openemr\n", " MYSQL_PASS: '******'RDSPassword'), "'\n", " OE_USER: admin\n", " OE_PASS: '******'AdminPassword'), "'\n", "volumes:\n", " logvolume01: {}\n", " sitevolume: {}\n" ] bootstrapInstall = cloudformation.InitConfig( files={ "/root/cloud-setup.sh": { "content": Join("", setupScript), "mode": "000500", "owner": "root", "group": "root" }, "/root/cloud-variables": { "content": Join("", stackPassthroughFile), "mode": "000500", "owner": "root", "group": "root" }, "/root/openemr-devops/packages/standard/docker-compose.yaml": { "content": Join("", dockerComposeFile), "mode": "000500", "owner": "root", "group": "root" } }, commands={"01_setup": { "command": "/root/cloud-setup.sh" }}) bootstrapMetadata = cloudformation.Metadata( cloudformation.Init(cloudformation.InitConfigSets(Setup=['Install']), Install=bootstrapInstall)) t.add_resource( ec2.Instance('WebserverInstance', Metadata=bootstrapMetadata, ImageId=FindInMap('RegionData', ref_region, 'OpenEMRMktPlaceAMI'), InstanceType=Ref('WebserverInstanceSize'), NetworkInterfaces=[ ec2.NetworkInterfaceProperty( AssociatePublicIpAddress=True, DeviceIndex="0", GroupSet=[ Ref('ApplicationSecurityGroup'), Ref('WebserverIngressSG'), Ref('SysAdminAccessSG') ], SubnetId=Ref('PublicSubnet1')) ], KeyName=Ref('EC2KeyPair'), IamInstanceProfile=Ref('WebserverInstanceProfile'), Volumes=[{ "Device": "/dev/sdd", "VolumeId": Ref('DockerVolume') }], Tags=Tags(Name='OpenEMR Cloud Standard'), InstanceInitiatedShutdownBehavior='stop', UserData=Base64(Join('', bootstrapScript)), CreationPolicy={"ResourceSignal": { "Timeout": "PT15M" }})) return t
"export ambari_pass="******"\n", "export deploy=", ref_deploy_cluster, "\n", ] return exports + bootstrap_script_body.splitlines(True) AmbariNode = t.add_resource( ec2.Instance( "AmbariNode", UserData=Base64( Join( "", my_bootstrap_script('AmbariNode', 'true', 'true', '127.0.0.1'))), ImageId=FindInMap("CENTOS7", Ref("AWS::Region"), "AMI"), BlockDeviceMappings=my_block_device_mappings_ephemeral(24, "/dev/sd"), CreationPolicy=CreationPolicy( ResourceSignal=ResourceSignal(Count=1, Timeout="PT15M")), Tags=Tags(Name=ref_stack_name, ), KeyName=Ref(KeyName), InstanceType=Ref(InstanceType), SubnetId=Ref(SubnetId), SecurityGroupIds=Ref(SecurityGroups), )) t.add_output([ Output(
def test_parameterized_codec_b64(self) -> None: """Test parameterized codec b64.""" expected = Base64(Join("", ["Test ", {"Ref": "Interpolation"}, " Here"])) out = parameterized_codec("Test {{Interpolation}} Here", True) self.assertEqual(Base64, out.__class__) self.assertTemplateEqual(expected, out)
WebServerGroup = t.add_resource(AutoScalingGroup( "WebServerGroup", DesiredCapacity=Ref(WebServerCapacity), LaunchConfigurationName=Ref("LaunchConfig"), MinSize="1", MaxSize="5", LoadBalancerNames=[Ref("ElasticLoadBalancer")], AvailabilityZones=GetAZs(""), )) init_config = { "config": InitConfig({})} LaunchConfig = t.add_resource(LaunchConfiguration( "LaunchConfig", Metadata=Init(init_config), UserData=Base64(Join("", ["#!/bin/bash -xe\n", "yum update -y aws-cfn-bootstrap\n", "# Install the files and packages from the metadata\n", "/opt/aws/bin/cfn-init -v ", " --stack ", Ref("AWS::StackName"), " --resource LaunchConfig ", " --region ", Ref("AWS::Region"), "\n", "# Signal the status from cfn-init\n", "/opt/aws/bin/cfn-signal -e $? ", " --stack ", Ref("AWS::StackName"), " --resource WebServerGroup ", " --region ", Ref("AWS::Region"), "\n"])), KeyName=Ref(KeyName), SecurityGroups=[Ref(WebServerSecurityGroup)], InstanceType=Ref(InstanceType), ImageId=FindInMap("AWSRegionArch2AMI", Ref("AWS::Region"), FindInMap("AWSInstanceType2Arch", Ref(InstanceType), "Arch")), )) DBEC2SecurityGroup = t.add_resource(SecurityGroup( "DBEC2SecurityGroup", SecurityGroupIngress=[{ "ToPort": "3306", "IpProtocol": "tcp", "SourceSecurityGroupName": Ref(WebServerSecurityGroup), "FromPort": "3306" }], GroupDescription="Open database for access", Condition="Is-EC2-VPC", )) ElasticLoadBalancer = t.add_resource(LoadBalancer( "ElasticLoadBalancer",
def add_resources(self): metadata = { "AWS::CloudFormation::Init": { "configSets": { "wordpress_install": [ "install_wordpress"] }, "install_wordpress": { "packages": { "apt": { "apache2": [], "php": [], "php-mysql": [], "php7.0": [], "php7.0-mysql": [], "libapache2-mod-php7.0": [], "php7.0-cli": [], "php7.0-cgi": [], "php7.0-gd": [], "mysql-client": [], "sendmail": [] } }, "sources": { "/var/www/html": "http://wordpress.org/latest.tar.gz" }, "files": { "/tmp/create-wp-config": { "content": { "Fn::Join": ["", [ "#!/bin/bash\n", "cp /var/www/html/wordpress/wp-config-sample.php /var/www/html/wordpress/wp-config.php\n", "sed -i \"s/'database_name_here'/'", Ref( self.DBName), "'/g\" wp-config.php\n", "sed -i \"s/'username_here'/'", Ref( self.DBUser), "'/g\" wp-config.php\n", "sed -i \"s/'password_here'/'", Ref( self.DBPass), "'/g\" wp-config.php\n", "sed -i \"s/'localhost'/'", Ref( self.RDSEndpoint), "'/g\" wp-config.php\n" ]] }, "mode": "000500", "owner": "root", "group": "root" } }, "commands": { "01_configure_wordpress": { "command": "/tmp/create-wp-config", "cwd": "/var/www/html/wordpress" } } } } } self.WaitHandle = self.template.add_resource(cloudformation.WaitConditionHandle( "WaitHandle", )) self.WaitCondition = self.template.add_resource(cloudformation.WaitCondition( "WaitCondition", Handle=Ref(self.WaitHandle), Timeout="600", DependsOn="WebServerAutoScalingGroup", )) self.WebServerLaunchConfiguration = self.template.add_resource(autoscaling.LaunchConfiguration( "WebServerLaunchConfiguration", Metadata=metadata, UserData=Base64(Join("", [ "#!/bin/bash -x\n", "apt-get update\n", "apt-get install python-pip nfs-common -y \n", "mkdir -p /var/www/html/\n", "EC2_AZ=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)\n", "echo \"$EC2_AZ.", Ref(self.FileSystemID), ".efs.", Ref( "AWS::Region"), ".amazonaws.com:/ /var/www/html/ nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0\" >> /etc/fstab\n" "mount -a\n", "pip install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz\n", # "exec > /tmp/userdata.log 2>&1\n", "/usr/local/bin/cfn-init -v --stack ", Ref("AWS::StackName"), " --resource WebServerLaunchConfiguration ", " --configsets wordpress_install ", " --region ", Ref("AWS::Region"), "\n", "/bin/mv /var/www/html/wordpress/* /var/www/html/\n", "/bin/rm -f /var/www/html/index.html\n", "/bin/rm -rf /var/www/html/wordpress/\n", "chown www-data:www-data /var/www/html/* -R\n", "/usr/sbin/service apache2 restart\n", "/usr/bin/curl -O https://raw.githubusercontent.com/wp-cli/builds/gh-pages/phar/wp-cli.phar\n", "/bin/chmod +x wp-cli.phar\n", "/bin/mv wp-cli.phar /usr/local/bin/wp\n", "cd /var/www/html/\n", "if ! $(sudo -u www-data /usr/local/bin/wp core is-installed); then\n", "sudo -u www-data /usr/local/bin/wp core install ", "--url='", Ref(self.Hostname), ".", Ref(self.Domain), "' ", "--title='Cloudreach Meetup - ", Ref( self.Environment), "' ", "--admin_user='******' ", "--admin_password='******' ", "--admin_email='*****@*****.**'\n", "wget https://s3-eu-west-1.amazonaws.com/sceptre-meetup-munich/header.jpg -O /var/www/html/wp-content/themes/twentyseventeen/assets/images/header.jpg\n", "chown www-data:www-data /var/www/html/wp-content/themes/twentyseventeen/assets/images/header.jpg\n", "fi\n", "/usr/local/bin/cfn-signal -e $? --stack ", Ref( "AWS::StackName"), " -r \"Webserver setup complete\" '", Ref(self.WaitHandle), "'\n" ] )), ImageId=FindInMap("AWSRegion2AMI", Ref("AWS::Region"), "AMI"), KeyName=Ref(self.KeyName), SecurityGroups=[Ref(self.WebSecurityGroup)], InstanceType=Ref(self.InstanceType), AssociatePublicIpAddress=True, )) self.WebServerAutoScalingGroup = self.template.add_resource(autoscaling.AutoScalingGroup( "WebServerAutoScalingGroup", MinSize=Ref(self.WebServerCapacity), DesiredCapacity=Ref(self.WebServerCapacity), MaxSize=Ref(self.WebServerCapacity), VPCZoneIdentifier=[Ref(self.Subnet1), Ref(self.Subnet2)], AvailabilityZones=[Ref(self.AvailabilityZone1), Ref(self.AvailabilityZone2)], Tags=autoscaling.Tags( Name=Join("-", [Ref(self.Project), "web", "asg"]), Environment=Ref(self.Environment), Project=Ref(self.Project), ), LoadBalancerNames=[Ref(self.ElasticLoadBalancer)], LaunchConfigurationName=Ref(self.WebServerLaunchConfiguration), )) self.WebServerScaleUpPolicy = self.template.add_resource(autoscaling.ScalingPolicy( "WebServerScaleUpPolicy", ScalingAdjustment="1", Cooldown="60", AutoScalingGroupName=Ref(self.WebServerAutoScalingGroup), AdjustmentType="ChangeInCapacity", )) self.WebServerScaleDownPolicy = self.template.add_resource(autoscaling.ScalingPolicy( "WebServerScaleDownPolicy", ScalingAdjustment="-1", Cooldown="60", AutoScalingGroupName=Ref(self.WebServerAutoScalingGroup), AdjustmentType="ChangeInCapacity", )) self.CPUAlarmLow = self.template.add_resource(cloudwatch.Alarm( "CPUAlarmLow", EvaluationPeriods="2", Dimensions=[ cloudwatch.MetricDimension( Name="AutoScalingGroupName", Value=Ref(self.WebServerAutoScalingGroup) ), ], AlarmActions=[Ref(self.WebServerScaleDownPolicy)], AlarmDescription="Scale-down if CPU < 70% for 1 minute", Namespace="AWS/EC2", Period="60", ComparisonOperator="LessThanThreshold", Statistic="Average", Threshold="70", MetricName="CPUUtilization", )) self.CPUAlarmHigh = self.template.add_resource(cloudwatch.Alarm( "CPUAlarmHigh", EvaluationPeriods="2", Dimensions=[ cloudwatch.MetricDimension( Name="AutoScalingGroupName", Value=Ref("WebServerAutoScalingGroup") ), ], AlarmActions=[Ref(self.WebServerScaleUpPolicy)], AlarmDescription="Scale-up if CPU > 50% for 1 minute", Namespace="AWS/EC2", Period="60", ComparisonOperator="GreaterThanThreshold", Statistic="Average", Threshold="50", MetricName="CPUUtilization", ))
def main(): template = Template() template.add_resource( ecs.Cluster("ECSCluster", ClusterName="WorldCheckCluster")) template.add_resource( iam.Role("ECSTaskRole", AssumeRolePolicyDocument={ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "Service": ["ecs-tasks.amazonaws.com"] }, "Action": ["sts:AssumeRole"] }] })) template.add_resource( iam.Role( "ECSServiceSchedulerRole", AssumeRolePolicyDocument={ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "Service": ["ecs.amazonaws.com"] }, "Action": ["sts:AssumeRole"] }] }, Policies=[ iam.Policy(PolicyDocument={ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Action": [ "ec2:Describe*", "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", "elasticloadbalancing:DeregisterTargets", "elasticloadbalancing:Describe*", "elasticloadbalancing:RegisterInstancesWithLoadBalancer", "elasticloadbalancing:RegisterTargets" ], "Resource": "*" }] }, PolicyName="ecs-service") ])) template.add_resource( iam.Role("EC2InstanceRole", AssumeRolePolicyDocument={ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "Service": ["ec2.amazonaws.com"] }, "Action": ["sts:AssumeRole"] }] }, Policies=[ iam.Policy(PolicyDocument={ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Action": [ "ecs:CreateCluster", "ecs:DeregisterContainerInstance", "ecs:DiscoverPollEndpoint", "ecs:Poll", "ecs:RegisterContainerInstance", "ecs:StartTelemetrySession", "ecr:GetAuthorizationToken", "ecr:BatchGetImage", "ecr:GetDownloadUrlForLayer", "ecs:Submit*", "logs:CreateLogStream", "logs:PutLogEvents", "ec2:DescribeTags", "cloudwatch:PutMetricData" ], "Resource": "*" }] }, PolicyName="ecs-service") ])) template.add_resource( iam.InstanceProfile("EC2InstanceProfile", Roles=[Ref("EC2InstanceRole")])) with open("user-data.sh", "r") as f: user_data_content = f.readlines() template.add_resource( ec2.Instance( "EC2Instance", ImageId="ami-13f7226a", InstanceType="t2.micro", SecurityGroups=["default"], UserData=Base64(Join('', [Sub(x) for x in user_data_content])), IamInstanceProfile=Ref("EC2InstanceProfile"), )) template.add_resource( ecs.TaskDefinition( "ECSTaskDefinition", TaskRoleArn=Ref("ECSTaskRole"), ContainerDefinitions=[ ecs.ContainerDefinition( Name="SimpleServer", Memory="128", Image="abbas123456/simple-server:latest", PortMappings=[ecs.PortMapping(ContainerPort=8000)], ) ])) template.add_resource( elb.TargetGroup( "ECSTargetGroup", VpcId="vpc-925497f6", Port=8000, Protocol="HTTP", )) template.add_resource( elb.LoadBalancer( "LoadBalancer", Subnets=["subnet-a321c8fb", "subnet-68fa271e", "subnet-689d350c"], SecurityGroups=["sg-0202bd65"])) template.add_resource( elb.Listener( "LoadBalancerListener", DefaultActions=[ elb.Action(Type="forward", TargetGroupArn=Ref("ECSTargetGroup")) ], LoadBalancerArn=Ref("LoadBalancer"), Port=80, Protocol="HTTP", )) template.add_resource( ecs.Service("ECSService", Cluster=Ref("ECSCluster"), DesiredCount=1, LoadBalancers=[ ecs.LoadBalancer(ContainerPort=8000, ContainerName="SimpleServer", TargetGroupArn=Ref("ECSTargetGroup")) ], Role=Ref("ECSServiceSchedulerRole"), TaskDefinition=Ref("ECSTaskDefinition"), DependsOn="LoadBalancerListener")) return template.to_json()
SecurityGroups=[Ref("LoadBalancerSecurityGroup")], )) t.add_output(Output( "WebUrl", Description="Application endpoint", Value=Join("", [ "http://", GetAtt("LoadBalancer", "DNSName"), ":", ApplicationPort ]), )) ud = Base64(Join("\n", [ "#!/bin/bash", "exec > /var/log/userdata.log 2>&1", "sudo yum install --enablerepo=epel -y git", "sudo pip install ansible", AnsiblePullCmd, "echo '*/10 * * * * {}' > /etc/cron.d/ansible-pull".format(AnsiblePullCmd) ])) t.add_resource(Role( "Role", AssumeRolePolicyDocument=Policy( Statement=[ Statement( Effect=Allow, Action=[AssumeRole], Principal=Principal("Service", ["ec2.amazonaws.com"]) ) ] )
def main(): t = Template("A template to create a load balanced autoscaled Web flask deployment using ansible.") addMapping(t) ### VPC CONFIGURATION ### vpc = ec2.VPC( "MainVPC", CidrBlock="10.1.0.0/16" ) t.add_resource(vpc) vpc_id = Ref(vpc) subnet_1 = ec2.Subnet( "WebAppSubnet1", t, AvailabilityZone="us-east-1a", CidrBlock="10.1.0.0/24", MapPublicIpOnLaunch=True, VpcId=vpc_id, ) subnet_1_id = Ref(subnet_1) subnet_2 = ec2.Subnet( "WebAppSubnet2", t, AvailabilityZone="us-east-1b", CidrBlock="10.1.1.0/24", MapPublicIpOnLaunch=True, VpcId=vpc_id, ) subnet_2_id = Ref(subnet_2) ### NETWORKING ### igw = ec2.InternetGateway("internetGateway", t) gateway_to_internet = ec2.VPCGatewayAttachment( "GatewayToInternet", t, VpcId=vpc_id, InternetGatewayId=Ref(igw) ) route_table = ec2.RouteTable( "subnetRouteTable", t, VpcId=vpc_id ) route_table_id = Ref(route_table) internet_route = ec2.Route( "routeToInternet", t, DependsOn=gateway_to_internet, DestinationCidrBlock="0.0.0.0/0", GatewayId=Ref(igw), RouteTableId=route_table_id ) subnet_1_route_assoc = ec2.SubnetRouteTableAssociation( "Subnet1RouteAssociation", t, RouteTableId=route_table_id, SubnetId=Ref(subnet_1) ) subnet_2_route_assoc = ec2.SubnetRouteTableAssociation( "Subnet2RouteAssociation", t, RouteTableId=route_table_id, SubnetId=Ref(subnet_2) ) http_ingress = { "CidrIp": "0.0.0.0/0", "Description": "Allow HTTP traffic in from internet.", "IpProtocol": "tcp", "FromPort": 80, "ToPort": 80, } ssh_ingress = { "CidrIp": "0.0.0.0/0", "Description": "Allow SSH traffic in from internet.", "IpProtocol": "tcp", "FromPort": 22, "ToPort": 22, } elb_sg = ec2.SecurityGroup( "elbSecurityGroup", t, GroupName="WebGroup", GroupDescription="Allow web traffic in from internet to ELB", VpcId=vpc_id, SecurityGroupIngress=[ http_ingress ]) ssh_sg = ec2.SecurityGroup( "sshSecurityGroup", t, GroupName="SSHGroup", GroupDescription="Allow SSH traffic in from internet", VpcId=vpc_id, SecurityGroupIngress=[ ssh_ingress ] ) elb_sg_id = Ref(elb_sg) ssh_sg_id = Ref(ssh_sg) autoscale_ingress = { "SourceSecurityGroupId": elb_sg_id, "Description": "Allow web traffic in from ELB", "IpProtocol": "tcp", "FromPort": 80, "ToPort": 80 } autoscale_sg = ec2.SecurityGroup( "WebAutoscaleSG", t, GroupName="AutoscaleGroup", GroupDescription="Allow web traffic in from elb on port 80", VpcId=vpc_id, SecurityGroupIngress=[ autoscale_ingress ] ) autoscale_sg_id = Ref(autoscale_sg) # BUCKETS app_bucket = s3.Bucket( "CodeDeployApplicationBucket", t, ) ### LOAD BALANCING ### Web_elb = elb.LoadBalancer( "WebElb", t, Name="WebElb", # TODO: Fix for name conflict Subnets=[subnet_1_id, subnet_2_id], SecurityGroups=[elb_sg_id] ) Web_target_group = elb.TargetGroup( "WebTargetGroup", t, DependsOn=Web_elb, HealthCheckPath="/health", HealthCheckPort=80, HealthCheckProtocol="HTTP", Matcher=elb.Matcher(HttpCode="200"), Name="NginxTargetGroup", Port=80, Protocol="HTTP", VpcId=vpc_id ) Web_listener = elb.Listener( "WebListener", t, LoadBalancerArn=Ref(Web_elb), DefaultActions=[ elb.Action("forwardAction", TargetGroupArn=Ref(Web_target_group), Type="forward" ) ], Port=80, Protocol="HTTP" ) ### AUTOSCALING ### # Everything after sudo -u ubuntu is one command # The sudo command is required to properly set file permissions when # running the ansible script as it assumes running from non root user lc_user_data = Base64(Join("\n", [ "#!/bin/bash", "apt-add-repository -y ppa:ansible/ansible", "apt-get update && sudo apt-get -y upgrade", "apt-get -y install git", "apt-get -y install ansible", "cd /home/ubuntu/", "sudo -H -u ubuntu bash -c '" "export LC_ALL=C.UTF-8 && " "export LANG=C.UTF-8 && " "ansible-pull -U https://github.com/DameonSmith/aws-meetup-ansible.git --extra-vars \"user=ubuntu\"'" ])) web_instance_role = iam.Role( "webInstanceCodeDeployRole", t, AssumeRolePolicyDocument={ 'Statement': [{ 'Effect': 'Allow', 'Principal': { 'Service': 'ec2.amazonaws.com' }, 'Action': 'sts:AssumeRole' }] }, Policies=[ iam.Policy( PolicyName="CodeDeployS3Policy", PolicyDocument=aws.Policy( Version='2012-10-17', Statement=[ aws.Statement( Sid='CodeDeployS3', Effect=aws.Allow, Action=[ aws_s3.PutObject, aws_s3.GetObject, aws_s3.GetObjectVersion, aws_s3.DeleteObject, aws_s3.ListObjects, aws_s3.ListBucket, aws_s3.ListBucketVersions, aws_s3.ListAllMyBuckets, aws_s3.ListMultipartUploadParts, aws_s3.ListBucketMultipartUploads, aws_s3.ListBucketByTags, ], Resource=[ GetAtt(app_bucket, 'Arn'), Join('', [ GetAtt(app_bucket, 'Arn'), '/*', ]), "arn:aws:s3:::aws-codedeploy-us-east-2/*", "arn:aws:s3:::aws-codedeploy-us-east-1/*", "arn:aws:s3:::aws-codedeploy-us-west-1/*", "arn:aws:s3:::aws-codedeploy-us-west-2/*", "arn:aws:s3:::aws-codedeploy-ca-central-1/*", "arn:aws:s3:::aws-codedeploy-eu-west-1/*", "arn:aws:s3:::aws-codedeploy-eu-west-2/*", "arn:aws:s3:::aws-codedeploy-eu-west-3/*", "arn:aws:s3:::aws-codedeploy-eu-central-1/*", "arn:aws:s3:::aws-codedeploy-ap-northeast-1/*", "arn:aws:s3:::aws-codedeploy-ap-northeast-2/*", "arn:aws:s3:::aws-codedeploy-ap-southeast-1/*", "arn:aws:s3:::aws-codedeploy-ap-southeast-2/*", "arn:aws:s3:::aws-codedeploy-ap-south-1/*", "arn:aws:s3:::aws-codedeploy-sa-east-1/*", ] ) ] ) ) ] ) web_instance_profile = iam.InstanceProfile( "webInstanceProfile", t, Path='/', Roles=[Ref(web_instance_role)], ) Web_launch_config = autoscaling.LaunchConfiguration( "webLaunchConfig", t, ImageId=FindInMap("RegionMap", Ref("AWS::Region"), "AMI"), # TODO: Remove magic string SecurityGroups=[ssh_sg_id, autoscale_sg_id], IamInstanceProfile=Ref(web_instance_profile), InstanceType="t2.micro", BlockDeviceMappings= [{ "DeviceName": "/dev/sdk", "Ebs": {"VolumeSize": "10"} }], UserData= lc_user_data, KeyName="advanced-cfn", ) Web_autoscaler = autoscaling.AutoScalingGroup( "WebAutoScaler", t, LaunchConfigurationName=Ref(Web_launch_config), MinSize="2", # TODO: Change to parameter MaxSize="2", VPCZoneIdentifier=[subnet_2_id, subnet_1_id], TargetGroupARNs= [Ref(Web_target_group)] ) t.add_output([ Output( "ALBDNS", Description="The DNS name for the application load balancer.", Value=GetAtt(Web_elb, "DNSName") ) ]) # DEVTOOLS CONFIG codebuild_service_role = iam.Role( "CMSCodeBuildServiceRole", t, AssumeRolePolicyDocument={ 'Statement': [{ 'Effect': 'Allow', 'Principal': { 'Service': ['codebuild.amazonaws.com'] }, 'Action': ['sts:AssumeRole'] }] }, Policies=[ iam.Policy( PolicyName="CloudWatchLogsPolicy", PolicyDocument=aws.Policy( Version="2012-10-17", Statement=[ aws.Statement( Sid='logs', Effect=aws.Allow, Action=[ aws_logs.CreateLogGroup, aws_logs.CreateLogStream, aws_logs.PutLogEvents ], Resource=['*'] ) ] ) ), iam.Policy( PolicyName="s3AccessPolicy", PolicyDocument=aws.Policy( Version="2012-10-17", Statement=[ aws.Statement( Sid='codebuilder', Effect=aws.Allow, Action=[ aws_s3.PutObject, aws_s3.GetObject, aws_s3.GetObjectVersion, aws_s3.DeleteObject ], Resource=[ GetAtt(app_bucket, 'Arn'), Join('', [ GetAtt(app_bucket, 'Arn'), '/*', ]) ] ) ] ) ) ] ) github_repo = Parameter( "GithubRepoLink", Description="Name of the repository you wish to connect to codebuild.", Type="String" ) artifact_key = Parameter( "ArtifactKey", Description="The key for the artifact that codebuild creates.", Type="String" ) t.add_parameter(github_repo) t.add_parameter(artifact_key) cms_code_build_project = codebuild.Project( "CMSBuild", t, Name="CMS-Build", Artifacts=codebuild.Artifacts( Location=Ref(app_bucket), Name=Ref(artifact_key), NamespaceType="BUILD_ID", Type="S3", Packaging="ZIP" ), Description="Code build for CMS", Environment=codebuild.Environment( ComputeType="BUILD_GENERAL1_SMALL", Image="aws/codebuild/python:3.6.5", Type="LINUX_CONTAINER", ), ServiceRole=GetAtt(codebuild_service_role, 'Arn'), Source=codebuild.Source( "CMSSourceCode", Auth=codebuild.SourceAuth( "GitHubAuth", Type="OAUTH" ), Location=Ref(github_repo), Type="GITHUB" ), Triggers=codebuild.ProjectTriggers( Webhook=True ) ) codedeploy_service_role = iam.Role( "CMSDeploymentGroupServiceRole", t, AssumeRolePolicyDocument={ 'Statement': [{ 'Effect': 'Allow', 'Principal': { 'Service': ['codedeploy.amazonaws.com'] }, 'Action': ['sts:AssumeRole'] }] }, Policies=[ iam.Policy( PolicyName="CloudWatchLogsPolicy", PolicyDocument=aws.Policy( Version="2012-10-17", Statement=[ aws.Statement( Sid='logs', Effect=aws.Allow, Action=[ aws_logs.CreateLogGroup, aws_logs.CreateLogStream, aws_logs.PutLogEvents ], Resource=['*'] ) ] ) ), iam.Policy( PolicyName="s3AccessPolicy", PolicyDocument=aws.Policy( Version="2012-10-17", Statement=[ aws.Statement( Sid='codebuilder', Effect=aws.Allow, Action=[ aws_s3.PutObject, aws_s3.GetObject, aws_s3.GetObjectVersion, aws_s3.DeleteObject ], Resource=[ GetAtt(app_bucket, 'Arn'), Join('', [ GetAtt(app_bucket, 'Arn'), '/*' ]) ] ) ] ) ), iam.Policy( PolicyName="autoscalingAccess", PolicyDocument=aws.Policy( Version="2012-10-17", Statement=[ aws.Statement( Sid='codebuilder', Effect=aws.Allow, Action=[ aws.Action('autoscaling', '*'), aws.Action('elasticloadbalancing', '*') ], Resource=[ '*' ] ) ] ) ) ] ) cms_codedeploy_application = codedeploy.Application( "CMSCodeDeployApplication", t, ) cms_deployment_group = codedeploy.DeploymentGroup( "CMSDeploymentGroup", t, DependsOn=[cms_codedeploy_application], ApplicationName=Ref(cms_codedeploy_application), AutoScalingGroups=[Ref(Web_autoscaler)], LoadBalancerInfo=codedeploy.LoadBalancerInfo( "CodeDeployLBInfo", TargetGroupInfoList=[ codedeploy.TargetGroupInfoList( "WebTargetGroup", Name=GetAtt(Web_target_group, "TargetGroupName") ) ] ), ServiceRoleArn=GetAtt(codedeploy_service_role, 'Arn') ) print(t.to_yaml())
def main(): # Initialize template template = Template() template.set_version("2010-09-09") template.set_description("""\ Configures autoscaling group for nginx app""") # Collect template properties through parameters InstanceType = template.add_parameter( Parameter( "InstanceType", Type="String", Description="WebServer EC2 instance type", Default="t2.small", AllowedValues=[ "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge" ], ConstraintDescription="Must be a valid EC2 instance type.", )) KeyName = template.add_parameter( Parameter( "KeyName", Type="String", Description="Name of an existing EC2 KeyPair to enable SSH access", MinLength="1", AllowedPattern="[\x20-\x7E]*", MaxLength="255", ConstraintDescription="Can contain only ASCII characters.", )) ScaleCapacity = template.add_parameter( Parameter( "ScaleCapacity", Default="1", Type="String", Description="Number of nginx servers to run", )) PublicSubnet1 = template.add_parameter( Parameter( "PublicSubnet1", Type="String", Description= "A public VPC subnet ID for the nginx app load balancer.", )) PublicSubnet2 = template.add_parameter( Parameter( "PublicSubnet2", Type="String", Description="A public VPC subnet ID for the nginx load balancer.", )) VPCAvailabilityZone2 = template.add_parameter( Parameter( "VPCAvailabilityZone2", MinLength="1", Type="String", Description="Second availability zone", MaxLength="255", )) VPCAvailabilityZone1 = template.add_parameter( Parameter( "VPCAvailabilityZone1", MinLength="1", Type="String", Description="First availability zone", MaxLength="255", )) PrivateSubnet2 = template.add_parameter( Parameter( "PrivateSubnet2", Type="String", Description="Second private VPC subnet ID for the nginx app.", )) PrivateSubnet1 = template.add_parameter( Parameter( "PrivateSubnet1", Type="String", Description="First private VPC subnet ID for the nginx app.", )) VpcId = template.add_parameter( Parameter( "VpcId", Type="String", Description="VPC Id.", )) # as of now only provide centos based ami id AmiId = template.add_parameter( Parameter( "AmiId", Type="String", Description="AMI Id.", )) # Create a common security group NginxInstanceSG = template.add_resource( ec2.SecurityGroup( "InstanceSecurityGroup", VpcId=Ref(VpcId), GroupDescription="Enable SSH and HTTP access on the inbound port", SecurityGroupIngress=[ ec2.SecurityGroupRule( IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="10.0.0.0/8", ), ec2.SecurityGroupRule( IpProtocol="tcp", FromPort="80", ToPort="80", CidrIp="0.0.0.0/0", ) ])) # Add the application LB ApplicationElasticLB = template.add_resource( elb.LoadBalancer("ApplicationElasticLB", Name="ApplicationElasticLB", Scheme="internet-facing", Subnets=[Ref(PublicSubnet1), Ref(PublicSubnet2)], SecurityGroups=[Ref(NginxInstanceSG)])) # Add Target Group for the ALB TargetGroupNginx = template.add_resource( elb.TargetGroup( "TargetGroupNginx", VpcId=Ref(VpcId), HealthCheckIntervalSeconds="30", HealthCheckProtocol="HTTP", HealthCheckTimeoutSeconds="10", HealthyThresholdCount="4", Matcher=elb.Matcher(HttpCode="200"), Name="NginxTarget", Port="80", Protocol="HTTP", UnhealthyThresholdCount="3", )) # Add ALB listener Listener = template.add_resource( elb.Listener("Listener", Port="80", Protocol="HTTP", LoadBalancerArn=Ref(ApplicationElasticLB), DefaultActions=[ elb.Action(Type="forward", TargetGroupArn=Ref(TargetGroupNginx)) ])) # Add launch configuration for auto scaling LaunchConfig = template.add_resource( LaunchConfiguration( "LaunchConfiguration", ImageId=Ref(AmiId), KeyName=Ref(KeyName), AssociatePublicIpAddress="False", LaunchConfigurationName="nginx-LC", UserData=Base64( Join('', [ "#!/bin/bash\n", "yum update\n", "yum -y install nginx\n", "chkconfig nginx on\n", "service nginx start" ])), SecurityGroups=[Ref(NginxInstanceSG)], InstanceType=Ref(InstanceType))) # Add auto scaling group AutoscalingGroup = template.add_resource( AutoScalingGroup( "AutoscalingGroup", DesiredCapacity=Ref(ScaleCapacity), LaunchConfigurationName=Ref(LaunchConfig), MinSize="1", TargetGroupARNs=[Ref(TargetGroupNginx)], MaxSize=Ref(ScaleCapacity), VPCZoneIdentifier=[Ref(PrivateSubnet1), Ref(PrivateSubnet2)], AvailabilityZones=[ Ref(VPCAvailabilityZone1), Ref(VPCAvailabilityZone2) ], HealthCheckType="EC2")) template.add_output( Output("URL", Description="URL of the sample website", Value=Join("", ["http://", GetAtt(ApplicationElasticLB, "DNSName")]))) #print(template.to_json()) with open('AutoScaling.yaml', 'w') as f: f.write(template.to_yaml())
), ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy( Enabled=True, Timeout=10, ), CrossZone=True, Subnets=Ref("PublicSubnet"), SecurityGroups=[Ref("LoadBalancerSecurityGroup")], )) ud = Base64( Join('\n', [ "#!/bin/bash", "yum install --enablerepo=epel -y git", "yum install ruby -y", "wget https://aws-codedeploy-us-west-2.s3.amazonaws.com/latest/install", "chmod +x ./install", "./install auto", "#yum install java-1.8.0-openjdk -y", "#echo 2 |/usr/sbin/alternatives --config java", "pip install ansible", AnsiblePullCmd, "echo '*/10 * * * * {}' > /etc/cron.d/ansible-pull". format(AnsiblePullCmd) ])) t.add_resource( Role("Role", AssumeRolePolicyDocument=Policy(Statement=[ Statement(Effect=Allow, Action=[AssumeRole], Principal=Principal("Service", ["ec2.amazonaws.com"])) ]))) t.add_resource(
Ref(instanceSecurityGroup)], AssociatePublicIpAddress='true', DeviceIndex='0', DeleteOnTermination='true', SubnetId=Ref(subnet))], UserData=Base64( Join( '', [ '#!/bin/bash -xe\n', 'yum update -y aws-cfn-bootstrap\n', '/opt/aws/bin/cfn-init -v ', ' --stack ', Ref('AWS::StackName'), ' --resource WebServerInstance ', ' --region ', Ref('AWS::Region'), '\n', '/opt/aws/bin/cfn-signal -e $? ', ' --stack ', Ref('AWS::StackName'), ' --resource WebServerInstance ', ' --region ', Ref('AWS::Region'), '\n', ])), CreationPolicy=CreationPolicy( ResourceSignal=ResourceSignal( Timeout='PT15M')), Tags=Tags( Application=ref_stack_id),
FromPort="22", ToPort="22", CidrIp=PublicCidrIp, ), ec2.SecurityGroupRule( IpProtocol="tcp", FromPort=ApplicationPort, ToPort=ApplicationPort, CidrIp="0.0.0.0/0", ), ], )) ud = Base64( Join('\n', [ "#!/bin/bash", "sudo yum install --enablerepo=epel -y git", "pip install ansible", AnsiblePullCmd ])) t.add_resource( Role("Role", AssumeRolePolicyDocument=Policy(Statement=[ Statement(Effect=Allow, Action=[AssumeRole], Principal=Principal("Service", ["ec2.amazonaws.com"])) ]))) t.add_resource( IAMPolicy("Policy", PolicyName="AllowCodePipeline", PolicyDocument=Policy(Statement=[
def create_auto_scaling_resources(self, worker_security_group, worker_lb): worker_launch_config_name = 'lcWorker' worker_launch_config = self.add_resource( asg.LaunchConfiguration( worker_launch_config_name, EbsOptimized=True, ImageId=Ref(self.worker_ami), IamInstanceProfile=Ref(self.worker_instance_profile), InstanceType=Ref(self.worker_instance_type), KeyName=Ref(self.keyname), SecurityGroups=[Ref(worker_security_group)], UserData=Base64( Join('', self.get_cloud_config())) )) worker_auto_scaling_group_name = 'asgWorker' worker_asg = self.add_resource( asg.AutoScalingGroup( worker_auto_scaling_group_name, AvailabilityZones=Ref(self.availability_zones), Cooldown=300, DesiredCapacity=Ref(self.worker_auto_scaling_desired), HealthCheckGracePeriod=600, HealthCheckType='ELB', LaunchConfigurationName=Ref(worker_launch_config), LoadBalancerNames=[Ref(worker_lb)], MaxSize=Ref(self.worker_auto_scaling_max), MinSize=Ref(self.worker_auto_scaling_min), NotificationConfigurations=[ asg.NotificationConfigurations( TopicARN=Ref(self.notification_topic_arn), NotificationTypes=[ asg.EC2_INSTANCE_LAUNCH, asg.EC2_INSTANCE_LAUNCH_ERROR, asg.EC2_INSTANCE_TERMINATE, asg.EC2_INSTANCE_TERMINATE_ERROR ] ) ], VPCZoneIdentifier=Ref(self.private_subnets), Tags=[asg.Tag('Name', 'Worker', True)] ) ) self.add_resource( asg.ScheduledAction( 'schedWorkerAutoScalingStart', AutoScalingGroupName=Ref(worker_asg), DesiredCapacity=Ref( self.worker_auto_scaling_schedule_start_capacity), Recurrence=Ref( self.worker_auto_scaling_schedule_start_recurrence) ) ) self.add_resource( asg.ScheduledAction( 'schedWorkerAutoScalingEnd', AutoScalingGroupName=Ref(worker_asg), DesiredCapacity=Ref( self.worker_auto_scaling_schedule_end_capacity), Recurrence=Ref( self.worker_auto_scaling_schedule_end_recurrence) ) )
GroupDescription="Allow SSH and TCP/{} access".format(ApplicationPort), SecurityGroupIngress=[ ec2.SecurityGroupRule(IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="0.0.0.0/0"), ec2.SecurityGroupRule(IpProtocol="tcp", FromPort=ApplicationPort, ToPort=ApplicationPort, CidrIp="0.0.0.0/0") ])) ud = Base64( Join('\n', [ "#!/bin/bash", "sudo yum install --enablerepo=epel -y nodejs", "wget http://bit.ly/2vESNuc -O /home/ec2-user/helloworld.js", "wget http://bit.ly/2vVvT18 -O /etc/init/helloworld.conf", "start helloworld" ])) t.add_resource( ec2.Instance("instance", ImageId="ami-bf4193c7", InstanceType="t2.micro", SecurityGroups=[Ref("SecurityGroup")], KeyName=Ref("KeyPair"), UserData=ud)) t.add_output( Output("InstancePublicIp", Description="Public Ip of our instance.",
), ec2.SecurityGroupRule( IpProtocol="tcp", FromPort=ApplicationPort, ToPort=ApplicationPort, CidrIp="0.0.0.0/0", ), ], )) ud = Base64(Join('\n', [ "#!/bin/bash", "sudo hostnamectl set-hostname test.localdomain", "sudo reboot", "sudo yum -y gcc-c++ make", "curl -sL https://rpm.nodesource.com/setup_11.x | sudo -E bash -", "sudo yum -y install nodejs", "sudo yum -y install wget", "wget https://raw.githubusercontent.com/szhouchoice/EffectiveDevOpsTemplates/master/helloworld.js -O /home/centos/helloworld.js", "wget https://raw.githubusercontent.com/szhouchoice/EffectiveDevOpsTemplates/master/helloworld.service -O /lib/systemd/system/helloworld.service", "service helloworld start " ])) t.add_resource(ec2.Instance( "instance", ImageId="ami-03e057d80a32a76ab", InstanceType="t2.micro", SecurityGroups=[Ref("SecurityGroup")], KeyName=Ref("KeyPair"), UserData=ud, ))