def configure_for_follower(instance, counter): subnet_index = counter % NUM_AZS if counter == 2: instance.DependsOn = "MonitorInstance" else: instance.DependsOn = BASE_NAME + str(counter - 1) #base_instance.title instance.SubnetId = Select(str(subnet_index), Ref(PrivateSubnetsToSpanParam)) # instance.AvailabilityZone = Select(str(subnet_index), Ref(AvailabilityZonesParam)) instance.Metadata = cloudformation.Metadata( cloudformation.Authentication({ "S3AccessCreds": cloudformation.AuthenticationBlock( type="S3", roleName=Ref(StorReduceHostRole), #Ref(HostRoleParam), buckets=[Ref(QSS3BucketNameParam)]) }), cloudformation.Init({ "config": cloudformation.InitConfig( files=cloudformation.InitFiles({ "/home/ec2-user/connect-srr.sh": cloudformation.InitFile(source=Sub( "https://${" + QSS3BucketNameParam.title + "}.${QSS3Region}.amazonaws.com/${" + QSS3KeyPrefixParam.title + "}scripts/connect-srr.sh", **{ "QSS3Region": If("GovCloudCondition", "s3-us-gov-west-1", "s3") }), mode="000550", owner="root", group="root") }), commands={ "connect-srr": { "command": Join("", [ "/home/ec2-user/connect-srr.sh \"", GetAtt(base_instance, "PrivateDnsName"), "\" \'", Ref(StorReducePasswordParam), "\' ", "\"", Ref(ShardsNumParam), "\" ", "\"", Ref(ReplicaShardsNumParam), "\" ", "\"", Ref(elasticLB), "\" ", "\"", Ref("AWS::Region"), "\" ", "\"", GetAtt("Eth0", "PrimaryPrivateIpAddress"), "\" ", "\"", Ref(NumSRRHostsParam), "\"" ]) } }) })) instance.Tags = [{"Key": "Name", "Value": "StorReduce-QS-Host"}]
def Meta(name, private_ip): return cfn.Metadata( cfn.Authentication({ "default": cfn.AuthenticationBlock(type="S3", roleName=Ref(InstanceRole), buckets=[ Ref(Bucket) ]) }), cfn.Init( cfn.InitConfigSets(default = ['SetupHost','SetupWebsite']), SetupHost = cfn.InitConfig( files = { "/etc/hostname":{ "content": Join(".",[ name, Ref(HostedZone) ]) }, "/root/set-hostname.sh":{ "content": Join("",[ "#!/bin/bash\n", "hostname --file /etc/hostname\n", "h=$(cat /etc/hostname)\n", "sed -i s/HOSTNAME=.*/HOSTNAME=$h/g /etc/sysconfig/network\n", "echo ", private_ip , " $h >>/etc/hosts \n", "service network restart" ]), "mode": "755" }, }, commands={ "1-set-hostname": { "command": "/root/set-hostname.sh " } } ), SetupWebsite = cfn.InitConfig( packages = { "yum" : { 'httpd' : []} }, sources = { "/var/www/html/": Join("", [ "https://", Ref(Bucket), ".s3.amazonaws.com/3dstreetartindia.zip" ]) }, services = { "sysvinit" : { "httpd" : { "enabled" : "true", "ensureRunning" : "true" }, } } ) ))
"sysvinit": cloudformation.InitServices({ "rsyslog": cloudformation.InitService( enabled=True, ensureRunning=True, files=["/etc/rsyslog.d/20-somethin.conf"], ) }) }, ) }), cloudformation.Authentication({ "DeployUserAuth": cloudformation.AuthenticationBlock( type="S3", accessKeyId=Ref(DeployUserAccessKey), secretKey=Ref(DeployUserSecretKey), ) }), ), UserData=Base64( Join( "", [ "#!/bin/bash\n", "cfn-signal -e 0", " --resource AutoscalingGroup", " --stack ", Ref("AWS::StackName"), " --region ", Ref("AWS::Region"),
def AS_LaunchTemplate(): cfg.use_cfn_init = True InitConfigSets = ASInitConfigSets() CfmInitArgs = {} IBoxEnvApp = [] Tags_List = [] UserDataApp = [] for n in cfg.Apps: name = f"Apps{n}" # Ex. Apps1 envname = f"EnvApp{n}Version" # Ex EnvApp1Version reponame = f"{name}RepoName" # Ex Apps1RepoName UserDataApp.extend(["#${%s}\n" % envname]) p_EnvAppVersion = Parameter( envname, Description=f"Application {n} version", AllowedPattern="^[a-zA-Z0-9-_.]*$", ) p_AppsRepoName = Parameter( reponame, Description=f"App {n} Repo Name - empty for default based on env/role", AllowedPattern="^[a-zA-Z0-9-_.]*$", ) # parameters add_obj( [ p_EnvAppVersion, p_AppsRepoName, ] ) # conditions add_obj( { name: And( Not(Equals(Ref(envname), "")), Not(get_condition("", "equals", "None", reponame)), ) } ) InitConfigApps = ASInitConfigApps(name) CfmInitArgs[name] = InitConfigApps InitConfigAppsBuilAmi = ASInitConfigAppsBuildAmi(name) # AUTOSPOT - Let cfn-init always prepare instances on boot # CfmInitArgs[name + 'BuildAmi'] = InitConfigAppsBuilAmi CfmInitArgs[name] = InitConfigAppsBuilAmi IBoxEnvApp.extend( [ f"export EnvApp{n}Version=", Ref(envname), "\n", f"export EnvRepo{n}Name=", get_endvalue(reponame), "\n", ] ) InitConfigSetsApp = If(name, name, Ref("AWS::NoValue")) InitConfigSetsAppBuilAmi = If(name, f"{name}BuildAmi", Ref("AWS::NoValue")) IndexSERVICES = InitConfigSets.data["default"].index("SERVICES") InitConfigSets.data["default"].insert(IndexSERVICES, InitConfigSetsApp) # AUTOSPOT - Let cfn-init always prepare instances on boot # InitConfigSets.data['buildamifull'].append( # InitConfigSetsAppBuilAmi) InitConfigSets.data["buildamifull"].append(InitConfigSetsApp) Tags_List.append(asg.Tag(envname, Ref(envname), True)) # outputs Output_app = Output(envname, Value=Ref(envname)) Output_repo = Output(reponame, Value=get_endvalue(reponame)) add_obj([Output_app, Output_repo]) InitConfigSetup = ASInitConfigSetup() InitConfigSetup.ibox_env_app = IBoxEnvApp InitConfigSetup.setup() InitConfigCodeDeploy = ASInitConfigCodeDeploy() CfmInitArgs["SETUP"] = InitConfigSetup CfmInitArgs["CWAGENT"] = ASInitConfigCloudWatchAgent("") if cfg.CodeDeploy: CfmInitArgs["CODEDEPLOY"] = InitConfigCodeDeploy if not getattr(cfg, "IBOX_LAUNCH_TEMPLATE_NO_WAIT_ELB_HEALTH", False): for lb in cfg.LoadBalancer: # LoadBalancerClassic if cfg.LoadBalancerType == "Classic": InitConfigELB = ASInitConfigELBClassic(scheme=lb) CfmInitArgs["ELBWAITER"] = InitConfigELB # LoadBalancerApplication if cfg.LoadBalancerType == "Application": InitConfigELB = ASInitConfigELBApplication(scheme=lb) CfmInitArgs["ELBWAITER"] = InitConfigELB # LoadBalancerNetwork if cfg.LoadBalancerType == "Network": for k in cfg.Listeners: InitConfigELB = ASInitConfigELBApplication( scheme=f"TargetGroupListeners{k}{lb}" ) CfmInitArgs["ELBWAITER"] = InitConfigELB if getattr(cfg, "IBOX_LAUNCH_TEMPLATE_NO_SG_EXTRA", False): SecurityGroups = [] else: SecurityGroups = cfg.SecurityGroupsImport # Resources R_LaunchTemplate = ec2.LaunchTemplate( "LaunchTemplate", LaunchTemplateName=Sub("${AWS::StackName}-${EnvRole}"), LaunchTemplateData=ASLaunchTemplateData( "LaunchTemplateData", UserDataApp=UserDataApp ), ) R_LaunchTemplate.LaunchTemplateData.NetworkInterfaces[0].Groups.extend( SecurityGroups ) # Import role specific cfn definition try: # Do not use role but direct cfg yaml configuration (ecs + cluster) cfn_envrole = f"cfn_{cfg.IBOX_ROLE_EX}" except Exception: cfn_envrole = f"cfn_{cfg.envrole}" cfn_envrole = cfn_envrole.replace("-", "_") if cfn_envrole in globals(): # Ex cfn_client_portal CfnRole = globals()[cfn_envrole]() CfmInitArgs.update(CfnRole) if cfg.use_cfn_init: R_LaunchTemplate.Metadata = cfm.Metadata( { "CloudFormationInitVersion": If( "CloudFormationInit", Ref("EnvStackVersion"), Ref("AWS::NoValue"), ) }, cfm.Init(InitConfigSets, **CfmInitArgs), cfm.Authentication( { "CfnS3Auth": cfm.AuthenticationBlock( type="S3", buckets=[ Sub(cfg.BucketNameAppRepository), Sub(cfg.BucketNameAppData), ], roleName=Ref("RoleInstance"), ) } ), ) add_obj(R_LaunchTemplate) Tags = asg.Tags() Tags.tags = Tags_List return Tags
def __init__(self): InitConfigSets = ASInitConfigSets() CfmInitArgs = {} IBoxEnvApp = [] Tags = [] UserDataApp = [] for n in cfg.Apps: name = f'Apps{n}' # Ex. Apps1 envname = f'EnvApp{n}Version' # Ex EnvApp1Version reponame = f'{name}RepoName' # Ex Apps1RepoName UserDataApp.extend(['#${%s}\n' % envname]) p_EnvAppVersion = Parameter(envname) p_EnvAppVersion.Description = f'Application {n} version' p_EnvAppVersion.AllowedPattern = '^[a-zA-Z0-9-_.]*$' p_AppsRepoName = Parameter(reponame) p_AppsRepoName.Description = ( f'App {n} Repo Name - empty for default based on env/role') p_AppsRepoName.AllowedPattern = '^[a-zA-Z0-9-_.]*$' # parameters add_obj([ p_EnvAppVersion, p_AppsRepoName, ]) # conditions add_obj({ name: And(Not(Equals(Ref(envname), '')), Not(get_condition('', 'equals', 'None', reponame))) }) InitConfigApps = ASInitConfigApps(name) CfmInitArgs[name] = InitConfigApps InitConfigAppsBuilAmi = ASInitConfigAppsBuildAmi(name) # AUTOSPOT - Let cfn-init always prepare instances on boot # CfmInitArgs[name + 'BuildAmi'] = InitConfigAppsBuilAmi CfmInitArgs[name] = InitConfigAppsBuilAmi IBoxEnvApp.extend([ f'export EnvApp{n}Version=', Ref(envname), "\n", f'export EnvRepo{n}Name=', get_endvalue(reponame), "\n", ]) InitConfigSetsApp = If(name, name, Ref('AWS::NoValue')) InitConfigSetsAppBuilAmi = If(name, f'{name}BuildAmi', Ref('AWS::NoValue')) IndexSERVICES = InitConfigSets.data['default'].index('SERVICES') InitConfigSets.data['default'].insert(IndexSERVICES, InitConfigSetsApp) # AUTOSPOT - Let cfn-init always prepare instances on boot # InitConfigSets.data['buildamifull'].append( # InitConfigSetsAppBuilAmi) InitConfigSets.data['buildamifull'].append(InitConfigSetsApp) Tags.append(asg.Tag(envname, Ref(envname), True)) # resources # FOR MULTIAPP CODEDEPLOY if len(cfg.Apps) > 1: r_DeploymentGroup = CDDeploymentGroup(f'DeploymentGroup{name}') r_DeploymentGroup.setup(index=n) add_obj(r_DeploymentGroup) # outputs Output_app = Output(envname) Output_app.Value = Ref(envname) add_obj(Output_app) Output_repo = Output(reponame) Output_repo.Value = get_endvalue(reponame) add_obj(Output_repo) InitConfigSetup = ASInitConfigSetup() InitConfigSetup.ibox_env_app = IBoxEnvApp InitConfigSetup.setup() InitConfigCodeDeploy = ASInitConfigCodeDeploy() CfmInitArgs['SETUP'] = InitConfigSetup CfmInitArgs['CWAGENT'] = ASInitConfigCloudWatchAgent('') if cfg.Apps: CfmInitArgs['CODEDEPLOY'] = InitConfigCodeDeploy CD_DeploymentGroup() # LoadBalancerClassic External if cfg.LoadBalancerClassicExternal: InitConfigELBExternal = ASInitConfigELBClassicExternal() CfmInitArgs['ELBWAITER'] = InitConfigELBExternal # LoadBalancerClassic Internal if cfg.LoadBalancerClassicInternal: InitConfigELBInternal = ASInitConfigELBClassicInternal() CfmInitArgs['ELBWAITER'] = InitConfigELBInternal # LoadBalancerApplication External if cfg.LoadBalancerApplicationExternal: InitConfigELBExternal = ASInitConfigELBApplicationExternal() CfmInitArgs['ELBWAITER'] = InitConfigELBExternal # LoadBalancerApplication Internal InitConfigELBInternal = ASInitConfigELBApplicationInternal() CfmInitArgs['ELBWAITER'] = InitConfigELBInternal SecurityGroups = SG_SecurityGroupsEC2().SecurityGroups # Resources R_LaunchConfiguration = ASLaunchConfiguration('LaunchConfiguration', UserDataApp=UserDataApp) R_LaunchConfiguration.SecurityGroups.extend(SecurityGroups) R_InstanceProfile = IAMInstanceProfile('InstanceProfile') # Import role specific cfn definition cfn_envrole = f'cfn_{cfg.classenvrole}' if cfn_envrole in globals(): # Ex cfn_client_portal CfnRole = globals()[cfn_envrole]() CfmInitArgs.update(CfnRole) R_LaunchConfiguration.Metadata = cfm.Metadata( { 'CloudFormationInitVersion': If( 'CloudFormationInit', Ref('EnvStackVersion'), Ref('AWS::NoValue'), ) }, cfm.Init(InitConfigSets, **CfmInitArgs), cfm.Authentication({ 'CfnS3Auth': cfm.AuthenticationBlock(type='S3', buckets=[ Sub(cfg.BucketAppRepository), Sub(cfg.BucketAppData) ], roleName=Ref('RoleInstance')) })) R_LaunchConfigurationSpot = ASLaunchConfiguration( 'LaunchConfigurationSpot', UserDataApp=UserDataApp, spot=True) R_LaunchConfigurationSpot.SecurityGroups = ( R_LaunchConfiguration.SecurityGroups) R_LaunchConfigurationSpot.SpotPrice = get_endvalue('SpotPrice') add_obj([ R_LaunchConfiguration, R_InstanceProfile, ]) if cfg.SpotASG: add_obj(R_LaunchConfigurationSpot) self.LaunchConfiguration = R_LaunchConfiguration self.Tags = Tags
def attach(self): """Attaches a bootstrapped Chef Node EC2 instance to an AWS CloudFormation template and returns the template. """ parameters = ec2_parameters.EC2Parameters(self.template) parameters.attach() resources = ec2_resources.EC2Resources(self.template) resources.attach() security_group = self.template.add_resource(ec2.SecurityGroup( 'SecurityGroup', GroupDescription='Allows SSH access from anywhere', SecurityGroupIngress=[ ec2.SecurityGroupRule( IpProtocol='tcp', FromPort=22, ToPort=22, CidrIp=Ref(self.template.parameters['SSHLocation']) ), ec2.SecurityGroupRule( IpProtocol='tcp', FromPort=80, ToPort=80, CidrIp='0.0.0.0/0' ), ec2.SecurityGroupRule( IpProtocol='tcp', FromPort=8080, ToPort=8080, CidrIp='0.0.0.0/0' ) ], VpcId=ImportValue("prod2-VPCID"), Tags=Tags( Name='{0}SecurityGroup'.format(EC2_INSTANCE_NAME) ) )) self.template.add_resource(ec2.Instance( EC2_INSTANCE_NAME, ImageId=If( 'IsCentos7', FindInMap( "AWSRegionArch2Centos7LinuxAMI", Ref("AWS::Region"), FindInMap("AWSInstanceType2Arch", Ref(self.template.parameters['InstanceType']), "Arch")), FindInMap( "AWSRegionArch2AmazonLinuxAMI", Ref("AWS::Region"), FindInMap("AWSInstanceType2Arch", Ref(self.template.parameters['InstanceType']), "Arch")) ), InstanceType=Ref(self.template.parameters['InstanceType']), KeyName=FindInMap('Region2KeyPair', Ref('AWS::Region'), 'key'), SecurityGroupIds=[Ref(security_group)], SubnetId=ImportValue("prod2-SubnetPublicAZ2"), IamInstanceProfile=Ref( self.template.resources['InstanceProfileResource']), UserData=Base64(Join('', [ If('IsCentos7', Join('\n', [ '#!/bin/bash ', 'sudo yum update -y ', 'sudo yum install -y vim ', 'sudo yum install -y epel-release ', 'sudo yum install -y awscli ', '# Install CFN-BootStrap ', ('/usr/bin/easy_install --script-dir /opt/aws/bin ' 'https://s3.amazonaws.com/cloudformation-examples/' 'aws-cfn-bootstrap-latest.tar.gz '), ('cp -v /usr/lib/python2*/site-packages/aws_cfn_' 'bootstrap*/init/redhat/cfn-hup /etc/init.d '), 'chmod +x /etc/init.d/cfn-hup ', ]), Join('\n', [ '#!/bin/bash -xe ', 'yum update -y ', '# Update CFN-BootStrap ', 'yum update -y aws-cfn-bootstrap', 'sudo yum install -y awslogs ', ])), Join('', [ '# Install the files and packages from the metadata\n' '/opt/aws/bin/cfn-init -v ', ' --stack ', Ref('AWS::StackName'), ' --resource ', EC2_INSTANCE_NAME, ' --configsets InstallAndRun', ' --region ', Ref('AWS::Region'), ' --role ', Ref(self.template.resources['RoleResource']), '\n', '# Signal the status from cfn-init\n', '/opt/aws/bin/cfn-signal -e $? ' ' --stack ', Ref('AWS::StackName'), ' --resource ', EC2_INSTANCE_NAME, ' --region ', Ref('AWS::Region'), ' --role ', Ref(self.template.resources['RoleResource']), '\n' ]), ] ) ), Metadata=cloudformation.Metadata( cloudformation.Init( cloudformation.InitConfigSets( InstallAndRun=['Install', 'InstallLogs', 'InstallChef', 'Configure'] ), Install=cloudformation.InitConfig( packages={ 'yum': { 'stress': [], 'docker': [] } }, files={ '/etc/cfn/cfn-hup.conf': { 'content': Join('\n', [ '[main]', 'stack={{stackid}}', 'region={{region}}', 'interval=1' ]), 'context': { 'stackid': Ref('AWS::StackId'), 'region': Ref('AWS::Region') }, 'mode': '000400', 'owner': 'root', 'group': 'root' }, '/etc/cfn/hooks.d/cfn-auto-reloader.conf': { 'content': Join('\n', [ '[cfn-auto-reloader-hook]', 'triggers=post.update', ('path=Resources.{{instance_name}}' '.Metadata' '.AWS::CloudFormation::Init'), ('action=/opt/aws/bin/cfn-init -v ' ' --stack {{stack_name}} ' ' --resource {{instance_name}} ' ' --configsets {{config_sets}} ' ' --region {{region}} '), 'runas={{run_as}}' ]), 'context': { 'instance_name': EC2_INSTANCE_NAME, 'stack_name': Ref('AWS::StackName'), 'region': Ref('AWS::Region'), 'config_sets': 'InstallAndRun', 'run_as': 'root' } } }, services={ 'sysvinit': { 'docker': { 'enabled': 'true', 'ensureRunning': 'true' }, 'cfn-hup': { 'enabled': 'true', 'ensureRunning': 'true' } } }, commands={ '01_test': { 'command': 'echo "$CFNTEST" > Install.txt', 'env': { 'CFNTEST': 'I come from Install.' }, 'cwd': '~' } } ), InstallLogs=cloudformation.InitConfig( files={ '/etc/awslogs/awslogs.conf': { 'content': Join('\n', [ '[general]', ('state_file= /var/awslogs/' 'state/agent-state'), '', '[/var/log/cloud-init.log]', 'file = /var/log/cloud-init.log', 'log_group_name = {{log_group_name}}', ('log_stream_name = ' '{instance_id}/cloud-init.log'), 'datetime_format = {{datetime_format}}', '', '[/var/log/cloud-init-output.log]', 'file = /var/log/cloud-init-output.log', 'log_group_name = {{log_group_name}}', ('log_stream_name = ' '{instance_id}/cloud-init-output.log'), 'datetime_format = {{datetime_format}}', '', '[/var/log/cfn-init.log]', 'file = /var/log/cfn-init.log', 'log_group_name = {{log_group_name}}', ('log_stream_name = ' '{instance_id}/cfn-init.log'), 'datetime_format = {{datetime_format}}', '', '[/var/log/cfn-hup.log]', 'file = /var/log/cfn-hup.log', 'log_group_name = {{log_group_name}}', ('log_stream_name = ' '{instance_id}/cfn-hup.log'), 'datetime_format = {{datetime_format}}', '', '[/var/log/cfn-wire.log]', 'file = /var/log/cfn-wire.log', 'log_group_name = {{log_group_name}}', ('log_stream_name = ' '{instance_id}/cfn-wire.log'), 'datetime_format = {{datetime_format}}', '', '[/var/log/httpd]', 'file = /var/log/httpd/*', 'log_group_name = {{log_group_name}}', ('log_stream_name = ' '{instance_id}/httpd'), 'datetime_format = {{datetime_format}}' ]), 'context': { 'log_group_name': Ref( self.template.resources[ 'LogGroupResource']), 'datetime_format': '%d/%b/%Y:%H:%M:%S' } }, '/etc/awslogs/awscli.conf': { 'content': Join('\n', [ '[plugins]', 'cwlogs = cwlogs', '[default]', 'region = {{region}}' ]), 'context': { 'region': Ref('AWS::Region') }, 'mode': '000444', 'owner': 'root', 'group': 'root' } }, commands={ '01_create_state_directory': { 'command' : 'mkdir -p /var/awslogs/state' }, '02_test': { 'command': 'echo "$CFNTEST" > InstallLogs.txt', 'env': { 'CFNTEST': 'I come from install_logs.' }, 'cwd': '~' }, '03_install_aws_logs_if_centos': { 'command': If('IsCentos7', Join('\n', [ ('curl https://s3.amazonaws.com/aws-' 'cloudwatch/downloads/latest/awslogs-' 'agent-setup.py -O'), Join('', [ 'sudo python ./awslogs-agent-setup.py', ' --configfile /etc/awslogs/awslogs', '.conf --non-interactive --region ', Ref('AWS::Region')]) ]), Join('', [ 'echo "not installing awslogs from ', 'from source"' ])) } }, services={ 'sysvinit': { 'awslogs': { 'enabled': 'true', 'ensureRunning': 'true', 'files': ['/etc/awslogs/awslogs.conf'] } } } ), InstallChef=cloudformation.InitConfig( commands={ '01_invoke_omnitruck_install': { 'command': ( 'curl -L ' 'https://omnitruck.chef.io/install.sh | ' 'bash' ), } }, files={ '/etc/chef/client.rb': { 'source': S3_CLIENT_RB, 'mode': '000600', 'owner': 'root', 'group': 'root', 'authentication': 'S3AccessCreds' }, '/etc/chef/jasondebolt-validator.pem': { 'source': S3_VALIDATOR_PEM, 'mode': '000600', 'owner': 'root', 'group': 'root', 'authentication': 'S3AccessCreds' }, '/etc/chef/first-run.json': { 'source': S3_FIRST_RUN, 'mode': '000600', 'owner': 'root', 'group': 'root', 'authentication': 'S3AccessCreds' } } ), Configure=cloudformation.InitConfig( commands={ '01_test': { 'command': 'echo "$CFNTEST" > Configure.txt', 'env': { 'CFNTEST': 'I come from Configure.' }, 'cwd': '~' }, '02_chef_bootstrap': { 'command': ( 'chef-client -j ' '/etc/chef/first-run.json' ) } } ) ), cloudformation.Authentication({ 'S3AccessCreds': cloudformation.AuthenticationBlock( type='S3', roleName=Ref(self.template.resources['RoleResource'])) }) ), Tags=Tags( Name=Ref('AWS::StackName'), env='ops' ) )) self.template.add_output(Output( 'PublicIp', Description='Public IP of the newly created EC2 instance', Value=GetAtt(EC2_INSTANCE_NAME, 'PublicIp') )) self.template.add_output(Output( 'LinuxType', Description='The linux type of the EC2 instance.', Value=If('IsCentos7', 'centos_7', 'amazon_linux') )) return self.template
def generate_stack_template(): template = Template() generate_description(template) generate_version(template) # ---Parameters------------------------------------------------------------ param_vpc_id = Parameter( 'VpcIdentifer', Description= 'The identity of the VPC (vpc-abcdwxyz) in which this stack shall be created.', Type='AWS::EC2::VPC::Id', ) template.add_parameter(param_vpc_id) param_vpc_cidr_block = Parameter( 'VpcCidrBlock', Description= 'The CIDR block of the VPC (w.x.y.z/n) in which this stack shall be created.', Type='String', Default='10.0.0.0/16') template.add_parameter(param_vpc_cidr_block) param_database_instance_subnet_id = Parameter( 'VpcSubnetIdentifer', Description= 'The identity of the private subnet (subnet-abcdwxyz) in which the database server shall be created.', Type='AWS::EC2::Subnet::Id', ) template.add_parameter(param_database_instance_subnet_id) param_keyname = Parameter( 'PemKeyName', Description= 'Name of an existing EC2 KeyPair file (.pem) to use to create EC2 instances', Type='AWS::EC2::KeyPair::KeyName') template.add_parameter(param_keyname) param_instance_type = Parameter( 'EC2InstanceType', Description= 'EC2 instance type, reference this parameter to insure consistency', Type='String', Default= 't2.medium', # Prices from (2015-12-03) (Windows, us-west (North CA)) AllowedValues=[ # Source : https://aws.amazon.com/ec2/pricing/ 't2.small', # $0.044/hour 't2.micro', # $0.022/hour 't2.medium', # $0.088/hour 't2.large', # $0.166/hour 'm3.medium', # $0.140/hour 'm3.large', # $0.28/hour 'c4.large' # $0.221/hour ], ConstraintDescription='Must be a valid EC2 instance type') template.add_parameter(param_instance_type) param_s3_bucket = Parameter( 'S3Bucket', Description='The bucket in which applicable content can be found.', Type='String', Default='author-it-deployment-test-us-east-1') template.add_parameter(param_s3_bucket) param_s3_key = Parameter( 'S3Key', Description= 'The key within the bucket in which relevant files are located.', Type='String', Default='source/database/postgresql/single') template.add_parameter(param_s3_key) param_database_admin_password = Parameter( 'PostgresAdminPassword', Description='The password to be used by user postgres.', Type='String', NoEcho=True) template.add_parameter(param_database_admin_password) #---Mappings--------------------------------------------------------------- mapping_environment_attribute_map = template.add_mapping( 'EnvironmentAttributeMap', { 'ap-southeast-1': { 'DatabaseServerAmi': 'ami-1ddc0b7e' }, 'ap-southeast-2': { 'DatabaseServerAmi': 'ami-0c95b86f' }, 'us-east-1': { 'DatabaseServerAmi': 'ami-a4827dc9' }, 'us-west-1': { 'DatabaseServerAmi': 'ami-f5f41398' } }) # ---Resources------------------------------------------------------------- ref_stack_id = Ref('AWS::StackId') ref_region = Ref('AWS::Region') ref_stack_name = Ref('AWS::StackName') path_database_admin_script = 'usr/ec2-user/postgresql/set_admin_password.sql' name_database_server_wait_handle = 'DatabaseServerWaitHandle' cmd_postgresql_initdb = dict(command='service postgresql-95 initdb') cmd_start_postgresql_service = dict(command='service postgresql-95 start') cmd_set_postgres_user_password = dict(command='psql -U postgres -f %s' % path_database_admin_script) cmd_start_postgresql_on_startup = dict(command='chkconfig postgresql on') cmd_signal_success = dict(command='cfn-signal --exit-code $?') # Create an instance of AWS::IAM::Role for the instance. # This allows: # - Access to S3 bucket content. # - Stack updates resource_instance_role = template.add_resource( iam.Role('InstanceRole', AssumeRolePolicyDocument=Policy(Statement=[ Statement(Action=[AssumeRole], Effect=Allow, Principal=Principal('Service', ['ec2.amazonaws.com'])) ]), Path='/')) # Create the S3 policy and attach it to the role. template.add_resource( iam.PolicyType( 'InstanceS3DownloadPolicy', PolicyName='S3Download', PolicyDocument={ 'Statement': [{ 'Effect': 'Allow', 'Action': ['s3:GetObject'], 'Resource': Join('', ['arn:aws:s3:::', Ref(param_s3_bucket), '/*']) }, { 'Effect': 'Allow', 'Action': ['cloudformation:DescribeStacks', 'ec2:DescribeInstances'], 'Resource': '*' }] }, Roles=[Ref(resource_instance_role)])) # Create the CloudFormation stack update policy and attach it to the role. template.add_resource( iam.PolicyType('InstanceStackUpdatePolicy', PolicyName='StackUpdate', PolicyDocument={ 'Statement': [{ "Effect": "Allow", "Action": "Update:*", "Resource": "*" }] }, Roles=[Ref(resource_instance_role)])) # Create the AWS::IAM::InstanceProfile from the role for reference in the # database server instance definition. resource_instance_profile = template.add_resource( iam.InstanceProfile('InstanceProfile', Path='/', Roles=[Ref(resource_instance_role)])) # Create a security group for the postgresql instance. # This must be internal to the VPC only. name_security_group_database = 'VpcDatabaseSecurityGroup' resource_database_security_group = ec2.SecurityGroup( name_security_group_database, GroupDescription=Join( ' ', ['Security group for VPC database', Ref(param_vpc_id)]), Tags=Tags(Name=name_security_group_database), VpcId=Ref(param_vpc_id)) template.add_resource(resource_database_security_group) template.add_output( Output('SecurityGroupForDatabase', Description='Security group created for database in VPC.', Value=Ref(resource_database_security_group))) # Add ingress rule from VPC to database security group for database traffic. database_port = 5432 ssh_port = 22 template.add_resource( ec2.SecurityGroupIngress('DatabaseSecurityGroupDatabaseIngress', CidrIp=Ref(param_vpc_cidr_block), FromPort=str(database_port), GroupId=Ref(resource_database_security_group), IpProtocol='tcp', ToPort=str(database_port))) # Add ingress rule from VPC to database security group for ssh traffic. ssh_port = 22 template.add_resource( ec2.SecurityGroupIngress('DatabaseSecurityGroupSshIngress', CidrIp=Ref(param_vpc_cidr_block), FromPort=str(ssh_port), GroupId=Ref(resource_database_security_group), IpProtocol='tcp', ToPort=str(ssh_port))) # Create the metadata for the database instance. name_database_server = 'DatabaseServer' database_instance_metadata = cloudformation.Metadata( cloudformation.Init({ 'config': cloudformation.InitConfig( packages={ 'rpm': { 'postgresql': 'https://download.postgresql.org/pub/repos/yum/9.5/redhat/rhel-6-x86_64/pgdg-ami201503-95-9.5-2.noarch.rpm' }, 'yum': { 'postgresql95': [], 'postgresql95-libs': [], 'postgresql95-server': [], 'postgresql95-devel': [], 'postgresql95-contrib': [], 'postgresql95-docs': [] } }, files=cloudformation.InitFiles({ # cfn-hup.conf initialization '/etc/cfn/cfn-hup.conf': cloudformation.InitFile(content=Join( '', [ '[main]\n', 'stack=', ref_stack_id, '\n', 'region=', ref_region, '\n', 'interval=2', '\n', 'verbose=true', '\n' ]), mode='000400', owner='root', group='root'), # cfn-auto-reloader.conf initialization '/etc/cfn/cfn-auto-reloader.conf': cloudformation.InitFile( content=Join( '', [ '[cfn-auto-reloader-hook]\n', 'triggers=post.update\n', 'path=Resources.%s.Metadata.AWS::CloudFormation::Init\n' % name_database_server, 'action=cfn-init.exe ', ' --verbose ' ' --stack ', ref_stack_name, ' --resource %s ' % name_database_server, # resource that defines the Metadata ' --region ', ref_region, '\n' ]), mode='000400', owner='root', group='root'), # # pg_hba.conf retrieval from S3 '/var/lib/pgsql9/data/pg_hba.conf': cloudformation.InitFile( source=Join( '/', [ # Join('', ['https://s3-', ref_region, '.', 'amazonaws.com']), 'https://s3.amazonaws.com', Ref(param_s3_bucket), Ref(param_s3_key), 'conf' 'pg_hba.conf' ]), mode='000400', owner='root', group='root'), # postgresql.conf retrieval from S3 '/var/lib/pgsql9/data/postgresql.conf': cloudformation.InitFile( source=Join( '/', [ #Join('', ['https://s3-', ref_region, '.', 'amazonaws.com']), 'https://s3.amazonaws.com', Ref(param_s3_bucket), Ref(param_s3_key), 'conf' 'postgresql.conf' ]), mode='000400', owner='root', group='root'), # pg_ident.conf retrieval from S3 '/var/lib/pgsql9/data/pg_ident.conf': cloudformation.InitFile( source=Join( '/', [ #Join('', ['https://s3-', ref_region, '.', 'amazonaws.com']), 'https://s3.amazonaws.com', Ref(param_s3_bucket), Ref(param_s3_key), 'conf' 'pg_ident.conf' ]), mode='000400', owner='root', group='root'), # script to set postgresql admin password. # (admin user = '******') path_database_admin_script: cloudformation.InitFile( source=Join('', [ 'ALTER USER postgres WITH PASSWORD ', Ref(param_database_admin_password), ';', '\n' ])) }), commands={ '10-postgresql_initdb': cmd_postgresql_initdb, '20-start_postgresql_service': cmd_start_postgresql_service, '30-set-postgres-user-password': cmd_set_postgres_user_password, '40-start-postgresql-on-startup': cmd_start_postgresql_on_startup, #'99-signal-success': cmd_signal_success }, services=dict(sysvinit=cloudformation.InitServices({ # start cfn-hup service - # required for CloudFormation stack update 'cfn-hup': cloudformation.InitService( enabled=True, ensureRunning=True, files=[ '/etc/cfn/cfn-hup.conf', '/etc/cfn/hooks.d/cfn-auto-reloader.conf' ]), # start postgresql service 'postgresql-9.5': cloudformation.InitService(enabled=True, ensureRunning=True), # Disable sendmail service - not required. 'sendmail': cloudformation.InitService(enabled=False, ensureRunning=False) }))) }), cloudformation.Authentication({ 'S3AccessCredentials': cloudformation.AuthenticationBlock( buckets=[Ref(param_s3_bucket)], roleName=Ref(resource_instance_role), type='S3') })) # Add a wait handle to receive the completion signal. #resource_database_server_wait_handle = template.add_resource( # cloudformation.WaitConditionHandle( # name_database_server_wait_handle # ) # ) #template.add_resource( # cloudformation.WaitCondition( # 'DatabaseServerWaitCondition', # DependsOn=name_database_server, # Handle=Ref(resource_database_server_wait_handle), # Timeout=300, # ) #) resource_database_server = ec2.Instance( name_database_server, DependsOn=name_security_group_database, IamInstanceProfile=Ref(resource_instance_profile), Metadata=database_instance_metadata, ImageId=FindInMap('EnvironmentAttributeMap', ref_region, 'DatabaseServerAmi'), InstanceType=Ref(param_instance_type), KeyName=Ref(param_keyname), SecurityGroupIds=[Ref(resource_database_security_group)], SubnetId=Ref(param_database_instance_subnet_id), Tags=Tags(Name=name_database_server, VPC=Ref(param_vpc_id)), UserData=Base64( Join('', [ '#!/bin/bash -xe\n', 'yum update -y aws-cfn-bootstrap\n', '/opt/aws/bin/cfn-init --verbose ', ' --stack ', ref_stack_name, ' --resource DatabaseServer ', ' --region ', ref_region, '\n', '/opt/aws/bin/cfn-signal --exit-code $? ', ' --stack ', ref_stack_name, ' --resource ', name_database_server, '\n' ]))) template.add_resource(resource_database_server) template.add_output( Output('DatabaseServer', Description='PostgreSQL single instance database server', Value=Ref(resource_database_server))) return template
def generate_new_instance(counter): # Create base StorReduce instance instance = ec2.Instance(BASE_NAME + str(counter)) instance.DependsOn = [ elasticLB.title, SrrBucket.title, StorReduceHostProfile.title, AllInternalAccessSecurityGroup.title ] instance.ImageId = FindInMap("AWSAMIRegion", Ref("AWS::Region"), "AMI") instance.IamInstanceProfile = Ref(StorReduceHostProfile) # instance.AvailabilityZone = Select("0", Ref(AvailabilityZonesParam)) instance.InstanceType = Ref(InstanceTypeParam) instance.KeyName = Ref(KeyPairNameParam) #instance.SecurityGroupIds = Ref(SecurityGroupIdsParam) instance.SecurityGroupIds = [Ref(AllInternalAccessSecurityGroup)] instance.SubnetId = Select("0", Ref(PrivateSubnetsToSpanParam)) instance.UserData = Base64( Join("", [ """ #!/bin/bash -xe /opt/aws/bin/cfn-init -v --stack """, Ref("AWS::StackName"), " --resource " + instance.title + " --region ", Ref("AWS::Region"), "\n", "/opt/aws/bin/cfn-signal -e $? --stack ", Ref("AWS::StackName"), " --resource " + instance.title + " --region ", Ref("AWS::Region") ])) instance.Metadata = cloudformation.Metadata( cloudformation.Authentication({ "S3AccessCreds": cloudformation.AuthenticationBlock( type="S3", roleName=Ref(StorReduceHostRole), #Ref(HostRoleParam), buckets=[Ref(QSS3BucketNameParam)]) }), cloudformation.Init({ "config": cloudformation.InitConfig( files=cloudformation.InitFiles({ "/home/ec2-user/init-srr.sh": cloudformation.InitFile(source=Sub( "https://${" + QSS3BucketNameParam.title + "}.${QSS3Region}.amazonaws.com/${" + QSS3KeyPrefixParam.title + "}scripts/init-srr.sh", **{ "QSS3Region": If("GovCloudCondition", "s3-us-gov-west-1", "s3") }), mode="000550", owner="root", group="root") }), commands={ "init-srr": { "command": Join("", [ "/home/ec2-user/init-srr.sh \"", Ref(BucketNameParam), "\" \'", Ref(StorReduceLicenseParam), "\' ", "\'", Ref(StorReducePasswordParam), "\' ", "\'", Ref(ShardsNumParam), "\' ", "\'", Ref(ReplicaShardsNumParam), "\' ", "\"", Ref(StorReduceHostNameParam), "\" ", "\"", GetAtt(elasticLB, "DNSName"), "\" ", "\"", Ref(elasticLB), "\" ", "\"", Ref("AWS::Region"), "\" ", "\"", GetAtt("Eth0", "PrimaryPrivateIpAddress"), "\" ", "\"", Ref(NumSRRHostsParam), "\"" ]) } }) })) instance.Tags = [{"Key": "Name", "Value": "StorReduce-QS-Base-Host"}] instance.CreationPolicy = CreationPolicy(ResourceSignal=ResourceSignal( Timeout='PT15M')) return instance
""" #!/bin/bash -xe /opt/aws/bin/cfn-init -v --stack """, Ref("AWS::StackName"), " --resource " + monitor_instance.title + " --region ", Ref("AWS::Region"), "\n", "/opt/aws/bin/cfn-signal -e $? --stack ", Ref("AWS::StackName"), " --resource " + monitor_instance.title + " --region ", Ref("AWS::Region") ])) monitor_instance.Metadata = cloudformation.Metadata( cloudformation.Authentication({ "S3AccessCreds": cloudformation.AuthenticationBlock( type="S3", roleName=Ref(StorReduceHostRole), #Ref(HostRoleParam), buckets=[Ref(QSS3BucketNameParam)]) }), cloudformation.Init({ "config": cloudformation.InitConfig( files=cloudformation.InitFiles({ "/home/ec2-user/monitor-srr.sh": cloudformation.InitFile(source=Sub( "https://${" + QSS3BucketNameParam.title + "}.${QSS3Region}.amazonaws.com/${" + QSS3KeyPrefixParam.title + "}scripts/monitor-srr.sh", **{ "QSS3Region": If("GovCloudCondition", "s3-us-gov-west-1", "s3") }),
InstanceProfile( "EC2InstanceProfile", Roles=[Ref(EC2Role)], Path="/", ) ) # Create an autoscaling group LaunchConfig = t.add_resource(LaunchConfiguration( "LaunchConfig", Metadata=cloudformation.Metadata( cloudformation.Authentication({ "DeployUserAuth": cloudformation.AuthenticationBlock( type="S3", roleName=Ref(EC2Role), buckets=[Ref(DeployBucket)] ) }), cloudformation.Init({ "config": cloudformation.InitConfig( files=cloudformation.InitFiles({ "/home/ec2-user/init-cron.sh": cloudformation.InitFile( source=Join('', [ "http://", Ref(DeployBucket), ".s3.", Ref("AWS::Region"), ".amazonaws.com/scripts/init-cron.sh" ]), mode="000550", owner="root", group="root",