def configure_for_follower(instance, counter):
subnet_index = counter % NUM_AZS
if counter == 2:
instance.DependsOn = "MonitorInstance"
else:
instance.DependsOn = BASE_NAME + str(counter - 1) #base_instance.title
instance.SubnetId = Select(str(subnet_index),
Ref(PrivateSubnetsToSpanParam))
# instance.AvailabilityZone = Select(str(subnet_index), Ref(AvailabilityZonesParam))
instance.Metadata = cloudformation.Metadata(
cloudformation.Authentication({
"S3AccessCreds":
cloudformation.AuthenticationBlock(
type="S3",
roleName=Ref(StorReduceHostRole), #Ref(HostRoleParam),
buckets=[Ref(QSS3BucketNameParam)])
}),
cloudformation.Init({
"config":
cloudformation.InitConfig(
files=cloudformation.InitFiles({
"/home/ec2-user/connect-srr.sh":
cloudformation.InitFile(source=Sub(
"https://${" + QSS3BucketNameParam.title +
"}.${QSS3Region}.amazonaws.com/${" +
QSS3KeyPrefixParam.title + "}scripts/connect-srr.sh",
**{
"QSS3Region":
If("GovCloudCondition", "s3-us-gov-west-1", "s3")
}),
mode="000550",
owner="root",
group="root")
}),
commands={
"connect-srr": {
"command":
Join("", [
"/home/ec2-user/connect-srr.sh \"",
GetAtt(base_instance, "PrivateDnsName"), "\" \'",
Ref(StorReducePasswordParam), "\' ", "\"",
Ref(ShardsNumParam), "\" ", "\"",
Ref(ReplicaShardsNumParam), "\" ", "\"",
Ref(elasticLB), "\" ", "\"",
Ref("AWS::Region"), "\" ", "\"",
GetAtt("Eth0", "PrimaryPrivateIpAddress"), "\" ",
"\"",
Ref(NumSRRHostsParam), "\""
])
}
})
}))
instance.Tags = [{"Key": "Name", "Value": "StorReduce-QS-Host"}]
def Meta(name, private_ip):
return cfn.Metadata(
cfn.Authentication({
"default": cfn.AuthenticationBlock(type="S3", roleName=Ref(InstanceRole), buckets=[ Ref(Bucket) ])
}),
cfn.Init(
cfn.InitConfigSets(default = ['SetupHost','SetupWebsite']),
SetupHost = cfn.InitConfig(
files = {
"/etc/hostname":{
"content": Join(".",[ name, Ref(HostedZone) ])
},
"/root/set-hostname.sh":{
"content": Join("",[
"#!/bin/bash\n",
"hostname --file /etc/hostname\n",
"h=$(cat /etc/hostname)\n",
"sed -i s/HOSTNAME=.*/HOSTNAME=$h/g /etc/sysconfig/network\n",
"echo ", private_ip , " $h >>/etc/hosts \n",
"service network restart"
]),
"mode": "755"
},
},
commands={
"1-set-hostname": {
"command": "/root/set-hostname.sh "
}
}
),
SetupWebsite = cfn.InitConfig(
packages = {
"yum" : { 'httpd' : []}
},
sources = {
"/var/www/html/": Join("", [ "https://", Ref(Bucket), ".s3.amazonaws.com/3dstreetartindia.zip" ])
},
services = {
"sysvinit" : {
"httpd" : { "enabled" : "true", "ensureRunning" : "true" },
}
}
)
))
"sysvinit":
cloudformation.InitServices({
"rsyslog":
cloudformation.InitService(
enabled=True,
ensureRunning=True,
files=["/etc/rsyslog.d/20-somethin.conf"],
)
})
},
)
}),
cloudformation.Authentication({
"DeployUserAuth":
cloudformation.AuthenticationBlock(
type="S3",
accessKeyId=Ref(DeployUserAccessKey),
secretKey=Ref(DeployUserSecretKey),
)
}),
),
UserData=Base64(
Join(
"",
[
"#!/bin/bash\n",
"cfn-signal -e 0",
" --resource AutoscalingGroup",
" --stack ",
Ref("AWS::StackName"),
" --region ",
Ref("AWS::Region"),
def AS_LaunchTemplate():
cfg.use_cfn_init = True
InitConfigSets = ASInitConfigSets()
CfmInitArgs = {}
IBoxEnvApp = []
Tags_List = []
UserDataApp = []
for n in cfg.Apps:
name = f"Apps{n}" # Ex. Apps1
envname = f"EnvApp{n}Version" # Ex EnvApp1Version
reponame = f"{name}RepoName" # Ex Apps1RepoName
UserDataApp.extend(["#${%s}\n" % envname])
p_EnvAppVersion = Parameter(
envname,
Description=f"Application {n} version",
AllowedPattern="^[a-zA-Z0-9-_.]*$",
)
p_AppsRepoName = Parameter(
reponame,
Description=f"App {n} Repo Name - empty for default based on env/role",
AllowedPattern="^[a-zA-Z0-9-_.]*$",
)
# parameters
add_obj(
[
p_EnvAppVersion,
p_AppsRepoName,
]
)
# conditions
add_obj(
{
name: And(
Not(Equals(Ref(envname), "")),
Not(get_condition("", "equals", "None", reponame)),
)
}
)
InitConfigApps = ASInitConfigApps(name)
CfmInitArgs[name] = InitConfigApps
InitConfigAppsBuilAmi = ASInitConfigAppsBuildAmi(name)
# AUTOSPOT - Let cfn-init always prepare instances on boot
# CfmInitArgs[name + 'BuildAmi'] = InitConfigAppsBuilAmi
CfmInitArgs[name] = InitConfigAppsBuilAmi
IBoxEnvApp.extend(
[
f"export EnvApp{n}Version=",
Ref(envname),
"\n",
f"export EnvRepo{n}Name=",
get_endvalue(reponame),
"\n",
]
)
InitConfigSetsApp = If(name, name, Ref("AWS::NoValue"))
InitConfigSetsAppBuilAmi = If(name, f"{name}BuildAmi", Ref("AWS::NoValue"))
IndexSERVICES = InitConfigSets.data["default"].index("SERVICES")
InitConfigSets.data["default"].insert(IndexSERVICES, InitConfigSetsApp)
# AUTOSPOT - Let cfn-init always prepare instances on boot
# InitConfigSets.data['buildamifull'].append(
# InitConfigSetsAppBuilAmi)
InitConfigSets.data["buildamifull"].append(InitConfigSetsApp)
Tags_List.append(asg.Tag(envname, Ref(envname), True))
# outputs
Output_app = Output(envname, Value=Ref(envname))
Output_repo = Output(reponame, Value=get_endvalue(reponame))
add_obj([Output_app, Output_repo])
InitConfigSetup = ASInitConfigSetup()
InitConfigSetup.ibox_env_app = IBoxEnvApp
InitConfigSetup.setup()
InitConfigCodeDeploy = ASInitConfigCodeDeploy()
CfmInitArgs["SETUP"] = InitConfigSetup
CfmInitArgs["CWAGENT"] = ASInitConfigCloudWatchAgent("")
if cfg.CodeDeploy:
CfmInitArgs["CODEDEPLOY"] = InitConfigCodeDeploy
if not getattr(cfg, "IBOX_LAUNCH_TEMPLATE_NO_WAIT_ELB_HEALTH", False):
for lb in cfg.LoadBalancer:
# LoadBalancerClassic
if cfg.LoadBalancerType == "Classic":
InitConfigELB = ASInitConfigELBClassic(scheme=lb)
CfmInitArgs["ELBWAITER"] = InitConfigELB
# LoadBalancerApplication
if cfg.LoadBalancerType == "Application":
InitConfigELB = ASInitConfigELBApplication(scheme=lb)
CfmInitArgs["ELBWAITER"] = InitConfigELB
# LoadBalancerNetwork
if cfg.LoadBalancerType == "Network":
for k in cfg.Listeners:
InitConfigELB = ASInitConfigELBApplication(
scheme=f"TargetGroupListeners{k}{lb}"
)
CfmInitArgs["ELBWAITER"] = InitConfigELB
if getattr(cfg, "IBOX_LAUNCH_TEMPLATE_NO_SG_EXTRA", False):
SecurityGroups = []
else:
SecurityGroups = cfg.SecurityGroupsImport
# Resources
R_LaunchTemplate = ec2.LaunchTemplate(
"LaunchTemplate",
LaunchTemplateName=Sub("${AWS::StackName}-${EnvRole}"),
LaunchTemplateData=ASLaunchTemplateData(
"LaunchTemplateData", UserDataApp=UserDataApp
),
)
R_LaunchTemplate.LaunchTemplateData.NetworkInterfaces[0].Groups.extend(
SecurityGroups
)
# Import role specific cfn definition
try:
# Do not use role but direct cfg yaml configuration (ecs + cluster)
cfn_envrole = f"cfn_{cfg.IBOX_ROLE_EX}"
except Exception:
cfn_envrole = f"cfn_{cfg.envrole}"
cfn_envrole = cfn_envrole.replace("-", "_")
if cfn_envrole in globals(): # Ex cfn_client_portal
CfnRole = globals()[cfn_envrole]()
CfmInitArgs.update(CfnRole)
if cfg.use_cfn_init:
R_LaunchTemplate.Metadata = cfm.Metadata(
{
"CloudFormationInitVersion": If(
"CloudFormationInit",
Ref("EnvStackVersion"),
Ref("AWS::NoValue"),
)
},
cfm.Init(InitConfigSets, **CfmInitArgs),
cfm.Authentication(
{
"CfnS3Auth": cfm.AuthenticationBlock(
type="S3",
buckets=[
Sub(cfg.BucketNameAppRepository),
Sub(cfg.BucketNameAppData),
],
roleName=Ref("RoleInstance"),
)
}
),
)
add_obj(R_LaunchTemplate)
Tags = asg.Tags()
Tags.tags = Tags_List
return Tags
def __init__(self):
InitConfigSets = ASInitConfigSets()
CfmInitArgs = {}
IBoxEnvApp = []
Tags = []
UserDataApp = []
for n in cfg.Apps:
name = f'Apps{n}' # Ex. Apps1
envname = f'EnvApp{n}Version' # Ex EnvApp1Version
reponame = f'{name}RepoName' # Ex Apps1RepoName
UserDataApp.extend(['#${%s}\n' % envname])
p_EnvAppVersion = Parameter(envname)
p_EnvAppVersion.Description = f'Application {n} version'
p_EnvAppVersion.AllowedPattern = '^[a-zA-Z0-9-_.]*$'
p_AppsRepoName = Parameter(reponame)
p_AppsRepoName.Description = (
f'App {n} Repo Name - empty for default based on env/role')
p_AppsRepoName.AllowedPattern = '^[a-zA-Z0-9-_.]*$'
# parameters
add_obj([
p_EnvAppVersion,
p_AppsRepoName,
])
# conditions
add_obj({
name:
And(Not(Equals(Ref(envname), '')),
Not(get_condition('', 'equals', 'None', reponame)))
})
InitConfigApps = ASInitConfigApps(name)
CfmInitArgs[name] = InitConfigApps
InitConfigAppsBuilAmi = ASInitConfigAppsBuildAmi(name)
# AUTOSPOT - Let cfn-init always prepare instances on boot
# CfmInitArgs[name + 'BuildAmi'] = InitConfigAppsBuilAmi
CfmInitArgs[name] = InitConfigAppsBuilAmi
IBoxEnvApp.extend([
f'export EnvApp{n}Version=',
Ref(envname),
"\n",
f'export EnvRepo{n}Name=',
get_endvalue(reponame),
"\n",
])
InitConfigSetsApp = If(name, name, Ref('AWS::NoValue'))
InitConfigSetsAppBuilAmi = If(name, f'{name}BuildAmi',
Ref('AWS::NoValue'))
IndexSERVICES = InitConfigSets.data['default'].index('SERVICES')
InitConfigSets.data['default'].insert(IndexSERVICES,
InitConfigSetsApp)
# AUTOSPOT - Let cfn-init always prepare instances on boot
# InitConfigSets.data['buildamifull'].append(
# InitConfigSetsAppBuilAmi)
InitConfigSets.data['buildamifull'].append(InitConfigSetsApp)
Tags.append(asg.Tag(envname, Ref(envname), True))
# resources
# FOR MULTIAPP CODEDEPLOY
if len(cfg.Apps) > 1:
r_DeploymentGroup = CDDeploymentGroup(f'DeploymentGroup{name}')
r_DeploymentGroup.setup(index=n)
add_obj(r_DeploymentGroup)
# outputs
Output_app = Output(envname)
Output_app.Value = Ref(envname)
add_obj(Output_app)
Output_repo = Output(reponame)
Output_repo.Value = get_endvalue(reponame)
add_obj(Output_repo)
InitConfigSetup = ASInitConfigSetup()
InitConfigSetup.ibox_env_app = IBoxEnvApp
InitConfigSetup.setup()
InitConfigCodeDeploy = ASInitConfigCodeDeploy()
CfmInitArgs['SETUP'] = InitConfigSetup
CfmInitArgs['CWAGENT'] = ASInitConfigCloudWatchAgent('')
if cfg.Apps:
CfmInitArgs['CODEDEPLOY'] = InitConfigCodeDeploy
CD_DeploymentGroup()
# LoadBalancerClassic External
if cfg.LoadBalancerClassicExternal:
InitConfigELBExternal = ASInitConfigELBClassicExternal()
CfmInitArgs['ELBWAITER'] = InitConfigELBExternal
# LoadBalancerClassic Internal
if cfg.LoadBalancerClassicInternal:
InitConfigELBInternal = ASInitConfigELBClassicInternal()
CfmInitArgs['ELBWAITER'] = InitConfigELBInternal
# LoadBalancerApplication External
if cfg.LoadBalancerApplicationExternal:
InitConfigELBExternal = ASInitConfigELBApplicationExternal()
CfmInitArgs['ELBWAITER'] = InitConfigELBExternal
# LoadBalancerApplication Internal
InitConfigELBInternal = ASInitConfigELBApplicationInternal()
CfmInitArgs['ELBWAITER'] = InitConfigELBInternal
SecurityGroups = SG_SecurityGroupsEC2().SecurityGroups
# Resources
R_LaunchConfiguration = ASLaunchConfiguration('LaunchConfiguration',
UserDataApp=UserDataApp)
R_LaunchConfiguration.SecurityGroups.extend(SecurityGroups)
R_InstanceProfile = IAMInstanceProfile('InstanceProfile')
# Import role specific cfn definition
cfn_envrole = f'cfn_{cfg.classenvrole}'
if cfn_envrole in globals(): # Ex cfn_client_portal
CfnRole = globals()[cfn_envrole]()
CfmInitArgs.update(CfnRole)
R_LaunchConfiguration.Metadata = cfm.Metadata(
{
'CloudFormationInitVersion':
If(
'CloudFormationInit',
Ref('EnvStackVersion'),
Ref('AWS::NoValue'),
)
}, cfm.Init(InitConfigSets, **CfmInitArgs),
cfm.Authentication({
'CfnS3Auth':
cfm.AuthenticationBlock(type='S3',
buckets=[
Sub(cfg.BucketAppRepository),
Sub(cfg.BucketAppData)
],
roleName=Ref('RoleInstance'))
}))
R_LaunchConfigurationSpot = ASLaunchConfiguration(
'LaunchConfigurationSpot', UserDataApp=UserDataApp, spot=True)
R_LaunchConfigurationSpot.SecurityGroups = (
R_LaunchConfiguration.SecurityGroups)
R_LaunchConfigurationSpot.SpotPrice = get_endvalue('SpotPrice')
add_obj([
R_LaunchConfiguration,
R_InstanceProfile,
])
if cfg.SpotASG:
add_obj(R_LaunchConfigurationSpot)
self.LaunchConfiguration = R_LaunchConfiguration
self.Tags = Tags
def attach(self):
"""Attaches a bootstrapped Chef Node EC2 instance to an
AWS CloudFormation template and returns the template.
"""
parameters = ec2_parameters.EC2Parameters(self.template)
parameters.attach()
resources = ec2_resources.EC2Resources(self.template)
resources.attach()
security_group = self.template.add_resource(ec2.SecurityGroup(
'SecurityGroup',
GroupDescription='Allows SSH access from anywhere',
SecurityGroupIngress=[
ec2.SecurityGroupRule(
IpProtocol='tcp',
FromPort=22,
ToPort=22,
CidrIp=Ref(self.template.parameters['SSHLocation'])
),
ec2.SecurityGroupRule(
IpProtocol='tcp',
FromPort=80,
ToPort=80,
CidrIp='0.0.0.0/0'
),
ec2.SecurityGroupRule(
IpProtocol='tcp',
FromPort=8080,
ToPort=8080,
CidrIp='0.0.0.0/0'
)
],
VpcId=ImportValue("prod2-VPCID"),
Tags=Tags(
Name='{0}SecurityGroup'.format(EC2_INSTANCE_NAME)
)
))
self.template.add_resource(ec2.Instance(
EC2_INSTANCE_NAME,
ImageId=If(
'IsCentos7',
FindInMap(
"AWSRegionArch2Centos7LinuxAMI", Ref("AWS::Region"),
FindInMap("AWSInstanceType2Arch",
Ref(self.template.parameters['InstanceType']),
"Arch")),
FindInMap(
"AWSRegionArch2AmazonLinuxAMI", Ref("AWS::Region"),
FindInMap("AWSInstanceType2Arch",
Ref(self.template.parameters['InstanceType']),
"Arch"))
),
InstanceType=Ref(self.template.parameters['InstanceType']),
KeyName=FindInMap('Region2KeyPair', Ref('AWS::Region'), 'key'),
SecurityGroupIds=[Ref(security_group)],
SubnetId=ImportValue("prod2-SubnetPublicAZ2"),
IamInstanceProfile=Ref(
self.template.resources['InstanceProfileResource']),
UserData=Base64(Join('', [
If('IsCentos7',
Join('\n', [
'#!/bin/bash ',
'sudo yum update -y ',
'sudo yum install -y vim ',
'sudo yum install -y epel-release ',
'sudo yum install -y awscli ',
'# Install CFN-BootStrap ',
('/usr/bin/easy_install --script-dir /opt/aws/bin '
'https://s3.amazonaws.com/cloudformation-examples/'
'aws-cfn-bootstrap-latest.tar.gz '),
('cp -v /usr/lib/python2*/site-packages/aws_cfn_'
'bootstrap*/init/redhat/cfn-hup /etc/init.d '),
'chmod +x /etc/init.d/cfn-hup ',
]),
Join('\n', [
'#!/bin/bash -xe ',
'yum update -y ',
'# Update CFN-BootStrap ',
'yum update -y aws-cfn-bootstrap',
'sudo yum install -y awslogs ',
])),
Join('', [
'# Install the files and packages from the metadata\n'
'/opt/aws/bin/cfn-init -v ',
' --stack ', Ref('AWS::StackName'),
' --resource ', EC2_INSTANCE_NAME,
' --configsets InstallAndRun',
' --region ', Ref('AWS::Region'),
' --role ',
Ref(self.template.resources['RoleResource']),
'\n',
'# Signal the status from cfn-init\n',
'/opt/aws/bin/cfn-signal -e $? '
' --stack ', Ref('AWS::StackName'),
' --resource ', EC2_INSTANCE_NAME,
' --region ', Ref('AWS::Region'),
' --role ',
Ref(self.template.resources['RoleResource']),
'\n'
]),
]
)
),
Metadata=cloudformation.Metadata(
cloudformation.Init(
cloudformation.InitConfigSets(
InstallAndRun=['Install', 'InstallLogs', 'InstallChef',
'Configure']
),
Install=cloudformation.InitConfig(
packages={
'yum': {
'stress': [],
'docker': []
}
},
files={
'/etc/cfn/cfn-hup.conf': {
'content': Join('\n', [
'[main]',
'stack={{stackid}}',
'region={{region}}',
'interval=1'
]),
'context': {
'stackid': Ref('AWS::StackId'),
'region': Ref('AWS::Region')
},
'mode': '000400',
'owner': 'root',
'group': 'root'
},
'/etc/cfn/hooks.d/cfn-auto-reloader.conf': {
'content': Join('\n', [
'[cfn-auto-reloader-hook]',
'triggers=post.update',
('path=Resources.{{instance_name}}'
'.Metadata'
'.AWS::CloudFormation::Init'),
('action=/opt/aws/bin/cfn-init -v '
' --stack {{stack_name}} '
' --resource {{instance_name}} '
' --configsets {{config_sets}} '
' --region {{region}} '),
'runas={{run_as}}'
]),
'context': {
'instance_name': EC2_INSTANCE_NAME,
'stack_name': Ref('AWS::StackName'),
'region': Ref('AWS::Region'),
'config_sets': 'InstallAndRun',
'run_as': 'root'
}
}
},
services={
'sysvinit': {
'docker': {
'enabled': 'true',
'ensureRunning': 'true'
},
'cfn-hup': {
'enabled': 'true',
'ensureRunning': 'true'
}
}
},
commands={
'01_test': {
'command': 'echo "$CFNTEST" > Install.txt',
'env': {
'CFNTEST': 'I come from Install.'
},
'cwd': '~'
}
}
),
InstallLogs=cloudformation.InitConfig(
files={
'/etc/awslogs/awslogs.conf': {
'content': Join('\n', [
'[general]',
('state_file= /var/awslogs/'
'state/agent-state'),
'',
'[/var/log/cloud-init.log]',
'file = /var/log/cloud-init.log',
'log_group_name = {{log_group_name}}',
('log_stream_name = '
'{instance_id}/cloud-init.log'),
'datetime_format = {{datetime_format}}',
'',
'[/var/log/cloud-init-output.log]',
'file = /var/log/cloud-init-output.log',
'log_group_name = {{log_group_name}}',
('log_stream_name = '
'{instance_id}/cloud-init-output.log'),
'datetime_format = {{datetime_format}}',
'',
'[/var/log/cfn-init.log]',
'file = /var/log/cfn-init.log',
'log_group_name = {{log_group_name}}',
('log_stream_name = '
'{instance_id}/cfn-init.log'),
'datetime_format = {{datetime_format}}',
'',
'[/var/log/cfn-hup.log]',
'file = /var/log/cfn-hup.log',
'log_group_name = {{log_group_name}}',
('log_stream_name = '
'{instance_id}/cfn-hup.log'),
'datetime_format = {{datetime_format}}',
'',
'[/var/log/cfn-wire.log]',
'file = /var/log/cfn-wire.log',
'log_group_name = {{log_group_name}}',
('log_stream_name = '
'{instance_id}/cfn-wire.log'),
'datetime_format = {{datetime_format}}',
'',
'[/var/log/httpd]',
'file = /var/log/httpd/*',
'log_group_name = {{log_group_name}}',
('log_stream_name = '
'{instance_id}/httpd'),
'datetime_format = {{datetime_format}}'
]),
'context': {
'log_group_name': Ref(
self.template.resources[
'LogGroupResource']),
'datetime_format': '%d/%b/%Y:%H:%M:%S'
}
},
'/etc/awslogs/awscli.conf': {
'content': Join('\n', [
'[plugins]',
'cwlogs = cwlogs',
'[default]',
'region = {{region}}'
]),
'context': {
'region': Ref('AWS::Region')
},
'mode': '000444',
'owner': 'root',
'group': 'root'
}
},
commands={
'01_create_state_directory': {
'command' : 'mkdir -p /var/awslogs/state'
},
'02_test': {
'command': 'echo "$CFNTEST" > InstallLogs.txt',
'env': {
'CFNTEST': 'I come from install_logs.'
},
'cwd': '~'
},
'03_install_aws_logs_if_centos': {
'command': If('IsCentos7', Join('\n', [
('curl https://s3.amazonaws.com/aws-'
'cloudwatch/downloads/latest/awslogs-'
'agent-setup.py -O'),
Join('', [
'sudo python ./awslogs-agent-setup.py',
' --configfile /etc/awslogs/awslogs',
'.conf --non-interactive --region ',
Ref('AWS::Region')])
]), Join('', [
'echo "not installing awslogs from ',
'from source"'
]))
}
},
services={
'sysvinit': {
'awslogs': {
'enabled': 'true',
'ensureRunning': 'true',
'files': ['/etc/awslogs/awslogs.conf']
}
}
}
),
InstallChef=cloudformation.InitConfig(
commands={
'01_invoke_omnitruck_install': {
'command': (
'curl -L '
'https://omnitruck.chef.io/install.sh | '
'bash'
),
}
},
files={
'/etc/chef/client.rb': {
'source': S3_CLIENT_RB,
'mode': '000600',
'owner': 'root',
'group': 'root',
'authentication': 'S3AccessCreds'
},
'/etc/chef/jasondebolt-validator.pem': {
'source': S3_VALIDATOR_PEM,
'mode': '000600',
'owner': 'root',
'group': 'root',
'authentication': 'S3AccessCreds'
},
'/etc/chef/first-run.json': {
'source': S3_FIRST_RUN,
'mode': '000600',
'owner': 'root',
'group': 'root',
'authentication': 'S3AccessCreds'
}
}
),
Configure=cloudformation.InitConfig(
commands={
'01_test': {
'command': 'echo "$CFNTEST" > Configure.txt',
'env': {
'CFNTEST': 'I come from Configure.'
},
'cwd': '~'
},
'02_chef_bootstrap': {
'command': (
'chef-client -j '
'/etc/chef/first-run.json'
)
}
}
)
),
cloudformation.Authentication({
'S3AccessCreds': cloudformation.AuthenticationBlock(
type='S3',
roleName=Ref(self.template.resources['RoleResource']))
})
),
Tags=Tags(
Name=Ref('AWS::StackName'),
env='ops'
)
))
self.template.add_output(Output(
'PublicIp',
Description='Public IP of the newly created EC2 instance',
Value=GetAtt(EC2_INSTANCE_NAME, 'PublicIp')
))
self.template.add_output(Output(
'LinuxType',
Description='The linux type of the EC2 instance.',
Value=If('IsCentos7', 'centos_7', 'amazon_linux')
))
return self.template
def generate_stack_template():
template = Template()
generate_description(template)
generate_version(template)
# ---Parameters------------------------------------------------------------
param_vpc_id = Parameter(
'VpcIdentifer',
Description=
'The identity of the VPC (vpc-abcdwxyz) in which this stack shall be created.',
Type='AWS::EC2::VPC::Id',
)
template.add_parameter(param_vpc_id)
param_vpc_cidr_block = Parameter(
'VpcCidrBlock',
Description=
'The CIDR block of the VPC (w.x.y.z/n) in which this stack shall be created.',
Type='String',
Default='10.0.0.0/16')
template.add_parameter(param_vpc_cidr_block)
param_database_instance_subnet_id = Parameter(
'VpcSubnetIdentifer',
Description=
'The identity of the private subnet (subnet-abcdwxyz) in which the database server shall be created.',
Type='AWS::EC2::Subnet::Id',
)
template.add_parameter(param_database_instance_subnet_id)
param_keyname = Parameter(
'PemKeyName',
Description=
'Name of an existing EC2 KeyPair file (.pem) to use to create EC2 instances',
Type='AWS::EC2::KeyPair::KeyName')
template.add_parameter(param_keyname)
param_instance_type = Parameter(
'EC2InstanceType',
Description=
'EC2 instance type, reference this parameter to insure consistency',
Type='String',
Default=
't2.medium', # Prices from (2015-12-03) (Windows, us-west (North CA))
AllowedValues=[ # Source : https://aws.amazon.com/ec2/pricing/
't2.small', # $0.044/hour
't2.micro', # $0.022/hour
't2.medium', # $0.088/hour
't2.large', # $0.166/hour
'm3.medium', # $0.140/hour
'm3.large', # $0.28/hour
'c4.large' # $0.221/hour
],
ConstraintDescription='Must be a valid EC2 instance type')
template.add_parameter(param_instance_type)
param_s3_bucket = Parameter(
'S3Bucket',
Description='The bucket in which applicable content can be found.',
Type='String',
Default='author-it-deployment-test-us-east-1')
template.add_parameter(param_s3_bucket)
param_s3_key = Parameter(
'S3Key',
Description=
'The key within the bucket in which relevant files are located.',
Type='String',
Default='source/database/postgresql/single')
template.add_parameter(param_s3_key)
param_database_admin_password = Parameter(
'PostgresAdminPassword',
Description='The password to be used by user postgres.',
Type='String',
NoEcho=True)
template.add_parameter(param_database_admin_password)
#---Mappings---------------------------------------------------------------
mapping_environment_attribute_map = template.add_mapping(
'EnvironmentAttributeMap', {
'ap-southeast-1': {
'DatabaseServerAmi': 'ami-1ddc0b7e'
},
'ap-southeast-2': {
'DatabaseServerAmi': 'ami-0c95b86f'
},
'us-east-1': {
'DatabaseServerAmi': 'ami-a4827dc9'
},
'us-west-1': {
'DatabaseServerAmi': 'ami-f5f41398'
}
})
# ---Resources-------------------------------------------------------------
ref_stack_id = Ref('AWS::StackId')
ref_region = Ref('AWS::Region')
ref_stack_name = Ref('AWS::StackName')
path_database_admin_script = 'usr/ec2-user/postgresql/set_admin_password.sql'
name_database_server_wait_handle = 'DatabaseServerWaitHandle'
cmd_postgresql_initdb = dict(command='service postgresql-95 initdb')
cmd_start_postgresql_service = dict(command='service postgresql-95 start')
cmd_set_postgres_user_password = dict(command='psql -U postgres -f %s' %
path_database_admin_script)
cmd_start_postgresql_on_startup = dict(command='chkconfig postgresql on')
cmd_signal_success = dict(command='cfn-signal --exit-code $?')
# Create an instance of AWS::IAM::Role for the instance.
# This allows:
# - Access to S3 bucket content.
# - Stack updates
resource_instance_role = template.add_resource(
iam.Role('InstanceRole',
AssumeRolePolicyDocument=Policy(Statement=[
Statement(Action=[AssumeRole],
Effect=Allow,
Principal=Principal('Service',
['ec2.amazonaws.com']))
]),
Path='/'))
# Create the S3 policy and attach it to the role.
template.add_resource(
iam.PolicyType(
'InstanceS3DownloadPolicy',
PolicyName='S3Download',
PolicyDocument={
'Statement': [{
'Effect':
'Allow',
'Action': ['s3:GetObject'],
'Resource':
Join('', ['arn:aws:s3:::',
Ref(param_s3_bucket), '/*'])
}, {
'Effect':
'Allow',
'Action':
['cloudformation:DescribeStacks', 'ec2:DescribeInstances'],
'Resource':
'*'
}]
},
Roles=[Ref(resource_instance_role)]))
# Create the CloudFormation stack update policy and attach it to the role.
template.add_resource(
iam.PolicyType('InstanceStackUpdatePolicy',
PolicyName='StackUpdate',
PolicyDocument={
'Statement': [{
"Effect": "Allow",
"Action": "Update:*",
"Resource": "*"
}]
},
Roles=[Ref(resource_instance_role)]))
# Create the AWS::IAM::InstanceProfile from the role for reference in the
# database server instance definition.
resource_instance_profile = template.add_resource(
iam.InstanceProfile('InstanceProfile',
Path='/',
Roles=[Ref(resource_instance_role)]))
# Create a security group for the postgresql instance.
# This must be internal to the VPC only.
name_security_group_database = 'VpcDatabaseSecurityGroup'
resource_database_security_group = ec2.SecurityGroup(
name_security_group_database,
GroupDescription=Join(
' ', ['Security group for VPC database',
Ref(param_vpc_id)]),
Tags=Tags(Name=name_security_group_database),
VpcId=Ref(param_vpc_id))
template.add_resource(resource_database_security_group)
template.add_output(
Output('SecurityGroupForDatabase',
Description='Security group created for database in VPC.',
Value=Ref(resource_database_security_group)))
# Add ingress rule from VPC to database security group for database traffic.
database_port = 5432
ssh_port = 22
template.add_resource(
ec2.SecurityGroupIngress('DatabaseSecurityGroupDatabaseIngress',
CidrIp=Ref(param_vpc_cidr_block),
FromPort=str(database_port),
GroupId=Ref(resource_database_security_group),
IpProtocol='tcp',
ToPort=str(database_port)))
# Add ingress rule from VPC to database security group for ssh traffic.
ssh_port = 22
template.add_resource(
ec2.SecurityGroupIngress('DatabaseSecurityGroupSshIngress',
CidrIp=Ref(param_vpc_cidr_block),
FromPort=str(ssh_port),
GroupId=Ref(resource_database_security_group),
IpProtocol='tcp',
ToPort=str(ssh_port)))
# Create the metadata for the database instance.
name_database_server = 'DatabaseServer'
database_instance_metadata = cloudformation.Metadata(
cloudformation.Init({
'config':
cloudformation.InitConfig(
packages={
'rpm': {
'postgresql':
'https://download.postgresql.org/pub/repos/yum/9.5/redhat/rhel-6-x86_64/pgdg-ami201503-95-9.5-2.noarch.rpm'
},
'yum': {
'postgresql95': [],
'postgresql95-libs': [],
'postgresql95-server': [],
'postgresql95-devel': [],
'postgresql95-contrib': [],
'postgresql95-docs': []
}
},
files=cloudformation.InitFiles({
# cfn-hup.conf initialization
'/etc/cfn/cfn-hup.conf':
cloudformation.InitFile(content=Join(
'', [
'[main]\n', 'stack=', ref_stack_id, '\n',
'region=', ref_region, '\n', 'interval=2', '\n',
'verbose=true', '\n'
]),
mode='000400',
owner='root',
group='root'),
# cfn-auto-reloader.conf initialization
'/etc/cfn/cfn-auto-reloader.conf':
cloudformation.InitFile(
content=Join(
'',
[
'[cfn-auto-reloader-hook]\n',
'triggers=post.update\n',
'path=Resources.%s.Metadata.AWS::CloudFormation::Init\n'
% name_database_server,
'action=cfn-init.exe ',
' --verbose '
' --stack ',
ref_stack_name,
' --resource %s ' %
name_database_server, # resource that defines the Metadata
' --region ',
ref_region,
'\n'
]),
mode='000400',
owner='root',
group='root'),
#
# pg_hba.conf retrieval from S3
'/var/lib/pgsql9/data/pg_hba.conf':
cloudformation.InitFile(
source=Join(
'/',
[
# Join('', ['https://s3-', ref_region, '.', 'amazonaws.com']),
'https://s3.amazonaws.com',
Ref(param_s3_bucket),
Ref(param_s3_key),
'conf'
'pg_hba.conf'
]),
mode='000400',
owner='root',
group='root'),
# postgresql.conf retrieval from S3
'/var/lib/pgsql9/data/postgresql.conf':
cloudformation.InitFile(
source=Join(
'/',
[
#Join('', ['https://s3-', ref_region, '.', 'amazonaws.com']),
'https://s3.amazonaws.com',
Ref(param_s3_bucket),
Ref(param_s3_key),
'conf'
'postgresql.conf'
]),
mode='000400',
owner='root',
group='root'),
# pg_ident.conf retrieval from S3
'/var/lib/pgsql9/data/pg_ident.conf':
cloudformation.InitFile(
source=Join(
'/',
[
#Join('', ['https://s3-', ref_region, '.', 'amazonaws.com']),
'https://s3.amazonaws.com',
Ref(param_s3_bucket),
Ref(param_s3_key),
'conf'
'pg_ident.conf'
]),
mode='000400',
owner='root',
group='root'),
# script to set postgresql admin password.
# (admin user = '******')
path_database_admin_script:
cloudformation.InitFile(
source=Join('', [
'ALTER USER postgres WITH PASSWORD ',
Ref(param_database_admin_password), ';', '\n'
]))
}),
commands={
'10-postgresql_initdb':
cmd_postgresql_initdb,
'20-start_postgresql_service':
cmd_start_postgresql_service,
'30-set-postgres-user-password':
cmd_set_postgres_user_password,
'40-start-postgresql-on-startup':
cmd_start_postgresql_on_startup,
#'99-signal-success': cmd_signal_success
},
services=dict(sysvinit=cloudformation.InitServices({
# start cfn-hup service -
# required for CloudFormation stack update
'cfn-hup':
cloudformation.InitService(
enabled=True,
ensureRunning=True,
files=[
'/etc/cfn/cfn-hup.conf',
'/etc/cfn/hooks.d/cfn-auto-reloader.conf'
]),
# start postgresql service
'postgresql-9.5':
cloudformation.InitService(enabled=True,
ensureRunning=True),
# Disable sendmail service - not required.
'sendmail':
cloudformation.InitService(enabled=False,
ensureRunning=False)
})))
}),
cloudformation.Authentication({
'S3AccessCredentials':
cloudformation.AuthenticationBlock(
buckets=[Ref(param_s3_bucket)],
roleName=Ref(resource_instance_role),
type='S3')
}))
# Add a wait handle to receive the completion signal.
#resource_database_server_wait_handle = template.add_resource(
# cloudformation.WaitConditionHandle(
# name_database_server_wait_handle
# )
# )
#template.add_resource(
# cloudformation.WaitCondition(
# 'DatabaseServerWaitCondition',
# DependsOn=name_database_server,
# Handle=Ref(resource_database_server_wait_handle),
# Timeout=300,
# )
#)
resource_database_server = ec2.Instance(
name_database_server,
DependsOn=name_security_group_database,
IamInstanceProfile=Ref(resource_instance_profile),
Metadata=database_instance_metadata,
ImageId=FindInMap('EnvironmentAttributeMap', ref_region,
'DatabaseServerAmi'),
InstanceType=Ref(param_instance_type),
KeyName=Ref(param_keyname),
SecurityGroupIds=[Ref(resource_database_security_group)],
SubnetId=Ref(param_database_instance_subnet_id),
Tags=Tags(Name=name_database_server, VPC=Ref(param_vpc_id)),
UserData=Base64(
Join('', [
'#!/bin/bash -xe\n', 'yum update -y aws-cfn-bootstrap\n',
'/opt/aws/bin/cfn-init --verbose ', ' --stack ',
ref_stack_name, ' --resource DatabaseServer ', ' --region ',
ref_region, '\n', '/opt/aws/bin/cfn-signal --exit-code $? ',
' --stack ', ref_stack_name, ' --resource ',
name_database_server, '\n'
])))
template.add_resource(resource_database_server)
template.add_output(
Output('DatabaseServer',
Description='PostgreSQL single instance database server',
Value=Ref(resource_database_server)))
return template
def generate_new_instance(counter):
# Create base StorReduce instance
instance = ec2.Instance(BASE_NAME + str(counter))
instance.DependsOn = [
elasticLB.title, SrrBucket.title, StorReduceHostProfile.title,
AllInternalAccessSecurityGroup.title
]
instance.ImageId = FindInMap("AWSAMIRegion", Ref("AWS::Region"), "AMI")
instance.IamInstanceProfile = Ref(StorReduceHostProfile)
# instance.AvailabilityZone = Select("0", Ref(AvailabilityZonesParam))
instance.InstanceType = Ref(InstanceTypeParam)
instance.KeyName = Ref(KeyPairNameParam)
#instance.SecurityGroupIds = Ref(SecurityGroupIdsParam)
instance.SecurityGroupIds = [Ref(AllInternalAccessSecurityGroup)]
instance.SubnetId = Select("0", Ref(PrivateSubnetsToSpanParam))
instance.UserData = Base64(
Join("", [
"""
#!/bin/bash -xe
/opt/aws/bin/cfn-init -v --stack """,
Ref("AWS::StackName"),
" --resource " + instance.title + " --region ",
Ref("AWS::Region"), "\n", "/opt/aws/bin/cfn-signal -e $? --stack ",
Ref("AWS::StackName"),
" --resource " + instance.title + " --region ",
Ref("AWS::Region")
]))
instance.Metadata = cloudformation.Metadata(
cloudformation.Authentication({
"S3AccessCreds":
cloudformation.AuthenticationBlock(
type="S3",
roleName=Ref(StorReduceHostRole), #Ref(HostRoleParam),
buckets=[Ref(QSS3BucketNameParam)])
}),
cloudformation.Init({
"config":
cloudformation.InitConfig(
files=cloudformation.InitFiles({
"/home/ec2-user/init-srr.sh":
cloudformation.InitFile(source=Sub(
"https://${" + QSS3BucketNameParam.title +
"}.${QSS3Region}.amazonaws.com/${" +
QSS3KeyPrefixParam.title + "}scripts/init-srr.sh", **{
"QSS3Region":
If("GovCloudCondition", "s3-us-gov-west-1", "s3")
}),
mode="000550",
owner="root",
group="root")
}),
commands={
"init-srr": {
"command":
Join("", [
"/home/ec2-user/init-srr.sh \"",
Ref(BucketNameParam), "\" \'",
Ref(StorReduceLicenseParam), "\' ", "\'",
Ref(StorReducePasswordParam), "\' ", "\'",
Ref(ShardsNumParam), "\' ", "\'",
Ref(ReplicaShardsNumParam), "\' ", "\"",
Ref(StorReduceHostNameParam), "\" ", "\"",
GetAtt(elasticLB, "DNSName"), "\" ", "\"",
Ref(elasticLB), "\" ", "\"",
Ref("AWS::Region"), "\" ", "\"",
GetAtt("Eth0", "PrimaryPrivateIpAddress"), "\" ",
"\"",
Ref(NumSRRHostsParam), "\""
])
}
})
}))
instance.Tags = [{"Key": "Name", "Value": "StorReduce-QS-Base-Host"}]
instance.CreationPolicy = CreationPolicy(ResourceSignal=ResourceSignal(
Timeout='PT15M'))
return instance
"""
#!/bin/bash -xe
/opt/aws/bin/cfn-init -v --stack """,
Ref("AWS::StackName"),
" --resource " + monitor_instance.title + " --region ",
Ref("AWS::Region"), "\n", "/opt/aws/bin/cfn-signal -e $? --stack ",
Ref("AWS::StackName"),
" --resource " + monitor_instance.title + " --region ",
Ref("AWS::Region")
]))
monitor_instance.Metadata = cloudformation.Metadata(
cloudformation.Authentication({
"S3AccessCreds":
cloudformation.AuthenticationBlock(
type="S3",
roleName=Ref(StorReduceHostRole), #Ref(HostRoleParam),
buckets=[Ref(QSS3BucketNameParam)])
}),
cloudformation.Init({
"config":
cloudformation.InitConfig(
files=cloudformation.InitFiles({
"/home/ec2-user/monitor-srr.sh":
cloudformation.InitFile(source=Sub(
"https://${" + QSS3BucketNameParam.title +
"}.${QSS3Region}.amazonaws.com/${" +
QSS3KeyPrefixParam.title + "}scripts/monitor-srr.sh", **{
"QSS3Region":
If("GovCloudCondition", "s3-us-gov-west-1", "s3")
}),
InstanceProfile(
"EC2InstanceProfile",
Roles=[Ref(EC2Role)],
Path="/",
)
)
# Create an autoscaling group
LaunchConfig = t.add_resource(LaunchConfiguration(
"LaunchConfig",
Metadata=cloudformation.Metadata(
cloudformation.Authentication({
"DeployUserAuth": cloudformation.AuthenticationBlock(
type="S3",
roleName=Ref(EC2Role),
buckets=[Ref(DeployBucket)]
)
}),
cloudformation.Init({
"config": cloudformation.InitConfig(
files=cloudformation.InitFiles({
"/home/ec2-user/init-cron.sh": cloudformation.InitFile(
source=Join('', [
"http://",
Ref(DeployBucket),
".s3.", Ref("AWS::Region"), ".amazonaws.com/scripts/init-cron.sh"
]),
mode="000550",
owner="root",
group="root",