def _fsx_factory(**kwargs):
# FSx stack
fsx_template = Template()
fsx_template.set_version()
fsx_template.set_description("Create FSx stack")
# Create security group. If using an existing file system
# It must be associated to a security group that allows inbound TCP traffic to port 988
fsx_sg = ec2.SecurityGroup(
"FSxSecurityGroup",
GroupDescription="SecurityGroup for testing existing FSx",
SecurityGroupIngress=[
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort="988",
ToPort="988",
CidrIp="0.0.0.0/0",
),
],
VpcId=vpc_stack.cfn_outputs["VpcId"],
)
fsx_filesystem = FileSystem(
SecurityGroupIds=[Ref(fsx_sg)], SubnetIds=[vpc_stack.cfn_outputs["PublicSubnetId"]], **kwargs
)
fsx_template.add_resource(fsx_sg)
fsx_template.add_resource(fsx_filesystem)
fsx_stack = CfnStack(
name=fsx_stack_name,
region=region,
template=fsx_template.to_json(),
)
cfn_stacks_factory.create_stack(fsx_stack)
return fsx_stack.cfn_resources[kwargs.get("title")]
def create_template():
template = Template(Description='DNS Validated ACM Certificate Example')
template.set_version()
certificate = template.add_resource(
certificatemanager.Certificate(
'ExampleCertificate',
ValidationMethod='DNS',
DomainName='test.example.com',
DomainValidationOptions=[
certificatemanager.DomainValidationOption(
DomainName='test.example.com',
HostedZoneId='Z2KZ5YTUFZNC7H')
],
Tags=[{
'Key': 'Name',
'Value': 'Example Certificate'
}]))
template.add_output(
Output('CertificateARN',
Value=Ref(certificate),
Description='The ARN of the example certificate'))
return template
def custom_security_group(vpc_stack, region, request, cfn_stacks_factory):
template = Template()
template.set_version("2010-09-09")
template.set_description("custom security group stack for testing additional_sg and vpc_security_group_id")
security_group = template.add_resource(
SecurityGroup(
"SecurityGroupResource",
GroupDescription="custom security group for testing additional_sg and vpc_security_group_id",
VpcId=vpc_stack.cfn_outputs["VpcId"],
)
)
template.add_resource(
SecurityGroupIngress(
"SecurityGroupIngressResource",
IpProtocol="-1",
FromPort=0,
ToPort=65535,
SourceSecurityGroupId=Ref(security_group),
GroupId=Ref(security_group),
)
)
stack = CfnStack(
name=generate_stack_name("integ-tests-custom-sg", request.config.getoption("stackname_suffix")),
region=region,
template=template.to_json(),
)
cfn_stacks_factory.create_stack(stack)
yield stack
if not request.config.getoption("no_delete"):
cfn_stacks_factory.delete_stack(stack.name, region)
def _write_file_into_efs(region, vpc_stack, efs_stack, request, key_name,
cfn_stacks_factory):
"""Write file stack contains a mount target and a instance to write a empty file with random name into the efs."""
write_file_template = Template()
write_file_template.set_version("2010-09-09")
write_file_template.set_description(
"Stack to write a file to the existing EFS")
default_security_group_id = get_default_vpc_security_group(
vpc_stack.cfn_outputs["VpcId"], region)
write_file_template.add_resource(
MountTarget(
"MountTargetResource",
FileSystemId=efs_stack.cfn_resources["FileSystemResource"],
SubnetId=vpc_stack.cfn_outputs["PublicSubnetId"],
SecurityGroups=[default_security_group_id],
))
random_file_name = random_alphanumeric()
user_data = (
"""
#cloud-config
package_update: true
package_upgrade: true
runcmd:
- yum install -y nfs-utils
- file_system_id_1=""" +
efs_stack.cfn_resources["FileSystemResource"] + """
- efs_mount_point_1=/mnt/efs/fs1
- mkdir -p "${!efs_mount_point_1}"
- mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport,_netdev """
+
""""${!file_system_id_1}.efs.${AWS::Region}.${AWS::URLSuffix}:/" "${!efs_mount_point_1}"
- touch ${!efs_mount_point_1}/""" + random_file_name + """
- umount ${!efs_mount_point_1}
- opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource InstanceToWriteEFS --region ${AWS::Region}
""")
write_file_template.add_resource(
Instance(
"InstanceToWriteEFS",
CreationPolicy={"ResourceSignal": {
"Timeout": "PT10M"
}},
ImageId=retrieve_latest_ami(region, "alinux2"),
InstanceType="c5.xlarge",
SubnetId=vpc_stack.cfn_outputs["PublicSubnetId"],
UserData=Base64(Sub(user_data)),
KeyName=key_name,
DependsOn=["MountTargetResource"],
))
write_file_stack = CfnStack(
name=generate_stack_name("integ-tests-efs-write-file",
request.config.getoption("stackname_suffix")),
region=region,
template=write_file_template.to_json(),
)
cfn_stacks_factory.create_stack(write_file_stack)
cfn_stacks_factory.delete_stack(write_file_stack.name, region)
return random_file_name
def template(config: Config, plugins: List[str]) -> Template:
t = Template()
t.set_version('2010-09-09')
for i in items(config, plugins):
if isinstance(i, Parameter):
t.add_parameter(i)
elif isinstance(i, Output):
t.add_output(i)
else:
t.add_resource(i)
return t
def _find_or_create_topic(self, context_name: str):
arn = f'arn:aws:sns:{self._region}:{self._account_id}:{self._topic_name(context_name)}'
try:
self._sns_client.get_topic_attributes(TopicArn=arn)
except ClientError:
template = Template()
template.set_version('2010-09-09')
template.add_resource(
Topic(self._topic_name(context_name),
TopicName=self._topic_name(context_name)))
self.info(f'Creating stack for context "{context_name}"')
self._create_stack(self._stack_name(context_name), template)
def _store_secret(secret):
template = Template()
template.set_version("2010-09-09")
template.set_description("stack to store a secret string")
template.add_resource(Secret("Secret", SecretString=secret))
stack = CfnStack(
name=secret_stack_name,
region=region,
template=template.to_json(),
)
cfn_stacks_factory.create_stack(stack)
return stack.cfn_resources["Secret"]
def create_autoscaling_stack(projectName):
cloudformationClient = boto3.client('cloudformation',
endpoint_url='http://localhost:4581',
region_name='us-west-2')
t = Template()
t.set_version('2010-09-09')
t.set_description(
"LocalStackTests Task two cloud formation template of AutoScalingGroup "
)
ssh_rule = create_security_group_rule("tcp", "22", "22", "0.0.0.0/0")
http_rule = create_security_group_rule(
"tcp", "80", "80", "0.0.0.0/0") # assume we are ruing on 80 port 80
ec2SecurityGroup = create_security_group_resource(
projectName + "EC2SecurityGroup", [ssh_rule, http_rule],
"EC2 Security Group")
launchConfigName = projectName + "LaunchConfiguration"
launchConfig = create_launch_configuration_resource(
launchConfigName, [Ref(ec2SecurityGroup)])
autoscaling = create_autoscaling_resource(projectName, launchConfig)
t.add_resource(ec2SecurityGroup)
t.add_resource(launchConfig)
t.add_resource(autoscaling)
stackName = projectName + 'Task2'
try:
task2_stack = cloudformationClient.create_stack(
StackName=stackName, TemplateBody=t.to_yaml())
except Exception as e:
# TODO: check other exceptions
pass
stackReady = wait_resource(cloudformationClient.describe_stacks,
check_cloudformation_stack_complete,
10,
StackName=stackName)
if stackReady:
res = cloudformationClient.describe_stacks(StackName=stackName)
return res['Stacks'][0]
else:
raise Exception(
"Fails to get recently created stack, try to wait for more time")
def bastion_instance(vpc_stack, cfn_stacks_factory, request, region, key_name):
"""Class to create bastion instance used to execute commands on cluster in private subnet."""
bastion_stack_name = utils.generate_stack_name(
"integ-tests-networking-bastion",
request.config.getoption("stackname_suffix"))
bastion_template = Template()
bastion_template.set_version()
bastion_template.set_description("Create Networking bastion stack")
bastion_sg = ec2.SecurityGroup(
"NetworkingTestBastionSG",
GroupDescription="SecurityGroup for Bastion",
SecurityGroupIngress=[
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort="22",
ToPort="22",
CidrIp="0.0.0.0/0",
),
],
VpcId=vpc_stack.cfn_outputs["VpcId"],
)
instance = ec2.Instance(
"NetworkingBastionInstance",
InstanceType="c5.xlarge",
ImageId=retrieve_latest_ami(region, "alinux2"),
KeyName=key_name,
SecurityGroupIds=[Ref(bastion_sg)],
SubnetId=vpc_stack.cfn_outputs["PublicSubnetId"],
)
bastion_template.add_resource(bastion_sg)
bastion_template.add_resource(instance)
bastion_template.add_output(
Output("BastionIP",
Value=GetAtt(instance, "PublicIp"),
Description="The Bastion Public IP"))
bastion_stack = CfnStack(
name=bastion_stack_name,
region=region,
template=bastion_template.to_json(),
)
cfn_stacks_factory.create_stack(bastion_stack)
bastion_ip = bastion_stack.cfn_outputs.get("BastionIP")
logging.info(f"Bastion_ip: {bastion_ip}")
yield f"ec2-user@{bastion_ip}"
if not request.config.getoption("no_delete"):
cfn_stacks_factory.delete_stack(bastion_stack_name, region)
def init_template(description=None):
"""Function to initialize the troposphere base template
:param description: Description used for the CFN
:type description: str
:returns: template
:rtype: Template
"""
if description is not None:
template = Template(description)
else:
template = Template(f"Template generated by ECS ComposeX")
template.set_metadata({"GeneratedOn": DATE})
template.set_version()
return template
def existing_eip(region, request, cfn_stacks_factory):
template = Template()
template.set_version("2010-09-09")
template.set_description("EIP stack for testing existing EIP")
template.add_resource(EIP("ElasticIP", Domain="vpc"))
stack = CfnStack(
name=generate_stack_name("integ-tests-eip",
request.config.getoption("stackname_suffix")),
region=region,
template=template.to_json(),
)
cfn_stacks_factory.create_stack(stack)
yield stack.cfn_resources["ElasticIP"]
if not request.config.getoption("no_delete"):
cfn_stacks_factory.delete_stack(stack.name, region)
def create_template():
template = Template()
template.set_description('This stack deploys the OTM communication stack')
template.set_version()
template.set_transform('AWS::Serverless-2016-10-31')
# DB
table = dynamodb_factory.add_dynamodb(template)
# ROLES
role_factory.add_role(template)
use_dynamodb_role = role_factory.add_use_dynamodb_role(template)
apigwlambda_role = role_factory.add_apigateway_role(template)
# LAMBDAS
add_record_lambda = lambda_factory.add_lambda(template, use_dynamodb_role,
"./src/lambdas/records/add",
"addRecordLambda", table)
get_record_lambda = lambda_factory.add_lambda(template, use_dynamodb_role,
"./src/lambdas/records/get",
"getRecordLambda", table)
remove_record_lambda = lambda_factory.add_lambda(
template, use_dynamodb_role, "./src/lambdas/records/remove",
"removeRecordLambda", table)
# API GATEWAY METHODS
rest_api = apigateway_factory.add_apigateway_to_lambda(template)
resource = apigateway_factory.add_resource(template, rest_api)
add_record_method = apigateway_factory.add_method_to_apigateway(
template, add_record_lambda, apigwlambda_role, rest_api, resource,
"GET")
get_record_method = apigateway_factory.add_method_to_apigateway(
template, get_record_lambda, apigwlambda_role, rest_api, resource,
"POST")
remove_record_method = apigateway_factory.add_method_to_apigateway(
template, remove_record_lambda, apigwlambda_role, rest_api, resource,
"DELETE")
apigateway_factory.add_deployment(
template, rest_api,
[add_record_method, get_record_method, remove_record_method])
bucket = s3_factory.add_bucket(template)
return template
def efs_stack(cfn_stacks_factory, request, region):
"""EFS stack contains a single efs resource."""
efs_template = Template()
efs_template.set_version("2010-09-09")
efs_template.set_description("EFS stack created for testing existing EFS")
efs_template.add_resource(FileSystem("FileSystemResource"))
stack = CfnStack(
name=generate_stack_name("integ-tests-efs",
request.config.getoption("stackname_suffix")),
region=region,
template=efs_template.to_json(),
)
cfn_stacks_factory.create_stack(stack)
yield stack
if not request.config.getoption("no_delete"):
cfn_stacks_factory.delete_stack(stack.name, region)
def placement_group_stack(cfn_stacks_factory, request, region):
"""Placement group stack contains a placement group."""
placement_group_template = Template()
placement_group_template.set_version()
placement_group_template.set_description(
"Placement group stack created for testing existing placement group")
placement_group_template.add_resource(
PlacementGroup("PlacementGroup", Strategy="cluster"))
stack = CfnStack(
name=generate_stack_name("integ-tests-placement-group",
request.config.getoption("stackname_suffix")),
region=region,
template=placement_group_template.to_json(),
)
cfn_stacks_factory.create_stack(stack)
yield stack
cfn_stacks_factory.delete_stack(stack.name, region)
def _bastion_factory():
"""Create bastion stack."""
bastion_template = Template()
bastion_template.set_version()
bastion_template.set_description("Create Networking bastion stack")
bastion_sg = ec2.SecurityGroup(
"NetworkingTestBastionSG",
GroupDescription="SecurityGroup for Bastion",
SecurityGroupIngress=[
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort="22",
ToPort="22",
CidrIp="0.0.0.0/0",
),
],
VpcId=vpc_stack.cfn_outputs["VpcId"],
)
bastion_instance = ec2.Instance(
"NetworkingBastionInstance",
InstanceType="c5.xlarge",
ImageId=retrieve_latest_ami(region, "alinux2"),
KeyName=key_name,
SecurityGroupIds=[Ref(bastion_sg)],
SubnetId=vpc_stack.cfn_outputs["PublicSubnetId"],
)
bastion_template.add_resource(bastion_sg)
bastion_template.add_resource(bastion_instance)
bastion_template.add_output(
Output("BastionIP",
Value=GetAtt(bastion_instance, "PublicIp"),
Description="The Bastion Public IP"))
bastion_stack = CfnStack(
name=bastion_stack_name,
region=region,
template=bastion_template.to_json(),
)
cfn_stacks_factory.create_stack(bastion_stack)
return bastion_stack.cfn_outputs.get("BastionIP")
def generate_template():
_template = request.get_json()
template = Template()
template.set_version('2010-09-09')
template.set_description(_template['description'])
params = _template['parameters']
params = generateParams(params)
for param in params:
template.add_parameter(param)
resources = _template['resources']
for resource in resources:
template.add_resource(dispatcher(resource))
return template.to_dict()
def create_hosted_zone():
hosted_zone_template = Template()
hosted_zone_template.set_version("2010-09-09")
hosted_zone_template.set_description(
"Hosted zone stack created for testing existing DNS")
hosted_zone_template.add_resource(
HostedZone(
"HostedZoneResource",
Name=domain_name,
VPCs=[
HostedZoneVPCs(VPCId=vpc_stack.cfn_outputs["VpcId"],
VPCRegion=region)
],
))
hosted_zone_stack = CfnStack(
name=hosted_zone_stack_name,
region=region,
template=hosted_zone_template.to_json(),
)
cfn_stacks_factory.create_stack(hosted_zone_stack)
return hosted_zone_stack.cfn_resources[
"HostedZoneResource"], domain_name
def init_template(description=None):
"""Function to initialize the troposphere base template
:param description: Description used for the CFN
:type description: str
:returns: template
:rtype: Template
"""
if description is not None:
template = Template(description)
else:
template = Template("Template generated by ECS ComposeX")
template.set_metadata(
deepcopy({
"Type": "ComposeX",
"Properties": {
"Version": version,
"GeneratedOn": DATE
},
}))
template.set_version()
return template
def main(**kwargs):
""" Create the CFN template and either write to screen or update/create boto3. """
codebuild = Template()
codebuild.set_version('2010-09-09')
# Create a single CloudWatch Event role to allow codebuild:startBuild
cw_event_role = build_cw_cb_role(codebuild)
# TODO: There is a problem with the resource statement
#logging.info('Creating github role: {}', build_github_role(codebuild))
for job in config.sections():
if 'CodeBuild:' in job:
job_title = job.split(':')[1]
service_role = build_codebuild_role(
template=codebuild, project_name=job_title).to_dict()
# Pull the env out of the section, and use the snippet for the other values.
if 'snippet' in config[job]:
build_project(template=codebuild,
project_name=job_title,
section=config.get(job, 'snippet'),
service_role=service_role['Ref'],
raw_env=config.get(job, 'env'))
else:
build_project(template=codebuild,
project_name=job_title,
section=job,
service_role=service_role['Ref'])
build_cw_event(template=codebuild,
project_name=job_title,
role=cw_event_role)
with (open(args.output_dir + "/s2n_codebuild_projects.yml", 'w')) as fh:
fh.write(codebuild.to_yaml())
if args.dry_run:
logging.debug('Dry Run: wrote cfn file, but not calling AWS.')
else:
print('Boto functionality goes here.')
def main(**params):
try:
# Metadata
t = Template()
t.set_version("2010-09-09")
t.set_description("(SOCA) - Base template to deploy compute nodes.")
allow_anonymous_data_collection = params["MetricCollectionAnonymous"]
debug = False
mip_usage = False
instances_list = params["InstanceType"].split("+")
asg_lt = asg_LaunchTemplate()
ltd = LaunchTemplateData("NodeLaunchTemplateData")
mip = MixedInstancesPolicy()
stack_name = Ref("AWS::StackName")
# Begin LaunchTemplateData
UserData = '''#!/bin/bash -xe
export PATH=$PATH:/usr/local/bin
if [[ "''' + params['BaseOS'] + '''" == "centos7" ]] || [[ "''' + params['BaseOS'] + '''" == "rhel7" ]];
then
EASY_INSTALL=$(which easy_install-2.7)
$EASY_INSTALL pip
PIP=$(which pip2.7)
$PIP install awscli
yum install -y nfs-utils # enforce install of nfs-utils
else
# Upgrade awscli on ALI (do not use yum)
EASY_INSTALL=$(which easy_install-2.7)
$EASY_INSTALL pip
PIP=$(which pip)
$PIP install awscli --upgrade
fi
if [[ "''' + params['BaseOS'] + '''" == "amazonlinux2" ]];
then
/usr/sbin/update-motd --disable
fi
GET_INSTANCE_TYPE=$(curl http://169.254.169.254/latest/meta-data/instance-type)
echo export "SOCA_CONFIGURATION="''' + str(params['ClusterId']) + '''"" >> /etc/environment
echo export "SOCA_BASE_OS="''' + str(params['BaseOS']) + '''"" >> /etc/environment
echo export "SOCA_JOB_QUEUE="''' + str(params['JobQueue']) + '''"" >> /etc/environment
echo export "SOCA_JOB_OWNER="''' + str(params['JobOwner']) + '''"" >> /etc/environment
echo export "SOCA_JOB_NAME="''' + str(params['JobName']) + '''"" >> /etc/environment
echo export "SOCA_JOB_PROJECT="''' + str(params['JobProject']) + '''"" >> /etc/environment
echo export "SOCA_VERSION="''' + str(params['Version']) + '''"" >> /etc/environment
echo export "SOCA_JOB_EFA="''' + str(params['Efa']).lower() + '''"" >> /etc/environment
echo export "SOCA_JOB_ID="''' + str(params['JobId']) + '''"" >> /etc/environment
echo export "SOCA_SCRATCH_SIZE=''' + str(params['ScratchSize']) + '''" >> /etc/environment
echo export "SOCA_INSTALL_BUCKET="''' + str(params['S3Bucket']) + '''"" >> /etc/environment
echo export "SOCA_INSTALL_BUCKET_FOLDER="''' + str(params['S3InstallFolder']) + '''"" >> /etc/environment
echo export "SOCA_FSX_LUSTRE_BUCKET="''' + str(params['FSxLustreConfiguration']['fsx_lustre']).lower() + '''"" >> /etc/environment
echo export "SOCA_FSX_LUSTRE_DNS="''' + str(params['FSxLustreConfiguration']['existing_fsx']).lower() + '''"" >> /etc/environment
echo export "SOCA_INSTANCE_TYPE=$GET_INSTANCE_TYPE" >> /etc/environment
echo export "SOCA_INSTANCE_HYPERTHREADING="''' + str(params['ThreadsPerCore']).lower() + '''"" >> /etc/environment
echo export "SOCA_HOST_SYSTEM_LOG="/apps/soca/''' + str(params['ClusterId']) + '''/cluster_node_bootstrap/logs/''' + str(params['JobId']) + '''/$(hostname -s)"" >> /etc/environment
echo export "AWS_STACK_ID=${AWS::StackName}" >> /etc/environment
echo export "AWS_DEFAULT_REGION=${AWS::Region}" >> /etc/environment
source /etc/environment
AWS=$(which aws)
# Give yum permission to the user on this specific machine
echo "''' + params['JobOwner'] + ''' ALL=(ALL) /bin/yum" >> /etc/sudoers
mkdir -p /apps
mkdir -p /data
# Mount EFS
echo "''' + params['EFSDataDns'] + ''':/ /data nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0" >> /etc/fstab
echo "''' + params['EFSAppsDns'] + ''':/ /apps nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0" >> /etc/fstab
mount -a
# Configure NTP
yum remove -y ntp
yum install -y chrony
mv /etc/chrony.conf /etc/chrony.conf.original
echo -e """
# use the local instance NTP service, if available
server 169.254.169.123 prefer iburst minpoll 4 maxpoll 4
# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
# !!! [BEGIN] SOCA REQUIREMENT
# You will need to open UDP egress traffic on your security group if you want to enable public pool
#pool 2.amazon.pool.ntp.org iburst
# !!! [END] SOCA REQUIREMENT
# Record the rate at which the system clock gains/losses time.
driftfile /var/lib/chrony/drift
# Allow the system clock to be stepped in the first three updates
# if its offset is larger than 1 second.
makestep 1.0 3
# Specify file containing keys for NTP authentication.
keyfile /etc/chrony.keys
# Specify directory for log files.
logdir /var/log/chrony
# save data between restarts for fast re-load
dumponexit
dumpdir /var/run/chrony
""" > /etc/chrony.conf
systemctl enable chronyd
# Prepare Log folder
mkdir -p $SOCA_HOST_SYSTEM_LOG
echo "@reboot /bin/bash /apps/soca/$SOCA_CONFIGURATION/cluster_node_bootstrap/ComputeNodePostReboot.sh >> $SOCA_HOST_SYSTEM_LOG/ComputeNodePostInstall.log 2>&1" | crontab -
$AWS s3 cp s3://$SOCA_INSTALL_BUCKET/$SOCA_INSTALL_BUCKET_FOLDER/scripts/config.cfg /root/
/bin/bash /apps/soca/$SOCA_CONFIGURATION/cluster_node_bootstrap/ComputeNode.sh ''' + params['SchedulerHostname'] + ''' >> $SOCA_HOST_SYSTEM_LOG/ComputeNode.sh.log 2>&1'''
ltd.EbsOptimized = True
for instance in instances_list:
if "t2." in instance:
ltd.EbsOptimized = False
else:
# t2 does not support CpuOptions
ltd.CpuOptions = CpuOptions(
CoreCount=int(params["CoreCount"]),
ThreadsPerCore=1 if params["ThreadsPerCore"] is False else 2)
ltd.IamInstanceProfile = IamInstanceProfile(Arn=params["ComputeNodeInstanceProfileArn"])
ltd.KeyName = params["SSHKeyPair"]
ltd.ImageId = params["ImageId"]
if params["SpotPrice"] is not False and params["SpotAllocationCount"] is False:
ltd.InstanceMarketOptions = InstanceMarketOptions(
MarketType="spot",
SpotOptions=SpotOptions(
MaxPrice=Ref("AWS::NoValue") if params["SpotPrice"] == "auto" else str(params["SpotPrice"])
# auto -> cap at OD price
)
)
ltd.InstanceType = instances_list[0]
ltd.NetworkInterfaces = [NetworkInterfaces(
InterfaceType="efa" if params["Efa"] is not False else Ref("AWS::NoValue"),
DeleteOnTermination=True,
DeviceIndex=0,
Groups=[params["SecurityGroupId"]]
)]
ltd.UserData = Base64(Sub(UserData))
ltd.BlockDeviceMappings = [
BlockDeviceMapping(
DeviceName="/dev/xvda" if params["BaseOS"] == "amazonlinux2" else "/dev/sda1",
Ebs=EBSBlockDevice(
VolumeSize=params["RootSize"],
VolumeType="gp2",
DeleteOnTermination="false" if params["KeepEbs"] is True else "true",
Encrypted=True))
]
if int(params["ScratchSize"]) > 0:
ltd.BlockDeviceMappings.append(
BlockDeviceMapping(
DeviceName="/dev/xvdbx",
Ebs=EBSBlockDevice(
VolumeSize=params["ScratchSize"],
VolumeType="io1" if int(params["VolumeTypeIops"]) > 0 else "gp2",
Iops=params["VolumeTypeIops"] if int(params["VolumeTypeIops"]) > 0 else Ref("AWS::NoValue"),
DeleteOnTermination="false" if params["KeepEbs"] is True else "true",
Encrypted=True))
)
# End LaunchTemplateData
# Begin Launch Template Resource
lt = LaunchTemplate("NodeLaunchTemplate")
lt.LaunchTemplateName = params["ClusterId"] + "-" + str(params["JobId"])
lt.LaunchTemplateData = ltd
t.add_resource(lt)
# End Launch Template Resource
asg_lt.LaunchTemplateSpecification = LaunchTemplateSpecification(
LaunchTemplateId=Ref(lt),
Version=GetAtt(lt, "LatestVersionNumber")
)
asg_lt.Overrides = []
for instance in instances_list:
asg_lt.Overrides.append(LaunchTemplateOverrides(
InstanceType=instance))
# Begin InstancesDistribution
if params["SpotPrice"] is not False and \
params["SpotAllocationCount"] is not False and \
(params["DesiredCapacity"] - params["SpotAllocationCount"]) > 0:
mip_usage = True
idistribution = InstancesDistribution()
idistribution.OnDemandAllocationStrategy = "prioritized" # only supported value
idistribution.OnDemandBaseCapacity = params["DesiredCapacity"] - params["SpotAllocationCount"]
idistribution.OnDemandPercentageAboveBaseCapacity = "0" # force the other instances to be SPOT
idistribution.SpotMaxPrice = Ref("AWS::NoValue") if params["SpotPrice"] == "auto" else str(
params["SpotPrice"])
idistribution.SpotAllocationStrategy = params['SpotAllocationStrategy']
mip.InstancesDistribution = idistribution
# End MixedPolicyInstance
# Begin FSx for Lustre
if params["FSxLustreConfiguration"]["fsx_lustre"] is not False:
if params["FSxLustreConfiguration"]["existing_fsx"] is False:
fsx_lustre = FileSystem("FSxForLustre")
fsx_lustre.FileSystemType = "LUSTRE"
fsx_lustre.StorageCapacity = params["FSxLustreConfiguration"]["capacity"]
fsx_lustre.SecurityGroupIds = [params["SecurityGroupId"]]
fsx_lustre.SubnetIds = params["SubnetId"]
if params["FSxLustreConfiguration"]["s3_backend"] is not False:
fsx_lustre_configuration = LustreConfiguration()
fsx_lustre_configuration.ImportPath = params["FSxLustreConfiguration"]["import_path"] if params["FSxLustreConfiguration"]["import_path"] is not False else params["FSxLustreConfiguration"]["s3_backend"]
fsx_lustre_configuration.ExportPath = params["FSxLustreConfiguration"]["import_path"] if params["FSxLustreConfiguration"]["import_path"] is not False else params["FSxLustreConfiguration"]["s3_backend"] + "/" + params["ClusterId"] + "-fsxoutput/job-" + params["JobId"] + "/"
fsx_lustre.LustreConfiguration = fsx_lustre_configuration
fsx_lustre.Tags = base_Tags(
# False disable PropagateAtLaunch
Name=str(params["ClusterId"] + "-compute-job-" + params["JobId"]),
_soca_JobId=str(params["JobId"]),
_soca_JobName=str(params["JobName"]),
_soca_JobQueue=str(params["JobQueue"]),
_soca_StackId=stack_name,
_soca_JobOwner=str(params["JobOwner"]),
_soca_JobProject=str(params["JobProject"]),
_soca_KeepForever=str(params["KeepForever"]).lower(),
_soca_FSx="true",
_soca_ClusterId=str(params["ClusterId"]),
)
t.add_resource(fsx_lustre)
# End FSx For Lustre
# Begin AutoScalingGroup Resource
asg = AutoScalingGroup("AutoScalingComputeGroup")
asg.DependsOn = "NodeLaunchTemplate"
if mip_usage is True or instances_list.__len__() > 1:
mip.LaunchTemplate = asg_lt
asg.MixedInstancesPolicy = mip
else:
asg.LaunchTemplate = LaunchTemplateSpecification(
LaunchTemplateId=Ref(lt),
Version=GetAtt(lt, "LatestVersionNumber"))
asg.MinSize = int(params["DesiredCapacity"])
asg.MaxSize = int(params["DesiredCapacity"])
asg.VPCZoneIdentifier = params["SubnetId"]
if params["PlacementGroup"] is True:
pg = PlacementGroup("ComputeNodePlacementGroup")
pg.Strategy = "cluster"
t.add_resource(pg)
asg.PlacementGroup = Ref(pg)
asg.Tags = Tags(
Name=str(params["ClusterId"]) + "-compute-job-" + str(params["JobId"]),
_soca_JobId=str(params["JobId"]),
_soca_JobName=str(params["JobName"]),
_soca_JobQueue=str(params["JobQueue"]),
_soca_StackId=stack_name,
_soca_JobOwner=str(params["JobOwner"]),
_soca_JobProject=str(params["JobProject"]),
_soca_KeepForever=str(params["KeepForever"]).lower(),
_soca_ClusterId=str(params["ClusterId"]),
_soca_NodeType="soca-compute-node")
t.add_resource(asg)
# End AutoScalingGroup Resource
# Begin Custom Resource
# Change Mapping to No if you want to disable this
if allow_anonymous_data_collection is True:
metrics = CustomResourceSendAnonymousMetrics("SendAnonymousData")
metrics.ServiceToken = params["SolutionMetricLambda"]
metrics.DesiredCapacity = str(params["DesiredCapacity"])
metrics.InstanceType = str(params["InstanceType"])
metrics.Efa = str(params["Efa"])
metrics.ScratchSize = str(params["ScratchSize"])
metrics.RootSize = str(params["RootSize"])
metrics.SpotPrice = str(params["SpotPrice"])
metrics.BaseOS = str(params["BaseOS"])
metrics.StackUUID = str(params["StackUUID"])
metrics.KeepForever = str(params["KeepForever"])
metrics.FsxLustre = str(params["FSxLustreConfiguration"])
t.add_resource(metrics)
# End Custom Resource
if debug is True:
print(t.to_json())
# Tags must use "soca:<Key>" syntax
template_output = t.to_yaml().replace("_soca_", "soca:")
return {'success': True,
'output': template_output}
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
return {'success': False,
'output': 'cloudformation_builder.py: ' + (
str(e) + ': error :' + str(exc_type) + ' ' + str(fname) + ' ' + str(exc_tb.tb_lineno))}
class VPCTemplateBuilder:
"""Build troposphere CFN templates for VPC creation."""
def __init__(self, vpc_config, description="VPC built by VPCBuilder"):
self.__template = Template()
self.__template.set_version("2010-09-09")
self.__template.set_description(description)
self.__vpc_config = vpc_config
def build(self):
"""Build the template."""
self.__build_template()
return self.__template
def __build_template(self):
vpc = self.__build_vpc()
internet_gateway = self.__build_internet_gateway(vpc)
nat_gateway = None
subnet_refs = []
for subnet in self.__vpc_config.subnets:
subnet_ref = self.__build_subnet(subnet, vpc)
subnet_refs.append(subnet_ref)
if subnet.has_nat_gateway:
nat_gateway = self.__build_nat_gateway(subnet, subnet_ref)
for subnet, subnet_ref in zip(self.__vpc_config.subnets, subnet_refs):
self.__build_route_table(subnet, subnet_ref, vpc, internet_gateway, nat_gateway)
def __build_vpc(self):
vpc_config = self.__vpc_config
vpc = self.__template.add_resource(
VPC(
vpc_config.name,
CidrBlock=vpc_config.cidr,
EnableDnsSupport=vpc_config.enable_dns_support,
EnableDnsHostnames=vpc_config.enable_dns_hostnames,
Tags=vpc_config.tags,
)
)
self.__template.add_output(Output("VpcId", Value=Ref(vpc), Description="VPC Id"))
return vpc
def __build_internet_gateway(self, vpc: VPC):
internet_gateway = self.__template.add_resource(
InternetGateway("InternetGateway", Tags=Tags(Name=Ref("AWS::StackName"), Stack=Ref("AWS::StackId")))
)
self.__template.add_resource(
VPCGatewayAttachment("VPCGatewayAttachment", VpcId=Ref(vpc), InternetGatewayId=Ref(internet_gateway))
)
return internet_gateway
def __build_subnet(self, subnet_config: SubnetConfig, vpc: VPC):
subnet = Subnet(
subnet_config.name,
CidrBlock=subnet_config.cidr,
VpcId=Ref(vpc),
MapPublicIpOnLaunch=subnet_config.map_public_ip_on_launch,
Tags=subnet_config.tags(),
)
if subnet_config.availability_zone:
subnet.AvailabilityZone = subnet_config.availability_zone
self.__template.add_resource(subnet)
self.__template.add_output(Output(subnet_config.name + "Id", Value=Ref(subnet)))
return subnet
def __build_nat_gateway(self, subnet_config: SubnetConfig, subnet_ref: Subnet):
nat_eip = self.__template.add_resource(EIP("NatEIP" + subnet_config.name, Domain="vpc"))
return self.__template.add_resource(
NatGateway(
"NatGateway" + subnet_config.name,
AllocationId=GetAtt(nat_eip, "AllocationId"),
SubnetId=Ref(subnet_ref),
)
)
def __build_route_table(
self,
subnet_config: SubnetConfig,
subnet_ref: Subnet,
vpc: VPC,
internet_gateway: InternetGateway,
nat_gateway: NatGateway,
):
route_table = self.__template.add_resource(
RouteTable(
"RouteTable" + subnet_config.name,
VpcId=Ref(vpc),
Tags=Tags(Name=Sub("${AWS::StackName}_route_table_" + subnet_config.name), Stack=Ref("AWS::StackId")),
)
)
self.__template.add_resource(
SubnetRouteTableAssociation(
"RouteAssociation" + subnet_config.name, SubnetId=Ref(subnet_ref), RouteTableId=Ref(route_table)
)
)
if subnet_config.default_gateway == Gateways.INTERNET_GATEWAY:
self.__template.add_resource(
Route(
"DefaultRoute" + subnet_config.name,
RouteTableId=Ref(route_table),
DestinationCidrBlock="0.0.0.0/0",
GatewayId=Ref(internet_gateway),
)
)
elif subnet_config.default_gateway == Gateways.NAT_GATEWAY:
self.__template.add_resource(
Route(
"NatRoute" + subnet_config.name,
RouteTableId=Ref(route_table),
DestinationCidrBlock="0.0.0.0/0",
NatGatewayId=Ref(nat_gateway),
)
)
def build(ssh_keypair_name):
template = Template()
template.set_version("2010-09-09")
keyname_param = template.add_parameter(
Parameter(
"KeyName",
ConstraintDescription="must be the name of an existing EC2 KeyPair.",
Description="Name of an existing EC2 KeyPair to enable SSH access to \
the instance",
Type="AWS::EC2::KeyPair::KeyName",
Default=ssh_keypair_name,
)
)
sshlocation_param = template.add_parameter(
Parameter(
"SSHLocation",
Description=" The IP address range that can be used to SSH to the EC2 \
instances",
Type="String",
MinLength="9",
MaxLength="18",
Default="0.0.0.0/0",
AllowedPattern=r"(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})",
ConstraintDescription=("must be a valid IP CIDR range of the form x.x.x.x/x."),
)
)
instanceType_param = template.add_parameter(
Parameter(
"InstanceType",
Type="String",
Description="WebServer EC2 instance type",
Default="t3a.small",
AllowedValues=[
"t2.micro",
"t2.small",
"t2.medium",
"t3a.small",
"m3.medium",
"m3.large",
"m3.xlarge",
"m3.2xlarge",
"c3.large",
"c3.xlarge",
"c3.2xlarge",
"c3.4xlarge",
"c3.8xlarge",
"g2.2xlarge",
"r3.large",
"r3.xlarge",
"r3.2xlarge",
"r3.4xlarge",
"r3.8xlarge",
"i2.xlarge",
"i2.2xlarge",
"i2.4xlarge",
"i2.8xlarge",
"hi1.4xlarge",
"hs1.8xlarge",
"cr1.8xlarge",
"cc2.8xlarge",
],
ConstraintDescription="must be a valid EC2 instance type.",
)
)
template.add_mapping(
"AWSRegion2AMI",
{
"us-east-1": {"image": "ami-0d915a031cabac0e0"},
"us-east-2": {"image": "ami-0b97435028ca44fcc"},
"us-west-1": {"image": "ami-068d0753a46192935"},
"us-west-2": {"image": "ami-0c457f229774da543"},
"eu-west-1": {"image": "ami-046c6a0123bf94619"},
"eu-west-2": {"image": "ami-0dbe8ba0cd21ea12b"},
"eu-west-3": {"image": "ami-041bf9180061ce7ea"},
"eu-central-1": {"image": "ami-0f8184e6f30cc0c33"},
"eu-north-1": {"image": "ami-08dd1b893371bcaac"},
"ap-south-1": {"image": "ami-0ff23052091536db2"},
"ap-southeast-1": {"image": "ami-0527e82bae7c51958"},
"ap-southeast-2": {"image": "ami-0bae8773e653a32ec"},
"ap-northeast-1": {"image": "ami-060741a96307668be"},
"ap-northeast-2": {"image": "ami-0d991ac4f545a6b34"},
"sa-east-1": {"image": "ami-076f350d5a5ec448d"},
"ca-central-1": {"image": "ami-0071deaa12b66d1bf"},
},
)
vpc = template.add_resource(VPC("VPC", CidrBlock="10.0.0.0/16"))
subnet = template.add_resource(Subnet("Subnet", CidrBlock="10.0.0.0/24", VpcId=Ref(vpc)))
internet_gateway = template.add_resource(InternetGateway("InternetGateway"))
attach_gateway = template.add_resource(
VPCGatewayAttachment("AttachGateway", VpcId=Ref(vpc), InternetGatewayId=Ref(internet_gateway))
)
route_table = template.add_resource(RouteTable("RouteTable", VpcId=Ref(vpc)))
template.add_resource(
Route(
"Route",
DependsOn=attach_gateway,
GatewayId=Ref(internet_gateway),
DestinationCidrBlock="0.0.0.0/0",
RouteTableId=Ref(route_table),
)
)
template.add_resource(
SubnetRouteTableAssociation("SubnetRouteTableAssociation", SubnetId=Ref(subnet), RouteTableId=Ref(route_table),)
)
network_acl = template.add_resource(NetworkAcl("NetworkAcl", VpcId=Ref(vpc),))
template.add_resource(
NetworkAclEntry(
"InboundHTTPNetworkAclEntry",
NetworkAclId=Ref(network_acl),
RuleNumber="100",
Protocol="6",
PortRange=PortRange(To="80", From="80"),
Egress="false",
RuleAction="allow",
CidrBlock="0.0.0.0/0",
)
)
template.add_resource(
NetworkAclEntry(
"InboundSSHNetworkAclEntry",
NetworkAclId=Ref(network_acl),
RuleNumber="101",
Protocol="6",
PortRange=PortRange(To="22", From="22"),
Egress="false",
RuleAction="allow",
CidrBlock="0.0.0.0/0",
)
)
template.add_resource(
NetworkAclEntry(
"InboundResponsePortsNetworkAclEntry",
NetworkAclId=Ref(network_acl),
RuleNumber="102",
Protocol="6",
PortRange=PortRange(To="65535", From="1024"),
Egress="false",
RuleAction="allow",
CidrBlock="0.0.0.0/0",
)
)
template.add_resource(
NetworkAclEntry(
"OutBoundHTTPNetworkAclEntry",
NetworkAclId=Ref(network_acl),
RuleNumber="100",
Protocol="6",
PortRange=PortRange(To="80", From="80"),
Egress="true",
RuleAction="allow",
CidrBlock="0.0.0.0/0",
)
)
template.add_resource(
NetworkAclEntry(
"OutBoundHTTPSNetworkAclEntry",
NetworkAclId=Ref(network_acl),
RuleNumber="101",
Protocol="6",
PortRange=PortRange(To="443", From="443"),
Egress="true",
RuleAction="allow",
CidrBlock="0.0.0.0/0",
)
)
template.add_resource(
NetworkAclEntry(
"OutBoundResponsePortsNetworkAclEntry",
NetworkAclId=Ref(network_acl),
RuleNumber="102",
Protocol="6",
PortRange=PortRange(To="65535", From="1024"),
Egress="true",
RuleAction="allow",
CidrBlock="0.0.0.0/0",
)
)
template.add_resource(
SubnetNetworkAclAssociation("SubnetNetworkAclAssociation", SubnetId=Ref(subnet), NetworkAclId=Ref(network_acl),)
)
instance_security_group = template.add_resource(
SecurityGroup(
"InstanceSecurityGroup",
GroupDescription="Enable SSH access via port 22",
SecurityGroupIngress=[
SecurityGroupRule(IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp=Ref(sshlocation_param)),
SecurityGroupRule(IpProtocol="tcp", FromPort="80", ToPort="80", CidrIp="0.0.0.0/0"),
],
VpcId=Ref(vpc),
)
)
server_instance = template.add_resource(
Instance(
"ServerInstance",
ImageId=FindInMap("AWSRegion2AMI", Ref("AWS::Region"), "image"),
InstanceType=Ref(instanceType_param),
KeyName=Ref(keyname_param),
NetworkInterfaces=[
NetworkInterfaceProperty(
GroupSet=[Ref(instance_security_group)],
AssociatePublicIpAddress="true",
DeviceIndex="0",
DeleteOnTermination="true",
SubnetId=Ref(subnet),
)
],
)
)
template.add_output([Output("ServerIP", Value=GetAtt(server_instance, "PublicIp"))])
return template
from troposphere import (Template, GetAtt)
from troposphere.awslambda import Function, Code
from troposphere.cloudformation import AWSCustomObject
t = Template('CustomResourceXke')
t.set_version()
class CustomResourceXke(AWSCustomObject):
resource_type = "Custom::CustomResourceXke"
props = {
'ServiceToken': (str, True),
'Var1': (str, True),
'Var2': (str, True),
}
lambda_definition = """
const response = require('cfn-response');
exports.handler = async (event, context) => {
var physicalResourceId = 'find_a_uniq_id';
try {
switch (event.RequestType) {
case 'Create':
case 'Update':
case 'Delete':
console.info(event.ResourceProperties.Var1);
console.info(event.ResourceProperties.Var2);
break;
}
securityWorkerEgress = [
SecurityGroupClass("Input", "0.0.0.0/0", "-1"),
]
ipPrivateList = ""
for f in vpc.subnets:
if f.ec2 is not None:
if len(ipPrivateList) > 0:
ipPrivateList += " "
ipPrivateList += f.ec2.ip
template = Template(
"This template deploys a VPC, with a pair of public and private subnets spread across two Availability Zones. It deploys an Internet Gateway, with a default route on the public subnets. It deploys a pair of NAT Gateways (one in each AZ), and default routes for them in the private subnets."
)
template.set_version("2010-09-09")
########################## MAPPING #####################################################
template.add_mapping(
'RegionMap', {
"ap-south-1": {
"AMI": "ami-0937dcc711d38ef3f"
},
"eu-west-3": {
"AMI": "ami-0854d53ce963f69d8"
},
"eu-north-1": {
"AMI": "ami-6d27a913"
},
"eu-west-2": {
"AMI": "ami-0664a710233d7c148"
def main(**launch_parameters):
try:
t = Template()
t.set_version("2010-09-09")
t.set_description("(SOCA) - Base template to deploy DCV nodes")
allow_anonymous_data_collection = launch_parameters["DefaultMetricCollection"]
# Launch Actual Capacity
instance = ec2.Instance(str(launch_parameters["session_name"]))
instance.BlockDeviceMappings = [{'DeviceName': "/dev/xvda" if launch_parameters["base_os"] == "amazonlinux2" else "/dev/sda1",
'Ebs': {
'DeleteOnTermination': True,
'VolumeSize': 30 if launch_parameters["disk_size"] is False else int(launch_parameters["disk_size"]),
'VolumeType': 'gp3',
'Encrypted': True}
}]
instance.ImageId = launch_parameters["image_id"]
instance.SecurityGroupIds = [launch_parameters["security_group_id"]]
if launch_parameters["hibernate"] is True:
instance.HibernationOptions = ec2.HibernationOptions(Configured=True)
instance.InstanceType = launch_parameters["instance_type"]
instance.SubnetId = random.choice(launch_parameters["soca_private_subnets"]) if len(launch_parameters["soca_private_subnets"]) > 1 else launch_parameters["soca_private_subnets"][0]
instance.IamInstanceProfile = launch_parameters["ComputeNodeInstanceProfileArn"].split("instance-profile/")[-1]
instance.UserData = Base64(Sub((launch_parameters["user_data"])))
instance.Tags = base_Tags(
Name=str(launch_parameters["cluster_id"] + "-" + launch_parameters["session_name"] + "-" + launch_parameters["user"]),
_soca_JobName=str(launch_parameters["session_name"]),
_soca_JobOwner=str(launch_parameters["user"]),
_soca_NodeType="dcv",
_soca_JobProject="desktop",
_soca_DCVSupportHibernate=str(launch_parameters["hibernate"]).lower(),
_soca_ClusterId=str(launch_parameters["cluster_id"]),
_soca_DCVSessionUUID=str(launch_parameters["session_uuid"]),
_soca_DCVSystem=str(launch_parameters["base_os"]))
t.add_resource(instance)
# Begin Custom Resource
# Change Mapping to No if you want to disable this
if allow_anonymous_data_collection is True:
metrics = CustomResourceSendAnonymousMetrics("SendAnonymousData")
metrics.ServiceToken = launch_parameters["SolutionMetricLambda"]
metrics.DesiredCapacity = "1"
metrics.InstanceType = str(launch_parameters["instance_type"])
metrics.Efa = "false"
metrics.ScratchSize = "0"
metrics.RootSize = str(launch_parameters["disk_size"])
metrics.SpotPrice = "false"
metrics.BaseOS = str(launch_parameters["base_os"])
metrics.StackUUID = str(launch_parameters["session_uuid"])
metrics.KeepForever = "false"
metrics.FsxLustre = str({"fsx_lustre": "false", "existing_fsx": "false", "s3_backend": "false", "import_path": "false", "export_path": "false",
"deployment_type": "false", "per_unit_throughput": "false", "capacity": 1200})
metrics.TerminateWhenIdle = "false"
metrics.Dcv = "true"
t.add_resource(metrics)
# End Custom Resource
# Tags must use "soca:<Key>" syntax
template_output = t.to_yaml().replace("_soca_", "soca:")
return {'success': True,
'output': template_output}
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
return {'success': False,
'output': 'cloudformation_builder.py: ' + (str(e) + ': error :' + str(exc_type) + ' ' + str(fname) + ' ' + str(exc_tb.tb_lineno))}
def main():
t = Template()
t.set_description("test instance launch")
t.set_version("2010-09-09")
InstUserData = [
'#!/usr/bin/env bash\n',
'\n',
'set -x\n',
'\n',
'my_wait_handle="',
Ref('InstanceWaitHandle'),
'"\n',
'curl -X PUT -H \'Content-Type:\' --data-binary \'{ "Status" : "SUCCESS", "Reason" : "Instance launched", "UniqueId" : "launch001", "Data" : "Instance launched."}\' "${my_wait_handle}"',
'\n',
'\n',
]
EC2KeyName = t.add_parameter(
Parameter(
'EC2KeyName',
Type="AWS::EC2::KeyPair::KeyName",
Description=
"Name of an existing EC2 KeyPair to enable SSH access to the instance.",
ConstraintDescription="REQUIRED: Must be a valud EC2 key pair",
))
OperatingSystem = t.add_parameter(
Parameter('OperatingSystem',
Type="String",
Description="Operating System",
Default="centos7",
AllowedValues=[
"alinux2",
"centos7",
"rhel7",
],
ConstraintDescription="Must be: alinux2, centos7, rhel7"))
myInstanceType = t.add_parameter(
Parameter(
'MyInstanceType',
Type="String",
Description="Instance type",
Default="m5.2xlarge",
))
VpcId = t.add_parameter(
Parameter(
'VpcId',
Type="AWS::EC2::VPC::Id",
Description="VPC Id for this instance",
))
Subnet = t.add_parameter(
Parameter('Subnet',
Type="AWS::EC2::Subnet::Id",
Description="Subnet IDs"))
ExistingSecurityGroup = t.add_parameter(
Parameter(
'ExistingSecurityGroup',
Type="AWS::EC2::SecurityGroup::Id",
Description=
"OPTIONAL: Choose an existing Security Group ID, e.g. sg-abcd1234")
)
UsePublicIp = t.add_parameter(
Parameter(
'UsePublicIp',
Type="String",
Description="Should a public IP address be given to the instance",
Default="true",
ConstraintDescription="true/false",
AllowedValues=["true", "false"]))
SshAccessCidr = t.add_parameter(
Parameter(
'SshAccessCidr',
Type="String",
Description="CIDR Block for SSH access, default 127.0.0.1/32",
Default="127.0.0.1/32",
AllowedPattern=
"(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
ConstraintDescription="Must be a valid CIDR x.x.x.x/x"))
RootRole = t.add_resource(
iam.Role("RootRole",
AssumeRolePolicyDocument={
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": ["ec2.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}]
}))
SshSecurityGroup = t.add_resource(
SecurityGroup("SshSecurityGroup",
VpcId=Ref(VpcId),
GroupDescription="SSH Secuirty group",
SecurityGroupIngress=[
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort="22",
ToPort="22",
CidrIp=Ref(SshAccessCidr),
),
]))
RootInstanceProfile = t.add_resource(
InstanceProfile("RootInstanceProfile", Roles=[Ref(RootRole)]))
tags = Tags(Name=Ref("AWS::StackName"))
myInstance = t.add_resource(
ec2.Instance(
'MyInstance',
ImageId=FindInMap("AWSRegionAMI", Ref("AWS::Region"),
Ref(OperatingSystem)),
KeyName=Ref(EC2KeyName),
InstanceType=(Ref(myInstanceType)),
NetworkInterfaces=[
NetworkInterfaceProperty(
GroupSet=If(
"not_existing_sg", [Ref(SshSecurityGroup)],
[Ref(SshSecurityGroup),
Ref(ExistingSecurityGroup)]),
AssociatePublicIpAddress=Ref(UsePublicIp),
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(Subnet))
],
IamInstanceProfile=(Ref(RootInstanceProfile)),
UserData=Base64(Join('', InstUserData)),
))
t.add_mapping(
'AWSRegionAMI', {
"ap-northeast-1": {
"centos7": "ami-8e8847f1",
"rhel7": "ami-6b0d5f0d"
},
"ap-northeast-2": {
"centos7": "ami-bf9c36d1",
"rhel7": "ami-3eee4150"
},
"ap-south-1": {
"centos7": "ami-1780a878",
"rhel7": "ami-5b673c34"
},
"ap-southeast-1": {
"centos7": "ami-8e0205f2",
"rhel7": "ami-76144b0a"
},
"ap-southeast-2": {
"centos7": "ami-d8c21dba",
"rhel7": "ami-67589505"
},
"ca-central-1": {
"centos7": "ami-e802818c",
"rhel7": "ami-49f0762d"
},
"eu-central-1": {
"centos7": "ami-dd3c0f36",
"rhel7": "ami-c86c3f23"
},
"eu-west-1": {
"centos7": "ami-3548444c",
"rhel7": "ami-7c491f05"
},
"eu-west-2": {
"centos7": "ami-00846a67",
"rhel7": "ami-7c1bfd1b"
},
"eu-west-3": {
"centos7": "ami-262e9f5b",
"rhel7": "ami-5026902d"
},
"sa-east-1": {
"centos7": "ami-cb5803a7",
"rhel7": "ami-b0b7e3dc"
},
"us-east-1": {
"centos7": "ami-9887c6e7",
"rhel7": "ami-6871a115"
},
"us-east-2": {
"centos7": "ami-9c0638f9",
"rhel7": "ami-03291866"
},
"us-west-1": {
"centos7": "ami-4826c22b",
"rhel7": "ami-18726478"
},
"us-west-2": {
"centos7": "ami-3ecc8f46",
"rhel7": "ami-28e07e50"
}
})
t.add_condition("not_existing_sg", Equals(Ref(ExistingSecurityGroup), ""))
t.add_condition("Has_Public_Ip", Equals(Ref(UsePublicIp), "true"))
mywaithandle = t.add_resource(WaitConditionHandle('InstanceWaitHandle'))
mywaitcondition = t.add_resource(
WaitCondition("InstanceWaitCondition",
Handle=Ref(mywaithandle),
Timeout="1500",
DependsOn="MyInstance"))
t.add_output([
Output("InstanceID", Description="Instance ID", Value=Ref(myInstance))
])
t.add_output(
[Output("InstancePrivateIP", Value=GetAtt('MyInstance', 'PrivateIp'))])
t.add_output([
Output("InstancePublicIP",
Value=GetAtt('MyInstance', 'PublicIp'),
Condition="Has_Public_Ip")
])
##print(t.to_yaml())
print(t.to_json(indent=2))
class VPCTemplate(): #pylint: disable=too-many-public-methods
'''
Create VPC template
'''
def __init__(self):
self.cfn_template = Template()
def add_descriptions(self, descriptions):
'''
Add descriptions to template
'''
self.cfn_template.set_description(descriptions)
return self.cfn_template
def add_version(self, version):
'''
Add a version of the template file to template
'''
self.cfn_template.set_version(version)
return self.cfn_template
def add_parameters(self):
'''
Add parameters to generated template
'''
self.cfn_template.add_parameter(
Parameter(
'Environment',
Type='String',
))
self.cfn_template.add_parameter(
Parameter(
'VPCCIDRBlock',
Type='String',
Description=
"Enter valid CIDR block values for VPC. Default is 10.0.0.0/16",
MinLength='9',
MaxLength='18',
ConstraintDescription=
"must be a valid IP CIDR range of the form x.x.x.x/x.",
))
self.cfn_template.add_parameter(
Parameter(
'PublicSubnet1CIDRBlock',
Type='String',
Description=
"Enter valid CIDR block values for public subnet-1. i.e 10.0.0.0/24",
MinLength='9',
MaxLength='18',
ConstraintDescription=
"must be a valid IP CIDR range of the form x.x.x.x/x.",
))
self.cfn_template.add_parameter(
Parameter(
'PrivateSubnet1CIDRBlock',
Type='String',
Description=
"Enter valid CIDR block values for private subnet-1.i.e 10.0.1.0/24",
MinLength='9',
MaxLength='18',
ConstraintDescription=
"must be a valid IP CIDR range of the form x.x.x.x/x.",
))
self.cfn_template.add_parameter(
Parameter(
'PublicSubnet2CIDRBlock',
Type='String',
Description=
"Enter valid CIDR block values for public subnet-2. i.e 10.0.2.0/24",
MinLength='9',
MaxLength='18',
ConstraintDescription=
"must be a valid IP CIDR range of the form x.x.x.x/x.",
))
self.cfn_template.add_parameter(
Parameter(
'PrivateSubnet2CIDRBlock',
Type='String',
Description=
"Enter valid CIDR block values for private subnet-2. i.e 10.0.3.0/24",
MinLength='9',
MaxLength='18',
ConstraintDescription=
"must be a valid IP CIDR range of the form x.x.x.x/x.",
))
return self.cfn_template
def add_outputs(self):
'''
Add outputs to generated template
'''
self.cfn_template.add_output(
Output(
"NetVPC",
Description="The VPC ID",
Export=Export(Sub('${Environment}-VPCID')),
Value=Ref(constants.VPC),
))
self.cfn_template.add_output(
Output(
"PublicNetSubnet1",
Description="The network's public subnet-1 ID",
Export=Export(Sub('${Environment}-PUBLIC-SUBNET-1')),
Value=Ref(constants.PUB_SUBNET1),
))
self.cfn_template.add_output(
Output(
"PrivateNetSubnet1",
Description="The network's private subnet-1 ID",
Export=Export(Sub('${Environment}-PRIVATE-SUBNET-1')),
Value=Ref(constants.PRIV_SUBNET1),
))
self.cfn_template.add_output(
Output(
"PublicNetSubnet2",
Description="The network's public subnet-2 ID",
Export=Export(Sub('${Environment}-PUBLIC-SUBNET-2')),
Value=Ref(constants.PUB_SUBNET2),
))
self.cfn_template.add_output(
Output(
'PrivateNetSubnet2',
Description="The network's private subnet-2 ID",
Export=Export(Sub('${Environment}-PRIVATE-SUBNET-2')),
Value=Ref(constants.PRIV_SUBNET2),
))
self.cfn_template.add_output(
Output(
'PublicSubnet1AZ',
Description="The public subnet-1 AZ to use for public servers",
Export=Export(Sub('${Environment}-PUBLIC-SUBNET-1-AZ')),
Value=GetAtt(constants.PUB_SUBNET1, 'AvailabilityZone'),
))
self.cfn_template.add_output(
Output(
'PrivateSubnet1AZ',
Description=
"The private subnet-1 AZ to use for private servers",
Export=Export(Sub('${Environment}-PRIVATE-SUBNET-1-AZ')),
Value=GetAtt(constants.PRIV_SUBNET1, 'AvailabilityZone'),
))
self.cfn_template.add_output(
Output(
'PublicSubnet2AZ',
Description='The public subnet-2 AZ to use for public servers',
Export=Export(Sub('${Environment}-PUBLIC-SUBNET-2-AZ')),
Value=GetAtt(constants.PUB_SUBNET2, 'AvailabilityZone'),
))
self.cfn_template.add_output(
Output(
'PrivateSubnet2AZ',
Description=
"The private subnet-2 AZ to use for private servers",
Export=Export(Sub('${Environment}-PRIVATE-SUBNET-2-AZ')),
Value=GetAtt(constants.PRIV_SUBNET2, 'AvailabilityZone'),
))
self.cfn_template.add_output(
Output(
'NATIPAddress',
Description="The VPC NAT ip address",
Export=Export(Sub('${Environment}-NATIPAddress')),
Value=Ref(constants.EIP),
))
self.cfn_template.add_output(
Output(
'AppSecurityGroup',
Description='The security group ID to use for apps',
Export=Export(Sub('${Environment}-AppSecurityGroup')),
Value=GetAtt(constants.APP_SG, 'GroupId'),
))
return self.cfn_template
def add_vpc(self):
'''
Add a VPC for Anchore Engine
'''
self.cfn_template.add_resource(
VPC(
title=constants.VPC,
CidrBlock=Ref('VPCCIDRBlock'),
EnableDnsSupport=bool('true'),
EnableDnsHostnames=bool('true'),
InstanceTenancy='default',
))
return self.cfn_template
def add_public_subnet1(self):
'''
Add a public subnet 1
'''
self.cfn_template.add_resource(
Subnet(title=constants.PUB_SUBNET1,
VpcId=Ref(constants.VPC),
CidrBlock=Ref('PublicSubnet1CIDRBlock'),
MapPublicIpOnLaunch="true",
AvailabilityZone=Select('0', GetAZs())))
return self.cfn_template
def add_private_subnet1(self):
'''
Add a private subnet 1
'''
self.cfn_template.add_resource(
Subnet(title=constants.PRIV_SUBNET1,
VpcId=Ref(constants.VPC),
CidrBlock=Ref('PrivateSubnet1CIDRBlock'),
AvailabilityZone=Select('0', GetAZs())))
return self.cfn_template
def add_public_subnet2(self):
'''
Add a public subnet 2
'''
self.cfn_template.add_resource(
Subnet(title=constants.PUB_SUBNET2,
VpcId=Ref(constants.VPC),
CidrBlock=Ref('PublicSubnet2CIDRBlock'),
MapPublicIpOnLaunch="true",
AvailabilityZone=Select('1', GetAZs())))
return self.cfn_template
def add_private_subnet2(self):
'''
Add a private subnet 2
'''
self.cfn_template.add_resource(
Subnet(title=constants.PRIV_SUBNET2,
VpcId=Ref(constants.VPC),
CidrBlock=Ref('PrivateSubnet2CIDRBlock'),
AvailabilityZone=Select('1', GetAZs())))
return self.cfn_template
def add_internet_gateway(self):
'''
Add an internet gateway
'''
self.cfn_template.add_resource(InternetGateway(title=constants.IGW, ))
return self.cfn_template
def add_attach_gateway(self):
'''
Add an internet gateway attachment
'''
self.cfn_template.add_resource(
VPCGatewayAttachment(title=constants.ATTACH_GW,
VpcId=Ref(constants.VPC),
InternetGatewayId=Ref(constants.IGW)))
return self.cfn_template
def add_vpc_eip(self):
'''
Add elastic ip to vpc
'''
self.cfn_template.add_resource(
EIP(
title=constants.EIP,
Domain='vpc',
DependsOn=constants.ATTACH_GW,
))
return self.cfn_template
def add_nat(self):
'''
Add a NAT Gateway
'''
self.cfn_template.add_resource(
NatGateway(title=constants.NAT,
AllocationId=GetAtt(constants.EIP, 'AllocationId'),
SubnetId=Ref(constants.PUB_SUBNET1)))
return self.cfn_template
def add_public_routetable(self):
'''
Add a public route-table
'''
self.cfn_template.add_resource(
RouteTable(
title=constants.PUB_RT,
VpcId=Ref(constants.VPC),
))
return self.cfn_template
def add_private_routetable(self):
'''
Add a private route-table
'''
self.cfn_template.add_resource(
RouteTable(
title=constants.PRIV_RT,
VpcId=Ref(constants.VPC),
))
return self.cfn_template
def add_public_route(self):
'''
Add a public route with internet gateway
'''
self.cfn_template.add_resource(
Route(
title=constants.PUB_ROUTE,
RouteTableId=Ref(constants.PUB_RT),
DestinationCidrBlock='0.0.0.0/0',
GatewayId=Ref(constants.IGW),
))
return self.cfn_template
def add_private_route(self):
'''
Add a private route with nat gateway
'''
self.cfn_template.add_resource(
Route(
title=constants.PRIV_ROUTE,
RouteTableId=Ref(constants.PRIV_RT),
DestinationCidrBlock='0.0.0.0/0',
NatGatewayId=Ref(constants.NAT),
))
return self.cfn_template
def add_public_subnet_routetable_association1(self):
'''
Add a public subnet route-table association
to public subnet 1
'''
self.cfn_template.add_resource(
SubnetRouteTableAssociation(
title=constants.PUB_RT_ASS1,
SubnetId=Ref(constants.PUB_SUBNET1),
RouteTableId=Ref(constants.PUB_RT),
))
return self.cfn_template
def add_private_subnet_routetable_association1(self):
'''
Add a private subnet route-table association
to private subnet 1
'''
self.cfn_template.add_resource(
SubnetRouteTableAssociation(
title=constants.PRIV_RT_ASS1,
SubnetId=Ref(constants.PRIV_SUBNET1),
RouteTableId=Ref(constants.PRIV_RT),
))
return self.cfn_template
def add_public_subnet_routetable_association2(self):
'''
Add a public subnet route-table association
to public subnet 2
'''
self.cfn_template.add_resource(
SubnetRouteTableAssociation(
title=constants.PUB_RT_ASS2,
SubnetId=Ref(constants.PUB_SUBNET2),
RouteTableId=Ref(constants.PUB_RT),
))
return self.cfn_template
def add_private_subnet_routetable_association2(self):
'''
Add a private subnet route-table association
to private subnet 2
'''
self.cfn_template.add_resource(
SubnetRouteTableAssociation(
title=constants.PRIV_RT_ASS2,
SubnetId=Ref(constants.PRIV_SUBNET2),
RouteTableId=Ref(constants.PRIV_RT),
))
return self.cfn_template
def add_app_secuirty_group(self):
'''
Add security group tfor all application
'''
self.cfn_template.add_resource(
SecurityGroup(title=constants.APP_SG,
GroupDescription=
'Allow communication between application servers',
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol='tcp',
FromPort=int('0'),
ToPort=int('65535'),
CidrIp=Ref('PublicSubnet1CIDRBlock'),
),
SecurityGroupRule(
IpProtocol='tcp',
FromPort=int('0'),
ToPort=int('65535'),
CidrIp=Ref('PrivateSubnet1CIDRBlock'),
),
SecurityGroupRule(
IpProtocol='tcp',
FromPort=int('0'),
ToPort=int('65535'),
CidrIp=Ref('PublicSubnet2CIDRBlock'),
),
SecurityGroupRule(
IpProtocol='tcp',
FromPort=int('0'),
ToPort=int('65535'),
CidrIp=Ref('PrivateSubnet2CIDRBlock'),
),
],
SecurityGroupEgress=[
SecurityGroupRule(IpProtocol='-1',
CidrIp='0.0.0.0/0')
],
VpcId=Ref(constants.VPC)))
return self.cfn_template
from troposphere import GetAtt, Join, Output, Parameter, Ref, Template
from troposphere.ec2 import SecurityGroup
from troposphere.rds import (DBSubnetGroup, DBCluster, ScalingConfiguration)
from troposphere.secretsmanager import (Secret, GenerateSecretString,
SecretTargetAttachment)
t = Template()
t.set_version('2010-09-09')
t.set_description("""\
AWS CloudFormation Template to launch an Aurora Serverless Relational \
Database (RDS) Cluster.""")
vpcid = t.add_parameter(
Parameter(
"VpcId",
Type="String",
Description="VpcId of your existing Virtual Private Cloud (VPC)"))
subnet = t.add_parameter(
Parameter(
"Subnets",
Type="CommaDelimitedList",
Description=(
"The list of SubnetIds, for at least two Availability Zones in the "
"region in your Virtual Private Cloud (VPC)")))
dbuser = t.add_parameter(
Parameter(
"DBUser",
NoEcho=True,
Default="admin",
Condition("NotOneEqualsFoo"),
Condition("BarEqualsTwo"),
Condition("OneEqualsFooAndNotBarEqualsTwo"),
Condition("OneIsQuzAndThreeEqualsFour")
),
"LaunchWithGusto": And(
Condition("LaunchInstance"),
Equals(Ref("One"), "Gusto")
)
}
# Create the object that will generate our template
t = Template()
t.set_description("Test to create first template using troposphere")
t.set_version("2012-10-17")
t.add_condition
#Add parameters, condition to template
for p in parameters.values():
t.add_parameter(p)
for k in conditions:
t.add_condition(k, conditions[k])
#for r in resources.values():
#t.add_resource(r)
# The subnet resource defined must be added to the template
cfnrole = t.add_resource(Role(
"my0cfl0role",
AssumeRolePolicyDocument=PolicyDocument(
Statement=[
from troposphere import Template, backup
from troposphere.iam import Role
template = Template("AWS Backup")
template.set_version()
backup_vault = template.add_resource(
backup.BackupVault(
"Vault",
BackupVaultName="my-backup-vault",
BackupVaultTags=dict(
Project="Project",
Environment="Environment",
Classifier="Classifier",
),
# EncryptionKeyArn="KmsKeyId",
)
)
backup_plan = template.add_resource(
backup.BackupPlan(
"Backup",
BackupPlan=backup.BackupPlanResourceType(
BackupPlanName="BackupPlan",
BackupPlanRule=[
backup.BackupRuleResourceType(
TargetBackupVault=backup_vault.ref(),
Lifecycle=backup.LifecycleResourceType(DeleteAfterDays=31),
RecoveryPointTags=dict(
Project="Project",
Environment="Environment",
def _custom_resource(image_id):
nonlocal stack_name_post_test
# custom resource stack
custom_resource_stack_name = generate_stack_name(
"-".join([image_id, "custom", "resource"]),
request.config.getoption("stackname_suffix"))
stack_name_post_test = custom_resource_stack_name
custom_resource_template = Template()
custom_resource_template.set_version()
custom_resource_template.set_description(
"Create build image custom resource stack")
# Create a instance role
managed_policy_arns = [
"arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore",
"arn:aws:iam::aws:policy/EC2InstanceProfileForImageBuilder",
]
policy_document = iam.Policy(
PolicyName="myInstanceRoleInlinePolicy",
PolicyDocument={
"Statement": [{
"Effect":
"Allow",
"Action": [
"ec2:CreateTags",
"ec2:ModifyImageAttribute",
"s3:GetObject",
"cloudformation:ListStacks",
],
"Resource":
"*",
}]
},
)
role_name = "".join(["dummyInstanceRole", generate_random_string()])
instance_role = iam.Role(
"CustomInstanceRole",
AssumeRolePolicyDocument={
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": ["ec2.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}]
},
Description="custom instance role for build image test.",
ManagedPolicyArns=managed_policy_arns,
Path="/parallelcluster/",
Policies=[policy_document],
RoleName=role_name,
)
custom_resource_template.add_resource(instance_role)
custom_resource_stack = CfnStack(
name=custom_resource_stack_name,
region=region,
template=custom_resource_template.to_json(),
capabilities=["CAPABILITY_NAMED_IAM"],
)
cfn_stacks_factory.create_stack(custom_resource_stack)
instance_role_arn = boto3.client("iam").get_role(
RoleName=role_name).get("Role").get("Arn")
logging.info("Custom instance role arn %s", instance_role_arn)
return instance_role_arn
