def ssm_global(): template = Template() ssm_role = iam.Role( 'SsmRole', RoleName="SsmRole", ManagedPolicyArns=[ "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore", "arn:aws:iam::aws:policy/AmazonS3FullAccess", "arn:aws:iam::aws:policy/AmazonEC2FullAccess" ], AssumeRolePolicyDocument=PolicyDocument(Statement=[ Statement(Effect=Allow, Action=[Action("sts", "AssumeRole")], Principal=Principal("Service", "ec2.amazonaws.com")) ])) ssm_profile = iam.InstanceProfile('SsmProfile', Roles=[Ref(ssm_role)], InstanceProfileName="SsmProfile") template.add_resource(ssm_role) template.add_resource(ssm_profile) with open( os.path.dirname(os.path.realpath(__file__)) + '/ssm_global.yml', 'w') as cf_file: cf_file.write(template.to_yaml()) return template.to_yaml()
def test_yaml_long_form(self): t = Template() t.add_resource(WaitCondition( "MyWaitCondition", Timeout=30, Handle=Sub(cond_string))) self.assertEqual(cond_normal, t.to_yaml()) self.assertEqual(cond_long, t.to_yaml(long_form=True)) self.assertEqual(cond_long, t.to_yaml(False, True)) self.assertEqual(cond_clean, t.to_yaml(clean_up=True)) self.assertEqual(cond_clean, t.to_yaml(True))
def test_UpdateReplacePolicy(self): t = Template() t.add_resource(Bucket( "S3Bucket", AccessControl=PublicRead, UpdateReplacePolicy='Retain', )) t.to_yaml() self.assertEqual(t.to_yaml(), test_updatereplacepolicy_yaml)
def test_yaml_long_form(self): t = Template() t.add_resource( WaitCondition("MyWaitCondition", Timeout=30, Handle=Sub(cond_string)) ) self.assertEqual(cond_normal, t.to_yaml()) self.assertEqual(cond_long, t.to_yaml(long_form=True)) self.assertEqual(cond_long, t.to_yaml(False, True)) self.assertEqual(cond_clean, t.to_yaml(clean_up=True)) self.assertEqual(cond_clean, t.to_yaml(True))
def create_stack_template(self): "sg-006bb1b84c5a7ea6c" ami_id="ami-0de53d8956e8dcf80" t = Template() instance = ec2.Instance("myinstance1") instance.ImageId = ami_id instance.SecurityGroupIds = [ "sg-006bb1b84c5a7ea6c" ] instance.KeyName = "eddie.christian" instance.InstanceType = "t1.micro" instance.Tags = self.instance_tags(1) instance2 = ec2.Instance("myinstance2") instance2.ImageId = ami_id instance2.SecurityGroupIds = [ "sg-006bb1b84c5a7ea6c" ] instance2.KeyName = "eddie.christian" instance2.InstanceType = "t1.micro" instance2.Tags = self.instance_tags(2) t.add_resource(instance) t.add_resource(instance2) print(t.to_yaml()) with open(self.template_name, 'wb') as f: f.write(t.to_yaml()) with open(self.template_name) as template_fileobj: template_data = template_fileobj.read() self.cf.validate_template(TemplateBody=template_data) params = { 'StackName': self.stack_name, 'TemplateBody': template_data, 'Parameters': [], } try: if self._stack_exists(): print('Updating {}'.format(self.stack_name)) stack_result = self.cf.update_stack(**params) waiter = self.cf.get_waiter('stack_update_complete') else: print('Creating {}'.format(self.stack_name)) stack_result = self.cf.create_stack(**params) waiter = self.cf.get_waiter('stack_create_complete') print("...waiting for stack to be ready...") waiter.wait(StackName=self.stack_name) except botocore.exceptions.ClientError as ex: error_message = ex.response['Error']['Message'] if error_message == 'No updates are to be performed.': print("No changes") else: raise
def create_peering(self): template = Template() template.add_version('2010-09-09') source_vpc_name_formatted = ''.join( e for e in self.source_vpc_name if e.isalnum()).capitalize() target_vpc_name_formatted = ''.join( e for e in self.target_vpc_name if e.isalnum()).capitalize() vpc_peering_connection = template.add_resource( VPCPeeringConnection( '{}{}{}VpcPeering'.format(self.stage,source_vpc_name_formatted,target_vpc_name_formatted), VpcId=ImportValue("{}{}VpcId".format(self.stage,source_vpc_name_formatted)), PeerVpcId=ImportValue("{}{}VpcId".format(self.stage,target_vpc_name_formatted)), Tags=Tags( Name="{}_{}_{}_peering".format(self.stage,source_vpc_name_formatted,target_vpc_name_formatted) ) ) ) template.add_resource( Route( '{}{}PublicRoutePeeringRule'.format(self.stage,source_vpc_name_formatted), VpcPeeringConnectionId=Ref(vpc_peering_connection), DestinationCidrBlock=self.target_cidr_block, RouteTableId=ImportValue("{}{}PublicRouteTableId".format(self.stage,source_vpc_name_formatted)) ) ) template.add_resource( Route( '{}{}PrivateRoutePeeringRule'.format(self.stage,source_vpc_name_formatted), VpcPeeringConnectionId=Ref(vpc_peering_connection), DestinationCidrBlock=self.target_cidr_block, RouteTableId=ImportValue("{}{}PrivateRouteTableId".format(self.stage,source_vpc_name_formatted)) ) ) template.add_resource( Route( '{}{}PublicRoutePeeringRule'.format(self.stage,target_vpc_name_formatted), VpcPeeringConnectionId=Ref(vpc_peering_connection), DestinationCidrBlock=self.source_cidr_block, RouteTableId=ImportValue("{}{}PublicRouteTableId".format(self.stage,target_vpc_name_formatted)) ) ) template.add_resource( Route( '{}{}PrivateRoutePeeringRule'.format(self.stage,target_vpc_name_formatted), VpcPeeringConnectionId=Ref(vpc_peering_connection), DestinationCidrBlock=self.source_cidr_block, RouteTableId=ImportValue("{}{}PrivateRouteTableId".format(self.stage,target_vpc_name_formatted)) ) ) f = open("modules/template_peer_vpcs.yaml", 'w') print(template.to_yaml(), file=f)
def main(**kwargs): """ Create the CFN template and either write to screen or update/create boto3. """ codebuild = Template() for job in config.sections(): if 'CodeBuild:' in job: job_title = job.split(':')[1] service_role = build_role(template=codebuild, project_name=job_title).to_dict() # Pull the env out of the section, and use the snippet for the other values. if 'snippet' in config[job]: build_project(template=codebuild, project_name=job_title, section=config.get(job, 'snippet'), service_role=service_role['Ref'], raw_env=config.get(job, 'env')) else: build_project(template=codebuild, project_name=job_title, section=job, service_role=service_role['Ref']) with (open("cfn/codebuild_test_projects.yml", 'w')) as fh: fh.write(codebuild.to_yaml()) if args.dry_run: logging.debug('Dry Run: wrote cfn file, but not calling AWS.') else: print('Boto functionality goes here.')
def dump_yaml(cfn_file): template = Template() vpc_cidr_param = template.add_parameter( Parameter( "vpcCidrParam", Description="string of vpc cidr block to use", Type="String", )) subnet_cidr_param = template.add_parameter( Parameter( "subnetCidrParam", Description="string of subnet cidr block to use", Type="String", )) resource_tags = Tags(Name=Sub("${AWS::StackName}"), user="******", stelligent_u_lesson='lesson-4-1', stelligent_u_lab='lab-1') vpc = template.add_resource( ec2.VPC( "Vpc", CidrBlock=Ref(vpc_cidr_param), EnableDnsSupport=True, EnableDnsHostnames=True, InstanceTenancy="default", Tags=resource_tags, )) subnet = template.add_resource( ec2.Subnet( "Subnet", VpcId=Ref(vpc), CidrBlock=Ref(subnet_cidr_param), MapPublicIpOnLaunch=False, AvailabilityZone=Select(0, GetAZs()), Tags=resource_tags, )) template.add_output([ Output( "vpcId", Description="InstanceId of the newly created EC2 instance", Value=Ref(vpc), ), Output( "SubnetId", Description="InstanceId of the newly created EC2 instance", Value=Ref(subnet), ), ]) with open(cfn_file, 'w') as f: f.write(template.to_yaml())
def provision_stack(stack_name_suffix: str, template: troposphere.Template) -> None: with betterboto_client.ClientContextManager( "cloudformation") as cloudformation: cloudformation.create_or_update( StackName=f"AWSOrganized-{stack_name_suffix}", TemplateBody=template.to_yaml(clean_up=True), Capabilities=["CAPABILITY_NAMED_IAM"], )
def create_pipeline_template(): template = Template() build_project = __create_build_project(template) __create_pipeline(template, build_project) with open('./pipeline.yml', mode='w') as file: file.write(template.to_yaml())
def main(): parser = argparse.ArgumentParser() parser.add_argument('location', type=str) parser.add_argument('--format', default='json', choices=['json', 'yaml']) group = parser.add_mutually_exclusive_group(required=True) group.add_argument( '--meta-git-repo', action='store_const', const=meta_git_repo, dest='func', help='indicates location is a url of a meta git repo', ) group.add_argument( '--services-file', action='store_const', const=services_file, dest='func', help='indicates location is a path of a sierra config file', ) if not sys.argv[1:]: parser.print_help() parser.exit() args = parser.parse_args() services = args.func(args.location) # start of template generation template = Template() bucket = template.add_parameter( Parameter( 'TemplateBucket', Description='The S3 bucket containing all of the templates.', Type='String', )) cluster = template.add_resource( cf.Stack( 'Cluster', TemplateURL=Sub( 'https://s3.amazonaws.com/${TemplateBucket}/templates/ecs-cluster.yaml' ), Parameters={ 'InstanceType': 't2.micro', })) if args.format == 'json': result = template.to_json() else: result = template.to_yaml() print(result)
def create_alb_template(): template = Template() vpc = template.add_parameter( parameter=Parameter(title='Vpc', Type='String')) subnet_a = template.add_parameter( parameter=Parameter(title='SubnetA', Type='String')) subnet_b = template.add_parameter( parameter=Parameter(title='SubnetB', Type='String')) ec2_instance = template.add_parameter( parameter=Parameter(title='Ec2Instance', Type='String')) security_group = template.add_resource( resource=SecurityGroup(title='SampleSecurityGroup', GroupDescription='sample-security-group', SecurityGroupIngress=[{ 'IpProtocol': 'tcp', 'FromPort': 80, 'ToPort': 80, 'CidrIp': '0.0.0.0/0' }], VpcId=Ref(vpc))) load_balancer = template.add_resource(resource=LoadBalancer( title='SampleLoadBalancer', Name='sample-alb', Subnets=[Ref(subnet_a), Ref(subnet_b)], SecurityGroups=[Ref(security_group)], )) target_group = template.add_resource(resource=TargetGroup( title='SampleTargetGroup', Targets=[TargetDescription( Id=Ref(ec2_instance), Port=80, )], VpcId=Ref(vpc), Name='sample-target-group', Port=80, Protocol='HTTP', )) template.add_resource(resource=Listener( title='SampleListener', DefaultActions=[ Action(TargetGroupArn=Ref(target_group), Type='forward') ], LoadBalancerArn=Ref(load_balancer), Port=80, Protocol='HTTP', )) with open('./alb.yml', mode='w') as file: file.write(template.to_yaml())
def test_s3_bucket(self): t = Template() t.add_description("S3 Bucket Example") s3bucket = t.add_resource(s3.Bucket( "S3Bucket", AccessControl=s3.PublicRead,)) t.add_output(Output( "BucketName", Value=Ref(s3bucket), Description="Name of S3 bucket to hold website content" )) self.assertEqual(s3_bucket_yaml, t.to_yaml())
def create_vpc_template(): template = Template() vpc_cidr = template.add_parameter(parameter=Parameter( title='VpcCidr', Type='String', Default='192.168.0.0/16')) subnet_cidr_a = template.add_parameter(parameter=Parameter( title='SubnetCidr1', Type='String', Default='192.168.1.0/24')) subnet_cidr_b = template.add_parameter(parameter=Parameter( title='SubnetCidr2', Type='String', Default='192.168.2.0/24')) vpc = template.add_resource(resource=VPC( title='SampleVpc', CidrBlock=Ref(vpc_cidr), EnableDnsHostnames=True)) igw = template.add_resource(resource=InternetGateway(title='SampleIgw')) template.add_resource(resource=VPCGatewayAttachment( title='SampleAttachment', VpcId=Ref(vpc), InternetGatewayId=Ref(igw))) subnet_a = template.add_resource( resource=Subnet(title='SampleSubnetA', AvailabilityZone='us-east-1a', CidrBlock=Ref(subnet_cidr_a), MapPublicIpOnLaunch=True, VpcId=Ref(vpc))) subnet_b = template.add_resource( resource=Subnet(title='SampleSubnetB', AvailabilityZone='us-east-1b', CidrBlock=Ref(subnet_cidr_b), MapPublicIpOnLaunch=True, VpcId=Ref(vpc))) route_table = template.add_resource( resource=RouteTable(title='SampleRoteTable', VpcId=Ref(vpc))) template.add_resource(resource=SubnetRouteTableAssociation( title='SampleRoteTableAssociationA', RouteTableId=Ref(route_table), SubnetId=Ref(subnet_a))) template.add_resource(resource=SubnetRouteTableAssociation( title='SampleRoteTableAssociationB', RouteTableId=Ref(route_table), SubnetId=Ref(subnet_b))) template.add_resource(resource=Route(title='SampleRoute', DestinationCidrBlock='0.0.0.0/0', GatewayId=Ref(igw), RouteTableId=Ref(route_table))) with open('./vpc.yml', mode='w') as file: file.write(template.to_yaml())
class Base(metaclass=ABCMeta): def __init__(self, sceptre_user_data: Dict): self.sceptre_user_data = sceptre_user_data self.tpl = Template() self.create_template() @abstractmethod def create_template(self): pass def to_yaml(self): return self.tpl.to_yaml()
def create_autoscaling_stack(projectName): cloudformationClient = boto3.client('cloudformation', endpoint_url='http://localhost:4581', region_name='us-west-2') t = Template() t.set_version('2010-09-09') t.set_description( "LocalStackTests Task two cloud formation template of AutoScalingGroup " ) ssh_rule = create_security_group_rule("tcp", "22", "22", "0.0.0.0/0") http_rule = create_security_group_rule( "tcp", "80", "80", "0.0.0.0/0") # assume we are ruing on 80 port 80 ec2SecurityGroup = create_security_group_resource( projectName + "EC2SecurityGroup", [ssh_rule, http_rule], "EC2 Security Group") launchConfigName = projectName + "LaunchConfiguration" launchConfig = create_launch_configuration_resource( launchConfigName, [Ref(ec2SecurityGroup)]) autoscaling = create_autoscaling_resource(projectName, launchConfig) t.add_resource(ec2SecurityGroup) t.add_resource(launchConfig) t.add_resource(autoscaling) stackName = projectName + 'Task2' try: task2_stack = cloudformationClient.create_stack( StackName=stackName, TemplateBody=t.to_yaml()) except Exception as e: # TODO: check other exceptions pass stackReady = wait_resource(cloudformationClient.describe_stacks, check_cloudformation_stack_complete, 10, StackName=stackName) if stackReady: res = cloudformationClient.describe_stacks(StackName=stackName) return res['Stacks'][0] else: raise Exception( "Fails to get recently created stack, try to wait for more time")
def generate(template: Template, template_path: str, template_format: TemplateFormat): print(f'Generating template: {template_path}') pathlib.Path(os.path.dirname(os.path.abspath(template_path))).mkdir( parents=True, exist_ok=True) if template_format == TemplateFormat.YAML: with open(template_path, 'w') as f: f.write(template.to_yaml()) else: with open(template_path, 'w') as f: f.write(template.to_json())
def create_vpc_template(): template = Template() vpc_cidr = template.add_parameter(parameter=Parameter( title='VpcCidr', Type='String', Default='192.168.0.0/16')) vpc = template.add_resource( resource=VPC(title='SampleVpc', CidrBlock=Ref(vpc_cidr))) public_subnet = __create_public_subnet(template, vpc) __create_private_subnet(template, vpc) __create_dmz_subnet(template, vpc, public_subnet) with open('./vpc.yml', mode='w') as file: file.write(template.to_yaml())
class TestCfTemplate(object): def setup_method(self): self.template = Template() @mock_cloudformation def test_ec2_resource(self, cf_client, vpc_data): ec2_resource = cf_resources.ec2_instance(name='test-ec2-bastion-cli', ami_id='ami-12345', keyname='foo', instance_type='t2.micro', sg_ids=['sg-12345'], subnet_id='subnet-12345') self.template.add_resource(ec2_resource) cf_client.create_stack(StackName='test_ec2_resource', TemplateBody=self.template.to_yaml())
def create_function_template(): template = Template() template.set_transform('AWS::Serverless-2016-10-31') template.add_resource(resource=Function( title='SampleLambdaFunction', CodeUri='.', FunctionName='sample-lambda-function', Handler='lambda_function.lambda_handler', Role=ImportValue( CommonResource.ExportName.LAMBDA_SERVICE_ROLE_ARN.value), Runtime='python3.7', )) with open('./function.yml', mode='w') as file: file.write(template.to_yaml())
def create_ec2_template(): template = Template() vpc = template.add_parameter( parameter=Parameter(title='Vpc', Type='String')) subnet = template.add_parameter( parameter=Parameter(title='Subnet', Type='String')) ami_image = template.add_parameter(parameter=Parameter( title='AmiImage', Default='ami-0e2ff28bfb72a4e45', Type='String')) key_name = template.add_parameter( parameter=Parameter(title='KeyName', Type='String')) my_ip = template.add_parameter( parameter=Parameter(title='MyIp', Type='String')) security_group = template.add_resource( resource=SecurityGroup(title='SampleSecurityGroup', GroupDescription='sample', VpcId=Ref(vpc), SecurityGroupIngress=[{ 'IpProtocol': 'tcp', 'FromPort': 80, 'ToPort': 80, 'CidrIp': '0.0.0.0/0', }, { 'IpProtocol': 'tcp', 'FromPort': 22, 'ToPort': 22, 'CidrIp': Ref(my_ip), }])) template.add_resource(resource=Instance( title='SampleEc2Instance', SubnetId=Ref(subnet), SecurityGroupIds=[Ref(security_group)], InstanceType='t2.micro', ImageId=Ref(ami_image), KeyName=Ref(key_name), )) with open('./ec2.yml', mode='w') as file: file.write(template.to_yaml())
def create_template(): template = Template() vpc = template.add_parameter( parameter=Parameter(title='Vpc', Type='String')) key_name = template.add_parameter( parameter=Parameter(title='KeyName', Type='String')) subnet_a = template.add_resource( resource=Subnet(title='SampleSubnetA', AvailabilityZone='us-east-1a', CidrBlock='192.168.10.0/24', MapPublicIpOnLaunch=True, VpcId=Ref(vpc))) template.add_resource(resource=Subnet(title='SampleSubnetB', AvailabilityZone='us-east-1b', CidrBlock='192.168.11.0/24', MapPublicIpOnLaunch=True, VpcId=Ref(vpc))) security_group = template.add_resource( resource=SecurityGroup(title='SampleSecurityGroup', GroupDescription='sample', VpcId=Ref(vpc), SecurityGroupIngress=[{ 'IpProtocol': 'tcp', 'FromPort': 80, 'ToPort': 80, 'CidrIp': '0.0.0.0/0', }])) template.add_resource(resource=Instance( title='SampleEc2Instance', SubnetId=Ref(subnet_a), SecurityGroupIds=[Ref(security_group)], InstanceType='t2.micro', ImageId='ami-0e2ff28bfb72a4e45', KeyName=Ref(key_name), )) with open('./driver.yml', mode='w') as file: file.write(template.to_yaml())
def create_cloud_front_template(): template = Template() template.set_transform('AWS::Serverless-2016-10-31') service_role = template.add_resource( resource=Role(title='SampleLambdaServiceRole', RoleName='sample-lambda-service-role', Path='/', AssumeRolePolicyDocument={ "Statement": [{ "Effect": "Allow", "Principal": { "Service": ['lambda.amazonaws.com'] }, "Action": ["sts:AssumeRole"] }] }, Policies=[ Policy(PolicyName="sample-policy", PolicyDocument={ "Version": "2012-10-17", "Statement": [{ "Action": 'lambda:*', "Resource": '*', "Effect": "Allow" }] }) ])) template.add_resource(resource=Function( title='SampleLambdaFunction', AutoPublishAlias='sample', CodeUri='.', FunctionName='sample-lambda-function', Handler='lambda_function.lambda_handler', Role=GetAtt(logicalName=service_role, attrName='Arn'), Runtime='python3.7', )) with open('./function.yml', mode='w') as file: file.write(template.to_yaml())
def main(): """main function""" t = Template() t.set_transform('AWS::Serverless-2016-10-31') # add resources to the template resources = [] \ + document_fsm.DocumentReviewMachine.cf_resources() \ + document_fsm.launch.cf_resources() \ + document_fsm.transition.cf_resources() \ + document_fsm.info.cf_resources() resources += document_tasks.archive_document.cf_resources() \ + document_tasks.delete_document.cf_resources() \ + document_tasks.notify_reviewer.cf_resources() \ + document_tasks.notify_uploader.cf_resources() \ + document_tasks.summarize_document.cf_resources() for r in resources: t.add_resource(r) # output yaml template print(t.to_yaml())
def main(): t = Template('GeoWave Admin UI dev server') t.add_parameter(PARAMS) t.add_resource(create_dnsrecords()) t.add_resource(create_loadbalancer()) t.add_resource(create_instance()) t.add_resource(create_routing()) t.add_resource(create_securitygroups()) t.add_resource(create_subnets()) t.add_resource(create_vpc()) t.add_output(Output('application', Value=_subdomain_for_application())) t.add_output(Output('instance', Value=_subdomain_for_instance())) t.add_output(Output('jenkins', Value=_subdomain_for_jenkins())) t.add_output(Output('loadbalancer', Value=GetAtt('tlsFrontend', 'DNSName'))) print(t.to_yaml())
def create_function_template(): template = Template() template.set_transform('AWS::Serverless-2016-10-31') bucket = template.add_resource(resource=Function( title='SampleLambdaFunction', AutoPublishAlias='sample', CodeUri='.', FunctionName='sample-lambda-edge-function', Handler='lambda_function.lambda_handler', Role=ImportValue( CommonResource.ExportName.LAMBDA_EDGE_SERVICE_ROLE_ARN.value), Runtime='python3.7', )) template.add_output(output=Output(title=bucket.title, Value=GetAtt(bucket, 'Arn'), Export=Export(name=get_export_name()))) with open('./function.yml', mode='w') as file: file.write(template.to_yaml())
def instantiate_CF_template(template: Template, stack_name: str = "unnamed", **params) -> None: client = boto3.client('cloudformation') logging.info(f"Validating stack {stack_name}") tpl_yaml = template.to_yaml() validate_result = client.validate_template(TemplateBody=tpl_yaml) logging.info(f"Creating stack {stack_name}") stack_params = dict( StackName=stack_name, TemplateBody=tpl_yaml, Parameters=[], Capabilities=['CAPABILITY_IAM'], #OnFailure = 'DELETE', ) stack_params.update(params) if stack_exists(client, stack_name): logging.warning(f"Stack '{stack_name}' already exists") stacks = client.describe_stacks(StackName=stack_name) status = stacks['Stacks'][0]['StackStatus'] if status == 'ROLLBACK_COMPLETE': # Stacks in Rollback complete can't be updated. #input("Press enter to delete the stack (is in ROLLBACK_COMPLETE state) or ^C to abort...") logging.info("Deleting stack...") delete_stack(client, stack_name) stack_result = client.create_stack(**stack_params) waiter = client.get_waiter('stack_create_complete') logging.info("Waiting for stack create...") waiter.wait(StackName=stack_name) else: stack_result = client.update_stack(**stack_params) waiter = client.get_waiter('stack_update_complete') logging.info("Waiting for stack update...") waiter.wait(StackName=stack_name) else: stack_result = client.create_stack(**stack_params) waiter = client.get_waiter('stack_create_complete') logging.info("Waiting for stack create...") waiter.wait(StackName=stack_name)
def main(**kwargs): """ Create the CFN template and either write to screen or update/create boto3. """ codebuild = Template() codebuild.set_version('2010-09-09') # Create a single CloudWatch Event role to allow codebuild:startBuild cw_event_role = build_cw_cb_role(codebuild) # TODO: There is a problem with the resource statement #logging.info('Creating github role: {}', build_github_role(codebuild)) for job in config.sections(): if 'CodeBuild:' in job: job_title = job.split(':')[1] service_role = build_codebuild_role( template=codebuild, project_name=job_title).to_dict() # Pull the env out of the section, and use the snippet for the other values. if 'snippet' in config[job]: build_project(template=codebuild, project_name=job_title, section=config.get(job, 'snippet'), service_role=service_role['Ref'], raw_env=config.get(job, 'env')) else: build_project(template=codebuild, project_name=job_title, section=job, service_role=service_role['Ref']) build_cw_event(template=codebuild, project_name=job_title, role=cw_event_role) with (open(args.output_dir + "/s2n_codebuild_projects.yml", 'w')) as fh: fh.write(codebuild.to_yaml()) if args.dry_run: logging.debug('Dry Run: wrote cfn file, but not calling AWS.') else: print('Boto functionality goes here.')
class CreateTemplate: def __init__(self): self.template = Template() def add_alarm(self, alarmName): self.template.add_resource(Alarm( alarmName, ComparisonOperator='GreaterThanThreshold', Statistic='Maximum', EvaluationPeriods=1, MetricName='BucketSizeBytes', Namespace='S3', Period='60', Threshold=0 )) def to_yaml(self): return self.template.to_yaml() def to_json(self): return json.dumps(self.to_dict()) def to_dict(self): return self.template.to_dict()
def main(**params): try: # Metadata t = Template() t.set_version("2010-09-09") t.set_description("(SOCA) - Base template to deploy compute nodes.") allow_anonymous_data_collection = params["MetricCollectionAnonymous"] debug = False mip_usage = False instances_list = params["InstanceType"].split("+") asg_lt = asg_LaunchTemplate() ltd = LaunchTemplateData("NodeLaunchTemplateData") mip = MixedInstancesPolicy() stack_name = Ref("AWS::StackName") # Begin LaunchTemplateData UserData = '''#!/bin/bash -xe export PATH=$PATH:/usr/local/bin if [[ "''' + params['BaseOS'] + '''" == "centos7" ]] || [[ "''' + params['BaseOS'] + '''" == "rhel7" ]]; then EASY_INSTALL=$(which easy_install-2.7) $EASY_INSTALL pip PIP=$(which pip2.7) $PIP install awscli yum install -y nfs-utils # enforce install of nfs-utils else # Upgrade awscli on ALI (do not use yum) EASY_INSTALL=$(which easy_install-2.7) $EASY_INSTALL pip PIP=$(which pip) $PIP install awscli --upgrade fi if [[ "''' + params['BaseOS'] + '''" == "amazonlinux2" ]]; then /usr/sbin/update-motd --disable fi GET_INSTANCE_TYPE=$(curl http://169.254.169.254/latest/meta-data/instance-type) echo export "SOCA_CONFIGURATION="''' + str(params['ClusterId']) + '''"" >> /etc/environment echo export "SOCA_BASE_OS="''' + str(params['BaseOS']) + '''"" >> /etc/environment echo export "SOCA_JOB_QUEUE="''' + str(params['JobQueue']) + '''"" >> /etc/environment echo export "SOCA_JOB_OWNER="''' + str(params['JobOwner']) + '''"" >> /etc/environment echo export "SOCA_JOB_NAME="''' + str(params['JobName']) + '''"" >> /etc/environment echo export "SOCA_JOB_PROJECT="''' + str(params['JobProject']) + '''"" >> /etc/environment echo export "SOCA_VERSION="''' + str(params['Version']) + '''"" >> /etc/environment echo export "SOCA_JOB_EFA="''' + str(params['Efa']).lower() + '''"" >> /etc/environment echo export "SOCA_JOB_ID="''' + str(params['JobId']) + '''"" >> /etc/environment echo export "SOCA_SCRATCH_SIZE=''' + str(params['ScratchSize']) + '''" >> /etc/environment echo export "SOCA_INSTALL_BUCKET="''' + str(params['S3Bucket']) + '''"" >> /etc/environment echo export "SOCA_INSTALL_BUCKET_FOLDER="''' + str(params['S3InstallFolder']) + '''"" >> /etc/environment echo export "SOCA_FSX_LUSTRE_BUCKET="''' + str(params['FSxLustreConfiguration']['fsx_lustre']).lower() + '''"" >> /etc/environment echo export "SOCA_FSX_LUSTRE_DNS="''' + str(params['FSxLustreConfiguration']['existing_fsx']).lower() + '''"" >> /etc/environment echo export "SOCA_INSTANCE_TYPE=$GET_INSTANCE_TYPE" >> /etc/environment echo export "SOCA_INSTANCE_HYPERTHREADING="''' + str(params['ThreadsPerCore']).lower() + '''"" >> /etc/environment echo export "SOCA_HOST_SYSTEM_LOG="/apps/soca/''' + str(params['ClusterId']) + '''/cluster_node_bootstrap/logs/''' + str(params['JobId']) + '''/$(hostname -s)"" >> /etc/environment echo export "AWS_STACK_ID=${AWS::StackName}" >> /etc/environment echo export "AWS_DEFAULT_REGION=${AWS::Region}" >> /etc/environment source /etc/environment AWS=$(which aws) # Give yum permission to the user on this specific machine echo "''' + params['JobOwner'] + ''' ALL=(ALL) /bin/yum" >> /etc/sudoers mkdir -p /apps mkdir -p /data # Mount EFS echo "''' + params['EFSDataDns'] + ''':/ /data nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0" >> /etc/fstab echo "''' + params['EFSAppsDns'] + ''':/ /apps nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0" >> /etc/fstab mount -a # Configure NTP yum remove -y ntp yum install -y chrony mv /etc/chrony.conf /etc/chrony.conf.original echo -e """ # use the local instance NTP service, if available server 169.254.169.123 prefer iburst minpoll 4 maxpoll 4 # Use public servers from the pool.ntp.org project. # Please consider joining the pool (http://www.pool.ntp.org/join.html). # !!! [BEGIN] SOCA REQUIREMENT # You will need to open UDP egress traffic on your security group if you want to enable public pool #pool 2.amazon.pool.ntp.org iburst # !!! [END] SOCA REQUIREMENT # Record the rate at which the system clock gains/losses time. driftfile /var/lib/chrony/drift # Allow the system clock to be stepped in the first three updates # if its offset is larger than 1 second. makestep 1.0 3 # Specify file containing keys for NTP authentication. keyfile /etc/chrony.keys # Specify directory for log files. logdir /var/log/chrony # save data between restarts for fast re-load dumponexit dumpdir /var/run/chrony """ > /etc/chrony.conf systemctl enable chronyd # Prepare Log folder mkdir -p $SOCA_HOST_SYSTEM_LOG echo "@reboot /bin/bash /apps/soca/$SOCA_CONFIGURATION/cluster_node_bootstrap/ComputeNodePostReboot.sh >> $SOCA_HOST_SYSTEM_LOG/ComputeNodePostInstall.log 2>&1" | crontab - $AWS s3 cp s3://$SOCA_INSTALL_BUCKET/$SOCA_INSTALL_BUCKET_FOLDER/scripts/config.cfg /root/ /bin/bash /apps/soca/$SOCA_CONFIGURATION/cluster_node_bootstrap/ComputeNode.sh ''' + params['SchedulerHostname'] + ''' >> $SOCA_HOST_SYSTEM_LOG/ComputeNode.sh.log 2>&1''' ltd.EbsOptimized = True for instance in instances_list: if "t2." in instance: ltd.EbsOptimized = False else: # t2 does not support CpuOptions ltd.CpuOptions = CpuOptions( CoreCount=int(params["CoreCount"]), ThreadsPerCore=1 if params["ThreadsPerCore"] is False else 2) ltd.IamInstanceProfile = IamInstanceProfile(Arn=params["ComputeNodeInstanceProfileArn"]) ltd.KeyName = params["SSHKeyPair"] ltd.ImageId = params["ImageId"] if params["SpotPrice"] is not False and params["SpotAllocationCount"] is False: ltd.InstanceMarketOptions = InstanceMarketOptions( MarketType="spot", SpotOptions=SpotOptions( MaxPrice=Ref("AWS::NoValue") if params["SpotPrice"] == "auto" else str(params["SpotPrice"]) # auto -> cap at OD price ) ) ltd.InstanceType = instances_list[0] ltd.NetworkInterfaces = [NetworkInterfaces( InterfaceType="efa" if params["Efa"] is not False else Ref("AWS::NoValue"), DeleteOnTermination=True, DeviceIndex=0, Groups=[params["SecurityGroupId"]] )] ltd.UserData = Base64(Sub(UserData)) ltd.BlockDeviceMappings = [ BlockDeviceMapping( DeviceName="/dev/xvda" if params["BaseOS"] == "amazonlinux2" else "/dev/sda1", Ebs=EBSBlockDevice( VolumeSize=params["RootSize"], VolumeType="gp2", DeleteOnTermination="false" if params["KeepEbs"] is True else "true", Encrypted=True)) ] if int(params["ScratchSize"]) > 0: ltd.BlockDeviceMappings.append( BlockDeviceMapping( DeviceName="/dev/xvdbx", Ebs=EBSBlockDevice( VolumeSize=params["ScratchSize"], VolumeType="io1" if int(params["VolumeTypeIops"]) > 0 else "gp2", Iops=params["VolumeTypeIops"] if int(params["VolumeTypeIops"]) > 0 else Ref("AWS::NoValue"), DeleteOnTermination="false" if params["KeepEbs"] is True else "true", Encrypted=True)) ) # End LaunchTemplateData # Begin Launch Template Resource lt = LaunchTemplate("NodeLaunchTemplate") lt.LaunchTemplateName = params["ClusterId"] + "-" + str(params["JobId"]) lt.LaunchTemplateData = ltd t.add_resource(lt) # End Launch Template Resource asg_lt.LaunchTemplateSpecification = LaunchTemplateSpecification( LaunchTemplateId=Ref(lt), Version=GetAtt(lt, "LatestVersionNumber") ) asg_lt.Overrides = [] for instance in instances_list: asg_lt.Overrides.append(LaunchTemplateOverrides( InstanceType=instance)) # Begin InstancesDistribution if params["SpotPrice"] is not False and \ params["SpotAllocationCount"] is not False and \ (params["DesiredCapacity"] - params["SpotAllocationCount"]) > 0: mip_usage = True idistribution = InstancesDistribution() idistribution.OnDemandAllocationStrategy = "prioritized" # only supported value idistribution.OnDemandBaseCapacity = params["DesiredCapacity"] - params["SpotAllocationCount"] idistribution.OnDemandPercentageAboveBaseCapacity = "0" # force the other instances to be SPOT idistribution.SpotMaxPrice = Ref("AWS::NoValue") if params["SpotPrice"] == "auto" else str( params["SpotPrice"]) idistribution.SpotAllocationStrategy = params['SpotAllocationStrategy'] mip.InstancesDistribution = idistribution # End MixedPolicyInstance # Begin FSx for Lustre if params["FSxLustreConfiguration"]["fsx_lustre"] is not False: if params["FSxLustreConfiguration"]["existing_fsx"] is False: fsx_lustre = FileSystem("FSxForLustre") fsx_lustre.FileSystemType = "LUSTRE" fsx_lustre.StorageCapacity = params["FSxLustreConfiguration"]["capacity"] fsx_lustre.SecurityGroupIds = [params["SecurityGroupId"]] fsx_lustre.SubnetIds = params["SubnetId"] if params["FSxLustreConfiguration"]["s3_backend"] is not False: fsx_lustre_configuration = LustreConfiguration() fsx_lustre_configuration.ImportPath = params["FSxLustreConfiguration"]["import_path"] if params["FSxLustreConfiguration"]["import_path"] is not False else params["FSxLustreConfiguration"]["s3_backend"] fsx_lustre_configuration.ExportPath = params["FSxLustreConfiguration"]["import_path"] if params["FSxLustreConfiguration"]["import_path"] is not False else params["FSxLustreConfiguration"]["s3_backend"] + "/" + params["ClusterId"] + "-fsxoutput/job-" + params["JobId"] + "/" fsx_lustre.LustreConfiguration = fsx_lustre_configuration fsx_lustre.Tags = base_Tags( # False disable PropagateAtLaunch Name=str(params["ClusterId"] + "-compute-job-" + params["JobId"]), _soca_JobId=str(params["JobId"]), _soca_JobName=str(params["JobName"]), _soca_JobQueue=str(params["JobQueue"]), _soca_StackId=stack_name, _soca_JobOwner=str(params["JobOwner"]), _soca_JobProject=str(params["JobProject"]), _soca_KeepForever=str(params["KeepForever"]).lower(), _soca_FSx="true", _soca_ClusterId=str(params["ClusterId"]), ) t.add_resource(fsx_lustre) # End FSx For Lustre # Begin AutoScalingGroup Resource asg = AutoScalingGroup("AutoScalingComputeGroup") asg.DependsOn = "NodeLaunchTemplate" if mip_usage is True or instances_list.__len__() > 1: mip.LaunchTemplate = asg_lt asg.MixedInstancesPolicy = mip else: asg.LaunchTemplate = LaunchTemplateSpecification( LaunchTemplateId=Ref(lt), Version=GetAtt(lt, "LatestVersionNumber")) asg.MinSize = int(params["DesiredCapacity"]) asg.MaxSize = int(params["DesiredCapacity"]) asg.VPCZoneIdentifier = params["SubnetId"] if params["PlacementGroup"] is True: pg = PlacementGroup("ComputeNodePlacementGroup") pg.Strategy = "cluster" t.add_resource(pg) asg.PlacementGroup = Ref(pg) asg.Tags = Tags( Name=str(params["ClusterId"]) + "-compute-job-" + str(params["JobId"]), _soca_JobId=str(params["JobId"]), _soca_JobName=str(params["JobName"]), _soca_JobQueue=str(params["JobQueue"]), _soca_StackId=stack_name, _soca_JobOwner=str(params["JobOwner"]), _soca_JobProject=str(params["JobProject"]), _soca_KeepForever=str(params["KeepForever"]).lower(), _soca_ClusterId=str(params["ClusterId"]), _soca_NodeType="soca-compute-node") t.add_resource(asg) # End AutoScalingGroup Resource # Begin Custom Resource # Change Mapping to No if you want to disable this if allow_anonymous_data_collection is True: metrics = CustomResourceSendAnonymousMetrics("SendAnonymousData") metrics.ServiceToken = params["SolutionMetricLambda"] metrics.DesiredCapacity = str(params["DesiredCapacity"]) metrics.InstanceType = str(params["InstanceType"]) metrics.Efa = str(params["Efa"]) metrics.ScratchSize = str(params["ScratchSize"]) metrics.RootSize = str(params["RootSize"]) metrics.SpotPrice = str(params["SpotPrice"]) metrics.BaseOS = str(params["BaseOS"]) metrics.StackUUID = str(params["StackUUID"]) metrics.KeepForever = str(params["KeepForever"]) metrics.FsxLustre = str(params["FSxLustreConfiguration"]) t.add_resource(metrics) # End Custom Resource if debug is True: print(t.to_json()) # Tags must use "soca:<Key>" syntax template_output = t.to_yaml().replace("_soca_", "soca:") return {'success': True, 'output': template_output} except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] return {'success': False, 'output': 'cloudformation_builder.py: ' + ( str(e) + ': error :' + str(exc_type) + ' ' + str(fname) + ' ' + str(exc_tb.tb_lineno))}
""" Your module description """ from troposphere import Ref, Template import troposphere.ec2 as ec2 template = Template() envs = ['dev', 'test', 'prod'] for x in envs: instancename = x + "Ec2" ec2_instance = template.add_resource(ec2.Instance( instancename, ImageId="ami-a7a242da", InstanceType="t2.nano", )) fh = open("template.yaml", "a") fh.writelines(template.to_yaml()) fh.close()