def _template_init(self):
t = Template()
t.add_version("2010-09-09")
#t.add_parameter(Parameter("StackName",
# Description="Name of this stack",
# Type="String",))
return t
def _generate_template(tms=1, within_vpc=False):
t = Template()
t.add_description(FLINK_TEMPLATE_DESCRIPTION)
t.add_version(FLINK_TEMPLATE_VERSION)
t.add_metadata({'LastUpdated': datetime.datetime.now().strftime('%c')})
# mappings
mappings.add_mappings(t)
# parameters
parameters.add_parameters(t)
vpc = None
subnet_pri = None
subnet_pub = None
if within_vpc:
# networking resources
vpc, subnet_pri, subnet_pub = _define_vpc(t)
# security groups
sg_ssh = t.add_resource(securitygroups.ssh(
parameters.ssh_location, vpc))
sg_jobmanager = t.add_resource(securitygroups.jobmanager(
parameters.http_location, vpc))
sg_taskmanager = t.add_resource(securitygroups.taskmanager(None, vpc))
jobmanager = t.add_resource(instances.jobmanager(
0,
[Ref(sg_ssh), Ref(sg_jobmanager)],
within_vpc,
subnet_pub
))
prefix = "JobManager00"
t.add_output(outputs.ssh_to(jobmanager, prefix))
t.add_output(Output(
"FlinkWebGui",
Description="Flink web interface",
Value=Join("", [
'http://', GetAtt(jobmanager, "PublicDnsName"), ':8081'
])
))
for index in range(0, tms):
i = t.add_resource(instances.taskmanager(
index,
jobmanager,
[Ref(sg_ssh), Ref(sg_taskmanager)],
within_vpc,
subnet_pri
))
prefix = "TaskManager%2.2d" % index
t.add_output(outputs.ssh_to(i, prefix, bastion=jobmanager))
return t.to_json()
def generate_env_template(app_env, env_dict):
sg_name = env_dict['sg_name']
vpc_id = 'vpc-a1d187c4' # query for this!
logger.debug('generating template for %s' % vpc_id)
t = Template()
t.add_version('2010-09-09')
t.add_description('env template for %s' % app_env)
app_sg = SecurityGroup('TestAppSecurityGroup')
app_sg.VpcId = vpc_id
app_sg.GroupDescription = 'testing'
app_sg.Tags = name_tag(sg_name)
t.add_resource(app_sg)
return t.to_json()
def generate_application_template(app_dict):
app_name = app_dict['app_name']
t = Template()
t.add_version()
t.add_description('app template for %s' % app_name)
app = Application(app_name, Description=app_name)
t.add_resource(app)
bucket_name = 'ehi-pcf-%s' % app_name
bucket = Bucket('AppBucket', BucketName=bucket_name, AccessControl=Private)
t.add_resource(bucket)
return t.to_json()
def lambda_handler(event, context):
#print("Received event: " + json.dumps(event, indent=2))
# Get the object from the event and show its content type
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(event['Records'][0]['s3']['object']['key']).decode('utf8')
print ("buket:" + bucket)
print ("key:" + key)
obj = s3.Object(bucket,key)
response= obj.get()
body = response['Body'].read()
body_list= body.splitlines()
# VPC関係
VPC_CidrBlockList = body_list[0].split(',')
VPC_EnableDnsSupportList = body_list[1].split(',')
VPC_TagsValueList = body_list[2].split(',')
t = Template()
t.add_version("2010-09-09")
t.add_description("ROOP&ROOP")
if len(VPC_CidrBlockList) > 1:
for (address,dns,value) in zip(VPC_CidrBlockList[1:],VPC_EnableDnsSupportList[1:],VPC_TagsValueList[1:]):
t.add_resource(VPC(
value,
EnableDnsSupport="true",
CidrBlock=address,
EnableDnsHostnames=dns,
Tags=Tags(Name=value)
))
json_template = t.to_json()
bucket = s3.Bucket('cf-templates-hokan')
obj = bucket.Object('json-template-' + basename + ' .txt')
response = obj.put(
Body=json_template.encode('utf-8'),
ContentEncoding='utf-8',
ContentType='text/plane'
)
print(json_template)
def add_resources(self, resources, config):
"""
Creates JSON-formatted string representation of stack resourcs
suitable for use with AWS Cloudformation
:param resources: Internal data structure of resources
:type resources: dict.
:param config: Config key/value pairs
:type config: dict.
:returns: string
:raises: :class:`pmcf.exceptions.ProvisionerException`
"""
LOG.info('Start building template')
data = Template()
desc = "%s %s stack" % (config['name'], config['environment'])
data.add_description(desc)
data.add_version()
self._add_streams(data, resources.get('stream', []), config)
self._add_queues(data, resources.get('queue', []), config)
self._add_nets(data, resources.get('network', []), config)
sgs = self._add_secgroups(data, resources['secgroup'], config)
self._add_caches(data, resources.get('cache', []), config, sgs)
lbs = self._add_lbs(data,
resources['load_balancer'],
config,
sgs,
resources['instance'])
self._add_instances(data,
resources['instance'],
config,
sgs,
lbs)
LOG.info('Finished building template')
return data.to_json(indent=None)
def main():
template = Template()
template.add_version("2010-09-09")
template.add_description(
"AWS CloudFormation Sample Template: ELB with 2 EC2 instances")
AddAMI(template)
# Add the Parameters
keyname_param = template.add_parameter(Parameter(
"KeyName",
Type="String",
Default="mark",
Description="Name of an existing EC2 KeyPair to "
"enable SSH access to the instance",
))
template.add_parameter(Parameter(
"InstanceType",
Type="String",
Description="WebServer EC2 instance type",
Default="m1.small",
AllowedValues=[
"t1.micro", "m1.small", "m1.medium", "m1.large", "m1.xlarge",
"m2.xlarge", "m2.2xlarge", "m2.4xlarge", "c1.medium", "c1.xlarge",
"cc1.4xlarge", "cc2.8xlarge", "cg1.4xlarge"
],
ConstraintDescription="must be a valid EC2 instance type.",
))
webport_param = template.add_parameter(Parameter(
"WebServerPort",
Type="String",
Default="8888",
Description="TCP/IP port of the web server",
))
apiport_param = template.add_parameter(Parameter(
"ApiServerPort",
Type="String",
Default="8889",
Description="TCP/IP port of the api server",
))
subnetA = template.add_parameter(Parameter(
"subnetA",
Type="String",
Default="subnet-096fd06d"
))
subnetB = template.add_parameter(Parameter(
"subnetB",
Type="String",
Default="subnet-1313ef4b"
))
VpcId = template.add_parameter(Parameter(
"VpcId",
Type="String",
Default="vpc-82c514e6"
))
# Define the instance security group
instance_sg = template.add_resource(
ec2.SecurityGroup(
"InstanceSecurityGroup",
GroupDescription="Enable SSH and HTTP access on the inbound port",
SecurityGroupIngress=[
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort="22",
ToPort="22",
CidrIp="0.0.0.0/0",
),
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=Ref(webport_param),
ToPort=Ref(webport_param),
CidrIp="0.0.0.0/0",
),
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=Ref(apiport_param),
ToPort=Ref(apiport_param),
CidrIp="0.0.0.0/0",
),
]
)
)
# Add the web server instance
WebInstance = template.add_resource(ec2.Instance(
"WebInstance",
SecurityGroups=[Ref(instance_sg)],
KeyName=Ref(keyname_param),
InstanceType=Ref("InstanceType"),
ImageId=FindInMap("RegionMap", Ref("AWS::Region"), "AMI"),
UserData=Base64(Ref(webport_param)),
))
# Add the api server instance
ApiInstance = template.add_resource(ec2.Instance(
"ApiInstance",
SecurityGroups=[Ref(instance_sg)],
KeyName=Ref(keyname_param),
InstanceType=Ref("InstanceType"),
ImageId=FindInMap("RegionMap", Ref("AWS::Region"), "AMI"),
UserData=Base64(Ref(apiport_param)),
))
# Add the application ELB
ApplicationElasticLB = template.add_resource(elb.LoadBalancer(
"ApplicationElasticLB",
Name="ApplicationElasticLB",
Scheme="internet-facing",
Subnets=[Ref(subnetA), Ref(subnetB)]
))
TargetGroupWeb = template.add_resource(elb.TargetGroup(
"TargetGroupWeb",
HealthCheckIntervalSeconds="30",
HealthCheckProtocol="HTTP",
HealthCheckTimeoutSeconds="10",
HealthyThresholdCount="4",
Matcher=elb.Matcher(
HttpCode="200"),
Name="WebTarget",
Port=Ref(webport_param),
Protocol="HTTP",
Targets=[elb.TargetDescription(
Id=Ref(WebInstance),
Port=Ref(webport_param))],
UnhealthyThresholdCount="3",
VpcId=Ref(VpcId)
))
TargetGroupApi = template.add_resource(elb.TargetGroup(
"TargetGroupApi",
HealthCheckIntervalSeconds="30",
HealthCheckProtocol="HTTP",
HealthCheckTimeoutSeconds="10",
HealthyThresholdCount="4",
Matcher=elb.Matcher(
HttpCode="200"),
Name="ApiTarget",
Port=Ref(apiport_param),
Protocol="HTTP",
Targets=[elb.TargetDescription(
Id=Ref(ApiInstance),
Port=Ref(apiport_param))],
UnhealthyThresholdCount="3",
VpcId=Ref(VpcId)
))
Listener = template.add_resource(elb.Listener(
"Listener",
Port="80",
Protocol="HTTP",
LoadBalancerArn=Ref(ApplicationElasticLB),
DefaultActions=[elb.Action(
Type="forward",
TargetGroupArn=Ref(TargetGroupWeb)
)]
))
template.add_resource(elb.ListenerRule(
"ListenerRuleApi",
ListenerArn=Ref(Listener),
Conditions=[elb.Condition(
Field="path-pattern",
Values=["/api/*"])],
Actions=[elb.Action(
Type="forward",
TargetGroupArn=Ref(TargetGroupApi)
)],
Priority="1"
))
template.add_output(Output(
"URL",
Description="URL of the sample website",
Value=Join("", ["http://", GetAtt(ApplicationElasticLB, "DNSName")])
))
print(template.to_json())
class CreateTemplate(object):
def __init__(self):
self.start_template()
self.add_mappings()
self.add_parameters()
self.add_security_groups()
self.add_kippo_rds()
self.add_kippo_sensors()
def start_template(self):
self.template = Template()
self.template.add_version('2010-09-09')
self.template.add_description('Kippo cluster CloudFormation stack')
def add_mappings(self):
# Ubuntu Trusty 14.04 LTS amd64
self.template.add_mapping('Ec2AmiMap', {
'ap-northeast-1': {'AmiId': 'ami-c011d4c0'},
'ap-southeast-1': {'AmiId': 'ami-76546924'},
'eu-central-1': {'AmiId': 'ami-00dae61d'},
'eu-west-1': {'AmiId': 'ami-2396f654'},
'sa-east-1': {'AmiId': 'ami-75b23768'},
'us-east-1': {'AmiId': 'ami-f63b3e9e'},
'us-west-1': {'AmiId': 'ami-057f9d41'},
'cn-north-1': {'AmiId': 'ami-78d84541'},
'us-gov-west-1': {'AmiId': 'ami-85fa9ba6'},
})
def add_parameters(self):
self.template.add_parameter(Parameter(
'Ec2InstanceType',
Default='t2.micro',
Description='Instance type of the EC2 instances',
Type='String',
))
self.template.add_parameter(Parameter(
'Ec2SubnetIdList',
Description='List of subnet IDs in which to create the EC2 instances',
Type='List<AWS::EC2::Subnet::Id>',
))
self.template.add_parameter(Parameter(
'ElbSubnetIdList',
Description='List of subnet IDs in which to create the ELB',
Type='List<AWS::EC2::Subnet::Id>',
))
self.template.add_parameter(Parameter(
'KeyName',
Description='Name of the keypair to install on the EC2 instances',
Type='AWS::EC2::KeyPair::KeyName',
))
self.template.add_parameter(Parameter(
'KippoSensorCount',
Default='1',
Description='Number of kippo sensors to create',
Type='Number',
))
self.template.add_parameter(Parameter(
'RdsInstanceType',
Default='t2.micro',
Description='Instance type of the RDS instance',
Type='String',
))
self.template.add_parameter(Parameter(
'RdsRootPassword',
Description='Password to use for the root RDS user',
Type='String',
))
self.template.add_parameter(Parameter(
'RdsStorage',
Default='20',
Description='Amount of storage (GB) for the RDS instance',
Type='Number',
))
self.template.add_parameter(Parameter(
'RdsSubnetIdList',
Description='List of subnet IDs in which to create the RDS instance',
Type='List<AWS::EC2::Subnet::Id>',
))
self.template.add_parameter(Parameter(
'RealSshPort',
Description='Port number to use for the real SSH service',
Type='Number',
))
self.template.add_parameter(Parameter(
'VpcId',
Description='ID of the VPC in which to create the kippo cluster',
Type='AWS::EC2::VPC::Id',
))
def add_security_groups(self):
# kippo sensor EC2 security group
self.template.add_resource(SecurityGroup(
'Ec2SecurityGroup',
GroupDescription='Security group for the kippo sensor ASG',
SecurityGroupIngress=[
# Allow SSH (to kippo) from anywhere
SecurityGroupRule(
CidrIp='0.0.0.0/0',
FromPort=22,
ToPort=22,
IpProtocol='tcp',
),
# Allow real SSH from anywhere
SecurityGroupRule(
CidrIp='0.0.0.0/0',
FromPort=Ref('RealSshPort'),
ToPort=Ref('RealSshPort'),
IpProtocol='tcp',
),
# Allow HTTP from the kippo ELB
SecurityGroupRule(
FromPort=80,
ToPort=80,
IpProtocol='tcp',
SourceSecurityGroupId=Ref('ElbSecurityGroup'),
),
# Allow HTTPS from the kippo ELB
SecurityGroupRule(
FromPort=443,
ToPort=443,
IpProtocol='tcp',
SourceSecurityGroupId=Ref('ElbSecurityGroup'),
),
],
VpcId=Ref('VpcId'),
))
# kippo sensor ELB security group
self.template.add_resource(SecurityGroup(
'ElbSecurityGroup',
GroupDescription='Security group for the kippo sensor ELB',
SecurityGroupIngress=[
# Allow HTTP from anywhere
SecurityGroupRule(
CidrIp='0.0.0.0/0',
FromPort=80,
ToPort=80,
IpProtocol='tcp',
),
# Allow HTTPS from anywhere
SecurityGroupRule(
CidrIp='0.0.0.0/0',
FromPort=443,
ToPort=443,
IpProtocol='tcp',
),
],
VpcId=Ref('VpcId'),
))
# RDS security group
self.template.add_resource(SecurityGroup(
'RdsSecurityGroup',
VpcId=Ref('VpcId'),
GroupDescription='Security group for the kippo RDS instance',
SecurityGroupIngress=[
# Allow MySQL from kippo EC2 instances
SecurityGroupRule(
FromPort=3306,
ToPort=3306,
IpProtocol='tcp',
SourceSecurityGroupId=Ref('Ec2SecurityGroup'),
),
],
))
def add_kippo_rds(self):
self.template.add_resource(DBSubnetGroup(
'RdsSubnetGroup',
DBSubnetGroupDescription='Subnet group for the kippo RDS instance',
SubnetIds=Ref('RdsSubnetIdList'),
))
self.template.add_resource(DBInstance(
'RdsInstance',
AllocatedStorage=Ref('RdsStorage'),
DBInstanceClass=Ref('RdsInstanceType'),
DBInstanceIdentifier='kippo-database',
DBSubnetGroupName=Ref('RdsSubnetGroup'),
Engine='MySQL',
EngineVersion='5.6.22',
MasterUsername='******',
MasterUserPassword=Ref('RdsRootPassword'),
MultiAZ=True,
Port=3306,
VPCSecurityGroups=[Ref('RdsSecurityGroup')],
))
self.template.add_output(Output(
'RdsEndpoint',
Description='RDS endpoint address',
Value=GetAtt('RdsInstance', 'Endpoint.Address'),
))
def add_kippo_sensors(self):
# Create the ELB
self.template.add_resource(LoadBalancer(
'Elb',
Listeners=[
Listener(
InstancePort=80,
LoadBalancerPort=80,
Protocol='http',
),
Listener(
InstancePort=443,
LoadBalancerPort=443,
Protocol='tcp', # Plain TCP forwarding for HTTPS/SSL
),
],
CrossZone=True,
Subnets=Ref('ElbSubnetIdList'),
SecurityGroups=[Ref('ElbSecurityGroup')],
Scheme='internet-facing',
HealthCheck=HealthCheck(
Target='HTTP:80/kippo-graph/',
HealthyThreshold=2,
UnhealthyThreshold=5,
Interval=120,
Timeout=60,
),
))
self.template.add_output(Output(
'ElbEndpoint',
Description='ELB endpoint address',
Value=GetAtt('Elb', 'DNSName'),
))
self.template.add_resource(LaunchConfiguration(
'LaunchConfiguration',
KeyName=Ref('KeyName'),
ImageId=FindInMap('Ec2AmiMap', Ref('AWS::Region'), 'AmiId'),
InstanceType=Ref('Ec2InstanceType'),
SecurityGroups=[Ref('Ec2SecurityGroup')],
AssociatePublicIpAddress=True,
UserData=Base64(Join('\n', [
'#cloud-config',
'repo_upgrade: security',
'runcmd:',
' - "/usr/bin/wget -O /tmp/configure_kippo_sensor.sh https://raw.githubusercontent.com/cdodd/aws-kippo-cluster/master/bootstrap/configure_kippo_sensor.sh"',
Join(
'',
[
' - "bash /tmp/configure_kippo_sensor.sh',
' ',
GetAtt('RdsInstance', 'Endpoint.Address'),
' ',
Ref('RdsRootPassword'),
' ',
Ref('RealSshPort'),
'"',
],
),
])),
))
self.template.add_resource(AutoScalingGroup(
'Asg',
DesiredCapacity=Ref('KippoSensorCount'),
HealthCheckGracePeriod=1800,
HealthCheckType='ELB',
LaunchConfigurationName=Ref('LaunchConfiguration'),
LoadBalancerNames=[Ref('Elb')],
MaxSize=Ref('KippoSensorCount'),
MinSize=Ref('KippoSensorCount'),
Tags=[Tag(key='Name', value='kippo-sensor', propogate='true')],
VPCZoneIdentifier=Ref('Ec2SubnetIdList'),
))
def output_template(self):
return self.template.to_json()
def generate_template(service_name):
t = Template()
t.add_version('2010-09-09')
t.add_description("""\
AWS CloudFormation Template for AWS Exploitation Lab """)
t.add_mapping("PublicRegionMap", ami_public_mapping)
t.add_mapping("PrivateRegionMap", ami_private_mapping)
keyname_param = t.add_parameter(
Parameter(
'KeyName',
ConstraintDescription='must be the name of an existing EC2 KeyPair.',
Description='Name of an existing EC2 KeyPair to enable SSH access to \
the instance',
Type='AWS::EC2::KeyPair::KeyName',
))
sshlocation_param = t.add_parameter(
Parameter(
'SSHLocation',
Description=' The IP address range that can be used to SSH to the EC2 \
instances',
Type='String',
MinLength='9',
MaxLength='18',
Default='0.0.0.0/0',
AllowedPattern="(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})",
ConstraintDescription=(
"must be a valid IP CIDR range of the form x.x.x.x/x."),
))
instanceType_param = t.add_parameter(Parameter(
'InstanceType',
Type='String',
Description='WebServer EC2 instance type',
Default='t2.micro',
AllowedValues=[
't2.micro', 't2.small', 't2.medium',
'm3.medium', 'm3.large', 'm3.xlarge', 'm3.2xlarge',
],
ConstraintDescription='must be a valid EC2 instance type.',
))
ref_stack_id = Ref('AWS::StackId')
ec2_role = t.add_resource(Role(
"%sEC2Role" % service_name,
AssumeRolePolicyDocument=awacs.aws.Policy(
Statement=[
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[awacs.aws.Action("sts", "AssumeRole")],
Principal=awacs.aws.Principal("Service", ["ec2.amazonaws.com"])
)
]
)
))
ec2_role.ManagedPolicyArns = [
"arn:aws:iam::aws:policy/ReadOnlyAccess"
]
ec2_snapshot_policy_document = awacs.aws.Policy(
Statement=[
awacs.aws.Statement(
Sid="PermitEC2Snapshots",
Effect=awacs.aws.Allow,
Action=[
awacs.aws.Action("ec2", "CreateSnapshot"),
awacs.aws.Action("ec2", "ModifySnapshotAttribute"),
],
Resource=["*"]
)
]
)
ec2_snapshot_policy = Policy(
PolicyName="EC2SnapshotPermissions",
PolicyDocument=ec2_snapshot_policy_document
)
priv_ec2_role = t.add_resource(Role(
"%sPrivEC2Role" % service_name,
AssumeRolePolicyDocument=awacs.aws.Policy(
Statement=[
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[awacs.aws.Action("sts", "AssumeRole")],
Principal=awacs.aws.Principal("Service", ["ec2.amazonaws.com"])
)
]
),
Policies=[ec2_snapshot_policy]
))
priv_ec2_role.ManagedPolicyArns = [
"arn:aws:iam::aws:policy/ReadOnlyAccess"
]
VPC_ref = t.add_resource(
VPC(
'VPC',
CidrBlock='10.0.0.0/16',
Tags=Tags(
Application=ref_stack_id)))
instanceProfile = t.add_resource(
InstanceProfile(
"InstanceProfile",
InstanceProfileName="%sInstanceRole" % (service_name),
Roles=[Ref(ec2_role)]))
privInstanceProfile = t.add_resource(
InstanceProfile(
"PrivInstanceProfile",
InstanceProfileName="%sPrivInstanceRole" % (service_name),
Roles=[Ref(priv_ec2_role)]))
public_subnet = t.add_resource(
Subnet(
'%sSubnetPublic' % service_name,
MapPublicIpOnLaunch=True,
CidrBlock='10.0.1.0/24',
VpcId=Ref(VPC_ref),
Tags=Tags(
Application=ref_stack_id,
Name="%sSubnet_public" % (service_name))
)
)
private_subnet = t.add_resource(
Subnet(
'%sSubnetPrivate' % service_name,
MapPublicIpOnLaunch=False,
CidrBlock='10.0.2.0/24',
VpcId=Ref(VPC_ref),
Tags=Tags(
Application=ref_stack_id,
Name="%sSubnet_private" % (service_name))
)
)
internetGateway = t.add_resource(
InternetGateway(
'InternetGateway',
Tags=Tags(
Application=ref_stack_id,
Name="%sInternetGateway" % service_name)))
gatewayAttachment = t.add_resource(
VPCGatewayAttachment(
'AttachGateway',
VpcId=Ref(VPC_ref),
InternetGatewayId=Ref(internetGateway)))
routeTable = t.add_resource(
RouteTable(
'RouteTable',
VpcId=Ref(VPC_ref),
Tags=Tags(
Application=ref_stack_id,
Name="%sRouteTable" % service_name)))
route = t.add_resource(
Route(
'Route',
DependsOn='AttachGateway',
GatewayId=Ref('InternetGateway'),
DestinationCidrBlock='0.0.0.0/0',
RouteTableId=Ref(routeTable),
))
# Only associate this Route Table with the public subnet
subnetRouteTableAssociation = t.add_resource(
SubnetRouteTableAssociation(
'SubnetRouteTableAssociation',
SubnetId=Ref(public_subnet),
RouteTableId=Ref(routeTable),
))
instanceSecurityGroup = t.add_resource(
SecurityGroup(
'InstanceSecurityGroup',
GroupDescription='%sSecurityGroup' % service_name,
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol='tcp',
FromPort='22',
ToPort='22',
CidrIp=Ref(sshlocation_param)),
SecurityGroupRule(
IpProtocol='tcp',
FromPort='80',
ToPort='80',
CidrIp='0.0.0.0/0'),
SecurityGroupRule(
IpProtocol='tcp',
FromPort='1080',
ToPort='1080',
CidrIp='0.0.0.0/0'),
SecurityGroupRule(
IpProtocol='tcp',
FromPort='443',
ToPort='443',
CidrIp='0.0.0.0/0'),
SecurityGroupRule(
IpProtocol='tcp',
FromPort='0',
ToPort='65535',
CidrIp="10.0.0.0/8"),
],
VpcId=Ref(VPC_ref),
)
)
public_instance = t.add_resource(
Instance(
"Public%sInstance" % service_name,
ImageId=FindInMap("PublicRegionMap", Ref("AWS::Region"), "AMI"),
InstanceType=Ref(instanceType_param),
KeyName=Ref(keyname_param),
NetworkInterfaces=[
NetworkInterfaceProperty(
GroupSet=[
Ref(instanceSecurityGroup)],
AssociatePublicIpAddress='true',
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(public_subnet))],
UserData=Base64(public_instance_userdata),
Tags=Tags(
Application=ref_stack_id,
Name='%sPublicInstance' % (service_name))
)
)
private_instance = t.add_resource(
Instance(
"Private%sInstance" % service_name,
ImageId=FindInMap("PrivateRegionMap", Ref("AWS::Region"), "AMI"),
InstanceType=Ref(instanceType_param),
KeyName=Ref(keyname_param),
NetworkInterfaces=[
NetworkInterfaceProperty(
GroupSet=[
Ref(instanceSecurityGroup)],
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(private_subnet))],
UserData=Base64(private_instance_userdata),
Tags=Tags(
Application=ref_stack_id,
Name='%sPrivateInstance' % (service_name)),
IamInstanceProfile="%sPrivInstanceRole" % (service_name)
)
)
outputs = []
outputs.append(
Output(
"PublicIP",
Description="IP Address of Public Instance",
Value=GetAtt(public_instance, "PublicIp"),
)
)
t.add_output(outputs)
# Set up S3 Bucket and CloudTrail
S3Bucket = t.add_resource(
Bucket(
"S3Bucket",
DeletionPolicy="Retain"
)
)
S3PolicyDocument=awacs.aws.PolicyDocument(
Id='EnforceServersideEncryption',
Version='2012-10-17',
Statement=[
awacs.aws.Statement(
Sid='PermitCTBucketPut',
Action=[s3.PutObject],
Effect=awacs.aws.Allow,
Principal=awacs.aws.Principal("Service", ["cloudtrail.amazonaws.com"]),
Resource=[Join('', [s3.ARN(''), Ref(S3Bucket), "/*"])],
),
awacs.aws.Statement(
Sid='PermitCTBucketACLRead',
Action=[s3.GetBucketAcl],
Effect=awacs.aws.Allow,
Principal=awacs.aws.Principal("Service", ["cloudtrail.amazonaws.com"]),
Resource=[Join('', [s3.ARN(''), Ref(S3Bucket)])],
)
]
)
S3BucketPolicy = t.add_resource(
BucketPolicy(
"BucketPolicy",
PolicyDocument=S3PolicyDocument,
Bucket=Ref(S3Bucket),
DependsOn=[S3Bucket]
)
)
myTrail = t.add_resource(
Trail(
"CloudTrail",
IsLogging=True,
S3BucketName=Ref(S3Bucket),
DependsOn=["BucketPolicy"],
)
)
myTrail.IsMultiRegionTrail = True
myTrail.IncludeGlobalServiceEvents = True
return t.to_json()
EMAIL_PATTERN = '^[\w!#$%&\'*+/=?`{|}~^-]+(?:\.[\w!#$%&\'*+/=?`{|}~^-]+)*@(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,6}$'
# pylint: enable=anomalous-backslash-in-string
ANSIBLE_PULL_URL = os.getenv('ANSIBLE_PULL_URL',
'https://github.com/trustedanalytics/ansible-playbooks.git')
ANSIBLE_PULL_CHECKOUT = os.getenv('ANSIBLE_PULL_CHECKOUT', 'master')
ANSIBLE_GROUP_VARS = [
'ntp_server=[\'0.amazon.pool.ntp.org\', \'1.amazon.pool.ntp.org\']\n',
'provider=aws\n',
]
TEMPLATE = Template()
TEMPLATE.add_version('2010-09-09')
TEMPLATE.add_description('Trusted Analytics Platform (TAP) is open source software, optimized for '
'performance and security, that accelerates the creation of Cloud-native '
'applications driven by Big Data Analytics. TAP makes it easier for '
'developers and data scientists, at enterprises, CSPs and SIs, to '
'collaborate by providing a shared, flexible environment for advanced '
'analytics in public and private Clouds. Data scientists get extensible '
'tools, scalable algorithms and powerful engines to train and deploy '
'predictive models. Developers get consistent APIs, services and runtimes'
' to quickly integrate these models into applications. System Operators '
'get an integrated stack that they can easily provision in a Cloud '
'infrastructure.')
TEMPLATE.add_mapping('Region2AMI', {
EU_WEST_1: {'Ubuntu': 'ami-cd0fd6be', 'RHEL': 'ami-78d29c0f'},
class CloudFormationBuilder(object):
def __init__(self, use_firehose=False, use_processor=False):
self._random_id = str(uuid.uuid4()).replace('-', '')[:12].upper()
self._template = Template()
self._template.add_version("2010-09-09")
desc = "AWS CloudFormation Push Stack"
if use_processor:
desc += " - with Firehose Logging + Processor + Push Messages API"
elif use_firehose:
desc += " - with Firehose Logging"
self._template.add_description(desc)
self.use_firehose = use_firehose or use_processor
self.use_processor = use_processor
self.add_resource = self._template.add_resource
self.add_parameter = self._template.add_parameter
self.AutopushVersion = self.add_parameter(Parameter(
"AutopushVersion",
Type="String",
Description="Autopush version to deploy",
Default="1.14.2",
AllowedValues=[
"latest",
"1.14.2",
"1.14.1",
]
))
if self.use_processor:
self.ProcessorLambdaBucket = self.add_parameter(Parameter(
"ProcessorLambdaBucket",
Type="String",
Default="push-lambda-funcs",
Description="S3 Bucket of lambda Message Processor",
))
self.ProcessorLambdaKey = self.add_parameter(Parameter(
"ProcessorLambdaKey",
Type="String",
Default="push_processor_0.4.zip",
Description="S3 Key of lambda Message Processor",
))
self.ProcessorVPCId = self.add_parameter(Parameter(
"ProcessorVPCId",
Type="AWS::EC2::VPC::Id",
Description="VPC to run the processor/Messages API in"
))
self.ProcessorSubnetIds = self.add_parameter(Parameter(
"ProcessorSubnetIds",
Type="List<AWS::EC2::Subnet::Id>",
Description=(
"Processor Subnet ID's, MUST have NAT Gateway "
"access for S3 loading"
),
))
self.MessagesSecurityGroup = self.add_parameter(Parameter(
"MessagesSecurityGroup",
Type="AWS::EC2::SecurityGroup::Id",
Description="Security Group to allow Messages API access from",
))
self.MessageAPISubnetId = self.add_parameter(Parameter(
"MessageApiEC2Subnet",
Type="AWS::EC2::Subnet::Id",
Description=(
"Subnet to run Push Messages EC2 Instance in, MUST "
"be in the same VPC as the Processor Subnets"
)
))
self.PushMessagesVersion = self.add_parameter(Parameter(
"PushMessagesVersion",
Type="String",
Description="Push-Messages API version to deploy",
Default="0.6",
AllowedValues=[
"latest",
"0.5",
"0.6"
]
))
self.PushCryptoKey = self.add_parameter(Parameter(
"AutopushCryptoKey",
Type="String",
Default=Fernet.generate_key(),
Description="Autopush crypto-key",
MinLength=44,
MaxLength=44,
))
self.KeyPair = self.add_parameter(Parameter(
"AutopushSSHKeyPair",
Type="AWS::EC2::KeyPair::KeyName",
Description="Name of an EC2 KeyPair to enable SSH access."
))
self.PushTablePrefix = self.add_parameter(Parameter(
"PushTablePrefix",
Type="String",
Default="autopush_" + self._random_id,
Description="Autopush DynamoDB Table Prefixes",
))
if self.use_firehose:
self._setup_firehose_custom_resource()
self._add_firehose()
self._add_autopush_security_group()
self._add_autopush_iam_roles()
self._add_autopush_servers()
if self.use_processor:
self._add_processor_databases()
self._setup_s3writer_custom_resource()
self._add_processor()
self._add_push_messages_api()
def _add_autopush_security_group(self):
self.InternalRouterSG = self.add_resource(SecurityGroup(
"AutopushInternalRouter",
GroupDescription="Internal Routing SG"
))
self.EndpointSG = self.add_resource(SecurityGroup(
"AutopushEndpointNode",
SecurityGroupIngress=[
allow_tcp(8082),
allow_tcp(22),
],
GroupDescription="Allow HTTP traffic to autoendpoint node",
))
self.ConnectionSG = self.add_resource(SecurityGroup(
"AutopushConnectionNode",
SecurityGroupIngress=[
allow_tcp(8080),
allow_tcp(22),
SecurityGroupRule(
IpProtocol="tcp",
FromPort=8081,
ToPort=8081,
SourceSecurityGroupName=Ref(self.InternalRouterSG)
)
],
GroupDescription=(
"Allow Websocket traffic to autopush node"
)
))
def _add_autopush_iam_roles(self):
firehose_extras = []
if self.use_firehose:
# Add in the firehose permissions
firehose_extras.append(Statement(
Effect=Allow,
Action=[
Action("firehose", "PutRecord"),
Action("firehose", "PutRecordBatch"),
],
Resource=[
GetAtt(self.FirehoseLogstream, "Arn"),
]
))
self.PushServerRole = self.add_resource(Role(
"AutopushServerRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", "ec2.amazonaws.com")
)
]
),
Path="/",
))
self.add_resource(PolicyType(
"AutopushServerRolePolicy",
PolicyName="AutopushServerRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[
ddb.BatchGetItem,
ddb.BatchWriteItem,
ddb.GetItem,
ddb.PutItem,
ddb.DeleteItem,
ddb.UpdateItem,
ddb.Query,
ddb.Scan,
],
Resource=[
Join("", ["arn:aws:dynamodb:us-east-1:*:table/",
Ref(self.PushTablePrefix),
"_*"]
)
]
),
Statement(
Effect=Allow,
Action=[
ddb.ListTables,
ddb.DescribeTable,
ddb.CreateTable,
],
Resource=["*"]
)
] + firehose_extras
),
Roles=[Ref(self.PushServerRole)]
))
def _aws_cfn_signal_service(self, wait_for, resource):
"""Returns an array suitable to join for UserData that signals after
the wait_for service has started
"""
return [
" - name: 'aws_cfn_signal.service'\n",
" command: 'start'\n",
" content: |\n",
" [Unit]\n",
" Description=AWS Cloud Formation Signaling\n",
" After=%s.service\n" % wait_for,
" Requires=%s.service\n" % wait_for,
" Type=oneshot\n",
" \n",
" [Service]\n",
" TimeoutStartSec=0\n",
" EnvironmentFile=/etc/environment\n",
" ExecStartPre=/usr/bin/docker pull ",
"aweber/cfn-signal\n",
" ExecStart=/usr/bin/docker run --name cfn-signal ",
"aweber/cfn-signal --success=true --reason='Registry Started'",
" --stack=", Ref("AWS::StackName"),
" --resource=", resource, "\n",
]
def _instance_tags(self, app, app_type):
return Tags(
App=app,
Datadog="false",
Env="testing",
Name=Join("", [Ref("AWS::StackName"), "-", app_type]),
Stack=Ref("AWS::StackName"),
Type=app_type,
)
def _add_autopush_servers(self):
self.PushServerInstanceProfile = self.add_resource(InstanceProfile(
"AutopushServerInstanceProfile",
Path="/",
Roles=[Ref(self.PushServerRole)]
))
# Extras is common options for UserData
extras = [
"--hostname $public_ipv4 ",
"--storage_tablename ", Ref(self.PushTablePrefix), "_storage ",
"--message_tablename ", Ref(self.PushTablePrefix), "_message ",
"--router_tablename ", Ref(self.PushTablePrefix), "_router ",
"--crypto_key '", Ref(self.PushCryptoKey), "' ",
]
if self.use_firehose:
extras.extend([
"--firehose_stream_name ", Ref(self.FirehoseLogstream), " "
])
self.PushEndpointServerInstance = self.add_resource(Instance(
"AutopushEndpointInstance",
ImageId="ami-2c393546",
InstanceType="t2.micro",
SecurityGroups=[
Ref(self.EndpointSG),
Ref(self.InternalRouterSG),
],
KeyName=Ref(self.KeyPair),
IamInstanceProfile=Ref(self.PushServerInstanceProfile),
CreationPolicy=CreationPolicy(
ResourceSignal=ResourceSignal(
Timeout='PT15M'
)
),
UserData=Base64(Join("", [
"#cloud-config\n\n",
"coreos:\n",
" units:\n",
] + self._aws_cfn_signal_service(
"autoendpoint", "AutopushEndpointInstance") + [
" - name: 'autoendpoint.service'\n",
" command: 'start'\n",
" content: |\n",
" [Unit]\n",
" Description=Autoendpoint container\n",
" Author=Mozilla Services\n",
" After=docker.service\n",
" \n",
" [Service]\n",
" Restart=always\n",
" ExecStartPre=-/usr/bin/docker kill autoendpoint\n",
" ExecStartPre=-/usr/bin/docker rm autoendpoint\n",
" ExecStartPre=/usr/bin/docker pull ",
"bbangert/autopush:", Ref(self.AutopushVersion), "\n",
" ExecStart=/usr/bin/docker run ",
"--name autoendpoint ",
"-p 8082:8082 ",
"-e 'AWS_DEFAULT_REGION=us-east-1' ",
"bbangert/autopush:", Ref(self.AutopushVersion), " ",
"./pypy/bin/autoendpoint ",
] + extras)),
DependsOn="AutopushServerRolePolicy",
Tags=self._instance_tags("autopush", "autoendpoint"),
))
self.PushConnectionServerInstance = self.add_resource(Instance(
"AutopushConnectionInstance",
ImageId="ami-2c393546",
InstanceType="t2.micro",
SecurityGroups=[
Ref(self.ConnectionSG),
Ref(self.InternalRouterSG),
],
KeyName=Ref(self.KeyPair),
IamInstanceProfile=Ref(self.PushServerInstanceProfile),
CreationPolicy=CreationPolicy(
ResourceSignal=ResourceSignal(
Timeout='PT15M'
)
),
UserData=Base64(Join("", [
"#cloud-config\n\n",
"coreos:\n",
" units:\n",
] + self._aws_cfn_signal_service(
"autopush", "AutopushConnectionInstance") + [
" - name: 'autopush.service'\n",
" command: 'start'\n",
" content: |\n",
" [Unit]\n",
" Description=Autopush container\n",
" Author=Mozilla Services\n",
" After=docker.service\n",
" \n",
" [Service]\n",
" Restart=always\n",
" ExecStartPre=-/usr/bin/docker kill autopush\n",
" ExecStartPre=-/usr/bin/docker rm autopush\n",
" ExecStartPre=/usr/bin/docker pull ",
"bbangert/autopush:", Ref(self.AutopushVersion), "\n",
" ExecStart=/usr/bin/docker run ",
"--name autopush ",
"-p 8080:8080 ",
"-p 8081:8081 ",
"-e 'AWS_DEFAULT_REGION=us-east-1' ",
"bbangert/autopush:", Ref(self.AutopushVersion), " ",
"./pypy/bin/autopush ",
"--router_hostname $private_ipv4 ",
"--endpoint_hostname ",
GetAtt(self.PushEndpointServerInstance, "PublicDnsName"),
" ",
] + extras)),
DependsOn="AutopushServerRolePolicy",
Tags=self._instance_tags("autopush", "autopush"),
))
self._template.add_output([
Output(
"PushServerURL",
Description="Push Websocket URL",
Value=Join("", [
"ws://",
GetAtt(self.PushConnectionServerInstance, "PublicDnsName"),
":8080/"
])
)
])
def _setup_firehose_custom_resource(self):
# Setup the FirehoseLambda CloudFormation Custom Resource
self.FirehoseLambdaCFExecRole = self.add_resource(Role(
"FirehoseLambdaCFRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", "lambda.amazonaws.com")
)
]
),
Path="/",
))
self.FirehoseLambdaPolicy = self.add_resource(PolicyType(
"FirehoseCFPolicy",
PolicyName="FirehoseLambdaCFRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[
Action("logs", "CreateLogGroup"),
Action("logs", "CreateLogStream"),
Action("logs", "PutLogEvents"),
],
Resource=[
"arn:aws:logs:*:*:*"
]
),
Statement(
Effect=Allow,
Action=[
Action("firehose", "CreateDeliveryStream"),
Action("firehose", "DeleteDeliveryStream"),
Action("firehose", "ListDeliveryStreams"),
Action("firehose", "DescribeDeliveryStream"),
Action("firehose", "UpdateDestination"),
],
Resource=["*"]
)
]
),
Roles=[Ref(self.FirehoseLambdaCFExecRole)],
DependsOn="FirehoseLambdaCFRole"
))
self.FirehoseCFCustomResource = self.add_resource(Function(
"FirehoseCustomResource",
Description=(
"Creates, updates, and deletes Firehose delivery streams"
),
Runtime="python2.7",
Timeout=300,
Handler="lambda_function.lambda_handler",
Role=GetAtt(self.FirehoseLambdaCFExecRole, "Arn"),
Code=Code(
S3Bucket="cloudformation-custom-resources",
S3Key="firehose_lambda.zip",
),
DependsOn="FirehoseCFPolicy"
))
def _setup_s3writer_custom_resource(self):
self.S3WriterLambdaCFExecRole = self.add_resource(Role(
"S3WriterLambdaCFRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", "lambda.amazonaws.com")
)
]
),
Path="/",
))
self.S3WriterCFPolicy = self.add_resource(PolicyType(
"S3WriterCFPolicy",
PolicyName="S3WriterLambdaCFRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[
Action("logs", "CreateLogGroup"),
Action("logs", "CreateLogStream"),
Action("logs", "PutLogEvents"),
],
Resource=[
"arn:aws:logs:*:*:*"
]
),
Statement(
Effect=Allow,
Action=[
s3.DeleteObject,
s3.ListBucket,
s3.PutObject,
s3.GetObject,
],
Resource=["*"]
)
]
),
Roles=[Ref(self.S3WriterLambdaCFExecRole)],
DependsOn="S3WriterLambdaCFRole"
))
self.S3WriterCFCustomResource = self.add_resource(Function(
"S3WriterCustomResource",
Description=(
"Creates, updates, and deletes S3 Files with custom content"
),
Runtime="python2.7",
Timeout=300,
Handler="lambda_function.lambda_handler",
Role=GetAtt(self.S3WriterLambdaCFExecRole, "Arn"),
Code=Code(
S3Bucket="cloudformation-custom-resources",
S3Key="s3writer_lambda.zip",
),
DependsOn="S3WriterCFPolicy"
))
def _add_firehose(self):
self.FirehoseLoggingBucket = self.add_resource(Bucket(
"FirehoseLoggingBucket",
DeletionPolicy="Retain",
))
self.FirehoseLoggingRole = self.add_resource(Role(
"FirehoseRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[AssumeRole],
Principal=Principal(
"Service", "firehose.amazonaws.com"
)
)
]
),
Path="/",
))
self.FirehosePolicy = self.add_resource(PolicyType(
"FirehosePolicy",
PolicyName="FirehoseRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[
s3.AbortMultipartUpload,
s3.GetBucketLocation,
s3.GetObject,
s3.ListBucket,
s3.ListBucketMultipartUploads,
s3.PutObject
],
Resource=[
Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
]),
Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
"/*",
])
]
),
]
),
Roles=[Ref(self.FirehoseLoggingRole)]
))
self.FirehoseLogstream = self.add_resource(CustomResource(
"FirehoseLogStream",
ServiceToken=GetAtt(self.FirehoseCFCustomResource, "Arn"),
S3DestinationConfiguration=dict(
RoleARN=GetAtt(self.FirehoseLoggingRole, "Arn"),
BucketARN=Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
]),
BufferingHints=dict(
SizeInMBs=5,
IntervalInSeconds=60,
)
),
DependsOn=[
"FirehosePolicy"
]
))
self._template.add_output([
Output(
"FirehoseLoggingBucket",
Description="Firehose Logging Bucket",
Value=Ref(self.FirehoseLoggingBucket),
)
])
def _add_processor_databases(self):
# Add the security group for Redis access
self.LambdaProcessorSG = self.add_resource(SecurityGroup(
"LambdaProcessorSG",
GroupDescription="Lambda Message Processor",
VpcId=Ref(self.ProcessorVPCId),
))
self.RedisClusterSG = self.add_resource(SecurityGroup(
"RedisClusterSG",
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol="tcp",
FromPort=6379,
ToPort=6379,
SourceSecurityGroupId=GetAtt(self.LambdaProcessorSG,
"GroupId")
)
],
GroupDescription="Allow HTTP traffic to redis",
VpcId=Ref(self.ProcessorVPCId),
))
self.RedisClusterSubnetGroup = self.add_resource(SubnetGroup(
"RedisClusterSubnetGroup",
Description="Subnet group for Redis Cluster",
SubnetIds=[Ref(self.MessageAPISubnetId)],
))
self.RedisCluster = self.add_resource(CacheCluster(
"RedisPushMessages",
Engine="redis",
CacheNodeType="cache.m3.medium",
NumCacheNodes=1,
CacheSubnetGroupName=Ref(self.RedisClusterSubnetGroup),
VpcSecurityGroupIds=[
GetAtt(self.RedisClusterSG, "GroupId"),
],
Tags=self._instance_tags("push-messages", "push-messages"),
))
def _add_processor(self):
self.ProcessorExecRole = self.add_resource(Role(
"ProcessorExecRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[AssumeRole],
Principal=Principal(
"Service", "lambda.amazonaws.com"
)
)
]
),
Path="/",
))
# Common statements for accessing Redis
self.PushMessageStatements = [
Statement(
Effect=Allow,
Action=[
elasticache.DescribeCacheClusters,
],
Resource=["*"]
),
Statement(
Effect=Allow,
Action=[
ddb.ListTables,
ddb.DescribeTable,
],
Resource=["*"]
),
]
self.ProcessorLambdaPolicy = self.add_resource(PolicyType(
"ProcessorLambdaPolicy",
PolicyName="ProcessorLambdaRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[
Action("logs", "CreateLogGroup"),
Action("logs", "CreateLogStream"),
Action("logs", "PutLogEvents"),
],
Resource=[
"arn:aws:logs:*:*:*"
]
),
Statement(
Effect=Allow,
Action=[
s3.GetBucketLocation,
s3.GetObject,
s3.ListBucket,
s3.ListBucketMultipartUploads,
],
Resource=[
Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
]),
Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
"/*",
])
]
),
Statement(
Effect=Allow,
Action=[
ec2.CreateNetworkInterface,
ec2.DescribeNetworkInterfaces,
ec2.DeleteNetworkInterface,
],
Resource=["*"]
),
] + self.PushMessageStatements
),
Roles=[Ref(self.ProcessorExecRole)],
DependsOn="ProcessorExecRole"
))
self.ProcessorS3Settings = self.add_resource(CustomResource(
"ProcessorS3Settings",
ServiceToken=GetAtt(self.S3WriterCFCustomResource, "Arn"),
Bucket=Ref(self.FirehoseLoggingBucket),
Key="processor_settings.json",
Content=dict(
redis_name=Ref(self.RedisCluster),
file_type="json",
),
DependsOn=[
"S3WriterCustomResource"
]
))
self.ProcessorLambda = self.add_resource(Function(
"ProcessorLambda",
Description=(
"Processes logfiles when they hit S3"
),
Runtime="python2.7",
Timeout=300,
Handler="lambda.handler",
Role=GetAtt(self.ProcessorExecRole, "Arn"),
Code=Code(
S3Bucket=Ref(self.ProcessorLambdaBucket),
S3Key=Ref(self.ProcessorLambdaKey),
),
VpcConfig=VPCConfig(
SecurityGroupIds=[
Ref(self.LambdaProcessorSG),
],
SubnetIds=Ref(self.ProcessorSubnetIds),
),
DependsOn=[
"ProcessorExecRole",
"ProcessorS3Settings",
]
))
def _add_push_messages_api(self):
self.MessagesServerRole = self.add_resource(Role(
"MessagesServerRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", "ec2.amazonaws.com")
)
]
),
Path="/",
))
self.MessagesServerPolicy = self.add_resource(PolicyType(
"MessagesServerPolicy",
PolicyName="MessagesServerRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=self.PushMessageStatements,
),
Roles=[Ref(self.MessagesServerRole)],
DependsOn="MessagesServerRole"
))
self.MessagesServerInstanceProfile = self.add_resource(InstanceProfile(
"MessagesServerInstanceProfile",
Path="/",
Roles=[Ref(self.MessagesServerRole)]
))
self.MessagesServerSG = self.add_resource(SecurityGroup(
"MessagesServerSG",
SecurityGroupIngress=[
allow_tcp(22),
allow_tcp(80),
],
VpcId=Ref(self.ProcessorVPCId),
GroupDescription="Allow HTTP traffic to Message Server node",
))
self.MessagesServerInstance = self.add_resource(Instance(
"MessagesServerInstance",
ImageId="ami-2c393546",
InstanceType="t2.micro",
SecurityGroupIds=[
GetAtt(self.LambdaProcessorSG, "GroupId"),
GetAtt(self.MessagesServerSG, "GroupId"),
],
KeyName=Ref(self.KeyPair),
IamInstanceProfile=Ref(self.MessagesServerInstanceProfile),
CreationPolicy=CreationPolicy(
ResourceSignal=ResourceSignal(
Timeout='PT15M'
)
),
UserData=Base64(Join("", [
"#cloud-config\n\n",
"coreos:\n",
" units:\n",
] + self._aws_cfn_signal_service(
"pushmessages", "MessagesServerInstance") + [
" - name: 'pushmessages.service'\n",
" command: 'start'\n",
" content: |\n",
" [Unit]\n",
" Description=Push Messages container\n",
" Author=Mozilla Services\n",
" After=docker.service\n",
" \n",
" [Service]\n",
" Restart=always\n",
" ExecStartPre=-/usr/bin/docker kill pushmessages\n",
" ExecStartPre=-/usr/bin/docker rm pushmessages\n",
" ExecStartPre=/usr/bin/docker pull ",
"bbangert/push-messages:", Ref(self.PushMessagesVersion), "\n",
" ExecStart=/usr/bin/docker run ",
"--name pushmessages ",
"-p 80:8000 ",
"-e 'AWS_DEFAULT_REGION=us-east-1' ",
"-e 'REDIS_ELASTICACHE=", Ref(self.RedisCluster), "' ",
"bbangert/push-messages:", Ref(self.PushMessagesVersion), "\n",
])),
SubnetId=Ref(self.MessageAPISubnetId),
DependsOn="MessagesServerPolicy",
Tags=self._instance_tags("push-messages", "push-messages"),
))
self._template.add_output([
Output(
"MessagesAPI",
Description="Push Messages API URL",
Value=Join("", [
"http://",
GetAtt(self.MessagesServerInstance, "PublicIp"),
"/"
])
)
])
def json(self):
return self._template.to_json()
def lambda_handler(event, context):
print("Received event: " + json.dumps(event, indent=2))
# lambda_handlerに渡されたイベントからｓ３バケットとキーを取り出してｓ３オブジェクトをgetする。
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(event['Records'][0]['s3']['object']['key']).decode('utf8')
print ("buket:" + bucket)
print ("key:" + key)
obj = s3.Object(bucket,key)
response= obj.get()
data = response['Body'].read()
# パラメータシートの読み込み。
# 作成したパラメータシートの先頭に書いてある文字列が変数名として使用されます。
patrameter_list= data.splitlines()
for i in range(len(parameter_list)):
parameter = parameter_list[i].split(',')
a = str(parameter[0])
ns = locals()
ns[a] = parameter
# テンプレートの作成
t = Template()
t.add_version("2010-09-09")
t.add_description("ROOP&ROOP")
# VPCの作成
if len(VPC_CidrBlockList) > 1:
for (address,dns,value) in zip(VPC_CidrBlockList[1:],VPC_EnableDnsHostnamesList[1:],VPC_TagsValueList[1:]):
t.add_resource(VPC(
value.translate(string.maketrans("", ""), "-_"),
EnableDnsSupport="true",
CidrBlock=address,
EnableDnsHostnames=dns,
Tags=Tags(
Name=value
)
))
# u'サブネットの作成'
if len(Subnet_CidrBlockList) > 1:
for (address,az,pip,value,vpcid) in zip(Subnet_CidrBlockList[1:],Subnet_AzList[1:],Subnet_MapPublicIpOnLaunchList[1:],\
Subnet_TagsValueList[1:],Subnet_VpcIdList[1:]):
t.add_resource(Subnet(
value.translate(string.maketrans("", ""), "-_"),
CidrBlock=address,
AvailabilityZone=az,
MapPublicIpOnLaunch=pip,
VpcId=Ref(vpcid.translate(string.maketrans("", ""), "-_")),
Tags=Tags(
Name=value
)
))
# u'各種GateWayの作成
if len(InternetGateway_TagsValueList) > 1:
for (value) in (InternetGateway_TagsValueList[1:]):
t.add_resource(InternetGateway(
value.translate(string.maketrans("", ""), "-_"),
Tags=Tags(
Name=value
)
))
if len(CustomerGateway_TagsValueList) > 1:
for (value,ipaddress,bgpasn) in zip(CustomerGateway_TagsValueList[1:],CustomerGateway_IpAddressList[1:],CustomerGateway_BgpAsnList[1:]):
t.add_resource(CustomerGateway(
value.translate(string.maketrans("", ""), "-_"),
Type="ipsec.1",
IpAddress=ipaddress,
BgpAsn=bgpasn,
Tags=Tags(
Name=value
)
))
if len(VPNGateway_TagsValueList) > 1:
for (value) in (VPNGateway_TagsValueList[1:]):
t.add_resource(VPNGateway(
value.translate(string.maketrans("", ""), "-_"),
Type="ipsec.1",
Tags=Tags(
Name=value
)
))
# u'各種GatewayのVPCへのアタッチ'
if len(VPCGatewayAttachment_InternetGatewayIdList) > 1:
for i,(gatewayid,vpcid) in enumerate(zip(VPCGatewayAttachment_InternetGatewayIdList[1:],VPCGatewayAttachment_VpcId_IGList[1:])):
t.add_resource(VPCGatewayAttachment(
"VPCGatewayAttachmentInternetGateway"+str(i),
InternetGatewayId=Ref(gatewayid.translate(string.maketrans("", ""), "-_")),
VpcId=Ref(vpcid.translate(string.maketrans("", ""), "-_"))
))
if len(VPCGatewayAttachment_VpnGatewayIdList) > 1:
for i,(gatewayid,vpcid) in enumerate(zip(VPCGatewayAttachment_VpnGatewayIdList[1:],VPCGatewayAttachment_VpcId_VPNList[1:])):
t.add_resource(VPCGatewayAttachment(
"VPCGatewayAttachmentVPNGateway"+str(i),
VpnGatewayId=Ref(gatewayid.translate(string.maketrans("", ""), "-_")),
VpcId=Ref(vpcid.translate(string.maketrans("", ""), "-_"))
))
# u'DHCPオプションの作成'
if len(DHCPOptions_DomainNameList) > 1:
for (domainname,domainnameservers,value,) in zip(DHCPOptions_DomainNameList[1:],DHCPOptions_DomainNameServersList[1:],DHCPOptions_ValueList[1:]):
t.add_resource(DHCPOptions(
value.translate(string.maketrans("", ""), "-_"),
DomainName=domainname,
DomainNameServers=['"' + domainnameservers + '"'],
Tags=Tags(
Name=value
)
))
# u'VPCへのDHCPオプションの関連付け'
if len(VPCDHCPOptionsAssociation_DhcpOptionsIdList) > 1:
for i,(dhcpoptionid,vpcid) in enumerate(zip(VPCDHCPOptionsAssociation_DhcpOptionsIdList[1:],VPCDHCPOptionsAssociation_VpcIdList[1:])):
t.add_resource(VPCDHCPOptionsAssociation(
"VPCDHCPOptionsAssociation" + str(i),
DhcpOptionsId=Ref(dhcpoptionid.translate(string.maketrans("", ""), "-_")),
VpcId=Ref(vpcid.translate(string.maketrans("", ""), "-_")),
))
# u'ルートテーブルの作成'
if len(RouteTable_VpcIdList) > 1:
for (vpcid,value) in zip(RouteTable_VpcIdList[1:],RouteTable_TagsValueList[1:]):
t.add_resource(RouteTable(
value.translate(string.maketrans("", ""), "-_"),
VpcId=Ref(vpcid.translate(string.maketrans("", ""), "-_")),
Tags=Tags(
Name=value
)
))
# u'ルートテーブルへのGatewayの関連付け'
if len(Route_GatewayIdList) > 1:
for i,(gatewayid,routetable) in enumerate(zip(Route_GatewayIdList[1:],Route_RouteTableId_GWList[1:])):
t.add_resource(Route(
"RouteGatewayId" + str(i),
DestinationCidrBlock="0.0.0.0/0",
GatewayId=Ref(gatewayid.translate(string.maketrans("", ""), "-_")),
RouteTableId=Ref(routetable.translate(string.maketrans("", ""), "-_"))
))
# u'サブネットへのルートテーブルの関連付け'
if len(SubnetRouteTableAssociation_RouteTableIdList) > 1:
for i,(routetableid,subnetid) in enumerate(zip(SubnetRouteTableAssociation_RouteTableIdList[1:],SubnetRouteTableAssociation_SubnetIdList[1:])):
t.add_resource(SubnetRouteTableAssociation(
"SubnetRouteTableAssociation" + str(i),
RouteTableId=Ref(routetableid.translate(string.maketrans("", ""), "-_")),
SubnetId=Ref(subnetid.translate(string.maketrans("", ""), "-_"))
))
# u'ネットワークACLの作成'
if len(NetworkAcl_TagsValueList) > 1:
for (value,vpcid) in zip(NetworkAcl_TagsValueList[1:],NetworkAcl_VpcIdList[1:]):
t.add_resource(NetworkAcl(
value.translate(string.maketrans("", ""), "-_"),
VpcId=Ref(vpcid.translate(string.maketrans("", ""), "-_")),
Tags=Tags(
Name=value
)
))
# 'ネットワークACLへのルールの追加'
if len(NetworkAclEntry_CidrBlockList) > 1:
for i,(cidr,egress,naclid,pfrom,pto,protocol,ruleaction,rulenumber) in enumerate(zip(NetworkAclEntry_CidrBlockList[1:],NetworkAclEntry_EgressList[1:],\
NetworkAclEntry_NetworkAclIdList[1:],NetworkAclEntry_PortRangeFromList[1:],\
NetworkAclEntry_PortRangeToList[1:],NetworkAclEntry_ProtocolList[1:],\
NetworkAclEntry_RuleActionList[1:],NetworkAclEntry_RuleNumberList[1:])):
t.add_resource(NetworkAclEntry(
"NetworkAclEntry" + str(i),
CidrBlock=cidr,
Egress=egress,
NetworkAclId=Ref(naclid.translate(string.maketrans("", ""), "-_")),
PortRange=PortRange(
From=pfrom,
To=pto
),
Protocol=protocol,
RuleAction=ruleaction,
RuleNumber=rulenumber
))
# 'サブネットへのネットワークACLの関連付け'
if len(SubnetNetworkAclAssociation_SubnetIdList) > 1:
for i,(subnetid,naclid) in enumerate(zip(SubnetNetworkAclAssociation_SubnetIdList[1:],SubnetNetworkAclAssociation_NetworkAclIdList[1:])):
t.add_resource(SubnetNetworkAclAssociation(
"SubnetNetworkAclAssociation" + str(i),
SubnetId=Ref(subnetid.translate(string.maketrans("", ""), "-_")),
NetworkAclId=Ref(naclid.translate(string.maketrans("", ""), "-_"))
))
# 'セキュリティグループの作成'
if len(SecurityGroup_TagsValueList) > 1:
for (value,description,vpcid) in zip(SecurityGroup_TagsValueList[1:],SecurityGroup_GroupDescriptionList[1:],SecurityGroup_VpcIdList[1:]):
t.add_resource(SecurityGroup(
value.translate(string.maketrans("", ""), "-_"),
GroupDescription=description,
VpcId=Ref(vpcid.translate(string.maketrans("", ""), "-_")),
Tags=Tags(
Name=value
)
))
# 'セキュリティグループへのインバウンドルールの設定_ソースにIPを指定'
if len(SGIngressIP_CidrIpList) > 1:
for i,(cidr,pfrom,pto,sgid,protocol) in enumerate(zip(SGIngressIP_CidrIpList[1:],SGIngressIP_FromPortList[1:],SGIngressIP_ToPortList[1:],\
SGIngressIP_GroupIdList[1:],SGIngressIP_IpProtocolList[1:])):
t.add_resource(SecurityGroupIngress(
"SecurityGroupIngressIP" + str(i),
CidrIp=cidr,
FromPort=pfrom,
ToPort=pto,
GroupId=Ref(sgid.translate(string.maketrans("", ""), "-_")),
IpProtocol=protocol
))
# 'セキュリティグループへのインバウンドルールの設定_ソースにセキュリティグループを指定'
if len(SGIngressSG_FromPortList) > 1:
for i,(pfrom,pto,sgid,protocol,sourcesgid) in enumerate(zip(SGIngressSG_FromPortList[1:],SGIngressSG_ToPortList[1:],SGIngressSG_GroupIdList[1:],\
SGIngressSG_IpProtocolList[1:],SGIngressSG_SourceSecurityGroupIdList[1:])):
t.add_resource(SecurityGroupIngress(
"SecurityGroupIngressSG" + str(i),
FromPort=pfrom,
ToPort=pto,
GroupId=Ref(sgid.translate(string.maketrans("", ""), "-_")),
IpProtocol=protocol,
SourceSecurityGroupId=Ref(sourcesgid.translate(string.maketrans("", ""), "-_"))
))
# 'セキュリティグループへのアウトバウンドルールの設定_宛先にIPを指定'
if len(SGEgressIP_CidrIpList) > 1:
for i,(cidr,pfrom,pto,sgid,protocol) in enumerate(zip(SGEgressIP_CidrIpList[1:],SGEgressIP_FromPortList[1:],SGEgressIP_ToPortList[1:],\
SGEgressIP_GroupIdList[1:],SGEgressIP_IpProtocolList[1:])):
t.add_resource(SecurityGroupEgress(
"SecurityGroupEgressIP" + str(i),
CidrIp=cidr,
FromPort=pfrom,
ToPort=pto,
GroupId=Ref(sgid.translate(string.maketrans("", ""), "-_")),
IpProtocol=protocol
))
# 'セキュリティグループへのアウトバウンドルールの設定_宛先にセキュリティグループを指定'
if len(SGEgressSG_FromPortList) > 1:
for i,(pfrom,pto,sgid,protocol,distsgid) in enumerate(zip(SGEgressSG_FromPortList[1:],SGEgressSG_ToPortList[1:],SGEgressSG_GroupIdList[1:],\
SGEgressSG_IpProtocolList[1:],SGEgressSG_DestinationSecurityGroupIdList[1:])):
t.add_resource(SecurityGroupEgress(
"SecurityGroupEgressSG" + str(i),
FromPort=pfrom,
ToPort=pto,
GroupId=Ref(sgid.translate(string.maketrans("", ""), "-_")),
IpProtocol=protocol,
DestinationSecurityGroupId=Ref(distsgid.translate(string.maketrans("", ""), "-_"))
))
# 作成したテンプレートファイルを整形
json_template = t.to_json()
# 保存先のｓ３バケットの指定と保存処理
bucket = s3.Bucket(SAVE_BUCKET)
obj = bucket.Object('json-template-' + BASENAME + ' .txt')
response = obj.put(
Body=json_template.encode('utf-8'),
ContentEncoding='utf-8',
ContentType='text/plane'
)
print(json_template)
def main():
"""Generates the CloudFormation template"""
template = Template()
template.add_version('2010-09-09')
template.add_description(
'This template deploys an ECS cluster to the ' +
'provided VPC and subnets using an Auto Scaling Group')
# Parameters
# EnvironmentName
env_name_param = template.add_parameter(
Parameter(
'EnvironmentName',
Type='String',
Description=
'An environment name that will be prefixed to resource names',
))
# InstanceType
instance_type_param = template.add_parameter(
Parameter(
'InstanceType',
Type='String',
Default='t2.nano',
Description=
'Which instance type should we use to build the ECS cluster?',
AllowedValues=[
't2.nano',
't2.micro',
't2.small',
't2.medium',
't2.large',
't2.xlarge',
't2.2xlarge',
],
))
# ClusterSize
cluster_size_param = template.add_parameter(
Parameter(
'ClusterSize',
Type='Number',
Description='How many ECS hosts do you want to initially deploy?',
Default='1',
))
# VPC
template.add_parameter(
Parameter(
'VPC',
Type='AWS::EC2::VPC::Id',
Description=
'Choose which VPC this ECS cluster should be deployed to',
))
# Subnets
subnets_param = template.add_parameter(
Parameter(
'Subnets',
Type='List<AWS::EC2::Subnet::Id>',
Description=
'Choose which subnets this ECS cluster should be deployed to',
))
# SecurityGroup
sg_param = template.add_parameter(
Parameter(
'SecurityGroup',
Type='AWS::EC2::SecurityGroup::Id',
Description=
'Select the Security Group to use for the ECS cluster hosts',
))
# Mappings
# AWSRegionToAMI
template.add_mapping(
'AWSRegionToAMI',
{
'us-east-1': {
'AMI': 'ami-a58760b3'
},
'us-east-2': {
'AMI': 'ami-a6e4bec3'
},
'us-west-1': {
'AMI': 'ami-74cb9b14'
},
'us-west-2': {
'AMI': 'ami-5b6dde3b'
},
'eu-west-1': {
'AMI': 'ami-e3fbd290'
},
'eu-west-2': {
'AMI': 'ami-77f6fc13'
},
'eu-central-1': {
'AMI': 'ami-38dc1157'
},
'ap-northeast-1': {
'AMI': 'ami-30bdce57'
},
'ap-southeast-1': {
'AMI': 'ami-9f75ddfc'
},
'ap-southeast-2': {
'AMI': 'ami-cf393cac'
},
'ca-central-1': {
'AMI': 'ami-1b01b37f'
},
},
)
# Resources
ecs_role = template.add_resource(
Role(
'ECSRole',
Path='/',
RoleName=Sub('${EnvironmentName}-ECSRole-${AWS::Region}'),
AssumeRolePolicyDocument=awacs.aws.Policy(Statement=[
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[awacs.aws.Action('sts', 'AssumeRole')],
Principal=awacs.aws.Principal('Service',
['ec2.amazonaws.com']),
),
]),
Policies=[
Policy(
PolicyName='ecs-service',
PolicyDocument=awacs.aws.Policy(Statement=[
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[
awacs.aws.Action('ecs', 'CreateCluster'),
awacs.aws.Action(
'ecs', 'DeregisterContainerInstance'),
awacs.aws.Action('ecs',
'DiscoverPollEndpoint'),
awacs.aws.Action('ecs', 'Poll'),
awacs.aws.Action('ecs',
'RegisterContainerInstance'),
awacs.aws.Action('ecs',
'StartTelemetrySession'),
awacs.aws.Action('ecs', 'Submit*'),
awacs.aws.Action('logs', 'CreateLogStream'),
awacs.aws.Action(
'ecr', 'BatchCheckLayerAvailability'),
awacs.aws.Action('ecr', 'BatchGetImage'),
awacs.aws.Action('ecr',
'GetDownloadUrlForLayer'),
awacs.aws.Action('ecr',
'GetAuthorizationToken'),
],
Resource=['*'],
),
], ),
),
],
))
ecs_instance_profile = template.add_resource(
InstanceProfile(
'ECSInstanceProfile',
Path='/',
Roles=[Ref(ecs_role)],
))
# ECSCluster
ecs_cluster = template.add_resource(
Cluster(
'ECSCluster',
ClusterName=Ref(env_name_param),
))
instance_metadata = Metadata(
Init({
'config':
InitConfig(
commands={
'01_add_instance_to_cluster': {
'command':
Join('', [
'#!/bin/bash\n', 'echo ECS_CLUSTER=',
Ref(ecs_cluster), ' >> /etc/ecs/ecs.config'
])
},
},
files=InitFiles({
'/etc/cfn/cfn-hup.conf':
InitFile(
mode='000400',
owner='root',
group='root',
content=Join('', [
'[main]\n', 'stack=',
Ref('AWS::StackId'), '\n', 'region=',
Ref('AWS::Region'), '\n'
]),
),
'/etc/cfn/hooks.d/cfn-auto-reloader.conf':
InitFile(
mode='000400',
owner='root',
group='root',
content=Join('', [
'[cfn-auto-reloader-hook]\n',
'triggers=post.update\n',
'path=Resources.ContainerInstances.Metadata.AWS::CloudFormation::Init\n'
'action=/opt/aws/bin/cfn-init -v --region ',
Ref('AWS::Region'), ' --stack ',
Ref('AWS::StackId'),
' --resource ECSLaunchConfiguration\n'
]),
)
}),
services=InitServices({
'cfn-hup':
InitService(enabled='true',
ensureRunning='true',
files=[
'/etc/cfn/cfn-hup.conf',
'/etc/cfn/hooks.d/cfn-auto-reloader.conf'
])
}),
)
}))
ecs_launch_config = template.add_resource(
LaunchConfiguration(
'ECSLaunchConfiguration',
ImageId=FindInMap('AWSRegionToAMI', Ref('AWS::Region'), 'AMI'),
InstanceType=Ref(instance_type_param),
SecurityGroups=[Ref(sg_param)],
IamInstanceProfile=Ref(ecs_instance_profile),
UserData=Base64(
Join('', [
'#!/bin/bash\n',
'yum install -y aws-cfn-bootstrap\n',
'/opt/aws/bin/cfn-init -v --region ',
Ref('AWS::Region'),
' --stack ',
Ref('AWS::StackName'),
' --resource ECSLaunchConfiguration\n',
'/opt/aws/bin/cfn-signal -e $? --region ',
Ref('AWS::Region'),
' --stack ',
Ref('AWS::StackName'),
' --resource ECSAutoScalingGroup\n',
])),
Metadata=instance_metadata,
))
# ECSAutoScalingGroup:
template.add_resource(
AutoScalingGroup(
'ECSAutoScalingGroup',
VPCZoneIdentifier=Ref(subnets_param),
LaunchConfigurationName=Ref(ecs_launch_config),
MinSize=Ref(cluster_size_param),
MaxSize=Ref(cluster_size_param),
DesiredCapacity=Ref(cluster_size_param),
Tags=ASTags(Name=(Sub('${EnvironmentName} ECS host'), True)),
CreationPolicy=CreationPolicy(
ResourceSignal=ResourceSignal(Timeout='PT15M'), ),
UpdatePolicy=UpdatePolicy(
AutoScalingRollingUpdate=AutoScalingRollingUpdate(
MinInstancesInService='1',
MaxBatchSize='1',
PauseTime='PT15M',
WaitOnResourceSignals=True,
)),
))
# Output
template.add_output(
Output(
'Cluster',
Description='A reference to the ECS cluster',
Value=Ref(ecs_cluster),
))
print(template.to_json())
def create_template(num_masters, num_agents, num_publicAgents):
#outfilename = "test.json"
outfilename = "cf_" + str(num_masters) + "." + str(num_agents) + "." + str(num_publicAgents) + ".json"
# Create the Template
t = Template()
t.add_version('2010-09-09')
t.add_description('Creates a set of Servers for DC/OS using CentOS 7.3 AMI. Creates a boot server to host the DC/OS installer and a NAT Instance for outbound connections from private agents. Creates ' + str(num_masters) + ' Master(s), ' \
+ str(num_agents) + ' Private Agent(s), and ' + str(num_publicAgents) + ' Public Agent(s). After creating the Stack; Log into the boot server and run the DCOS Bash Script installer for AWS')
# Amazon Linux AMI 2016.09.1.20170119 x86_64 VPC NAT HVM EBS
# amzn-ami-vpc-nat-hvm-2016.09.1.20170119-x86_64-ebs -
# ami-dd3dd7cb us-east-1 (N. Virginia)
# ami-564b6e33 us-east-2 (Ohio)
# ami-7d54061d us-west-1 (N. Cal)
# ami-3b6fd05b us-west-2 (Oregon)
t.add_mapping('NATAmi', {
'us-east-1': {'default': 'ami-dd3dd7cb'},
'us-east-2': {'default': 'ami-564b6e33'},
'us-west-1': {'default': 'ami-7d54061d'},
'us-west-2': {'default': 'ami-3b6fd05b'},
})
# The c73 AMI pre created and deployed on each region
t.add_mapping('c73Ami', {
'us-east-1': {'default': 'ami-46c1b650'},
'us-east-2': {'default': 'ami-18f8df7d'},
'us-west-1': {'default': 'ami-f5d7f195'},
'us-west-2': {'default': 'ami-f4533694'},
})
# CloudFormation Parameters
# Sometimes when I deployed stack on us-east-1; it would fail on av zone us-east-1c with error messages instance type not support on this AZ. I added this parameter to fix all of the components in on AZ for now
avzone_param = t.add_parameter(Parameter(
"AVZoneName",
ConstraintDescription='Must be the name of an an Availability Zone',
Description='Name of an Availability Zone',
Type='AWS::EC2::AvailabilityZone::Name',
))
# Every agent will get a data drive of this size
dataDriveSizeGB_param = t.add_parameter(Parameter(
"dataDriveSizeGB",
Default="100",
MinValue=20,
MaxValue=1000,
Description='Size of data drive to add to private agents from 20 to 1000GB',
Type='Number'
))
# The key will be added to the centos user so you can login to centos using the key
keyname_param = t.add_parameter(Parameter(
"KeyName",
ConstraintDescription='Must be the name of an existing EC2 KeyPair.',
Description='Name of an existing EC2 KeyPair to enable SSH access to the instance',
Type='AWS::EC2::KeyPair::KeyName',
))
# While you can allow everyone it's more secure to just allow a single machine or subnet of machines; web port will also be opened to this CIDR
sshlocation_param = t.add_parameter(Parameter(
"sshlocation",
Type="String",
Description="Subnet allowed to ssh to these servers. 0.0.0.0/0 to allow all."
))
# Instance type for Master
instanceTypeMaster_param = t.add_parameter(Parameter(
'InstanceTypeMaster',
Type='String',
Description='EC2 instance type for ' + str(num_masters) + ' Masters(s)',
Default='m4.xlarge',
AllowedValues=[
't2.xlarge', 't2.2xlarge',
'm4.xlarge', 'm4.2xlarge',
'm4.4xlarge', 'm4.10xlarge',
'c4.xlarge', 'c4.2xlarge',
'c4.4xlarge', 'c4.8xlarge',
],
ConstraintDescription='Must be a valid EC2 instance type.',
))
# Instance type for Agents
instanceTypeAgent_param = t.add_parameter(Parameter(
'InstanceTypeAgent',
Type='String',
Description='EC2 instance type for ' + str(num_agents) + ' Private Agent(s)',
Default='m4.2xlarge',
AllowedValues=[
't2.xlarge', 't2.2xlarge',
'm4.xlarge', 'm4.2xlarge',
'm4.4xlarge', 'm4.10xlarge',
'c4.xlarge', 'c4.2xlarge',
'c4.4xlarge', 'c4.8xlarge',
],
ConstraintDescription='Must be a valid EC2 instance type.',
))
# Instance type for Public Agents
instanceTypePublicAgent_param = t.add_parameter(Parameter(
'InstanceTypePublicAgent',
Type='String',
Description='EC2 instance type for ' + str(num_publicAgents) + ' Public Agent(s)',
Default='m4.xlarge',
AllowedValues=[
't2.xlarge', 't2.2xlarge',
'm4.xlarge', 'm4.2xlarge',
'm4.4xlarge', 'm4.10xlarge',
'c4.xlarge', 'c4.2xlarge',
'c4.4xlarge', 'c4.8xlarge',
],
ConstraintDescription='Must be a valid EC2 instance type.',
))
# Adding Resources
ref_stack_id = Ref('AWS::StackId')
ref_region = Ref('AWS::Region')
ref_stack_name = Ref('AWS::StackName')
# Create VPC
nm='vpc'
vpc = t.add_resource(
VPC(
nm,
CidrBlock='10.10.0.0/16',
EnableDnsSupport=True,
EnableDnsHostnames=True,
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
# Create Subnet for Masters
nm='mastersSubnet'
subnetMasters = t.add_resource(
Subnet(
nm,
AvailabilityZone=Ref(avzone_param),
CidrBlock='10.10.0.0/24',
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
# Create Subnet for Agents
nm='agentsSubnet'
subnetAgents = t.add_resource(
Subnet(
nm,
AvailabilityZone=Ref(avzone_param),
CidrBlock='10.10.16.0/24',
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
# Create Subnet for Public Agents
nm='publicAgentsSubnet'
subnetPublicAgents = t.add_resource(
Subnet(
nm,
AvailabilityZone=Ref(avzone_param),
CidrBlock='10.10.32.0/24',
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
# Create Gateway; route to the outside world (Internet)
nm='ig'
internetGateway = t.add_resource(
InternetGateway(
nm,
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
# Attach Gateway to VPC
nm='gatewayAttachment'
gatewayAttachment = t.add_resource(
VPCGatewayAttachment(
nm,
VpcId=Ref(vpc),
InternetGatewayId=Ref(internetGateway)
))
# Create Route Table
nm='routeTable'
routeTable = t.add_resource(
RouteTable(
nm,
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
# Add Routes
# Allow all outbound traffic
nm='route'
route = t.add_resource(
Route(
nm,
DependsOn=gatewayAttachment.title,
GatewayId=Ref(internetGateway),
DestinationCidrBlock='0.0.0.0/0',
RouteTableId=Ref(routeTable),
))
# Associate RouteTable to Master and Public Subnets
nm='subnetRTAMasters'
subnetRouteTableAssociation = t.add_resource(
SubnetRouteTableAssociation(
nm,
SubnetId=Ref(subnetMasters),
RouteTableId=Ref(routeTable),
))
nm='subnetRTAPublicAgents'
subnetRouteTableAssociation = t.add_resource(
SubnetRouteTableAssociation(
nm,
SubnetId=Ref(subnetPublicAgents),
RouteTableId=Ref(routeTable),
))
# Create Security Group (General access to ssh and internal connectionsn between masters, agents, and public agents)
nm='securityGroup'
securityGroup = t.add_resource(
SecurityGroup(
nm,
GroupDescription='Security Group',
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol='tcp',
FromPort='22',
ToPort='22',
CidrIp=Ref(sshlocation_param)),
SecurityGroupRule(
IpProtocol='-1',
CidrIp='10.10.0.0/16')
],
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
# Create Security Group Public Agents
nm='securityGroupPublicAgents'
publicAgentsSG = t.add_resource(
SecurityGroup(
nm,
GroupDescription='Security Group Public Agents',
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol='tcp',
FromPort='80',
ToPort='80',
CidrIp='0.0.0.0/0'),
SecurityGroupRule(
IpProtocol='tcp',
FromPort='443',
ToPort='443',
CidrIp='0.0.0.0/0'),
SecurityGroupRule(
IpProtocol='tcp',
FromPort='10000',
ToPort='10010',
CidrIp='0.0.0.0/0'),
SecurityGroupRule(
IpProtocol='tcp',
FromPort='9090',
ToPort='9090',
CidrIp='0.0.0.0/0')
],
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
# Create Security Group Masters Allow Access from sshlocation param as test
nm='securityGroupMasters'
mastersSG = t.add_resource(
SecurityGroup(
nm,
GroupDescription='Security Group Masters',
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol='tcp',
FromPort='80',
ToPort='80',
CidrIp=Ref(sshlocation_param)),
SecurityGroupRule(
IpProtocol='tcp',
FromPort='443',
ToPort='443',
CidrIp=Ref(sshlocation_param))
],
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
if useNatInstance:
# **** Also change in natRoute ****
# Create NAT instance; This allows private agents to get out to the Internet
nm='nat'
nat = t.add_resource(
Instance(
nm,
SourceDestCheck="false",
ImageId=FindInMap("NATAmi", Ref("AWS::Region"), "default"),
InstanceType="m4.large",
AvailabilityZone=Ref(avzone_param),
KeyName=Ref(keyname_param),
DependsOn=internetGateway.title,
NetworkInterfaces=[
NetworkInterfaceProperty(
GroupSet=[
Ref(securityGroup)],
AssociatePublicIpAddress='true',
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(subnetMasters),
PrivateIpAddress='10.10.0.9')],
BlockDeviceMappings=[
BlockDeviceMapping(
DeviceName="/dev/xvda",
Ebs=EBSBlockDevice(
DeleteOnTermination='true',
))],
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
else:
# Create Elastic IP for NatGateay
nm='natIP'
nat_eip = t.add_resource(
EIP(
nm,
Domain="vpc",
))
# Create NAT Gateway
nm='natGateway'
nat = t.add_resource(
NatGateway(
nm,
AllocationId=GetAtt(nat_eip, 'AllocationId'),
SubnetId=Ref(subnetMasters),
))
# Create Route Table for NAT
nm='natRouteTable'
routeTableNAT = t.add_resource(
RouteTable(
nm,
VpcId=Ref(vpc),
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
# Associate Agent Subnet to NAT
nm='subnetRTAAgents'
subnetRouteTableAssociation = t.add_resource(
SubnetRouteTableAssociation(
nm,
SubnetId=Ref(subnetAgents),
RouteTableId=Ref(routeTableNAT),
))
# Add Routes (Agents can reach out anywhere)
nm='natRoute'
if useNatInstance:
route = t.add_resource(
Route(
nm,
RouteTableId=Ref(routeTableNAT),
DestinationCidrBlock='0.0.0.0/0',
InstanceId=Ref(nat),
))
else:
route = t.add_resource(
Route(
nm,
RouteTableId=Ref(routeTableNAT),
DestinationCidrBlock='0.0.0.0/0',
NatGatewayId=Ref(nat),
))
# ****************************************
# NOTE: I am using static PrivateIPAddresses; this may not be a good choice; however, it simplified the install script. The range of IP's for the master and agents are limited to 24 subnet and I start at 11
# With this configuration the max number of agents is around 240.
# ****************************************
# Create boot instance
# Installs on AWS so far have taken longer than on Azure. Takes about 10 minutes for the boot server to configure.
# Tried several InstanceType from t2.micro to m4.large; all take about 10 minutes for boot to load. The docker start of mesosphere/dcos-genconf seems to be taking longer than it did on azure.
nm='boot'
boot = t.add_resource(
Instance(
nm,
ImageId=FindInMap("c73Ami", Ref("AWS::Region"), "default"),
InstanceType="m4.xlarge",
AvailabilityZone=Ref(avzone_param),
KeyName=Ref(keyname_param),
NetworkInterfaces=[
NetworkInterfaceProperty(
GroupSet=[
Ref(securityGroup)],
AssociatePublicIpAddress='true',
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(subnetMasters),
PrivateIpAddress='10.10.0.10')],
BlockDeviceMappings=[
BlockDeviceMapping(
DeviceName="/dev/sda1",
Ebs=EBSBlockDevice(
VolumeSize="100",
DeleteOnTermination='true',
))],
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
# Create master instance(s)
masters = []
i = 1
while i <= num_masters:
nm='m' + str(i)
private_ip = "10.10.0." + str(i+10)
instance = t.add_resource(
Instance(
nm,
ImageId=FindInMap("c73Ami", Ref("AWS::Region"), "default"),
InstanceType=Ref(instanceTypeMaster_param),
AvailabilityZone=Ref(avzone_param),
KeyName=Ref(keyname_param),
NetworkInterfaces=[
NetworkInterfaceProperty(
GroupSet=[
Ref(securityGroup),
Ref(mastersSG)],
AssociatePublicIpAddress='true',
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(subnetMasters),
PrivateIpAddress=private_ip)],
BlockDeviceMappings=[
BlockDeviceMapping(
DeviceName="/dev/sda1",
Ebs=EBSBlockDevice(
VolumeSize="100",
DeleteOnTermination='true',
))],
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
masters.append(instance)
i += 1
# Create agent instance(s)
i = 1
while i <= num_agents:
nm='a' + str(i)
private_ip = "10.10.16." + str(i+10)
instance = t.add_resource(
Instance(
nm,
ImageId=FindInMap("c73Ami", Ref("AWS::Region"), "default"),
InstanceType=Ref(instanceTypeAgent_param),
AvailabilityZone=Ref(avzone_param),
KeyName=Ref(keyname_param),
NetworkInterfaces=[
NetworkInterfaceProperty(
GroupSet=[
Ref(securityGroup)],
AssociatePublicIpAddress='false',
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(subnetAgents),
PrivateIpAddress=private_ip)],
BlockDeviceMappings=[
BlockDeviceMapping(
DeviceName="/dev/sda1",
Ebs=EBSBlockDevice(
VolumeSize="100",
DeleteOnTermination='true',
))],
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
volume = t.add_resource(
Volume(
nm+"data",
AvailabilityZone=Ref(avzone_param),
Size=Ref(dataDriveSizeGB_param),
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm+"data"]))
))
volattach = t.add_resource(
VolumeAttachment(
nm+"dataattach",
InstanceId=Ref(instance),
VolumeId=Ref(volume),
Device="/dev/sdc"
))
i += 1
# Create public agent instance(s)
publicAgents = []
i = 1
nm="p1"
while i <= num_publicAgents:
nm='p' + str(i)
private_ip = "10.10.32." + str(i+10)
instance = t.add_resource(
Instance(
nm,
ImageId=FindInMap("c73Ami", Ref("AWS::Region"), "default"),
InstanceType=Ref(instanceTypePublicAgent_param),
AvailabilityZone=Ref(avzone_param),
KeyName=Ref(keyname_param),
NetworkInterfaces=[
NetworkInterfaceProperty(
GroupSet=[
Ref(securityGroup),
Ref(publicAgentsSG)],
AssociatePublicIpAddress='true',
DeviceIndex='0',
DeleteOnTermination='true',
SubnetId=Ref(subnetPublicAgents),
PrivateIpAddress=private_ip)],
BlockDeviceMappings=[
BlockDeviceMapping(
DeviceName="/dev/sda1",
Ebs=EBSBlockDevice(
VolumeSize="100",
DeleteOnTermination='true',
))],
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
publicAgents.append(instance)
i += 1
# Load Balancer Masters
nm="masters"
elasticLBMasters = t.add_resource(elb.LoadBalancer(
nm,
Instances=[Ref(r) for r in masters],
Subnets=[Ref(subnetMasters)],
SecurityGroups=[Ref(mastersSG)],
CrossZone=False,
Listeners=[
elb.Listener(
LoadBalancerPort="80",
InstancePort="80",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="443",
InstancePort="443",
Protocol="TCP",
),
],
# Health Checking on port 80 which should be there after DCOS has been installed.
HealthCheck=elb.HealthCheck(
Target="TCP:80",
HealthyThreshold="2",
UnhealthyThreshold="2",
Interval="30",
Timeout="5",
),
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
# Load Balancer Public Agents
nm="publicagents"
elasticLBPublicAgents = t.add_resource(elb.LoadBalancer(
nm,
#AvailabilityZones=GetAZs(""),
Instances=[Ref(r) for r in publicAgents],
Subnets=[Ref(subnetPublicAgents)],
SecurityGroups=[Ref(publicAgentsSG)],
CrossZone=False,
Listeners=[
elb.Listener(
LoadBalancerPort="10000",
InstancePort="10000",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10001",
InstancePort="10001",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10002",
InstancePort="10002",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10003",
InstancePort="10003",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10004",
InstancePort="10004",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10005",
InstancePort="10005",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10006",
InstancePort="10006",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10007",
InstancePort="10007",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10008",
InstancePort="10008",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10009",
InstancePort="10009",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="10010",
InstancePort="10010",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="9090",
InstancePort="9090",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="80",
InstancePort="80",
Protocol="TCP",
),
elb.Listener(
LoadBalancerPort="443",
InstancePort="443",
Protocol="TCP",
)
],
# I've added health check for port 9090; becomes healthy after Marathon-LB is installed.
HealthCheck=elb.HealthCheck(
Target="TCP:9090",
HealthyThreshold="2",
UnhealthyThreshold="2",
Interval="30",
Timeout="5",
),
Tags=Tags(Application=ref_stack_id,Name=Join("",[Ref('AWS::StackName'),"-",nm]))
))
# Outputs
t.add_output(Output(
"BootServer",
Description="Name/IP of Boot Server",
Value=Join("/", [GetAtt(boot, "PublicDnsName"), GetAtt(boot, "PublicIp")])
))
t.add_output(Output(
"MastersURL",
Description="URL of the Masters",
Value=Join("", ["http://", GetAtt(elasticLBMasters, "DNSName")])
))
t.add_output(Output(
"PublicAgentsURL",
Description="URL of the Public Agents haproxy stats.",
Value=Join("", ["http://", GetAtt(elasticLBPublicAgents, "DNSName"), ":9090/haproxy?stats"])
))
# Write json to file
jsonStr = t.to_json()
fout = open(outfilename,"w")
fout.write(jsonStr)
fout.close()
# Print the json to screen
print(jsonStr)
def generate(dry_run, file_location=None):
"""CloudFormation template generator to apply to all accounts which configures log sources to publish to the centralized log target(s) specified"""
t = Template()
t.add_version("2010-09-09")
t.add_description(
"UCSD Log Source AWS CloudFormation Template - this template is meant to be applied to pre-approved accounts and configures CloudWatch Logs to forward to the UCSD log aggregation process."
)
#
# CloudWatch Logs setup - Set up shipping to 'centralized' account
#
# Parameters
delivery_stream_arn = t.add_parameter(
Parameter('LogDeliveryDestinationArn',
Type="String",
Default="",
Description="ARN of the Log Destination to send logs to."))
# resources
cwl_group_retention = t.add_parameter(
Parameter("LogGroupRetentionInDays",
Type="Number",
Description=
"Number of days to retain logs in the CloudWatch Log Group",
MinValue=1,
MaxValue=14,
Default=1))
cwl_group = t.add_resource(
cwl.LogGroup('SecurityLogShippingGroup',
LogGroupName=security_log_shipping_group_name,
RetentionInDays=Ref(cwl_group_retention)))
cwl_subscription = t.add_resource(
cwl.SubscriptionFilter('SecurityLogShippingFilter',
DestinationArn=Ref(delivery_stream_arn),
LogGroupName=Ref(cwl_group),
FilterPattern=""))
cwl_primary_stream = t.add_resource(
cwl.LogStream('PrimaryLogStream',
LogGroupName=Ref(cwl_group),
LogStreamName='PrimaryLogStream'))
# Create IAM role to allow VPC Flow Logs within this account to push data to CloudWatch Logs per https://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/flow-logs.html#flow-logs-iam
vpc_flow_log_iam_role = t.add_resource(
iam.Role('VPCFlowLogToCWLIAMRole',
AssumeRolePolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal(
"Service", "vpc-flow-logs.amazonaws.com"))
])))
vpc_flow_log_policies = t.add_resource(
iam.PolicyType(
'VPCFlowLogToCWLPolicy',
PolicyName='vpcflowlogtocwlpolicy20180213',
Roles=[Ref(vpc_flow_log_iam_role)],
PolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[
CreateLogGroup, CreateLogStream, PutLogEvents,
DescribeLogGroups, DescribeLogStreams
],
Resource=["*"])
])))
# outputs
t.add_output(
Output(
'CloudWatchLogGroupName',
Value=Ref(cwl_group),
Description=
"Name of the CloudWatch Log Group created to flow logs to the centralized logging stream."
))
t.add_output(
Output(
'CloudWatchLogGroupARN',
Value=GetAtt(cwl_group, "Arn"),
Description=
"ARN of the CloudWatch Log Group created to flow logs to the centralized logging stream."
))
t.add_output(
Output(
'VPCFlowLogDeliveryLogsPermissionArn',
Value=GetAtt(vpc_flow_log_iam_role, "Arn"),
Description=
"ARN of the IAM role for VPC Flow Logs to use within this account to ship VPC flow logs through."
))
#
# CloudTrail setup - ship to S3 in 'central account' as well as cloudtrail logs if it'll let us :)
#
# parameters
ct_is_logging = t.add_parameter(
Parameter(
'CloudTrailIsLogging',
Type="String",
Default="false",
AllowedValues=["true", "false"],
Description=
"Flag indicating that CloudTrail is configured to send logs."))
ct_include_global = t.add_parameter(
Parameter(
'CloudTrailIncludeGlobal',
Type="String",
Default="true",
AllowedValues=["true", "false"],
Description=
"Flag indicating that CloudTrail is configured to capture global service events."
))
ct_multi_region = t.add_parameter(
Parameter(
'CloudTrailMultiRegion',
Type="String",
Default="true",
AllowedValues=["true", "false"],
Description=
"Flag indicating that CloudTrail is to be configured in multi-region mode"
))
ct_s3_key_prefix = t.add_parameter(
Parameter('CloudTrailKeyPrefix',
Type='String',
Default='',
Description='Key name prefix for logs being sent to S3'))
ct_bucket_name = t.add_parameter(
Parameter(
'CloudTrailBucketName',
Type='String',
Default='',
Description='Name of the S3 Bucket for delivery of CloudTrail logs'
))
# resources
ct_trail = t.add_resource(
ct.Trail("SecurityTrail",
TrailName=Join("-", ["SecurityTrail", Region]),
S3BucketName=Ref(ct_bucket_name),
S3KeyPrefix=Ref(ct_s3_key_prefix),
IncludeGlobalServiceEvents=Ref(ct_include_global),
IsMultiRegionTrail=Ref(ct_multi_region),
IsLogging=Ref(ct_is_logging)))
# outputs
t.add_output(
Output(
'CloudTrailARN',
Description=
"ARN of the CloudTrail Trail configured for this log source deployment.",
Value=GetAtt(ct_trail, "Arn")))
# Splunk Addon User and Policies per http://docs.splunk.com/Documentation/AddOns/released/AWS/ConfigureAWSpermissions
addon_user = t.add_resource(
iam.User('SplunkAddonUser', UserName='******'))
# http://docs.splunk.com/Documentation/AddOns/released/AWS/ConfigureAWSpermissions#Configure_CloudTrail_permissions
ct_splunk_user_policy = t.add_resource(
iam.PolicyType('cloudtrailSplunkPolicy',
PolicyName='cloudtrailsplunkuser20180213',
Roles=[Ref(vpc_flow_log_iam_role)],
PolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[
asqs.GetQueueAttributes,
asqs.ListQueues, asqs.ReceiveMessage,
asqs.GetQueueUrl, asqs.DeleteMessage,
as3.Action('Get*'),
as3.Action('List*'),
as3.Action('Delete*')
],
Resource=["*"])
])))
# http://docs.splunk.com/Documentation/AddOns/released/AWS/ConfigureAWSpermissions#Configure_CloudWatch_permissions
cw_splunk_user_policy = t.add_resource(
iam.PolicyType('cloudwatchSplunkPolicy',
PolicyName='cloudwatchsplunkuser20180213',
Roles=[Ref(vpc_flow_log_iam_role)],
PolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[
aas.Action("Describe*"),
acw.Action("Describe*"),
acw.Action("Get*"),
acw.Action("List*"),
asns.Action("Get*"),
asns.Action("List*")
],
Resource=['*'])
])))
if dry_run:
print(t.to_json())
else:
save_path = file_location if file_location else os.path.join(
log_aggregation_cf, 'log_sources.json')
with open(save_path, 'w') as f:
f.write(t.to_json())
# -*- encoding: utf-8 -*-
__author__ = 'ray'
__date__ = '1/3/17'
from troposphere import Template, Parameter, Output, Retain
from troposphere import GetAtt, Export, Equals, Ref, If, AWS_NO_VALUE, Not, Sub
from troposphere import logs
import cfnutil
t = Template()
t.add_version()
t.add_description('CloudWatch Logs log group')
#
# Parameter
#
param_name = t.add_parameter(
Parameter('LogGroupName',
Type='String',
Description='Name of the Log Group',
MaxLength=512,
AllowedPattern='[\.\-_/#A-Za-z0-9]*',
Default=''))
param_retention = t.add_parameter(
Parameter('RetentionDays',
Type='Number',
from troposphere import Base64, Join, Parameter, Ref, Template, Tags
import troposphere.ec2 as ec2
# template
template = Template()
template.add_version("2010-09-09")
template.add_description("AWS CloudFormation template for base instance")
app_name = 'taxmap'
app_name_capitalize = 'TaxMap'
# parameters
subnet_id = Parameter(
"SubnetId",
Type="AWS::EC2::Subnet::Id",
Default="subnet-0b3b0d6e",
Description="Id of the Subnet which will hold the instance")
template.add_parameter(subnet_id)
vpc_id = Parameter("VpcId",
Type="AWS::EC2::VPC::Id",
Default="vpc-ebe0c38e",
Description="Id of the VPC which will hold the instance")
template.add_parameter(vpc_id)
# security group
sg = ec2.SecurityGroup(app_name_capitalize + 'SG',
GroupDescription="Security Group for base instance.",
Tags=Tags(Name=app_name + '-base-instance-sg',
Custo=app_name),
SecurityGroupIngress=[
def main():
template = Template()
template.add_version("2010-09-09")
template.set_description(
"AWS CloudFormation Sample Template: NLB with 1 EC2 instance")
AddAMI(template)
# Add the Parameters
keyname_param = template.add_parameter(
Parameter(
"KeyName",
Type="String",
Default="mark",
Description="Name of an existing EC2 KeyPair to "
"enable SSH access to the instance",
))
template.add_parameter(
Parameter(
"InstanceType",
Type="String",
Description="WebServer EC2 instance type",
Default="m1.small",
AllowedValues=[
"t1.micro", "m1.small", "m1.medium", "m1.large", "m1.xlarge",
"m2.xlarge", "m2.2xlarge", "m2.4xlarge", "c1.medium",
"c1.xlarge", "cc1.4xlarge", "cc2.8xlarge", "cg1.4xlarge"
],
ConstraintDescription="must be a valid EC2 instance type.",
))
webport_param = template.add_parameter(
Parameter(
"WebServerPort",
Type="String",
Default="8888",
Description="TCP/IP port of the web server",
))
subnetA = template.add_parameter(
Parameter("subnetA", Type="String", Default="subnet-096fd06d"))
subnetB = template.add_parameter(
Parameter("subnetB", Type="String", Default="subnet-1313ef4b"))
VpcId = template.add_parameter(
Parameter("VpcId", Type="String", Default="vpc-82c514e6"))
# Define the instance security group
instance_sg = template.add_resource(
ec2.SecurityGroup(
"InstanceSecurityGroup",
GroupDescription="Enable SSH and HTTP access on the inbound port",
SecurityGroupIngress=[
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort="22",
ToPort="22",
CidrIp="0.0.0.0/0",
),
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=Ref(webport_param),
ToPort=Ref(webport_param),
CidrIp="0.0.0.0/0",
),
]))
eipA = template.add_resource(ec2.EIP(
'eipA',
Domain='vpc',
))
eipB = template.add_resource(ec2.EIP(
'eipB',
Domain='vpc',
))
# Add the web server instance
WebInstance = template.add_resource(
ec2.Instance(
"WebInstance",
SecurityGroups=[Ref(instance_sg)],
KeyName=Ref(keyname_param),
InstanceType=Ref("InstanceType"),
ImageId=FindInMap("RegionMap", Ref("AWS::Region"), "AMI"),
UserData=Base64(Ref(webport_param)),
))
# Add the network LB
NetworkLB = template.add_resource(
elb.LoadBalancer(
"NetworkLB",
Name="NetworkLB",
Scheme="internet-facing",
SubnetMappings=[
elb.SubnetMapping(AllocationId=GetAtt(eipA, 'AllocationId'),
SubnetId=Ref(subnetA)),
elb.SubnetMapping(AllocationId=GetAtt(eipB, 'AllocationId'),
SubnetId=Ref(subnetB))
],
Type='network'))
TargetGroupWeb = template.add_resource(
elb.TargetGroup("TargetGroupWeb",
HealthCheckIntervalSeconds="30",
HealthCheckProtocol="HTTP",
HealthCheckTimeoutSeconds="10",
HealthyThresholdCount="4",
Matcher=elb.Matcher(HttpCode="200"),
Name="WebTarget",
Port=Ref(webport_param),
Protocol="HTTP",
Targets=[
elb.TargetDescription(Id=Ref(WebInstance),
Port=Ref(webport_param))
],
UnhealthyThresholdCount="3",
VpcId=Ref(VpcId)))
template.add_resource(
elb.Listener("Listener",
Port="80",
Protocol="HTTP",
LoadBalancerArn=Ref(NetworkLB),
DefaultActions=[
elb.Action(Type="forward",
TargetGroupArn=Ref(TargetGroupWeb))
]))
template.add_output(
Output("URL",
Description="URL of the sample website",
Value=Join("",
["http://", GetAtt(NetworkLB, "DNSName")])))
print(template.to_json())
class aws_resources:
"""Initialise the class and start the creationg of the CloudFormation Template"""
def __init__(self):
self.data = []
self.t = Template()
self.t.add_version('2010-09-09')
self.t.add_description ("""Base template to build out of band Jenkins and Public, Private, Dmz and DB subnets.""")
"""Consume the arguments provided to create the VPC specification"""
def create_vpc(self, vpc_name, company, project, cidr):
aVPC = self.t.add_resource(
VPC(
vpc_name,
CidrBlock=cidr,
Tags=Tags(
Name=vpc_name + '-' + company + '-' + project,
StackType=project,
Company=company,
Project=project)))
self.data.append(aVPC)
"""Create an InternetGateway in the template"""
def create_internet_gateway(self, company, project, igw_name):
internetGateway = self.t.add_resource(
InternetGateway(
igw_name,
Tags=Tags(
Name=igw_name + '-' + company + '-' + project,
Project=company,
Company=project)))
self.data.append(internetGateway)
"""Create a Gateway Attachment"""
def create_gateway_attachment(self, vpc_name, igw_name):
gatewayAttachment = self.t.add_resource(
VPCGatewayAttachment(
'AttachGateway',
VpcId=Ref(vpc_name),
InternetGatewayId=Ref(igw_name)))
self.data.append(gatewayAttachment)
def create_route_table(self, name, company, project, vpc_name):
routeTable = self.t.add_resource(
RouteTable(
name,
VpcId=Ref(vpc_name),
Tags=Tags(
Name=name + '-' + company + '-' + project,
Project=company,
Company=project)))
self.data.append(routeTable)
return routeTable
def create_route(self, name, depends, igw_name, dst_block, rt_id):
route = self.t.add_resource(
Route(
name,
DependsOn=depends,
GatewayId=Ref(igw_name),
DestinationCidrBlock=dst_block,
RouteTableId=Ref(rt_id),
))
self.data.append(route)
return route
def create_subnet(self, name, vpc_name, network, type, az, company, project, zone_name):
subnet = self.t.add_resource(
Subnet(
name,
CidrBlock=network,
AvailabilityZone=Join("", [ Ref("AWS::Region"), az ] ),
VpcId=Ref(vpc_name),
Tags=Tags(
Name=name,
Public=type,
Project=project,
Company=company,
Zone=zone_name)))
self.data.append(subnet)
return subnet
def create_subnet_association(self, name, subnet, rt_id):
subnetRouteTableAssociation = self.t.add_resource(
SubnetRouteTableAssociation(
name,
SubnetId=Ref(subnet),
RouteTableId=Ref(rt_id),
))
self.data.append(subnetRouteTableAssociation)
def create_sg(self, name, vpc_id, description):
vpcSecurityGroup = self.t.add_resource(
SecurityGroup(
name,
VpcId=vpc_id,
GroupDescription=description,
Tags=Tags(
Name=name + '-security-group'
)))
self.data.append(vpcSecurityGroup)
return vpcSecurityGroup
def create_sg_ingress(self, name, from_port, to_port, source_sg, protocol):
vpcSecurityGroupIngress = self.t.add_resource(
SecurityGroupIngress(
name,
FromPort=from_port,
ToPort=to_port,
IpProtocol=protocol,
SourceSecurityGroupName=Ref(source_sg)
))
self.data.append(SecurityGroupIngress)
return vpcSecurityGroupIngress
def complete_cfn(self):
return(self.t.to_json())
def main():
"""Generates the CloudFormation template"""
template = Template()
template.add_version("2010-09-09")
template.add_description(
'This template deploys a VPC, with a pair of public and private subnets spread ' +
'across two Availabilty Zones. It deploys an Internet Gateway, with a default ' +
'route on the public subnets. It deploys a pair of NAT Gateways (one in each AZ), ' +
'and default routes for them in the private subnets.'
)
# Parameters
# EnvironmentName
env_param = template.add_parameter(Parameter(
'EnvironmentName',
Type='String',
Description='An environment name that will be prefixed to resource names',
))
# VPC CIDR
vpc_cidr_param = template.add_parameter(Parameter(
'VpcCIDR',
Type='String',
Description='Please enter the IP range (CIDR notation) for this VPC',
Default='10.192.0.0/16',
))
# PublicSubnet1CIDR
pub_subnet_1_param = template.add_parameter(Parameter(
'PublicSubnet1CIDR',
Type='String',
Description='Please enter the IP range (CIDR notation) for the public subnet ' +
'in the first Availability Zone',
Default='10.192.10.0/24',
))
# PublicSubnet2CIDR
pub_subnet_2_param = template.add_parameter(Parameter(
'PublicSubnet2CIDR',
Type='String',
Description='Please enter the IP range (CIDR notation) for the public subnet ' +
'in the second Availability Zone',
Default='10.192.11.0/24',
))
# PrivateSubnet1CIDR
prvt_subnet_1_param = template.add_parameter(Parameter(
'PrivateSubnet1CIDR',
Type='String',
Description='Please enter the IP range (CIDR notation) for the private subnet ' +
'in the first Availability Zone',
Default='10.192.20.0/24',
))
# PrivateSubnet2CIDR
prvt_subnet_2_param = template.add_parameter(Parameter(
'PrivateSubnet2CIDR',
Type='String',
Description='Please enter the IP range (CIDR notation) for the private subnet ' +
'in the second Availability Zone',
Default='10.192.21.0/24',
))
# Resources
# VPC
vpc = template.add_resource(
VPC(
'VPC',
CidrBlock=Ref(vpc_cidr_param),
Tags=Tags(Name=Ref(env_param)),
)
)
# InternetGateway
internet_gateway = template.add_resource(
InternetGateway(
'InternetGateway',
Tags=Tags(Name=Ref(env_param)),
)
)
# InternetGatewayAttachment
template.add_resource(
VPCGatewayAttachment(
'InternetGatewayAttachment',
InternetGatewayId=Ref(internet_gateway),
VpcId=Ref(vpc),
)
)
# PublicSubnet1
pub_subnet1 = template.add_resource(
Subnet(
'PublicSubnet1',
VpcId=Ref(vpc),
AvailabilityZone=Select('0', GetAZs("")),
CidrBlock=Ref(pub_subnet_1_param),
MapPublicIpOnLaunch=False,
Tags=Tags(Name=Sub('${EnvironmentName} Public Subnet (AZ1)')),
)
)
# PublicSubnet2
pub_subnet2 = template.add_resource(
Subnet(
'PublicSubnet2',
VpcId=Ref(vpc),
AvailabilityZone=Select('1', GetAZs("")),
CidrBlock=Ref(pub_subnet_2_param),
MapPublicIpOnLaunch=False,
Tags=Tags(Name=Sub('${EnvironmentName} Public Subnet (AZ2)')),
)
)
# PrivateSubnet1
prvt_subnet1 = template.add_resource(
Subnet(
'PrivateSubnet1',
VpcId=Ref(vpc),
AvailabilityZone=Select('0', GetAZs("")),
CidrBlock=Ref(prvt_subnet_1_param),
MapPublicIpOnLaunch=False,
Tags=Tags(Name=Sub('${EnvironmentName} Private Subnet (AZ1)')),
)
)
# PrivateSubnet2
prvt_subnet2 = template.add_resource(
Subnet(
'PrivateSubnet2',
VpcId=Ref(vpc),
AvailabilityZone=Select('1', GetAZs("")),
CidrBlock=Ref(prvt_subnet_2_param),
MapPublicIpOnLaunch=False,
Tags=Tags(Name=Sub('${EnvironmentName} Private Subnet (AZ2)')),
)
)
# NatGateway1EIP
nat_gateway1_eip = template.add_resource(
EIP(
'NatGateway1EIP',
DependsOn='InternetGatewayAttachment',
Domain='vpc',
)
)
# NatGateway2EIP
nat_gateway2_eip = template.add_resource(
EIP(
'NatGateway2EIP',
DependsOn='InternetGatewayAttachment',
Domain='vpc',
)
)
# NatGateway1
nat_gateway1 = template.add_resource(
NatGateway(
'NatGateway1',
AllocationId=GetAtt(nat_gateway1_eip, 'AllocationId'),
SubnetId=Ref(pub_subnet1),
)
)
# NatGateway2
nat_gateway2 = template.add_resource(
NatGateway(
'NatGateway2',
AllocationId=GetAtt(nat_gateway2_eip, 'AllocationId'),
SubnetId=Ref(pub_subnet2),
)
)
# PublicRouteTable
pub_route_table = template.add_resource(
RouteTable(
'PublicRouteTable',
VpcId=Ref(vpc),
Tags=Tags(Name=Sub('${EnvironmentName} Public Routes')),
)
)
# DefaultPublicRoute
template.add_resource(
Route(
'DefaultPublicRoute',
RouteTableId=Ref(pub_route_table),
DestinationCidrBlock='0.0.0.0/0',
GatewayId=Ref(internet_gateway),
)
)
# PublicSubnet1RouteTableAssociation
template.add_resource(
SubnetRouteTableAssociation(
'PublicSubnet1RouteTableAssociation',
RouteTableId=Ref(pub_route_table),
SubnetId=Ref(pub_subnet1),
)
)
# PublicSubnet2RouteTableAssociation
template.add_resource(
SubnetRouteTableAssociation(
'PublicSubnet2RouteTableAssociation',
RouteTableId=Ref(pub_route_table),
SubnetId=Ref(pub_subnet2),
)
)
# PrivateRouteTable1
prvt_route_table1 = template.add_resource(
RouteTable(
'PrivateRouteTable1',
VpcId=Ref(vpc),
Tags=Tags(Name=Sub('${EnvironmentName} Private Routes (AZ1)')),
)
)
# DefaultPrivateRoute1
template.add_resource(
Route(
'DefaultPrivateRoute1',
RouteTableId=Ref(prvt_route_table1),
DestinationCidrBlock='0.0.0.0/0',
NatGatewayId=Ref(nat_gateway1),
)
)
# PrivateSubnet1RouteTableAssociation
template.add_resource(
SubnetRouteTableAssociation(
'PrivateSubnet1RouteTableAssociation',
RouteTableId=Ref(prvt_route_table1),
SubnetId=Ref(prvt_subnet1),
)
)
# PrivateRouteTable2
prvt_route_table2 = template.add_resource(
RouteTable(
'PrivateRouteTable2',
VpcId=Ref(vpc),
Tags=Tags(Name=Sub('${EnvironmentName} Private Routes (AZ2)')),
)
)
# DefaultPrivateRoute2
template.add_resource(
Route(
'DefaultPrivateRoute2',
RouteTableId=Ref(prvt_route_table2),
DestinationCidrBlock='0.0.0.0/0',
NatGatewayId=Ref(nat_gateway2),
)
)
# PrivateSubnet1RouteTableAssociation
template.add_resource(
SubnetRouteTableAssociation(
'PrivateSubnet2RouteTableAssociation',
RouteTableId=Ref(prvt_route_table2),
SubnetId=Ref(prvt_subnet2),
)
)
# Outputs
template.add_output(Output(
'VPC',
Description='A reference to the created VPC',
Value=Ref(vpc),
))
template.add_output(Output(
'PublicSubnets',
Description='A list of the public subnets',
Value=Join(',', [Ref(pub_subnet1), Ref(pub_subnet2)]),
))
template.add_output(Output(
'PrivateSubnets',
Description='A list of the private subnets',
Value=Join(',', [Ref(prvt_subnet1), Ref(prvt_subnet2)]),
))
template.add_output(Output(
'PublicSubnet1',
Description='A reference to the public subnet in the 1st Availability Zone',
Value=Ref(pub_subnet1),
))
template.add_output(Output(
'PublicSubnet2',
Description='A reference to the public subnet in the 2nd Availability Zone',
Value=Ref(pub_subnet2),
))
template.add_output(Output(
'PrivateSubnet1',
Description='A reference to the private subnet in the 1st Availability Zone',
Value=Ref(prvt_subnet1),
))
template.add_output(Output(
'PrivateSubnet2',
Description='A reference to the private subnet in the 2nd Availability Zone',
Value=Ref(prvt_subnet2),
))
print(template.to_json())
class CloudFormationBuilder(object):
def __init__(self, use_firehose=False, use_processor=False):
self._random_id = str(uuid.uuid4()).replace('-', '')[:12].upper()
self._template = Template()
self._template.add_version("2010-09-09")
desc = "AWS CloudFormation Push Stack"
if use_processor:
desc += " - with Firehose Logging + Processor + Push Messages API"
elif use_firehose:
desc += " - with Firehose Logging"
self._template.add_description(desc)
self.use_firehose = use_firehose or use_processor
self.use_processor = use_processor
self.add_resource = self._template.add_resource
self.add_parameter = self._template.add_parameter
self.AutopushVersion = self.add_parameter(
Parameter("AutopushVersion",
Type="String",
Description="Autopush version to deploy",
Default="1.14.2",
AllowedValues=[
"latest",
"1.14.2",
"1.14.1",
]))
if self.use_processor:
self.ProcessorLambdaBucket = self.add_parameter(
Parameter(
"ProcessorLambdaBucket",
Type="String",
Default="push-lambda-funcs",
Description="S3 Bucket of lambda Message Processor",
))
self.ProcessorLambdaKey = self.add_parameter(
Parameter(
"ProcessorLambdaKey",
Type="String",
Default="push_processor_0.4.zip",
Description="S3 Key of lambda Message Processor",
))
self.ProcessorVPCId = self.add_parameter(
Parameter(
"ProcessorVPCId",
Type="AWS::EC2::VPC::Id",
Description="VPC to run the processor/Messages API in"))
self.ProcessorSubnetIds = self.add_parameter(
Parameter(
"ProcessorSubnetIds",
Type="List<AWS::EC2::Subnet::Id>",
Description=(
"Processor Subnet ID's, MUST have NAT Gateway "
"access for S3 loading"),
))
self.MessagesSecurityGroup = self.add_parameter(
Parameter(
"MessagesSecurityGroup",
Type="AWS::EC2::SecurityGroup::Id",
Description=
"Security Group to allow Messages API access from",
))
self.MessageAPISubnetId = self.add_parameter(
Parameter(
"MessageApiEC2Subnet",
Type="AWS::EC2::Subnet::Id",
Description=(
"Subnet to run Push Messages EC2 Instance in, MUST "
"be in the same VPC as the Processor Subnets")))
self.PushMessagesVersion = self.add_parameter(
Parameter("PushMessagesVersion",
Type="String",
Description="Push-Messages API version to deploy",
Default="0.6",
AllowedValues=["latest", "0.5", "0.6"]))
self.PushCryptoKey = self.add_parameter(
Parameter(
"AutopushCryptoKey",
Type="String",
Default=Fernet.generate_key(),
Description="Autopush crypto-key",
MinLength=44,
MaxLength=44,
))
self.KeyPair = self.add_parameter(
Parameter(
"AutopushSSHKeyPair",
Type="AWS::EC2::KeyPair::KeyName",
Description="Name of an EC2 KeyPair to enable SSH access."))
self.PushTablePrefix = self.add_parameter(
Parameter(
"PushTablePrefix",
Type="String",
Default="autopush_" + self._random_id,
Description="Autopush DynamoDB Table Prefixes",
))
if self.use_firehose:
self._setup_firehose_custom_resource()
self._add_firehose()
self._add_autopush_security_group()
self._add_autopush_iam_roles()
self._add_autopush_servers()
if self.use_processor:
self._add_processor_databases()
self._setup_s3writer_custom_resource()
self._add_processor()
self._add_push_messages_api()
def _add_autopush_security_group(self):
self.InternalRouterSG = self.add_resource(
SecurityGroup("AutopushInternalRouter",
GroupDescription="Internal Routing SG"))
self.EndpointSG = self.add_resource(
SecurityGroup(
"AutopushEndpointNode",
SecurityGroupIngress=[
allow_tcp(8082),
allow_tcp(22),
],
GroupDescription="Allow HTTP traffic to autoendpoint node",
))
self.ConnectionSG = self.add_resource(
SecurityGroup(
"AutopushConnectionNode",
SecurityGroupIngress=[
allow_tcp(8080),
allow_tcp(22),
SecurityGroupRule(IpProtocol="tcp",
FromPort=8081,
ToPort=8081,
SourceSecurityGroupName=Ref(
self.InternalRouterSG))
],
GroupDescription=("Allow Websocket traffic to autopush node")))
def _add_autopush_iam_roles(self):
firehose_extras = []
if self.use_firehose:
# Add in the firehose permissions
firehose_extras.append(
Statement(Effect=Allow,
Action=[
Action("firehose", "PutRecord"),
Action("firehose", "PutRecordBatch"),
],
Resource=[
GetAtt(self.FirehoseLogstream, "Arn"),
]))
self.PushServerRole = self.add_resource(
Role(
"AutopushServerRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service",
"ec2.amazonaws.com"))
]),
Path="/",
))
self.add_resource(
PolicyType(
"AutopushServerRolePolicy",
PolicyName="AutopushServerRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[
ddb.BatchGetItem,
ddb.BatchWriteItem,
ddb.GetItem,
ddb.PutItem,
ddb.DeleteItem,
ddb.UpdateItem,
ddb.Query,
ddb.Scan,
],
Resource=[
Join("", [
"arn:aws:dynamodb:us-east-1:*:table/",
Ref(self.PushTablePrefix), "_*"
])
]),
Statement(Effect=Allow,
Action=[
ddb.ListTables,
ddb.DescribeTable,
ddb.CreateTable,
],
Resource=["*"])
] + firehose_extras),
Roles=[Ref(self.PushServerRole)]))
def _aws_cfn_signal_service(self, wait_for, resource):
"""Returns an array suitable to join for UserData that signals after
the wait_for service has started
"""
return [
" - name: 'aws_cfn_signal.service'\n",
" command: 'start'\n",
" content: |\n",
" [Unit]\n",
" Description=AWS Cloud Formation Signaling\n",
" After=%s.service\n" % wait_for,
" Requires=%s.service\n" % wait_for,
" Type=oneshot\n",
" \n",
" [Service]\n",
" TimeoutStartSec=0\n",
" EnvironmentFile=/etc/environment\n",
" ExecStartPre=/usr/bin/docker pull ",
"aweber/cfn-signal\n",
" ExecStart=/usr/bin/docker run --name cfn-signal ",
"aweber/cfn-signal --success=true --reason='Registry Started'",
" --stack=",
Ref("AWS::StackName"),
" --resource=",
resource,
"\n",
]
def _instance_tags(self, app, app_type):
return Tags(
App=app,
Datadog="false",
Env="testing",
Name=Join("", [Ref("AWS::StackName"), "-", app_type]),
Stack=Ref("AWS::StackName"),
Type=app_type,
)
def _add_autopush_servers(self):
self.PushServerInstanceProfile = self.add_resource(
InstanceProfile("AutopushServerInstanceProfile",
Path="/",
Roles=[Ref(self.PushServerRole)]))
# Extras is common options for UserData
extras = [
"--hostname $public_ipv4 ",
"--storage_tablename ",
Ref(self.PushTablePrefix),
"_storage ",
"--message_tablename ",
Ref(self.PushTablePrefix),
"_message ",
"--router_tablename ",
Ref(self.PushTablePrefix),
"_router ",
"--crypto_key '",
Ref(self.PushCryptoKey),
"' ",
]
if self.use_firehose:
extras.extend(
["--firehose_stream_name ",
Ref(self.FirehoseLogstream), " "])
self.PushEndpointServerInstance = self.add_resource(
Instance(
"AutopushEndpointInstance",
ImageId="ami-2c393546",
InstanceType="t2.micro",
SecurityGroups=[
Ref(self.EndpointSG),
Ref(self.InternalRouterSG),
],
KeyName=Ref(self.KeyPair),
IamInstanceProfile=Ref(self.PushServerInstanceProfile),
CreationPolicy=CreationPolicy(ResourceSignal=ResourceSignal(
Timeout='PT15M')),
UserData=Base64(
Join("", [
"#cloud-config\n\n",
"coreos:\n",
" units:\n",
] + self._aws_cfn_signal_service(
"autoendpoint", "AutopushEndpointInstance"
) + [
" - name: 'autoendpoint.service'\n",
" command: 'start'\n",
" content: |\n",
" [Unit]\n",
" Description=Autoendpoint container\n",
" Author=Mozilla Services\n",
" After=docker.service\n",
" \n",
" [Service]\n",
" Restart=always\n",
" ExecStartPre=-/usr/bin/docker kill autoendpoint\n",
" ExecStartPre=-/usr/bin/docker rm autoendpoint\n",
" ExecStartPre=/usr/bin/docker pull ",
"bbangert/autopush:",
Ref(self.AutopushVersion),
"\n",
" ExecStart=/usr/bin/docker run ",
"--name autoendpoint ",
"-p 8082:8082 ",
"-e 'AWS_DEFAULT_REGION=us-east-1' ",
"bbangert/autopush:",
Ref(self.AutopushVersion),
" ",
"./pypy/bin/autoendpoint ",
] + extras)),
DependsOn="AutopushServerRolePolicy",
Tags=self._instance_tags("autopush", "autoendpoint"),
))
self.PushConnectionServerInstance = self.add_resource(
Instance(
"AutopushConnectionInstance",
ImageId="ami-2c393546",
InstanceType="t2.micro",
SecurityGroups=[
Ref(self.ConnectionSG),
Ref(self.InternalRouterSG),
],
KeyName=Ref(self.KeyPair),
IamInstanceProfile=Ref(self.PushServerInstanceProfile),
CreationPolicy=CreationPolicy(ResourceSignal=ResourceSignal(
Timeout='PT15M')),
UserData=Base64(
Join("", [
"#cloud-config\n\n",
"coreos:\n",
" units:\n",
] + self._aws_cfn_signal_service(
"autopush", "AutopushConnectionInstance"
) + [
" - name: 'autopush.service'\n",
" command: 'start'\n",
" content: |\n",
" [Unit]\n",
" Description=Autopush container\n",
" Author=Mozilla Services\n",
" After=docker.service\n",
" \n",
" [Service]\n",
" Restart=always\n",
" ExecStartPre=-/usr/bin/docker kill autopush\n",
" ExecStartPre=-/usr/bin/docker rm autopush\n",
" ExecStartPre=/usr/bin/docker pull ",
"bbangert/autopush:",
Ref(self.AutopushVersion),
"\n",
" ExecStart=/usr/bin/docker run ",
"--name autopush ",
"-p 8080:8080 ",
"-p 8081:8081 ",
"-e 'AWS_DEFAULT_REGION=us-east-1' ",
"bbangert/autopush:",
Ref(self.AutopushVersion),
" ",
"./pypy/bin/autopush ",
"--router_hostname $private_ipv4 ",
"--endpoint_hostname ",
GetAtt(self.PushEndpointServerInstance,
"PublicDnsName"),
" ",
] + extras)),
DependsOn="AutopushServerRolePolicy",
Tags=self._instance_tags("autopush", "autopush"),
))
self._template.add_output([
Output("PushServerURL",
Description="Push Websocket URL",
Value=Join("", [
"ws://",
GetAtt(self.PushConnectionServerInstance,
"PublicDnsName"), ":8080/"
]))
])
def _setup_firehose_custom_resource(self):
# Setup the FirehoseLambda CloudFormation Custom Resource
self.FirehoseLambdaCFExecRole = self.add_resource(
Role(
"FirehoseLambdaCFRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service",
"lambda.amazonaws.com"))
]),
Path="/",
))
self.FirehoseLambdaPolicy = self.add_resource(
PolicyType("FirehoseCFPolicy",
PolicyName="FirehoseLambdaCFRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[
Action("logs", "CreateLogGroup"),
Action("logs", "CreateLogStream"),
Action("logs", "PutLogEvents"),
],
Resource=["arn:aws:logs:*:*:*"]),
Statement(Effect=Allow,
Action=[
Action("firehose",
"CreateDeliveryStream"),
Action("firehose",
"DeleteDeliveryStream"),
Action("firehose",
"ListDeliveryStreams"),
Action("firehose",
"DescribeDeliveryStream"),
Action("firehose",
"UpdateDestination"),
],
Resource=["*"])
]),
Roles=[Ref(self.FirehoseLambdaCFExecRole)],
DependsOn="FirehoseLambdaCFRole"))
self.FirehoseCFCustomResource = self.add_resource(
Function(
"FirehoseCustomResource",
Description=(
"Creates, updates, and deletes Firehose delivery streams"),
Runtime="python2.7",
Timeout=300,
Handler="lambda_function.lambda_handler",
Role=GetAtt(self.FirehoseLambdaCFExecRole, "Arn"),
Code=Code(
S3Bucket="cloudformation-custom-resources",
S3Key="firehose_lambda.zip",
),
DependsOn="FirehoseCFPolicy"))
def _setup_s3writer_custom_resource(self):
self.S3WriterLambdaCFExecRole = self.add_resource(
Role(
"S3WriterLambdaCFRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service",
"lambda.amazonaws.com"))
]),
Path="/",
))
self.S3WriterCFPolicy = self.add_resource(
PolicyType("S3WriterCFPolicy",
PolicyName="S3WriterLambdaCFRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[
Action("logs", "CreateLogGroup"),
Action("logs", "CreateLogStream"),
Action("logs", "PutLogEvents"),
],
Resource=["arn:aws:logs:*:*:*"]),
Statement(Effect=Allow,
Action=[
s3.DeleteObject,
s3.ListBucket,
s3.PutObject,
s3.GetObject,
],
Resource=["*"])
]),
Roles=[Ref(self.S3WriterLambdaCFExecRole)],
DependsOn="S3WriterLambdaCFRole"))
self.S3WriterCFCustomResource = self.add_resource(
Function(
"S3WriterCustomResource",
Description=(
"Creates, updates, and deletes S3 Files with custom content"
),
Runtime="python2.7",
Timeout=300,
Handler="lambda_function.lambda_handler",
Role=GetAtt(self.S3WriterLambdaCFExecRole, "Arn"),
Code=Code(
S3Bucket="cloudformation-custom-resources",
S3Key="s3writer_lambda.zip",
),
DependsOn="S3WriterCFPolicy"))
def _add_firehose(self):
self.FirehoseLoggingBucket = self.add_resource(
Bucket(
"FirehoseLoggingBucket",
DeletionPolicy="Retain",
))
self.FirehoseLoggingRole = self.add_resource(
Role(
"FirehoseRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal(
"Service", "firehose.amazonaws.com"))
]),
Path="/",
))
self.FirehosePolicy = self.add_resource(
PolicyType("FirehosePolicy",
PolicyName="FirehoseRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[
s3.AbortMultipartUpload,
s3.GetBucketLocation, s3.GetObject,
s3.ListBucket,
s3.ListBucketMultipartUploads,
s3.PutObject
],
Resource=[
Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
]),
Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
"/*",
])
]),
]),
Roles=[Ref(self.FirehoseLoggingRole)]))
self.FirehoseLogstream = self.add_resource(
CustomResource("FirehoseLogStream",
ServiceToken=GetAtt(self.FirehoseCFCustomResource,
"Arn"),
S3DestinationConfiguration=dict(
RoleARN=GetAtt(self.FirehoseLoggingRole, "Arn"),
BucketARN=Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
]),
BufferingHints=dict(
SizeInMBs=5,
IntervalInSeconds=60,
)),
DependsOn=["FirehosePolicy"]))
self._template.add_output([
Output(
"FirehoseLoggingBucket",
Description="Firehose Logging Bucket",
Value=Ref(self.FirehoseLoggingBucket),
)
])
def _add_processor_databases(self):
# Add the security group for Redis access
self.LambdaProcessorSG = self.add_resource(
SecurityGroup(
"LambdaProcessorSG",
GroupDescription="Lambda Message Processor",
VpcId=Ref(self.ProcessorVPCId),
))
self.RedisClusterSG = self.add_resource(
SecurityGroup(
"RedisClusterSG",
SecurityGroupIngress=[
SecurityGroupRule(IpProtocol="tcp",
FromPort=6379,
ToPort=6379,
SourceSecurityGroupId=GetAtt(
self.LambdaProcessorSG, "GroupId"))
],
GroupDescription="Allow HTTP traffic to redis",
VpcId=Ref(self.ProcessorVPCId),
))
self.RedisClusterSubnetGroup = self.add_resource(
SubnetGroup(
"RedisClusterSubnetGroup",
Description="Subnet group for Redis Cluster",
SubnetIds=[Ref(self.MessageAPISubnetId)],
))
self.RedisCluster = self.add_resource(
CacheCluster(
"RedisPushMessages",
Engine="redis",
CacheNodeType="cache.m3.medium",
NumCacheNodes=1,
CacheSubnetGroupName=Ref(self.RedisClusterSubnetGroup),
VpcSecurityGroupIds=[
GetAtt(self.RedisClusterSG, "GroupId"),
],
Tags=self._instance_tags("push-messages", "push-messages"),
))
def _add_processor(self):
self.ProcessorExecRole = self.add_resource(
Role(
"ProcessorExecRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service",
"lambda.amazonaws.com"))
]),
Path="/",
))
# Common statements for accessing Redis
self.PushMessageStatements = [
Statement(Effect=Allow,
Action=[
elasticache.DescribeCacheClusters,
],
Resource=["*"]),
Statement(Effect=Allow,
Action=[
ddb.ListTables,
ddb.DescribeTable,
],
Resource=["*"]),
]
self.ProcessorLambdaPolicy = self.add_resource(
PolicyType(
"ProcessorLambdaPolicy",
PolicyName="ProcessorLambdaRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[
Action("logs", "CreateLogGroup"),
Action("logs", "CreateLogStream"),
Action("logs", "PutLogEvents"),
],
Resource=["arn:aws:logs:*:*:*"]),
Statement(Effect=Allow,
Action=[
s3.GetBucketLocation,
s3.GetObject,
s3.ListBucket,
s3.ListBucketMultipartUploads,
],
Resource=[
Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
]),
Join("", [
"arn:aws:s3:::",
Ref(self.FirehoseLoggingBucket),
"/*",
])
]),
Statement(Effect=Allow,
Action=[
ec2.CreateNetworkInterface,
ec2.DescribeNetworkInterfaces,
ec2.DeleteNetworkInterface,
],
Resource=["*"]),
] + self.PushMessageStatements),
Roles=[Ref(self.ProcessorExecRole)],
DependsOn="ProcessorExecRole"))
self.ProcessorS3Settings = self.add_resource(
CustomResource("ProcessorS3Settings",
ServiceToken=GetAtt(self.S3WriterCFCustomResource,
"Arn"),
Bucket=Ref(self.FirehoseLoggingBucket),
Key="processor_settings.json",
Content=dict(
redis_name=Ref(self.RedisCluster),
file_type="json",
),
DependsOn=["S3WriterCustomResource"]))
self.ProcessorLambda = self.add_resource(
Function("ProcessorLambda",
Description=("Processes logfiles when they hit S3"),
Runtime="python2.7",
Timeout=300,
Handler="lambda.handler",
Role=GetAtt(self.ProcessorExecRole, "Arn"),
Code=Code(
S3Bucket=Ref(self.ProcessorLambdaBucket),
S3Key=Ref(self.ProcessorLambdaKey),
),
VpcConfig=VPCConfig(
SecurityGroupIds=[
Ref(self.LambdaProcessorSG),
],
SubnetIds=Ref(self.ProcessorSubnetIds),
),
DependsOn=[
"ProcessorExecRole",
"ProcessorS3Settings",
]))
def _add_push_messages_api(self):
self.MessagesServerRole = self.add_resource(
Role(
"MessagesServerRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service",
"ec2.amazonaws.com"))
]),
Path="/",
))
self.MessagesServerPolicy = self.add_resource(
PolicyType("MessagesServerPolicy",
PolicyName="MessagesServerRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=self.PushMessageStatements,
),
Roles=[Ref(self.MessagesServerRole)],
DependsOn="MessagesServerRole"))
self.MessagesServerInstanceProfile = self.add_resource(
InstanceProfile("MessagesServerInstanceProfile",
Path="/",
Roles=[Ref(self.MessagesServerRole)]))
self.MessagesServerSG = self.add_resource(
SecurityGroup(
"MessagesServerSG",
SecurityGroupIngress=[
allow_tcp(22),
allow_tcp(80),
],
VpcId=Ref(self.ProcessorVPCId),
GroupDescription="Allow HTTP traffic to Message Server node",
))
self.MessagesServerInstance = self.add_resource(
Instance(
"MessagesServerInstance",
ImageId="ami-2c393546",
InstanceType="t2.micro",
SecurityGroupIds=[
GetAtt(self.LambdaProcessorSG, "GroupId"),
GetAtt(self.MessagesServerSG, "GroupId"),
],
KeyName=Ref(self.KeyPair),
IamInstanceProfile=Ref(self.MessagesServerInstanceProfile),
CreationPolicy=CreationPolicy(ResourceSignal=ResourceSignal(
Timeout='PT15M')),
UserData=Base64(
Join("", [
"#cloud-config\n\n",
"coreos:\n",
" units:\n",
] + self._aws_cfn_signal_service(
"pushmessages", "MessagesServerInstance"
) + [
" - name: 'pushmessages.service'\n",
" command: 'start'\n",
" content: |\n",
" [Unit]\n",
" Description=Push Messages container\n",
" Author=Mozilla Services\n",
" After=docker.service\n",
" \n",
" [Service]\n",
" Restart=always\n",
" ExecStartPre=-/usr/bin/docker kill pushmessages\n",
" ExecStartPre=-/usr/bin/docker rm pushmessages\n",
" ExecStartPre=/usr/bin/docker pull ",
"bbangert/push-messages:",
Ref(self.PushMessagesVersion),
"\n",
" ExecStart=/usr/bin/docker run ",
"--name pushmessages ",
"-p 80:8000 ",
"-e 'AWS_DEFAULT_REGION=us-east-1' ",
"-e 'REDIS_ELASTICACHE=",
Ref(self.RedisCluster),
"' ",
"bbangert/push-messages:",
Ref(self.PushMessagesVersion),
"\n",
])),
SubnetId=Ref(self.MessageAPISubnetId),
DependsOn="MessagesServerPolicy",
Tags=self._instance_tags("push-messages", "push-messages"),
))
self._template.add_output([
Output("MessagesAPI",
Description="Push Messages API URL",
Value=Join("", [
"http://",
GetAtt(self.MessagesServerInstance, "PublicIp"), "/"
]))
])
def json(self):
return self._template.to_json()
def create_template():
template = BaseTemplate('Create the infrastructure needed to run the API')
template.add_version('2010-09-09')
return template
def create_vpc(self):
template = Template()
template.add_version('2010-09-09')
vpc_name_formatted = ''.join(
e for e in self.vpc_name if e.isalnum()).capitalize()
private_vpc = template.add_resource(
VPC(
'{}{}'.format(self.stage, vpc_name_formatted),
CidrBlock=self.vpc_cidr_block,
EnableDnsHostnames="true",
Tags=Tags(
Name="{}_{}".format(self.stage,vpc_name_formatted)
)
)
)
internet_gateway = template.add_resource(
InternetGateway(
'{}{}InternetGateway'.format(self.stage, vpc_name_formatted),
Tags=Tags(
Name="{}_{}_internet_gateway".format(self.stage,vpc_name_formatted)
)
)
)
template.add_resource(
VPCGatewayAttachment(
'{}{}InternetGatewayAttachment'.format(self.stage, vpc_name_formatted),
VpcId=Ref(private_vpc),
InternetGatewayId=Ref(internet_gateway)
)
)
public_route_table = template.add_resource(
RouteTable(
'{}{}PublicRouteTable'.format(self.stage, vpc_name_formatted),
VpcId=Ref(private_vpc),
Tags=Tags(
Name="{}_{}_public_route_table".format(self.stage,vpc_name_formatted)
)
)
)
template.add_resource(
Route(
'{}{}PublicRouteIGRule'.format(self.stage,vpc_name_formatted),
DependsOn='{}{}InternetGatewayAttachment'.format(self.stage, vpc_name_formatted),
GatewayId=Ref(internet_gateway),
DestinationCidrBlock='0.0.0.0/0',
RouteTableId=Ref(public_route_table)
)
)
private_route_table = template.add_resource(
RouteTable(
'{}{}PrivateRouteTable'.format(self.stage, vpc_name_formatted),
VpcId=Ref(private_vpc),
Tags=Tags(
Name="{}_{}_private_route_table".format(self.stage,vpc_name_formatted)
)
)
)
for sub in self.subnets:
subnet_name = sub["name"]
subnet_type = sub["type"]
subnet_cidr_block = sub["cidr_block"]
subnet_availability_zone = sub["availability_zone"]
subnet_map_ip_on_launch = sub["map_ip_on_launch"]
subnet_nat_gateway = sub["nat_gateway"]
subnet_name_formatted = ''.join(
e for e in subnet_name if e.isalnum()).capitalize()
subnet = template.add_resource(
Subnet(
'{}{}{}Subnet'.format(self.stage,vpc_name_formatted,subnet_name_formatted),
CidrBlock=subnet_cidr_block,
AvailabilityZone=subnet_availability_zone,
MapPublicIpOnLaunch=subnet_map_ip_on_launch,
VpcId=Ref(private_vpc),
Tags=Tags(
Name="{}_{}_{}".format(self.stage,vpc_name_formatted,subnet_name)
)
)
)
self.output_list.append(
Output(
'{}{}{}SubnetId'.format(self.stage, vpc_name_formatted,subnet_name_formatted),
Description="ID of the Subnet",
Value=Ref(subnet),
Export=Export('{}{}{}SubnetId'.format(self.stage, vpc_name_formatted,subnet_name_formatted))
)
)
if subnet_nat_gateway == "true":
nat_eip = template.add_resource(EIP(
'{}{}{}NatEip'.format(self.stage, vpc_name_formatted,subnet_name_formatted),
Domain="private_vpc"
))
nat_gateway = template.add_resource(NatGateway(
'{}{}{}NatGateway'.format(self.stage,vpc_name_formatted,subnet_name_formatted),
AllocationId=GetAtt(nat_eip, 'AllocationId'),
SubnetId=Ref(subnet)
))
template.add_resource(
Route(
'{}{}{}NatGatewayRouteRule'.format(self.stage,vpc_name_formatted,subnet_name_formatted),
NatGatewayId=Ref(nat_gateway),
DestinationCidrBlock='0.0.0.0/0',
RouteTableId=Ref(private_route_table)
)
)
if subnet_type == "public":
template.add_resource(
SubnetRouteTableAssociation(
'{}{}{}SubnetRouteAssociation'.format(self.stage,vpc_name_formatted,subnet_name_formatted),
SubnetId=Ref(subnet),
RouteTableId=Ref(public_route_table)
)
)
elif subnet_type == "private":
template.add_resource(
SubnetRouteTableAssociation(
'{}{}{}SubnetRouteAssociation'.format(self.stage,vpc_name_formatted,subnet_name_formatted),
SubnetId=Ref(subnet),
RouteTableId=Ref(private_route_table)
)
)
if self.vpc_endpoint_s3 == True:
template.add_resource(
VPCEndpoint(
'{}{}VPCEndpointS3'.format(self.stage, vpc_name_formatted),
VpcId=Ref(private_vpc),
ServiceName="com.amazonaws.us-east-1.s3",
RouteTableIds=[Ref(public_route_table), Ref(private_route_table)]
)
)
# #################### OUTPUTS #####################
self.output_list.append(
Output(
"{}{}VpcId".format(self.stage, vpc_name_formatted),
Description="ID of {} VPC".format(vpc_name_formatted),
Value=Ref(private_vpc),
Export=Export("{}{}VpcId".format(self.stage, vpc_name_formatted))
)
)
self.output_list.append(
Output(
"{}{}PublicRouteTableId".format(self.stage,vpc_name_formatted),
Description="ID of {} VPC".format(vpc_name_formatted),
Value=Ref(public_route_table),
Export=Export("{}{}PublicRouteTableId".format(self.stage,vpc_name_formatted))
)
)
self.output_list.append(
Output(
"{}{}PrivateRouteTableId".format(self.stage,vpc_name_formatted),
Description="ID of {} VPC".format(vpc_name_formatted),
Value=Ref(private_route_table),
Export=Export("{}{}PrivateRouteTableId".format(self.stage,vpc_name_formatted))
)
)
template.add_output(self.output_list)
f = open("modules/template_vpc_{}.yaml".format(vpc_name_formatted), 'w')
print(template.to_yaml(), file=f)
class StandUp:
region = "ap-southeast-2"
subnet = "subnet-cb5facae"
type = "m3.xlarge"
ami = "ami-e95c31d3"
stackname = "TestIOStack"
name = "windows-2012-test"
instance_template = "TestIO"
timezone = "Australia/Brisbane"
environment = "Development"
comment = "Comments"
keypair = "IOTEST"
iamrole = "IAM-EC2-Default"
securitygroups = ["sg-02b36667"]
monitoring = False
rollback = False
volume = ""
ec2conn = ""
cfnconn = ""
template = ""
def __init__(self):
self.build()
self.output_to_file()
def output_to_file(self):
output = self.template.to_json()
fd = open('standup.json', 'w')
fd.write(output)
fd.close()
print(output)
def metadata(self):
m = MetadataObject()
m.add_configkeys(
'AWS::CloudFormation::Init', 'InitRAID', 'files', {
"C:\\cfn\\scripts\\striperaidebs.txt": {
"content":
"""select disk 1
clean
convert dynamic
select disk 2
clean
convert dynamic
create volume stripe disk=1,2
list volume
select volume 2
assign letter=e
format fs=ntfs quick"""
},
"C:\\cfn\\scripts\\striperaidephemeral.txt": {
"content":
"""select disk 2
clean
convert dynamic
select disk 3
clean
convert dynamic
create volume stripe disk=2,3
select volume 2
assign letter=E
format fs=ntfs quick"""
}
})
m.add_configkeys(
'AWS::CloudFormation::Init', 'InitRAID', 'commands', {
"1-initialize-raid-1": {
"command":
"""diskpart /s C:\\cfn\\scripts\\striperaidebs.txt""",
"waitAfterCompletion": 0
}
})
m.add_configkeys(
'AWS::CloudFormation::Init', 'TestIO', 'packages', {
"msi": {
"python":
"""https://www.python.org/ftp/python/3.4.2/python-3.4.2.amd64.msi"""
}
})
m.add_configkeys(
'AWS::CloudFormation::Init', 'TestIO', 'files', {
"C:\\cfn\\scripts\\DiskRobIOt.py":
"""https://raw.githubusercontent.com/monk-ee/DiskRobIOt/master/DiskRobIOt.py"""
})
m.add_configkeys(
'AWS::CloudFormation::Init', 'TestIO', 'commands', {
"1-python-path": {
"command": """setx path "%path%;C:\\Python34" """,
"waitAfterCompletion": 0
},
"2-run-disktest": {
"command":
"""c:\\cfn\\scripts\DiskRobIOt.py --path e:\\ """,
"waitAfterCompletion": 0
}
})
m.add_configkeys('AWS::CloudFormation::Init', 'configSets', 'config',
["InitRAID", "TestIO"])
return m
def build(self):
self.template = Template()
self.template.add_version()
self.template.add_description(self.comment)
m = self.metadata()
ec2_instance = self.template.add_resource(
ec2.Instance(
self.instance_template,
ImageId=self.ami,
InstanceType=self.type,
KeyName=self.keypair,
SubnetId=self.subnet,
SecurityGroupIds=self.securitygroups,
Monitoring=self.monitoring,
IamInstanceProfile=self.iamrole,
UserData=Base64("""<script> cfn-init -v -s """ +
self.stackname + """ -r """ +
self.instance_template + """ --region """ +
self.region +
""" --configset config</script>"""),
Metadata=m.JSONrepr(),
BlockDeviceMappings=[
ec2.BlockDeviceMapping(
DeviceName="/dev/xvdca",
VirtualName="ephemeral0",
),
ec2.BlockDeviceMapping(
DeviceName="/dev/xvdcb",
VirtualName="ephemeral1",
),
ec2.BlockDeviceMapping(
DeviceName="/dev/xvdb",
Ebs=ec2.EBSBlockDevice(DeleteOnTermination=True,
VolumeSize="45",
VolumeType="gp2"),
),
ec2.BlockDeviceMapping(
DeviceName="/dev/xvdc",
Ebs=ec2.EBSBlockDevice(DeleteOnTermination=True,
VolumeSize="45",
VolumeType="gp2"),
)
],
Tags=Tags(
Name=self.name,
Environment=self.environment,
Comment=self.comment,
Role=self.iamrole,
),
))
self.template.add_resource(
ec2.EIP(
"EIP",
InstanceId=Ref(ec2_instance),
Domain='vpc',
))
self.template.add_output([
Output(
"InstanceId",
Description="InstanceId of the newly created EC2 instance",
Value=Ref(ec2_instance),
),
Output(
"PrivateIP",
Description=
"Private IP address of the newly created EC2 instance",
Value=GetAtt(ec2_instance, "PrivateIp"),
),
Output(
"PrivateDNS",
Description="Private DNSName of the newly created EC2 instance",
Value=GetAtt(ec2_instance, "PrivateDnsName"),
)
])
def cloudform(self):
try:
self.cfnconn.create_stack(self.stackname,
template_body=self.template.to_json(),
disable_rollback=self.rollback)
output = self.template.to_json()
return output
except Exception as e:
raise
def ec2_connect_to_region(self):
try:
self.ec2conn = boto.ec2.connect_to_region(self.region)
except:
raise
def vpc_connect_to_region(self):
try:
self.vpcconn = boto.vpc.connect_to_region(self.region)
except:
raise
def iam_connect_to_region(self):
try:
self.iamconn = boto.iam.connect_to_region(self.region)
except:
raise
def cfn_connect_to_region(self):
try:
self.cfnconn = boto.cloudformation.connect_to_region(self.region)
except:
raise
class EC2Instance(JetstreamTemplate):
'''EC2 Template'''
def __init__(self):
self.name = 'ec2.template'
self.template = Template()
self.template.add_version("2010-09-09")
self.test_parameter_groups = TestParameterGroups()
default_test_params = TestParameterGroup()
self.test_parameter_groups.add(default_test_params)
Environment = self.template.add_parameter(Parameter(
"Environment",
Default="Development",
Type="String",
Description="Application environment",
AllowedValues=["Development", "Integration",
"PreProduction", "Production", "Staging", "Test"],
))
default_test_params.add(TestParameter("Environment", "Integration"))
Bucket = self.template.add_parameter(Parameter(
"S3Bucket",
Type="String",
Description="S3 Bucket",
))
default_test_params.add(TestParameter("S3Bucket", "Arn", S3Bucket()))
ImageId = self.template.add_parameter(Parameter(
"ImageId",
Type="String",
Description="Image Id"
))
default_test_params.add(TestParameter("ImageId", "ami-6869aa05"))
self.template.add_resource(Instance(
"EC2Instance",
Tags=Tags(
Name=Ref("AWS::StackName"),
ServiceProvider="Rackspace",
Environment=Ref(Environment),
),
InstanceType="t2.small",
ImageId=Ref(ImageId),
))
EC2Policy = Policy(
PolicyName="EC2_S3_Access",
PolicyDocument={
"Statement": [{
"Effect": "Allow",
"Action": "s3:*",
"Resource": Ref(Bucket)
}]
})
EC2InstanceRole = self.template.add_resource(Role(
"EC2InstanceRole",
AssumeRolePolicyDocument={
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": ["ec2.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}]
},
Path="/",
Policies=[EC2Policy],
))
self.template.add_resource(InstanceProfile(
"EC2InstanceProfile",
Path="/",
Roles=[Ref(EC2InstanceRole)]
))
def main(argv):
FILE = None
try:
opts, args = getopt.getopt(argv,"hf:",["FILE="])
except getopt.GetoptError:
print sys.argv[0], ' -f <metric-csv-file>'
sys.exit(2)
# An array to contain any parameters for the template.
parameters = []
# An array to contain any conditions for the template.
conditions = {}
# An array to contain any key value maps for the template.
maps = []
# An array to contain any resource objects for the template.
resources = []
# An array to contain any output objects for the template.
outputs = []
with open(FILE, 'rbU') as f:
reader = csv.reader(f)
try:
for row in islice(reader, 1, None):
resources.append(Alarm(
"QueueDepthAlarm",
AlarmDescription="Alarm if queue depth grows beyond 10 messages",
Namespace="AWS/SQS",
MetricName="ApproximateNumberOfMessagesVisible",
Dimensions=[
MetricDimension(
Name="QueueName",
Value=GetAtt(myqueue, "QueueName")
),
],
Statistic="Sum",
Period="300",
EvaluationPeriods="1",
Threshold="10",
ComparisonOperator="GreaterThanThreshold",
AlarmActions=[Ref(alarmtopic), ],
InsufficientDataActions=[Ref(alarmtopic), ],
))
except csv.Error as e:
sys.exit('file %s, line %d: %s' % (VPC_ID, reader.line_num, e))
t = Template()
t.add_version('2010-09-09')
t.add_description(
"This is an AWS CloudFormation template that provisions metric filters "
"based on a spreadsheet of applicable metric filters. ***WARNING*** This "
"template creates many Amazon CloudWatch alarms based on a Amazon "
"CloudWatch Logs Log Group. You will be billed for the AWS resources used "
"if you create a stack from this template."
)
for p in parameters:
t.add_parameter(p)
for k in conditions:
t.add_condition(k, conditions[k])
for r in resources:
t.add_resource(r)
for o in outputs:
t.add_output(o)
# Print the template to JSON
print(t.to_json())
class CloudFormationBuilder(object):
def __init__(self):
self._random_id = str(uuid.uuid4()).replace('-', '')[:12].upper()
self._template = Template()
self._template.add_version("2010-09-09")
self.add_resource = self._template.add_resource
self.add_parameter = self._template.add_parameter
self.PushVPC = self.add_parameter(Parameter(
"ExistingVPC",
Type="AWS::EC2::VPC::Id",
Description=(
"The VPC Id so launch the Autopush server into"
),
))
self.PushCryptoKey = self.add_parameter(Parameter(
"AutopushCryptoKey",
Type="String",
Default=Fernet.generate_key(),
Description="Autopush crypto-key",
))
self.KeyPair = self.add_parameter(Parameter(
"AutopushSSHKeyPair",
Type="AWS::EC2::KeyPair::KeyName",
Description="Name of an EC2 KeyPair to enable SSH access."
))
self.PushTablePrefix = self.add_parameter(Parameter(
"PushTablePrefix",
Type="String",
Default="autopush_" + self._random_id,
Description="Autopush DynamoDB Table Prefixes",
))
self._add_autopush_security_group()
self._add_autopush_iam_roles()
self._add_autopush_servers()
def _add_autopush_security_group(self):
self.InternalRouterSG = self.add_resource(SecurityGroup(
"AutopushInternalRouter",
GroupDescription="Internal Routing SG"
))
self.EndpointSG = self.add_resource(SecurityGroup(
"AutopushEndpointNode",
SecurityGroupIngress=[
allow_tcp(8082),
allow_tcp(22),
],
GroupDescription="Allow HTTP traffic to autoendpoint node",
))
self.ConnectionSG = self.add_resource(SecurityGroup(
"AutopushConnectionNode",
SecurityGroupIngress=[
allow_tcp(8080),
allow_tcp(22),
SecurityGroupRule(
IpProtocol="tcp",
FromPort=8081,
ToPort=8081,
SourceSecurityGroupName=Ref(self.InternalRouterSG)
)
],
GroupDescription=(
"Allow Websocket traffic to autopush node"
)
))
def _add_autopush_iam_roles(self):
self.PushServerRole = self.add_resource(Role(
"AutopushServerRole",
AssumeRolePolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[AssumeRole],
Principal=Principal("Service", "ec2.amazonaws.com")
)
]
),
Path="/",
))
self.add_resource(PolicyType(
"AutopushServerRolePolicy",
PolicyName="AutopushServerRole",
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[
ddb.BatchGetItem,
ddb.BatchWriteItem,
ddb.GetItem,
ddb.PutItem,
ddb.DeleteItem,
ddb.UpdateItem,
ddb.Query,
ddb.Scan,
],
Resource=[
Join("", ["arn:aws:dynamodb:us-east-1:*:table/",
Ref(self.PushTablePrefix),
"_*"]
)
]
),
Statement(
Effect=Allow,
Action=[
ddb.ListTables,
ddb.DescribeTable,
ddb.CreateTable,
],
Resource=["*"]
)
]
),
Roles=[Ref(self.PushServerRole)]
))
def _add_autopush_servers(self):
self.PushServerInstanceProfile = self.add_resource(InstanceProfile(
"AutopushServerInstanceProfile",
Path="/",
Roles=[Ref(self.PushServerRole)]
))
self.PushEndpointServerInstance = self.add_resource(Instance(
"AutopushEndpointInstance",
ImageId="ami-2c393546",
InstanceType="t2.micro",
SecurityGroups=[
Ref(self.EndpointSG),
Ref(self.InternalRouterSG),
],
KeyName=Ref(self.KeyPair),
IamInstanceProfile=Ref(self.PushServerInstanceProfile),
UserData=Base64(Join("", [
"#cloud-config\n\n",
"coreos:\n",
" units:\n",
" - name: 'autoendpoint.service'\n",
" command: 'start'\n",
" content: |\n",
" [Unit]\n",
" Description=Autoendpoint container\n",
" Author=Mozilla Services\n",
" After=docker.service\n",
" \n",
" [Service]\n",
" Restart=always\n",
" ExecStartPre=-/usr/bin/docker kill autoendpoint\n",
" ExecStartPre=-/usr/bin/docker rm autoendpoint\n",
" ExecStartPre=/usr/bin/docker pull ",
"bbangert/autopush:1.14.1\n",
" ExecStart=/usr/bin/docker run ",
"--name autoendpoint ",
"-p 8082:8082 ",
"bbangert/autopush:1.14.1 ./pypy/bin/autoendpoint ",
"--hostname $public_ipv4 ",
"--storage_tablename ", Ref(self.PushTablePrefix),
"_storage ",
"--message_tablename ", Ref(self.PushTablePrefix),
"_message ",
"--router_tablename ", Ref(self.PushTablePrefix),
"_router ",
"--crypto_key '", Ref(self.PushCryptoKey), "' ",
"\n"
]))
))
self.PushConnectionServerInstance = self.add_resource(Instance(
"AutopushConectionInstance",
ImageId="ami-2c393546",
InstanceType="t2.micro",
SecurityGroups=[
Ref(self.ConnectionSG),
Ref(self.InternalRouterSG),
],
KeyName=Ref(self.KeyPair),
IamInstanceProfile=Ref(self.PushServerInstanceProfile),
UserData=Base64(Join("", [
"#cloud-config\n\n",
"coreos:\n",
" units:\n",
" - name: 'autopush.service'\n",
" command: 'start'\n",
" content: |\n",
" [Unit]\n",
" Description=Autopush container\n",
" Author=Mozilla Services\n",
" After=docker.service\n",
" \n",
" [Service]\n",
" Restart=always\n",
" ExecStartPre=-/usr/bin/docker kill autopush\n",
" ExecStartPre=-/usr/bin/docker rm autopush\n",
" ExecStartPre=/usr/bin/docker pull ",
"bbangert/autopush:1.14.1\n",
" ExecStart=/usr/bin/docker run ",
"--name autopush ",
"-p 8080:8080 ",
"-p 8081:8081 ",
"bbangert/autopush:1.14.1 ./pypy/bin/autopush ",
"--hostname $public_ipv4 ",
"--storage_tablename ", Ref(self.PushTablePrefix),
"_storage ",
"--message_tablename ", Ref(self.PushTablePrefix),
"_message ",
"--router_tablename ", Ref(self.PushTablePrefix),
"_router ",
"--router_hostname $private_ipv4 ",
"--endpoint_hostname ",
GetAtt(self.PushEndpointServerInstance, "PublicDnsName"),
" ",
"--crypto_key '", Ref(self.PushCryptoKey), "' ",
"\n"
]))
))
self._template.add_output([
Output(
"PushServerURL",
Description="Push Websocket URL",
Value=Join("", [
"ws://",
GetAtt(self.PushConnectionServerInstance, "PublicDnsName"),
":8080/"
])
)
])
def json(self):
return self._template.to_json()
def main():
template = Template()
template.add_version("2010-09-09")
template.set_description("AWS CloudFormation ECS Service")
# Add the Parameters
Application = template.add_parameter(
Parameter(
"Application",
Type="String",
))
DockerImage = template.add_parameter(
Parameter(
"DockerImage",
Type="String",
))
ClusterName = template.add_parameter(
Parameter(
"ClusterName",
Type="String",
))
ContainerPort = template.add_parameter(
Parameter(
"ContainerPort",
Type="String",
))
HostPort = template.add_parameter(Parameter(
"HostPort",
Type="String",
))
HostedZoneName = template.add_parameter(
Parameter(
"HostedZoneName",
Type="String",
))
CertArn = template.add_parameter(Parameter(
"CertArn",
Type="String",
))
ExecutionRoleArn = template.add_parameter(
Parameter("ExecutionRoleArn",
Type="String",
Description="Execution Role to get creadentials from ssm"))
HealthCheckPath = template.add_parameter(
Parameter(
"HealthCheckPath",
Type="String",
))
HealthCheckIntervalSeconds = template.add_parameter(
Parameter(
"HealthCheckIntervalSeconds",
Type="String",
))
HealthyThresholdCount = template.add_parameter(
Parameter(
"HealthyThresholdCount",
Type="String",
))
HealthCheckTimeoutSeconds = template.add_parameter(
Parameter(
"HealthCheckTimeoutSeconds",
Type="String",
))
UnhealthyThresholdCount = template.add_parameter(
Parameter(
"UnhealthyThresholdCount",
Type="String",
))
VpcId = template.add_parameter(Parameter(
"VpcId",
Type="String",
))
Subnets = template.add_parameter(
Parameter(
"Subnets",
Type="List<AWS::EC2::Subnet::Id>",
))
PrivateSubnets = template.add_parameter(
Parameter(
"PrivateSubnets",
Type="List<AWS::EC2::Subnet::Id>",
))
# Add the application ELB
NetworkLB = template.add_resource(
elb.LoadBalancer("NetworkLB",
Name=Join("", [Ref(Application), "-nlb"]),
Scheme="internet-facing",
Subnets=Ref(Subnets),
Type='network'))
NlbTargetGroup = template.add_resource(
elb.TargetGroup(
"NlbTargetGroup",
Name='ecs-fargate-service-targetgroup',
HealthCheckIntervalSeconds=Ref(HealthCheckIntervalSeconds),
HealthCheckProtocol="TCP",
HealthyThresholdCount=Ref(HealthyThresholdCount),
Port=80,
Protocol="TCP",
TargetType="ip",
UnhealthyThresholdCount=Ref(UnhealthyThresholdCount),
VpcId=Ref(VpcId)))
NlbListener = template.add_resource(
elb.Listener(
"Listener",
DependsOn=["NlbTargetGroup", "NetworkLB"],
Certificates=[elb.Certificate(CertificateArn=Ref(CertArn))],
Port="443",
Protocol="TLS",
LoadBalancerArn=Ref(NetworkLB),
DefaultActions=[
elb.Action(Type="forward", TargetGroupArn=Ref(NlbTargetGroup))
]))
Task_Definition = template.add_resource(
TaskDefinition(
'TaskDefinition',
Memory='512',
Cpu='256',
RequiresCompatibilities=['FARGATE'],
NetworkMode='awsvpc',
ExecutionRoleArn=Ref(ExecutionRoleArn),
ContainerDefinitions=[
ContainerDefinition(
Name=Join("", [Ref(Application)]),
Image=Ref(DockerImage),
Essential=True,
Environment=[Environment(Name="MY_ENV_VAR", Value="true")],
DockerLabels={
'aws-account': Ref("AWS::AccountId"),
'region': Ref("AWS::Region"),
'stack': Ref("AWS::StackName")
},
PortMappings=[
PortMapping(ContainerPort=Ref(ContainerPort))
])
]))
AwsVpcSg = template.add_resource(
ec2.SecurityGroup('SecurityGroup',
GroupDescription='Security Group',
SecurityGroupIngress=[
ec2.SecurityGroupRule(IpProtocol='-1',
CidrIp='10.0.0.0/8')
],
SecurityGroupEgress=[
ec2.SecurityGroupRule(IpProtocol="-1",
CidrIp="0.0.0.0/0")
],
VpcId=Ref(VpcId)))
app_service = template.add_resource(
Service("AppService",
DependsOn=["Listener", "TaskDefinition"],
Cluster=Ref(ClusterName),
LaunchType='FARGATE',
DesiredCount=1,
TaskDefinition=Ref(Task_Definition),
ServiceName=Join("", [Ref(Application), "-ecs-service"]),
LoadBalancers=[
ecs.LoadBalancer(ContainerName=Join(
"", [Ref(Application)]),
ContainerPort=Ref(ContainerPort),
TargetGroupArn=Ref(NlbTargetGroup))
],
NetworkConfiguration=NetworkConfiguration(
AwsvpcConfiguration=AwsvpcConfiguration(
Subnets=Ref(PrivateSubnets),
SecurityGroups=[Ref(AwsVpcSg)]))))
AppDNSRecord = template.add_resource(
RecordSetType(
"AppDNSRecord",
DependsOn=["AppService"],
HostedZoneName=Join("", [Ref(HostedZoneName), "."]),
Name=Join("", [Ref(Application), ".",
Ref(HostedZoneName), "."]),
Type="CNAME",
TTL="900",
ResourceRecords=[GetAtt(NetworkLB, "DNSName")]))
template.add_output(
Output("URL",
Description="DomainName",
Value=Join("", ["https://", Ref(AppDNSRecord)])))
with open("ecs-fargate-service-cf.yaml", "w") as yamlout:
yamlout.write(template.to_yaml())
from troposphere import Template
from troposphere.codedeploy import AutoRollbackConfiguration, \
DeploymentStyle, DeploymentGroup, ElbInfoList, LoadBalancerInfo
template = Template()
template.add_version('2010-09-09')
auto_rollback_configuration = AutoRollbackConfiguration(
Enabled=True,
Events=['DEPLOYMENT_FAILURE']
)
deployment_style = DeploymentStyle(
DeploymentOption='WITH_TRAFFIC_CONTROL'
)
elb_info_list = ElbInfoList(
Name='DemoLoadBalancer'
)
load_balancer_info = LoadBalancerInfo(
ElbInfoList=[elb_info_list]
)
deployment_group = DeploymentGroup(
"DemoDeploymentGroup",
ApplicationName='DemoApplication',
AutoRollbackConfiguration=auto_rollback_configuration,
DeploymentStyle=deployment_style,
def make_bigfix_awscf_template():
template = Template()
# http://stackoverflow.com/questions/843277/how-do-i-check-if-a-variable-exists-in-python
# http://www.tutorialspoint.com/python/string_endswith.htm
# Only run the function to include the Meraki MSI installation if Windows Server + MERAKI_MSI_URL defined properly
# MERAKI_MSI_URL should be defined in the bf_cf_config.py file
if "Windows" == BES_ROOT_SERVER_TYPE and 'MERAKI_MSI_URL' in globals() and MERAKI_MSI_URL.endswith('/MerakiPCCAgent.msi'):
meraki_msi_url = MERAKI_MSI_URL
else:
meraki_msi_url = ""
template.add_description("""\
BigFix Eval AWS CloudFormation Template within the resource limits of the AWS free tier. \
**WARNING** This template creates Amazon EC2 & RDS instances. You may be billed \
for the AWS resources used if you create a stack from this template.""")
template.add_version('2010-09-09')
# define subnet
# define DB Subnet Group
# define RDS instance
# https://github.com/cloudtools/troposphere/blob/master/troposphere/rds.py#L13
# define EBS volume (to be attached to the EC2 instance)
# define Metadata for EC2 instance
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-init.html#aws-resource-init-files
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/deploying.applications.html
# https://github.com/cloudtools/troposphere/issues/3
metadata = {
"AWS::CloudFormation::Init": {
"config": {
"packages": {
"msi" : {
#"awscli" : "https://s3.amazonaws.com/aws-cli/AWSCLI64.msi",
"meraki" : meraki_msi_url
},
},
}
}
}
# define EC2 instance with Metadata
ec2_instance = ec2.Instance("BigFixEval", Metadata=metadata)
ec2_instance.ImageId = "ami-6502e021"
ec2_instance.InstanceType = "t2.micro"
# https://github.com/cloudtools/troposphere/blob/master/troposphere/ec2.py#L134
#ec2_instance.SubnetId = "" # Ref(SubnetID)
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-instance.html#cfn-ec2-instance-volumes
# https://github.com/cloudtools/troposphere/blob/master/troposphere/ec2.py#L138
#ec2_instance.Volumes = "" # Ref(Volume)
template.add_resource(ec2_instance)
template.add_output([
Output(
"AZ",
Description="Availability Zone of the newly created EC2 instance",
Value=GetAtt(ec2_instance, "AvailabilityZone"),
),
Output(
"PublicDNS",
Description="Public DNSName of the newly created EC2 instance",
Value=GetAtt(ec2_instance, "PublicDnsName"),
),
])
template.add_parameter(
Parameter(
'RootServerDomain',
Description=' The domain name the BigFix root server will use ',
Type='String',
Default='bigfixeval.organization.tld'
))
return template.to_json()
"""
Batch.
Create the AWS Batch Compute environment and roles.
"""
from troposphere import GetAtt, Output, Parameter, Ref, Tags, Template
from troposphere.batch import (ComputeEnvironment, ComputeEnvironmentOrder,
ComputeResources, JobQueue,
LaunchTemplateSpecification)
from troposphere.ec2 import SecurityGroup
from troposphere.iam import InstanceProfile, Role
t = Template('AWS Batch')
t.add_version()
PrivateSubnetA = t.add_parameter(Parameter(
'PrivateSubnetA',
Type='String'
))
PrivateSubnetB = t.add_parameter(Parameter(
'PrivateSubnetB',
Type='String'
))
Vpc = t.add_parameter(Parameter(
'Vpc',
ConstraintDescription='Must be a valid VPC ID.',
Type='AWS::EC2::VPC::Id'
))
env = arguments["ENV"].lower()
with open('conf/aws_config.json') as aws_config_file:
aws_config = json.load(aws_config_file)
with open(arguments["-f"]) as config_file:
config = json.load(config_file)
vpc = {
"VpcId": aws_config["VpcId"]
}
zones = config.get("Zones", aws_config["Zones"])
template = Template()
template.add_version("2010-09-09")
min_size = Parameter(
"MinSize",
Default="1",
AllowedValues=config["MinMaxNumberOfInstancesPerZone"],
Type="Number",
Description="Set MinSize and MaxSize to 0 to stop instances. Set it to 1 to start instances",
)
template.add_parameter(min_size)
max_size = Parameter(
"MaxSize",
Default="1",
AllowedValues=config["MinMaxNumberOfInstancesPerZone"],
Type="Number",
class CreateTemplate(object):
def __init__(self):
self.start_template()
self.add_mappings()
self.add_parameters()
self.add_security_groups()
self.add_kippo_rds()
self.add_kippo_sensors()
def start_template(self):
self.template = Template()
self.template.add_version('2010-09-09')
self.template.add_description('Kippo cluster CloudFormation stack')
def add_mappings(self):
# Ubuntu Trusty 14.04 LTS amd64
self.template.add_mapping(
'Ec2AmiMap', {
'ap-northeast-1': {
'AmiId': 'ami-c011d4c0'
},
'ap-southeast-1': {
'AmiId': 'ami-76546924'
},
'eu-central-1': {
'AmiId': 'ami-00dae61d'
},
'eu-west-1': {
'AmiId': 'ami-2396f654'
},
'sa-east-1': {
'AmiId': 'ami-75b23768'
},
'us-east-1': {
'AmiId': 'ami-f63b3e9e'
},
'us-west-1': {
'AmiId': 'ami-057f9d41'
},
'cn-north-1': {
'AmiId': 'ami-78d84541'
},
'us-gov-west-1': {
'AmiId': 'ami-85fa9ba6'
},
})
def add_parameters(self):
self.template.add_parameter(
Parameter(
'Ec2InstanceType',
Default='t2.micro',
Description='Instance type of the EC2 instances',
Type='String',
))
self.template.add_parameter(
Parameter(
'Ec2SubnetIdList',
Description=
'List of subnet IDs in which to create the EC2 instances',
Type='List<AWS::EC2::Subnet::Id>',
))
self.template.add_parameter(
Parameter(
'ElbSubnetIdList',
Description='List of subnet IDs in which to create the ELB',
Type='List<AWS::EC2::Subnet::Id>',
))
self.template.add_parameter(
Parameter(
'KeyName',
Description=
'Name of the keypair to install on the EC2 instances',
Type='AWS::EC2::KeyPair::KeyName',
))
self.template.add_parameter(
Parameter(
'KippoSensorCount',
Default='1',
Description='Number of kippo sensors to create',
Type='Number',
))
self.template.add_parameter(
Parameter(
'RdsInstanceType',
Default='t2.micro',
Description='Instance type of the RDS instance',
Type='String',
))
self.template.add_parameter(
Parameter(
'RdsRootPassword',
Description='Password to use for the root RDS user',
Type='String',
))
self.template.add_parameter(
Parameter(
'RdsStorage',
Default='20',
Description='Amount of storage (GB) for the RDS instance',
Type='Number',
))
self.template.add_parameter(
Parameter(
'RdsSubnetIdList',
Description=
'List of subnet IDs in which to create the RDS instance',
Type='List<AWS::EC2::Subnet::Id>',
))
self.template.add_parameter(
Parameter(
'RealSshPort',
Description='Port number to use for the real SSH service',
Type='Number',
))
self.template.add_parameter(
Parameter(
'VpcId',
Description=
'ID of the VPC in which to create the kippo cluster',
Type='AWS::EC2::VPC::Id',
))
def add_security_groups(self):
# kippo sensor EC2 security group
self.template.add_resource(
SecurityGroup(
'Ec2SecurityGroup',
GroupDescription='Security group for the kippo sensor ASG',
SecurityGroupIngress=[
# Allow SSH (to kippo) from anywhere
SecurityGroupRule(
CidrIp='0.0.0.0/0',
FromPort=22,
ToPort=22,
IpProtocol='tcp',
),
# Allow real SSH from anywhere
SecurityGroupRule(
CidrIp='0.0.0.0/0',
FromPort=Ref('RealSshPort'),
ToPort=Ref('RealSshPort'),
IpProtocol='tcp',
),
# Allow HTTP from the kippo ELB
SecurityGroupRule(
FromPort=80,
ToPort=80,
IpProtocol='tcp',
SourceSecurityGroupId=Ref('ElbSecurityGroup'),
),
# Allow HTTPS from the kippo ELB
SecurityGroupRule(
FromPort=443,
ToPort=443,
IpProtocol='tcp',
SourceSecurityGroupId=Ref('ElbSecurityGroup'),
),
],
VpcId=Ref('VpcId'),
))
# kippo sensor ELB security group
self.template.add_resource(
SecurityGroup(
'ElbSecurityGroup',
GroupDescription='Security group for the kippo sensor ELB',
SecurityGroupIngress=[
# Allow HTTP from anywhere
SecurityGroupRule(
CidrIp='0.0.0.0/0',
FromPort=80,
ToPort=80,
IpProtocol='tcp',
),
# Allow HTTPS from anywhere
SecurityGroupRule(
CidrIp='0.0.0.0/0',
FromPort=443,
ToPort=443,
IpProtocol='tcp',
),
],
VpcId=Ref('VpcId'),
))
# RDS security group
self.template.add_resource(
SecurityGroup(
'RdsSecurityGroup',
VpcId=Ref('VpcId'),
GroupDescription='Security group for the kippo RDS instance',
SecurityGroupIngress=[
# Allow MySQL from kippo EC2 instances
SecurityGroupRule(
FromPort=3306,
ToPort=3306,
IpProtocol='tcp',
SourceSecurityGroupId=Ref('Ec2SecurityGroup'),
),
],
))
def add_kippo_rds(self):
self.template.add_resource(
DBSubnetGroup(
'RdsSubnetGroup',
DBSubnetGroupDescription=
'Subnet group for the kippo RDS instance',
SubnetIds=Ref('RdsSubnetIdList'),
))
self.template.add_resource(
DBInstance(
'RdsInstance',
AllocatedStorage=Ref('RdsStorage'),
DBInstanceClass=Ref('RdsInstanceType'),
DBInstanceIdentifier='kippo-database',
DBSubnetGroupName=Ref('RdsSubnetGroup'),
Engine='MySQL',
EngineVersion='5.6.22',
MasterUsername='******',
MasterUserPassword=Ref('RdsRootPassword'),
MultiAZ=True,
Port=3306,
VPCSecurityGroups=[Ref('RdsSecurityGroup')],
))
self.template.add_output(
Output(
'RdsEndpoint',
Description='RDS endpoint address',
Value=GetAtt('RdsInstance', 'Endpoint.Address'),
))
def add_kippo_sensors(self):
# Create the ELB
self.template.add_resource(
LoadBalancer(
'Elb',
Listeners=[
Listener(
InstancePort=80,
LoadBalancerPort=80,
Protocol='http',
),
Listener(
InstancePort=443,
LoadBalancerPort=443,
Protocol='tcp', # Plain TCP forwarding for HTTPS/SSL
),
],
CrossZone=True,
Subnets=Ref('ElbSubnetIdList'),
SecurityGroups=[Ref('ElbSecurityGroup')],
Scheme='internet-facing',
HealthCheck=HealthCheck(
Target='HTTP:80/kippo-graph/',
HealthyThreshold=2,
UnhealthyThreshold=5,
Interval=120,
Timeout=60,
),
))
self.template.add_output(
Output(
'ElbEndpoint',
Description='ELB endpoint address',
Value=GetAtt('Elb', 'DNSName'),
))
self.template.add_resource(
LaunchConfiguration(
'LaunchConfiguration',
KeyName=Ref('KeyName'),
ImageId=FindInMap('Ec2AmiMap', Ref('AWS::Region'), 'AmiId'),
InstanceType=Ref('Ec2InstanceType'),
SecurityGroups=[Ref('Ec2SecurityGroup')],
AssociatePublicIpAddress=True,
UserData=Base64(
Join('\n', [
'#cloud-config',
'repo_upgrade: security',
'runcmd:',
' - "/usr/bin/wget -O /tmp/configure_kippo_sensor.sh https://raw.githubusercontent.com/cdodd/aws-kippo-cluster/master/bootstrap/configure_kippo_sensor.sh"',
Join(
'',
[
' - "bash /tmp/configure_kippo_sensor.sh',
' ',
GetAtt('RdsInstance', 'Endpoint.Address'),
' ',
Ref('RdsRootPassword'),
' ',
Ref('RealSshPort'),
'"',
],
),
])),
))
self.template.add_resource(
AutoScalingGroup(
'Asg',
DesiredCapacity=Ref('KippoSensorCount'),
HealthCheckGracePeriod=1800,
HealthCheckType='ELB',
LaunchConfigurationName=Ref('LaunchConfiguration'),
LoadBalancerNames=[Ref('Elb')],
MaxSize=Ref('KippoSensorCount'),
MinSize=Ref('KippoSensorCount'),
Tags=[Tag(key='Name', value='kippo-sensor', propogate='true')],
VPCZoneIdentifier=Ref('Ec2SubnetIdList'),
))
def output_template(self):
return self.template.to_json()
class Infra:
def __init__(self, config):
self.log = logging.getLogger(__name__)
self.aws = config['config:aws']
self.t = Template()
self.t.add_version('2010-09-09')
self.t.add_description(
'TTT Infra: VPC, IGW, RouteTable, DefaultRoute, S3'
)
# TODO: add OUTPUTs here too so we can accumulate them in the resources
def __str__(self):
# is this a string of json?
return self.t.to_json()
def _name_tags(self, component):
"""Return logical name and tags based on short name of component.
If 'component' is 'vpc', look in config for 'app', 'env', and
'vpc.name'. Create a LogicalName from app and component, and tags based
on that and the env. If app='TTTdiazo', env='prod', component='vpc'
and 'vpc.name'='VPC' we would return tuple::
('TTTdiazoVPC', {'TTTdiazo-VPC-prod', app='TTTdiazo', env='prod'})
Logical names may not have dashes or underscores, so we use
punctation-free name for those, but tags are more readable with dashes.
:param str component: name of the component to use as key in .ini file
:returns: `tuple` (name, tags)
"""
app = self.aws['app']
env = self.aws['env']
sname = self.aws[component + '.name']
name = app + sname
tags = {'Name': '{}-{}-{}'.format(app, sname, env),
'env': env, 'app': app}
return (name, tags)
def add_vpc(self):
"""Add VPC to template and return the resource."""
name, tags = self._name_tags('vpc')
self.vpc = self.t.add_resource(
ec2.VPC(
name,
CidrBlock=self.aws['vpc.cidr_block'],
EnableDnsHostnames=True,
Tags=Tags(**tags)
))
self.t.add_output(Output(
"VpcId", Value=Ref(self.vpc)
))
###########################################################################
# SGs
def add_sg_ssh(self):
name, tags = self._name_tags('sg_ssh')
sg_rules = []
for port in self.aws['sg_ssh.ports'].split(','):
sg_rules.append(
ec2.SecurityGroupRule(
IpProtocol='tcp',
FromPort=port,
ToPort=port,
CidrIp=self.aws['sg_ssh.cidr_block'],
),
)
self.sg_ssh = self.t.add_resource(
ec2.SecurityGroup(
name,
VpcId=Ref(self.vpc),
GroupDescription=self.aws['sg_ssh.desc'],
SecurityGroupIngress=sg_rules,
Tags=Tags(**tags)
))
self.t.add_output(Output(
"SGSSH", Value=Ref(self.sg_ssh)
))
def add_sg_elb(self):
"""Allow world to ELB, then allow priv subnet EC2s from this SG."""
name, tags = self._name_tags('sg_elb')
sg_rules = []
for port in self.aws['sg_elb.ports'].split(','):
sg_rules.append(
ec2.SecurityGroupRule(
IpProtocol='tcp',
FromPort=port,
ToPort=port,
CidrIp=self.aws['sg_elb.cidr_block'],
),
)
self.sg_elb = self.t.add_resource(
ec2.SecurityGroup(
name,
VpcId=Ref(self.vpc),
GroupDescription=self.aws['sg_elb.desc'],
SecurityGroupIngress=sg_rules,
Tags=Tags(**tags)
))
self.t.add_output(Output(
"SGELB", Value=Ref(self.sg_elb)
))
###########################################################################
# IGW
def add_igw_gateway(self):
name, tags = self._name_tags('igw_gateway')
self.igw_gateway = self.t.add_resource(
ec2.InternetGateway(
name,
Tags=Tags(**tags),
))
def add_igw_attachment(self):
name, tags = self._name_tags('igw_attachment')
self.igw_attachment = self.t.add_resource(
ec2.VPCGatewayAttachment(
name,
VpcId=Ref(self.vpc),
InternetGatewayId=Ref(self.igw_gateway),
# Doesn't support: Tags=Tags(**tags),
))
def add_igw_route_table(self):
name, tags = self._name_tags('igw_route_table')
self.igw_route_table = self.t.add_resource(
ec2.RouteTable(
name,
VpcId=Ref(self.vpc),
Tags=Tags(**tags),
))
self.t.add_output(Output(
"IGWRoute", Value=Ref(self.igw_route_table)
))
def add_igw_default_route(self):
name, tags = self._name_tags('igw_default_route')
self.igw_default_route = self.t.add_resource(
ec2.Route(
name,
DependsOn=self.igw_attachment.name, # above
GatewayId=Ref(self.igw_gateway),
DestinationCidrBlock="0.0.0.0/0",
RouteTableId=Ref(self.igw_route_table),
# Doesn't support: Tags=Tags(**tags),
))
###########################################################################
# Subnets:
# - public subnet for ELB public access
# - internal public subnet for EC2 so it can use IGW to get out to TTT
# (if on private subnet, needs separate NAT instance to get out: extra
# cost and a single point of failure)
def add_subnet_public(self):
name, tags = self._name_tags('subnet_public')
self.subnet_public = self.t.add_resource(
ec2.Subnet(
name,
AvailabilityZone=self.aws['subnet_public.az'],
CidrBlock=self.aws['subnet_public.cidr_block'],
VpcId=Ref(self.vpc),
Tags=Tags(**tags),
))
self.t.add_output(Output(
"PublicSubnet", Value=Ref(self.subnet_public)
))
def add_subnet_app(self):
name, tags = self._name_tags('subnet_app')
self.subnet_app = self.t.add_resource(
ec2.Subnet(
name,
AvailabilityZone=self.aws['subnet_app.az'],
CidrBlock=self.aws['subnet_app.cidr_block'],
VpcId=Ref(self.vpc),
Tags=Tags(**tags),
))
self.t.add_output(Output(
"AppSubnet", Value=Ref(self.subnet_app)
))
def add_subnet_db1(self):
name, tags = self._name_tags('subnet_db1')
self.subnet_db1 = self.t.add_resource(
ec2.Subnet(
name,
AvailabilityZone=self.aws['subnet_db1.az'],
CidrBlock=self.aws['subnet_db1.cidr_block'],
VpcId=Ref(self.vpc),
Tags=Tags(**tags),
))
self.t.add_output(Output(
"DB1Subnet", Value=Ref(self.subnet_db1)
))
def add_subnet_db2(self):
name, tags = self._name_tags('subnet_db2')
self.subnet_db2 = self.t.add_resource(
ec2.Subnet(
name,
AvailabilityZone=self.aws['subnet_db2.az'],
CidrBlock=self.aws['subnet_db2.cidr_block'],
VpcId=Ref(self.vpc),
Tags=Tags(**tags),
))
self.t.add_output(Output(
"DB2Subnet", Value=Ref(self.subnet_db2)
))
def add_subnet_public_rta(self):
name, tags = self._name_tags('subnet_public_rta')
self.subnet_public_rta = self.t.add_resource(
ec2.SubnetRouteTableAssociation(
name,
SubnetId=Ref(self.subnet_public),
RouteTableId=Ref(self.igw_route_table),
# Doesn't support: Tags=Tags(**tags),
))
def add_subnet_app_rta(self):
name, tags = self._name_tags('subnet_app_rta')
self.subnet_app_rta = self.t.add_resource(
ec2.SubnetRouteTableAssociation(
name,
SubnetId=Ref(self.subnet_app),
RouteTableId=Ref(self.igw_route_table),
# Doesn't support: Tags=Tags(**tags),
))
def add_subnet_db1_rta(self):
name, tags = self._name_tags('subnet_db1_rta')
self.subnet_db1_rta = self.t.add_resource(
ec2.SubnetRouteTableAssociation(
name,
SubnetId=Ref(self.subnet_db1),
RouteTableId=Ref(self.igw_route_table),
# Doesn't support: Tags=Tags(**tags),
))
def add_subnet_db2_rta(self):
name, tags = self._name_tags('subnet_db2_rta')
self.subnet_db1_rta = self.t.add_resource(
ec2.SubnetRouteTableAssociation(
name,
SubnetId=Ref(self.subnet_db2),
RouteTableId=Ref(self.igw_route_table),
# Doesn't support: Tags=Tags(**tags),
))
###########################################################################
# Role and Policy: allow EC2 to access S3 to get code
# You must specify at invocation: --capabilities CAPABILITY_IAM
# Test with python and boto3 in a virtualenv on the instance:
# python -c "import boto3; print(boto3.resource('s3').Object('CODEBUCKET.aws.v-studios.com', 'index.html').get())"
# I want to allow just our role or EC2 profile to be able to get objects
# via HTTP, e.g., with curl on the static website that's set up. However, I
# can't see how to do this. This Bucket Policy is accepted, but curl on the
# instance is denied for index.html. ("Principal":"*" works).
# {
# "Version": "2012-10-17",
# "Statement": [
# {
# "Sid": "TRY role/policy access ",
# "Effect": "Allow",
# "Principal": {
# "AWS": [
# "arn:aws:iam::############:role/tttdiazoprod-TTTdiazoRoleEc2S3-1LR1T7POA0ZL7"
# ]
# },
# "Action": "s3:GetObject",
# "Resource": "arn:aws:s3:::code.aws.v-studios.com/*"
# }
# ]
# }
#
# Trying Instance Profile name is rejected: Invalid principal in policy:
# "Principal": {
# "AWS": ["arn:aws:iam::############:instance-profile/tttdiazoprod-TTTdiazoProfileDiazo-D0A10MLPODFX"]
# }, ...
def add_role(self):
name, tags = self._name_tags('role')
self.role = self.t.add_resource(
iam.Role(
name,
AssumeRolePolicyDocument=Policy(
Statement=[
Statement(
Action=[awacs.sts.AssumeRole],
Effect=Allow,
Principal=Principal('Service', 'ec2.amazonaws.com'),
Sid=name, # redundant?
),
],
),
ManagedPolicyArns=['arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess'],
Path='/',
Policies=[iam.Policy(
name + 'Policy',
PolicyDocument=Policy(
Statement=[
Statement(
Action=[
awacs.s3.GetObject,
awacs.s3.ListBucket,
],
Effect=Allow,
Resource=[awacs.s3.ARN(self.aws['s3.bucket']),
awacs.s3.ARN(self.aws['s3.bucket'] + '/*')],
),
],
),
PolicyName=name + 'Policy',
)],
# Doesn't support: Tags=Tags(**tags)
))
def add_profile(self):
name, tags = self._name_tags('profile')
self.profile = self.t.add_resource(
iam.InstanceProfile(
name,
Path='/',
Roles=[Ref(self.role.name)],
# Doesn't support: Tags=Tags(**tags),
))
###########################################################################
# S3 bucket for code, with DNS name for HTTP access
def add_s3(self):
name, tags = self._name_tags('s3')
self.s3 = self.t.add_resource(
s3.Bucket(
name,
AccessControl=s3.BucketOwnerFullControl,
BucketName=self.aws['s3.bucket'],
Tags=Tags(**tags),
WebsiteConfiguration=s3.WebsiteConfiguration(IndexDocument='index.html'),
))
def add_s3_dns(self):
name, tags = self._name_tags('s3_dns')
self.s3_dns = self.t.add_resource(
route53.RecordSetType(
name,
HostedZoneName=self.aws['s3_dns.zone'],
Comment='CNAME to public ELB',
Name=self.aws['s3_dns.record'] + '.' + self.aws['s3_dns.zone'],
TTL=self.aws['s3_dns.ttl'],
Type='CNAME',
ResourceRecords=[GetAtt(self.s3.name, 'DomainName')],
# Doesn't support: Tags=Tags(**tags),
))
###########################################################################
# Route53 DNS zones and records
def add_r53_dns(self):
zone_name_index = 0
name, tags = self._name_tags('r53_dns')
for zone in self.aws['r53_dns.zones'].split(','):
zone_name = name + str(zone_name_index)
self.r53_dns = self.t.add_resource(
route53.HostedZone(
zone_name,
Name=zone,
))
zone_output_name = "HostedZoneName" + str(zone_name_index)
self.t.add_output(Output(
zone_output_name, Value=Ref(self.r53_dns)
))
zone_name_index += 1
###########################################################################
# CodeDeploy environment which deploys to instances in the app ASGs
# Iterates over contents of cd_application.names to create multiple
# CodeDeploy Apps with an IAM user and role per App
def add_cd_applications(self):
cdapp_index = 0
for cdapp in self.aws['cd_application.names'].split(','):
name, tags = self._name_tags('cd_application')
cdapp_name = name + str(cdapp_index)
self.cd_application = self.t.add_resource(
codedeploy.Application(
cdapp_name,
ApplicationName=cdapp,
# Doesn't support: Tags=Tags(**tags),
))
cdapp_output_name = "CDApp" + str(cdapp_index)
self.t.add_output(Output(
cdapp_output_name, Value=Ref(self.cd_application)
))
cd_role_name = cdapp + "CDRole"
self.cd_role = self.t.add_resource(
iam.Role(
cd_role_name,
AssumeRolePolicyDocument=Policy(
Statement=[
Statement(
Action=[awacs.sts.AssumeRole],
Effect=Allow,
Principal=Principal('Service', 'codedeploy.amazonaws.com'),
Sid=cd_role_name, # redundant?
),
],
),
ManagedPolicyArns=['arn:aws:iam::aws:policy/service-role/AWSCodeDeployRole'],
Path='/',
))
cd_role_output_name = "CDRole" + str(cdapp_index)
self.t.add_output(Output(
cd_role_output_name, Value=Ref(self.cd_role)
))
cd_iam_user_name = cdapp + "CDUser"
self.cd_iam_user = self.t.add_resource(
iam.User(
cd_iam_user_name,
# Can't attach policy here,
# must create policy and attach to user from app stack
# ManagedPolicyArns=[Ref(self.cd_iam_user_policy.name)]
))
cd_iam_user_output_name = "CDUser" + str(cdapp_index)
self.t.add_output(Output(
cd_iam_user_output_name, Value=Ref(self.cd_iam_user)
))
cdapp_index += 1
def main():
template = Template()
template.add_version("2010-09-09")
template.set_description(
"AWS CloudFormation Sample Template: ELB with 2 EC2 instances")
AddAMI(template)
# Add the Parameters
keyname_param = template.add_parameter(
Parameter(
"KeyName",
Type="String",
Default="mark",
Description="Name of an existing EC2 KeyPair to "
"enable SSH access to the instance",
))
template.add_parameter(
Parameter(
"InstanceType",
Type="String",
Description="WebServer EC2 instance type",
Default="m1.small",
AllowedValues=[
"t1.micro", "m1.small", "m1.medium", "m1.large", "m1.xlarge",
"m2.xlarge", "m2.2xlarge", "m2.4xlarge", "c1.medium",
"c1.xlarge", "cc1.4xlarge", "cc2.8xlarge", "cg1.4xlarge"
],
ConstraintDescription="must be a valid EC2 instance type.",
))
webport_param = template.add_parameter(
Parameter(
"WebServerPort",
Type="String",
Default="8888",
Description="TCP/IP port of the web server",
))
# Define the instance security group
instance_sg = template.add_resource(
ec2.SecurityGroup(
"InstanceSecurityGroup",
GroupDescription="Enable SSH and HTTP access on the inbound port",
SecurityGroupIngress=[
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort="22",
ToPort="22",
CidrIp="0.0.0.0/0",
),
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=Ref(webport_param),
ToPort=Ref(webport_param),
CidrIp="0.0.0.0/0",
),
]))
# Add the web server instances
web_instances = []
for name in ("Ec2Instance1", "Ec2Instance2"):
instance = template.add_resource(
ec2.Instance(
name,
SecurityGroups=[Ref(instance_sg)],
KeyName=Ref(keyname_param),
InstanceType=Ref("InstanceType"),
ImageId=FindInMap("RegionMap", Ref("AWS::Region"), "AMI"),
UserData=Base64(Ref(webport_param)),
))
web_instances.append(instance)
elasticLB = template.add_resource(
elb.LoadBalancer(
'ElasticLoadBalancer',
AccessLoggingPolicy=elb.AccessLoggingPolicy(
EmitInterval=5,
Enabled=True,
S3BucketName="logging",
S3BucketPrefix="myELB",
),
AvailabilityZones=GetAZs(""),
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=300,
),
CrossZone=True,
Instances=[Ref(r) for r in web_instances],
Listeners=[
elb.Listener(
LoadBalancerPort="80",
InstancePort=Ref(webport_param),
Protocol="HTTP",
),
],
HealthCheck=elb.HealthCheck(
Target=Join("", ["HTTP:", Ref(webport_param), "/"]),
HealthyThreshold="3",
UnhealthyThreshold="5",
Interval="30",
Timeout="5",
)))
template.add_output(
Output("URL",
Description="URL of the sample website",
Value=Join("",
["http://", GetAtt(elasticLB, "DNSName")])))
print(template.to_json())
SecureTransport, SourceArn, SourceIp, UserAgent
import awacs.sts
import awacs.cloudformation
import awacs.iam
import awacs.ec2
import awacs.logs
import csv
import cfnutil
import six
#
# Template
#
t = Template()
t.add_version('2010-09-09')
t.add_description('Amazon EC2 instance running the Ubuntu Linux.')
#
# Interface
#
parameter_groups = [
{
'Label': {
'default': 'Network Configuration'
},
'Parameters': [
'VpcId',
'SubnetId',
'AllocateElasticIp',
'AssociatePublicIp',
def generate_cf_template():
"""
Returns an entire CloudFormation stack by using troposphere to construct
each piece
"""
# Header of CloudFormation template
t = Template()
t.add_version("2010-09-09")
t.add_description("Lambda Chat AWS Resources")
# Paramters
description = "should match [0-9]+-[a-z0-9]+.apps.googleusercontent.com"
google_oauth_client_id = t.add_parameter(Parameter(
"GoogleOAuthClientID",
AllowedPattern="[0-9]+-[a-z0-9]+.apps.googleusercontent.com",
Type="String",
Description="The Client ID of your Google project",
ConstraintDescription=description
))
website_s3_bucket_name = t.add_parameter(Parameter(
"WebsiteS3BucketName",
AllowedPattern="[a-zA-Z0-9\-]*",
Type="String",
Description="Name of S3 bucket to store the website in",
ConstraintDescription="can contain only alphanumeric characters and dashes.",
))
# The SNS topic the website will publish chat messages to
website_sns_topic = t.add_resource(sns.Topic(
'WebsiteSnsTopic',
TopicName='lambda-chat',
DisplayName='Lambda Chat'
))
t.add_output(Output(
"WebsiteSnsTopic",
Description="sns_topic_arn",
Value=Ref(website_sns_topic),
))
# The IAM Role and Policy the website will assume to publish to SNS
website_role = t.add_resource(iam.Role(
"WebsiteRole",
Path="/",
AssumeRolePolicyDocument=Policy(
Statement=[
Statement(
Effect=Allow,
Action=[Action("sts", "AssumeRoleWithWebIdentity")],
Principal=Principal("Federated", "accounts.google.com"),
Condition=Condition(
StringEquals(
"accounts.google.com:aud",
Ref(google_oauth_client_id)
)
),
),
],
),
))
t.add_resource(iam.PolicyType(
"WebsitePolicy",
PolicyName="lambda-chat-website-policy",
Roles=[Ref(website_role)],
PolicyDocument=Policy(
Version="2012-10-17",
Statement=[
Statement(
Effect=Allow,
Action=[Action("sns", "Publish")],
Resource=[
Ref(website_sns_topic)
],
),
],
)
))
t.add_output(Output(
"WebsiteRole",
Description="website_iam_role_arn",
Value=GetAtt(website_role, "Arn"),
))
website_bucket = t.add_resource(s3.Bucket(
'WebsiteS3Bucket',
BucketName=Ref(website_s3_bucket_name),
WebsiteConfiguration=s3.WebsiteConfiguration(
ErrorDocument="error.html",
IndexDocument="index.html"
)
))
t.add_output(Output(
"S3Bucket",
Description="s3_bucket",
Value=Ref(website_bucket),
))
t.add_resource(s3.BucketPolicy(
'WebsiteS3BucketPolicy',
Bucket=Ref(website_bucket),
PolicyDocument={
"Version": "2012-10-17",
"Statement": [
{
"Sid": "PublicAccess",
"Effect": "Allow",
"Principal": "*",
"Action": ["s3:GetObject"],
"Resource": [{
"Fn::Join": [
"",
[
"arn:aws:s3:::",
{
"Ref": "WebsiteS3Bucket",
},
"/*"
]
]
}]
}
]
}
))
return t
# This is just hardcoded in right now, and really only fits us-east-1's mold
availability_zones = ['a', 'b', 'c']
def get_availability_zones():
return availability_zones
def get_asg_azs():
return [Join('', [Ref('AWS::Region'), az]) for az in availability_zones]
# Initialize the Cloudformation template
template = Template()
template.add_version('2010-09-09')
template.add_description(
'This is a cloudformation script that creates our specific VPCs. Each VPC spans 3 availability zones.'
)
# These are all Ubuntu 14.04 AMIs (see: http://cloud-images.ubuntu.com/locator/ec2/)
template.add_mapping(
'RegionMap', {
'us-east-1': {
int(Amis.NAT): 'ami-184dc970',
int(Amis.EBS): 'ami-86562dee',
int(Amis.INSTANCE): 'ami-cc5229a4'
},
'us-west-1': {
int(Amis.NAT): 'ami-a98396ec'
},
def main():
template = Template()
template.add_version("2010-09-09")
template.add_description(
"AWS CloudFormation Sample Template: ELB with 2 EC2 instances")
AddAMI(template)
# Add the Parameters
keyname_param = template.add_parameter(Parameter(
"KeyName",
Type="String",
Default="mark",
Description="Name of an existing EC2 KeyPair to "
"enable SSH access to the instance",
))
template.add_parameter(Parameter(
"InstanceType",
Type="String",
Description="WebServer EC2 instance type",
Default="m1.small",
AllowedValues=[
"t1.micro", "m1.small", "m1.medium", "m1.large", "m1.xlarge",
"m2.xlarge", "m2.2xlarge", "m2.4xlarge", "c1.medium", "c1.xlarge",
"cc1.4xlarge", "cc2.8xlarge", "cg1.4xlarge"
],
ConstraintDescription="must be a valid EC2 instance type.",
))
webport_param = template.add_parameter(Parameter(
"WebServerPort",
Type="String",
Default="8888",
Description="TCP/IP port of the web server",
))
# Define the instance security group
instance_sg = template.add_resource(
ec2.SecurityGroup(
"InstanceSecurityGroup",
GroupDescription="Enable SSH and HTTP access on the inbound port",
SecurityGroupIngress=[
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort="22",
ToPort="22",
CidrIp="0.0.0.0/0",
),
ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=Ref(webport_param),
ToPort=Ref(webport_param),
CidrIp="0.0.0.0/0",
),
]
)
)
# Add the web server instances
web_instances = []
for name in ("Ec2Instance1", "Ec2Instance2"):
instance = template.add_resource(ec2.Instance(
name,
SecurityGroups=[Ref(instance_sg)],
KeyName=Ref(keyname_param),
InstanceType=Ref("InstanceType"),
ImageId=FindInMap("RegionMap", Ref("AWS::Region"), "AMI"),
UserData=Base64(Ref(webport_param)),
))
web_instances.append(instance)
elasticLB = template.add_resource(elb.LoadBalancer(
'ElasticLoadBalancer',
AccessLoggingPolicy=elb.AccessLoggingPolicy(
EmitInterval=5,
Enabled=True,
S3BucketName="logging",
S3BucketPrefix="myELB",
),
AvailabilityZones=GetAZs(""),
ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy(
Enabled=True,
Timeout=300,
),
CrossZone=True,
Instances=[Ref(r) for r in web_instances],
Listeners=[
elb.Listener(
LoadBalancerPort="80",
InstancePort=Ref(webport_param),
Protocol="HTTP",
),
],
HealthCheck=elb.HealthCheck(
Target=Join("", ["HTTP:", Ref(webport_param), "/"]),
HealthyThreshold="3",
UnhealthyThreshold="5",
Interval="30",
Timeout="5",
)
))
template.add_output(Output(
"URL",
Description="URL of the sample website",
Value=Join("", ["http://", GetAtt(elasticLB, "DNSName")])
))
print(template.to_json())
ref_stack_id = Ref('AWS::StackId')
ref_region = Ref('AWS::Region')
ref_stack_name = Ref('AWS::StackName')
ref_ambariserver = GetAtt('AmbariNode', 'PrivateDnsName')
ref_java_provider = Ref('JavaProvider')
ref_java_version = Ref('JavaVersion')
ref_additional_instance_count = Ref('AdditionalInstanceCount')
ref_ambari_pass = Ref('AmbariPass')
ref_ambari_services = Ref('AmbariServices')
ref_deploy_cluster = Ref('DeployCluster')
ref_wait_ambari = Ref('waitHandleAmbari')
# now the work begins
t = Template()
t.add_version("2010-09-09")
t.add_description("""\
CloudFormation template to Deploy Hortonworks Data Platform on VPC with a public subnet"""
)
## Parameters
BootDiskSize = t.add_parameter(
Parameter(
"BootDiskSize",
Type="Number",
Default="80",
MinValue=10,
MaxValue=2000,
Description="Size of boot disk.",
from troposphere import Template, Parameter, Ref, Tags, Output, GetAtt, ec2
import template_utils as utils
import troposphere.autoscaling as asg
import troposphere.cloudwatch as cw
import troposphere.elasticloadbalancing as elb
t = Template()
t.add_version('2010-09-09')
t.add_description('An application server stack for the nyc-trees project.')
#
# Parameters
#
color_param = t.add_parameter(Parameter(
'StackColor', Type='String',
Description='Stack color', AllowedValues=['Blue', 'Green'],
ConstraintDescription='must be either Blue or Green'
))
vpc_param = t.add_parameter(Parameter(
'VpcId', Type='String', Description='Name of an existing VPC'
))
keyname_param = t.add_parameter(Parameter(
'KeyName', Type='String', Default='nyc-trees-stg',
Description='Name of an existing EC2 key pair'
))
notification_arn_param = t.add_parameter(Parameter(
def main():
"""Generates the CloudFormation template"""
template = Template()
template.add_version("2010-09-09")
template.add_description(
'This template contains the security groups required by our ' +
'entire stack. We create them in a seperate nested template, ' +
'so they can be referenced by all of the other nested templates')
# Parameters
# EnvironmentName
template.add_parameter(
Parameter(
'EnvironmentName',
Type='String',
Description=
'An environment name that will be prefixed to resource names',
))
# VPC
vpc_param = template.add_parameter(
Parameter(
'VPC',
Type='AWS::EC2::VPC::Id',
Description=
'Choose which VPC this ECS cluster should be deployed to',
))
# Resources
# LoadBalancerSecurityGroup
elb_security_group = template.add_resource(
SecurityGroup('LoadBalancerSecurityGroup',
VpcId=Ref(vpc_param),
GroupDescription=
'Access to the load balancer that sits in front of ECS',
SecurityGroupIngress=[
SecurityGroupRule(
CidrIp='0.0.0.0/0',
IpProtocol='-1',
)
],
Tags=[{
'Key': 'Name',
'Value': Sub('${EnvironmentName}-LoadBalancers')
}]))
# ECSHostSecurityGroup
ecs_security_group = template.add_resource(
SecurityGroup(
'ECSHostSecurityGroup',
VpcId=Ref(vpc_param),
GroupDescription=
'Access to the ECS hosts and the tasks/containers that run on them',
SecurityGroupIngress=[
SecurityGroupRule(
SourceSecurityGroupId=Ref(elb_security_group),
IpProtocol='-1',
)
],
Tags=[{
'Key': 'Name',
'Value': Sub('${EnvironmentName}-ECS-Hosts')
}]))
# Output
template.add_output(
Output(
'ECSHostSecurityGroup',
Description='A reference to the security group for ECS hosts',
Value=Ref(ecs_security_group),
))
template.add_output(
Output(
'LoadBalancerSecurityGroup',
Description='A reference to the security group for load balancers',
Value=Ref(elb_security_group),
))
print(template.to_json())
from troposphere import Base64, Select, FindInMap, GetAtt, GetAZs, Join, Output
from troposphere import Parameter, Ref, Tags, Template
from troposphere.cloudformation import Init
from troposphere.cloudfront import Distribution, DistributionConfig
from troposphere.cloudfront import Origin, DefaultCacheBehavior
from troposphere.ec2 import PortRange
from troposphere.iam import InstanceProfile
from troposphere.iam import Role
from troposphere.iam import Group
t = Template()
t.add_version("2010-09-09")
t.add_description("""\
IAM""")
craftInstanceProfile = t.add_resource(InstanceProfile(
"craftInstanceProfile",
Path="/",
Roles=[Ref("craftIamRole")],
))
BastionInstanceProfile = t.add_resource(InstanceProfile(
"BastionInstanceProfile",
Path="/",
Roles=[Ref("BastionIamRole")],
))
BastionIamRole = t.add_resource(Role(
"BastionIamRole",
def generate_template(d):
# Set template metadata
t = Template()
t.add_version("2010-09-09")
t.set_description(d["cf_template_description"])
# ref_stack_id = Ref('AWS::StackId')
# ref_region = Ref('AWS::Region')
# ref_stack_name = Ref('AWS::StackName')
# Create VPC
vpc = t.add_resource(
VPC(
"VPC",
EnableDnsSupport=True,
EnableDnsHostnames=True,
CidrBlock=d["vpc_cidr_block"],
Tags=Tags(d["tags"], {"Name": d["project_name"]}),
)
)
# Create Public Subnets
public_subnet_count = 1
public_subnets = []
az_count = 0
for public_subnet_cidr_block in d["public_subnet_cidr_blocks"]:
public_subnets.append(
t.add_resource(
Subnet(
"PublicSubnet" + str(public_subnet_count),
CidrBlock=public_subnet_cidr_block,
VpcId=Ref(vpc),
AvailabilityZone=d["availability_zones"][az_count],
Tags=Tags(
d["tags"],
{
"Name": d["project_name"]
+ "-public-subnet"
+ str(public_subnet_count)
},
),
)
)
)
public_subnet_count += 1
az_count += 1
# Create Private Subnets
private_subnet_count = 1
private_subnets = []
az_count = 0
for private_subnet_cidr_block in d["private_subnet_cidr_blocks"]:
private_subnets.append(
t.add_resource(
Subnet(
"PrivateSubnet" + str(private_subnet_count),
CidrBlock=private_subnet_cidr_block,
VpcId=Ref(vpc),
AvailabilityZone=d["availability_zones"][az_count],
Tags=Tags(
d["tags"],
{
"Name": d["project_name"]
+ "-private-subnet"
+ str(private_subnet_count)
},
),
)
)
)
private_subnet_count += 1
az_count += 1
# Create Internet Gateway
internetGateway = t.add_resource(
InternetGateway(
"InternetGateway",
Tags=Tags(
d["tags"],
{
"Name": d["project_name"]
+ "-igw"
}
),
)
)
# Attach Internet Gateway to VPC
t.add_resource(
VPCGatewayAttachment(
"AttachmentInternetGateway",
VpcId=Ref(vpc),
InternetGatewayId=Ref(internetGateway),
)
)
# Create Two EIP for each Nat Gateway
eip_count = 1
for eip in range(2):
t.add_resource(
EIP(
"Eip" + str(eip_count),
Domain="vpc",
Tags=Tags(
d["tags"],
{
"Name": d["project_name"]
+ "-eip"
+ str(eip_count)
}
),
)
)
eip_count += 1
# Create Public Route Table
publicRouteTable = t.add_resource(
RouteTable(
"PublicRouteTable",
VpcId=Ref(vpc),
Tags=Tags(
d["tags"],
{
"Name": d["project_name"]
+ "-public-rtb"
}
),
)
)
# Create Route to Internet Gateway on Public Route Table
t.add_resource(
Route(
"RouteIgw",
DependsOn="AttachmentInternetGateway",
GatewayId=Ref("InternetGateway"),
DestinationCidrBlock="0.0.0.0/0",
RouteTableId=Ref(publicRouteTable),
)
)
# Create Private Route Table1
PrivateRouteTable1 = t.add_resource(
RouteTable(
"PrivateRouteTable1",
VpcId=Ref(vpc),
Tags=Tags(
d["tags"],
{
"Name": d["project_name"]
+ "-private-rtb1"
}
),
)
)
# Create Private Route Table2
PrivateRouteTable2 = t.add_resource(
RouteTable(
"PrivateRouteTable2",
VpcId=Ref(vpc),
Tags=Tags(
d["tags"],
{
"Name": d["project_name"]
+ "-private-rtb2"
}
),
)
)
# Associate Public Subnets with Public Route Table
public_subnet_count = 1
for public_subnet in range(len(public_subnets)):
t.add_resource(
SubnetRouteTableAssociation(
"PublicSubnetRouteTableAssociation" + str(public_subnet_count),
SubnetId=Ref("PublicSubnet" + str(public_subnet_count)),
RouteTableId=Ref(publicRouteTable),
)
)
public_subnet_count += 1
# Associate Private Subnet1 with Private Route Table1
t.add_resource(
SubnetRouteTableAssociation(
"PrivateSubnetPrivateRouteTableAssociation1",
SubnetId=Ref("PrivateSubnet1"),
RouteTableId=Ref(PrivateRouteTable1),
)
)
# Associate Private Subnet2 with Private Route Table2
t.add_resource(
SubnetRouteTableAssociation(
"PrivateSubnetPrivateRouteTableAssociation2",
SubnetId=Ref("PrivateSubnet2"),
RouteTableId=Ref(PrivateRouteTable2),
)
)
# Create Two Nat Gateways one for each Private Route Table
nat_gtw_count = 1
public_subnet_count = 1
for nat_gtw in range(2):
t.add_resource(
NatGateway(
"Nat" + str(nat_gtw_count),
AllocationId=GetAtt("Eip" + str(nat_gtw_count), "AllocationId"),
SubnetId=Ref("PublicSubnet" + str(public_subnet_count)),
Tags=Tags(
d["tags"],
{
"Name": d["project_name"]
+ "-nat"
+ str(public_subnet_count)
},
),
)
)
nat_gtw_count += 1
public_subnet_count += 1
# Create route in rtb1 to nat1
t.add_resource(
Route(
"RouteNat1",
DependsOn="Nat1",
NatGatewayId=Ref("Nat1"),
DestinationCidrBlock="0.0.0.0/0",
RouteTableId=Ref("PrivateRouteTable1"),
)
)
# Create route in rtb2 to nat2
t.add_resource(
Route(
"RouteNat2",
DependsOn="Nat2",
NatGatewayId=Ref("Nat2"),
DestinationCidrBlock="0.0.0.0/0",
RouteTableId=Ref("PrivateRouteTable2"),
)
)
# Outputs
t.add_output(
Output(
"VPCId",
Description="VPCId of the newly created VPC",
Export=Export(Sub("${AWS::StackName}-VPCId")),
Value=Ref(vpc),
)
)
public_subnet_count = 1
count = 0
for subnet in public_subnets:
t.add_output(
Output(
"PublicSubnet" + str(public_subnet_count),
Description=f"PublicSubnetId{public_subnet_count}for cross reference.",
Export=Export(
Sub("${AWS::StackName}" + f"-PublicSubnetId{public_subnet_count}")
),
Value=Ref(public_subnets[count]),
)
)
count += 1
public_subnet_count += 1
private_subnet_count = 1
count = 0
for subnet in private_subnets:
t.add_output(
Output(
"PrivateSubnet" + str(private_subnet_count),
Description=f"PrivateSubnetId{private_subnet_count} for cross reference.",
Export=Export(
Sub("${AWS::StackName}" + f"-PrivateSubnetId{private_subnet_count}")
),
Value=Ref(private_subnets[count]),
)
)
count += 1
private_subnet_count += 1
return t
