def transform_reference(v):
"""
Transform ref and ref_att in dictionary to CloudFormation ref or get_att
"""
if v.startswith('ref('):
v = v[len('ref('):-1].strip()
v = functions.ref(v)
elif v.startswith('get_att('):
v = [s.strip() for s in v[len('get_att('):-1].split(',')]
v = functions.get_att(v[0], v[1])
return v
def transform_reference(v):
"""
Transform ref and ref_att in dictionary to CloudFormation ref or get_att
"""
if v.startswith('ref('):
v = v[len('ref('):-1].strip()
v = functions.ref(v)
elif v.startswith('get_att('):
v = [s.strip() for s in v[len('get_att('):-1].split(',')]
v = functions.get_att(v[0], v[1])
return v
def test_get_att(self):
ret = functions.get_att('ThingName', 'AttrName')
self.assertEqual(ret['Fn::GetAtt'], ['ThingName', 'AttrName'])
def main():
config, config_yaml = load_config()
derive_config(config)
unique_suffix = datetime.datetime.utcnow().strftime("%Y%m%dT%H%M")
# We discover the current git branch/commit
# so that the deployment script can use it
# to clone the same commit.
commit = os.popen("""git rev-parse HEAD""").read().rstrip()
assert commit
assert "'" not in config['SITE_NAME']
instance_tags = tags.load()
# Set the `Name` as it appears on the EC2 web UI.
instance_tags.append({'Key': 'Name',
'Value': "refinery-web-" + unique_suffix})
config['tags'] = instance_tags
config_uri = save_s3_config(config, unique_suffix)
sys.stderr.write("Configuration saved to {}\n".format(config_uri))
# The userdata script is executed via CloudInit.
# It's made by concatenating a block of parameter variables,
# with the bootstrap.sh script,
# and the aws.sh script.
user_data_script = functions.join(
"",
"#!/bin/sh\n",
"CONFIG_YAML=", base64.b64encode(config_yaml), "\n",
"CONFIG_JSON=", base64.b64encode(json.dumps(config)), "\n",
"AWS_DEFAULT_REGION=", functions.ref("AWS::Region"), "\n",
"RDS_ID=", functions.ref('RDSInstance'), "\n",
"RDS_ENDPOINT_ADDRESS=",
functions.get_att('RDSInstance', 'Endpoint.Address'),
"\n",
"RDS_ENDPOINT_PORT=",
functions.get_att('RDSInstance', 'Endpoint.Port'),
"\n",
"RDS_SUPERUSER_PASSWORD="******"\n",
"RDS_ROLE=", config['RDS_ROLE'], "\n",
"ADMIN=", config['ADMIN'], "\n",
"DEFAULT_FROM_EMAIL=", config['DEFAULT_FROM_EMAIL'], "\n",
"SERVER_EMAIL=", config['SERVER_EMAIL'], "\n",
"IAM_SMTP_USER="******"\n",
"S3_CONFIG_URI=", config['S3_CONFIG_URI'], "\n",
"SITE_URL=", config['SITE_URL'], "\n",
# May contain spaces, but can't contain "'"
"SITE_NAME='", config['SITE_NAME'], "'\n",
"GIT_BRANCH=", commit, "\n",
"\n",
open('bootstrap.sh').read(),
open('aws.sh').read())
cft = core.CloudFormationTemplate(description="refinery platform.")
rds_properties = {
"AllocatedStorage": "5",
"AvailabilityZone": config['AVAILABILITY_ZONE'],
"BackupRetentionPeriod": "0",
"DBInstanceClass": "db.t2.small", # todo:?
"DBInstanceIdentifier": config['RDS_NAME'],
"Engine": "postgres",
"EngineVersion": "9.3.10",
# "KmsKeyId" ?
"MasterUsername": "******",
"MasterUserPassword": "******",
"MultiAZ": False,
"Port": "5432",
"PubliclyAccessible": False,
"StorageType": "gp2",
"Tags": instance_tags, # todo: Should be different?
}
if 'RDS_SNAPSHOT' in config:
rds_properties['DBSnapshotIdentifier'] = config['RDS_SNAPSHOT']
cft.resources.rds_instance = core.Resource(
'RDSInstance', 'AWS::RDS::DBInstance',
core.Properties(rds_properties),
core.DeletionPolicy("Snapshot"),
)
volume_properties = {
'AvailabilityZone': config['AVAILABILITY_ZONE'],
'Encrypted': True,
'Size': config['DATA_VOLUME_SIZE'],
'Tags': tags.load(),
'VolumeType': config['DATA_VOLUME_TYPE'],
}
if 'DATA_SNAPSHOT' in config:
volume_properties['SnapshotId'] = config['DATA_SNAPSHOT']
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-ebs-volume.html
cft.resources.ebs = core.Resource(
'RefineryData', 'AWS::EC2::Volume',
core.Properties(volume_properties),
core.DeletionPolicy("Snapshot"),
)
cft.resources.ec2_instance = core.Resource(
'WebInstance', 'AWS::EC2::Instance',
core.Properties({
'AvailabilityZone': config['AVAILABILITY_ZONE'],
'ImageId': 'ami-d05e75b8',
'InstanceType': 'm3.medium',
'UserData': functions.base64(user_data_script),
'KeyName': config['KEY_NAME'],
'IamInstanceProfile': functions.ref('WebInstanceProfile'),
'Tags': instance_tags,
}),
core.DependsOn('RDSInstance'),
)
cft.resources.instance_profile = core.Resource(
'WebInstanceProfile', 'AWS::IAM::InstanceProfile',
core.Properties({
'Path': '/',
'Roles': [
functions.ref('WebInstanceRole')
]
})
)
cft.resources.web_role = core.Resource(
'WebInstanceRole', 'AWS::IAM::Role',
core.Properties({
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html#cfn-iam-role-templateexamples
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": ["ec2.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}]
},
'ManagedPolicyArns': [
'arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess',
'arn:aws:iam::aws:policy/AmazonRDSReadOnlyAccess',
'arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess'
],
'Path': '/',
'Policies': [
{
'PolicyName': "CreateAccessKey",
'PolicyDocument': {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"iam:CreateAccessKey"
],
"Resource": [
"*"
]
}
]
}
},
{
'PolicyName': "CreateTags",
'PolicyDocument': {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"ec2:CreateTags"
],
"Resource": "*"
}
]
}
}
]
})
)
cft.resources.smtp_user = core.Resource(
'RefinerySMTPUser', 'AWS::IAM::User',
core.Properties({
'Policies': [{
'PolicyName': "SESSendingAccess",
'PolicyDocument': {
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "ses:SendRawEmail",
"Resource": "*"
}]
}
}]
})
)
cft.resources.mount = core.Resource(
'RefineryVolume', 'AWS::EC2::VolumeAttachment',
core.Properties({
'Device': '/dev/xvdr',
'InstanceId': functions.ref('WebInstance'),
'VolumeId': functions.ref('RefineryData'),
})
)
print(str(cft))
def make_template(config, config_yaml):
"""Make a fresh CloudFormation template object and return it"""
stack_name = config['STACK_NAME']
# We discover the current git branch/commit so that the deployment script
# can use it to clone the same commit
commit = os.popen("""git rev-parse HEAD""").read().rstrip()
assert commit
assert "'" not in config['SITE_NAME']
instance_tags = load_tags()
# Stack Name is also used for instances.
instance_tags.append({'Key': 'Name', 'Value': stack_name})
# This tag is variable and can be specified by
# template Parameter.
instance_tags.append({
'Key': functions.ref('SnapshotSchedulerTag'),
'Value': 'default'
})
config['tags'] = instance_tags
config_uri = save_s3_config(config)
sys.stdout.write("Configuration saved to {}\n".format(config_uri))
tls_rewrite = "false"
if 'TLS_CERTIFICATE' in config:
tls_rewrite = "true"
# The userdata script is executed via CloudInit
# It's made by concatenating a block of parameter variables,
# with the bootstrap.sh script, and the aws.sh script
user_data_script = functions.join(
"",
"#!/bin/sh\n",
"CONFIG_YAML=",
base64.b64encode(config_yaml),
"\n",
"CONFIG_JSON=",
base64.b64encode(json.dumps(config)),
"\n",
"AWS_DEFAULT_REGION=",
functions.ref("AWS::Region"),
"\n",
"RDS_ENDPOINT_ADDRESS=",
functions.get_att('RDSInstance', 'Endpoint.Address'),
"\n",
"RDS_ENDPOINT_PORT=",
functions.get_att('RDSInstance', 'Endpoint.Port'),
"\n",
"RDS_SUPERUSER_PASSWORD="******"\n",
"RDS_ROLE=",
config['RDS_ROLE'],
"\n",
"ADMIN=",
config['ADMIN'],
"\n",
"DEFAULT_FROM_EMAIL=",
config['DEFAULT_FROM_EMAIL'],
"\n",
"SERVER_EMAIL=",
config['SERVER_EMAIL'],
"\n",
"IAM_SMTP_USER="******"\n",
"export FACTER_TLS_REWRITE=",
tls_rewrite,
"\n",
"S3_CONFIG_URI=",
config['S3_CONFIG_URI'],
"\n",
"SITE_URL=",
config['SITE_URL'],
"\n",
# May contain spaces, but can't contain "'"
"SITE_NAME='",
config['SITE_NAME'],
"'\n",
"GIT_BRANCH=",
commit,
"\n",
"\n",
open('bootstrap.sh').read(),
open('aws.sh').read())
cft = core.CloudFormationTemplate(description="Refinery Platform main")
# This parameter tags the EC2 instances, and is intended to be used
# with the AWS Reference Implementation EBS Snapshot Scheduler:
# http://docs.aws.amazon.com/solutions/latest/ebs-snapshot-scheduler/welcome.html
cft.parameters.add(
core.Parameter(
'SnapshotSchedulerTag', 'String', {
'Default':
'scheduler:ebs-snapshot',
'Description':
"Tag added to EC2 Instances so that "
"the EBS Snapshot Scheduler will recognise them.",
}))
cft.parameters.add(
core.Parameter(
'IdentityPoolName', 'String', {
'Default': 'Refinery Platform',
'Description': 'Name of Cognito identity pool for S3 uploads',
}))
cft.parameters.add(
core.Parameter(
'DeveloperProviderName', 'String', {
'Default':
'login.refinery',
'Description':
'"domain" by which Cognito will refer to users',
'AllowedPattern':
'[a-z\-\.]+',
'ConstraintDescription':
'must only contain lower case letters, periods, '
'underscores, and hyphens'
}))
cft.parameters.add(
core.Parameter(
'StorageStackName', 'String', {
'Default':
'${AWS::StackName}Storage',
'Description':
'Name of the S3 storage stack for Django '
'static and media files',
}))
rds_properties = {
"AllocatedStorage": "5",
"AutoMinorVersionUpgrade": False,
"BackupRetentionPeriod": "15",
"CopyTagsToSnapshot": True,
"DBInstanceClass": "db.t2.small", # todo:?
"DBInstanceIdentifier": config['RDS_NAME'],
"Engine": "postgres",
"EngineVersion": "9.3.14",
# "KmsKeyId" ?
"MasterUsername": "******",
"MasterUserPassword": config['RDS_SUPERUSER_PASSWORD'],
"MultiAZ": False,
"Port": "5432",
"PubliclyAccessible": False,
"StorageType": "gp2",
"Tags": instance_tags, # todo: Should be different?
"VPCSecurityGroups":
[functions.get_att('RDSSecurityGroup', 'GroupId')],
}
if 'RDS_SNAPSHOT' in config:
rds_properties['DBSnapshotIdentifier'] = config['RDS_SNAPSHOT']
cft.resources.rds_instance = core.Resource(
'RDSInstance',
'AWS::RDS::DBInstance',
core.Properties(rds_properties),
core.DeletionPolicy("Snapshot"),
)
volume_properties = {
'Encrypted': True,
'Size': config['DATA_VOLUME_SIZE'],
'Tags': load_tags(),
'AvailabilityZone': functions.get_att('WebInstance',
'AvailabilityZone'),
'VolumeType': config['DATA_VOLUME_TYPE'],
}
if 'DATA_SNAPSHOT' in config:
volume_properties['SnapshotId'] = config['DATA_SNAPSHOT']
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-ebs-volume.html
cft.resources.ebs = core.Resource(
'RefineryData',
'AWS::EC2::Volume',
core.Properties(volume_properties),
core.DeletionPolicy("Snapshot"),
)
cft.resources.ec2_instance = core.Resource(
'WebInstance',
'AWS::EC2::Instance',
core.Properties({
'ImageId':
'ami-d05e75b8',
'InstanceType':
'm3.medium',
'UserData':
functions.base64(user_data_script),
'KeyName':
config['KEY_NAME'],
'IamInstanceProfile':
functions.ref('WebInstanceProfile'),
'SecurityGroups': [functions.ref("InstanceSecurityGroup")],
'Tags':
instance_tags,
}),
core.DependsOn(['RDSInstance']),
)
cft.resources.instance_profile = core.Resource(
'WebInstanceProfile', 'AWS::IAM::InstanceProfile',
core.Properties({
'Path': '/',
'Roles': [functions.ref('WebInstanceRole')]
}))
cft.resources.web_role = core.Resource(
'WebInstanceRole',
'AWS::IAM::Role',
core.Properties({
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html#cfn-iam-role-templateexamples
"AssumeRolePolicyDocument": {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": ["ec2.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}]
},
'ManagedPolicyArns': [
'arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess',
'arn:aws:iam::aws:policy/AmazonRDSReadOnlyAccess',
'arn:aws:iam::aws:policy/AmazonS3FullAccess'
],
'Path':
'/',
'Policies': [{
'PolicyName': "CreateAccessKey",
'PolicyDocument': {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["iam:CreateAccessKey"],
"Resource": ["*"]
}]
},
}, {
'PolicyName': "CreateSnapshot",
'PolicyDocument': {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["ec2:CreateSnapshot"],
"Resource": ["*"]
}]
}
}, {
'PolicyName': "CreateDBSnapshot",
'PolicyDocument': {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["rds:CreateDBSnapshot"],
"Resource": ["*"]
}]
}
}, {
'PolicyName': "CreateTags",
'PolicyDocument': {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["ec2:CreateTags"],
"Resource": "*"
}]
}
}, {
"PolicyName": "CognitoAccess",
"PolicyDocument": {
"Version":
"2012-10-17",
"Statement": [{
"Effect":
"Allow",
"Action": [
"cognito-identity:ListIdentityPools",
],
"Resource":
"arn:aws:cognito-identity:*"
}, {
"Effect":
"Allow",
"Action": [
"cognito-identity:"
"GetOpenIdTokenForDeveloperIdentity"
],
"Resource": {
"Fn::Sub": [
"arn:aws:cognito-identity:"
"${AWS::Region}:${AWS::AccountId}:"
"identitypool/${Pool}", {
"Pool": functions.ref('IdentityPool')
}
]
}
}]
}
}]
}))
cft.resources.smtp_user = core.Resource(
'RefinerySMTPUser', 'AWS::IAM::User',
core.Properties({
'Policies': [{
'PolicyName': "SESSendingAccess",
'PolicyDocument': {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "ses:SendRawEmail",
"Resource": "*"
}]
}
}]
}))
cft.resources.mount = core.Resource(
'RefineryVolume', 'AWS::EC2::VolumeAttachment',
core.Properties({
'Device': '/dev/xvdr',
'InstanceId': functions.ref('WebInstance'),
'VolumeId': functions.ref('RefineryData'),
}))
# Security Group for Elastic Load Balancer
# (public facing).
cft.resources.elbsg = core.Resource(
'ELBSecurityGroup',
'AWS::EC2::SecurityGroup',
core.Properties({
'GroupDescription':
"Refinery ELB",
# Egress Rule defined via
# AWS::EC2::SecurityGroupEgress resource,
# to avoid circularity (below).
# See http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-security-group.html # noqa: E501
'SecurityGroupIngress': [
{
"IpProtocol": "tcp",
"FromPort": "80",
"ToPort": "80",
"CidrIp": "0.0.0.0/0",
},
{
"IpProtocol": "tcp",
"FromPort": "443",
"ToPort": "443",
"CidrIp": "0.0.0.0/0",
},
],
}))
cft.resources.elbegress = core.Resource(
'ELBEgress', 'AWS::EC2::SecurityGroupEgress',
core.Properties({
"GroupId":
functions.get_att('ELBSecurityGroup', 'GroupId'),
"IpProtocol":
"tcp",
"FromPort":
"80",
"ToPort":
"80",
"DestinationSecurityGroupId":
functions.get_att('InstanceSecurityGroup', 'GroupId'),
}))
# Security Group for EC2- instance.
cft.resources.instancesg = core.Resource(
'InstanceSecurityGroup',
'AWS::EC2::SecurityGroup',
core.Properties({
'GroupDescription':
"Refinery EC2 Instance",
'SecurityGroupEgress': [],
'SecurityGroupIngress': [
{
"IpProtocol": "tcp",
"FromPort": "22",
"ToPort": "22",
"CidrIp": "0.0.0.0/0",
},
{
"IpProtocol":
"tcp",
"FromPort":
"80",
"ToPort":
"80",
# "CidrIp": "0.0.0.0/0",
# Only accept connections from the ELB.
"SourceSecurityGroupId":
functions.get_att('ELBSecurityGroup', 'GroupId'),
},
],
}))
# Security Group for RDS instance.
cft.resources.rdssg = core.Resource(
'RDSSecurityGroup',
'AWS::EC2::SecurityGroup',
core.Properties({
'GroupDescription':
"Refinery RDS",
'SecurityGroupEgress': [
# We would like to remove all egress rules here,
# but you can't do that with this version
# of CloudFormation.
# We decided that the hacky workarounds are
# not worth it.
],
'SecurityGroupIngress': [
{
"IpProtocol":
"tcp",
"FromPort":
"5432",
"ToPort":
"5432",
# Only accept connections from the
# Instance Security Group.
"SourceSecurityGroupId":
functions.get_att('InstanceSecurityGroup', 'GroupId'),
},
],
}))
# ELB per
# http://cfn-pyplates.readthedocs.io/en/latest/examples/options/template.html
# Insecure, Port 80, HTTP listener
http_listener = {
'LoadBalancerPort': '80',
'Protocol': 'HTTP',
'InstanceProtocol': 'HTTP',
'InstancePort': '80',
'PolicyNames': []
}
listeners = [http_listener]
if 'TLS_CERTIFICATE' in config:
# Secure, Port 443, HTTPS listener
https_listener = {
'LoadBalancerPort': '443',
'Protocol': 'HTTPS',
'InstanceProtocol': 'HTTP',
'InstancePort': '80',
'PolicyNames': [],
'SSLCertificateId': config['TLS_CERTIFICATE']
}
listeners.append(https_listener)
cft.resources.elb = core.Resource(
'LoadBalancer',
'AWS::ElasticLoadBalancing::LoadBalancer',
{
'AccessLoggingPolicy': {
'EmitInterval': functions.ref('LogInterval'),
'Enabled': True,
'S3BucketName': config['S3_LOG_BUCKET'],
# 'S3BucketPrefix' unused
},
'AvailabilityZones':
[functions.get_att('WebInstance', 'AvailabilityZone')],
'ConnectionSettings': {
'IdleTimeout': 1800 # seconds
},
'HealthCheck': {
'HealthyThreshold': '2',
'Interval': '30',
'Target': 'HTTP:80/',
'Timeout': '5',
'UnhealthyThreshold': '4'
},
'Instances': [functions.ref('WebInstance')],
'LoadBalancerName':
config['STACK_NAME'],
'Listeners':
listeners,
'SecurityGroups':
[functions.get_att('ELBSecurityGroup', 'GroupId')],
'Tags':
load_tags(),
})
cft.parameters.add(
core.Parameter(
'LogInterval', 'Number', {
'Default':
60,
'Description':
"How often, in minutes, the ELB emits its logs to the "
"configured S3 bucket. The ELB log facility restricts "
"this to be 5 or 60.",
}))
# Cognito Identity Pool for Developer Authenticated Identities Authflow
# http://docs.aws.amazon.com/cognito/latest/developerguide/authentication-flow.html
cft.resources.add(
core.Resource(
'IdentityPool', 'AWS::Cognito::IdentityPool',
core.Properties({
'IdentityPoolName':
functions.ref('IdentityPoolName'),
'AllowUnauthenticatedIdentities':
False,
'DeveloperProviderName':
functions.ref('DeveloperProviderName'),
})))
cft.resources.add(
core.Resource(
'IdentityPoolAuthenticatedRole',
'AWS::Cognito::IdentityPoolRoleAttachment',
core.Properties({
'IdentityPoolId': functions.ref('IdentityPool'),
'Roles': {
'authenticated':
functions.get_att('CognitoS3UploadRole', 'Arn'),
}
})))
upload_role_trust_policy = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Federated": "cognito-identity.amazonaws.com"
},
"Action": "sts:AssumeRoleWithWebIdentity",
"Condition": {
"StringEquals": {
"cognito-identity.amazonaws.com:aud":
functions.ref('IdentityPool')
},
"ForAnyValue:StringLike": {
"cognito-identity.amazonaws.com:amr": "authenticated"
}
}
}]
}
upload_access_policy = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["cognito-identity:*"],
"Resource": "*"
}, {
"Action": ["s3:PutObject", "s3:AbortMultipartUpload"],
"Effect": "Allow",
"Resource": {
"Fn::Sub": [
"arn:aws:s3:::${MediaBucket}/uploads/"
"${!cognito-identity.amazonaws.com:sub}/*", {
"MediaBucket": {
"Fn::ImportValue": {
"Fn::Sub": "${StorageStackName}Media"
}
}
}
]
}
}]
}
cft.resources.add(
core.Resource(
'CognitoS3UploadRole', 'AWS::IAM::Role',
core.Properties({
'AssumeRolePolicyDocument':
upload_role_trust_policy,
'Policies': [{
'PolicyName': 'AuthenticatedS3UploadPolicy',
'PolicyDocument': upload_access_policy,
}]
})))
# See http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-access-logs.html#attach-bucket-policy # noqa: E501
# for full list of region--principal identifiers.
cft.mappings.region = core.Mapping(
'Region', {'us-east-1': {
'ELBPrincipal': '127311923021'
}})
cft.resources.log_policy = core.Resource(
'LogBucketPolicy', 'AWS::S3::BucketPolicy',
core.Properties({
'Bucket': config['S3_LOG_BUCKET'],
'PolicyDocument': {
'Statement': [{
"Action": ["s3:PutObject"],
"Effect":
"Allow",
"Resource":
functions.join("", "arn:aws:s3:::",
config['S3_LOG_BUCKET'], "/AWSLogs/",
functions.ref("AWS::AccountId"), "/*"),
"Principal": {
"AWS": [
functions.find_in_map('Region',
functions.ref("AWS::Region"),
'ELBPrincipal'),
]
}
}]
}
}))
return cft
def test_get_att(self):
ret = functions.get_att('ThingName', 'AttrName')
self.assertEqual(ret['Fn::GetAtt'], ['ThingName', 'AttrName'])
def make_storage_template():
cft = CloudFormationTemplate(description="Refinery Platform storage")
# Parameters
cft.parameters.add(
Parameter('StaticBucketName', 'String', {
'Description': 'Name of S3 bucket for Django static files',
}))
cft.parameters.add(
Parameter(
'MediaBucketName',
'String',
{
'Description':
'Name of S3 bucket for Django media files',
# make names DNS-compliant without periods (".") for
# compatibility with virtual-hosted-style access and S3
# Transfer Acceleration
'AllowedPattern':
'[a-z0-9\-]+',
'ConstraintDescription':
'must only contain lower case letters, numbers, and '
'hyphens',
}))
cft.parameters.add(
Parameter(
'IdentityPoolName', 'String', {
'Default': 'Refinery Platform',
'Description': 'Name of Cognito identity pool for S3 uploads',
}))
cft.parameters.add(
Parameter(
'DeveloperProviderName', 'String', {
'Default':
'login.refinery',
'Description':
'"domain" by which Cognito will refer to users',
'AllowedPattern':
'[a-z\-\.]+',
'ConstraintDescription':
'must only contain lower case letters, periods, '
'underscores, and hyphens'
}))
# Resources
cft.resources.add(
Resource(
'StaticStorageBucket',
'AWS::S3::Bucket',
Properties({
'BucketName': ref('StaticBucketName'),
'AccessControl': 'PublicRead',
'CorsConfiguration': {
'CorsRules': [{
'AllowedOrigins': ['*'],
'AllowedMethods': ['GET'],
'AllowedHeaders': ['Authorization'],
'MaxAge': 3000,
}]
},
}),
DeletionPolicy('Retain'),
))
cft.resources.add(
Resource(
'MediaStorageBucket',
'AWS::S3::Bucket',
Properties({
'BucketName': ref('MediaBucketName'),
'AccessControl': 'PublicRead',
'CorsConfiguration': {
'CorsRules': [{
'AllowedOrigins': ['*'],
'AllowedMethods': ['POST', 'PUT', 'DELETE'],
'AllowedHeaders': ['*'],
'ExposedHeaders': ['ETag'],
'MaxAge': 3000,
}]
}
}),
DeletionPolicy('Retain'),
))
# Cognito Identity Pool for Developer Authenticated Identities Authflow
# http://docs.aws.amazon.com/cognito/latest/developerguide/authentication-flow.html
cft.resources.add(
Resource(
'IdentityPool', 'AWS::Cognito::IdentityPool',
Properties({
'IdentityPoolName': ref('IdentityPoolName'),
'AllowUnauthenticatedIdentities': False,
'DeveloperProviderName': ref('DeveloperProviderName'),
})))
cft.resources.add(
Resource(
'IdentityPoolAuthenticatedRole',
'AWS::Cognito::IdentityPoolRoleAttachment',
Properties({
'IdentityPoolId': ref('IdentityPool'),
'Roles': {
'authenticated': get_att('CognitoS3UploadRole', 'Arn'),
}
})))
upload_role_trust_policy = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Federated": "cognito-identity.amazonaws.com"
},
"Action": "sts:AssumeRoleWithWebIdentity",
"Condition": {
"StringEquals": {
"cognito-identity.amazonaws.com:aud": ref('IdentityPool')
},
"ForAnyValue:StringLike": {
"cognito-identity.amazonaws.com:amr": "authenticated"
}
}
}]
}
upload_access_policy = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["cognito-identity:*"],
"Resource": "*"
}, {
"Action": ["s3:PutObject", "s3:AbortMultipartUpload"],
"Effect": "Allow",
"Resource": {
"Fn::Sub":
"arn:aws:s3:::${MediaStorageBucket}/uploads/"
"${!cognito-identity.amazonaws.com:sub}/*"
}
}]
}
cft.resources.add(
Resource(
'CognitoS3UploadRole', 'AWS::IAM::Role',
Properties({
'AssumeRolePolicyDocument':
upload_role_trust_policy,
'Policies': [{
'PolicyName': 'AuthenticatedS3UploadPolicy',
'PolicyDocument': upload_access_policy,
}]
})))
# Outputs
cft.outputs.add(
Output('IdentityPoolId', ref('IdentityPool'),
{'Fn::Sub': '${AWS::StackName}IdentityPoolId'},
'Cognito identity pool ID'))
return cft
def test_get_att(self):
ret = functions.get_att("ThingName", "AttrName")
self.assertEqual(ret["Fn::GetAtt"], ["ThingName", "AttrName"])
def main():
config, config_yaml = load_config()
derive_config(config)
unique_suffix = datetime.datetime.utcnow().strftime("%Y%m%dT%H%M")
# We discover the current git branch/commit
# so that the deployment script can use it
# to clone the same commit.
commit = os.popen("""git rev-parse HEAD""").read().rstrip()
assert commit
assert "'" not in config['SITE_NAME']
instance_tags = tags.load()
# Set the `Name` as it appears on the EC2 web UI.
instance_tags.append({'Key': 'Name',
'Value': "refinery-web-" + unique_suffix})
config['tags'] = instance_tags
config_uri = save_s3_config(config, unique_suffix)
sys.stderr.write("Configuration saved to {}\n".format(config_uri))
tls_rewrite = "false"
if 'TLS_CERTIFICATE' in config:
tls_rewrite = "true"
# The userdata script is executed via CloudInit.
# It's made by concatenating a block of parameter variables,
# with the bootstrap.sh script,
# and the aws.sh script.
user_data_script = functions.join(
"",
"#!/bin/sh\n",
"CONFIG_YAML=", base64.b64encode(config_yaml), "\n",
"CONFIG_JSON=", base64.b64encode(json.dumps(config)), "\n",
"AWS_DEFAULT_REGION=", functions.ref("AWS::Region"), "\n",
"RDS_ID=", functions.ref('RDSInstance'), "\n",
"RDS_ENDPOINT_ADDRESS=",
functions.get_att('RDSInstance', 'Endpoint.Address'),
"\n",
"RDS_ENDPOINT_PORT=",
functions.get_att('RDSInstance', 'Endpoint.Port'),
"\n",
"RDS_SUPERUSER_PASSWORD="******"\n",
"RDS_ROLE=", config['RDS_ROLE'], "\n",
"ADMIN=", config['ADMIN'], "\n",
"DEFAULT_FROM_EMAIL=", config['DEFAULT_FROM_EMAIL'], "\n",
"SERVER_EMAIL=", config['SERVER_EMAIL'], "\n",
"IAM_SMTP_USER="******"\n",
"export FACTER_TLS_REWRITE=", tls_rewrite, "\n",
"S3_CONFIG_URI=", config['S3_CONFIG_URI'], "\n",
"SITE_URL=", config['SITE_URL'], "\n",
# May contain spaces, but can't contain "'"
"SITE_NAME='", config['SITE_NAME'], "'\n",
"GIT_BRANCH=", commit, "\n",
"\n",
open('bootstrap.sh').read(),
open('aws.sh').read())
cft = core.CloudFormationTemplate(description="refinery platform.")
rds_properties = {
"AllocatedStorage": "5",
"AutoMinorVersionUpgrade": False,
"AvailabilityZone": config['AVAILABILITY_ZONE'],
"BackupRetentionPeriod": "0",
"DBInstanceClass": "db.t2.small", # todo:?
"DBInstanceIdentifier": config['RDS_NAME'],
"Engine": "postgres",
"EngineVersion": "9.3.14",
# "KmsKeyId" ?
"MasterUsername": "******",
"MasterUserPassword": "******",
"MultiAZ": False,
"Port": "5432",
"PubliclyAccessible": False,
"StorageType": "gp2",
"Tags": instance_tags, # todo: Should be different?
}
if 'RDS_SNAPSHOT' in config:
rds_properties['DBSnapshotIdentifier'] = config['RDS_SNAPSHOT']
cft.resources.rds_instance = core.Resource(
'RDSInstance', 'AWS::RDS::DBInstance',
core.Properties(rds_properties),
core.DeletionPolicy("Snapshot"),
)
volume_properties = {
'AvailabilityZone': config['AVAILABILITY_ZONE'],
'Encrypted': True,
'Size': config['DATA_VOLUME_SIZE'],
'Tags': tags.load(),
'VolumeType': config['DATA_VOLUME_TYPE'],
}
if 'DATA_SNAPSHOT' in config:
volume_properties['SnapshotId'] = config['DATA_SNAPSHOT']
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-ebs-volume.html
cft.resources.ebs = core.Resource(
'RefineryData', 'AWS::EC2::Volume',
core.Properties(volume_properties),
core.DeletionPolicy("Snapshot"),
)
cft.resources.ec2_instance = core.Resource(
'WebInstance', 'AWS::EC2::Instance',
core.Properties({
'AvailabilityZone': config['AVAILABILITY_ZONE'],
'ImageId': 'ami-d05e75b8',
'InstanceType': 'm3.medium',
'UserData': functions.base64(user_data_script),
'KeyName': config['KEY_NAME'],
'IamInstanceProfile': functions.ref('WebInstanceProfile'),
'Tags': instance_tags,
}),
core.DependsOn('RDSInstance'),
)
cft.resources.instance_profile = core.Resource(
'WebInstanceProfile', 'AWS::IAM::InstanceProfile',
core.Properties({
'Path': '/',
'Roles': [
functions.ref('WebInstanceRole')
]
})
)
cft.resources.web_role = core.Resource(
'WebInstanceRole', 'AWS::IAM::Role',
core.Properties({
# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html#cfn-iam-role-templateexamples
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": ["ec2.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}]
},
'ManagedPolicyArns': [
'arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess',
'arn:aws:iam::aws:policy/AmazonRDSReadOnlyAccess',
'arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess'
],
'Path': '/',
'Policies': [
{
'PolicyName': "CreateAccessKey",
'PolicyDocument': {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"iam:CreateAccessKey"
],
"Resource": [
"*"
]
}
]
},
},
{
'PolicyName': "CreateSnapshot",
'PolicyDocument': {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"ec2:CreateSnapshot"
],
"Resource": [
"*"
]
}
]
}
},
{
'PolicyName': "CreateDBSnapshot",
'PolicyDocument': {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"rds:CreateDBSnapshot"
],
"Resource": [
"*"
]
}
]
}
},
{
'PolicyName': "CreateTags",
'PolicyDocument': {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"ec2:CreateTags"
],
"Resource": "*"
}
]
}
}
]
})
)
cft.resources.smtp_user = core.Resource(
'RefinerySMTPUser', 'AWS::IAM::User',
core.Properties({
'Policies': [{
'PolicyName': "SESSendingAccess",
'PolicyDocument': {
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "ses:SendRawEmail",
"Resource": "*"
}]
}
}]
})
)
cft.resources.mount = core.Resource(
'RefineryVolume', 'AWS::EC2::VolumeAttachment',
core.Properties({
'Device': '/dev/xvdr',
'InstanceId': functions.ref('WebInstance'),
'VolumeId': functions.ref('RefineryData'),
})
)
cft.resources.elbsg = core.Resource(
'ELBSecurityGroup', 'AWS::EC2::SecurityGroup',
core.Properties({
'GroupDescription': "Refinery ELB",
'SecurityGroupEgress': [],
'SecurityGroupIngress': [
{
"IpProtocol": "tcp",
"FromPort": "80",
"ToPort": "80",
"CidrIp": "0.0.0.0/0",
},
{
"IpProtocol": "tcp",
"FromPort": "443",
"ToPort": "443",
"CidrIp": "0.0.0.0/0",
},
],
})
)
# ELB per
# http://cfn-pyplates.readthedocs.io/en/latest/examples/options/template.html
# Insecure, Port 80, HTTP listener
http_listener = {
'LoadBalancerPort': '80',
'Protocol': 'HTTP',
'InstanceProtocol': 'HTTP',
'InstancePort': '80',
'PolicyNames': []
}
listeners = [http_listener]
if 'TLS_CERTIFICATE' in config:
# Secure, Port 443, HTTPS listener
https_listener = {
'LoadBalancerPort': '443',
'Protocol': 'HTTPS',
'InstanceProtocol': 'HTTP',
'InstancePort': '80',
'PolicyNames': [],
'SSLCertificateId': config['TLS_CERTIFICATE']
}
listeners.append(https_listener)
cft.resources.elb = core.Resource(
'LoadBalancer', 'AWS::ElasticLoadBalancing::LoadBalancer',
{
'AvailabilityZones': [config['AVAILABILITY_ZONE']],
'HealthCheck': {
'HealthyThreshold': '2',
'Interval': '30',
'Target': 'HTTP:80/',
'Timeout': '5',
'UnhealthyThreshold': '4'
},
'Instances': [functions.ref('WebInstance')],
'Listeners': listeners,
'SecurityGroups': [
functions.get_att('ELBSecurityGroup', 'GroupId')],
"Tags": instance_tags, # todo: Should be different?
})
sys.stdout.write(str(cft))
