PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "sqs:SendMessage",
"Resource": GetAtt(start_events, 'Arn')
}]
})
lambda_role = t.add_resource(
Role(
"SchedulerExecutionRole",
Path="/service-role/",
ManagedPolicyArns=[
"arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole",
"arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
],
Policies=[sns_policy, sqs_policy],
AssumeRolePolicyDocument=utils.get_static_policy('lambda-policy-doc'),
))
scheduler = utils.make_lambda_function(
name="scheduler",
role=lambda_role,
lambda_params={
"Environment":
Environment(
Variables={
'SNS_ARN': Ref(finish_sns),
'DB_HOST': utils.get_host_address(),
from template import t
from troposphere import GetAtt, Ref, Sub
from troposphere.awslambda import Permission
from troposphere.sns import Subscription, Topic
from troposphere.ssm import Parameter
from .hyp3_send_email import send_email
print(' adding sns')
finish_sns = t.add_resource(Topic(
"HyP3FinishEventSNSTopic",
Subscription=[
Subscription(
Protocol="lambda",
Endpoint=GetAtt(send_email, "Arn")
)
]
))
sns_invoke_permissions = t.add_resource(Permission(
"SNSSchedulerInvokePermissions",
Action="lambda:InvokeFunction",
Principal="sns.amazonaws.com",
SourceArn=Ref(finish_sns),
FunctionName=GetAtt(send_email, "Arn")
))
ssm_sns_arn = t.add_resource(Parameter(
"HyP3SSMParameterFinishEventSNSArn",
def get_security_group(name: str):
return ec2.SecurityGroupRule(name,
IpProtocol="tcp",
FromPort="5432",
ToPort="5432",
CidrIp="0.0.0.0/0")
inrule, outrule = [
get_security_group(n + 'PostgresTcpRule') for n in ('In', 'Out')
]
security_group = t.add_resource(
ec2.SecurityGroup(
"HyP3TCPAll",
GroupDescription="Allow for all tcp traffic through port 5432",
VpcId=Ref(hyp3_vpc),
SecurityGroupIngress=[inrule],
SecurityGroupEgress=[outrule]))
mydbsubnetgroup = t.add_resource(
rds.DBSubnetGroup(
"MyDBSubnetGroup",
DBSubnetGroupDescription="Subnets available for the RDS DB Instance",
SubnetIds=[Ref(subnet) for subnet in get_public_subnets()],
))
# Only certain versions of postgres are supported on the smaller instance types
# https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.DBInstanceClass.html
db = rds.DBInstance(
"HyP3DB",
Type="Number"))
t.add_mapping("Region2AMI", get_map('region2ami'))
security_group = t.add_resource(
SecurityGroup(
"HyP3ProcessingInstancesSecurityGroup",
GroupDescription="Allow ssh to processing instances",
VpcId=Ref(hyp3_vpc),
SecurityGroupIngress=[
SecurityGroupRule("HyP3ProcessingInstancesSecurityGroupSSHIn",
IpProtocol="tcp",
FromPort="22",
ToPort="22",
CidrIp="0.0.0.0/0")
],
SecurityGroupEgress=[
SecurityGroupRule("HyP3ProcessingInstancesSecurityGroupWebOut",
IpProtocol="tcp",
FromPort="80",
ToPort="80",
CidrIp="0.0.0.0/0"),
SecurityGroupRule("HyP3ProcessingInstancesSecurityGroupWebSOut",
IpProtocol="tcp",
FromPort="443",
ToPort="443",
CidrIp="0.0.0.0/0")
]))
products_bucket_access = IAMPolicy(
PolicyName="ProductsPutObject",
PolicyDocument=Policy(Statement=[
* Restricted Subnet
* **NetworkACL:** Control traffic for the restricted subnet
"""
from template import t
from troposphere import (GetAZs, Output, Ref, Select, Tags, ec2, Parameter,
GetAtt)
print(' adding vpc')
hyp3_vpc = t.add_resource(
ec2.VPC('HyP3VPC',
CidrBlock='10.0.0.0/16',
InstanceTenancy='default',
EnableDnsSupport=True,
EnableDnsHostnames=True,
Tags=Tags(Name="HyP3 VPC")))
igw = t.add_resource(ec2.InternetGateway('HyP3InternetGateway', ))
net_gw_vpc_attachment = t.add_resource(
ec2.VPCGatewayAttachment(
"HyP3GatewayAttachment",
VpcId=Ref(hyp3_vpc),
InternetGatewayId=Ref(igw),
))
public_route_table = t.add_resource(
ec2.RouteTable(
"""
Troposphere template responsible for the product bucket
Resources
~~~~~~~~~
* **S3 Bucket:** Where all the hyp3 products are stored
* **SSM Parameter:** Stores the name of the products bucket
"""
from troposphere import s3, Sub, Ref
from troposphere.ssm import Parameter as SSMParameter
from template import t
products_bucket = t.add_resource(
s3.Bucket("S3Bucket",
BucketName=Sub("${StackName}-products-bucket",
StackName=Ref("AWS::StackName"))))
ssm_products_bucket_name = t.add_resource(
SSMParameter("HyP3SSMParameterProductsBucket",
Name=Sub("/${StackName}/ProductsS3Bucket",
StackName=Ref("AWS::StackName")),
Type="String",
Value=Ref(products_bucket)))
"Resource": Join(":", [
"arn:aws:ssm",
Ref("AWS::Region"),
Ref("AWS::AccountId"),
"parameter/*"
])
}
]
}
)
role = t.add_resource(Role(
"SetupDbExecutionRole",
Path="/",
ManagedPolicyArns=[
"arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
],
Policies=[default_processes_s3_read, ssm_param_read_write],
AssumeRolePolicyDocument=utils.get_static_policy('lambda-policy-doc')
))
admin_email = t.add_parameter(Parameter(
"HyP3AdminEmail",
Description=(
"Email for the admin hyp3 user. "
"This is where emails will be sent to."
),
Type="String",
AllowedPattern=utils.get_email_pattern()
))
Statement(
Effect=Allow,
Action=[PutMetricData],
Resource=["*"]
)
]
)
)
role = t.add_resource(Role(
"CustomMetricExecutionRole",
Path="/",
ManagedPolicyArns=[
"arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
],
Policies=[
describe_autoscale,
get_queue_attributes,
put_metric_data
],
AssumeRolePolicyDocument=utils.get_static_policy('lambda-policy-doc'),
))
custom_metric = t.add_resource(utils.make_lambda_function(
name='custom_metric',
role=role,
lambda_params={
"KmsKeyArn": GetAtt(kms_key, "Arn"),
"Environment": Environment(
Variables={
"HyP3StackName": Ref("AWS::StackName")
"""
from troposphere import Parameter, Ref, Sub
from troposphere.ssm import Parameter as SSMParameter
from template import t
earthdata_username = t.add_parameter(
Parameter(
"EarthdataUsername",
Description="Username for Earthdata account HyP3 should use",
Type="String",
))
earthdata_password = t.add_parameter(
Parameter("EarthdataPassword",
NoEcho=True,
Description="Password for Earthdata account HyP3 should use",
Type="String"))
ssm_earthdata_creds = t.add_resource(
SSMParameter("HyP3SSMParameterEarthdataCreds",
Name=Sub("/${StackName}/EarthdataCredentials",
StackName=Ref("AWS::StackName")),
Type="String",
Value=Sub(
'{"username": "******", "password": "******"}',
Username=Ref(earthdata_username),
Password=Ref(earthdata_password))))
"""
from troposphere import GetAtt, Ref, awslambda, events, iam, Sub
from troposphere.ssm import Parameter as SSMParameter
from template import t
from . import hyp3_scheduler, utils
source_zip = "find_new_granules.zip"
print(' adding find_new lambda')
ssm_previous_time = t.add_resource(
SSMParameter("HyP3SSMParameterPerviousTime",
Name=Sub("/${StackName}/previous_time.json",
StackName=Ref("AWS::StackName")),
Type="String",
Value="_"))
logs_policy = iam.Policy(PolicyName="LogAccess",
PolicyDocument=utils.get_static_policy('logs-policy'))
ssm_arn = "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${ParamName}"
prev_time_s3_policy = iam.Policy(
PolicyName='PreviousTimeSSMReadWriteAccess',
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Effect":
"Allow",
print(' adding hyp3 dashboard')
def get_dashboard_json():
with (pl.Path(__file__).parent / 'dashboard.json').open('r') as f:
return minifed(f.read())
def minifed(json_str):
return json.dumps(json.loads(json_str))
hyp3_dashboard = t.add_resource(
Dashboard('HyP3Dashboard',
DashboardName=Sub('${StackName}-hyp3-monitoring',
StackName=Ref('AWS::StackName')),
DashboardBody=Sub(get_dashboard_json(),
FindNewGranulesName=Ref(find_new_granules),
SchedulerName=Ref(scheduler),
SendEmailName=Ref(send_email),
SetupDBName=Ref(setup_db),
HyP3DBInsatnceIdentifier=Ref(db),
StartEventQueue=GetAtt(start_events,
"QueueName"),
AutoscalingGroup=Ref(processing_group))))
t.add_output(
Output("MonitoringDashboard",
Description="Monitoring dashboard for hyp3",
Value=Ref(hyp3_dashboard)))
from template import t
from troposphere import GetAtt, Ref, Sub
from troposphere.sqs import Queue, RedrivePolicy
from troposphere.ssm import Parameter
print(' adding sqs')
def hours_in_seconds(n):
return n * 60 * 60
failed_start_evets = t.add_resource(
Queue(
"FailedStartEvents",
FifoQueue=True,
ContentBasedDeduplication=True,
))
start_events = t.add_resource(
Queue("HyP3StartEvents",
FifoQueue=True,
ContentBasedDeduplication=True,
RedrivePolicy=RedrivePolicy(
deadLetterTargetArn=GetAtt(failed_start_evets, "Arn"),
maxReceiveCount=1,
),
VisibilityTimeout=hours_in_seconds(3)))
ssm_queue_name = t.add_resource(
Parameter("HyP3SSMParameterStartEventQueueName",
source_email = t.add_parameter(
Parameter("VerifiedSourceEmail",
Description=("Verified source email. "
"This is where emails will be sent from."),
Type="String",
AllowedPattern=utils.get_email_pattern()))
lambda_policy = Policy(
PolicyName="SESSendEmail",
PolicyDocument=utils.get_static_policy('ses-send-email'))
send_email_role = t.add_resource(
Role(
"SendEmailExecutionRole",
Path="/service-role/",
ManagedPolicyArns=[
"arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
],
Policies=[lambda_policy],
AssumeRolePolicyDocument=utils.get_static_policy('lambda-policy-doc'),
))
send_email = t.add_resource(
utils.make_lambda_function(name='send_email',
role=send_email_role,
lambda_params={
"Environment":
Environment(
Variables={
'SOURCE_EMAIL': Ref(source_email),
'DB_HOST': utils.get_host_address(),
'DB_USER': Ref(db_user),
from awacs.aws import Action, Allow, PolicyDocument, Principal, Statement
from template import t
from troposphere import AWS_ACCOUNT_ID, Join, Ref, kms
print(' adding KMS key')
key_policy = PolicyDocument(
Version="2012-10-17",
Statement=[
Statement(
Sid="1",
Effect=Allow,
Principal=Principal(
"AWS", Join(":", ["arn:aws:iam:", Ref(AWS_ACCOUNT_ID), "root"])
),
Action=[Action("kms", "*")],
Resource=["*"]
)
]
)
kms_key = t.add_resource(
kms.Key(
"HyP3KMSKey",
Description="KMS Key used by HyP3 to encrypt sensitive data",
KeyPolicy=key_policy
)
)
t.add_mapping("Region2Principal", get_map('region2principal'))
solution_stack_name = t.add_parameter(
Parameter("SolutionStackName",
Description=
"ElasticBeanstalk configuration to run the API in. This should \
be set to the latest version of the Python 3.6 container.",
Type="String",
Default="64bit Amazon Linux 2018.03 v2.7.3 running Python 3.6"))
role = t.add_resource(
Role("HyP3ApiWebServerRole",
AssumeRolePolicyDocument=get_ec2_assume_role_policy(
FindInMap("Region2Principal", Ref("AWS::Region"),
"EC2Principal")),
Path="/",
ManagedPolicyArns=[
"arn:aws:iam::aws:policy/AWSElasticBeanstalkWebTier"
]))
instance_profile = t.add_resource(
InstanceProfile("WebServerInstanceProfile", Path="/", Roles=[Ref(role)]))
app = t.add_resource(
Application("HyP3Api",
ApplicationName=Sub("${StackName}-hyp3-api",
StackName=Ref('AWS::StackName')),
Description=("AWS Elastic Beanstalk API for "
"interacting with the HyP3 system")))
