def __init__(self, scope: core.Stack, id: str, **kwargs):
super().__init__(scope, id, **kwargs)
self.base_platform = BasePlatform(self, self.stack_name)
self.fargate_queue_processing_service = aws_ecs_patterns.QueueProcessingFargateService(
self,
"QueueProcessingService",
service_name='queue-processing',
cluster=self.base_platform.ecs_cluster,
cpu=256,
desired_task_count=0,
max_scaling_capacity=10,
memory_limit_mib=512,
#image=aws_ecs.ContainerImage.from_registry("625806755153.dkr.ecr.us-east-1.amazonaws.com/python-workers:latest"),
image=aws_ecs.ContainerImage.from_ecr_repository(
aws_ecr.Repository.from_repository_name(
self, "mi_repositorio", "python-workers")),
environment={
"REGION": getenv('AWS_DEFAULT_REGION'),
"AWS_ACCOUNT_ID": getenv('AWS_ACCOUNT_ID'),
},
)
self.fargate_queue_processing_service.task_definition.add_to_task_role_policy(
aws_iam.PolicyStatement(actions=['ec2:DescribeSubnets'],
resources=['*']))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
self.fargate_queue_service = aws_ecs_patterns.QueueProcessingFargateService(
self,
"QueueService",
cpu=512,
memory_limit_mib=2048,
image=aws_ecs.ContainerImage.from_asset(
directory='.',
file='Dockerfile.queue_service',
exclude=["cdk.out"]),
desired_task_count=1)
def __init__(self, scope, id, cluster: ecs.Cluster,
tracks_table: dynamodb.Table, input_bucket: s3.Bucket,
output_bucket: s3.Bucket, **kwargs):
super().__init__(scope, id, **kwargs)
worker_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'worker'))
self.service = ecs_patterns.QueueProcessingFargateService(
self,
'separator-service',
cluster=cluster,
cpu=2048,
memory_limit_mib=8192,
image=ecs.ContainerImage.from_asset(directory=worker_dir),
environment={
'TRACKS_TABLE_NAME': tracks_table.table_name,
'OUTPUT_BUCKET_NAME': output_bucket.bucket_name
})
input_bucket.grant_read(self.service.task_definition.task_role)
output_bucket.grant_write(self.service.task_definition.task_role)
tracks_table.grant_read_write_data(
self.service.task_definition.task_role)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# API Gateway needs to have resource policy granting FHIR Works on AWS lambda
# execute permissions. Lambda function ARN will be passed during deployment as CDK context variable
# FHIR Works lambda will need to have policy attached to its execution role
# allowing it to invoke API
# From --context resource-router-lambda-role="arn:aws:iam::123456789012:role/rolename"
imported_resource_router_lambda_role = self.node.try_get_context(
"resource-router-lambda-role"
)
# Amazon ECS on AWS Fargate container implementing connection manager
# will be launched into a VPC that needs to have private and public subnets
# and NAT gateway or instance
# From --context vpc-id="vpc-123456"
vpc_id = self.node.try_get_context("vpc-id")
# The following parameters specify name of the HL7 server
# that will be receiving transformed HL7v2 messages and TCP port
# that it will be listening on
# From --context hl7-server-name="hl7.example.com"
# From --context hl7-port="2575"
hl7_server_name = self.node.try_get_context("hl7-server-name")
hl7_port = self.node.try_get_context("hl7-port")
# In this proof of concept source of data for read interactions
# is S3 bucket where mock HL7 server stores processed HL7 messages
# From --context test-server-output-bucket-name="DOC-EXAMPLE-BUCKET"
test_server_output_bucket_name = self.node.try_get_context(
"test-server-output-bucket-name"
)
# SQS queue
# Custom transform lambda communicates with Connectivity Manager using this SQS queue
queue = sqs.Queue(
self, f"{COMPONENT_PREFIX}Queue", encryption=sqs.QueueEncryption.KMS_MANAGED
)
# S3 Bucket to retrieve HL7v2 messages in proof of concept deployment
test_server_output_bucket = s3.Bucket.from_bucket_name(
self, f"{COMPONENT_PREFIX}OutputBucket", test_server_output_bucket_name
)
# Transform Lambda
# Reference implementation of Custom Transform component of Transform Execution Environment
transform_lambda = lambda_.Function(
self,
f"{COMPONENT_PREFIX}TransformLambda",
handler="transform.handler",
runtime=lambda_.Runtime.PYTHON_3_8,
code=lambda_.Code.from_asset(
path.join(dirname, "../../lambda"),
bundling={
"image": lambda_.Runtime.PYTHON_3_8.bundling_docker_image,
"command": [
"bash",
"-c",
" && ".join(
[
"pip install --no-cache-dir -r requirements.txt -t /asset-output",
"(tar -c --exclude-from=exclude.lst -f - .)|(cd /asset-output; tar -xf -)",
]
),
],
},
),
timeout=core.Duration.seconds(60),
environment=dict(
SQS_QUEUE=queue.queue_url,
# The following parameter is optional
S3_BUCKET_NAME=test_server_output_bucket_name,
),
)
queue.grant_send_messages(transform_lambda)
# API Gateway with Lambda construct (using https://aws.amazon.com/solutions/constructs/patterns)
# Reference implementation of Custom Transform component of Transform Execution Environment
api_lambda = apigw_lambda.ApiGatewayToLambda(
self,
"ApiGw",
existing_lambda_obj=transform_lambda,
api_gateway_props=apigw.LambdaRestApiProps(
handler=transform_lambda,
proxy=False,
rest_api_name=f"{COMPONENT_PREFIX_DASHES}-api",
endpoint_export_name=f"{COMPONENT_PREFIX}ApiEndPoint",
description=f"{COMPONENT_PREFIX} APIGW with Transform Lambda (FHIR to HL7v2)",
default_method_options=apigw.MethodOptions(
authorization_type=apigw.AuthorizationType.IAM,
),
policy=iam.PolicyDocument(
statements=[
iam.PolicyStatement(
actions=["execute-api:Invoke"],
effect=iam.Effect.ALLOW,
principals=[
iam.ArnPrincipal(imported_resource_router_lambda_role),
],
resources=["execute-api:/*/*/*"],
)
]
),
),
)
rest_api = api_lambda.api_gateway
persistence = rest_api.root.add_resource("persistence")
resource_type = persistence.add_resource("{resource_type}")
resource_type.add_method("POST")
resource_id = resource_type.add_resource("{id}")
resource_id.add_method("GET")
resource_id.add_method("PUT")
resource_id.add_method("DELETE")
# ECS Fargate Container (HL7v2 sender)
# This container implements Connectivity Manager component
# of Transform Execution Environment
vpc = ec2.Vpc.from_lookup(self, "DefaultVpc", vpc_id=vpc_id)
cluster = ecs.Cluster(self, f"{COMPONENT_PREFIX}Cluster", vpc=vpc)
ecs_patterns.QueueProcessingFargateService(
self,
f"{COMPONENT_PREFIX}Service",
cluster=cluster,
image=ecs.ContainerImage.from_asset(path.join(dirname, "../../container")),
queue=queue,
desired_task_count=1,
log_driver=ecs.LogDriver.aws_logs(
stream_prefix=f"{COMPONENT_PREFIX}HL7Client",
log_retention=logs.RetentionDays.ONE_DAY,
),
environment=dict(
SERVER_NAME=hl7_server_name,
PORT_NUMBER=hl7_port,
),
)
# The following permission grants are needed to support
# read interactions with integration transform
test_server_output_bucket.grant_read(transform_lambda)
transform_lambda.add_to_role_policy(
iam.PolicyStatement(
actions=["s3:ListBucket"],
effect=iam.Effect.ALLOW,
resources=[test_server_output_bucket.bucket_arn],
)
)
transform_lambda.add_to_role_policy(
iam.PolicyStatement(
actions=["s3:GetObject"],
effect=iam.Effect.ALLOW,
resources=[test_server_output_bucket.arn_for_objects("*")],
)
)
# CloudFormation Stack outputs
# The following outputs needed to configure FHIR Works on AWS API interface
core.CfnOutput(
self,
"TransformApiRootUrl",
value=rest_api.url,
export_name="TransformApiRootUrl",
)
core.CfnOutput(
self,
"TransformApiRegion",
value=self.region,
export_name="TransformApiRegion",
)
core.CfnOutput(
self,
"TransformApiAccountId",
value=self.account,
export_name="TransformApiAccountId",
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
### S3 ###
source_csv_bucket = _s3.Bucket(self,
"BYODValidationSourceBucket",
versioned=True)
target_csv_bucket = _s3.Bucket(
self,
"BYODValidationTargetBucket",
removal_policy=core.RemovalPolicy.RETAIN)
webtool_bucket = _s3.Bucket(
self,
"WebToolBucket",
website_index_document="index.html",
website_error_document="index.html",
public_read_access=True,
)
core.CfnOutput(self, "DVTRegion", value=self.region)
core.CfnOutput(self,
"SourceS3Bucket",
value=source_csv_bucket.bucket_name)
core.CfnOutput(self,
"TargetS3Bucket",
value=target_csv_bucket.bucket_name)
core.CfnOutput(self,
"WebToolS3Bucket",
value=webtool_bucket.bucket_name)
core.CfnOutput(self,
"WebToolUrl",
value=webtool_bucket.bucket_website_url)
### Stager Function ###
stager_function = _lambda.Function(self,
"StagerFunction",
runtime=_lambda.Runtime.NODEJS_12_X,
code=_lambda.Code.from_asset(
os.path.join(
dirname, "lambda",
"stager")),
handler='index.handler')
stager_function.add_environment("REGION", self.region)
stager_function.add_environment("SOURCE_BUCKET",
source_csv_bucket.bucket_name)
stager_function.add_environment("STAGE_BUCKET",
target_csv_bucket.bucket_name)
source_csv_bucket.grant_read(stager_function)
target_csv_bucket.grant_put(stager_function)
core.CfnOutput(self,
"StagerLambdaFunction",
value=stager_function.function_name)
### Profiling Queue
profiling_job_queue = _sqs.Queue(self, "ProfilingJobQueue")
core.CfnOutput(self,
"SQSProfileQueue",
value=profiling_job_queue.queue_url)
### Cognito ###
userpool = _cognito.UserPool(self,
"WebToolUserPool",
user_pool_name="byod-webtool-userpool",
self_sign_up_enabled=True,
auto_verify={
"email": True,
"phone": False
},
user_verification={
"email_subject":
"Your verification code",
"email_body":
"Your verification code is {####}",
"email_style":
_cognito.VerificationEmailStyle.CODE
},
standard_attributes={
"email": {
"required": True,
"mutable": False
}
},
password_policy={})
client = userpool.add_client(
"webtool-app-client",
auth_flows={
"custom": True,
"user_password": True,
"user_srp": True,
#"refresh_token": True
})
identity_pool = _cognito.CfnIdentityPool(
self,
"WebToolCognitoIdentityPool",
allow_unauthenticated_identities=True)
identity_pool.add_property_override(
"CognitoIdentityProviders",
[{
"ClientId": client.user_pool_client_id,
"ProviderName": userpool.user_pool_provider_name
}])
auth_role = _iam.Role(
self,
"CognitoAuthRole",
assumed_by=WebIdentityPrincipal(
"cognito-identity.amazonaws.com", {
"StringEquals": {
"cognito-identity.amazonaws.com:aud": identity_pool.ref
},
"ForAnyValue:StringLike": {
"cognito-identity.amazonaws.com:amr": "authenticated"
}
}))
auth_role.add_to_policy(
PolicyStatement(effect=Effect.ALLOW,
actions=["s3:GetObject", "s3:PutObject"],
resources=["%s/*" % target_csv_bucket.bucket_arn]))
auth_role.add_to_policy(
PolicyStatement(effect=Effect.ALLOW,
actions=["lambda:invokeFunction"],
resources=[stager_function.function_arn]))
auth_role.add_to_policy(
PolicyStatement(effect=Effect.ALLOW,
actions=["sqs:*"],
resources=[profiling_job_queue.queue_arn]))
unauth_role = _iam.Role(
self,
"CognitoUnauthRole",
assumed_by=_iam.WebIdentityPrincipal(
"cognito-identity.amazonaws.com",
conditions={
"StringEquals": {
"cognito-identity.amazonaws.com:aud": identity_pool.ref
},
"ForAnyValue:StringLike": {
"cognito-identity.amazonaws.com:amr": "unauthenticated"
}
}))
identity_pool_policy = _cognito.CfnIdentityPoolRoleAttachment(
self,
"WebToolCognitoIdentityPoolPolicy",
identity_pool_id=identity_pool.ref,
roles={
'unauthenticated': unauth_role.role_arn,
'authenticated': auth_role.role_arn
})
core.CfnOutput(self, "UserPoolId", value=userpool.user_pool_id)
core.CfnOutput(self, "IdentityPoolId", value=identity_pool.ref)
core.CfnOutput(self, "ClientId", value=client.user_pool_client_id)
core.CfnOutput(self,
"ProviderName",
value=userpool.user_pool_provider_name)
### DynamoDB ###
validation_job_table = _dynamodb.Table(
self,
"ValidationJobTable",
partition_key=_dynamodb.Attribute(
name="id", type=_dynamodb.AttributeType.STRING))
## AppSync ###
api = _appsync.GraphqlApi(
self,
"Api",
name="validation-job-api",
schema=_appsync.Schema.from_asset(
os.path.join(dirname, "api", "schema.graphql")),
authorization_config=AuthorizationConfig(
default_authorization=AuthorizationMode(
authorization_type=AuthorizationType.USER_POOL,
user_pool_config=UserPoolConfig(user_pool=userpool))),
log_config=LogConfig(exclude_verbose_content=False,
field_log_level=FieldLogLevel.ALL))
api_ds = api.add_dynamo_db_data_source("ValidationJobDataSource",
validation_job_table)
from aws_cdk.aws_appsync import MappingTemplate
api_ds.create_resolver(
type_name="Query",
field_name="listJobss",
request_mapping_template=MappingTemplate.from_file(
os.path.join(dirname, "api", "resolvers",
"Query.listJobss.req.vtl")),
response_mapping_template=MappingTemplate.from_file(
os.path.join(dirname, "api", "resolvers",
"Query.listJobss.res.vtl")))
api_ds.create_resolver(
type_name="Query",
field_name="getJobs",
request_mapping_template=MappingTemplate.from_file(
os.path.join(dirname, "api", "resolvers",
"Query.getJobs.req.vtl")),
response_mapping_template=MappingTemplate.from_file(
os.path.join(dirname, "api", "resolvers",
"Query.getJobs.res.vtl")))
core.CfnOutput(self, "GraphQLEndpoint", value=api.graphql_url)
### SQS ###
validation_job_queue = _sqs.Queue(self, "ValidationJobQueue")
### Lambda ###
validation_trigger_function = _lambda.Function(
self,
"ValidationTriggerFunction",
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset(
os.path.join(dirname, "lambda", "validation_trigger")),
handler='lambda_function.lambda_handler')
validation_trigger_function.add_environment(
"TABLE_NAME", validation_job_table.table_name)
validation_trigger_function.add_environment(
"QUEUE_URL", validation_job_queue.queue_url)
validation_trigger_function.add_event_source(
_S3EventSource(source_csv_bucket,
events=[_s3.EventType.OBJECT_CREATED]))
source_csv_bucket.grant_read(validation_trigger_function)
validation_job_table.grant_read_write_data(validation_trigger_function)
validation_job_queue.grant_send_messages(validation_trigger_function)
### ECS Fargate ###
validation_fargate_asset = _ecr_assets.DockerImageAsset(
self,
"ValidationBuildImage",
directory=os.path.join(dirname, "fargate", "validation"))
profiling_fargate_asset = _ecr_assets.DockerImageAsset(
self,
"ProfilingBuildImage",
directory=os.path.join(dirname, "fargate", "profiling"))
vpc = _ec2.Vpc(self, "VPC", max_azs=3)
cluster = _ecs.Cluster(self, "ECSCluster", vpc=vpc)
validation_fargate_service = _ecs_patterns.QueueProcessingFargateService(
self,
"ValidationFargateService",
cluster=cluster,
cpu=4096,
memory_limit_mib=30720,
enable_logging=True,
image=_ecs.ContainerImage.from_docker_image_asset(
validation_fargate_asset),
environment={
"TABLE_NAME": validation_job_table.table_name,
"QUEUE_URL": validation_job_queue.queue_url,
"SOURCE_BUCKET_NAME": source_csv_bucket.bucket_name,
"TARGET_BUCKET_NAME": target_csv_bucket.bucket_name,
"REGION": self.region
},
queue=validation_job_queue,
max_scaling_capacity=2,
max_healthy_percent=200,
min_healthy_percent=66)
validation_fargate_service.task_definition.task_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonDynamoDBFullAccess"))
validation_fargate_service.task_definition.task_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3FullAccess"))
profiling_fargate_service = _ecs_patterns.QueueProcessingFargateService(
self,
"ProfilingFargateService",
cluster=cluster,
cpu=4096,
memory_limit_mib=30720,
enable_logging=True,
image=_ecs.ContainerImage.from_docker_image_asset(
profiling_fargate_asset),
environment={
"TABLE_NAME": validation_job_table.table_name,
"QUEUE_URL": profiling_job_queue.queue_url,
"SOURCE_BUCKET_NAME": source_csv_bucket.bucket_name,
"TARGET_BUCKET_NAME": target_csv_bucket.bucket_name,
"REGION": self.region
},
queue=profiling_job_queue,
max_scaling_capacity=2,
max_healthy_percent=200,
min_healthy_percent=66)
profiling_fargate_service.task_definition.task_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonDynamoDBFullAccess"))
profiling_fargate_service.task_definition.task_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3FullAccess"))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
### VPC
# create a new VPC with 2 AZ's and two NAT gateways
# TODO - include VPC SQS endpoint so the NAT gateway isn't needed anymore
vpc = aws_ec2.Vpc(self,
"Vpc",
max_azs=2,
nat_gateways=2,
subnet_configuration=[
aws_ec2.SubnetConfiguration(
name="private",
cidr_mask=24,
subnet_type=aws_ec2.SubnetType.PRIVATE),
aws_ec2.SubnetConfiguration(
name="public",
cidr_mask=28,
subnet_type=aws_ec2.SubnetType.PUBLIC)
])
# create a new ECS cluster
cluster = aws_ecs.Cluster(self, "FargateSQS", vpc=vpc)
### SQS
# create a new SQS queue
msg_queue = aws_sqs.Queue(self,
"SQSQueue",
visibility_timeout=core.Duration.seconds(0),
retention_period=core.Duration.minutes(30))
### FARGATE
# build the docker image from local "./docker" directory
sqscontainer = aws_ecs.ContainerImage.from_asset(directory="docker")
# add the aws-xray-daemon as a sidecar running on UDP/2000
xraycontainer = aws_ecs.ContainerImage.from_registry(
"amazon/aws-xray-daemon")
# create the queue processing service on fargate with a locally built container
# the pattern automatically adds an environment variable with the queue name for the container to read
fargate_service = aws_ecs_patterns.QueueProcessingFargateService(
self,
"Service",
cluster=cluster,
memory_limit_mib=512,
cpu=256,
image=sqscontainer,
enable_logging=True,
desired_task_count=0,
max_scaling_capacity=5,
scaling_steps=
[{
"upper": 0,
"change": -5
}, {
"lower": 1,
"change": +1
}
# disabled metric based scaling to test scaling on cpu usage only
# this may potentially lower cost as fargate will scale in smaller steps
#{"lower": 50000, "change": +2},
#{"lower": 250000, "change": +4}
],
queue=msg_queue,
environment={"sqs_queue_url": msg_queue.queue_url})
# add the standard aws xray sidecar to the container task
xray_sidecar = fargate_service.task_definition.add_container(
"xraycontainer",
image=xraycontainer,
logging=fargate_service.log_driver)
# expose the sidecar on port UDP/2000
xray_sidecar.add_port_mappings(
aws_ecs.PortMapping(container_port=2000,
protocol=aws_ecs.Protocol.UDP))
### LAMBDA
# build the go binary for the lambda SQS generator and retrieve the unix timestamp of when the file was modified
# since CDK cannot natively build Go binaries yet, we need to do this manually through build_lambda_zip.py
os.system("python loadgen/build_lambda_zip.py")
filets = str(int(os.path.getctime("./loadgen/lambda.zip")))
# create a lambda function to generate load, using the filets value as a source hash for the zip
sqs_lambda = aws_lambda.Function(
self,
"GenerateLoadSQS",
runtime=aws_lambda.Runtime.GO_1_X,
code=aws_lambda.Code.from_asset("./loadgen/lambda.zip",
source_hash=filets),
handler="loadgensqs",
timeout=core.Duration.seconds(20),
memory_size=128,
retry_attempts=0,
tracing=aws_lambda.Tracing.ACTIVE,
environment={
"sqs_queue_url": msg_queue.queue_url,
"total_message_count": "100"
})
### CLOUDWATCH RULE
# create a new cloudwatch rule running every minute to trigger the lambda function
eventRuleMinu = aws_events.Rule(
self,
"lambda-generator-minute-rule",
enabled=True,
schedule=aws_events.Schedule.cron(minute="*"))
eventRuleMinu.add_target(aws_events_targets.LambdaFunction(sqs_lambda))
### IAM
# add the Lambda IAM permission to send SQS messages
msg_queue.grant_send_messages(sqs_lambda)
# add XRay permissions to Fargate task and Lambda
xray_policy = PolicyStatement(resources=["*"],
actions=[
"xray:GetGroup", "xray:GetGroups",
"xray:GetSampling*", "xray:GetTime*",
"xray:GetService*",
"xray:PutTelemetryRecords",
"xray:PutTraceSegments"
])
fargate_service.task_definition.add_to_task_role_policy(xray_policy)
sqs_lambda.add_to_role_policy(xray_policy)