def __init__(self, scope: core.Construct, id: str, props,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
################################################################################
# Set up permissions
ro_buckets = set()
for bucket in props['ro_buckets']:
tmp_bucket = s3.Bucket.from_bucket_name(self,
bucket,
bucket_name=bucket)
ro_buckets.add(tmp_bucket)
rw_buckets = set()
for bucket in props['rw_buckets']:
tmp_bucket = s3.Bucket.from_bucket_name(self,
bucket,
bucket_name=bucket)
rw_buckets.add(tmp_bucket)
batch_service_role = iam.Role(
self,
'BatchServiceRole',
assumed_by=iam.ServicePrincipal('batch.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSBatchServiceRole')
])
spotfleet_role = iam.Role(
self,
'AmazonEC2SpotFleetRole',
assumed_by=iam.ServicePrincipal('spotfleet.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonEC2SpotFleetTaggingRole')
])
# Create role for Batch instances
batch_instance_role = iam.Role(
self,
'BatchInstanceRole',
role_name='UmccriseBatchInstanceRole',
assumed_by=iam.CompositePrincipal(
iam.ServicePrincipal('ec2.amazonaws.com'),
iam.ServicePrincipal('ecs.amazonaws.com')),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonEC2RoleforSSM'),
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonEC2ContainerServiceforEC2Role')
])
batch_instance_role.add_to_policy(
iam.PolicyStatement(actions=[
"ec2:Describe*", "ec2:AttachVolume", "ec2:CreateVolume",
"ec2:CreateTags", "ec2:ModifyInstanceAttribute"
],
resources=["*"]))
batch_instance_role.add_to_policy(
iam.PolicyStatement(actions=["ecs:ListClusters"], resources=["*"]))
for bucket in ro_buckets:
bucket.grant_read(batch_instance_role)
for bucket in rw_buckets:
# restirct write to paths with */umccrise/*
bucket.grant_read_write(batch_instance_role, '*/umccrised/*')
# Turn the instance role into a Instance Profile
batch_instance_profile = iam.CfnInstanceProfile(
self,
'BatchInstanceProfile',
instance_profile_name='UmccriseBatchInstanceProfile',
roles=[batch_instance_role.role_name])
################################################################################
# Minimal networking
# TODO: import resource created with TF
vpc = props['vpc']
################################################################################
# Setup Batch compute resources
# Configure BlockDevice to expand instance disk space (if needed?)
block_device_mappings = [{
'deviceName': '/dev/xvdf',
'ebs': {
'deleteOnTermination': True,
'volumeSize': 1024,
'volumeType': 'gp2'
}
}]
launch_template = ec2.CfnLaunchTemplate(
self,
'UmccriseBatchComputeLaunchTemplate',
launch_template_name='UmccriseBatchComputeLaunchTemplate',
launch_template_data={
'userData': core.Fn.base64(user_data_script),
'blockDeviceMappings': block_device_mappings
})
launch_template_spec = batch.LaunchTemplateSpecification(
launch_template_name=launch_template.launch_template_name,
version='$Latest')
my_compute_res = batch.ComputeResources(
type=batch.ComputeResourceType.SPOT,
allocation_strategy=batch.AllocationStrategy.BEST_FIT_PROGRESSIVE,
desiredv_cpus=0,
maxv_cpus=128,
minv_cpus=0,
image=ec2.MachineImage.generic_linux(
ami_map={'ap-southeast-2': props['compute_env_ami']}),
launch_template=launch_template_spec,
spot_fleet_role=spotfleet_role,
instance_role=batch_instance_profile.instance_profile_name,
vpc=vpc,
#compute_resources_tags=core.Tag('Creator', 'Batch')
)
# XXX: How to add more than one tag above??
# core.Tag.add(my_compute_res, 'Foo', 'Bar')
my_compute_env = batch.ComputeEnvironment(
self,
'UmccriseBatchComputeEnv',
compute_environment_name="cdk-umccrise-batch-compute-env",
service_role=batch_service_role,
compute_resources=my_compute_res)
job_queue = batch.JobQueue(self,
'UmccriseJobQueue',
job_queue_name='cdk-umccrise_job_queue',
compute_environments=[
batch.JobQueueComputeEnvironment(
compute_environment=my_compute_env,
order=1)
],
priority=10)
job_container = batch.JobDefinitionContainer(
image=ecs.ContainerImage.from_registry(
name=props['container_image']),
vcpus=2,
memory_limit_mib=2048,
command=["/opt/container/umccrise-wrapper.sh", "Ref::vcpus"],
mount_points=[
ecs.MountPoint(container_path='/work',
read_only=False,
source_volume='work'),
ecs.MountPoint(container_path='/opt/container',
read_only=True,
source_volume='container')
],
volumes=[
ecs.Volume(name='container',
host=ecs.Host(source_path='/opt/container')),
ecs.Volume(name='work', host=ecs.Host(source_path='/mnt'))
],
privileged=True)
job_definition = batch.JobDefinition(
self,
'UmccriseJobDefinition',
job_definition_name='cdk-umccrise-job-definition',
parameters={'vcpus': '1'},
container=job_container,
timeout=core.Duration.hours(5))
################################################################################
# Set up job submission Lambda
lambda_role = iam.Role(
self,
'UmccriseLambdaRole',
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'),
iam.ManagedPolicy.from_aws_managed_policy_name(
'AWSBatchFullAccess') # TODO: restrict!
])
for bucket in ro_buckets:
bucket.grant_read(lambda_role)
for bucket in rw_buckets:
bucket.grant_read(lambda_role)
# TODO: support dev/prod split, i.e. image being configurable on dev, but fixed on prod
# may need a default JobDefinition to be set up
lmbda.Function(self,
'UmccriseLambda',
function_name='umccrise_batch_lambda',
handler='umccrise.lambda_handler',
runtime=lmbda.Runtime.PYTHON_3_7,
code=lmbda.Code.from_asset('lambdas/umccrise'),
environment={
'JOBNAME_PREFIX': "UMCCRISE_",
'JOBQUEUE': job_queue.job_queue_name,
'REFDATA_BUCKET': props['refdata_bucket'],
'DATA_BUCKET': props['data_bucket'],
'UMCCRISE_MEM': '50000',
'UMCCRISE_VCPUS': '16'
},
role=lambda_role)
def __init__(self, scope: core.Stack, id: str, cluster, vpc, worker,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.cluster = cluster
self.vpc = vpc
self.worker = worker
# Building a custom image for jenkins master.
self.container_image = ecr.DockerImageAsset(
self, "JenkinsMasterDockerImage", directory='./docker/master/')
if config['DEFAULT']['fargate_enabled'] == "yes" or not config[
'DEFAULT']['ec2_enabled'] == "yes":
# Task definition details to define the Jenkins master container
self.jenkins_task = ecs_patterns.ApplicationLoadBalancedTaskImageOptions(
# image=ecs.ContainerImage.from_ecr_repository(self.container_image.repository),
image=ecs.ContainerImage.from_docker_image_asset(
self.container_image),
container_port=8080,
enable_logging=True,
environment={
# https://github.com/jenkinsci/docker/blob/master/README.md#passing-jvm-parameters
'JAVA_OPTS':
'-Djenkins.install.runSetupWizard=false',
# https://github.com/jenkinsci/configuration-as-code-plugin/blob/master/README.md#getting-started
'CASC_JENKINS_CONFIG':
'/config-as-code.yaml',
'network_stack':
self.vpc.stack_name,
'cluster_stack':
self.cluster.stack_name,
'worker_stack':
self.worker.stack_name,
'cluster_arn':
self.cluster.cluster.cluster_arn,
'aws_region':
config['DEFAULT']['region'],
'jenkins_url':
config['DEFAULT']['jenkins_url'],
'subnet_ids':
",".join(
[x.subnet_id for x in self.vpc.vpc.private_subnets]),
'security_group_ids':
self.worker.worker_security_group.security_group_id,
'execution_role_arn':
self.worker.worker_execution_role.role_arn,
'task_role_arn':
self.worker.worker_task_role.role_arn,
'worker_log_group':
self.worker.worker_logs_group.log_group_name,
'worker_log_stream_prefix':
self.worker.worker_log_stream.log_stream_name
},
)
# Create the Jenkins master service
self.jenkins_master_service_main = ecs_patterns.ApplicationLoadBalancedFargateService(
self,
"JenkinsMasterService",
cpu=int(config['DEFAULT']['fargate_cpu']),
memory_limit_mib=int(
config['DEFAULT']['fargate_memory_limit_mib']),
cluster=self.cluster.cluster,
desired_count=1,
enable_ecs_managed_tags=True,
task_image_options=self.jenkins_task,
cloud_map_options=ecs.CloudMapOptions(
name="master", dns_record_type=sd.DnsRecordType('A')))
self.jenkins_master_service = self.jenkins_master_service_main.service
self.jenkins_master_task = self.jenkins_master_service.task_definition
if config['DEFAULT']['ec2_enabled'] == "yes":
self.jenkins_load_balancer = elb.ApplicationLoadBalancer(
self,
"JenkinsMasterELB",
vpc=self.vpc.vpc,
internet_facing=True,
)
self.listener = self.jenkins_load_balancer.add_listener("Listener",
port=80)
self.jenkins_master_task = ecs.Ec2TaskDefinition(
self,
"JenkinsMasterTaskDef",
network_mode=ecs.NetworkMode.AWS_VPC,
volumes=[
ecs.Volume(name="efs_mount",
host=ecs.Host(source_path='/mnt/efs'))
],
)
self.jenkins_master_task.add_container(
"JenkinsMasterContainer",
image=ecs.ContainerImage.from_ecr_repository(
self.container_image.repository),
cpu=int(config['DEFAULT']['ec2_cpu']),
memory_limit_mib=int(
config['DEFAULT']['ec2_memory_limit_mib']),
environment={
# https://github.com/jenkinsci/docker/blob/master/README.md#passing-jvm-parameters
'JAVA_OPTS':
'-Djenkins.install.runSetupWizard=false',
'CASC_JENKINS_CONFIG':
'/config-as-code.yaml',
'network_stack':
self.vpc.stack_name,
'cluster_stack':
self.cluster.stack_name,
'worker_stack':
self.worker.stack_name,
'cluster_arn':
self.cluster.cluster.cluster_arn,
'aws_region':
config['DEFAULT']['region'],
'jenkins_url':
config['DEFAULT']['jenkins_url'],
'subnet_ids':
",".join(
[x.subnet_id for x in self.vpc.vpc.private_subnets]),
'security_group_ids':
self.worker.worker_security_group.security_group_id,
'execution_role_arn':
self.worker.worker_execution_role.role_arn,
'task_role_arn':
self.worker.worker_task_role.role_arn,
'worker_log_group':
self.worker.worker_logs_group.log_group_name,
'worker_log_stream_prefix':
self.worker.worker_log_stream.log_stream_name
},
logging=ecs.LogDriver.aws_logs(
stream_prefix="JenkinsMaster",
log_retention=logs.RetentionDays.ONE_WEEK),
)
self.jenkins_master_task.default_container.add_mount_points(
ecs.MountPoint(container_path='/var/jenkins_home',
source_volume="efs_mount",
read_only=False))
self.jenkins_master_task.default_container.add_port_mappings(
ecs.PortMapping(container_port=8080, host_port=8080))
self.jenkins_master_service = ecs.Ec2Service(
self,
"EC2MasterService",
task_definition=self.jenkins_master_task,
cloud_map_options=ecs.CloudMapOptions(
name="master", dns_record_type=sd.DnsRecordType('A')),
desired_count=1,
min_healthy_percent=0,
max_healthy_percent=100,
enable_ecs_managed_tags=True,
cluster=self.cluster.cluster,
)
self.target_group = self.listener.add_targets(
"JenkinsMasterTarget",
port=80,
targets=[
self.jenkins_master_service.load_balancer_target(
container_name=self.jenkins_master_task.
default_container.container_name,
container_port=8080,
)
],
deregistration_delay=core.Duration.seconds(10))
# Opening port 5000 for master <--> worker communications
self.jenkins_master_service.task_definition.default_container.add_port_mappings(
ecs.PortMapping(container_port=50000, host_port=50000))
# Enable connection between Master and Worker
self.jenkins_master_service.connections.allow_from(
other=self.worker.worker_security_group,
port_range=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation='Master to Worker 50000',
from_port=50000,
to_port=50000))
# Enable connection between Master and Worker on 8080
self.jenkins_master_service.connections.allow_from(
other=self.worker.worker_security_group,
port_range=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation='Master to Worker 8080',
from_port=8080,
to_port=8080))
# IAM Statements to allow jenkins ecs plugin to talk to ECS as well as the Jenkins cluster #
self.jenkins_master_task.add_to_task_role_policy(
iam.PolicyStatement(
actions=[
"ecs:RegisterTaskDefinition",
"ecs:DeregisterTaskDefinition", "ecs:ListClusters",
"ecs:DescribeContainerInstances",
"ecs:ListTaskDefinitions", "ecs:DescribeTaskDefinition",
"ecs:DescribeTasks"
],
resources=["*"],
))
self.jenkins_master_task.add_to_task_role_policy(
iam.PolicyStatement(actions=["ecs:ListContainerInstances"],
resources=[self.cluster.cluster.cluster_arn]))
self.jenkins_master_task.add_to_task_role_policy(
iam.PolicyStatement(
actions=["ecs:RunTask"],
resources=[
"arn:aws:ecs:{0}:{1}:task-definition/fargate-workers*".
format(
self.region,
self.account,
)
]))
self.jenkins_master_task.add_to_task_role_policy(
iam.PolicyStatement(actions=["ecs:StopTask"],
resources=[
"arn:aws:ecs:{0}:{1}:task/*".format(
self.region, self.account)
],
conditions={
"ForAnyValue:ArnEquals": {
"ecs:cluster":
self.cluster.cluster.cluster_arn
}
}))
self.jenkins_master_task.add_to_task_role_policy(
iam.PolicyStatement(actions=["iam:PassRole"],
resources=[
self.worker.worker_task_role.role_arn,
self.worker.worker_execution_role.role_arn
]))
# END OF JENKINS ECS PLUGIN IAM POLICIES #
self.jenkins_master_task.add_to_task_role_policy(
iam.PolicyStatement(
actions=["*"],
resources=[self.worker.worker_logs_group.log_group_arn]))
def __init__(self, scope: core.Construct, id: str, vpc: ec2.Vpc, cluster: ecs.Cluster, repository: ecr.Repository, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
namespace = servicediscovery.PrivateDnsNamespace(
scope=self,
id="PRIVATE-DNS",
vpc=vpc,
name="private",
description="a private dns"
)
sg = ec2.SecurityGroup(
scope=self,
id="SG",
vpc=vpc,
allow_all_outbound=True,
description="open 9200 and 9300 ports",
security_group_name="es-group"
)
sg.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port.tcp(port=9200),
)
sg.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port.tcp(port=9300),
)
#####################################################
elastic_task_def = ecs.Ec2TaskDefinition(
scope=self,
id="ES-TASK-DEF",
network_mode=ecs.NetworkMode.AWS_VPC,
volumes=[ecs.Volume(
name="esdata",
host=ecs.Host(source_path="/usr/share/elasticsearch/data"),
)],
)
elastic = ecs.ContainerDefinition(
scope=self,
id=constants.ES_CONTAINER_NAME,
start_timeout=core.Duration.seconds(amount=30),
task_definition=elastic_task_def,
memory_limit_mib=4500,
essential=True,
image=ecs.ContainerImage.from_ecr_repository(
repository=repository, tag='latest'),
environment={
"cluster.name": constants.ES_CLUSTER_NAME,
"bootstrap.memory_lock": "true",
# "discovery.zen.ping.unicast.hosts": "elasticsearch",
"node.name": constants.ES_CONTAINER_NAME,
"node.master": "true",
"node.data": "true",
"ES_JAVA_OPTS": "-Xms4g -Xmx4g",
},
logging=ecs.AwsLogDriver(
stream_prefix="ES",
log_retention=logs.RetentionDays.ONE_DAY,
),
)
elastic.add_ulimits(ecs.Ulimit(
name=ecs.UlimitName.NOFILE, hard_limit=65535, soft_limit=65535))
elastic.add_ulimits(ecs.Ulimit(
name=ecs.UlimitName.MEMLOCK, hard_limit=-1, soft_limit=-1))
elastic.add_port_mappings(ecs.PortMapping(container_port=9200))
elastic.add_port_mappings(ecs.PortMapping(container_port=9300))
elastic.add_mount_points(ecs.MountPoint(
container_path="/usr/share/elasticsearch/data",
source_volume="esdata",
read_only=False,
))
# elastic.add_volumes_from(ecs.VolumeFrom(
# source_container="esdata",
# read_only=False,
# ))
es_service = ecs.Ec2Service(
scope=self,
id="ES-SERVICE",
cluster=cluster,
task_definition=elastic_task_def,
desired_count=1,
service_name="ES",
security_group=sg,
)
es_lb = elbv2.ApplicationLoadBalancer(
scope=self,
id="ES-ELB",
vpc=vpc,
internet_facing=True,
)
es_listener = es_lb.add_listener(
id="ES-LISTENER",
port=80,
)
es_service.register_load_balancer_targets(
ecs.EcsTarget(
new_target_group_id="ES-GRP",
container_name=elastic.container_name,
listener=ecs.ListenerConfig.application_listener(
listener=es_listener,
protocol=elbv2.ApplicationProtocol.HTTP),
))
service = es_service.enable_cloud_map(
cloud_map_namespace=namespace,
dns_record_type=servicediscovery.DnsRecordType.A,
# dns_ttl=core.Duration.seconds(amount=30),
failure_threshold=1,
name="elastic",
)
core.CfnOutput(
scope=self,
id="DNS-ES",
value=es_lb.load_balancer_dns_name,
)
#####################################################
node_task_def = ecs.Ec2TaskDefinition(
scope=self,
id="NODE-TASK-DEF",
network_mode=ecs.NetworkMode.AWS_VPC,
volumes=[ecs.Volume(
name="esdata",
host=ecs.Host(source_path="/usr/share/elasticsearch/data"),
)],
)
node = ecs.ContainerDefinition(
scope=self,
id=constants.ES_NODE_CONTAINER_NAME,
start_timeout=core.Duration.seconds(amount=40),
task_definition=node_task_def,
memory_limit_mib=4500,
essential=True,
image=ecs.ContainerImage.from_ecr_repository(
repository=repository, tag='latest'),
environment={
"cluster.name": constants.ES_CLUSTER_NAME,
"bootstrap.memory_lock": "true",
"discovery.zen.ping.unicast.hosts": "elastic.private",
"node.name": constants.ES_NODE_CONTAINER_NAME,
"node.master": "false",
"node.data": "true",
"ES_JAVA_OPTS": "-Xms4g -Xmx4g",
},
logging=ecs.LogDrivers.aws_logs(
stream_prefix="NODE",
log_retention=logs.RetentionDays.ONE_DAY,
))
node.add_port_mappings(ecs.PortMapping(container_port=9200))
node.add_port_mappings(ecs.PortMapping(container_port=9300))
node.add_ulimits(ecs.Ulimit(
name=ecs.UlimitName.NOFILE, hard_limit=65536, soft_limit=65536))
node.add_ulimits(ecs.Ulimit(
name=ecs.UlimitName.MEMLOCK, hard_limit=-1, soft_limit=-1))
node.add_mount_points(ecs.MountPoint(
container_path="/usr/share/elasticsearch/data",
source_volume="esdata",
read_only=False,
))
node_service = ecs.Ec2Service(
scope=self,
id="ES-NODE-SERVICE",
cluster=cluster,
task_definition=node_task_def,
desired_count=1,
service_name="NODE",
security_group=sg,
)
node_lb = elbv2.ApplicationLoadBalancer(
scope=self,
id="NODE-ELB",
vpc=vpc,
internet_facing=True,
)
node_listener = node_lb.add_listener(
id="NODE-LISTENER",
port=80,
)
node_service.register_load_balancer_targets(
ecs.EcsTarget(
new_target_group_id="NODE-GRP",
container_name=node.container_name,
listener=ecs.ListenerConfig.application_listener(
listener=node_listener,
protocol=elbv2.ApplicationProtocol.HTTP),
))
core.CfnOutput(
scope=self,
id="DNS-NODE",
value=node_lb.load_balancer_dns_name,
)
def __init__(self, scope: core.Construct, id: str, props, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
dirname = os.path.dirname(__file__)
ecr_repo = ecr.Repository.from_repository_name(
self,
'UmccriseEcrRepo',
repository_name='umccrise'
)
################################################################################
# Set up permissions
ro_buckets = set()
for bucket in props['ro_buckets']:
tmp_bucket = s3.Bucket.from_bucket_name(
self,
bucket,
bucket_name=bucket
)
ro_buckets.add(tmp_bucket)
rw_buckets = set()
for bucket in props['rw_buckets']:
tmp_bucket = s3.Bucket.from_bucket_name(
self,
bucket,
bucket_name=bucket
)
rw_buckets.add(tmp_bucket)
batch_service_role = iam.Role(
self,
'BatchServiceRole',
assumed_by=iam.ServicePrincipal('batch.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AWSBatchServiceRole')
]
)
spotfleet_role = iam.Role(
self,
'AmazonEC2SpotFleetRole',
assumed_by=iam.ServicePrincipal('spotfleet.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AmazonEC2SpotFleetTaggingRole')
]
)
# Create role for Batch instances
batch_instance_role = iam.Role(
self,
'BatchInstanceRole',
role_name='UmccriseBatchInstanceRole',
assumed_by=iam.CompositePrincipal(
iam.ServicePrincipal('ec2.amazonaws.com'),
iam.ServicePrincipal('ecs.amazonaws.com')
),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AmazonEC2RoleforSSM'),
iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AmazonEC2ContainerServiceforEC2Role')
]
)
batch_instance_role.add_to_policy(
iam.PolicyStatement(
actions=[
"ec2:Describe*",
"ec2:AttachVolume",
"ec2:CreateVolume",
"ec2:CreateTags",
"ec2:ModifyInstanceAttribute"
],
resources=["*"]
)
)
batch_instance_role.add_to_policy(
iam.PolicyStatement(
actions=[
"ecs:ListClusters"
],
resources=["*"]
)
)
for bucket in ro_buckets:
bucket.grant_read(batch_instance_role)
for bucket in rw_buckets:
# restirct write to paths with */umccrise/*
bucket.grant_read_write(batch_instance_role, '*/umccrised/*')
# Turn the instance role into a Instance Profile
batch_instance_profile = iam.CfnInstanceProfile(
self,
'BatchInstanceProfile',
instance_profile_name='UmccriseBatchInstanceProfile',
roles=[batch_instance_role.role_name]
)
################################################################################
# Network
# (Import common infrastructure (maintained via TerraForm)
# VPC
vpc = ec2.Vpc.from_lookup(
self,
'UmccrMainVpc',
tags={'Name': 'main-vpc', 'Stack': 'networking'}
)
batch_security_group = ec2.SecurityGroup(
self,
"BatchSecurityGroup",
vpc=vpc,
description="Allow all outbound, no inbound traffic"
)
################################################################################
# Setup Batch compute resources
# Configure BlockDevice to expand instance disk space (if needed?)
block_device_mappings = [
{
'deviceName': '/dev/xvdf',
'ebs': {
'deleteOnTermination': True,
'encrypted': True,
'volumeSize': 2048,
'volumeType': 'gp2'
}
}
]
# Set up custom user data to configure the Batch instances
umccrise_wrapper_asset = assets.Asset(
self,
'UmccriseWrapperAsset',
path=os.path.join(dirname, '..', 'assets', "umccrise-wrapper.sh")
)
umccrise_wrapper_asset.grant_read(batch_instance_role)
user_data_asset = assets.Asset(
self,
'UserDataAsset',
path=os.path.join(dirname, '..', 'assets', "batch-user-data.sh")
)
user_data_asset.grant_read(batch_instance_role)
user_data = ec2.UserData.for_linux()
local_path = user_data.add_s3_download_command(
bucket=user_data_asset.bucket,
bucket_key=user_data_asset.s3_object_key
)
user_data.add_execute_file_command(
file_path=local_path,
arguments=f"s3://{umccrise_wrapper_asset.bucket.bucket_name}/{umccrise_wrapper_asset.s3_object_key}"
)
# Generate user data wrapper to comply with LaunchTemplate required MIME multi-part archive format for user data
mime_wrapper = ec2.UserData.custom('MIME-Version: 1.0')
mime_wrapper.add_commands('Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="')
mime_wrapper.add_commands('')
mime_wrapper.add_commands('--==MYBOUNDARY==')
mime_wrapper.add_commands('Content-Type: text/x-shellscript; charset="us-ascii"')
mime_wrapper.add_commands('')
# install AWS CLI, as it's unexpectedly missing from the AWS Linux 2 AMI...
mime_wrapper.add_commands('yum -y install unzip')
mime_wrapper.add_commands('cd /opt')
mime_wrapper.add_commands('curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"')
mime_wrapper.add_commands('unzip awscliv2.zip')
mime_wrapper.add_commands('sudo ./aws/install --bin-dir /usr/bin')
# insert our actual user data payload
mime_wrapper.add_commands(user_data.render())
mime_wrapper.add_commands('--==MYBOUNDARY==--')
launch_template = ec2.CfnLaunchTemplate(
self,
'UmccriseBatchComputeLaunchTemplate',
launch_template_name='UmccriseBatchComputeLaunchTemplate',
launch_template_data={
'userData': core.Fn.base64(mime_wrapper.render()),
'blockDeviceMappings': block_device_mappings
}
)
launch_template_spec = batch.LaunchTemplateSpecification(
launch_template_name=launch_template.launch_template_name,
version='$Latest'
)
my_compute_res = batch.ComputeResources(
type=(batch.ComputeResourceType.SPOT if props['compute_env_type'].lower() == 'spot' else batch.ComputeResourceType.ON_DEMAND),
allocation_strategy=batch.AllocationStrategy.BEST_FIT_PROGRESSIVE,
desiredv_cpus=0,
maxv_cpus=320,
minv_cpus=0,
image=ec2.MachineImage.generic_linux(ami_map={'ap-southeast-2': props['compute_env_ami']}),
launch_template=launch_template_spec,
spot_fleet_role=spotfleet_role,
instance_role=batch_instance_profile.instance_profile_name,
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE,
# availability_zones=["ap-southeast-2a"]
),
security_groups=[batch_security_group]
# compute_resources_tags=core.Tag('Creator', 'Batch')
)
# XXX: How to add more than one tag above??
# https://github.com/aws/aws-cdk/issues/7350
# core.Tag.add(my_compute_res, 'Foo', 'Bar')
my_compute_env = batch.ComputeEnvironment(
self,
'UmccriseBatchComputeEnv',
compute_environment_name="cdk-umccr_ise-batch-compute-env",
service_role=batch_service_role,
compute_resources=my_compute_res
)
# child = my_compute_env.node.default_child
# child_comp_res = child.compute_resources
# child_comp_res.tags = "{'Foo': 'Bar'}"
job_queue = batch.JobQueue(
self,
'UmccriseJobQueue',
job_queue_name='cdk-umccrise_job_queue',
compute_environments=[
batch.JobQueueComputeEnvironment(
compute_environment=my_compute_env,
order=1
)
],
priority=10
)
job_container = batch.JobDefinitionContainer(
image=ecs.ContainerImage.from_registry(name=props['container_image']),
vcpus=32,
memory_limit_mib=100000,
command=[
"/opt/container/umccrise-wrapper.sh",
"Ref::vcpus"
],
mount_points=[
ecs.MountPoint(
container_path='/work',
read_only=False,
source_volume='work'
),
ecs.MountPoint(
container_path='/opt/container',
read_only=True,
source_volume='container'
)
],
volumes=[
ecs.Volume(
name='container',
host=ecs.Host(
source_path='/opt/container'
)
),
ecs.Volume(
name='work',
host=ecs.Host(
source_path='/mnt'
)
)
],
privileged=True
)
job_definition = batch.JobDefinition(
self,
'UmccriseJobDefinition',
job_definition_name='cdk-umccrise-job-definition',
parameters={'vcpus': '1'},
container=job_container,
timeout=core.Duration.hours(5)
)
################################################################################
# Set up job submission Lambda
lambda_role = iam.Role(
self,
'UmccriseLambdaRole',
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AWSLambdaBasicExecutionRole'),
iam.ManagedPolicy.from_aws_managed_policy_name('AWSBatchFullAccess') # TODO: restrict!
]
)
for bucket in ro_buckets:
bucket.grant_read(lambda_role)
for bucket in rw_buckets:
bucket.grant_read(lambda_role)
ecr_repo.grant(lambda_role, 'ecr:ListImages')
# TODO: support dev/prod split, i.e. image being configurable on dev, but fixed on prod
# may need a default JobDefinition to be set up
lmbda.Function(
self,
'UmccriseLambda',
function_name='umccrise_batch_lambda',
handler='umccrise.lambda_handler',
runtime=lmbda.Runtime.PYTHON_3_7,
code=lmbda.Code.from_asset('lambdas/umccrise'),
environment={
'JOBNAME_PREFIX': "UMCCRISE_",
'JOBQUEUE': job_queue.job_queue_name,
'UMCCRISE_MEM': '100000',
'UMCCRISE_VCPUS': '32',
'JOBDEF': job_definition.job_definition_name,
'REFDATA_BUCKET': props['refdata_bucket'],
'INPUT_BUCKET': props['input_bucket'],
'RESULT_BUCKET': props['result_bucket'],
'IMAGE_CONFIGURABLE': props['image_configurable']
},
role=lambda_role
)
def __init__(self, scope: core.Construct, id_: str, props,
**kwargs) -> None:
super().__init__(scope, id_, **kwargs)
namespace = props['namespace']
htsget_refserver_ecr_repo: ecr.Repository = props['ecr_repo']
htsget_refserver_image_tag = props['htsget_refserver_image_tag']
cors_allowed_origins = props['cors_allowed_origins']
# --- Query deployment env specific config from SSM Parameter Store
cert_apse2_arn = ssm.StringParameter.from_string_parameter_name(
self,
"SSLCertAPSE2ARN",
string_parameter_name="/htsget/acm/apse2_arn",
)
cert_apse2 = acm.Certificate.from_certificate_arn(
self,
"SSLCertAPSE2",
certificate_arn=cert_apse2_arn.string_value,
)
hosted_zone_id = ssm.StringParameter.from_string_parameter_name(
self, "HostedZoneID", string_parameter_name="hosted_zone_id")
hosted_zone_name = ssm.StringParameter.from_string_parameter_name(
self, "HostedZoneName", string_parameter_name="hosted_zone_name")
domain_name = ssm.StringParameter.from_string_parameter_name(
self,
"DomainName",
string_parameter_name="/htsget/domain",
)
# --- Cognito parameters are from data portal terraform stack
cog_user_pool_id = ssm.StringParameter.from_string_parameter_name(
self,
"CogUserPoolID",
string_parameter_name="/data_portal/client/cog_user_pool_id",
)
cog_app_client_id_stage = ssm.StringParameter.from_string_parameter_name(
self,
"CogAppClientIDStage",
string_parameter_name="/data_portal/client/cog_app_client_id_stage",
)
cog_app_client_id_local = ssm.StringParameter.from_string_parameter_name(
self,
"CogAppClientIDLocal",
string_parameter_name="/data_portal/client/cog_app_client_id_local",
)
# --- Query main VPC and setup Security Groups
vpc = ec2.Vpc.from_lookup(
self,
"VPC",
vpc_name="main-vpc",
tags={
'Stack': "networking",
},
)
private_subnets = ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE,
availability_zones=["ap-southeast-2a"],
)
sg_elb = ec2.SecurityGroup(
self,
"ELBSecurityGroup",
vpc=vpc,
description=f"Security Group for ELB in {namespace} stack",
security_group_name=f"{namespace} ELB Security Group",
allow_all_outbound=False,
)
sg_elb.add_ingress_rule(peer=ec2.Peer.any_ipv4(),
connection=ec2.Port.tcp(80),
description="Allow http inbound within VPC")
sg_ecs_service = ec2.SecurityGroup(
self,
"ECSServiceSecurityGroup",
vpc=vpc,
description=f"Security Group for ECS Service in {namespace} stack",
security_group_name=f"{namespace} ECS Security Group",
)
sg_ecs_service.add_ingress_rule(
peer=sg_elb,
connection=ec2.Port.tcp(3000),
description="Allow traffic from Load balancer to ECS service")
# --- Setup ECS Fargate cluster
config_vol = ecs.Volume(
name="config-vol",
host=ecs.Host(),
)
task_execution_role = iam.Role(
self,
"ecsTaskExecutionRole",
assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"))
task_execution_role.add_to_policy(
iam.PolicyStatement(
actions=[
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:ListMultipartUploadParts",
"s3:GetObjectTagging",
"s3:GetObjectVersionTagging",
"logs:CreateLogStream",
"logs:PutLogEvents",
"ssm:GetParameterHistory",
"ssm:GetParametersByPath",
"ssm:GetParameters",
"ssm:GetParameter",
],
resources=["*"],
))
task_execution_role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonECSTaskExecutionRolePolicy'))
task = ecs.FargateTaskDefinition(
self,
f"{namespace}-task",
cpu=512,
memory_limit_mib=1024,
volumes=[config_vol],
task_role=task_execution_role,
execution_role=task_execution_role,
)
cmd_ssm = "ssm get-parameter --name '/htsget/refserver/config' --output text --query Parameter.Value"
sidecar_container: ecs.ContainerDefinition = task.add_container(
f"{namespace}-sidecar",
image=ecs.ContainerImage.from_registry(
"quay.io/victorskl/aws-cli:2.1.3"),
essential=False,
entry_point=[
"/bin/bash",
"-c",
f"aws {cmd_ssm} > config.json",
],
logging=ecs.LogDriver.aws_logs(stream_prefix=f"{namespace}", ),
)
sidecar_container.add_mount_points(
ecs.MountPoint(
container_path="/aws",
read_only=False,
source_volume=config_vol.name,
))
main_container: ecs.ContainerDefinition = task.add_container(
namespace,
image=ecs.ContainerImage.from_ecr_repository(
repository=htsget_refserver_ecr_repo,
tag=htsget_refserver_image_tag,
),
essential=True,
command=[
"./htsget-refserver", "-config",
"/usr/src/app/config/config.json"
],
logging=ecs.LogDriver.aws_logs(stream_prefix=f"{namespace}", ),
)
main_container.add_port_mappings(
ecs.PortMapping(
container_port=3000,
protocol=ecs.Protocol.TCP,
))
main_container.add_mount_points(
ecs.MountPoint(
container_path="/usr/src/app/config",
read_only=True,
source_volume=config_vol.name,
))
main_container.add_container_dependencies(
ecs.ContainerDependency(
container=sidecar_container,
condition=ecs.ContainerDependencyCondition.COMPLETE,
))
cluster = ecs.Cluster(self, f"{namespace}-cluster", vpc=vpc)
service = ecs.FargateService(
self,
f"{namespace}-service",
platform_version=ecs.FargatePlatformVersion.VERSION1_4,
task_definition=task,
cluster=cluster,
vpc_subnets=private_subnets,
desired_count=1,
security_groups=[
sg_ecs_service,
],
)
# --- Setup Application Load Balancer in front of ECS cluster
lb = elbv2.ApplicationLoadBalancer(
self,
f"{namespace}-lb",
vpc=vpc,
internet_facing=False,
security_group=sg_elb,
deletion_protection=True,
)
http_listener = lb.add_listener(
"HttpLBListener",
port=80,
)
health_check = elbv2.HealthCheck(interval=core.Duration.seconds(30),
path="/reads/service-info",
timeout=core.Duration.seconds(5))
http_listener.add_targets(
"LBtoECS",
port=3000,
protocol=elbv2.ApplicationProtocol.HTTP,
targets=[service],
health_check=health_check,
)
core.CfnOutput(self,
"LoadBalancerDNS",
value=lb.load_balancer_dns_name)
# --- Setup APIGatewayv2 HttpApi using VpcLink private integration to ALB/ECS in private subnets
vpc_link = apigwv2.VpcLink(self,
f"{namespace}-VpcLink",
vpc=vpc,
security_groups=[
sg_ecs_service,
sg_elb,
])
self.apigwv2_alb_integration = apigwv2i.HttpAlbIntegration(
listener=http_listener,
vpc_link=vpc_link,
)
custom_domain = apigwv2.DomainName(
self,
"CustomDomain",
certificate=cert_apse2,
domain_name=domain_name.string_value,
)
self.http_api = apigwv2.HttpApi(
self,
f"{namespace}-apigw",
default_domain_mapping=apigwv2.DomainMappingOptions(
domain_name=custom_domain),
cors_preflight=apigwv2.CorsPreflightOptions(
allow_origins=cors_allowed_origins,
allow_headers=["*"],
allow_methods=[
apigwv2.CorsHttpMethod.ANY,
],
allow_credentials=True,
))
core.CfnOutput(self, "ApiEndpoint", value=self.http_api.api_endpoint)
# --- Setup DNS for the custom domain
hosted_zone = route53.HostedZone.from_hosted_zone_attributes(
self,
"HostedZone",
hosted_zone_id=hosted_zone_id.string_value,
zone_name=hosted_zone_name.string_value,
)
route53.ARecord(
self,
"ApiCustomDomainAlias",
zone=hosted_zone,
record_name="htsget",
target=route53.RecordTarget.from_alias(
route53t.ApiGatewayv2DomainProperties(
regional_domain_name=custom_domain.regional_domain_name,
regional_hosted_zone_id=custom_domain.
regional_hosted_zone_id)),
)
core.CfnOutput(
self,
"HtsgetEndpoint",
value=custom_domain.name,
)
cognito_authzr = apigwv2.CfnAuthorizer(
self,
"CognitoAuthorizer",
api_id=self.http_api.http_api_id,
authorizer_type="JWT",
identity_source=[
"$request.header.Authorization",
],
name="CognitoAuthorizer",
jwt_configuration=apigwv2.CfnAuthorizer.JWTConfigurationProperty(
audience=[
cog_app_client_id_stage.string_value,
cog_app_client_id_local.string_value,
],
issuer=
f"https://cognito-idp.{self.region}.amazonaws.com/{cog_user_pool_id.string_value}"
))
# Add catch all routes
rt_catchall = apigwv2.HttpRoute(
self,
"CatchallRoute",
http_api=self.http_api,
route_key=apigwv2.HttpRouteKey.with_(
path="/{proxy+}", method=apigwv2.HttpMethod.GET),
integration=self.apigwv2_alb_integration)
rt_catchall_cfn: apigwv2.CfnRoute = rt_catchall.node.default_child
rt_catchall_cfn.authorizer_id = cognito_authzr.ref
rt_catchall_cfn.authorization_type = "JWT"
# Comment this to opt-out setting up experimental Passport + htsget
self.setup_ga4gh_passport()
