emit_tf({
"data": [{
"aws_route53_zone": {
zone.slug: {
"name": zone.name,
"private_zone": False
}
}
} for zone in set(zones_by_domain.values())],
# Note that ${} references exist to interpolate a value AND express a dependency.
"resource": [
{
"aws_api_gateway_deployment": {
lambda_.name: {
"rest_api_id":
"${module.chalice_%s.rest_api_id}" % lambda_.name,
"stage_name": config.deployment_stage
}
},
"aws_api_gateway_base_path_mapping": {
f"{lambda_.name}_{i}": {
"api_id":
"${module.chalice_%s.rest_api_id}" % lambda_.name,
"stage_name":
"${aws_api_gateway_deployment.%s.stage_name}" %
lambda_.name,
"domain_name":
"${aws_api_gateway_domain_name.%s_%i.domain_name}" %
(lambda_.name, i)
}
for i, domain in enumerate(lambda_.domains)
},
"aws_api_gateway_domain_name": {
f"{lambda_.name}_{i}": {
"domain_name":
"${aws_acm_certificate.%s_%i.domain_name}" %
(lambda_.name, i),
"certificate_arn":
"${aws_acm_certificate_validation.%s_%i.certificate_arn}" %
(lambda_.name, i)
}
for i, domain in enumerate(lambda_.domains)
},
"aws_acm_certificate": {
f"{lambda_.name}_{i}": {
"domain_name": domain,
"validation_method": "DNS",
"provider": "aws.us-east-1",
# I tried using SANs for the alias domains (like the DRS domain) but Terraform kept swapping the
# zones, I think because the order of elements in `aws_acm_certificate.domain_validation_options`
# is not deterministic. The alternative is to use separate certs, one for each domain, the main
# one as well as for each alias.
#
"subject_alternative_names": [],
"lifecycle": {
"create_before_destroy": True
}
}
for i, domain in enumerate(lambda_.domains)
},
"aws_acm_certificate_validation": {
f"{lambda_.name}_{i}": {
"certificate_arn":
"${aws_acm_certificate.%s_%i.arn}" % (lambda_.name, i),
"validation_record_fqdns": [
"${aws_route53_record.%s_domain_validation_%i.fqdn}" %
(lambda_.name, i)
],
"provider":
"aws.us-east-1"
}
for i, domain in enumerate(lambda_.domains)
},
"aws_route53_record": {
**{
f"{lambda_.name}_domain_validation_{i}": {
**{
key: "${aws_acm_certificate.%s_%i.domain_validation_options.0.resource_record_%s}" % (lambda_.name, i, key)
for key in ('name', 'type')
}, "zone_id":
"${data.aws_route53_zone.%s.id}" % zones_by_domain[domain].slug,
"records": [
"${aws_acm_certificate.%s_%i.domain_validation_options.0.resource_record_value}" % (lambda_.name, i)
],
"ttl":
60
}
for i, domain in enumerate(lambda_.domains)
},
**{
f"{lambda_.name}_{i}": {
"zone_id":
"${data.aws_route53_zone.%s.id}" % zones_by_domain[domain].slug,
"name":
"${aws_api_gateway_domain_name.%s_%i.domain_name}" % (lambda_.name, i),
"type":
"A",
"alias": {
"name":
"${aws_api_gateway_domain_name.%s_%i.cloudfront_domain_name}" % (lambda_.name, i),
"zone_id":
"${aws_api_gateway_domain_name.%s_%i.cloudfront_zone_id}" % (lambda_.name, i),
"evaluate_target_health":
True,
}
}
for i, domain in enumerate(lambda_.domains)
}
},
**({
"aws_cloudwatch_log_group": {
lambda_.name: {
"name":
"/aws/apigateway/" + config.qualified_resource_name(lambda_.name),
"retention_in_days":
1827,
"provisioner": {
"local-exec": {
"command":
' '.join(
map(shlex.quote, [
"python", config.project_root + "/scripts/log_api_gateway.py",
"${module.chalice_%s.rest_api_id}" % lambda_.name, config.deployment_stage,
"${aws_cloudwatch_log_group.%s.arn}" % lambda_.name
]))
}
}
}
}
} if config.enable_monitoring else {}),
"aws_iam_role": {
lambda_.name: {
"name":
config.qualified_resource_name(lambda_.name),
"assume_role_policy":
json.dumps({
"Version":
"2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "sts:AssumeRole",
"Principal": {
"Service": "lambda.amazonaws.com"
}
},
*(
{
"Effect": "Allow",
"Action": "sts:AssumeRole",
"Principal": {
"AWS": f"arn:aws:iam::{account}:root"
},
# Wildcards are not supported in `Principal`, but they are in `Condition`
"Condition": {
"StringLike": {
"aws:PrincipalArn": [
f"arn:aws:iam::{account}:role/{role}"
for role in roles
]
}
}
} for account, roles in
config.external_lambda_role_assumptors.items())
]
}),
**aws.permissions_boundary_tf
}
},
"aws_iam_role_policy": {
lambda_.name: {
"name": lambda_.name,
"policy": lambda_.policy,
"role": "${aws_iam_role.%s.id}" % lambda_.name
},
}
} for lambda_ in lambdas
]
})
emit_tf({
"resource": [
{
"aws_s3_bucket": {
"storage": {
"bucket": config.s3_bucket,
"acl": "private",
"force_destroy": True,
"lifecycle_rule": {
"id": "manifests",
"enabled": True,
"prefix": "manifests/",
"expiration": {
"days": config.manifest_expiration
}
}
},
"urls": {
"bucket": config.url_redirect_full_domain_name,
"force_destroy": not config.is_main_deployment(),
"acl": "public-read",
"website": {
# index_document is required; pointing to a non-existent file to return a 404
"index_document": "404.html"
}
}
}
},
*([{
"aws_route53_record": {
"url_redirect_record": {
"zone_id": "${data.aws_route53_zone.azul_url.zone_id}",
"name": config.url_redirect_full_domain_name,
"type": "CNAME",
"ttl": "300",
"records": ["${aws_s3_bucket.urls.website_endpoint}"]
}
}
}] if config.url_redirect_base_domain_name else [])
],
**({
"data": [{
"aws_route53_zone": {
"azul_url": {
"name": config.url_redirect_base_domain_name + ".",
"private_zone": False
}
}
}]
} if config.url_redirect_base_domain_name else {})
})
from azul import (
config,
)
from azul.deployment import (
aws,
emit_tf,
)
emit_tf(
{
"terraform": {
"backend": {
"s3": {
"bucket": config.terraform_backend_bucket,
"key": f"azul-{config.terraform_component}-{config.deployment_stage}.tfstate",
"region": aws.region_name,
**(
{
"profile": aws.profile['source_profile'],
"role_arn": aws.profile['role_arn']
} if 'role_arn' in aws.profile else {
}
)
}
}
}
}
)
actual_component_path = Path(__file__).absolute().parent
require(
expected_component_path.samefile(actual_component_path),
f"The current Terraform component is set to '{config.terraform_component}'. "
f"You should therefore be in '{expected_component_path}'")
emit_tf({
"data": [{
"aws_caller_identity": {
"current": {}
}
}, {
"aws_region": {
"current": {}
}
}, *([{
"google_client_config": {
"current": {}
}
}] if config.enable_gcp() else [])],
"locals": {
"account_id":
"${data.aws_caller_identity.current.account_id}",
"region":
"${data.aws_region.current.name}",
"google_project":
"${data.google_client_config.current.project}"
if config.enable_gcp() else None
},
})
emit_tf({
"resource": {
"aws_iam_role": {
"states": {
"name":
config.qualified_resource_name("statemachine"),
"assume_role_policy":
json.dumps({
"Version":
"2012-10-17",
"Statement": [{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "states.amazonaws.com"
},
"Action": "sts:AssumeRole"
}]
}),
**aws.permissions_boundary_tf
}
},
"aws_iam_role_policy": {
"states": {
"name":
config.qualified_resource_name("statemachine"),
"role":
"${aws_iam_role.states.id}",
"policy":
json.dumps({
"Version":
"2012-10-17",
"Statement": [{
"Effect":
"Allow",
"Action": ["lambda:InvokeFunction"],
"Resource": [
aws.get_lambda_arn(
config.service_name,
config.manifest_lambda_basename),
aws.get_lambda_arn(
config.service_name,
config.cart_item_write_lambda_basename),
aws.get_lambda_arn(
config.service_name,
config.cart_export_dss_push_lambda_basename)
]
}]
})
}
},
"aws_sfn_state_machine": {
"manifest": {
"name":
config.manifest_state_machine_name,
"role_arn":
"${aws_iam_role.states.arn}",
"definition":
json.dumps(
{
"StartAt": "WriteManifest",
"States": {
"WriteManifest": {
"Type":
"Task",
"Resource":
aws.get_lambda_arn(
config.service_name,
config.manifest_lambda_basename),
"End":
True
}
}
},
indent=2)
},
"cart_item": {
"name":
config.cart_item_state_machine_name,
"role_arn":
"${aws_iam_role.states.arn}",
"definition":
json.dumps(
{
"StartAt": "WriteBatch",
"States": cart_item_states()
},
indent=2)
},
"cart_export": {
"name":
config.cart_export_state_machine_name,
"role_arn":
"${aws_iam_role.states.arn}",
"definition":
json.dumps(
{
"StartAt": "SendToCollectionAPI",
"States": {
"SendToCollectionAPI": {
"Type":
"Task",
"Resource":
aws.get_lambda_arn(
config.service_name, config.
cart_export_dss_push_lambda_basename),
"Next":
"NextBatch",
"ResultPath":
"$"
},
"NextBatch": {
"Type":
"Choice",
"Choices": [{
"Variable": "$.resumable",
"BooleanEquals": False,
"Next": "SuccessState"
}],
"Default":
"SendToCollectionAPI"
},
"SuccessState": {
"Type": "Succeed"
}
}
},
indent=2)
}
}
}
})
emit_tf({
"resource": [{
"aws_sqs_queue": {
config.unqual_notifications_queue_name(): {
"name":
config.notifications_queue_name(),
"visibility_timeout_seconds":
config.contribution_lambda_timeout + 10,
"message_retention_seconds":
24 * 60 * 60,
"redrive_policy":
json.dumps({
"maxReceiveCount":
10,
"deadLetterTargetArn":
"${aws_sqs_queue.%s.arn}" %
config.unqual_notifications_queue_name(fail=True)
})
},
**{
config.unqual_tallies_queue_name(retry=retry): {
"name":
config.tallies_queue_name(retry=retry),
"fifo_queue":
True,
"delay_seconds":
config.es_refresh_interval + 9,
"visibility_timeout_seconds":
config.aggregation_lambda_timeout(retry=retry) + 10,
"message_retention_seconds":
24 * 60 * 60,
"redrive_policy":
json.dumps({
"maxReceiveCount":
9 if retry else 1,
"deadLetterTargetArn":
"${aws_sqs_queue.%s.arn}" % config.unqual_tallies_queue_name(retry=not retry,
fail=retry)
})
}
for retry in (False, True)
},
config.unqual_notifications_queue_name(fail=True): {
"name": config.notifications_queue_name(fail=True),
"message_retention_seconds": 14 * 24 * 60 * 60,
},
config.unqual_tallies_queue_name(fail=True): {
"fifo_queue": True,
"name": config.tallies_queue_name(fail=True),
"message_retention_seconds": 14 * 24 * 60 * 60,
}
}
}]
})
emit_tf(None if config.share_es_domain else {
"resource": [
*({
"aws_cloudwatch_log_group": {
f"{log}_log": {
"name": f"/aws/aes/domains/{domain}/{log}-logs",
"retention_in_days": 30 if log == 'error' else 1827
}
}
} for log in logs.keys()),
{
"aws_cloudwatch_log_resource_policy": {
"index": {
"policy_name": domain,
"policy_document": json.dumps(
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "es.amazonaws.com"
},
"Action": [
"logs:PutLogEvents",
"logs:CreateLogStream"
],
"Resource": [
"${aws_cloudwatch_log_group." + log + "_log.arn}" for log in logs.keys()
]
}
]
}
)
}
}
},
{
"aws_elasticsearch_domain": {
"index": {
"access_policies": json.dumps({
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::${local.account_id}:root"
},
"Action": "es:*",
"Resource": "arn:aws:es:${local.region}:${local.account_id}:domain/" + domain + "/*"
},
{
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": "es:*",
"Resource": "arn:aws:es:${local.region}:${local.account_id}:domain/" + domain + "/*",
"Condition": {
"IpAddress": {
"aws:SourceIp": []
}
}
}
]
}),
"advanced_options": {
"rest.action.multi.allow_explicit_index": "true"
},
"cluster_config": {
"instance_count": config.es_instance_count,
"instance_type": config.es_instance_type
},
"domain_name": domain,
"ebs_options": {
"ebs_enabled": "true",
"volume_size": config.es_volume_size,
"volume_type": "gp2"
},
"elasticsearch_version": "6.8",
"log_publishing_options": [
{
"cloudwatch_log_group_arn": "${aws_cloudwatch_log_group." + log + "_log.arn}",
"enabled": "true" if enabled else "false",
"log_type": log_type
} for log, (log_type, enabled) in logs.items()
],
"snapshot_options": {
"automated_snapshot_start_hour": 8 # midnight PST
}
}
}
} if domain else {}
]
})
emit_tf({} if config.terraform_component != 'gitlab' else {
"data": {
"aws_availability_zones": {
"available": {}
},
"aws_ebs_volume": {
"gitlab": {
"filter": [{
"name": "volume-type",
"values": ["gp2"]
}, {
"name": "tag:Name",
"values": [ebs_volume_name]
}],
"most_recent":
True
}
},
# This Route53 zone also has to exist.
"aws_route53_zone": {
"gitlab": {
"name": config.domain_name + ".",
"private_zone": False
}
},
"aws_ami": {
"rancheros": {
"owners": ['605812595337'],
"filter": [{
"name": "name",
"values": ["rancheros-v1.4.2-hvm-1"]
}]
}
},
"aws_iam_policy_document": {
# This policy is really close to the policy size limit, if you get LimitExceeded: Cannot exceed quota for
# PolicySize: 6144, you need to strip the existing policy down by essentially replacing the calls to the
# helper functions like allow_service() with a hand-curated list of actions, potentially by starting from
# a copy of the template output.
"gitlab_boundary": {
"statement": [
allow_global_actions(
'S3',
types={ServiceActionType.read, ServiceActionType.
list}),
{
"actions":
aws_service_actions('S3'),
"resources":
merge(
aws_service_arns(
'S3', BucketName=bucket_name, ObjectName='*')
for bucket_name in ([
'edu-ucsc-gi-singlecell-azul-*',
'*.url.singlecell.gi.ucsc.edu',
'url.singlecell.gi.ucsc.edu'
] if 'singlecell' in config.domain_name else [
'edu-ucsc-gi-azul-*',
'*.azul.data.humancellatlas.org',
]))
},
*allow_service('KMS',
action_types={
ServiceActionType.
read, ServiceActionType.list
},
KeyId='*',
Alias='*'),
*allow_service('SQS', QueueName='azul-*'),
# API Gateway ARNs refer to APIs by ID so we cannot restrict to name or prefix
*allow_service('API Gateway', ApiGatewayResourcePath="*"),
*allow_service('Elasticsearch Service',
global_action_types={
ServiceActionType.
read, ServiceActionType.list
},
DomainName="azul-*"),
{
'actions': ['es:ListTags'],
'resources':
aws_service_arns('Elasticsearch Service',
DomainName='*')
},
*allow_service('STS',
action_types={
ServiceActionType.
read, ServiceActionType.list
},
RelativeId='*',
RoleNameWithPath='*',
UserNameWithPath='*'),
dss_direct_access_policy_statement,
*allow_service(
'Certificate Manager',
# ACM ARNs refer to certificates by ID so we cannot restrict to name or prefix
CertificateId='*',
# API Gateway certs must reside in us-east-1, so we'll always add that region
Region={aws.region_name, 'us-east-1'}),
*allow_service('DynamoDB',
'table',
'index',
global_action_types={
ServiceActionType.
list, ServiceActionType.read
},
TableName='azul-*',
IndexName='*'),
# Lambda ARNs refer to event source mappings by UUID so we cannot restrict to name or prefix
*allow_service('Lambda',
LayerName="azul-*",
FunctionName='azul-*',
UUID='*',
LayerVersion='*'),
# CloudWatch does not describe any resource-level permissions
{
"actions": ["cloudwatch:*"],
"resources": ["*"]
},
*allow_service('CloudWatch Events',
global_action_types={
ServiceActionType.
list, ServiceActionType.read
},
RuleName='azul-*'),
# Route 53 ARNs refer to resources by ID so we cannot restrict to name or prefix
# FIXME: this is obviously problematic
{
"actions": ["route53:*"],
"resources": ["*"]
},
# Secret Manager ARNs refer to secrets by UUID so we cannot restrict to name or prefix
# FIXME: this is obviously problematic
*allow_service('Secrets Manager', SecretId='*'),
{
"actions": ['ssm:GetParameter'],
"resources":
aws_service_arns(
'Systems Manager',
'parameter',
FullyQualifiedParameterName='dcp/dss/*')
},
{
"actions": ["states:*"],
"resources":
aws_service_arns('Step Functions',
'execution',
'statemachine',
StateMachineName='azul-*',
ExecutionId='*')
},
{
"actions": [
"states:ListStateMachines",
"states:CreateStateMachine"
],
"resources": ["*"]
},
# CloudFront does not define any ARNs. We need it for friendly domain names for API Gateways
{
"actions": ["cloudfront:*"],
"resources": ["*"]
},
allow_global_actions('CloudWatch Logs'),
{
"actions":
aws_service_actions('CloudWatch Logs',
types={ServiceActionType.list}),
"resources":
aws_service_arns('CloudWatch Logs',
LogGroupName='*',
LogStream='*',
LogStreamName='*')
},
{
"actions":
aws_service_actions('CloudWatch Logs'),
"resources":
merge(
aws_service_arns('CloudWatch Logs',
LogGroupName=log_group_name,
LogStream='*',
LogStreamName='*')
for log_group_name in [
'/aws/apigateway/azul-*', '/aws/lambda/azul-*',
'/aws/aes/domains/azul-*'
])
}
]
},
"gitlab_iam": {
"statement": [
# Let Gitlab manage roles as long as they specify the permissions boundary
# This prevent privilege escalation.
{
"actions": [
"iam:CreateRole", "iam:PutRolePolicy",
"iam:DeleteRolePolicy", "iam:AttachRolePolicy",
"iam:DetachRolePolicy",
"iam:PutRolePermissionsBoundary"
],
"resources":
aws_service_arns(
'IAM', 'role', RoleNameWithPath='azul-*'),
"condition": {
"test": "StringEquals",
"variable": "iam:PermissionsBoundary",
"values": [aws.permissions_boundary_arn]
}
},
dss_direct_access_policy_statement,
{
"actions": [
"iam:UpdateAssumeRolePolicy",
"iam:DeleteRole",
"iam:PassRole" # FIXME: consider iam:PassedToService condition
],
"resources":
aws_service_arns(
'IAM', 'role', RoleNameWithPath='azul-*')
},
{
"actions":
aws_service_actions('IAM',
types={
ServiceActionType.
read, ServiceActionType.list
}),
"resources": ["*"]
},
*(
# Permissions required to deploy Data Browser and Portal
[{
"actions": [
"s3:PutObject", "s3:GetObject",
"s3:ListBucket", "s3:DeleteObject",
"s3:PutObjectAcl"
],
"resources": [
"arn:aws:s3:::dev.singlecell.gi.ucsc.edu/*",
"arn:aws:s3:::dev.explore.singlecell.gi.ucsc.edu/*",
"arn:aws:s3:::dev.explore.singlecell.gi.ucsc.edu",
"arn:aws:s3:::dev.singlecell.gi.ucsc.edu"
]
}, {
"actions": ["cloudfront:CreateInvalidation"],
"resources": [
"arn:aws:cloudfront::122796619775:distribution/E3562WJBOLN8W8"
]
}] if config.domain_name ==
'dev.singlecell.gi.ucsc.edu' else [])
]
}
},
},
"resource": {
"aws_vpc": {
"gitlab": {
"cidr_block": vpc_cidr,
"tags": {
"Name": "azul-gitlab"
}
}
},
"aws_subnet": { # a public and a private subnet per availability zone
f"gitlab_{subnet_name(public)}_{zone}": {
"availability_zone":
f"${{data.aws_availability_zones.available.names[{zone}]}}",
"cidr_block":
f"${{cidrsubnet(aws_vpc.gitlab.cidr_block, 8, {subnet_number(zone, public)})}}",
"map_public_ip_on_launch": public,
"vpc_id": "${aws_vpc.gitlab.id}",
"tags": {
"Name":
f"azul-gitlab-{subnet_name(public)}-{subnet_number(zone, public)}"
}
}
for public in (False, True) for zone in range(num_zones)
},
"aws_internet_gateway": {
"gitlab": {
"vpc_id": "${aws_vpc.gitlab.id}",
"tags": {
"Name": "azul-gitlab"
}
}
},
"aws_route": {
"gitlab": {
"destination_cidr_block": "0.0.0.0/0",
"gateway_id": "${aws_internet_gateway.gitlab.id}",
"route_table_id": "${aws_vpc.gitlab.main_route_table_id}"
}
},
"aws_eip": {
f"gitlab_{zone}": {
"depends_on": ["aws_internet_gateway.gitlab"],
"vpc": True,
"tags": {
"Name": f"azul-gitlab-{zone}"
}
}
for zone in range(num_zones)
},
"aws_nat_gateway": {
f"gitlab_{zone}": {
"allocation_id": f"${{aws_eip.gitlab_{zone}.id}}",
"subnet_id": f"${{aws_subnet.gitlab_public_{zone}.id}}",
"tags": {
"Name": f"azul-gitlab-{zone}"
}
}
for zone in range(num_zones)
},
"aws_route_table": {
f"gitlab_{zone}": {
"route": [{
"cidr_block": "0.0.0.0/0",
"nat_gateway_id": f"${{aws_nat_gateway.gitlab_{zone}.id}}",
"egress_only_gateway_id": None,
"gateway_id": None,
"instance_id": None,
"ipv6_cidr_block": None,
"network_interface_id": None,
"transit_gateway_id": None,
"vpc_peering_connection_id": None
}],
"vpc_id":
"${aws_vpc.gitlab.id}",
"tags": {
"Name": f"azul-gitlab-{zone}"
}
}
for zone in range(num_zones)
},
"aws_route_table_association": {
f"gitlab_{zone}": {
"route_table_id": f"${{aws_route_table.gitlab_{zone}.id}}",
"subnet_id": f"${{aws_subnet.gitlab_private_{zone}.id}}"
}
for zone in range(num_zones)
},
"aws_security_group": {
"gitlab_alb": {
"name":
"azul-gitlab-alb",
"vpc_id":
"${aws_vpc.gitlab.id}",
"egress": [{
**ingress_egress_block, "cidr_blocks": ["0.0.0.0/0"],
"protocol": -1,
"from_port": 0,
"to_port": 0
}],
"ingress": [
{
**ingress_egress_block, "cidr_blocks": ["0.0.0.0/0"],
"protocol": "tcp",
"from_port": 443,
"to_port": 443
}, *({
**ingress_egress_block, "cidr_blocks": ["0.0.0.0/0"],
"protocol": "tcp",
"from_port": ext_port,
"to_port": ext_port
} for ext_port, int_port, name in nlb_ports)
]
},
"gitlab": {
"name":
"azul-gitlab",
"vpc_id":
"${aws_vpc.gitlab.id}",
"egress": [{
**ingress_egress_block, "cidr_blocks": ["0.0.0.0/0"],
"protocol": -1,
"from_port": 0,
"to_port": 0
}],
"ingress": [{
**ingress_egress_block,
"from_port":
80,
"protocol":
"tcp",
"security_groups": ["${aws_security_group.gitlab_alb.id}"],
"to_port":
80,
}, *({
**ingress_egress_block, "cidr_blocks": [
"0.0.0.0/0" if nlb_preserve_source_ip else
"${aws_vpc.gitlab.cidr_block}"
],
"protocol":
"tcp",
"from_port":
int_port,
"to_port":
int_port
} for ext_port, int_port, name in nlb_ports)]
}
},
"aws_lb": {
"gitlab_nlb": {
"name":
"azul-gitlab-nlb",
"load_balancer_type":
"network",
"subnets": [
f"${{aws_subnet.gitlab_public_{zone}.id}}"
for zone in range(num_zones)
],
"tags": {
"Name": "azul-gitlab"
}
},
"gitlab_alb": {
"name":
"azul-gitlab-alb",
"load_balancer_type":
"application",
"subnets": [
f"${{aws_subnet.gitlab_public_{zone}.id}}"
for zone in range(num_zones)
],
"security_groups": ["${aws_security_group.gitlab_alb.id}"],
"tags": {
"Name": "azul-gitlab"
}
}
},
"aws_lb_listener": {
**({
"gitlab_" + name: {
"port":
ext_port,
"protocol":
"TCP",
"default_action": [{
"target_group_arn":
"${aws_lb_target_group.gitlab_" + name + ".id}",
"type":
"forward"
}],
"load_balancer_arn":
"${aws_lb.gitlab_nlb.id}"
}
for ext_port, int_port, name in nlb_ports
}), "gitlab_http": {
"port":
443,
"protocol":
"HTTPS",
"ssl_policy":
"ELBSecurityPolicy-2016-08",
"certificate_arn":
"${aws_acm_certificate.gitlab.arn}",
"default_action": [{
"target_group_arn":
"${aws_lb_target_group.gitlab_http.id}",
"type": "forward"
}],
"load_balancer_arn":
"${aws_lb.gitlab_alb.id}"
}
},
"aws_lb_target_group": {
**({
"gitlab_" + name: {
"name": "azul-gitlab-" + name,
"port": int_port,
"protocol": "TCP",
"target_type": "instance" if nlb_preserve_source_ip else "ip",
"stickiness": {
"type": "lb_cookie",
"enabled": False
},
"vpc_id": "${aws_vpc.gitlab.id}"
}
for ext_port, int_port, name in nlb_ports
}), "gitlab_http": {
"name": "azul-gitlab-http",
"port": 80,
"protocol": "HTTP",
"target_type": "instance",
"stickiness": {
"type": "lb_cookie",
"enabled": False
},
"vpc_id": "${aws_vpc.gitlab.id}",
"health_check": {
"protocol": "HTTP",
"path": "/",
"port": "traffic-port",
"healthy_threshold": 5,
"unhealthy_threshold": 2,
"timeout": 5,
"interval": 30,
"matcher": "302"
},
"tags": {
"Name": "azul-gitlab-http"
}
}
},
"aws_lb_target_group_attachment": {
**({
"gitlab_" + name: {
"target_group_arn":
"${aws_lb_target_group.gitlab_" + name + ".arn}",
"target_id":
f"${{aws_instance.gitlab.{'id' if nlb_preserve_source_ip else 'private_ip'}}}"
}
for ext_port, int_port, name in nlb_ports
}), "gitlab_http": {
"target_group_arn": "${aws_lb_target_group.gitlab_http.arn}",
"target_id": "${aws_instance.gitlab.id}"
}
},
"aws_acm_certificate": {
"gitlab": {
"domain_name":
"${aws_route53_record.gitlab.name}",
"subject_alternative_names":
["${aws_route53_record.gitlab_docker.name}"],
"validation_method":
"DNS",
"tags": {
"Name": "azul-gitlab"
},
"lifecycle": {
"create_before_destroy": True
}
}
},
"aws_acm_certificate_validation": {
"gitlab": {
"certificate_arn":
"${aws_acm_certificate.gitlab.arn}",
"validation_record_fqdns": [
"${aws_route53_record.gitlab_validation.fqdn}",
"${aws_route53_record.gitlab_validation_docker.fqdn}"
],
}
},
"aws_route53_record": {
**dict_merge({
departition('gitlab_validation', '_', subdomain): {
"name":
f"${{aws_acm_certificate.gitlab.domain_validation_options.{i}.resource_record_name}}",
"type":
f"${{aws_acm_certificate.gitlab.domain_validation_options.{i}.resource_record_type}}",
"zone_id":
"${data.aws_route53_zone.gitlab.id}",
"records": [
f"${{aws_acm_certificate.gitlab.domain_validation_options.{i}.resource_record_value}}"
],
"ttl":
60
},
departition('gitlab', '_', subdomain): {
"zone_id":
"${data.aws_route53_zone.gitlab.id}",
"name":
departition(subdomain, '.', f"gitlab.{config.domain_name}"),
"type":
"A",
"alias": {
"name": "${aws_lb.gitlab_alb.dns_name}",
"zone_id": "${aws_lb.gitlab_alb.zone_id}",
"evaluate_target_health": False
}
}
} for i, subdomain in enumerate([None, 'docker'])), "gitlab_ssh": {
"zone_id": "${data.aws_route53_zone.gitlab.id}",
"name": f"ssh.gitlab.{config.domain_name}",
"type": "A",
"alias": {
"name": "${aws_lb.gitlab_nlb.dns_name}",
"zone_id": "${aws_lb.gitlab_nlb.zone_id}",
"evaluate_target_health": False
}
}
},
"aws_network_interface": {
"gitlab": {
"subnet_id": "${aws_subnet.gitlab_private_0.id}",
"security_groups": ["${aws_security_group.gitlab.id}"],
"tags": {
"Name": "azul-gitlab"
}
}
},
"aws_volume_attachment": {
"gitlab": {
"device_name": "/dev/sdf",
"volume_id": "${data.aws_ebs_volume.gitlab.id}",
"instance_id": "${aws_instance.gitlab.id}",
"provisioner": {
"local-exec": {
"when":
"destroy",
"command":
"aws ec2 stop-instances --instance-ids ${self.instance_id}"
" && aws ec2 wait instance-stopped --instance-ids ${self.instance_id}"
}
}
}
},
"aws_key_pair": {
"gitlab": {
"key_name": "azul-gitlab",
"public_key": public_key
}
},
"aws_iam_role": {
"gitlab": {
"name":
"azul-gitlab",
"path":
"/",
"assume_role_policy":
json.dumps({
"Version":
"2012-10-17",
"Statement": [{
"Action": "sts:AssumeRole",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Effect": "Allow",
"Sid": ""
}]
})
}
},
"aws_iam_instance_profile": {
"gitlab": {
"name": "azul-gitlab",
"role": "${aws_iam_role.gitlab.name}",
}
},
"aws_iam_policy": {
"gitlab_iam": {
"name": "azul-gitlab-iam",
"path": "/",
"policy": "${data.aws_iam_policy_document.gitlab_iam.json}"
},
"gitlab_boundary": {
"name": config.permissions_boundary_name,
"path": "/",
"policy":
"${data.aws_iam_policy_document.gitlab_boundary.json}"
}
},
"aws_iam_role_policy_attachment": {
"gitlab_iam": {
"role": "${aws_iam_role.gitlab.name}",
"policy_arn": "${aws_iam_policy.gitlab_iam.arn}"
},
# Since we are using the boundary as a policy Gitlab can explicitly
# do everything within the boundary
"gitlab_boundary": {
"role": "${aws_iam_role.gitlab.name}",
"policy_arn": "${aws_iam_policy.gitlab_boundary.arn}"
}
},
"google_service_account": {
"gitlab": {
"project": "${local.google_project}",
"account_id": name,
"display_name": name,
}
for name in [
"azul-gitlab-sc" if
(os.environ["GOOGLE_PROJECT"] == "human-cell-atlas-travis-test"
and "singlecell" in config.domain_name) else "azul-gitlab"
]
},
"google_project_iam_member": {
"gitlab_" + name: {
"project": "${local.google_project}",
"role": role,
"member":
"serviceAccount:${google_service_account.gitlab.email}"
}
for name, role in [("write",
"${google_project_iam_custom_role.gitlab.id}"
), ("read", "roles/viewer")]
},
"google_project_iam_custom_role": {
"gitlab": {
"role_id":
"azul_gitlab",
"title":
"azul_gitlab",
"permissions": [
"resourcemanager.projects.setIamPolicy", *[
f"iam.{resource}.{operation}"
for operation in ("create", "delete", "get", "list",
"update", "undelete")
for resource in ("roles", "serviceAccountKeys",
"serviceAccounts")
if resource != "serviceAccountKeys" or operation not in
("update", "undelete")
]
]
}
},
"aws_instance": {
"gitlab": {
"iam_instance_profile":
"${aws_iam_instance_profile.gitlab.name}",
"ami":
"${data.aws_ami.rancheros.id}",
"instance_type":
"t3a.xlarge",
"key_name":
"${aws_key_pair.gitlab.key_name}",
"network_interface": {
"network_interface_id":
"${aws_network_interface.gitlab.id}",
"device_index": 0
},
"user_data":
dedent(
rf"""
#cloud-config
mounts:
- ["/dev/nvme1n1", "/mnt/gitlab", "ext4", ""]
rancher:
ssh_authorized_keys: {other_public_keys}
write_files:
- path: /etc/rc.local
permissions: "0755"
owner: root
content: |
#!/bin/bash
wait-for-docker
docker network \
create gitlab-runner-net
docker run \
--detach \
--name gitlab-dind \
--privileged \
--restart always \
--network gitlab-runner-net \
--volume /mnt/gitlab/docker:/var/lib/docker \
--volume /mnt/gitlab/runner/config:/etc/gitlab-runner \
docker:18.03.1-ce-dind
docker run \
--detach \
--name gitlab \
--hostname ${{aws_route53_record.gitlab.name}} \
--publish 80:80 \
--publish 2222:22 \
--restart always \
--volume /mnt/gitlab/config:/etc/gitlab \
--volume /mnt/gitlab/logs:/var/log/gitlab \
--volume /mnt/gitlab/data:/var/opt/gitlab \
gitlab/gitlab-ce:13.4.3-ce.0
docker run \
--detach \
--name gitlab-runner \
--restart always \
--volume /mnt/gitlab/runner/config:/etc/gitlab-runner \
--network gitlab-runner-net \
--env DOCKER_HOST=tcp://gitlab-dind:2375 \
gitlab/gitlab-runner:v13.2.4
"""[1:]
), # trim newline char at the beginning as dedent() only removes indent common to all lines
"tags": {
"Name": "azul-gitlab",
"Owner": config.owner
}
}
}
}
})
emit_tf({
"resource": [{
"aws_dynamodb_table": {
"users": {
"name": config.dynamo_user_table_name,
"billing_mode": "PAY_PER_REQUEST",
"hash_key": "UserId",
"attribute": [{
"name": "UserId",
"type": "S"
}]
},
"carts": {
"name":
config.dynamo_cart_table_name,
"billing_mode":
"PAY_PER_REQUEST",
"hash_key":
"UserId",
"range_key":
"CartId",
"attribute": [{
"name": "CartId",
"type": "S"
}, {
"name": "UserId",
"type": "S"
}, {
"name": "CartName",
"type": "S"
}],
"global_secondary_index": [{
"name": "UserIndex",
"hash_key": "UserId",
"projection_type": "ALL"
}, {
"name": "UserCartNameIndex",
"hash_key": "UserId",
"range_key": "CartName",
"projection_type": "ALL"
}]
},
"cart_items": {
"name":
config.dynamo_cart_item_table_name,
"billing_mode":
"PAY_PER_REQUEST",
"hash_key":
"CartId",
"range_key":
"CartItemId",
"attribute": [{
"name": "CartItemId",
"type": "S"
}, {
"name": "CartId",
"type": "S"
}]
},
"object_versions": {
"name": config.dynamo_object_version_table_name,
"billing_mode": "PAY_PER_REQUEST",
"hash_key": VersionService.key_name,
"attribute": [{
"name": VersionService.key_name,
"type": "S"
}]
}
}
}]
})
from azul import (
config,
)
from azul.deployment import (
aws,
emit_tf,
)
emit_tf({
"module": {
# Not using config.project_root because, "A local path must begin with
# either ./ or ../"
# https://www.terraform.io/docs/modules/sources.html#local-paths
f"chalice_{lambda_name}": {
"source": f"./{lambda_name}",
"role_arn": "${aws_iam_role." + lambda_name + ".arn}",
"layer_arn": "${aws_lambda_layer_version.dependencies.arn}",
"es_endpoint":
aws.es_endpoint
if config.share_es_domain else
("${aws_elasticsearch_domain.index.endpoint}", 443),
"es_instance_count":
aws.es_instance_count
if config.share_es_domain else
"${aws_elasticsearch_domain.index.cluster_config[0].instance_count}",
} for lambda_name in config.lambda_names()
}
})
emit_tf(
None if config.disable_monitoring else {
"resource": [
*[
{
"aws_route53_health_check": {
name: {
"fqdn": config.api_lambda_domain(name),
"port": 443,
"type": "HTTPS",
"resource_path": "/health/cached",
"failure_threshold": "3",
"request_interval": "30",
"tags": {
"Name": full_name
},
"regions": ['us-west-2', 'us-east-1', 'eu-west-1'],
"measure_latency": True,
# This is necessary only because of a Terraform bug:
# https://github.com/hashicorp/terraform/issues/22171
"lifecycle": {
"create_before_destroy": True
}
},
}
} for name, full_name in (('indexer', config.indexer_name),
('service', config.service_name))
],
{
"aws_route53_health_check": {
"composite-azul": {
# NOTE: There is a 64 character limit on reference name. Terraform adds long string at the end so
# we must be economical about what we add.
"reference_name":
f"azul-{config.deployment_stage}",
"type":
"CALCULATED",
"child_health_threshold":
2,
"child_healthchecks": [
"${aws_route53_health_check." + "indexer" +
".id}", "${aws_route53_health_check." + "service" +
".id}"
],
"measure_latency":
True,
"cloudwatch_alarm_region":
aws.region_name,
"tags": {
"Name": f"azul-composite-{config.deployment_stage}"
}
}
}
},
*[
{
"aws_route53_health_check": {
name: {
"fqdn": domain,
"port": 443,
"type": "HTTPS",
"resource_path": path,
"failure_threshold": "3",
"request_interval": "30",
"tags": {
"Name": full_name
},
"measure_latency": True,
# This is necessary only because of a Terraform bug:
# https://github.com/hashicorp/terraform/issues/22171
"lifecycle": {
"create_before_destroy": True
}
}
}
} for name, domain, full_name, path in (
("data-browser", config.data_browser_domain,
config.data_browser_name, '/explore'),
("data-portal", config.data_browser_domain,
config.data_portal_name, '/'))
],
{
"aws_route53_health_check": {
"composite-portal": {
"reference_name":
f"portal-{config.deployment_stage}",
"type":
"CALCULATED",
"child_health_threshold":
2,
"child_healthchecks": [
"${aws_route53_health_check." + "data-browser" +
".id}", "${aws_route53_health_check." +
"data-portal" + ".id}"
],
"measure_latency":
True,
"cloudwatch_alarm_region":
aws.region_name,
"tags": {
"Name":
f"azul-portal-composite-{config.deployment_stage}"
}
}
}
}
]
})
emit_tf({
"provider":
[{
"null": {
'version': "~> 2.1"
}
}, {
"google": {
'version': "~> 2.10"
}
}, *({
"aws": {
'version':
"~> 2.20",
**({
'region': region,
'alias': region
} if region else {}),
**({
'profile': aws.profile['source_profile'],
'assume_role': {
'role_arn': aws.profile['role_arn']
}
} if 'role_arn' in aws.profile else {})
}
} for region in (None, 'us-east-1'))
# Generate a default `aws` provider and one that pins the region for the certificates of the API Gateway
# custom domain names. Certificates of edge-optimized custom domain names have to reside in us-east-1.
]
})
emit_tf({
"resource": [
{
"google_service_account": {
"azul": {
"project":
"${local.google_project}",
"account_id":
config.google_service_account,
"display_name":
config.google_service_account,
"description":
f"Azul service account in {config.deployment_stage}",
"provisioner": [{
"local-exec": {
"command":
' '.join(
map(shlex.quote, [
"python",
config.project_root +
"/scripts/provision_credentials.py",
"google-key",
"--build",
"${self.email}",
]))
}
}, {
"local-exec": {
"when":
"destroy",
"command":
' '.join(
map(shlex.quote, [
"python",
config.project_root +
"/scripts/provision_credentials.py",
"google-key",
"--destroy",
"${self.email}",
]))
}
}]
}
},
"google_project_iam_member": {
"azul": {
"project": "${local.google_project}",
"role": "${google_project_iam_custom_role.azul.id}",
"member":
"serviceAccount:${google_service_account.azul.email}"
}
},
"google_project_iam_custom_role": {
"azul": {
"role_id":
f"azul_{config.deployment_stage}",
"title":
f"azul_{config.deployment_stage}",
"permissions": ["bigquery.jobs.create"]
if config.is_tdr_enabled() else []
}
},
"null_resource": {
"hmac_secret": {
"provisioner": [{
"local-exec": {
"command":
' '.join(
map(shlex.quote, [
"python",
config.project_root +
"/scripts/provision_credentials.py",
"hmac-key",
"--build",
]))
}
}, {
"local-exec": {
"when":
"destroy",
"command":
' '.join(
map(shlex.quote, [
"python",
config.project_root +
"/scripts/provision_credentials.py",
"hmac-key",
"--destroy",
]))
}
}]
}
}
},
]
})
from azul import (
config, )
from azul.deployment import (
emit_tf, )
from azul.lambda_layer import (
DependenciesLayer, )
layer = DependenciesLayer()
emit_tf({
"resource": [{
"aws_lambda_layer_version": {
"dependencies": {
"layer_name": config.qualified_resource_name("dependencies"),
"s3_bucket": config.lambda_layer_bucket,
"s3_key": layer.object_key
}
}
}],
})
emit_tf(
None if config.disable_monitoring else {
"resource": [
*([] if config.share_es_domain else [{
"aws_cloudwatch_metric_alarm": {
"CPUUtilization": {
"alarm_name":
config.es_domain + "-CPUUtilization",
"actions_enabled":
True,
"comparison_operator":
"GreaterThanOrEqualToThreshold",
"evaluation_periods":
"2",
"metric_name":
"CPUUtilization",
"namespace":
"AWS/ES",
"period":
"3600",
"statistic":
"Average",
"threshold":
"85",
"alarm_description":
json.dumps({
"slack_channel":
"dcp-ops-alerts",
"description":
config.es_domain + " CPUUtilization alarm"
}),
"dimensions": {
"ClientId": aws.account,
"DomainName": config.es_domain
},
"alarm_actions": [
f"arn:aws:sns:{aws.region_name}:{aws.account}:cloudwatch-alarms",
f"arn:aws:sns:{aws.region_name}:{aws.account}:dcp-events"
],
"ok_actions": [
f"arn:aws:sns:{aws.region_name}:{aws.account}:cloudwatch-alarms",
f"arn:aws:sns:{aws.region_name}:{aws.account}:dcp-events"
],
}
}
}, {
"aws_cloudwatch_metric_alarm": {
"FreeStorageSpace": {
"alarm_name":
config.es_domain + "-FreeStorageSpace",
"actions_enabled":
True,
"comparison_operator":
"LessThanOrEqualToThreshold",
"evaluation_periods":
"1",
"metric_name":
"FreeStorageSpace",
"namespace":
"AWS/ES",
"period":
"300",
"statistic":
"Average",
"threshold":
"14000",
"alarm_description":
json.dumps({
"slack_channel":
"dcp-ops-alerts",
"description":
config.es_domain + " FreeStorageSpace alarm"
}),
"dimensions": {
"ClientId": aws.account,
"DomainName": config.es_domain
},
"alarm_actions": [
f"arn:aws:sns:{aws.region_name}:{aws.account}:cloudwatch-alarms",
f"arn:aws:sns:{aws.region_name}:{aws.account}:dcp-events"
],
"ok_actions": [
f"arn:aws:sns:{aws.region_name}:{aws.account}:cloudwatch-alarms",
f"arn:aws:sns:{aws.region_name}:{aws.account}:dcp-events"
],
}
}
}, {
"aws_cloudwatch_metric_alarm": {
"JVMMemoryPressure": {
"alarm_name":
config.es_domain + "-JVMMemoryPressure",
"actions_enabled":
True,
"comparison_operator":
"GreaterThanOrEqualToThreshold",
"evaluation_periods":
"1",
"metric_name":
"JVMMemoryPressure",
"namespace":
"AWS/ES",
"period":
"300",
"statistic":
"Minimum",
"threshold":
"65",
"alarm_description":
json.dumps({
"slack_channel":
"dcp-ops-alerts",
"description":
config.es_domain + " JVMMemoryPressure alarm"
}),
"dimensions": {
"ClientId": aws.account,
"DomainName": config.es_domain
},
"alarm_actions": [
f"arn:aws:sns:{aws.region_name}:{aws.account}:cloudwatch-alarms",
f"arn:aws:sns:{aws.region_name}:{aws.account}:dcp-events"
],
"ok_actions": [
f"arn:aws:sns:{aws.region_name}:{aws.account}:cloudwatch-alarms",
f"arn:aws:sns:{aws.region_name}:{aws.account}:dcp-events"
],
}
}
}]),
{
"aws_cloudwatch_metric_alarm": {
"azul_health": {
"alarm_name":
f"azul-{config.deployment_stage}",
"actions_enabled":
True,
"comparison_operator":
"LessThanThreshold",
"evaluation_periods":
"1",
"metric_name":
"HealthCheckStatus",
"namespace":
"AWS/Route53",
"period":
"120",
"statistic":
"Minimum",
"threshold":
"1.0",
"alarm_description":
json.dumps({
"slack_channel":
"azul-dev",
"environment":
config.deployment_stage,
"description":
f"azul-{config.deployment_stage} HealthCheckStatus alarm"
}),
"alarm_actions": [
f"arn:aws:sns:{aws.region_name}:{aws.account}:cloudwatch-alarms",
f"arn:aws:sns:{aws.region_name}:{aws.account}:dcp-events"
],
"ok_actions": [
f"arn:aws:sns:{aws.region_name}:{aws.account}:cloudwatch-alarms",
f"arn:aws:sns:{aws.region_name}:{aws.account}:dcp-events"
],
"dimensions": {
"HealthCheckId":
"${aws_route53_health_check.composite-azul.id}",
}
},
"data_portal_health": {
"alarm_name":
f"data-browser-{config.deployment_stage}",
"actions_enabled":
True,
"comparison_operator":
"LessThanThreshold",
"evaluation_periods":
"1",
"metric_name":
"HealthCheckStatus",
"namespace":
"AWS/Route53",
"period":
"120",
"statistic":
"Minimum",
"threshold":
"1.0",
"alarm_description":
json.dumps({
"slack_channel":
"data-browser",
"environment":
config.deployment_stage,
"description":
f"data-browser-{config.deployment_stage} HealthCheckStatus alarm"
}),
"alarm_actions": [
f"arn:aws:sns:{aws.region_name}:{aws.account}:cloudwatch-alarms",
f"arn:aws:sns:{aws.region_name}:{aws.account}:dcp-events"
],
"ok_actions": [
f"arn:aws:sns:{aws.region_name}:{aws.account}:cloudwatch-alarms",
f"arn:aws:sns:{aws.region_name}:{aws.account}:dcp-events"
],
"dimensions": {
"HealthCheckId":
"${aws_route53_health_check.composite-portal.id}",
}
},
**{
f"{queue.replace('.', '-')}-queue": {
"alarm_name":
f"{queue}-message-count",
"actions_enabled":
True,
"comparison_operator":
"GreaterThanThreshold",
"evaluation_periods":
"1",
"metric_name":
"ApproximateNumberOfMessagesVisible",
"namespace":
"AWS/SQS",
"period":
"300", # SQS pushes metrics at most every 5 min, lower periods wouldn't make sense
"statistic":
"Maximum",
"threshold":
"0.0",
"alarm_description":
json.dumps({
"slack_channel":
"azul-dev",
"environment":
config.deployment_stage,
"description":
f"{queue} ApproximateNumberOfMessagesVisible alarm"
}),
"dimensions": {
"QueueName": queue
},
"alarm_actions": [
f"arn:aws:sns:{aws.region_name}:{aws.account}:cloudwatch-alarms", f"arn:aws:sns:{aws.region_name}:{aws.account}:dcp-events"
],
"ok_actions": [
f"arn:aws:sns:{aws.region_name}:{aws.account}:cloudwatch-alarms", f"arn:aws:sns:{aws.region_name}:{aws.account}:dcp-events"
],
}
for queue in config.fail_queue_names
}
}
}
]
})
