VersionService, )
emit_tf({
"resource": [{
"aws_dynamodb_table": {
"object_versions": {
"name": config.dynamo_object_version_table_name,
"billing_mode": "PAY_PER_REQUEST",
"hash_key": VersionService.key_name,
"attribute": [{
"name": VersionService.key_name,
"type": "S"
}]
},
"sources_cache_by_auth": {
"name": config.dynamo_sources_cache_table_name,
"billing_mode": "PAY_PER_REQUEST",
"hash_key": SourceService.key_attribute,
"attribute": [{
"name": SourceService.key_attribute,
"type": "S"
}],
"ttl": {
"attribute_name": SourceService.ttl_attribute,
"enabled": True
}
}
}
}]
})
emit_tf(
None if config.disable_monitoring else {
"output": {
"grafana_dashboard_azul": {
"sensitive":
True,
"value":
json.dumps({
"annotations": {
"list": [{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": True,
"hide": True,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}]
},
"editable":
True,
"gnetId":
None,
"graphTooltip":
0,
"id":
None,
"links": [],
"panels": [{
"cacheTimeout":
None,
"colorBackground":
True,
"colorValue":
False,
"colors":
["#bf1b00", "#629e51", "rgba(237, 129, 40, 0.89)"],
"datasource":
"account-cloudwatch",
"format":
"none",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": False,
"thresholdLabels": False,
"thresholdMarkers": True
},
"gridPos": {
"h": 10,
"w": 6,
"x": 0,
"y": 0
},
"id":
4,
"interval":
None,
"links": [],
"mappingType":
1,
"mappingTypes": [{
"name": "value to text",
"value": 1
}, {
"name": "range to text",
"value": 2
}],
"maxDataPoints":
100,
"NonePointMode":
"connected",
"NoneText":
None,
"postfix":
"",
"postfixFontSize":
"50%",
"prefix":
"AZUL",
"prefixFontSize":
"120%",
"rangeMaps": [{
"from": "None",
"text": "N/A",
"to": "None"
}],
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": False,
"lineColor": "rgb(31, 120, 193)",
"show": True
},
"tableColumn":
"",
"targets": [{
"dimensions": {
"HealthCheckId":
"${aws_route53_health_check.composite-azul.id}",
},
"highResolution": False,
"metricName": "HealthCheckStatus",
"namespace": "AWS/Route53",
"period": "",
"refId": "A",
"region": aws.region_name,
"statistics": ["Minimum"]
}],
"thresholds":
"0.5",
"title":
"Azul Health",
"type":
"singlestat",
"valueFontSize":
"120%",
"valueMaps": [{
"op": "=",
"text": "OK",
"value": "1"
}, {
"op": "=",
"text": "ERR",
"value": "0"
}],
"valueName":
"current"
}, {
"aliasColors": {},
"bars":
False,
"dashLength":
10,
"dashes":
False,
"datasource":
"account-cloudwatch",
"fill":
1,
"gridPos": {
"h": 10,
"w": 6,
"x": 6,
"y": 0
},
"id":
14,
"legend": {
"alignAsTable": False,
"avg": False,
"current": False,
"max": True,
"min": True,
"show": True,
"total": False,
"values": True
},
"lines":
True,
"linewidth":
1,
"links": [],
"NonePointMode":
"None as zero",
"percentage":
False,
"pointradius":
5,
"points":
False,
"renderer":
"flot",
"seriesOverrides": [],
"spaceLength":
10,
"stack":
False,
"steppedLine":
False,
"targets": [{
"dimensions": {
"ClientId": aws.account,
"DomainName": config.es_domain
},
"highResolution": True,
"metricName": "CPUUtilization",
"namespace": "AWS/ES",
"period": "",
"refId": "A",
"region": "default",
"statistics": ["Average"]
}, {
"dimensions": {
"ClientId": aws.account,
"DomainName": config.es_domain
},
"highResolution": True,
"metricName": "JVMMemoryPressure",
"namespace": "AWS/ES",
"period": "",
"refId": "B",
"region": "default",
"statistics": ["Average"]
}],
"thresholds": [],
"timeFrom":
None,
"timeRegions": [],
"timeShift":
None,
"title":
"ES - CPU",
"tooltip": {
"shared": True,
"sort": 0,
"value_type": "individual"
},
"type":
"graph",
"xaxis": {
"buckets": None,
"mode": "time",
"name": None,
"show": True,
"values": []
},
"yaxes": [{
"format": "percent",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}, {
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}],
"yaxis": {
"align": False,
"alignLevel": None
}
}, {
"aliasColors": {},
"bars":
False,
"dashLength":
10,
"dashes":
False,
"datasource":
"account-cloudwatch",
"fill":
1,
"gridPos": {
"h": 10,
"w": 6,
"x": 12,
"y": 0
},
"id":
16,
"legend": {
"alignAsTable": True,
"avg": False,
"current": False,
"max": True,
"min": True,
"show": True,
"total": False,
"values": True
},
"lines":
True,
"linewidth":
1,
"links": [],
"NonePointMode":
"None as zero",
"percentage":
False,
"pointradius":
5,
"points":
False,
"renderer":
"flot",
"seriesOverrides": [],
"spaceLength":
10,
"stack":
False,
"steppedLine":
False,
"targets": [{
"dimensions": {
"ClientId": aws.account,
"DomainName": config.es_domain
},
"highResolution": True,
"metricName": "ReadLatency",
"namespace": "AWS/ES",
"period": "",
"refId": "A",
"region": "default",
"statistics": ["Average"]
}, {
"dimensions": {
"ClientId": aws.account,
"DomainName": config.es_domain
},
"highResolution": True,
"metricName": "WriteLatency",
"namespace": "AWS/ES",
"period": "",
"refId": "B",
"region": "default",
"statistics": ["Average"]
}],
"thresholds": [],
"timeFrom":
None,
"timeRegions": [],
"timeShift":
None,
"title":
"ES - I/O Latency",
"tooltip": {
"shared": True,
"sort": 0,
"value_type": "individual"
},
"type":
"graph",
"xaxis": {
"buckets": None,
"mode": "time",
"name": None,
"show": True,
"values": []
},
"yaxes": [{
"format": "s",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}, {
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}],
"yaxis": {
"align": False,
"alignLevel": None
}
}, {
"alert": {
"conditions": [{
"evaluator": {
"params": [1073741824],
"type": "lt"
},
"operator": {
"type": "and"
},
"query": {
"params": ["A", "5m", "now"]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}],
"executionErrorState":
"alerting",
"frequency":
"15m",
"handler":
1,
"name":
"ES - Free Storage Space alert",
"noDataState":
"no_data",
"notifications": []
},
"aliasColors": {},
"bars":
False,
"dashLength":
10,
"dashes":
False,
"datasource":
"account-cloudwatch",
"fill":
1,
"gridPos": {
"h": 10,
"w": 6,
"x": 18,
"y": 0
},
"id":
18,
"legend": {
"alignAsTable": True,
"avg": False,
"current": False,
"max": True,
"min": True,
"show": True,
"total": False,
"values": True
},
"lines":
True,
"linewidth":
1,
"links": [],
"NonePointMode":
"None as zero",
"percentage":
False,
"pointradius":
5,
"points":
False,
"renderer":
"flot",
"seriesOverrides": [],
"spaceLength":
10,
"stack":
False,
"steppedLine":
False,
"targets": [{
"dimensions": {
"ClientId": aws.account,
"DomainName": config.es_domain
},
"highResolution": True,
"metricName": "FreeStorageSpace",
"namespace": "AWS/ES",
"period": "",
"refId": "A",
"region": "default",
"statistics": ["Minimum"]
}],
"thresholds": [{
"colorMode": "critical",
"fill": True,
"line": True,
"op": "lt",
"value": 1073741824
}],
"timeFrom":
None,
"timeRegions": [],
"timeShift":
None,
"title":
"ES - Free Storage Space",
"tooltip": {
"shared": True,
"sort": 0,
"value_type": "individual"
},
"type":
"graph",
"xaxis": {
"buckets": None,
"mode": "time",
"name": None,
"show": True,
"values": []
},
"yaxes": [{
"format": "mbytes",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}, {
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}],
"yaxis": {
"align": False,
"alignLevel": None
}
}, {
"aliasColors": {},
"bars":
False,
"dashLength":
10,
"dashes":
False,
"datasource":
"account-cloudwatch",
"decimals":
None,
"fill":
1,
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 10
},
"id":
6,
"legend": {
"alignAsTable": True,
"avg": True,
"current": True,
"hideEmpty": False,
"hideZero": False,
"max": True,
"min": True,
"show": True,
"sortDesc": True,
"total": True,
"values": True
},
"lines":
True,
"linewidth":
1,
"links": [],
"NonePointMode":
"None as zero",
"percentage":
False,
"pointradius":
1,
"points":
False,
"renderer":
"flot",
"seriesOverrides": [],
"spaceLength":
10,
"stack":
False,
"steppedLine":
True,
"targets": [{
"alias": "Total Requests",
"dimensions": {
"ApiName": config.service_name,
"Stage": config.deployment_stage
},
"highResolution": True,
"metricName": "Count",
"namespace": "AWS/ApiGateway",
"period": "",
"refId": "A",
"region": "default",
"statistics": ["Sum"]
}, {
"alias": "HTTP 4XX",
"dimensions": {
"ApiName": config.service_name,
"Stage": config.deployment_stage
},
"highResolution": True,
"metricName": "4XXError",
"namespace": "AWS/ApiGateway",
"period": "",
"refId": "B",
"region": "default",
"statistics": ["Sum"]
}, {
"alias": "HTTP 5XX",
"dimensions": {
"ApiName": config.service_name,
"Stage": config.deployment_stage
},
"highResolution": True,
"metricName": "5XXError",
"namespace": "AWS/ApiGateway",
"period": "",
"refId": "C",
"region": "default",
"statistics": ["Sum"]
}],
"thresholds": [],
"timeFrom":
None,
"timeRegions": [],
"timeShift":
None,
"title":
"Web Service API Request/Error Rates",
"tooltip": {
"shared": True,
"sort": 0,
"value_type": "individual"
},
"type":
"graph",
"xaxis": {
"buckets": None,
"mode": "time",
"name": None,
"show": True,
"values": []
},
"yaxes": [{
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}, {
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}],
"yaxis": {
"align": False,
"alignLevel": None
}
}, {
"aliasColors": {},
"bars":
False,
"dashLength":
10,
"dashes":
False,
"datasource":
"account-cloudwatch",
"decimals":
None,
"fill":
1,
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 10
},
"id":
8,
"legend": {
"alignAsTable": True,
"avg": True,
"current": True,
"hideEmpty": False,
"hideZero": False,
"max": True,
"min": True,
"show": True,
"total": True,
"values": True
},
"lines":
True,
"linewidth":
1,
"links": [],
"NonePointMode":
"None as zero",
"percentage":
False,
"pointradius":
1,
"points":
False,
"renderer":
"flot",
"seriesOverrides": [],
"spaceLength":
10,
"stack":
False,
"steppedLine":
True,
"targets": [{
"alias": "Total Requests",
"dimensions": {
"ApiName": config.indexer_name,
"Stage": config.deployment_stage
},
"highResolution": True,
"metricName": "Count",
"namespace": "AWS/ApiGateway",
"period": "",
"refId": "A",
"region": "default",
"statistics": ["Sum"]
}, {
"alias": "HTTP 4XX",
"dimensions": {
"ApiName": config.indexer_name,
"Stage": config.deployment_stage
},
"highResolution": True,
"metricName": "4XXError",
"namespace": "AWS/ApiGateway",
"period": "",
"refId": "B",
"region": "default",
"statistics": ["Sum"]
}, {
"alias": "HTTP 5XX",
"dimensions": {
"ApiName": config.indexer_name,
"Stage": config.deployment_stage
},
"highResolution": True,
"metricName": "5XXError",
"namespace": "AWS/ApiGateway",
"period": "",
"refId": "C",
"region": "default",
"statistics": ["Sum"]
}],
"thresholds": [],
"timeFrom":
None,
"timeRegions": [],
"timeShift":
None,
"title":
"Indexer API Request/Error Rates",
"tooltip": {
"shared": True,
"sort": 0,
"value_type": "individual"
},
"type":
"graph",
"xaxis": {
"buckets": None,
"mode": "time",
"name": None,
"show": True,
"values": []
},
"yaxes": [{
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}, {
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}],
"yaxis": {
"align": False,
"alignLevel": None
}
}, {
"aliasColors": {},
"bars":
False,
"dashLength":
10,
"dashes":
False,
"datasource":
"account-cloudwatch",
"fill":
1,
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 20
},
"id":
10,
"legend": {
"alignAsTable": True,
"avg": True,
"current": True,
"max": True,
"min": True,
"show": True,
"total": True,
"values": True
},
"lines":
True,
"linewidth":
1,
"links": [],
"NonePointMode":
"None as zero",
"percentage":
False,
"pointradius":
5,
"points":
False,
"renderer":
"flot",
"seriesOverrides": [],
"spaceLength":
10,
"stack":
False,
"steppedLine":
False,
"targets": [{
"alias":
"Web Service - Total Invocation Count",
"bucketAggs": [{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}],
"dimensions": {
"FunctionName": config.service_name
},
"highResolution":
True,
"metricName":
"Invocations",
"metrics": [{
"field": "select field",
"id": "1",
"type": "count"
}],
"namespace":
"AWS/Lambda",
"period":
"",
"refId":
"A",
"region":
"default",
"statistics": ["Sum"],
"timeField":
"@timestamp"
}, {
"alias": "Web Service - Total Error Count",
"dimensions": {
"FunctionName": config.service_name
},
"highResolution": True,
"metricName": "Errors",
"namespace": "AWS/Lambda",
"period": "",
"refId": "B",
"region": "default",
"statistics": ["Sum"]
}],
"thresholds": [],
"timeFrom":
None,
"timeRegions": [],
"timeShift":
None,
"title":
"Web Service Lambda",
"tooltip": {
"shared": True,
"sort": 0,
"value_type": "individual"
},
"type":
"graph",
"xaxis": {
"buckets": None,
"mode": "time",
"name": None,
"show": True,
"values": []
},
"yaxes": [{
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}, {
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}],
"yaxis": {
"align": False,
"alignLevel": None
}
}, {
"aliasColors": {},
"bars":
False,
"dashLength":
10,
"dashes":
False,
"datasource":
"account-cloudwatch",
"fill":
1,
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 20
},
"id":
12,
"legend": {
"alignAsTable": True,
"avg": True,
"current": True,
"max": True,
"min": True,
"show": True,
"total": True,
"values": True
},
"lines":
True,
"linewidth":
1,
"links": [],
"NonePointMode":
"None as zero",
"percentage":
False,
"pointradius":
5,
"points":
False,
"renderer":
"flot",
"seriesOverrides": [],
"spaceLength":
10,
"stack":
False,
"steppedLine":
False,
"targets": [{
"alias":
"Indexer - Total Invocation Count",
"bucketAggs": [{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}],
"dimensions": {
"FunctionName": config.indexer_name
},
"highResolution":
True,
"metricName":
"Invocations",
"metrics": [{
"field": "select field",
"id": "1",
"type": "count"
}],
"namespace":
"AWS/Lambda",
"period":
"",
"refId":
"A",
"region":
"default",
"statistics": ["Sum"],
"timeField":
"@timestamp"
}, {
"alias": "Indexer - Total Error Count",
"dimensions": {
"FunctionName": config.indexer_name
},
"highResolution": True,
"metricName": "Errors",
"namespace": "AWS/Lambda",
"period": "",
"refId": "B",
"region": "default",
"statistics": ["Sum"]
}],
"thresholds": [],
"timeFrom":
None,
"timeRegions": [],
"timeShift":
None,
"title":
"Indexer Lambda",
"tooltip": {
"shared": True,
"sort": 0,
"value_type": "individual"
},
"type":
"graph",
"xaxis": {
"buckets": None,
"mode": "time",
"name": None,
"show": True,
"values": []
},
"yaxes": [{
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}, {
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True
}],
"yaxis": {
"align": False,
"alignLevel": None
}
}],
"refresh":
"1m",
"schemaVersion":
16,
"style":
"dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h",
"2h", "1d"
],
"time_options": [
"5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d",
"30d"
]
},
"timezone":
"utc",
"title":
f"Azul [{config.deployment_stage.upper()}]",
"uid":
f'azul-{config.deployment_stage}',
"version":
1
})
},
"grafana_dashboard_data_portal": {
"sensitive":
True,
"value":
json.dumps({
"annotations": {
"list": [{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": True,
"hide": True,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}]
},
"editable":
True,
"gnetId":
None,
"graphTooltip":
0,
"id":
None,
"links": [],
"panels": [{
"cacheTimeout":
None,
"colorBackground":
True,
"colorValue":
False,
"colors": ["#d44a3a", "#629e51", "#c15c17"],
"datasource":
"account-cloudwatch",
"format":
"none",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": False,
"thresholdLabels": False,
"thresholdMarkers": True
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 0
},
"id":
2,
"interval":
None,
"links": [],
"mappingType":
1,
"mappingTypes": [{
"name": "value to text",
"value": 1
}, {
"name": "range to text",
"value": 2
}],
"maxDataPoints":
100,
"nullPointMode":
"connected",
"nullText":
None,
"postfix":
"",
"postfixFontSize":
"50%",
"prefix":
"DATA BROWSER & PORTAL",
"prefixFontSize":
"120%",
"rangeMaps": [{
"from": "null",
"text": "N/A",
"to": "null"
}],
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": False,
"lineColor": "rgb(31, 120, 193)",
"show": True
},
"tableColumn":
"",
"targets": [{
"dimensions": {
"HealthCheckId":
"${aws_route53_health_check.composite-portal.id}"
},
"expression": "",
"highResolution": False,
"id": "",
"metricName": "HealthCheckStatus",
"namespace": "AWS/Route53",
"period": "",
"refId": "A",
"region": "us-east-1",
"returnData": False,
"statistics": ["Minimum"]
}],
"thresholds":
"0.5",
"title":
"Data Browser & Portal Health",
"type":
"singlestat",
"valueFontSize":
"120%",
"valueMaps": [{
"op": "=",
"text": "OK",
"value": "1"
}, {
"op": "=",
"text": "ERR",
"value": "0"
}],
"valueName":
"current"
}],
"schemaVersion":
16,
"style":
"dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h",
"2h", "1d"
],
"time_options": [
"5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d",
"30d"
]
},
"timezone":
"",
"title":
f"Data Browser & Portal [{config.deployment_stage.upper()}]",
"uid":
f"data-portal-{config.deployment_stage}",
"version":
1
})
}
}
})
emit_tf({
"provider": [
{
"template": {
'version': '2.2.0'
}
},
{
"null": {
'version': '2.1.2'
}
},
{
"google": {
'version': '2.20.3'
}
},
*({
"aws": {
'version': '4.3.0',
**(
{
'region': region,
'alias': region
} if region else {
}
),
**(
{
'assume_role': {
'role_arn': aws.profile['role_arn']
}
} if 'role_arn' in aws.profile else {
}
)
}
} for region in (None, 'us-east-1'))
# Generate a default `aws` provider and one that pins the region for the certificates of the API Gateway
# custom domain names. Certificates of edge-optimized custom domain names have to reside in us-east-1.
]
})
emit_tf({
"resource": [
{
"aws_s3_bucket": {
"storage": {
"bucket": config.s3_bucket,
"force_destroy": True
},
"urls": {
"bucket": config.url_redirect_full_domain_name,
"force_destroy": not config.is_main_deployment(),
}
},
"aws_s3_bucket_lifecycle_configuration": {
"storage": {
"bucket": "${aws_s3_bucket.storage.id}",
"rule": {
"id": "manifests",
"status": "Enabled",
"filter": {
"prefix": "manifests/"
},
"expiration": {
"days": config.manifest_expiration
},
"abort_incomplete_multipart_upload": {
"days_after_initiation": 1
}
}
}
},
"aws_s3_bucket_acl": {
"storage": {
"bucket": "${aws_s3_bucket.storage.id}",
"acl": "private",
},
"urls": {
"bucket": "${aws_s3_bucket.urls.id}",
"acl": "public-read",
}
},
"aws_s3_bucket_website_configuration": {
"urls": {
"bucket": "${aws_s3_bucket.urls.id}",
"index_document": {
"suffix": "404.html"
}
}
}
},
*([{
"aws_route53_record": {
"url_redirect_record": {
"zone_id": "${data.aws_route53_zone.azul_url.zone_id}",
"name": config.url_redirect_full_domain_name,
"type": "CNAME",
"ttl": "300",
"records": ["${aws_s3_bucket_website_configuration.urls.website_endpoint}"]
}
}
}] if config.url_redirect_base_domain_name else [])
],
**({
"data": [
{
"aws_route53_zone": {
"azul_url": {
"name": config.url_redirect_base_domain_name + ".",
"private_zone": False
}
}
}
]
} if config.url_redirect_base_domain_name else {})
})
emit_tf({
"data": [{
"aws_route53_zone": {
zone.slug: {
"name": zone.name,
"private_zone": False
}
}
} for zone in set(zones_by_domain.values())],
# Note that ${} references exist to interpolate a value AND express a dependency.
"resource": [
{
"aws_api_gateway_base_path_mapping": {
f"{lambda_.name}_{i}": {
"api_id":
"${module.chalice_%s.RestAPIId}" % lambda_.name,
"stage_name":
"${module.chalice_%s.stage_name}" % lambda_.name,
"domain_name":
"${aws_api_gateway_domain_name.%s_%i.domain_name}" %
(lambda_.name, i)
}
for i, domain in enumerate(lambda_.domains)
},
"aws_api_gateway_domain_name": {
f"{lambda_.name}_{i}": {
"domain_name":
"${aws_acm_certificate.%s_%i.domain_name}" %
(lambda_.name, i),
"certificate_arn":
"${aws_acm_certificate_validation.%s_%i.certificate_arn}" %
(lambda_.name, i)
}
for i, domain in enumerate(lambda_.domains)
},
"aws_api_gateway_method_settings": {
f"{lambda_.name}_{i}": {
"rest_api_id":
"${module.chalice_%s.RestAPIId}" % lambda_.name,
"stage_name":
"${module.chalice_%s.stage_name}" % lambda_.name,
"method_path": "*/*", # every URL path, every HTTP method
"settings": {
"metrics_enabled": True,
"data_trace_enabled": config.debug == 2,
"logging_level":
"ERROR" if config.debug == 0 else "INFO"
}
}
for i, domain in enumerate(lambda_.domains)
},
"aws_acm_certificate": {
f"{lambda_.name}_{i}": {
"domain_name": domain,
"validation_method": "DNS",
"provider": "aws.us-east-1",
# I tried using SANs for the alias domains (like the DRS
# domain) but Terraform kept swapping the zones, I think
# because the order of elements in
# `aws_acm_certificate.domain_validation_options` is not
# deterministic. The alternative is to use separate certs,
# one for each domain, the main one as well as for each
# alias.
#
# Update 03/07/2022: My guess about the non-determinism was
# correct. That bug was "fixed" in Terraform by making the
# domain_validation_options a set so that elements can't be
# accessed via numeric index. The Terraform documentation
# recommends looping over the elements in that set. That's
# what we do for GitLab. To do the same here would require
# bigger refactoring that I don't think is worth it. The
# current solution works, too.
"subject_alternative_names": [],
"lifecycle": {
"create_before_destroy": True
}
}
for i, domain in enumerate(lambda_.domains)
},
"aws_acm_certificate_validation": {
f"{lambda_.name}_{i}": {
"certificate_arn":
"${aws_acm_certificate.%s_%i.arn}" % (lambda_.name, i),
"validation_record_fqdns": [
"${aws_route53_record.%s_domain_validation_%i.fqdn}" %
(lambda_.name, i)
],
"provider":
"aws.us-east-1"
}
for i, domain in enumerate(lambda_.domains)
},
"aws_route53_record": {
**{
f"{lambda_.name}_domain_validation_{i}": {
**{
# We know there is only one. See comment above.
key: "${tolist(aws_acm_certificate.%s_%i.domain_validation_options).0.resource_record_%s}" % (lambda_.name, i, key)
for key in ('name', 'type')
},
"zone_id":
"${data.aws_route53_zone.%s.id}" % zones_by_domain[domain].slug,
"records": [
# We know there is only one. See comment above.
"${tolist(aws_acm_certificate.%s_%i.domain_validation_options).0.resource_record_value}" % (lambda_.name, i)
],
"ttl":
60
}
for i, domain in enumerate(lambda_.domains)
},
**{
f"{lambda_.name}_{i}": {
"zone_id":
"${data.aws_route53_zone.%s.id}" % zones_by_domain[domain].slug,
"name":
"${aws_api_gateway_domain_name.%s_%i.domain_name}" % (lambda_.name, i),
"type":
"A",
"alias": {
"name":
"${aws_api_gateway_domain_name.%s_%i.cloudfront_domain_name}" % (lambda_.name, i),
"zone_id":
"${aws_api_gateway_domain_name.%s_%i.cloudfront_zone_id}" % (lambda_.name, i),
"evaluate_target_health":
True,
}
}
for i, domain in enumerate(lambda_.domains)
}
},
"aws_cloudwatch_log_group": {
lambda_.name: {
"name":
"/aws/apigateway/" +
config.qualified_resource_name(lambda_.name),
"retention_in_days":
1827,
}
},
"null_resource": {
f'{lambda_.name}_log_group_provisioner': {
"triggers": {
"file_sha1":
file_sha1(config.project_root +
"/scripts/log_api_gateway.py"),
"log_group_id":
f"${{aws_cloudwatch_log_group.{lambda_.name}.id}}"
},
# FIXME: Use Terraform to configure API Gateway access logs
# https://github.com/DataBiosphere/azul/issues/3412
"provisioner": {
"local-exec": {
"command":
' '.join(
map(shlex.quote, [
"python", config.project_root +
"/scripts/log_api_gateway.py",
"${module.chalice_%s.RestAPIId}" %
lambda_.name, config.deployment_stage,
"${aws_cloudwatch_log_group.%s.arn}" %
lambda_.name
]))
}
}
}
},
"aws_iam_role": {
lambda_.name: {
"name":
config.qualified_resource_name(lambda_.name),
"assume_role_policy":
json.dumps({
"Version":
"2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "sts:AssumeRole",
"Principal": {
"Service": "lambda.amazonaws.com"
}
},
*(
{
"Effect": "Allow",
"Action": "sts:AssumeRole",
"Principal": {
"AWS": f"arn:aws:iam::{account}:root"
},
# Wildcards are not supported in `Principal`, but they are in `Condition`
"Condition": {
"StringLike": {
"aws:PrincipalArn": [
f"arn:aws:iam::{account}:role/{role}"
for role in roles
]
}
}
} for account, roles in
config.external_lambda_role_assumptors.items())
]
}),
**aws.permissions_boundary_tf
}
},
"aws_iam_role_policy": {
lambda_.name: {
"name": lambda_.name,
"policy": lambda_.policy,
"role": "${aws_iam_role.%s.id}" % lambda_.name
},
}
} for lambda_ in lambdas
]
})
emit_tf({
"resource": [
{
"google_service_account": {
'azul' + service_account.value: {
"project":
"${local.google_project}",
"account_id":
service_account.id,
"display_name":
service_account.id,
"description":
f"Azul service account in {config.deployment_stage}",
"provisioner": [{
"local-exec": {
"command":
' '.join(
map(shlex.quote, [
"python", config.project_root +
"/scripts/provision_credentials.py",
"google-key", "--build", "${self.email}",
service_account.secret_name
]))
}
}, {
"local-exec": {
"when":
"destroy",
"command":
' '.join(
map(shlex.quote, [
"python", config.project_root +
"/scripts/provision_credentials.py",
"google-key", "--destroy", "${self.email}",
service_account.secret_name
]))
}
}]
}
for service_account in config.ServiceAccount
},
**({
"google_project_iam_member": {
"azul": {
"project":
"${local.google_project}",
"role":
"${google_project_iam_custom_role.azul.id}",
"member":
"serviceAccount:${google_service_account.azul.email}"
},
},
"google_project_iam_custom_role": {
"azul": {
"role_id":
f"azul_{config.deployment_stage}",
"title":
f"azul_{config.deployment_stage}",
"permissions": [
"bigquery.jobs.create", "bigquery.reservations.get", "bigquery.capacityCommitments.get", *[
f'bigquery.{resource}.{action}' for resource in ('capacityCommitments', 'reservations', 'reservationAssignments') for action in ('create', 'list', 'delete')
]
]
},
}
} if config.is_tdr_enabled() else {}), "null_resource": {
"hmac_secret": {
"provisioner": [{
"local-exec": {
"command":
' '.join(
map(shlex.quote, [
"python",
config.project_root +
"/scripts/provision_credentials.py",
"hmac-key",
"--build",
]))
}
}, {
"local-exec": {
"when":
"destroy",
"command":
' '.join(
map(shlex.quote, [
"python",
config.project_root +
"/scripts/provision_credentials.py",
"hmac-key",
"--destroy",
]))
}
}]
}
}
},
]
})
from azul import (
config,
)
from azul.deployment import (
aws,
)
from azul.terraform import (
emit_tf,
)
emit_tf(
{
"terraform": {
"backend": {
"s3": {
"bucket": config.terraform_backend_bucket,
"key": f"azul-{config.terraform_component}-{config.deployment_stage}.tfstate",
"region": aws.region_name,
**(
{
"profile": aws.profile['source_profile'],
"role_arn": aws.profile['role_arn']
} if 'role_arn' in aws.profile else {
}
)
}
}
}
}
)
from azul import (
config, )
from azul.lambda_layer import (
DependenciesLayer, )
from azul.terraform import (
emit_tf, )
layer = DependenciesLayer()
emit_tf({
"resource": [{
"aws_lambda_layer_version": {
"dependencies": {
"layer_name": config.qualified_resource_name("dependencies"),
"s3_bucket": config.lambda_layer_bucket,
"s3_key": layer.object_key
}
}
}],
})
config, )
from azul.deployment import (
aws, )
from azul.terraform import (
emit_tf, )
emit_tf({
"module": {
# Not using config.project_root because, "A local path must begin with
# either ./ or ../"
# https://www.terraform.io/docs/modules/sources.html#local-paths
f"chalice_{lambda_name}": {
"source":
f"./{lambda_name}",
"role_arn":
"${aws_iam_role." + lambda_name + ".arn}",
"layer_arn":
"${aws_lambda_layer_version.dependencies.arn}",
"es_endpoint":
aws.es_endpoint if config.share_es_domain else
("${aws_elasticsearch_domain.index.endpoint}", 443),
"es_instance_count":
aws.es_instance_count if config.share_es_domain else
"${aws_elasticsearch_domain.index.cluster_config[0].instance_count}",
"cloudwatch_log_group_provisioner":
f"{lambda_name}_log_group_provisioner"
}
for lambda_name in config.lambda_names()
}
})
emit_tf(
None if config.disable_monitoring else {
"resource": [
*[
{
"aws_route53_health_check": {
name: {
"fqdn": config.api_lambda_domain(name),
"port": 443,
"type": "HTTPS",
"resource_path": "/health/cached",
"failure_threshold": "3",
"request_interval": "30",
"tags": {
"Name": full_name
},
"regions": ['us-west-2', 'us-east-1', 'eu-west-1'],
"measure_latency": True,
# This is necessary only because of a Terraform bug:
# https://github.com/hashicorp/terraform/issues/22171
"lifecycle": {
"create_before_destroy": True
}
},
}
} for name, full_name in (('indexer', config.indexer_name),
('service', config.service_name))
],
{
"aws_route53_health_check": {
"composite-azul": {
# NOTE: There is a 64 character limit on reference name. Terraform adds long string at the end so
# we must be economical about what we add.
"reference_name":
f"azul-{config.deployment_stage}",
"type":
"CALCULATED",
"child_health_threshold":
2,
"child_healthchecks": [
"${aws_route53_health_check." + "indexer" +
".id}", "${aws_route53_health_check." + "service" +
".id}"
],
"measure_latency":
True,
"cloudwatch_alarm_region":
aws.region_name,
"tags": {
"Name": f"azul-composite-{config.deployment_stage}"
}
}
}
},
*[
{
"aws_route53_health_check": {
name: {
"fqdn": domain,
"port": 443,
"type": "HTTPS",
"resource_path": path,
"failure_threshold": "3",
"request_interval": "30",
"tags": {
"Name": full_name
},
"measure_latency": True,
# This is necessary only because of a Terraform bug:
# https://github.com/hashicorp/terraform/issues/22171
"lifecycle": {
"create_before_destroy": True
}
}
}
} for name, domain, full_name, path in (
("data-browser", config.data_browser_domain,
config.data_browser_name, '/explore'),
("data-portal", config.data_browser_domain,
config.data_portal_name, '/'))
],
{
"aws_route53_health_check": {
"composite-portal": {
"reference_name":
f"portal-{config.deployment_stage}",
"type":
"CALCULATED",
"child_health_threshold":
2,
"child_healthchecks": [
"${aws_route53_health_check." + "data-browser" +
".id}", "${aws_route53_health_check." +
"data-portal" + ".id}"
],
"measure_latency":
True,
"cloudwatch_alarm_region":
aws.region_name,
"tags": {
"Name":
f"azul-portal-composite-{config.deployment_stage}"
}
}
}
}
]
})
emit_tf({
"resource": [{
"aws_sqs_queue": {
**{
config.unqual_notifications_queue_name(retry=retry): {
"name":
config.notifications_queue_name(retry=retry),
"visibility_timeout_seconds":
config.contribution_lambda_timeout(retry=retry) + 10,
"message_retention_seconds":
24 * 60 * 60,
"redrive_policy":
json.dumps({
"maxReceiveCount":
9 if retry else 1,
"deadLetterTargetArn":
"${aws_sqs_queue.%s.arn}" % config.unqual_notifications_queue_name(retry=not retry,
fail=retry)
})
}
for retry in (False, True)
},
**{
config.unqual_tallies_queue_name(retry=retry): {
"name":
config.tallies_queue_name(retry=retry),
"fifo_queue":
True,
"delay_seconds":
config.es_refresh_interval + 9,
"visibility_timeout_seconds":
config.aggregation_lambda_timeout(retry=retry) + 10,
"message_retention_seconds":
24 * 60 * 60,
"redrive_policy":
json.dumps({
"maxReceiveCount":
9 if retry else 1,
"deadLetterTargetArn":
"${aws_sqs_queue.%s.arn}" % config.unqual_tallies_queue_name(retry=not retry,
fail=retry)
})
}
for retry in (False, True)
},
config.unqual_notifications_queue_name(fail=True): {
"name": config.notifications_queue_name(fail=True),
"message_retention_seconds": 14 * 24 * 60 * 60,
},
config.unqual_tallies_queue_name(fail=True): {
"fifo_queue": True,
"name": config.tallies_queue_name(fail=True),
"message_retention_seconds": 14 * 24 * 60 * 60,
}
}
}]
})
actual_component_path = Path(__file__).absolute().parent
require(
expected_component_path.samefile(actual_component_path),
f"The current Terraform component is set to '{config.terraform_component}'. "
f"You should therefore be in '{expected_component_path}'")
emit_tf({
"data": [{
"aws_caller_identity": {
"current": {}
}
}, {
"aws_region": {
"current": {}
}
}, *([{
"google_client_config": {
"current": {}
}
}] if config.enable_gcp() else [])],
"locals": {
"account_id":
"${data.aws_caller_identity.current.account_id}",
"region":
"${data.aws_region.current.name}",
"google_project":
"${data.google_client_config.current.project}"
if config.enable_gcp() else None
},
})
