コード例 #1
0
    VersionService, )

emit_tf({
    "resource": [{
        "aws_dynamodb_table": {
            "object_versions": {
                "name": config.dynamo_object_version_table_name,
                "billing_mode": "PAY_PER_REQUEST",
                "hash_key": VersionService.key_name,
                "attribute": [{
                    "name": VersionService.key_name,
                    "type": "S"
                }]
            },
            "sources_cache_by_auth": {
                "name": config.dynamo_sources_cache_table_name,
                "billing_mode": "PAY_PER_REQUEST",
                "hash_key": SourceService.key_attribute,
                "attribute": [{
                    "name": SourceService.key_attribute,
                    "type": "S"
                }],
                "ttl": {
                    "attribute_name": SourceService.ttl_attribute,
                    "enabled": True
                }
            }
        }
    }]
})
コード例 #2
0
emit_tf(
    None if config.disable_monitoring else {
        "output": {
            "grafana_dashboard_azul": {
                "sensitive":
                True,
                "value":
                json.dumps({
                    "annotations": {
                        "list": [{
                            "builtIn": 1,
                            "datasource": "-- Grafana --",
                            "enable": True,
                            "hide": True,
                            "iconColor": "rgba(0, 211, 255, 1)",
                            "name": "Annotations & Alerts",
                            "type": "dashboard"
                        }]
                    },
                    "editable":
                    True,
                    "gnetId":
                    None,
                    "graphTooltip":
                    0,
                    "id":
                    None,
                    "links": [],
                    "panels": [{
                        "cacheTimeout":
                        None,
                        "colorBackground":
                        True,
                        "colorValue":
                        False,
                        "colors":
                        ["#bf1b00", "#629e51", "rgba(237, 129, 40, 0.89)"],
                        "datasource":
                        "account-cloudwatch",
                        "format":
                        "none",
                        "gauge": {
                            "maxValue": 100,
                            "minValue": 0,
                            "show": False,
                            "thresholdLabels": False,
                            "thresholdMarkers": True
                        },
                        "gridPos": {
                            "h": 10,
                            "w": 6,
                            "x": 0,
                            "y": 0
                        },
                        "id":
                        4,
                        "interval":
                        None,
                        "links": [],
                        "mappingType":
                        1,
                        "mappingTypes": [{
                            "name": "value to text",
                            "value": 1
                        }, {
                            "name": "range to text",
                            "value": 2
                        }],
                        "maxDataPoints":
                        100,
                        "NonePointMode":
                        "connected",
                        "NoneText":
                        None,
                        "postfix":
                        "",
                        "postfixFontSize":
                        "50%",
                        "prefix":
                        "AZUL",
                        "prefixFontSize":
                        "120%",
                        "rangeMaps": [{
                            "from": "None",
                            "text": "N/A",
                            "to": "None"
                        }],
                        "sparkline": {
                            "fillColor": "rgba(31, 118, 189, 0.18)",
                            "full": False,
                            "lineColor": "rgb(31, 120, 193)",
                            "show": True
                        },
                        "tableColumn":
                        "",
                        "targets": [{
                            "dimensions": {
                                "HealthCheckId":
                                "${aws_route53_health_check.composite-azul.id}",
                            },
                            "highResolution": False,
                            "metricName": "HealthCheckStatus",
                            "namespace": "AWS/Route53",
                            "period": "",
                            "refId": "A",
                            "region": aws.region_name,
                            "statistics": ["Minimum"]
                        }],
                        "thresholds":
                        "0.5",
                        "title":
                        "Azul Health",
                        "type":
                        "singlestat",
                        "valueFontSize":
                        "120%",
                        "valueMaps": [{
                            "op": "=",
                            "text": "OK",
                            "value": "1"
                        }, {
                            "op": "=",
                            "text": "ERR",
                            "value": "0"
                        }],
                        "valueName":
                        "current"
                    }, {
                        "aliasColors": {},
                        "bars":
                        False,
                        "dashLength":
                        10,
                        "dashes":
                        False,
                        "datasource":
                        "account-cloudwatch",
                        "fill":
                        1,
                        "gridPos": {
                            "h": 10,
                            "w": 6,
                            "x": 6,
                            "y": 0
                        },
                        "id":
                        14,
                        "legend": {
                            "alignAsTable": False,
                            "avg": False,
                            "current": False,
                            "max": True,
                            "min": True,
                            "show": True,
                            "total": False,
                            "values": True
                        },
                        "lines":
                        True,
                        "linewidth":
                        1,
                        "links": [],
                        "NonePointMode":
                        "None as zero",
                        "percentage":
                        False,
                        "pointradius":
                        5,
                        "points":
                        False,
                        "renderer":
                        "flot",
                        "seriesOverrides": [],
                        "spaceLength":
                        10,
                        "stack":
                        False,
                        "steppedLine":
                        False,
                        "targets": [{
                            "dimensions": {
                                "ClientId": aws.account,
                                "DomainName": config.es_domain
                            },
                            "highResolution": True,
                            "metricName": "CPUUtilization",
                            "namespace": "AWS/ES",
                            "period": "",
                            "refId": "A",
                            "region": "default",
                            "statistics": ["Average"]
                        }, {
                            "dimensions": {
                                "ClientId": aws.account,
                                "DomainName": config.es_domain
                            },
                            "highResolution": True,
                            "metricName": "JVMMemoryPressure",
                            "namespace": "AWS/ES",
                            "period": "",
                            "refId": "B",
                            "region": "default",
                            "statistics": ["Average"]
                        }],
                        "thresholds": [],
                        "timeFrom":
                        None,
                        "timeRegions": [],
                        "timeShift":
                        None,
                        "title":
                        "ES - CPU",
                        "tooltip": {
                            "shared": True,
                            "sort": 0,
                            "value_type": "individual"
                        },
                        "type":
                        "graph",
                        "xaxis": {
                            "buckets": None,
                            "mode": "time",
                            "name": None,
                            "show": True,
                            "values": []
                        },
                        "yaxes": [{
                            "format": "percent",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }, {
                            "format": "short",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }],
                        "yaxis": {
                            "align": False,
                            "alignLevel": None
                        }
                    }, {
                        "aliasColors": {},
                        "bars":
                        False,
                        "dashLength":
                        10,
                        "dashes":
                        False,
                        "datasource":
                        "account-cloudwatch",
                        "fill":
                        1,
                        "gridPos": {
                            "h": 10,
                            "w": 6,
                            "x": 12,
                            "y": 0
                        },
                        "id":
                        16,
                        "legend": {
                            "alignAsTable": True,
                            "avg": False,
                            "current": False,
                            "max": True,
                            "min": True,
                            "show": True,
                            "total": False,
                            "values": True
                        },
                        "lines":
                        True,
                        "linewidth":
                        1,
                        "links": [],
                        "NonePointMode":
                        "None as zero",
                        "percentage":
                        False,
                        "pointradius":
                        5,
                        "points":
                        False,
                        "renderer":
                        "flot",
                        "seriesOverrides": [],
                        "spaceLength":
                        10,
                        "stack":
                        False,
                        "steppedLine":
                        False,
                        "targets": [{
                            "dimensions": {
                                "ClientId": aws.account,
                                "DomainName": config.es_domain
                            },
                            "highResolution": True,
                            "metricName": "ReadLatency",
                            "namespace": "AWS/ES",
                            "period": "",
                            "refId": "A",
                            "region": "default",
                            "statistics": ["Average"]
                        }, {
                            "dimensions": {
                                "ClientId": aws.account,
                                "DomainName": config.es_domain
                            },
                            "highResolution": True,
                            "metricName": "WriteLatency",
                            "namespace": "AWS/ES",
                            "period": "",
                            "refId": "B",
                            "region": "default",
                            "statistics": ["Average"]
                        }],
                        "thresholds": [],
                        "timeFrom":
                        None,
                        "timeRegions": [],
                        "timeShift":
                        None,
                        "title":
                        "ES - I/O Latency",
                        "tooltip": {
                            "shared": True,
                            "sort": 0,
                            "value_type": "individual"
                        },
                        "type":
                        "graph",
                        "xaxis": {
                            "buckets": None,
                            "mode": "time",
                            "name": None,
                            "show": True,
                            "values": []
                        },
                        "yaxes": [{
                            "format": "s",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }, {
                            "format": "short",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }],
                        "yaxis": {
                            "align": False,
                            "alignLevel": None
                        }
                    }, {
                        "alert": {
                            "conditions": [{
                                "evaluator": {
                                    "params": [1073741824],
                                    "type": "lt"
                                },
                                "operator": {
                                    "type": "and"
                                },
                                "query": {
                                    "params": ["A", "5m", "now"]
                                },
                                "reducer": {
                                    "params": [],
                                    "type": "avg"
                                },
                                "type": "query"
                            }],
                            "executionErrorState":
                            "alerting",
                            "frequency":
                            "15m",
                            "handler":
                            1,
                            "name":
                            "ES - Free Storage Space alert",
                            "noDataState":
                            "no_data",
                            "notifications": []
                        },
                        "aliasColors": {},
                        "bars":
                        False,
                        "dashLength":
                        10,
                        "dashes":
                        False,
                        "datasource":
                        "account-cloudwatch",
                        "fill":
                        1,
                        "gridPos": {
                            "h": 10,
                            "w": 6,
                            "x": 18,
                            "y": 0
                        },
                        "id":
                        18,
                        "legend": {
                            "alignAsTable": True,
                            "avg": False,
                            "current": False,
                            "max": True,
                            "min": True,
                            "show": True,
                            "total": False,
                            "values": True
                        },
                        "lines":
                        True,
                        "linewidth":
                        1,
                        "links": [],
                        "NonePointMode":
                        "None as zero",
                        "percentage":
                        False,
                        "pointradius":
                        5,
                        "points":
                        False,
                        "renderer":
                        "flot",
                        "seriesOverrides": [],
                        "spaceLength":
                        10,
                        "stack":
                        False,
                        "steppedLine":
                        False,
                        "targets": [{
                            "dimensions": {
                                "ClientId": aws.account,
                                "DomainName": config.es_domain
                            },
                            "highResolution": True,
                            "metricName": "FreeStorageSpace",
                            "namespace": "AWS/ES",
                            "period": "",
                            "refId": "A",
                            "region": "default",
                            "statistics": ["Minimum"]
                        }],
                        "thresholds": [{
                            "colorMode": "critical",
                            "fill": True,
                            "line": True,
                            "op": "lt",
                            "value": 1073741824
                        }],
                        "timeFrom":
                        None,
                        "timeRegions": [],
                        "timeShift":
                        None,
                        "title":
                        "ES - Free Storage Space",
                        "tooltip": {
                            "shared": True,
                            "sort": 0,
                            "value_type": "individual"
                        },
                        "type":
                        "graph",
                        "xaxis": {
                            "buckets": None,
                            "mode": "time",
                            "name": None,
                            "show": True,
                            "values": []
                        },
                        "yaxes": [{
                            "format": "mbytes",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }, {
                            "format": "short",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }],
                        "yaxis": {
                            "align": False,
                            "alignLevel": None
                        }
                    }, {
                        "aliasColors": {},
                        "bars":
                        False,
                        "dashLength":
                        10,
                        "dashes":
                        False,
                        "datasource":
                        "account-cloudwatch",
                        "decimals":
                        None,
                        "fill":
                        1,
                        "gridPos": {
                            "h": 10,
                            "w": 12,
                            "x": 0,
                            "y": 10
                        },
                        "id":
                        6,
                        "legend": {
                            "alignAsTable": True,
                            "avg": True,
                            "current": True,
                            "hideEmpty": False,
                            "hideZero": False,
                            "max": True,
                            "min": True,
                            "show": True,
                            "sortDesc": True,
                            "total": True,
                            "values": True
                        },
                        "lines":
                        True,
                        "linewidth":
                        1,
                        "links": [],
                        "NonePointMode":
                        "None as zero",
                        "percentage":
                        False,
                        "pointradius":
                        1,
                        "points":
                        False,
                        "renderer":
                        "flot",
                        "seriesOverrides": [],
                        "spaceLength":
                        10,
                        "stack":
                        False,
                        "steppedLine":
                        True,
                        "targets": [{
                            "alias": "Total Requests",
                            "dimensions": {
                                "ApiName": config.service_name,
                                "Stage": config.deployment_stage
                            },
                            "highResolution": True,
                            "metricName": "Count",
                            "namespace": "AWS/ApiGateway",
                            "period": "",
                            "refId": "A",
                            "region": "default",
                            "statistics": ["Sum"]
                        }, {
                            "alias": "HTTP 4XX",
                            "dimensions": {
                                "ApiName": config.service_name,
                                "Stage": config.deployment_stage
                            },
                            "highResolution": True,
                            "metricName": "4XXError",
                            "namespace": "AWS/ApiGateway",
                            "period": "",
                            "refId": "B",
                            "region": "default",
                            "statistics": ["Sum"]
                        }, {
                            "alias": "HTTP 5XX",
                            "dimensions": {
                                "ApiName": config.service_name,
                                "Stage": config.deployment_stage
                            },
                            "highResolution": True,
                            "metricName": "5XXError",
                            "namespace": "AWS/ApiGateway",
                            "period": "",
                            "refId": "C",
                            "region": "default",
                            "statistics": ["Sum"]
                        }],
                        "thresholds": [],
                        "timeFrom":
                        None,
                        "timeRegions": [],
                        "timeShift":
                        None,
                        "title":
                        "Web Service API Request/Error Rates",
                        "tooltip": {
                            "shared": True,
                            "sort": 0,
                            "value_type": "individual"
                        },
                        "type":
                        "graph",
                        "xaxis": {
                            "buckets": None,
                            "mode": "time",
                            "name": None,
                            "show": True,
                            "values": []
                        },
                        "yaxes": [{
                            "format": "short",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }, {
                            "format": "short",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }],
                        "yaxis": {
                            "align": False,
                            "alignLevel": None
                        }
                    }, {
                        "aliasColors": {},
                        "bars":
                        False,
                        "dashLength":
                        10,
                        "dashes":
                        False,
                        "datasource":
                        "account-cloudwatch",
                        "decimals":
                        None,
                        "fill":
                        1,
                        "gridPos": {
                            "h": 10,
                            "w": 12,
                            "x": 12,
                            "y": 10
                        },
                        "id":
                        8,
                        "legend": {
                            "alignAsTable": True,
                            "avg": True,
                            "current": True,
                            "hideEmpty": False,
                            "hideZero": False,
                            "max": True,
                            "min": True,
                            "show": True,
                            "total": True,
                            "values": True
                        },
                        "lines":
                        True,
                        "linewidth":
                        1,
                        "links": [],
                        "NonePointMode":
                        "None as zero",
                        "percentage":
                        False,
                        "pointradius":
                        1,
                        "points":
                        False,
                        "renderer":
                        "flot",
                        "seriesOverrides": [],
                        "spaceLength":
                        10,
                        "stack":
                        False,
                        "steppedLine":
                        True,
                        "targets": [{
                            "alias": "Total Requests",
                            "dimensions": {
                                "ApiName": config.indexer_name,
                                "Stage": config.deployment_stage
                            },
                            "highResolution": True,
                            "metricName": "Count",
                            "namespace": "AWS/ApiGateway",
                            "period": "",
                            "refId": "A",
                            "region": "default",
                            "statistics": ["Sum"]
                        }, {
                            "alias": "HTTP 4XX",
                            "dimensions": {
                                "ApiName": config.indexer_name,
                                "Stage": config.deployment_stage
                            },
                            "highResolution": True,
                            "metricName": "4XXError",
                            "namespace": "AWS/ApiGateway",
                            "period": "",
                            "refId": "B",
                            "region": "default",
                            "statistics": ["Sum"]
                        }, {
                            "alias": "HTTP 5XX",
                            "dimensions": {
                                "ApiName": config.indexer_name,
                                "Stage": config.deployment_stage
                            },
                            "highResolution": True,
                            "metricName": "5XXError",
                            "namespace": "AWS/ApiGateway",
                            "period": "",
                            "refId": "C",
                            "region": "default",
                            "statistics": ["Sum"]
                        }],
                        "thresholds": [],
                        "timeFrom":
                        None,
                        "timeRegions": [],
                        "timeShift":
                        None,
                        "title":
                        "Indexer API Request/Error Rates",
                        "tooltip": {
                            "shared": True,
                            "sort": 0,
                            "value_type": "individual"
                        },
                        "type":
                        "graph",
                        "xaxis": {
                            "buckets": None,
                            "mode": "time",
                            "name": None,
                            "show": True,
                            "values": []
                        },
                        "yaxes": [{
                            "format": "short",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }, {
                            "format": "short",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }],
                        "yaxis": {
                            "align": False,
                            "alignLevel": None
                        }
                    }, {
                        "aliasColors": {},
                        "bars":
                        False,
                        "dashLength":
                        10,
                        "dashes":
                        False,
                        "datasource":
                        "account-cloudwatch",
                        "fill":
                        1,
                        "gridPos": {
                            "h": 10,
                            "w": 12,
                            "x": 0,
                            "y": 20
                        },
                        "id":
                        10,
                        "legend": {
                            "alignAsTable": True,
                            "avg": True,
                            "current": True,
                            "max": True,
                            "min": True,
                            "show": True,
                            "total": True,
                            "values": True
                        },
                        "lines":
                        True,
                        "linewidth":
                        1,
                        "links": [],
                        "NonePointMode":
                        "None as zero",
                        "percentage":
                        False,
                        "pointradius":
                        5,
                        "points":
                        False,
                        "renderer":
                        "flot",
                        "seriesOverrides": [],
                        "spaceLength":
                        10,
                        "stack":
                        False,
                        "steppedLine":
                        False,
                        "targets": [{
                            "alias":
                            "Web Service - Total Invocation Count",
                            "bucketAggs": [{
                                "field": "@timestamp",
                                "id": "2",
                                "settings": {
                                    "interval": "auto",
                                    "min_doc_count": 0,
                                    "trimEdges": 0
                                },
                                "type": "date_histogram"
                            }],
                            "dimensions": {
                                "FunctionName": config.service_name
                            },
                            "highResolution":
                            True,
                            "metricName":
                            "Invocations",
                            "metrics": [{
                                "field": "select field",
                                "id": "1",
                                "type": "count"
                            }],
                            "namespace":
                            "AWS/Lambda",
                            "period":
                            "",
                            "refId":
                            "A",
                            "region":
                            "default",
                            "statistics": ["Sum"],
                            "timeField":
                            "@timestamp"
                        }, {
                            "alias": "Web Service - Total Error Count",
                            "dimensions": {
                                "FunctionName": config.service_name
                            },
                            "highResolution": True,
                            "metricName": "Errors",
                            "namespace": "AWS/Lambda",
                            "period": "",
                            "refId": "B",
                            "region": "default",
                            "statistics": ["Sum"]
                        }],
                        "thresholds": [],
                        "timeFrom":
                        None,
                        "timeRegions": [],
                        "timeShift":
                        None,
                        "title":
                        "Web Service Lambda",
                        "tooltip": {
                            "shared": True,
                            "sort": 0,
                            "value_type": "individual"
                        },
                        "type":
                        "graph",
                        "xaxis": {
                            "buckets": None,
                            "mode": "time",
                            "name": None,
                            "show": True,
                            "values": []
                        },
                        "yaxes": [{
                            "format": "short",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }, {
                            "format": "short",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }],
                        "yaxis": {
                            "align": False,
                            "alignLevel": None
                        }
                    }, {
                        "aliasColors": {},
                        "bars":
                        False,
                        "dashLength":
                        10,
                        "dashes":
                        False,
                        "datasource":
                        "account-cloudwatch",
                        "fill":
                        1,
                        "gridPos": {
                            "h": 10,
                            "w": 12,
                            "x": 12,
                            "y": 20
                        },
                        "id":
                        12,
                        "legend": {
                            "alignAsTable": True,
                            "avg": True,
                            "current": True,
                            "max": True,
                            "min": True,
                            "show": True,
                            "total": True,
                            "values": True
                        },
                        "lines":
                        True,
                        "linewidth":
                        1,
                        "links": [],
                        "NonePointMode":
                        "None as zero",
                        "percentage":
                        False,
                        "pointradius":
                        5,
                        "points":
                        False,
                        "renderer":
                        "flot",
                        "seriesOverrides": [],
                        "spaceLength":
                        10,
                        "stack":
                        False,
                        "steppedLine":
                        False,
                        "targets": [{
                            "alias":
                            "Indexer - Total Invocation Count",
                            "bucketAggs": [{
                                "field": "@timestamp",
                                "id": "2",
                                "settings": {
                                    "interval": "auto",
                                    "min_doc_count": 0,
                                    "trimEdges": 0
                                },
                                "type": "date_histogram"
                            }],
                            "dimensions": {
                                "FunctionName": config.indexer_name
                            },
                            "highResolution":
                            True,
                            "metricName":
                            "Invocations",
                            "metrics": [{
                                "field": "select field",
                                "id": "1",
                                "type": "count"
                            }],
                            "namespace":
                            "AWS/Lambda",
                            "period":
                            "",
                            "refId":
                            "A",
                            "region":
                            "default",
                            "statistics": ["Sum"],
                            "timeField":
                            "@timestamp"
                        }, {
                            "alias": "Indexer - Total Error Count",
                            "dimensions": {
                                "FunctionName": config.indexer_name
                            },
                            "highResolution": True,
                            "metricName": "Errors",
                            "namespace": "AWS/Lambda",
                            "period": "",
                            "refId": "B",
                            "region": "default",
                            "statistics": ["Sum"]
                        }],
                        "thresholds": [],
                        "timeFrom":
                        None,
                        "timeRegions": [],
                        "timeShift":
                        None,
                        "title":
                        "Indexer Lambda",
                        "tooltip": {
                            "shared": True,
                            "sort": 0,
                            "value_type": "individual"
                        },
                        "type":
                        "graph",
                        "xaxis": {
                            "buckets": None,
                            "mode": "time",
                            "name": None,
                            "show": True,
                            "values": []
                        },
                        "yaxes": [{
                            "format": "short",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }, {
                            "format": "short",
                            "label": None,
                            "logBase": 1,
                            "max": None,
                            "min": None,
                            "show": True
                        }],
                        "yaxis": {
                            "align": False,
                            "alignLevel": None
                        }
                    }],
                    "refresh":
                    "1m",
                    "schemaVersion":
                    16,
                    "style":
                    "dark",
                    "tags": [],
                    "templating": {
                        "list": []
                    },
                    "time": {
                        "from": "now-6h",
                        "to": "now"
                    },
                    "timepicker": {
                        "refresh_intervals": [
                            "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h",
                            "2h", "1d"
                        ],
                        "time_options": [
                            "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d",
                            "30d"
                        ]
                    },
                    "timezone":
                    "utc",
                    "title":
                    f"Azul [{config.deployment_stage.upper()}]",
                    "uid":
                    f'azul-{config.deployment_stage}',
                    "version":
                    1
                })
            },
            "grafana_dashboard_data_portal": {
                "sensitive":
                True,
                "value":
                json.dumps({
                    "annotations": {
                        "list": [{
                            "builtIn": 1,
                            "datasource": "-- Grafana --",
                            "enable": True,
                            "hide": True,
                            "iconColor": "rgba(0, 211, 255, 1)",
                            "name": "Annotations & Alerts",
                            "type": "dashboard"
                        }]
                    },
                    "editable":
                    True,
                    "gnetId":
                    None,
                    "graphTooltip":
                    0,
                    "id":
                    None,
                    "links": [],
                    "panels": [{
                        "cacheTimeout":
                        None,
                        "colorBackground":
                        True,
                        "colorValue":
                        False,
                        "colors": ["#d44a3a", "#629e51", "#c15c17"],
                        "datasource":
                        "account-cloudwatch",
                        "format":
                        "none",
                        "gauge": {
                            "maxValue": 100,
                            "minValue": 0,
                            "show": False,
                            "thresholdLabels": False,
                            "thresholdMarkers": True
                        },
                        "gridPos": {
                            "h": 9,
                            "w": 24,
                            "x": 0,
                            "y": 0
                        },
                        "id":
                        2,
                        "interval":
                        None,
                        "links": [],
                        "mappingType":
                        1,
                        "mappingTypes": [{
                            "name": "value to text",
                            "value": 1
                        }, {
                            "name": "range to text",
                            "value": 2
                        }],
                        "maxDataPoints":
                        100,
                        "nullPointMode":
                        "connected",
                        "nullText":
                        None,
                        "postfix":
                        "",
                        "postfixFontSize":
                        "50%",
                        "prefix":
                        "DATA BROWSER & PORTAL",
                        "prefixFontSize":
                        "120%",
                        "rangeMaps": [{
                            "from": "null",
                            "text": "N/A",
                            "to": "null"
                        }],
                        "sparkline": {
                            "fillColor": "rgba(31, 118, 189, 0.18)",
                            "full": False,
                            "lineColor": "rgb(31, 120, 193)",
                            "show": True
                        },
                        "tableColumn":
                        "",
                        "targets": [{
                            "dimensions": {
                                "HealthCheckId":
                                "${aws_route53_health_check.composite-portal.id}"
                            },
                            "expression": "",
                            "highResolution": False,
                            "id": "",
                            "metricName": "HealthCheckStatus",
                            "namespace": "AWS/Route53",
                            "period": "",
                            "refId": "A",
                            "region": "us-east-1",
                            "returnData": False,
                            "statistics": ["Minimum"]
                        }],
                        "thresholds":
                        "0.5",
                        "title":
                        "Data Browser & Portal Health",
                        "type":
                        "singlestat",
                        "valueFontSize":
                        "120%",
                        "valueMaps": [{
                            "op": "=",
                            "text": "OK",
                            "value": "1"
                        }, {
                            "op": "=",
                            "text": "ERR",
                            "value": "0"
                        }],
                        "valueName":
                        "current"
                    }],
                    "schemaVersion":
                    16,
                    "style":
                    "dark",
                    "tags": [],
                    "templating": {
                        "list": []
                    },
                    "time": {
                        "from": "now-6h",
                        "to": "now"
                    },
                    "timepicker": {
                        "refresh_intervals": [
                            "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h",
                            "2h", "1d"
                        ],
                        "time_options": [
                            "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d",
                            "30d"
                        ]
                    },
                    "timezone":
                    "",
                    "title":
                    f"Data Browser & Portal [{config.deployment_stage.upper()}]",
                    "uid":
                    f"data-portal-{config.deployment_stage}",
                    "version":
                    1
                })
            }
        }
    })
コード例 #3
0
emit_tf({
    "provider": [
        {
            "template": {
                'version': '2.2.0'
            }
        },
        {
            "null": {
                'version': '2.1.2'
            }
        },
        {
            "google": {
                'version': '2.20.3'
            }
        },
        *({
            "aws": {
                'version': '4.3.0',
                **(
                    {
                        'region': region,
                        'alias': region
                    } if region else {
                    }
                ),
                **(
                    {
                        'assume_role': {
                            'role_arn': aws.profile['role_arn']
                        }
                    } if 'role_arn' in aws.profile else {
                    }
                )
            }
        } for region in (None, 'us-east-1'))
        # Generate a default `aws` provider and one that pins the region for the certificates of the API Gateway
        # custom domain names. Certificates of edge-optimized custom domain names have to reside in us-east-1.
    ]
})
コード例 #4
0
emit_tf({
    "resource": [
        {
            "aws_s3_bucket": {
                "storage": {
                    "bucket": config.s3_bucket,
                    "force_destroy": True
                },
                "urls": {
                    "bucket": config.url_redirect_full_domain_name,
                    "force_destroy": not config.is_main_deployment(),
                }
            },
            "aws_s3_bucket_lifecycle_configuration": {
                "storage": {
                    "bucket": "${aws_s3_bucket.storage.id}",
                    "rule": {
                        "id": "manifests",
                        "status": "Enabled",
                        "filter": {
                            "prefix": "manifests/"
                        },
                        "expiration": {
                            "days": config.manifest_expiration
                        },
                        "abort_incomplete_multipart_upload": {
                            "days_after_initiation": 1
                        }
                    }
                }
            },
            "aws_s3_bucket_acl": {
                "storage": {
                    "bucket": "${aws_s3_bucket.storage.id}",
                    "acl": "private",
                },
                "urls": {
                    "bucket": "${aws_s3_bucket.urls.id}",
                    "acl": "public-read",
                }
            },
            "aws_s3_bucket_website_configuration": {
                "urls": {
                    "bucket": "${aws_s3_bucket.urls.id}",
                    "index_document": {
                        "suffix": "404.html"
                    }
                }
            }
        },
        *([{
            "aws_route53_record": {
                "url_redirect_record": {
                    "zone_id": "${data.aws_route53_zone.azul_url.zone_id}",
                    "name": config.url_redirect_full_domain_name,
                    "type": "CNAME",
                    "ttl": "300",
                    "records": ["${aws_s3_bucket_website_configuration.urls.website_endpoint}"]
                }
            }
        }] if config.url_redirect_base_domain_name else [])
    ],
    **({
           "data": [
               {
                   "aws_route53_zone": {
                       "azul_url": {
                           "name": config.url_redirect_base_domain_name + ".",
                           "private_zone": False
                       }
                   }
               }
           ]
       } if config.url_redirect_base_domain_name else {})
})
コード例 #5
0
emit_tf({
    "data": [{
        "aws_route53_zone": {
            zone.slug: {
                "name": zone.name,
                "private_zone": False
            }
        }
    } for zone in set(zones_by_domain.values())],
    # Note that ${} references exist to interpolate a value AND express a dependency.
    "resource": [
        {
            "aws_api_gateway_base_path_mapping": {
                f"{lambda_.name}_{i}": {
                    "api_id":
                    "${module.chalice_%s.RestAPIId}" % lambda_.name,
                    "stage_name":
                    "${module.chalice_%s.stage_name}" % lambda_.name,
                    "domain_name":
                    "${aws_api_gateway_domain_name.%s_%i.domain_name}" %
                    (lambda_.name, i)
                }
                for i, domain in enumerate(lambda_.domains)
            },
            "aws_api_gateway_domain_name": {
                f"{lambda_.name}_{i}": {
                    "domain_name":
                    "${aws_acm_certificate.%s_%i.domain_name}" %
                    (lambda_.name, i),
                    "certificate_arn":
                    "${aws_acm_certificate_validation.%s_%i.certificate_arn}" %
                    (lambda_.name, i)
                }
                for i, domain in enumerate(lambda_.domains)
            },
            "aws_api_gateway_method_settings": {
                f"{lambda_.name}_{i}": {
                    "rest_api_id":
                    "${module.chalice_%s.RestAPIId}" % lambda_.name,
                    "stage_name":
                    "${module.chalice_%s.stage_name}" % lambda_.name,
                    "method_path": "*/*",  # every URL path, every HTTP method
                    "settings": {
                        "metrics_enabled": True,
                        "data_trace_enabled": config.debug == 2,
                        "logging_level":
                        "ERROR" if config.debug == 0 else "INFO"
                    }
                }
                for i, domain in enumerate(lambda_.domains)
            },
            "aws_acm_certificate": {
                f"{lambda_.name}_{i}": {
                    "domain_name": domain,
                    "validation_method": "DNS",
                    "provider": "aws.us-east-1",
                    # I tried using SANs for the alias domains (like the DRS
                    # domain) but Terraform kept swapping the zones, I think
                    # because the order of elements in
                    # `aws_acm_certificate.domain_validation_options` is not
                    # deterministic. The alternative is to use separate certs,
                    # one for each domain, the main one as well as for each
                    # alias.
                    #
                    # Update 03/07/2022: My guess about the non-determinism was
                    # correct. That bug was "fixed" in Terraform by making the
                    # domain_validation_options a set so that elements can't be
                    # accessed via numeric index. The Terraform documentation
                    # recommends looping over the elements in that set. That's
                    # what we do for GitLab. To do the same here would require
                    # bigger refactoring that I don't think is worth it. The
                    # current solution works, too.
                    "subject_alternative_names": [],
                    "lifecycle": {
                        "create_before_destroy": True
                    }
                }
                for i, domain in enumerate(lambda_.domains)
            },
            "aws_acm_certificate_validation": {
                f"{lambda_.name}_{i}": {
                    "certificate_arn":
                    "${aws_acm_certificate.%s_%i.arn}" % (lambda_.name, i),
                    "validation_record_fqdns": [
                        "${aws_route53_record.%s_domain_validation_%i.fqdn}" %
                        (lambda_.name, i)
                    ],
                    "provider":
                    "aws.us-east-1"
                }
                for i, domain in enumerate(lambda_.domains)
            },
            "aws_route53_record": {
                **{
                    f"{lambda_.name}_domain_validation_{i}": {
                        **{
                            # We know there is only one. See comment above.
                            key: "${tolist(aws_acm_certificate.%s_%i.domain_validation_options).0.resource_record_%s}" % (lambda_.name, i, key)
                            for key in ('name', 'type')
                        },
                        "zone_id":
                        "${data.aws_route53_zone.%s.id}" % zones_by_domain[domain].slug,
                        "records": [
                            # We know there is only one. See comment above.
                            "${tolist(aws_acm_certificate.%s_%i.domain_validation_options).0.resource_record_value}" % (lambda_.name, i)
                        ],
                        "ttl":
                        60
                    }
                    for i, domain in enumerate(lambda_.domains)
                },
                **{
                    f"{lambda_.name}_{i}": {
                        "zone_id":
                        "${data.aws_route53_zone.%s.id}" % zones_by_domain[domain].slug,
                        "name":
                        "${aws_api_gateway_domain_name.%s_%i.domain_name}" % (lambda_.name, i),
                        "type":
                        "A",
                        "alias": {
                            "name":
                            "${aws_api_gateway_domain_name.%s_%i.cloudfront_domain_name}" % (lambda_.name, i),
                            "zone_id":
                            "${aws_api_gateway_domain_name.%s_%i.cloudfront_zone_id}" % (lambda_.name, i),
                            "evaluate_target_health":
                            True,
                        }
                    }
                    for i, domain in enumerate(lambda_.domains)
                }
            },
            "aws_cloudwatch_log_group": {
                lambda_.name: {
                    "name":
                    "/aws/apigateway/" +
                    config.qualified_resource_name(lambda_.name),
                    "retention_in_days":
                    1827,
                }
            },
            "null_resource": {
                f'{lambda_.name}_log_group_provisioner': {
                    "triggers": {
                        "file_sha1":
                        file_sha1(config.project_root +
                                  "/scripts/log_api_gateway.py"),
                        "log_group_id":
                        f"${{aws_cloudwatch_log_group.{lambda_.name}.id}}"
                    },
                    # FIXME: Use Terraform to configure API Gateway access logs
                    #        https://github.com/DataBiosphere/azul/issues/3412
                    "provisioner": {
                        "local-exec": {
                            "command":
                            ' '.join(
                                map(shlex.quote, [
                                    "python", config.project_root +
                                    "/scripts/log_api_gateway.py",
                                    "${module.chalice_%s.RestAPIId}" %
                                    lambda_.name, config.deployment_stage,
                                    "${aws_cloudwatch_log_group.%s.arn}" %
                                    lambda_.name
                                ]))
                        }
                    }
                }
            },
            "aws_iam_role": {
                lambda_.name: {
                    "name":
                    config.qualified_resource_name(lambda_.name),
                    "assume_role_policy":
                    json.dumps({
                        "Version":
                        "2012-10-17",
                        "Statement": [
                            {
                                "Effect": "Allow",
                                "Action": "sts:AssumeRole",
                                "Principal": {
                                    "Service": "lambda.amazonaws.com"
                                }
                            },
                            *(
                                {
                                    "Effect": "Allow",
                                    "Action": "sts:AssumeRole",
                                    "Principal": {
                                        "AWS": f"arn:aws:iam::{account}:root"
                                    },
                                    # Wildcards are not supported in `Principal`, but they are in `Condition`
                                    "Condition": {
                                        "StringLike": {
                                            "aws:PrincipalArn": [
                                                f"arn:aws:iam::{account}:role/{role}"
                                                for role in roles
                                            ]
                                        }
                                    }
                                } for account, roles in
                                config.external_lambda_role_assumptors.items())
                        ]
                    }),
                    **aws.permissions_boundary_tf
                }
            },
            "aws_iam_role_policy": {
                lambda_.name: {
                    "name": lambda_.name,
                    "policy": lambda_.policy,
                    "role": "${aws_iam_role.%s.id}" % lambda_.name
                },
            }
        } for lambda_ in lambdas
    ]
})
コード例 #6
0
emit_tf({
    "resource": [
        {
            "google_service_account": {
                'azul' + service_account.value: {
                    "project":
                    "${local.google_project}",
                    "account_id":
                    service_account.id,
                    "display_name":
                    service_account.id,
                    "description":
                    f"Azul service account in {config.deployment_stage}",
                    "provisioner": [{
                        "local-exec": {
                            "command":
                            ' '.join(
                                map(shlex.quote, [
                                    "python", config.project_root +
                                    "/scripts/provision_credentials.py",
                                    "google-key", "--build", "${self.email}",
                                    service_account.secret_name
                                ]))
                        }
                    }, {
                        "local-exec": {
                            "when":
                            "destroy",
                            "command":
                            ' '.join(
                                map(shlex.quote, [
                                    "python", config.project_root +
                                    "/scripts/provision_credentials.py",
                                    "google-key", "--destroy", "${self.email}",
                                    service_account.secret_name
                                ]))
                        }
                    }]
                }
                for service_account in config.ServiceAccount
            },
            **({
                "google_project_iam_member": {
                    "azul": {
                        "project":
                        "${local.google_project}",
                        "role":
                        "${google_project_iam_custom_role.azul.id}",
                        "member":
                        "serviceAccount:${google_service_account.azul.email}"
                    },
                },
                "google_project_iam_custom_role": {
                    "azul": {
                        "role_id":
                        f"azul_{config.deployment_stage}",
                        "title":
                        f"azul_{config.deployment_stage}",
                        "permissions": [
                            "bigquery.jobs.create", "bigquery.reservations.get", "bigquery.capacityCommitments.get", *[
                                f'bigquery.{resource}.{action}' for resource in ('capacityCommitments', 'reservations', 'reservationAssignments') for action in ('create', 'list', 'delete')
                            ]
                        ]
                    },
                }
            } if config.is_tdr_enabled() else {}), "null_resource": {
                "hmac_secret": {
                    "provisioner": [{
                        "local-exec": {
                            "command":
                            ' '.join(
                                map(shlex.quote, [
                                    "python",
                                    config.project_root +
                                    "/scripts/provision_credentials.py",
                                    "hmac-key",
                                    "--build",
                                ]))
                        }
                    }, {
                        "local-exec": {
                            "when":
                            "destroy",
                            "command":
                            ' '.join(
                                map(shlex.quote, [
                                    "python",
                                    config.project_root +
                                    "/scripts/provision_credentials.py",
                                    "hmac-key",
                                    "--destroy",
                                ]))
                        }
                    }]
                }
            }
        },
    ]
})
コード例 #7
0
from azul import (
    config,
)
from azul.deployment import (
    aws,
)
from azul.terraform import (
    emit_tf,
)

emit_tf(
    {
        "terraform": {
            "backend": {
                "s3": {
                    "bucket": config.terraform_backend_bucket,
                    "key": f"azul-{config.terraform_component}-{config.deployment_stage}.tfstate",
                    "region": aws.region_name,
                    **(
                        {
                            "profile": aws.profile['source_profile'],
                            "role_arn": aws.profile['role_arn']
                        } if 'role_arn' in aws.profile else {
                        }
                    )
                }
            }
        }
    }
)
コード例 #8
0
from azul import (
    config, )
from azul.lambda_layer import (
    DependenciesLayer, )
from azul.terraform import (
    emit_tf, )

layer = DependenciesLayer()

emit_tf({
    "resource": [{
        "aws_lambda_layer_version": {
            "dependencies": {
                "layer_name": config.qualified_resource_name("dependencies"),
                "s3_bucket": config.lambda_layer_bucket,
                "s3_key": layer.object_key
            }
        }
    }],
})
コード例 #9
0
    config, )
from azul.deployment import (
    aws, )
from azul.terraform import (
    emit_tf, )

emit_tf({
    "module": {
        # Not using config.project_root because, "A local path must begin with
        # either ./ or ../"
        # https://www.terraform.io/docs/modules/sources.html#local-paths
        f"chalice_{lambda_name}": {
            "source":
            f"./{lambda_name}",
            "role_arn":
            "${aws_iam_role." + lambda_name + ".arn}",
            "layer_arn":
            "${aws_lambda_layer_version.dependencies.arn}",
            "es_endpoint":
            aws.es_endpoint if config.share_es_domain else
            ("${aws_elasticsearch_domain.index.endpoint}", 443),
            "es_instance_count":
            aws.es_instance_count if config.share_es_domain else
            "${aws_elasticsearch_domain.index.cluster_config[0].instance_count}",
            "cloudwatch_log_group_provisioner":
            f"{lambda_name}_log_group_provisioner"
        }
        for lambda_name in config.lambda_names()
    }
})
コード例 #10
0
emit_tf(
    None if config.disable_monitoring else {
        "resource": [
            *[
                {
                    "aws_route53_health_check": {
                        name: {
                            "fqdn": config.api_lambda_domain(name),
                            "port": 443,
                            "type": "HTTPS",
                            "resource_path": "/health/cached",
                            "failure_threshold": "3",
                            "request_interval": "30",
                            "tags": {
                                "Name": full_name
                            },
                            "regions": ['us-west-2', 'us-east-1', 'eu-west-1'],
                            "measure_latency": True,
                            # This is necessary only because of a Terraform bug:
                            # https://github.com/hashicorp/terraform/issues/22171
                            "lifecycle": {
                                "create_before_destroy": True
                            }
                        },
                    }
                } for name, full_name in (('indexer', config.indexer_name),
                                          ('service', config.service_name))
            ],
            {
                "aws_route53_health_check": {
                    "composite-azul": {
                        # NOTE: There is a 64 character limit on reference name. Terraform adds long string at the end so
                        #  we must be economical about what we add.
                        "reference_name":
                        f"azul-{config.deployment_stage}",
                        "type":
                        "CALCULATED",
                        "child_health_threshold":
                        2,
                        "child_healthchecks": [
                            "${aws_route53_health_check." + "indexer" +
                            ".id}", "${aws_route53_health_check." + "service" +
                            ".id}"
                        ],
                        "measure_latency":
                        True,
                        "cloudwatch_alarm_region":
                        aws.region_name,
                        "tags": {
                            "Name": f"azul-composite-{config.deployment_stage}"
                        }
                    }
                }
            },
            *[
                {
                    "aws_route53_health_check": {
                        name: {
                            "fqdn": domain,
                            "port": 443,
                            "type": "HTTPS",
                            "resource_path": path,
                            "failure_threshold": "3",
                            "request_interval": "30",
                            "tags": {
                                "Name": full_name
                            },
                            "measure_latency": True,
                            # This is necessary only because of a Terraform bug:
                            # https://github.com/hashicorp/terraform/issues/22171
                            "lifecycle": {
                                "create_before_destroy": True
                            }
                        }
                    }
                } for name, domain, full_name, path in (
                    ("data-browser", config.data_browser_domain,
                     config.data_browser_name, '/explore'),
                    ("data-portal", config.data_browser_domain,
                     config.data_portal_name, '/'))
            ],
            {
                "aws_route53_health_check": {
                    "composite-portal": {
                        "reference_name":
                        f"portal-{config.deployment_stage}",
                        "type":
                        "CALCULATED",
                        "child_health_threshold":
                        2,
                        "child_healthchecks": [
                            "${aws_route53_health_check." + "data-browser" +
                            ".id}", "${aws_route53_health_check." +
                            "data-portal" + ".id}"
                        ],
                        "measure_latency":
                        True,
                        "cloudwatch_alarm_region":
                        aws.region_name,
                        "tags": {
                            "Name":
                            f"azul-portal-composite-{config.deployment_stage}"
                        }
                    }
                }
            }
        ]
    })
コード例 #11
0
emit_tf({
    "resource": [{
        "aws_sqs_queue": {
            **{
                config.unqual_notifications_queue_name(retry=retry): {
                    "name":
                    config.notifications_queue_name(retry=retry),
                    "visibility_timeout_seconds":
                    config.contribution_lambda_timeout(retry=retry) + 10,
                    "message_retention_seconds":
                    24 * 60 * 60,
                    "redrive_policy":
                    json.dumps({
                        "maxReceiveCount":
                        9 if retry else 1,
                        "deadLetterTargetArn":
                        "${aws_sqs_queue.%s.arn}" % config.unqual_notifications_queue_name(retry=not retry,
                                                                                           fail=retry)
                    })
                }
                for retry in (False, True)
            },
            **{
                config.unqual_tallies_queue_name(retry=retry): {
                    "name":
                    config.tallies_queue_name(retry=retry),
                    "fifo_queue":
                    True,
                    "delay_seconds":
                    config.es_refresh_interval + 9,
                    "visibility_timeout_seconds":
                    config.aggregation_lambda_timeout(retry=retry) + 10,
                    "message_retention_seconds":
                    24 * 60 * 60,
                    "redrive_policy":
                    json.dumps({
                        "maxReceiveCount":
                        9 if retry else 1,
                        "deadLetterTargetArn":
                        "${aws_sqs_queue.%s.arn}" % config.unqual_tallies_queue_name(retry=not retry,
                                                                                     fail=retry)
                    })
                }
                for retry in (False, True)
            },
            config.unqual_notifications_queue_name(fail=True): {
                "name": config.notifications_queue_name(fail=True),
                "message_retention_seconds": 14 * 24 * 60 * 60,
            },
            config.unqual_tallies_queue_name(fail=True): {
                "fifo_queue": True,
                "name": config.tallies_queue_name(fail=True),
                "message_retention_seconds": 14 * 24 * 60 * 60,
            }
        }
    }]
})
コード例 #12
0
actual_component_path = Path(__file__).absolute().parent
require(
    expected_component_path.samefile(actual_component_path),
    f"The current Terraform component is set to '{config.terraform_component}'. "
    f"You should therefore be in '{expected_component_path}'")

emit_tf({
    "data": [{
        "aws_caller_identity": {
            "current": {}
        }
    }, {
        "aws_region": {
            "current": {}
        }
    }, *([{
        "google_client_config": {
            "current": {}
        }
    }] if config.enable_gcp() else [])],
    "locals": {
        "account_id":
        "${data.aws_caller_identity.current.account_id}",
        "region":
        "${data.aws_region.current.name}",
        "google_project":
        "${data.google_client_config.current.project}"
        if config.enable_gcp() else None
    },
})