def _setup_elasticsearch_7_10_fgac(self) -> None:
domain_name = "wrangler-es-7-10-fgac"
validate_domain_name(domain_name)
domain_arn = f"arn:aws:es:{self.region}:{self.account}:domain/{domain_name}"
domain = opensearch.Domain(
self,
domain_name,
domain_name=domain_name,
version=opensearch.EngineVersion.ELASTICSEARCH_7_10,
capacity=opensearch.CapacityConfig(
data_node_instance_type="t3.small.search", data_nodes=1),
access_policies=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=["es:*"],
principals=[iam.AnyPrincipal()], # FGACs
resources=[f"{domain_arn}/*"],
)
],
fine_grained_access_control=opensearch.AdvancedSecurityOptions(
master_user_name=self.username,
master_user_password=self.password_secret,
),
node_to_node_encryption=True,
encryption_at_rest=opensearch.EncryptionAtRestOptions(
enabled=True, kms_key=self.key),
enforce_https=True,
removal_policy=RemovalPolicy.DESTROY,
)
CfnOutput(self,
f"DomainEndpoint-{domain_name}",
value=domain.domain_endpoint)
def _setup_opensearch_1_0(self) -> None:
domain_name = "wrangler-os-1-0"
validate_domain_name(domain_name)
domain_arn = f"arn:aws:es:{self.region}:{self.account}:domain/{domain_name}"
domain = opensearch.Domain(
self,
domain_name,
domain_name=domain_name,
version=opensearch.EngineVersion.OPENSEARCH_1_0,
capacity=opensearch.CapacityConfig(
data_node_instance_type="t3.small.search", data_nodes=1),
access_policies=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=["es:*"],
principals=[iam.AccountRootPrincipal()],
resources=[f"{domain_arn}/*"],
)
],
removal_policy=RemovalPolicy.DESTROY,
)
CfnOutput(self,
f"DomainEndpoint-{domain_name}",
value=domain.domain_endpoint)
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
OPENSEARCH_DOMAIN_NAME = cdk.CfnParameter(
self,
'OpenSearchDomainName',
type='String',
description='Amazon OpenSearch Service domain name',
default='opensearch-{}'.format(''.join(
random.sample((string.ascii_letters), k=5))),
allowed_pattern='[a-z]+[A-Za-z0-9\-]+')
EC2_KEY_PAIR_NAME = cdk.CfnParameter(
self,
'EC2KeyPairName',
type='String',
description='Amazon EC2 Instance KeyPair name')
#XXX: For createing Amazon MWAA in the existing VPC,
# remove comments from the below codes and
# comments out vpc = aws_ec2.Vpc(..) codes,
# then pass -c vpc_name=your-existing-vpc to cdk command
# for example,
# cdk -c vpc_name=your-existing-vpc syth
#
# vpc_name = self.node.try_get_context('vpc_name')
# vpc = aws_ec2.Vpc.from_lookup(self, 'ExistingVPC',
# is_default=True,
# vpc_name=vpc_name
# )
vpc = aws_ec2.Vpc(
self,
"OpenSearchVPC",
max_azs=3,
gateway_endpoints={
"S3":
aws_ec2.GatewayVpcEndpointOptions(
service=aws_ec2.GatewayVpcEndpointAwsService.S3)
})
#XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/InstanceClass.html
#XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/InstanceSize.html#aws_cdk.aws_ec2.InstanceSize
ec2_instance_type = aws_ec2.InstanceType.of(
aws_ec2.InstanceClass.BURSTABLE3, aws_ec2.InstanceSize.MEDIUM)
sg_bastion_host = aws_ec2.SecurityGroup(
self,
"BastionHostSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for an bastion host',
security_group_name='bastion-host-sg')
cdk.Tags.of(sg_bastion_host).add('Name', 'bastion-host-sg')
#TODO: SHOULD restrict IP range allowed to ssh acces
sg_bastion_host.add_ingress_rule(peer=aws_ec2.Peer.ipv4("0.0.0.0/0"),
connection=aws_ec2.Port.tcp(22),
description='SSH access')
bastion_host = aws_ec2.Instance(
self,
"BastionHost",
vpc=vpc,
instance_type=ec2_instance_type,
machine_image=aws_ec2.MachineImage.latest_amazon_linux(),
vpc_subnets=aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.PUBLIC),
security_group=sg_bastion_host,
key_name=EC2_KEY_PAIR_NAME.value_as_string)
sg_use_opensearch = aws_ec2.SecurityGroup(
self,
"OpenSearchClientSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for an opensearch client',
security_group_name='use-opensearch-cluster-sg')
cdk.Tags.of(sg_use_opensearch).add('Name', 'use-opensearch-cluster-sg')
sg_opensearch_cluster = aws_ec2.SecurityGroup(
self,
"OpenSearchSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for an opensearch cluster',
security_group_name='opensearch-cluster-sg')
cdk.Tags.of(sg_opensearch_cluster).add('Name', 'opensearch-cluster-sg')
sg_opensearch_cluster.add_ingress_rule(
peer=sg_opensearch_cluster,
connection=aws_ec2.Port.all_tcp(),
description='opensearch-cluster-sg')
sg_opensearch_cluster.add_ingress_rule(
peer=sg_use_opensearch,
connection=aws_ec2.Port.tcp(443),
description='use-opensearch-cluster-sg')
sg_opensearch_cluster.add_ingress_rule(
peer=sg_use_opensearch,
connection=aws_ec2.Port.tcp_range(9200, 9300),
description='use-opensearch-cluster-sg')
sg_opensearch_cluster.add_ingress_rule(
peer=sg_bastion_host,
connection=aws_ec2.Port.tcp(443),
description='bastion-host-sg')
sg_opensearch_cluster.add_ingress_rule(
peer=sg_bastion_host,
connection=aws_ec2.Port.tcp_range(9200, 9300),
description='bastion-host-sg')
master_user_secret = aws_secretsmanager.Secret(
self,
"OpenSearchMasterUserSecret",
generate_secret_string=aws_secretsmanager.SecretStringGenerator(
secret_string_template=json.dumps({"username": "******"}),
generate_string_key="password",
# Master password must be at least 8 characters long and contain at least one uppercase letter,
# one lowercase letter, one number, and one special character.
password_length=8))
#XXX: aws cdk elastsearch example - https://github.com/aws/aws-cdk/issues/2873
# You should camelCase the property names instead of PascalCase
opensearch_domain = aws_opensearchservice.Domain(
self,
"OpenSearch",
domain_name=OPENSEARCH_DOMAIN_NAME.value_as_string,
version=aws_opensearchservice.EngineVersion.OPENSEARCH_1_0,
capacity={
"master_nodes": 3,
"master_node_instance_type": "r6g.large.search",
"data_nodes": 3,
"data_node_instance_type": "r6g.large.search"
},
ebs={
"volume_size": 10,
"volume_type": aws_ec2.EbsDeviceVolumeType.GP2
},
#XXX: az_count must be equal to vpc subnets count.
zone_awareness={"availability_zone_count": 3},
logging={
"slow_search_log_enabled": True,
"app_log_enabled": True,
"slow_index_log_enabled": True
},
fine_grained_access_control=aws_opensearchservice.
AdvancedSecurityOptions(
master_user_name=master_user_secret.secret_value_from_json(
"username").to_string(),
master_user_password=master_user_secret.secret_value_from_json(
"password")),
# Enforce HTTPS is required when fine-grained access control is enabled.
enforce_https=True,
# Node-to-node encryption is required when fine-grained access control is enabled
node_to_node_encryption=True,
# Encryption-at-rest is required when fine-grained access control is enabled.
encryption_at_rest={"enabled": True},
use_unsigned_basic_auth=True,
security_groups=[sg_opensearch_cluster],
automated_snapshot_start_hour=17, # 2 AM (GTM+9)
vpc=vpc,
vpc_subnets=[
aws_ec2.SubnetSelection(
one_per_az=True,
subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT)
],
removal_policy=cdk.RemovalPolicy.
DESTROY # default: cdk.RemovalPolicy.RETAIN
)
cdk.Tags.of(opensearch_domain).add(
'Name', f'{OPENSEARCH_DOMAIN_NAME.value_as_string}')
cdk.CfnOutput(self,
'BastionHostId',
value=bastion_host.instance_id,
export_name='BastionHostId')
cdk.CfnOutput(self,
'OpenSearchDomainEndpoint',
value=opensearch_domain.domain_endpoint,
export_name='OpenSearchDomainEndpoint')
cdk.CfnOutput(
self,
'OpenSearchDashboardsURL',
value=f"{opensearch_domain.domain_endpoint}/_dashboards/",
export_name='OpenSearchDashboardsURL')
cdk.CfnOutput(self,
'MasterUserSecretId',
value=master_user_secret.secret_name,
export_name='MasterUserSecretId')
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
OPENSEARCH_DOMAIN_NAME = cdk.CfnParameter(
self,
'OpenSearchDomainName',
type='String',
description='Amazon OpenSearch Service domain name',
default='opensearch-{}'.format(''.join(
random.sample((string.ascii_letters), k=5))),
allowed_pattern='[a-z]+[A-Za-z0-9\-]+')
OPENSEARCH_INDEX_NAME = cdk.CfnParameter(
self,
'SearchIndexName',
type='String',
description='Amazon OpenSearch Service index name')
EC2_KEY_PAIR_NAME = cdk.CfnParameter(
self,
'EC2KeyPairName',
type='String',
description='Amazon EC2 Instance KeyPair name')
# vpc_name = self.node.try_get_context("vpc_name")
# vpc = aws_ec2.Vpc.from_lookup(self, "ExistingVPC",
# is_default=True,
# vpc_name=vpc_name)
#
vpc = aws_ec2.Vpc(
self,
"EKKStackVPC",
max_azs=3,
gateway_endpoints={
"S3":
aws_ec2.GatewayVpcEndpointOptions(
service=aws_ec2.GatewayVpcEndpointAwsService.S3)
})
#XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/InstanceClass.html
#XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/InstanceSize.html#aws_cdk.aws_ec2.InstanceSize
ec2_instance_type = aws_ec2.InstanceType.of(
aws_ec2.InstanceClass.BURSTABLE3, aws_ec2.InstanceSize.MEDIUM)
sg_bastion_host = aws_ec2.SecurityGroup(
self,
"BastionHostSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for an bastion host',
security_group_name='bastion-host-sg')
cdk.Tags.of(sg_bastion_host).add('Name', 'bastion-host-sg')
#TODO: SHOULD restrict IP range allowed to ssh acces
sg_bastion_host.add_ingress_rule(peer=aws_ec2.Peer.ipv4("0.0.0.0/0"),
connection=aws_ec2.Port.tcp(22),
description='SSH access')
bastion_host = aws_ec2.Instance(
self,
"BastionHost",
vpc=vpc,
instance_type=ec2_instance_type,
machine_image=aws_ec2.MachineImage.latest_amazon_linux(),
vpc_subnets=aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.PUBLIC),
security_group=sg_bastion_host,
key_name=EC2_KEY_PAIR_NAME.value_as_string)
sg_use_opensearch = aws_ec2.SecurityGroup(
self,
"OpenSearchClientSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for an opensearch client',
security_group_name='use-opensearch-cluster-sg')
cdk.Tags.of(sg_use_opensearch).add('Name', 'use-opensearch-cluster-sg')
sg_opensearch_cluster = aws_ec2.SecurityGroup(
self,
"OpenSearchSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for an opensearch cluster',
security_group_name='opensearch-cluster-sg')
cdk.Tags.of(sg_opensearch_cluster).add('Name', 'opensearch-cluster-sg')
sg_opensearch_cluster.add_ingress_rule(
peer=sg_opensearch_cluster,
connection=aws_ec2.Port.all_tcp(),
description='opensearch-cluster-sg')
sg_opensearch_cluster.add_ingress_rule(
peer=sg_use_opensearch,
connection=aws_ec2.Port.tcp(443),
description='use-opensearch-cluster-sg')
sg_opensearch_cluster.add_ingress_rule(
peer=sg_use_opensearch,
connection=aws_ec2.Port.tcp_range(9200, 9300),
description='use-opensearch-cluster-sg')
sg_opensearch_cluster.add_ingress_rule(
peer=sg_bastion_host,
connection=aws_ec2.Port.tcp(443),
description='bastion-host-sg')
sg_opensearch_cluster.add_ingress_rule(
peer=sg_bastion_host,
connection=aws_ec2.Port.tcp_range(9200, 9300),
description='bastion-host-sg')
master_user_secret = aws_secretsmanager.Secret(
self,
"OpenSearchMasterUserSecret",
generate_secret_string=aws_secretsmanager.SecretStringGenerator(
secret_string_template=json.dumps({"username": "******"}),
generate_string_key="password",
# Master password must be at least 8 characters long and contain at least one uppercase letter,
# one lowercase letter, one number, and one special character.
password_length=8))
#XXX: aws cdk elastsearch example - https://github.com/aws/aws-cdk/issues/2873
# You should camelCase the property names instead of PascalCase
opensearch_domain = aws_opensearchservice.Domain(
self,
"OpenSearch",
domain_name=OPENSEARCH_DOMAIN_NAME.value_as_string,
version=aws_opensearchservice.EngineVersion.OPENSEARCH_1_0,
#XXX: You cannot use graviton instances with non-graviton instances.
# Use graviton instances as data nodes or use non-graviton instances as master nodes.
capacity={
"master_nodes": 3,
"master_node_instance_type": "r6g.large.search",
"data_nodes": 3,
"data_node_instance_type": "r6g.large.search"
},
ebs={
"volume_size": 10,
"volume_type": aws_ec2.EbsDeviceVolumeType.GP2
},
#XXX: az_count must be equal to vpc subnets count.
zone_awareness={"availability_zone_count": 3},
logging={
"slow_search_log_enabled": True,
"app_log_enabled": True,
"slow_index_log_enabled": True
},
fine_grained_access_control=aws_opensearchservice.
AdvancedSecurityOptions(
master_user_name=master_user_secret.secret_value_from_json(
"username").to_string(),
master_user_password=master_user_secret.secret_value_from_json(
"password")),
# Enforce HTTPS is required when fine-grained access control is enabled.
enforce_https=True,
# Node-to-node encryption is required when fine-grained access control is enabled
node_to_node_encryption=True,
# Encryption-at-rest is required when fine-grained access control is enabled.
encryption_at_rest={"enabled": True},
use_unsigned_basic_auth=True,
security_groups=[sg_opensearch_cluster],
automated_snapshot_start_hour=17, # 2 AM (GTM+9)
vpc=vpc,
vpc_subnets=[
aws_ec2.SubnetSelection(
one_per_az=True,
subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT)
],
removal_policy=cdk.RemovalPolicy.
DESTROY # default: cdk.RemovalPolicy.RETAIN
)
cdk.Tags.of(opensearch_domain).add(
'Name', f'{OPENSEARCH_DOMAIN_NAME.value_as_string}')
S3_BUCKET_SUFFIX = ''.join(
random.sample((string.ascii_lowercase + string.digits), k=7))
s3_bucket = s3.Bucket(
self,
"s3bucket",
removal_policy=cdk.RemovalPolicy.
DESTROY, #XXX: Default: core.RemovalPolicy.RETAIN - The bucket will be orphaned
bucket_name="opskk-stack-{region}-{suffix}".format(
region=cdk.Aws.REGION, suffix=S3_BUCKET_SUFFIX))
firehose_role_policy_doc = aws_iam.PolicyDocument()
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"effect":
aws_iam.Effect.ALLOW,
"resources": [
s3_bucket.bucket_arn, "{}/*".format(
s3_bucket.bucket_arn)
],
"actions": [
"s3:AbortMultipartUpload", "s3:GetBucketLocation",
"s3:GetObject", "s3:ListBucket",
"s3:ListBucketMultipartUploads", "s3:PutObject"
]
}))
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
resources=["*"],
actions=[
"ec2:DescribeVpcs", "ec2:DescribeVpcAttribute",
"ec2:DescribeSubnets", "ec2:DescribeSecurityGroups",
"ec2:DescribeNetworkInterfaces",
"ec2:CreateNetworkInterface",
"ec2:CreateNetworkInterfacePermission",
"ec2:DeleteNetworkInterface"
]))
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
resources=[
opensearch_domain.domain_arn,
"{}/*".format(opensearch_domain.domain_arn)
],
actions=[
"es:DescribeElasticsearchDomain",
"es:DescribeElasticsearchDomains",
"es:DescribeElasticsearchDomainConfig", "es:ESHttpPost",
"es:ESHttpPut"
]))
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
#XXX: https://aws.amazon.com/premiumsupport/knowledge-center/kinesis-data-firehose-delivery-failure/
resources=[
opensearch_domain.domain_arn,
f"{opensearch_domain.domain_arn}/_all/_settings",
f"{opensearch_domain.domain_arn}/_cluster/stats",
f"{opensearch_domain.domain_arn}/{OPENSEARCH_INDEX_NAME.value_as_string}*/_mapping/%FIREHOSE_POLICY_TEMPLATE_PLACEHOLDER%",
f"{opensearch_domain.domain_arn}/_nodes",
f"{opensearch_domain.domain_arn}/_nodes/stats",
f"{opensearch_domain.domain_arn}/_nodes/*/stats",
f"{opensearch_domain.domain_arn}/_stats",
f"{opensearch_domain.domain_arn}/{OPENSEARCH_INDEX_NAME.value_as_string}*/_stats"
],
actions=["es:ESHttpGet"]))
firehose_log_group_name = f"/aws/kinesisfirehose/{OPENSEARCH_INDEX_NAME.value_as_string}"
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
#XXX: The ARN will be formatted as follows:
# arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name}
resources=[
self.format_arn(
service="logs",
resource="log-group",
resource_name="{}:log-stream:*".format(
firehose_log_group_name),
arn_format=cdk.ArnFormat.COLON_RESOURCE_NAME)
],
actions=["logs:PutLogEvents"]))
firehose_role = aws_iam.Role(
self,
"KinesisFirehoseServiceRole",
role_name=
f"KinesisFirehoseServiceRole-{OPENSEARCH_INDEX_NAME.value_as_string}-{cdk.Aws.REGION}",
assumed_by=aws_iam.ServicePrincipal("firehose.amazonaws.com"),
#XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221
inline_policies={"firehose_role_policy": firehose_role_policy_doc})
opensearch_dest_vpc_config = aws_kinesisfirehose.CfnDeliveryStream.VpcConfigurationProperty(
role_arn=firehose_role.role_arn,
security_group_ids=[sg_use_opensearch.security_group_id],
subnet_ids=vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT).subnet_ids)
opensearch_dest_config = aws_kinesisfirehose.CfnDeliveryStream.ElasticsearchDestinationConfigurationProperty(
index_name=OPENSEARCH_INDEX_NAME.value_as_string,
role_arn=firehose_role.role_arn,
s3_configuration={
"bucketArn": s3_bucket.bucket_arn,
"bufferingHints": {
"intervalInSeconds": 60,
"sizeInMBs": 1
},
"cloudWatchLoggingOptions": {
"enabled": True,
"logGroupName": firehose_log_group_name,
"logStreamName": "S3Backup"
},
"compressionFormat":
"UNCOMPRESSED", # [GZIP | HADOOP_SNAPPY | Snappy | UNCOMPRESSED | ZIP]
# Kinesis Data Firehose automatically appends the “YYYY/MM/dd/HH/” UTC prefix to delivered S3 files. You can also specify
# an extra prefix in front of the time format and add "/" to the end to have it appear as a folder in the S3 console.
"prefix": f"{OPENSEARCH_INDEX_NAME.value_as_string}/",
"roleArn": firehose_role.role_arn
},
buffering_hints={
"intervalInSeconds": 60,
"sizeInMBs": 1
},
cloud_watch_logging_options={
"enabled": True,
"logGroupName": firehose_log_group_name,
"logStreamName": "ElasticsearchDelivery"
},
domain_arn=opensearch_domain.domain_arn,
index_rotation_period=
"NoRotation", # [NoRotation | OneDay | OneHour | OneMonth | OneWeek]
retry_options={"durationInSeconds": 60},
s3_backup_mode=
"FailedDocumentsOnly", # [AllDocuments | FailedDocumentsOnly]
vpc_configuration=opensearch_dest_vpc_config)
firehose_to_ops_delivery_stream = aws_kinesisfirehose.CfnDeliveryStream(
self,
"KinesisFirehoseToES",
delivery_stream_name=OPENSEARCH_INDEX_NAME.value_as_string,
delivery_stream_type="DirectPut",
elasticsearch_destination_configuration=opensearch_dest_config,
tags=[{
"key": "Name",
"value": OPENSEARCH_INDEX_NAME.value_as_string
}])
cdk.CfnOutput(self,
'BastionHostId',
value=bastion_host.instance_id,
export_name='BastionHostId')
cdk.CfnOutput(self,
'OpenSearchDomainEndpoint',
value=opensearch_domain.domain_endpoint,
export_name='OpenSearchDomainEndpoint')
cdk.CfnOutput(
self,
'OpenSearchDashboardsURL',
value=f"{opensearch_domain.domain_endpoint}/_dashboards/",
export_name='OpenSearchDashboardsURL')
cdk.CfnOutput(self,
'MasterUserSecretId',
value=master_user_secret.secret_name,
export_name='MasterUserSecretId')
cdk.CfnOutput(self,
'{}_S3DestBucket'.format(self.stack_name),
value=s3_bucket.bucket_name,
export_name='S3DestBucket')
cdk.CfnOutput(self,
'FirehoseRoleArn',
value=firehose_role.role_arn,
export_name='FirehoseRoleArn')
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
################################################################################
# VPC
vpc = ec2.Vpc(self, "Monitoring VPC", max_azs=3)
################################################################################
# Amazon OpenSearch Service domain
es_sec_grp = ec2.SecurityGroup(
self,
'OpenSearchSecGrpMonitoring',
vpc=vpc,
allow_all_outbound=True,
security_group_name='OpenSearchSecGrpMonitoring')
es_sec_grp.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(80))
es_sec_grp.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(443))
domain = opensearch.Domain(
self,
'opensearch-service-monitor',
version=opensearch.EngineVersion.
OPENSEARCH_1_0, # Upgrade when CDK upgrades
domain_name=DOMAIN_NAME,
removal_policy=core.RemovalPolicy.DESTROY,
capacity=opensearch.CapacityConfig(
data_node_instance_type=DOMAIN_DATA_NODE_INSTANCE_TYPE,
data_nodes=DOMAIN_DATA_NODE_INSTANCE_COUNT,
master_node_instance_type=DOMAIN_MASTER_NODE_INSTANCE_TYPE,
master_nodes=DOMAIN_MASTER_NODE_INSTANCE_COUNT,
warm_instance_type=DOMAIN_UW_NODE_INSTANCE_TYPE,
warm_nodes=DOMAIN_UW_NODE_INSTANCE_COUNT),
ebs=opensearch.EbsOptions(enabled=True,
volume_size=DOMAIN_INSTANCE_VOLUME_SIZE,
volume_type=ec2.EbsDeviceVolumeType.GP2),
vpc=vpc,
vpc_subnets=[ec2.SubnetType.PUBLIC],
security_groups=[es_sec_grp],
zone_awareness=opensearch.ZoneAwarenessConfig(
enabled=True, availability_zone_count=DOMAIN_AZ_COUNT),
enforce_https=True,
node_to_node_encryption=True,
encryption_at_rest={"enabled": True},
use_unsigned_basic_auth=True,
fine_grained_access_control={
"master_user_name":
DOMAIN_ADMIN_UNAME,
"master_user_password":
core.SecretValue.plain_text(DOMAIN_ADMIN_PW)
})
core.CfnOutput(
self,
"MasterUser",
value=DOMAIN_ADMIN_UNAME,
description="Master User Name for Amazon OpenSearch Service")
core.CfnOutput(
self,
"MasterPW",
value=DOMAIN_ADMIN_PW,
description="Master User Password for Amazon OpenSearch Service")
################################################################################
# Dynamo DB table for time stamp tracking
table = ddb.Table(
self,
'opensearch-monitor-lambda-timestamp',
table_name=TABLE_NAME,
partition_key=ddb.Attribute(name="domain",
type=ddb.AttributeType.STRING),
sort_key=ddb.Attribute(name='region',
type=ddb.AttributeType.STRING),
removal_policy=core.RemovalPolicy.DESTROY)
################################################################################
# Lambda monitoring function
lambda_func = lambda_.Function(
self,
'CWMetricsToOpenSearch',
function_name="CWMetricsToOpenSearch_monitoring",
runtime=lambda_.Runtime.PYTHON_3_8,
code=lambda_.Code.asset('CWMetricsToOpenSearch'),
handler='handler.handler',
memory_size=1024,
timeout=core.Duration.minutes(10),
vpc=vpc)
table.grant_read_data(lambda_func)
table.grant_write_data(lambda_func)
lambda_func.add_environment('TABLE', table.table_name)
lambda_func.add_environment('DOMAIN_ENDPOINT',
'https://' + domain.domain_endpoint)
lambda_func.add_environment('DOMAIN_ADMIN_UNAME', DOMAIN_ADMIN_UNAME)
lambda_func.add_environment('DOMAIN_ADMIN_PW', DOMAIN_ADMIN_PW)
lambda_func.add_environment('REGIONS', REGIONS_TO_MONITOR)
# When the domain is created here, restrict access
lambda_func.add_to_role_policy(
iam.PolicyStatement(actions=['es:*'], resources=['*']))
# The function needs to read CW events. Restrict
lambda_func.add_to_role_policy(
iam.PolicyStatement(actions=['cloudwatch:*'], resources=['*']))
lambda_schedule = events.Schedule.rate(
core.Duration.seconds(LAMBDA_INTERVAL))
event_lambda_target = targets.LambdaFunction(handler=lambda_func)
events.Rule(self,
"Monitoring",
enabled=True,
schedule=lambda_schedule,
targets=[event_lambda_target])
################################################################################
# Lambda for CW Logs
lambda_func_cw_logs = lambda_.Function(
self,
'CWLogsToOpenSearch',
function_name="CWLogsToOpenSearch_monitoring",
runtime=lambda_.Runtime.NODEJS_12_X,
code=lambda_.Code.asset('CWLogsToOpenSearch'),
handler='index.handler',
vpc=vpc)
# # Load Amazon OpenSearch Service Domain to env variable
lambda_func_cw_logs.add_environment('DOMAIN_ENDPOINT',
domain.domain_endpoint)
# # When the domain is created here, restrict access
lambda_func_cw_logs.add_to_role_policy(
iam.PolicyStatement(actions=['es:*'], resources=['*']))
# # The function needs to read CW Logs. Restrict
lambda_func_cw_logs.add_to_role_policy(
iam.PolicyStatement(actions=['logs:*'], resources=['*']))
# Add permission to create CW logs trigger for all specified region and current account, as region does not have an option to be wildcard
account_id = boto3.client("sts").get_caller_identity()["Account"]
for region in json.loads(REGIONS_TO_MONITOR):
lambda_func_cw_logs.add_permission(
id="lambda-cw-logs-permission-" + region,
principal=iam.ServicePrincipal("logs.amazonaws.com"),
action="lambda:InvokeFunction",
source_arn="arn:aws:logs:" + region + ":" + account_id +
":*:*:*")
################################################################################
# Jump host for SSH tunneling and direct access
sn_public = ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC)
amzn_linux = ec2.MachineImage.latest_amazon_linux(
generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2,
edition=ec2.AmazonLinuxEdition.STANDARD,
virtualization=ec2.AmazonLinuxVirt.HVM,
storage=ec2.AmazonLinuxStorage.GENERAL_PURPOSE)
# Instance Role and SSM Managed Policy
role = iam.Role(self,
"InstanceSSM",
assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"))
role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonEC2RoleforSSM"))
role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonSSMManagedInstanceCore"))
instance = ec2.Instance(
self,
'instance',
instance_type=ec2.InstanceType(EC2_INSTANCE_TYPE),
vpc=vpc,
machine_image=amzn_linux,
vpc_subnets=sn_public,
key_name=EC2_KEY_NAME,
role=role,
)
instance.connections.allow_from_any_ipv4(ec2.Port.tcp(22), 'SSH')
instance.connections.allow_from_any_ipv4(ec2.Port.tcp(443), 'HTTPS')
stmt = iam.PolicyStatement(actions=['es:*'],
resources=[domain.domain_arn])
instance.add_to_role_policy(stmt)
# Create SNS topic, subscription, IAM roles, Policies
sns_topic = sns.Topic(self, "cdk_monitoring_topic")
sns_topic.add_subscription(
subscriptions.EmailSubscription(SNS_NOTIFICATION_EMAIL))
sns_policy_statement = iam.PolicyStatement(
actions=["sns:publish"],
resources=[sns_topic.topic_arn],
effect=iam.Effect.ALLOW)
sns_policy = iam.ManagedPolicy(self, "cdk_monitoring_policy")
sns_policy.add_statements(sns_policy_statement)
sns_role = iam.Role(
self,
"cdk_monitoring_sns_role",
assumed_by=iam.ServicePrincipal("es.amazonaws.com"))
sns_role.add_managed_policy(sns_policy)
dirname = os.path.dirname(__file__)
dashboards_asset = Asset(
self,
"DashboardsAsset",
path=os.path.join(dirname,
'export_opensearch_dashboards_V1_0.ndjson'))
dashboards_asset.grant_read(instance.role)
dashboards_asset_path = instance.user_data.add_s3_download_command(
bucket=dashboards_asset.bucket,
bucket_key=dashboards_asset.s3_object_key,
)
nginx_asset = Asset(self,
"NginxAsset",
path=os.path.join(dirname,
'nginx_opensearch.conf'))
nginx_asset.grant_read(instance.role)
nginx_asset_path = instance.user_data.add_s3_download_command(
bucket=nginx_asset.bucket,
bucket_key=nginx_asset.s3_object_key,
)
alerting_asset = Asset(self,
"AlertingAsset",
path=os.path.join(dirname, 'create_alerts.sh'))
alerting_asset.grant_read(instance.role)
alerting_asset_path = instance.user_data.add_s3_download_command(
bucket=alerting_asset.bucket,
bucket_key=alerting_asset.s3_object_key,
)
instance.user_data.add_commands(
"yum update -y",
"yum install jq -y",
"amazon-linux-extras install nginx1.12",
"cd /tmp/assets",
"mv {} export_opensearch_dashboards_V1_0.ndjson".format(
dashboards_asset_path),
"mv {} nginx_opensearch.conf".format(nginx_asset_path),
"mv {} create_alerts.sh".format(alerting_asset_path),
"openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/nginx/cert.key -out /etc/nginx/cert.crt -subj /C=US/ST=./L=./O=./CN=.\n"
"cp nginx_opensearch.conf /etc/nginx/conf.d/",
"sed -i 's/DEFAULT_DOMAIN_NAME/" + DOMAIN_NAME +
"/g' /tmp/assets/export_opensearch_dashboards_V1_0.ndjson",
"sed -i 's/DOMAIN_ENDPOINT/" + domain.domain_endpoint +
"/g' /etc/nginx/conf.d/nginx_opensearch.conf",
"sed -i 's/DOMAIN_ENDPOINT/" + domain.domain_endpoint +
"/g' /tmp/assets/create_alerts.sh",
"sed -i 's=LAMBDA_CW_LOGS_ROLE_ARN=" +
lambda_func_cw_logs.role.role_arn +
"=g' /tmp/assets/create_alerts.sh",
"sed -i 's=SNS_ROLE_ARN=" + sns_role.role_arn +
"=g' /tmp/assets/create_alerts.sh",
"sed -i 's/SNS_TOPIC_ARN/" + sns_topic.topic_arn +
"/g' /tmp/assets/create_alerts.sh",
"sed -i 's=DOMAIN_ADMIN_UNAME=" + DOMAIN_ADMIN_UNAME +
"=g' /tmp/assets/create_alerts.sh",
"sed -i 's=DOMAIN_ADMIN_PW=" + DOMAIN_ADMIN_PW +
"=g' /tmp/assets/create_alerts.sh",
"systemctl restart nginx.service",
"chmod 500 create_alerts.sh",
"sleep 5",
"bash --verbose create_alerts.sh",
)
core.CfnOutput(self,
"Dashboards URL (via Jump host)",
value="https://" + instance.instance_public_ip,
description="Dashboards URL via Jump host")
core.CfnOutput(
self,
"SNS Subscription Alert Message",
value=SNS_NOTIFICATION_EMAIL,
description="Please confirm your SNS subscription receievedt at")