def rds_cluster(self, vpc, ec2sg, rds_type, rds_param=None):
postgres = rds.DatabaseCluster(
self,
"adl-" + rds_type,
default_database_name="adldb",
engine=getattr(rds.DatabaseClusterEngine, rds_type),
instance_props=rds.InstanceProps(
vpc=vpc['vpc'],
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE),
instance_type=ec2.InstanceType(
instance_type_identifier="t3.medium")),
master_user=rds.Login(username="******"),
backup=rds.BackupProps(retention=core.Duration.days(7),
preferred_window='01:00-02:00'),
parameter_group=rds_param,
preferred_maintenance_window="Sun:23:45-Mon:00:15",
removal_policy=core.RemovalPolicy.DESTROY,
storage_encrypted=True)
postgres.connections.allow_from(ec2sg['ec2-sg'], ec2.Port.all_tcp())
def _setup_mysql(self) -> None:
port = 3306
database = "test"
schema = "test"
aurora_mysql = rds.DatabaseCluster(
self,
"aws-data-wrangler-aurora-cluster-mysql",
removal_policy=RemovalPolicy.DESTROY,
engine=rds.DatabaseClusterEngine.aurora_mysql(
version=rds.AuroraMysqlEngineVersion.VER_5_7_12,
),
cluster_identifier="mysql-cluster-wrangler",
instances=1,
default_database_name=database,
credentials=rds.Credentials.from_password(
username=self.db_username,
password=self.db_password_secret,
),
port=port,
backup=rds.BackupProps(retention=Duration.days(1)),
instance_props=rds.InstanceProps(
vpc=self.vpc,
security_groups=[self.db_security_group],
publicly_accessible=True,
),
subnet_group=self.rds_subnet_group,
s3_import_buckets=[self.bucket],
s3_export_buckets=[self.bucket],
)
glue.Connection(
self,
"aws-data-wrangler-mysql-glue-connection",
description="Connect to Aurora (MySQL).",
type=glue.ConnectionType.JDBC,
connection_name="aws-data-wrangler-mysql",
properties={
"JDBC_CONNECTION_URL": f"jdbc:mysql://{aurora_mysql.cluster_endpoint.hostname}:{port}/{database}",
"USERNAME": self.db_username,
"PASSWORD": self.db_password,
},
subnet=self.vpc.private_subnets[0],
security_groups=[self.db_security_group],
)
glue.Connection(
self,
"aws-data-wrangler-mysql-glue-connection-ssl",
description="Connect to Aurora (MySQL) with SSL.",
type=glue.ConnectionType.JDBC,
connection_name="aws-data-wrangler-mysql-ssl",
properties={
"JDBC_CONNECTION_URL": f"jdbc:mysql://{aurora_mysql.cluster_endpoint.hostname}:{port}/{database}",
"USERNAME": self.db_username,
"PASSWORD": self.db_password,
"JDBC_ENFORCE_SSL": "true",
"CUSTOM_JDBC_CERT": "s3://rds-downloads/rds-combined-ca-bundle.pem",
},
subnet=self.vpc.private_subnets[0],
security_groups=[self.db_security_group],
)
secrets.Secret(
self,
"aws-data-wrangler-mysql-secret",
secret_name="aws-data-wrangler/mysql",
description="MySQL credentials",
generate_secret_string=secrets.SecretStringGenerator(
generate_string_key="dummy",
secret_string_template=json.dumps(
{
"username": self.db_username,
"password": self.db_password,
"engine": "mysql",
"host": aurora_mysql.cluster_endpoint.hostname,
"port": port,
"dbClusterIdentifier": aurora_mysql.cluster_identifier,
"dbname": database,
}
),
),
)
CfnOutput(self, "MysqlAddress", value=aurora_mysql.cluster_endpoint.hostname)
CfnOutput(self, "MysqlPort", value=str(port))
CfnOutput(self, "MysqlDatabase", value=database)
CfnOutput(self, "MysqlSchema", value=schema)
def _setup_postgresql(self) -> None:
port = 3306
database = "postgres"
schema = "public"
pg = rds.ParameterGroup(
self,
"aws-data-wrangler-postgresql-params",
engine=rds.DatabaseClusterEngine.aurora_postgres(
version=rds.AuroraPostgresEngineVersion.VER_11_13,
),
parameters={
"apg_plan_mgmt.capture_plan_baselines": "off",
},
)
aurora_pg = rds.DatabaseCluster(
self,
"aws-data-wrangler-aurora-cluster-postgresql",
removal_policy=RemovalPolicy.DESTROY,
engine=rds.DatabaseClusterEngine.aurora_postgres(
version=rds.AuroraPostgresEngineVersion.VER_11_13,
),
cluster_identifier="postgresql-cluster-wrangler",
instances=1,
credentials=rds.Credentials.from_password(
username=self.db_username,
password=self.db_password_secret,
),
port=port,
backup=rds.BackupProps(retention=Duration.days(1)),
parameter_group=pg,
s3_import_buckets=[self.bucket],
s3_export_buckets=[self.bucket],
instance_props=rds.InstanceProps(
vpc=self.vpc,
security_groups=[self.db_security_group],
publicly_accessible=True,
),
subnet_group=self.rds_subnet_group,
)
glue.Connection(
self,
"aws-data-wrangler-postgresql-glue-connection",
description="Connect to Aurora (PostgreSQL).",
type=glue.ConnectionType.JDBC,
connection_name="aws-data-wrangler-postgresql",
properties={
"JDBC_CONNECTION_URL": f"jdbc:postgresql://{aurora_pg.cluster_endpoint.hostname}:{port}/{database}",
"USERNAME": self.db_username,
"PASSWORD": self.db_password,
},
subnet=self.vpc.private_subnets[0],
security_groups=[self.db_security_group],
)
secrets.Secret(
self,
"aws-data-wrangler-postgresql-secret",
secret_name="aws-data-wrangler/postgresql",
description="Postgresql credentials",
generate_secret_string=secrets.SecretStringGenerator(
generate_string_key="dummy",
secret_string_template=json.dumps(
{
"username": self.db_username,
"password": self.db_password,
"engine": "postgresql",
"host": aurora_pg.cluster_endpoint.hostname,
"port": port,
"dbClusterIdentifier": aurora_pg.cluster_identifier,
"dbname": database,
}
),
),
)
CfnOutput(self, "PostgresqlAddress", value=aurora_pg.cluster_endpoint.hostname)
CfnOutput(self, "PostgresqlPort", value=str(port))
CfnOutput(self, "PostgresqlDatabase", value=database)
CfnOutput(self, "PostgresqlSchema", value=schema)
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# The code that defines your stack goes here
vpc_name = self.node.try_get_context("vpc_name")
vpc = aws_ec2.Vpc.from_lookup(self,
"ExistingVPC",
is_default=True,
vpc_name=vpc_name)
sg_use_mysql = aws_ec2.SecurityGroup(
self,
'MySQLClientSG',
vpc=vpc,
allow_all_outbound=True,
description='security group for mysql client',
security_group_name='use-mysql-sg')
core.Tags.of(sg_use_mysql).add('Name', 'mysql-client-sg')
sg_mysql_server = aws_ec2.SecurityGroup(
self,
'MySQLServerSG',
vpc=vpc,
allow_all_outbound=True,
description='security group for mysql',
security_group_name='mysql-server-sg')
sg_mysql_server.add_ingress_rule(peer=sg_use_mysql,
connection=aws_ec2.Port.tcp(3306),
description='use-mysql-sg')
core.Tags.of(sg_mysql_server).add('Name', 'mysql-server-sg')
rds_subnet_group = aws_rds.SubnetGroup(
self,
'RdsSubnetGroup',
description='subnet group for mysql',
subnet_group_name='aurora-mysql',
vpc_subnets=aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.PRIVATE),
vpc=vpc)
rds_engine = aws_rds.DatabaseClusterEngine.aurora_mysql(
version=aws_rds.AuroraMysqlEngineVersion.VER_2_08_1)
rds_cluster_param_group = aws_rds.ParameterGroup(
self,
'AuroraMySQLClusterParamGroup',
engine=rds_engine,
description='Custom cluster parameter group for aurora-mysql5.7',
parameters={
'innodb_flush_log_at_trx_commit': '2',
'slow_query_log': '1',
'tx_isolation': 'READ-COMMITTED',
'wait_timeout': '300',
'character-set-client-handshake': '0',
'character_set_server': 'utf8mb4',
'collation_server': 'utf8mb4_unicode_ci',
'init_connect': 'SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci'
})
rds_db_param_group = aws_rds.ParameterGroup(
self,
'AuroraMySQLDBParamGroup',
engine=rds_engine,
description='Custom parameter group for aurora-mysql5.7',
parameters={
'slow_query_log': '1',
'tx_isolation': 'READ-COMMITTED',
'wait_timeout': '300',
'init_connect': 'SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci'
})
db_cluster_name = self.node.try_get_context('db_cluster_name')
# #XXX: aws_rds.Credentials.from_username(username, ...) can not be given user specific Secret name
# #XXX: therefore, first create Secret and then use it to create database
# db_secret_name = self.node.try_get_context('db_secret_name')
# #XXX: arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name}
# db_secret_arn = 'arn:aws:secretsmanager:{region}:{account}:secret:{resource_name}'.format(
# region=core.Aws.REGION, account=core.Aws.ACCOUNT_ID, resource_name=db_secret_name)
# db_secret = aws_secretsmanager.Secret.from_secret_arn(self, 'DBSecretFromArn', db_secret_arn)
# rds_credentials = aws_rds.Credentials.from_secret(db_secret)
rds_credentials = aws_rds.Credentials.from_generated_secret("admin")
db_cluster = aws_rds.DatabaseCluster(
self,
'Database',
engine=rds_engine,
credentials=rds_credentials,
instance_props={
'instance_type':
aws_ec2.InstanceType.of(aws_ec2.InstanceClass.BURSTABLE3,
aws_ec2.InstanceSize.MEDIUM),
'parameter_group':
rds_db_param_group,
'vpc_subnets': {
'subnet_type': aws_ec2.SubnetType.PRIVATE
},
'vpc':
vpc,
'auto_minor_version_upgrade':
False,
'security_groups': [sg_mysql_server]
},
instances=2,
parameter_group=rds_cluster_param_group,
cloudwatch_logs_retention=aws_logs.RetentionDays.THREE_DAYS,
cluster_identifier=db_cluster_name,
subnet_group=rds_subnet_group,
backup=aws_rds.BackupProps(retention=core.Duration.days(3),
preferred_window="03:00-04:00"))
sagemaker_notebook_role_policy_doc = aws_iam.PolicyDocument()
sagemaker_notebook_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"effect": aws_iam.Effect.ALLOW,
"resources": [db_cluster.secret.secret_full_arn],
"actions": ["secretsmanager:GetSecretValue"]
}))
sagemaker_notebook_role = aws_iam.Role(
self,
'SageMakerNotebookRoleForRDS',
role_name='AWSSageMakerNotebookRoleForRDS',
assumed_by=aws_iam.ServicePrincipal('sagemaker.amazonaws.com'),
inline_policies={
'AuroraMySQLSecretPolicy': sagemaker_notebook_role_policy_doc
})
cf_readonly_access_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'AWSCloudFormationReadOnlyAccess')
sagemaker_notebook_role.add_managed_policy(cf_readonly_access_policy)
#XXX: skip downloading rds-combined-ca-bundle.pem if not use SSL with a MySQL DB instance
# https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_MySQL.html#MySQL.Concepts.SSLSupport
rds_wb_lifecycle_content = '''#!/bin/bash
sudo -u ec2-user -i <<'EOF'
echo "export AWS_REGION={AWS_Region}" >> ~/.bashrc
source /home/ec2-user/anaconda3/bin/activate python3
pip install --upgrade ipython-sql
pip install --upgrade PyMySQL
pip install --upgrade pretty_errors
source /home/ec2-user/anaconda3/bin/deactivate
cd /home/ec2-user/SageMaker
wget -N https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem
wget -N https://raw.githubusercontent.com/ksmin23/my-aws-cdk-examples/main/rds/sagemaker-aurora_mysql/ipython-sql.ipynb
EOF
'''.format(AWS_Region=core.Aws.REGION)
rds_wb_lifecycle_config_prop = aws_sagemaker.CfnNotebookInstanceLifecycleConfig.NotebookInstanceLifecycleHookProperty(
content=core.Fn.base64(rds_wb_lifecycle_content))
rds_wb_lifecycle_config = aws_sagemaker.CfnNotebookInstanceLifecycleConfig(
self,
'MySQLWorkbenchLifeCycleConfig',
notebook_instance_lifecycle_config_name=
'MySQLWorkbenchLifeCycleConfig',
on_start=[rds_wb_lifecycle_config_prop])
rds_workbench = aws_sagemaker.CfnNotebookInstance(
self,
'AuroraMySQLWorkbench',
instance_type='ml.t3.xlarge',
role_arn=sagemaker_notebook_role.role_arn,
lifecycle_config_name=rds_wb_lifecycle_config.
notebook_instance_lifecycle_config_name,
notebook_instance_name='AuroraMySQLWorkbench',
root_access='Disabled',
security_group_ids=[sg_use_mysql.security_group_name],
subnet_id=vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids[0])
core.CfnOutput(self,
'StackName',
value=self.stack_name,
export_name='StackName')
core.CfnOutput(self, 'VpcId', value=vpc.vpc_id, export_name='VpcId')
core.CfnOutput(self,
'DBClusterName',
value=db_cluster.cluster_identifier,
export_name='DBClusterName')
core.CfnOutput(self,
'DBCluster',
value=db_cluster.cluster_endpoint.socket_address,
export_name='DBCluster')
#XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_secretsmanager/README.html
# secret_arn="arn:aws:secretsmanager:<region>:<account-id-number>:secret:<secret-name>-<random-6-characters>",
core.CfnOutput(self,
'DBSecret',
value=db_cluster.secret.secret_name,
export_name='DBSecret')
core.CfnOutput(self,
'SageMakerRole',
value=sagemaker_notebook_role.role_name,
export_name='SageMakerRole')
core.CfnOutput(self,
'SageMakerNotebookInstance',
value=rds_workbench.notebook_instance_name,
export_name='SageMakerNotebookInstance')
core.CfnOutput(self,
'SageMakerNotebookInstanceLifecycleConfig',
value=rds_workbench.lifecycle_config_name,
export_name='SageMakerNotebookInstanceLifecycleConfig')
def __init__(self, scope: Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
vpc_name = self.node.try_get_context("vpc_name")
vpc = aws_ec2.Vpc.from_lookup(self, "ExistingVPC",
is_default=True,
vpc_name=vpc_name)
sg_use_mysql = aws_ec2.SecurityGroup(self, 'MySQLClientSG',
vpc=vpc,
allow_all_outbound=True,
description='security group for mysql client',
security_group_name='default-mysql-client-sg'
)
cdk.Tags.of(sg_use_mysql).add('Name', 'default-mysql-client-sg')
sg_mysql_server = aws_ec2.SecurityGroup(self, 'MySQLServerSG',
vpc=vpc,
allow_all_outbound=True,
description='security group for mysql',
security_group_name='default-mysql-server-sg'
)
sg_mysql_server.add_ingress_rule(peer=sg_use_mysql, connection=aws_ec2.Port.tcp(3306),
description='default-mysql-client-sg')
sg_mysql_server.add_ingress_rule(peer=sg_mysql_server, connection=aws_ec2.Port.all_tcp(),
description='default-mysql-server-sg')
cdk.Tags.of(sg_mysql_server).add('Name', 'default-mysql-server-sg')
rds_subnet_group = aws_rds.SubnetGroup(self, 'MySQLSubnetGroup',
description='subnet group for mysql',
subnet_group_name='aurora-mysql',
vpc_subnets=aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT),
vpc=vpc
)
rds_engine = aws_rds.DatabaseClusterEngine.aurora_mysql(version=aws_rds.AuroraMysqlEngineVersion.VER_3_01_0)
#XXX: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraMySQL.Reference.html#AuroraMySQL.Reference.Parameters.Cluster
rds_cluster_param_group = aws_rds.ParameterGroup(self, 'AuroraMySQLClusterParamGroup',
engine=rds_engine,
description='Custom cluster parameter group for aurora-mysql8.x',
parameters={
# For Aurora MySQL version 3, Aurora always uses the default value of 1.
# 'innodb_flush_log_at_trx_commit': '2',
'slow_query_log': '1',
# Removed from Aurora MySQL version 3.
# 'tx_isolation': 'READ-COMMITTED',
'wait_timeout': '300',
'character-set-client-handshake': '0',
'character_set_server': 'utf8mb4',
'collation_server': 'utf8mb4_unicode_ci',
'init_connect': 'SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci'
}
)
#XXX: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraMySQL.Reference.html#AuroraMySQL.Reference.Parameters.Instance
rds_db_param_group = aws_rds.ParameterGroup(self, 'AuroraMySQLDBParamGroup',
engine=rds_engine,
description='Custom parameter group for aurora-mysql8.x',
parameters={
'slow_query_log': '1',
# Removed from Aurora MySQL version 3.
# 'tx_isolation': 'READ-COMMITTED',
'wait_timeout': '300',
'init_connect': 'SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci'
}
)
db_cluster_name = self.node.try_get_context('db_cluster_name')
#XXX: aws_rds.Credentials.from_username(username, ...) can not be given user specific Secret name
# therefore, first create Secret and then use it to create database
db_secret_name = self.node.try_get_context('db_secret_name')
#XXX: arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name}
db_secret_arn = 'arn:aws:secretsmanager:{region}:{account}:secret:{resource_name}'.format(
region=cdk.Aws.REGION, account=cdk.Aws.ACCOUNT_ID, resource_name=db_secret_name)
db_secret = aws_secretsmanager.Secret.from_secret_partial_arn(self, 'DBSecretFromArn', db_secret_arn)
rds_credentials = aws_rds.Credentials.from_secret(db_secret)
db_cluster = aws_rds.DatabaseCluster(self, 'Database',
engine=rds_engine,
credentials=rds_credentials, # A username of 'admin' (or 'postgres' for PostgreSQL) and SecretsManager-generated password
instance_props={
'instance_type': aws_ec2.InstanceType.of(aws_ec2.InstanceClass.BURSTABLE3, aws_ec2.InstanceSize.MEDIUM),
'parameter_group': rds_db_param_group,
'vpc_subnets': {
'subnet_type': aws_ec2.SubnetType.PRIVATE_WITH_NAT
},
'vpc': vpc,
'auto_minor_version_upgrade': False,
'security_groups': [sg_mysql_server]
},
instances=2,
parameter_group=rds_cluster_param_group,
cloudwatch_logs_retention=aws_logs.RetentionDays.THREE_DAYS,
cluster_identifier=db_cluster_name,
subnet_group=rds_subnet_group,
backup=aws_rds.BackupProps(
retention=cdk.Duration.days(3),
preferred_window="03:00-04:00"
)
)
cdk.CfnOutput(self, 'DBClusterEndpoint', value=db_cluster.cluster_endpoint.socket_address, export_name='DBClusterEndpoint')
cdk.CfnOutput(self, 'DBClusterReadEndpoint', value=db_cluster.cluster_read_endpoint.socket_address, export_name='DBClusterReadEndpoint')
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
vpc_name = self.node.try_get_context("vpc_name")
vpc = aws_ec2.Vpc.from_lookup(self,
"ExistingVPC",
is_default=True,
vpc_name=vpc_name)
sg_use_mysql = aws_ec2.SecurityGroup(
self,
'MySQLClientSG',
vpc=vpc,
allow_all_outbound=True,
description='security group for mysql client',
security_group_name='use-default-mysql')
core.Tags.of(sg_use_mysql).add('Name', 'use-default-mysql')
sg_mysql_server = aws_ec2.SecurityGroup(
self,
'MySQLServerSG',
vpc=vpc,
allow_all_outbound=True,
description='security group for mysql',
security_group_name='default-mysql-server')
sg_mysql_server.add_ingress_rule(peer=sg_use_mysql,
connection=aws_ec2.Port.tcp(3306),
description='use-default-mysql')
core.Tags.of(sg_mysql_server).add('Name', 'mysql-server')
rds_subnet_group = aws_rds.SubnetGroup(
self,
'RdsSubnetGroup',
description='subnet group for mysql',
subnet_group_name='aurora-mysql',
vpc_subnets=aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.PRIVATE),
vpc=vpc)
rds_engine = aws_rds.DatabaseClusterEngine.aurora_mysql(
version=aws_rds.AuroraMysqlEngineVersion.VER_2_08_1)
rds_cluster_param_group = aws_rds.ParameterGroup(
self,
'AuroraMySQLClusterParamGroup',
engine=rds_engine,
description='Custom cluster parameter group for aurora-mysql5.7',
parameters={
'innodb_flush_log_at_trx_commit': '2',
'slow_query_log': '1',
'tx_isolation': 'READ-COMMITTED',
'wait_timeout': '300',
'character-set-client-handshake': '0',
'character_set_server': 'utf8mb4',
'collation_server': 'utf8mb4_unicode_ci',
'init_connect': 'SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci'
})
rds_db_param_group = aws_rds.ParameterGroup(
self,
'AuroraMySQLDBParamGroup',
engine=rds_engine,
description='Custom parameter group for aurora-mysql5.7',
parameters={
'slow_query_log': '1',
'tx_isolation': 'READ-COMMITTED',
'wait_timeout': '300',
'init_connect': 'SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci'
})
db_cluster_name = self.node.try_get_context('db_cluster_name')
#XXX: aws_rds.Credentials.from_username(username, ...) can not be given user specific Secret name
#XXX: therefore, first create Secret and then use it to create database
db_secret_name = self.node.try_get_context('db_secret_name')
#XXX: arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name}
db_secret_arn = 'arn:aws:secretsmanager:{region}:{account}:secret:{resource_name}'.format(
region=core.Aws.REGION,
account=core.Aws.ACCOUNT_ID,
resource_name=db_secret_name)
db_secret = aws_secretsmanager.Secret.from_secret_arn(
self, 'DBSecretFromArn', db_secret_arn)
rds_credentials = aws_rds.Credentials.from_secret(db_secret)
db_cluster = aws_rds.DatabaseCluster(
self,
'Database',
engine=rds_engine,
credentials=rds_credentials,
instance_props={
'instance_type':
aws_ec2.InstanceType.of(aws_ec2.InstanceClass.BURSTABLE3,
aws_ec2.InstanceSize.MEDIUM),
'parameter_group':
rds_db_param_group,
'vpc_subnets': {
'subnet_type': aws_ec2.SubnetType.PRIVATE
},
'vpc':
vpc,
'auto_minor_version_upgrade':
False,
'security_groups': [sg_mysql_server]
},
instances=2,
parameter_group=rds_cluster_param_group,
cloudwatch_logs_retention=aws_logs.RetentionDays.THREE_DAYS,
cluster_identifier=db_cluster_name,
subnet_group=rds_subnet_group,
backup=aws_rds.BackupProps(retention=core.Duration.days(3),
preferred_window="03:00-04:00"))
sg_mysql_public_proxy = aws_ec2.SecurityGroup(
self,
'MySQLPublicProxySG',
vpc=vpc,
allow_all_outbound=True,
description='security group for mysql public proxy',
security_group_name='default-mysql-public-proxy')
sg_mysql_public_proxy.add_ingress_rule(
peer=aws_ec2.Peer.any_ipv4(),
connection=aws_ec2.Port.tcp(3306),
description='mysql public proxy')
core.Tags.of(sg_mysql_public_proxy).add('Name', 'mysql-public-proxy')
#XXX: Datbase Proxy use only Secret Arn of target database or database cluster
#XXX: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-rds-dbproxy-authformat.html
#XXX: If new Secret for database user is created, it is necessary to update Resource of Proxy IAM Role to access new Secret.
#XXX: Otherwise, new database user can not connect to database by RDS Proxy.
db_proxy = aws_rds.DatabaseProxy(
self,
'DBProxy',
proxy_target=aws_rds.ProxyTarget.from_cluster(db_cluster),
secrets=[db_secret],
vpc=vpc,
db_proxy_name='{}-proxy'.format(db_cluster_name),
idle_client_timeout=core.Duration.minutes(10),
max_connections_percent=90,
max_idle_connections_percent=10,
security_groups=[sg_use_mysql, sg_mysql_public_proxy],
vpc_subnets=aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.PUBLIC))
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
vpc_name = self.node.try_get_context("vpc_name")
vpc = aws_ec2.Vpc.from_lookup(self,
"ExistingVPC",
is_default=True,
vpc_name=vpc_name)
sg_postgresql_client = aws_ec2.SecurityGroup(
self,
'PostgreSQLClientSG',
vpc=vpc,
allow_all_outbound=True,
description='security group for postgresql client',
security_group_name='default-postgresql-client-sg')
cdk.Tags.of(sg_postgresql_client).add('Name',
'default-postgresql-client-sg')
sg_postgresql_server = aws_ec2.SecurityGroup(
self,
'PostgreSQLServerSG',
vpc=vpc,
allow_all_outbound=True,
description='security group for postgresql',
security_group_name='default-postgresql-server-sg')
sg_postgresql_server.add_ingress_rule(
peer=sg_postgresql_client,
connection=aws_ec2.Port.tcp(5432),
description='default-postgresql-client-sg')
sg_postgresql_server.add_ingress_rule(
peer=sg_postgresql_server,
connection=aws_ec2.Port.all_tcp(),
description='default-postgresql-server-sg')
cdk.Tags.of(sg_postgresql_server).add('Name',
'default-postgresql-server-sg')
rds_subnet_group = aws_rds.SubnetGroup(
self,
'PostgreSQLSubnetGroup',
description='subnet group for postgresql',
subnet_group_name='aurora-postgresql',
vpc_subnets=aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT),
vpc=vpc)
db_cluster_name = self.node.try_get_context('db_cluster_name')
#XXX: aws_rds.Credentials.from_username(username, ...) can not be given user specific Secret name
# therefore, first create Secret and then use it to create database
db_secret_name = self.node.try_get_context('db_secret_name')
#XXX: arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name}
db_secret_arn = 'arn:aws:secretsmanager:{region}:{account}:secret:{resource_name}'.format(
region=cdk.Aws.REGION,
account=cdk.Aws.ACCOUNT_ID,
resource_name=db_secret_name)
db_secret = aws_secretsmanager.Secret.from_secret_partial_arn(
self, 'DBSecretFromArn', db_secret_arn)
rds_credentials = aws_rds.Credentials.from_secret(db_secret)
rds_engine = aws_rds.DatabaseClusterEngine.aurora_postgres(
version=aws_rds.AuroraPostgresEngineVersion.VER_13_4)
#XXX: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraPostgreSQL.Reference.ParameterGroups.html#AuroraPostgreSQL.Reference.Parameters.Cluster
rds_cluster_param_group = aws_rds.ParameterGroup(
self,
'AuroraPostgreSQLClusterParamGroup',
engine=rds_engine,
description=
'Custom cluster parameter group for aurora-postgresql13',
parameters={
'log_min_duration_statement': '15000', # 15 sec
'default_transaction_isolation': 'read committed',
'client_encoding': 'UTF8'
})
#XXX: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraPostgreSQL.Reference.ParameterGroups.html#AuroraPostgreSQL.Reference.Parameters.Instance
rds_db_param_group = aws_rds.ParameterGroup(
self,
'AuroraPostgreSQLDBParamGroup',
engine=rds_engine,
description='Custom parameter group for aurora-postgresql13',
parameters={
'log_min_duration_statement': '15000', # 15 sec
'default_transaction_isolation': 'read committed'
})
db_cluster = aws_rds.DatabaseCluster(
self,
'Database',
engine=rds_engine,
credentials=
rds_credentials, # A username of 'admin' (or 'postgres' for PostgreSQL) and SecretsManager-generated password
instance_props={
'instance_type':
aws_ec2.InstanceType.of(aws_ec2.InstanceClass.BURSTABLE3,
aws_ec2.InstanceSize.MEDIUM),
'parameter_group':
rds_db_param_group,
'vpc_subnets': {
'subnet_type': aws_ec2.SubnetType.PRIVATE_WITH_NAT
},
'vpc':
vpc,
'auto_minor_version_upgrade':
False,
'security_groups': [sg_postgresql_server]
},
instances=2,
parameter_group=rds_cluster_param_group,
cloudwatch_logs_retention=aws_logs.RetentionDays.THREE_DAYS,
cluster_identifier=db_cluster_name,
subnet_group=rds_subnet_group,
backup=aws_rds.BackupProps(retention=cdk.Duration.days(3),
preferred_window="03:00-04:00"))
cdk.CfnOutput(self,
'DBClusterEndpoint',
value=db_cluster.cluster_endpoint.socket_address,
export_name='DBClusterEndpoint')
cdk.CfnOutput(self,
'DBClusterReadEndpoint',
value=db_cluster.cluster_read_endpoint.socket_address,
export_name='DBClusterReadEndpoint')
def __init__(self, scope: core.Construct, id: str, props,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/Vpc.html
vpc = ec2.Vpc(self,
"vpc",
cidr=props['vpc_CIDR'],
max_azs=3,
subnet_configuration=[{
'cidrMask': 28,
'name': 'public',
'subnetType': ec2.SubnetType.PUBLIC
}, {
'cidrMask':
28,
'name':
'private',
'subnetType':
ec2.SubnetType.PRIVATE
}, {
'cidrMask':
28,
'name':
'db',
'subnetType':
ec2.SubnetType.ISOLATED
}])
rds_subnetGroup = rds.SubnetGroup(
self,
"rds_subnetGroup",
description=
f"Group for {props['environment']}-{props['application']}-{props['unit']} DB",
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.ISOLATED))
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_rds/DatabaseCluster.html
##TODO:ADD Aurora Serverless Option
rds_instance = rds.DatabaseCluster(
self,
'wordpress-db',
engine=rds.DatabaseClusterEngine.aurora_mysql(
version=rds.AuroraMysqlEngineVersion.VER_2_07_2),
instances=1,
instance_props=rds.InstanceProps(
vpc=vpc,
enable_performance_insights=props[
'rds_enable_performance_insights'],
instance_type=ec2.InstanceType(
instance_type_identifier=props['rds_instance_type'])),
subnet_group=rds_subnetGroup,
storage_encrypted=props['rds_storage_encrypted'],
backup=rds.BackupProps(retention=core.Duration.days(
props['rds_automated_backup_retention_days'])))
EcsToRdsSeurityGroup = ec2.SecurityGroup(
self,
"EcsToRdsSeurityGroup",
vpc=vpc,
description="Allow WordPress containers to talk to RDS")
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_lambda/Function.html
db_cred_generator = _lambda.Function(
self,
'db_creds_generator',
runtime=_lambda.Runtime.PYTHON_3_8,
handler='db_creds_generator.handler',
code=_lambda.Code.asset('lambda/db_creds_generator'),
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.ISOLATED
), #vpc.select_subnets(subnet_type = ec2.SubnetType("ISOLATED")).subnets ,
environment={
'SECRET_NAME': rds_instance.secret.secret_name,
})
#Set Permissions and Sec Groups
rds_instance.connections.allow_from(
EcsToRdsSeurityGroup,
ec2.Port.tcp(3306)) #Open hole to RDS in RDS SG
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_efs/FileSystem.html
file_system = efs.FileSystem(
self,
"MyEfsFileSystem",
vpc=vpc,
encrypted=True, # file system is not encrypted by default
lifecycle_policy=props['efs_lifecycle_policy'],
performance_mode=efs.PerformanceMode.GENERAL_PURPOSE,
throughput_mode=efs.ThroughputMode.BURSTING,
removal_policy=core.RemovalPolicy(props['efs_removal_policy']),
enable_automatic_backups=props['efs_automatic_backups'])
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/Cluster.html?highlight=ecs%20cluster#aws_cdk.aws_ecs.Cluster
cluster = ecs.Cluster(
self,
"Cluster",
vpc=vpc,
container_insights=props['ecs_enable_container_insights'])
if props['deploy_bastion_host']:
#ToDo: Deploy bastion host with a key file
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/BastionHostLinux.html
bastion_host = ec2.BastionHostLinux(self, 'bastion_host', vpc=vpc)
rds_instance.connections.allow_from(bastion_host,
ec2.Port.tcp(3306))
#######################
### Developer Tools ###
# SFTP into the EFS Shared File System
NetToolsSecret = secretsmanager.Secret(
self,
"NetToolsSecret",
generate_secret_string=secretsmanager.SecretStringGenerator(
secret_string_template=json.dumps({
"username": '******',
"ip": ''
}),
generate_string_key="password",
exclude_characters='/"'))
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_efs/FileSystem.html#aws_cdk.aws_efs.FileSystem.add_access_point
AccessPoint = file_system.add_access_point(
"access-point",
path="/",
create_acl=efs.Acl(
owner_uid=
"100", #https://aws.amazon.com/blogs/containers/developers-guide-to-using-amazon-efs-with-amazon-ecs-and-aws-fargate-part-2/
owner_gid="101",
permissions="0755"))
EfsVolume = ecs.Volume(
name="efs",
efs_volume_configuration=ecs.EfsVolumeConfiguration(
file_system_id=file_system.file_system_id,
transit_encryption="ENABLED",
authorization_config=ecs.AuthorizationConfig(
access_point_id=AccessPoint.access_point_id)))
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/FargateTaskDefinition.html
NetToolsTask = ecs.FargateTaskDefinition(self,
"TaskDefinition",
cpu=256,
memory_limit_mib=512,
volumes=[EfsVolume])
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/FargateTaskDefinition.html#aws_cdk.aws_ecs.FargateTaskDefinition.add_container
NetToolsContainer = NetToolsTask.add_container(
"NetTools",
image=ecs.ContainerImage.from_registry('netresearch/sftp'),
command=['test:test:100:101:efs'])
NetToolsContainer.add_port_mappings(
ecs.PortMapping(container_port=22, protocol=ecs.Protocol.TCP))
NetToolsContainer.add_mount_points(
ecs.MountPoint(
container_path=
"/home/test/efs", #ToDo build path out with username from secret
read_only=False,
source_volume=EfsVolume.name,
))
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/FargateService.html?highlight=fargateservice#aws_cdk.aws_ecs.FargateService
service = ecs.FargateService(
self,
"Service",
cluster=cluster,
task_definition=NetToolsTask,
platform_version=ecs.FargatePlatformVersion(
"VERSION1_4"), #Required for EFS
)
#ToDo somehow store container's IP on deploy
#Allow traffic to EFS Volume from Net Tools container
service.connections.allow_to(file_system, ec2.Port.tcp(2049))
#ToDo allow bastion host into container on port 22
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_lambda/Function.html
bastion_ip_locator = _lambda.Function(
self,
'bastion_ip_locator',
function_name=
f"{props['environment']}-{props['application']}-{props['unit']}-SFTP-IP",
runtime=_lambda.Runtime.PYTHON_3_8,
handler='bastion_ip_locator.handler',
code=_lambda.Code.asset('lambda/bastion_ip_locator'),
environment={
'CLUSTER_NAME': cluster.cluster_arn,
'SERVICE_NAME': service.service_name
})
#Give needed perms to bastion_ip_locator for reading info from ECS
bastion_ip_locator.add_to_role_policy(
iam.PolicyStatement(
actions=["ecs:DescribeTasks"],
resources=[
#f"arn:aws:ecs:us-east-1:348757191778:service/{cluster.cluster_name}/{service.service_name}",
f"arn:aws:ecs:us-east-1:348757191778:task/{cluster.cluster_name}/*"
]))
bastion_ip_locator.add_to_role_policy(
iam.PolicyStatement(actions=[
"ecs:ListTasks",
],
resources=["*"],
conditions={
'ArnEquals': {
'ecs:cluster': cluster.cluster_arn
}
}))
self.output_props = props.copy()
self.output_props["vpc"] = vpc
self.output_props["rds_instance"] = rds_instance
self.output_props["EcsToRdsSeurityGroup"] = EcsToRdsSeurityGroup
self.output_props["file_system"] = file_system
self.output_props["cluster"] = cluster
def __init__(self, scope: core.Construct, id: str, props, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/Vpc.html
vpc = ec2.Vpc(self, "vpc",
cidr=props['vpc_CIDR'],
max_azs=3,
subnet_configuration=[
{
'cidrMask': 28,
'name': 'public',
'subnetType': ec2.SubnetType.PUBLIC
},
{
'cidrMask': 28,
'name': 'private',
'subnetType': ec2.SubnetType.PRIVATE
},
{
'cidrMask': 28,
'name': 'db',
'subnetType': ec2.SubnetType.ISOLATED
}
]
)
rds_subnetGroup = rds.SubnetGroup(self, "rds_subnetGroup",
description = f"Group for {props['environment']}-{props['application']}-{props['unit']} DB",
vpc = vpc,
vpc_subnets = ec2.SubnetSelection(subnet_type= ec2.SubnetType.ISOLATED)
)
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_rds/DatabaseCluster.html
##TODO:ADD Aurora Serverless Option
rds_instance = rds.DatabaseCluster(self,'wordpress-db',
engine=rds.DatabaseClusterEngine.aurora_mysql(
version=rds.AuroraMysqlEngineVersion.VER_2_07_2
),
instances=1,
instance_props=rds.InstanceProps(
vpc=vpc,
enable_performance_insights=props['rds_enable_performance_insights'],
instance_type=ec2.InstanceType(instance_type_identifier=props['rds_instance_type'])
),
subnet_group=rds_subnetGroup,
storage_encrypted=props['rds_storage_encrypted'],
backup=rds.BackupProps(
retention=core.Duration.days(props['rds_automated_backup_retention_days'])
)
)
EcsToRdsSeurityGroup= ec2.SecurityGroup(self, "EcsToRdsSeurityGroup",
vpc = vpc,
description = "Allow WordPress containers to talk to RDS"
)
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_lambda/Function.html
db_cred_generator = _lambda.Function(
self, 'db_creds_generator',
runtime=_lambda.Runtime.PYTHON_3_8,
handler='db_creds_generator.handler',
code=_lambda.Code.asset('lambda'),
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(subnet_type= ec2.SubnetType.ISOLATED), #vpc.select_subnets(subnet_type = ec2.SubnetType("ISOLATED")).subnets ,
environment={
'SECRET_NAME': rds_instance.secret.secret_name,
}
)
#Set Permissions and Sec Groups
rds_instance.connections.allow_from(EcsToRdsSeurityGroup, ec2.Port.tcp(3306)) #Open hole to RDS in RDS SG
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_efs/FileSystem.html
file_system = efs.FileSystem(self, "MyEfsFileSystem",
vpc = vpc,
encrypted=True, # file system is not encrypted by default
lifecycle_policy = props['efs_lifecycle_policy'],
performance_mode = efs.PerformanceMode.GENERAL_PURPOSE,
throughput_mode = efs.ThroughputMode.BURSTING,
removal_policy = core.RemovalPolicy(props['efs_removal_policy']),
enable_automatic_backups = props['efs_automatic_backups']
)
if props['deploy_bastion_host']:
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/BastionHostLinux.html
bastion_host = ec2.BastionHostLinux(self, 'bastion_host',
vpc = vpc
)
rds_instance.connections.allow_from(bastion_host, ec2.Port.tcp(3306))
self.output_props = props.copy()
self.output_props["vpc"] = vpc
self.output_props["rds_instance"] = rds_instance
self.output_props["EcsToRdsSeurityGroup"] = EcsToRdsSeurityGroup
self.output_props["file_system"] = file_system
