def __init__(self, scope: core.Construct, id: str, vpc: aws_ec2.Vpc,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
_subnets = []
_subnets.append(
aws_ec2.Subnet(self,
'sbn-redshift-1',
availability_zone=vpc.availability_zones[0],
vpc_id=vpc.vpc_id,
cidr_block='10.0.4.0/25'))
_subnets.append(
aws_ec2.Subnet(self,
'sbn-redshift-2',
availability_zone=vpc.availability_zones[1],
vpc_id=vpc.vpc_id,
cidr_block='10.0.4.128/25'))
_cluster_subnet_group = aws_redshift.ClusterSubnetGroup(
self,
'deta-pipeline-redshift-subnet',
description='redshift cluster subnet',
vpc=vpc,
vpc_subnets=aws_ec2.SubnetSelection(subnets=_subnets))
aws_redshift.Cluster(
self,
'destination-redshift',
master_user=aws_redshift.Login(master_username='******'),
vpc=vpc,
subnet_group=_cluster_subnet_group)
def __init__(self, scope: core.Construct, id: str, vpc: VpcStack,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
subnet_group = redshift.ClusterSubnetGroup(
self,
id="RedshiftSubnetGroup",
description="Redshift private subnet group",
vpc=vpc.instance,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.ISOLATED),
)
self.redshift_secret = sm.Secret(
self,
"redshift-credentials",
secret_name="redshift-credentials",
description="Credentials for Amazon Redshift cluster.",
generate_secret_string=sm.SecretStringGenerator(
secret_string_template='{"username": "******"}',
generate_string_key="password",
password_length=32,
exclude_characters='"@\\\/',
exclude_punctuation=True,
),
)
redshift_login = redshift.Login(
master_username="******",
master_password=self.redshift_secret.secret_value_from_json(
"password"),
)
redshift_s3_read_access_role = iam.Role(
self,
"redshiftS3AccessRole",
role_name="redshiftS3AccessRole",
assumed_by=iam.ServicePrincipal("redshift.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3ReadOnlyAccess")
],
)
redshift_cluster = redshift.Cluster(
self,
id="redshift-cluster",
master_user=redshift_login,
vpc=vpc,
cluster_type=redshift.ClusterType.SINGLE_NODE,
default_database_name="redshift-db",
encrypted=True,
node_type=redshift.NodeType.DC2_LARGE,
port=5439,
roles=[redshift_s3_read_access_role],
security_groups=[vpc.redshift_sg],
subnet_group=subnet_group,
removal_policy=core.RemovalPolicy.DESTROY,
)
self._instance = redshift_cluster
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
nat_instance = aws_ec2.NatProvider.instance(
instance_type=aws_ec2.InstanceType('t3a.nano'))
vpc = aws_ec2.Vpc(self,
'VPC',
max_azs=1,
nat_gateway_provider=nat_instance)
cluster_security_group = aws_ec2.SecurityGroup(
self,
'ClusterSecurityGroup',
vpc=vpc,
allow_all_outbound=True,
description="Allow Glue Job to access redshift",
security_group_name="serverless-redshift-query-testing-redshift")
cluster = aws_redshift.Cluster(
self,
'Cluster',
cluster_name='serverless-redshift-query-testing',
master_user=aws_redshift.Login(master_username='******'),
vpc=vpc,
removal_policy=core.RemovalPolicy.DESTROY,
security_groups=[cluster_security_group])
app = aws_sam.CfnApplication(
self,
'RedshiftQueryGlueJob',
location=
'https://redshift-query.s3-eu-west-1.amazonaws.com/glue-job-template.yaml',
parameters={
'ClusterId': 'serverless-redshift-query-testing',
'SQLStatements': "select 1;",
'Loglevel': "ERROR"
})
glue_job_security_group_ref = app.get_att(
'Outputs.SecurityGroup').to_string()
glue_job_security_group = aws_ec2.SecurityGroup.from_security_group_id(
self, 'GlueSecurityGroup', glue_job_security_group_ref)
cluster_security_group.add_ingress_rule(
peer=glue_job_security_group,
connection=aws_ec2.Port(protocol=aws_ec2.Protocol.TCP,
from_port=5439,
to_port=5439,
string_representation='Redshift Port'))
app.node.add_dependency(cluster)
def __init__(
self,
scope: core.Construct,
common_stack: CommonResourcesStack,
data_lake_silver_bucket: BaseDataLakeBucket,
data_lake_gold_bucket: BaseDataLakeBucket,
**kwargs,
) -> None:
self.common_stack = common_stack
self.deploy_env = active_environment
self.data_lake_silver_bucket = data_lake_silver_bucket
self.data_lake_gold_bucket = data_lake_gold_bucket
super().__init__(scope,
id=f"{self.deploy_env.value}-redshift-stack",
**kwargs)
self.redshift_sg = ec2.SecurityGroup(
self,
f"redshift-{self.deploy_env.value}-sg",
vpc=self.common_stack.custom_vpc,
allow_all_outbound=True,
security_group_name=f"redshift-{self.deploy_env.value}-sg",
)
self.redshift_sg.add_ingress_rule(peer=ec2.Peer.ipv4("0.0.0.0/0"),
connection=ec2.Port.tcp(5439))
for subnet in self.common_stack.custom_vpc.private_subnets:
self.redshift_sg.add_ingress_rule(
peer=ec2.Peer.ipv4(subnet.ipv4_cidr_block),
connection=ec2.Port.tcp(5439),
)
self.redshift_cluster = redshift.Cluster(
self,
f"ecommerce-{self.deploy_env.value}-redshift",
cluster_name=f"ecommerce-{self.deploy_env.value}-redshift",
vpc=self.common_stack.custom_vpc,
cluster_type=redshift.ClusterType.SINGLE_NODE,
node_type=redshift.NodeType.DC2_LARGE,
default_database_name="dw",
number_of_nodes=1,
removal_policy=core.RemovalPolicy.DESTROY,
master_user=redshift.Login(master_username="******"),
publicly_accessible=True,
roles=[
SpectrumRole(self, self.data_lake_silver_bucket,
self.data_lake_gold_bucket)
],
security_groups=[self.redshift_sg],
vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC),
)
def __init__(self, scope: core.Construct, id: str, vpc: ec2.Vpc,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
kms_policy = iam.PolicyDocument()
kms_policy.add_statements(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=["kms:*"],
resources=['*'],
principals=[iam.AccountPrincipal(account_id=self.account)]))
redshift_key = kms.Key(self,
"volumeKey",
enable_key_rotation=True,
policy=kms_policy,
removal_policy=core.RemovalPolicy.RETAIN)
redshift_bucket = s3.Bucket(self, "redshiftBucket")
redshift_bucket.add_to_resource_policy(permission=iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=["s3:*"],
resources=[
f"{redshift_bucket.bucket_arn}/*", redshift_bucket.bucket_arn
],
principals=[
iam.ArnPrincipal(f"arn:aws:iam::193672423079:user/logs")
]))
self._cluster = redshift.Cluster(
self,
id,
master_user=redshift.Login(master_username="******",
encryption_key=redshift_key),
port=5439,
vpc=vpc,
cluster_name="dwh",
cluster_type=redshift.ClusterType.MULTI_NODE,
number_of_nodes=2,
default_database_name="aml",
encrypted=True,
encryption_key=redshift_key,
logging_bucket=redshift_bucket,
logging_key_prefix="dwh",
node_type=redshift.NodeType.DC2_LARGE,
removal_policy=core.RemovalPolicy.DESTROY,
security_groups=[self.redshift_sg(vpc)],
vpc_subnets=ec2.SubnetSelection(subnet_group_name="DBS"))
def __init__(self, scope: core.Construct, common: Common,
data_lake: DataLake, **kwargs) -> None:
self.env = common.env
super().__init__(scope, id=f'{self.env}-data-warehouse', **kwargs)
self.redshift_sg = ec2.SecurityGroup(
self,
f'redshift-{self.env}-sg',
vpc=common.custom_vpc,
allow_all_outbound=True,
security_group_name=f'redshift-{self.env}-sg',
)
self.redshift_sg.add_ingress_rule(
peer=ec2.Peer.ipv4('37.156.75.55/32'),
connection=ec2.Port.tcp(5439))
for subnet in common.custom_vpc.private_subnets:
self.redshift_sg.add_ingress_rule(peer=ec2.Peer.ipv4(
subnet.ipv4_cidr_block),
connection=ec2.Port.tcp(5439))
self.redshift_cluster = redshift.Cluster(
self,
f'belisco-{self.env}-redshift',
cluster_name=f'belisco-{self.env}-redshift',
vpc=common.custom_vpc,
cluster_type=redshift.ClusterType.MULTI_NODE,
node_type=redshift.NodeType.DC2_LARGE,
default_database_name='dw',
number_of_nodes=2,
removal_policy=core.RemovalPolicy.DESTROY,
master_user=redshift.Login(master_username='******'),
publicly_accessible=True,
roles=[SpectrumRole(self, data_lake)],
security_groups=[self.redshift_sg],
vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.current_dir = os.path.dirname(__file__)
self.vpc = ec2.Vpc(
self,
"VPC",
cidr="10.0.0.0/21",
max_azs=2,
subnet_configuration=[
ec2.SubnetConfiguration(
cidr_mask=28,
name="Database",
subnet_type=ec2.SubnetType.ISOLATED,
),
ec2.SubnetConfiguration(cidr_mask=28,
name="Private",
subnet_type=ec2.SubnetType.PRIVATE),
ec2.SubnetConfiguration(cidr_mask=28,
name="Public",
subnet_type=ec2.SubnetType.PUBLIC)
],
nat_gateways=3)
self.qs_security_group = ec2.SecurityGroup(
self,
"quicksight-sg",
vpc=self.vpc,
allow_all_outbound=True,
description="QuickSight security group")
self.bastion = ec2.BastionHostLinux(
self,
"BastionHost",
vpc=self.vpc,
subnet_selection=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PUBLIC))
self.bastion.connections.allow_from_any_ipv4(ec2.Port.tcp(22),
"Internet access SSH")
self.vpc.add_interface_endpoint(
"redshift_endpoint",
service=ec2.InterfaceVpcEndpointAwsService("redshift"))
self.vpc.add_interface_endpoint(
"rds_endpoint", service=ec2.InterfaceVpcEndpointAwsService("rds"))
self.redshift_secret = secrets.Secret(
self,
'redshift-admin',
secret_name='redshift-admin',
description=
"This secret has generated admin secret password for Redshift cluster",
generate_secret_string=secrets.SecretStringGenerator(
secret_string_template='{"username": "******"}',
generate_string_key='password',
password_length=32,
exclude_characters='"@\\\/',
exclude_punctuation=True))
self.rs_security_group = ec2.SecurityGroup(self,
"redshift-sg",
vpc=self.vpc,
allow_all_outbound=True,
description="Redshift SG")
self.rs_security_group.add_ingress_rule(self.rs_security_group,
ec2.Port.all_tcp(),
'Redshift-basic')
self.rs_security_group.add_ingress_rule(
# https://docs.aws.amazon.com/quicksight/latest/user/regions.html
ec2.Peer.ipv4('52.23.63.224/27'),
ec2.Port.tcp(5439),
'QuickSight-IP')
self.rs_security_group.add_ingress_rule(self.qs_security_group,
ec2.Port.tcp(5439),
'QuickSight-sg')
# self.rs_security_group.add_egress_rule(
# self.rs_security_group,
# ec2.Port.all_tcp(),
# 'Allow outbound for QuickSight'
# )
self.redshift_cluster = redshift.Cluster(
self,
"datasource-redshift",
master_user=redshift.Login(
master_username="******",
master_password=self.redshift_secret.secret_value_from_json(
'password')),
vpc=self.vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.ISOLATED),
security_groups=[self.rs_security_group])
self.rds_secret = secrets.Secret(
self,
'rds-admin',
secret_name='rds-admin',
description=
"This secret has generated admin secret password for RDS cluster",
generate_secret_string=secrets.SecretStringGenerator(
secret_string_template='{"username": "******"}',
generate_string_key='password',
password_length=32,
exclude_characters='"@\\\/',
exclude_punctuation=True))
self.rds_cluster = rds.DatabaseCluster(
self,
"datasource-rds",
engine=rds.DatabaseClusterEngine.aurora_postgres(
version=rds.AuroraPostgresEngineVersion.VER_11_9),
instance_props={
"vpc_subnets": {
"subnet_type": ec2.SubnetType.ISOLATED
},
"vpc": self.vpc
},
credentials=rds.Credentials.from_secret(self.rds_secret))
self.rds_cluster.connections.allow_default_port_from(
self.bastion, "EC2 Bastion access Aurora")
self.rds_cluster.connections.allow_default_port_from(
self.qs_security_group, "QuickSight-sg")
self.rds_cluster.connections.allow_default_port_from(
# https://docs.aws.amazon.com/quicksight/latest/user/regions.html
ec2.Peer.ipv4('52.23.63.224/27'),
"QuickSight-IP")
self.qs_security_group.add_ingress_rule(self.rs_security_group,
ec2.Port.all_tcp(), 'AllTCP')
for rds_group in self.rds_cluster.connections.security_groups:
self.qs_security_group.add_ingress_rule(rds_group,
ec2.Port.all_tcp(),
'AllTCP')
# self.qs_security_group.add_egress_rule(
# self.rs_security_group,
# ec2.Port.all_tcp(),
# 'AllTCP'
# )
core.CfnOutput(self, "vpcId", value=self.vpc.vpc_id)
core.CfnOutput(self, "redshiftUsername", value="admin")
core.CfnOutput(self, "redshiftPassword", value="redshift-admin")
core.CfnOutput(self,
"redshiftClusterId",
value=self.redshift_cluster.cluster_name)
core.CfnOutput(self,
"redshiftHost",
value=self.redshift_cluster.cluster_endpoint.hostname)
core.CfnOutput(self, "redshiftDB", value="dev")
core.CfnOutput(self, "rdsUsername", value="administrator")
core.CfnOutput(self, "rdsPassword", value="rds-admin")
core.CfnOutput(self,
"rdsClusterId",
value=self.rds_cluster.cluster_identifier)
core.CfnOutput(self, "namespace", value="default")
core.CfnOutput(self, "version", value="1")
core.CfnOutput(self,
"quicksightSecurityGroupId",
value=self.qs_security_group.security_group_id)
def __init__(self, scope: core.Construct, id: str, vpc, bastion_sg,
glue_sg, clean_bucket: s3.Bucket, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.__vpc = vpc
self.__clean_bucket = clean_bucket
self.__glue_sg = glue_sg
self.__redshift_sg = ec2.SecurityGroup(
self,
id="redshift-sg",
vpc=self.__vpc,
allow_all_outbound=None,
description=None,
security_group_name="redshift-sg")
self.__lambda_sg = ec2.SecurityGroup(
self,
id="redshift-lambda-sg",
vpc=vpc,
allow_all_outbound=None,
description=None,
security_group_name="redshift-lambda-sg")
self.__redshift_sg.add_ingress_rule(bastion_sg, ec2.Port.tcp(5439))
self.__redshift_sg.add_ingress_rule(self.__lambda_sg,
ec2.Port.tcp(5439))
self.__redshift_sg.add_ingress_rule(self.__glue_sg, ec2.Port.tcp(5439))
self.__security_groups_list = [self.__redshift_sg]
self.__master_user = {'master_username': "******"}
self.__subnets_selection = ec2.SubnetSelection(
availability_zones=None,
one_per_az=None,
subnet_group_name=None,
subnet_name=None,
subnets=None,
subnet_type=ec2.SubnetType.PRIVATE)
# Create role that is used by the Redshift to read data from clean bucket
self.__s3role = iam.Role(
self,
"RedshiftClean",
assumed_by=iam.ServicePrincipal("redshift.amazonaws.com"))
self.__s3role.add_to_principal_policy(
iam.PolicyStatement(actions=[
's3:GetObject', 's3:ListBucket', 's3:GetBucketLocation',
's3:ListMultipartUploadParts', 's3:ListBucketMultipartUploads'
],
resources=[
self.__clean_bucket.arn_for_objects('*'),
self.__clean_bucket.bucket_arn
]))
self.__s3role.add_to_principal_policy(
iam.PolicyStatement(actions=[
'glue:CreateDatabase', 'glue:CreateDatabase',
'glue:DeleteDatabase', 'glue:GetDatabase', 'glue:GetDatabases',
'glue:UpdateDatabase', 'glue:CreateTable', 'glue:DeleteTable',
'glue:BatchDeleteTable', 'glue:UpdateTable', 'glue:GetTable',
'glue:GetTables', 'glue:BatchCreatePartition',
'glue:CreatePartition', 'glue:DeletePartition',
'glue:BatchDeletePartition', 'glue:UpdatePartition',
'glue:GetPartition', 'glue:GetPartitions',
'glue:BatchGetPartition'
],
resources=['*']))
self.__roles_list = [self.__s3role]
self.__redshift_cluster = redshift.Cluster(
self,
"redshift",
master_user=self.__master_user,
vpc=vpc,
cluster_name="ara-cdk-cluster",
cluster_type=ClusterType.MULTI_NODE,
default_database_name=_config.RedshiftDeploy.REDSHIFT_DB_NAME,
encrypted=None,
encryption_key=None,
logging_bucket=None,
logging_key_prefix=None,
node_type=NodeType.DC2_LARGE,
number_of_nodes=2,
parameter_group=None,
port=None,
preferred_maintenance_window=None,
removal_policy=RemovalPolicy.DESTROY,
roles=self.__roles_list,
security_groups=self.__security_groups_list,
vpc_subnets=self.__subnets_selection,
)
self.__redshift_endpoint = self.__redshift_cluster.cluster_endpoint
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.current_dir = os.path.dirname(__file__)
self.vpc = ec2.Vpc(self,
"VPC",
cidr="10.0.0.0/21",
max_azs=3,
subnet_configuration=[
ec2.SubnetConfiguration(
cidr_mask=28,
name="Database",
subnet_type=ec2.SubnetType.ISOLATED,
)
])
self.vpc.add_interface_endpoint(
"redshift_endpoint",
service=ec2.InterfaceVpcEndpointAwsService("redshift"))
self.vpc.add_interface_endpoint(
"rds_endpoint", service=ec2.InterfaceVpcEndpointAwsService("rds"))
self.redshift_secret = secrets.Secret(
self,
'redshift-admin',
secret_name='redshift-admin',
description=
"This secret has generated admin secret password for Redshift cluster",
generate_secret_string=secrets.SecretStringGenerator(
secret_string_template='{"username": "******"}',
generate_string_key='password',
password_length=32,
exclude_characters='"@\\\/',
exclude_punctuation=True))
self.redshift_cluster = redshift.Cluster(
self,
"datasource-redshift",
master_user=redshift.Login(
master_username="******",
master_password=self.redshift_secret.secret_value_from_json(
'password')),
vpc=self.vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.ISOLATED))
self.rds_secret = secrets.Secret(
self,
'rds-admin',
secret_name='rds-admin',
description=
"This secret has generated admin secret password for RDS cluster",
generate_secret_string=secrets.SecretStringGenerator(
secret_string_template='{"username": "******"}',
generate_string_key='password',
password_length=32,
exclude_characters='"@\\\/',
exclude_punctuation=True))
self.rds_cluster = rds.DatabaseCluster(
self,
"datasource-rds",
engine=rds.DatabaseClusterEngine.aurora_mysql(
version=rds.AuroraMysqlEngineVersion.VER_2_08_1),
instance_props={
"vpc_subnets": {
"subnet_type": ec2.SubnetType.ISOLATED
},
"vpc": self.vpc
},
credentials=rds.Credentials.from_secret(self.rds_secret))
core.CfnOutput(self, "vpcId", value=self.vpc.vpc_id)
core.CfnOutput(self, "redshiftUsername", value="admin")
core.CfnOutput(self,
"redshiftPassword",
value=self.redshift_secret.secret_name)
core.CfnOutput(self,
"redshiftClusterId",
value=self.redshift_cluster.cluster_name)
core.CfnOutput(self,
"redshiftHost",
value=self.redshift_cluster.cluster_endpoint.hostname)
core.CfnOutput(self, "redshiftDB", value="dev")
core.CfnOutput(self, "rdsUsername", value="admin")
core.CfnOutput(self, "rdsPassword", value=self.rds_secret.secret_name)
core.CfnOutput(self,
"rdsClusterId",
value=self.rds_cluster.cluster_identifier)
core.CfnOutput(self, "namespace", value="default")
core.CfnOutput(self, "version", value="1")
def __init__(self, scope: core.Construct, id: str, vpc, props, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# create s3 bucket that redshift will use. if this bucket exists
# this cdk app will fail, so ensure this has not been created yet
redshift_bucket = s3.Bucket(
self,
"mwaa-redshift import",
bucket_name=f"{props['redshifts3location'].lower()}",
versioned=True,
block_public_access=s3.BlockPublicAccess.BLOCK_ALL
)
# create the files folder in the bucket - this is empty but needed in the DAG
s3deploy.BucketDeployment(self, "File",
sources=[s3deploy.Source.asset("./files")],
destination_bucket=redshift_bucket,
destination_key_prefix="files/",
prune=False,
retain_on_delete=False
)
redshift_bucket_arn = redshift_bucket.bucket_arn
# get arn of dags bucket - not sure if this is needed so may remove
dags_bucket = s3.Bucket.from_bucket_name(self, "mwaa-dag-bucket", f"{props['mwaadag'].lower()}")
dags_bucket_arn = dags_bucket.bucket_arn
# create redshift secret and redshift user
# create redshift iam role/policy that we will attach to the RedShift cluster
# that has the right level of access to a specific S3 bucket
# you can further lockdown this policy by just specifying s3 actions.
mwaa_redshift_policy_document = iam.PolicyDocument(
statements=[
iam.PolicyStatement(
actions=[
"s3:*"
],
effect=iam.Effect.ALLOW,
resources=[
f"{redshift_bucket_arn}/*",
f"{redshift_bucket_arn}",
f"{dags_bucket_arn}/*",
f"{dags_bucket_arn}",
]
)
]
)
mwaa_redshift_service_role = iam.Role(
self,
"mwaa-redshift-service-role2nd",
assumed_by=iam.ServicePrincipal("redshift.amazonaws.com"),
inline_policies={"mwaaRedshiftPolicyDocument": mwaa_redshift_policy_document}
)
mwaa_redshift_service_role_arn = mwaa_redshift_service_role.role_arn
# Setup Security Group
default_redshift_security_group = ec2.SecurityGroup.from_security_group_id(
self,
"MWAARedshiftSG",
security_group_id=vpc.vpc_default_security_group
)
default_redshift_security_group.add_ingress_rule(
peer=default_redshift_security_group,
connection=ec2.Port.tcp(5439)
)
# Modify MWAA security group to enable Redshift access
mwaa_security_group = ec2.SecurityGroup.from_security_group_id(
self,
"SG",
props['mwaa-sg']
#mutable=False
)
mwaa_security_group.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(5439), "allow redshift access")
# create subnet groups - one for RedShift and one for the VPE we will create
# the VPE subnet will take in parameters we provide that are the subnet-ids
# of the VPC where MWAA is deployed
redshift_cluster_subnet_group = redshift.ClusterSubnetGroup(
self,
"RedshiftCSG",
vpc = vpc,
#vpc_subnets = ec2.SubnetSelection(subnets=vpc.private_subnets),
vpc_subnets = ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE),
description="Redshift Cluster Subnet Group"
)
## get all the subnet ids from the MWAA VPC
subnet_ids = []
mwaavpc = ec2.Vpc.from_lookup(
self,
"MWAA VPC",
vpc_id=props['mwaa-vpc-id']
)
for subnet in mwaavpc.private_subnets:
subnet_ids.append(subnet.subnet_id)
for subnet in mwaavpc.public_subnets:
subnet_ids.append(subnet.subnet_id)
vpe_redshift_cluster_subnet_group = redshift.CfnClusterSubnetGroup(
self,
"MWAAVPERedshiftCSG",
subnet_ids = subnet_ids,
description="MWAA VPE Redshift Cluster Subnet Group"
)
redshiftclustername = f"{props['redshiftclustername'].lower()}"
cluster = redshift.Cluster(
self,
"MWAARedshiftCluster",
master_user=redshift.Login(
master_username=props['redshiftusername']
),
vpc = vpc,
security_groups=[default_redshift_security_group],
node_type=redshift.NodeType.RA3_4XLARGE,
number_of_nodes=2,
cluster_name=redshiftclustername,
default_database_name=props['redshiftdb'],
removal_policy=core.RemovalPolicy.DESTROY,
roles=[mwaa_redshift_service_role],
publicly_accessible=False,
subnet_group=redshift_cluster_subnet_group
)
redshift_secret_arn = cluster.secret.secret_arn
# Display some useful output
core.CfnOutput(
self,
id="RedshiftSecretARN :",
value=redshift_secret_arn,
description="This is the Redshift Secret ARN"
)
core.CfnOutput(
self,
id="RedshiftIAMARN :",
value=mwaa_redshift_service_role_arn,
description="This is the Redshift IAM ARN"
)
core.CfnOutput(
self,
id="RedshiftClusterEndpoint :",
value=cluster.cluster_endpoint.hostname,
description="This is the Redshift Cluster Endpoint"
)
core.CfnOutput(
self,
id="MWAAVPCESG :",
value=vpe_redshift_cluster_subnet_group.ref,
description="This is the VPE Subnet Group to use when creating the VPC Endpoint"
)
core.CfnOutput(
self,
id="redshiftvpcendpointcli",
value="aws redshift create-endpoint-access --cluster-identifier "+redshiftclustername+" --resource-owner "+self.account+ " --endpoint-name mwaa-redshift-endpoint --subnet-group-name "+vpe_redshift_cluster_subnet_group.ref+" --vpc-security-group-ids "+props['mwaa-sg'],
description="Use this command to create your vpce"
)
def _setup_redshift(self) -> None:
port = 5439
database = "test"
schema = "public"
redshift_role = iam.Role(
self,
"aws-data-wrangler-redshift-role",
assumed_by=iam.ServicePrincipal("redshift.amazonaws.com"),
inline_policies={
"KMS":
iam.PolicyDocument(statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"kms:Encrypt",
"kms:Decrypt",
"kms:GenerateDataKey",
],
resources=[self.key.key_arn],
)
]),
"S3":
iam.PolicyDocument(statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"s3:Get*",
"s3:List*",
"s3:Put*",
],
resources=[
self.bucket.bucket_arn,
f"{self.bucket.bucket_arn}/*",
],
)
]),
"LakeFormation":
iam.PolicyDocument(statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"lakeformation:GetDataAccess",
"lakeformation:GrantPermissions",
"lakeformation:GetWorkUnits",
"lakeformation:StartQueryPlanning",
"lakeformation:GetWorkUnitResults",
"lakeformation:GetQueryState",
],
resources=["*"],
)
]),
"Glue":
iam.PolicyDocument(statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"glue:SearchTables",
"glue:GetConnections",
"glue:GetDataCatalogEncryptionSettings",
"glue:GetTables",
"glue:GetTableVersions",
"glue:GetPartitions",
"glue:DeleteTableVersion",
"glue:BatchGetPartition",
"glue:GetDatabases",
"glue:GetTags",
"glue:GetTable",
"glue:GetDatabase",
"glue:GetPartition",
"glue:GetTableVersion",
"glue:GetConnection",
"glue:GetUserDefinedFunction",
"glue:GetUserDefinedFunctions",
],
resources=["*"],
)
]),
},
)
lf.CfnPermissions(
self,
"CodeBuildTestRoleLFPermissions",
data_lake_principal=lf.CfnPermissions.DataLakePrincipalProperty(
data_lake_principal_identifier=redshift_role.role_arn),
resource=lf.CfnPermissions.ResourceProperty(
table_resource=lf.CfnPermissions.TableResourceProperty(
database_name="aws_data_wrangler",
table_wildcard={}, # type: ignore
)),
permissions=[
"SELECT", "ALTER", "DESCRIBE", "DROP", "DELETE", "INSERT"
],
)
redshift.ClusterSubnetGroup(
self,
"aws-data-wrangler-redshift-subnet-group",
description="AWS Data Wrangler Test Arena - Redshift Subnet Group",
vpc=self.vpc,
vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC),
)
redshift_cluster = redshift.Cluster(
self,
"aws-data-wrangler-redshift-cluster",
default_database_name=database,
master_user=redshift.Login(
master_username=self.db_username,
master_password=self.db_password_secret,
),
cluster_type=redshift.ClusterType.SINGLE_NODE,
publicly_accessible=True,
port=port,
vpc=self.vpc,
vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC),
security_groups=[self.db_security_group],
roles=[redshift_role],
)
glue.Connection(
self,
"aws-data-wrangler-redshift-glue-connection",
description="Connect to Redshift.",
type=glue.ConnectionType.JDBC,
connection_name="aws-data-wrangler-redshift",
properties={
"JDBC_CONNECTION_URL":
f"jdbc:redshift://{redshift_cluster.cluster_endpoint.hostname}:{port}/{database}", # noqa: E501
"USERNAME": self.db_username,
"PASSWORD": self.db_password,
},
subnet=self.vpc.private_subnets[0],
security_groups=[self.db_security_group],
)
secret = secrets.Secret(
self,
"aws-data-wrangler-redshift-secret",
secret_name="aws-data-wrangler/redshift",
description="Redshift credentials",
generate_secret_string=secrets.SecretStringGenerator(
generate_string_key="dummy",
secret_string_template=json.dumps({
"username":
self.db_username,
"password":
self.db_password,
"engine":
"redshift",
"host":
redshift_cluster.cluster_endpoint.hostname,
"port":
port,
"dbClusterIdentifier":
redshift_cluster.cluster_name,
}),
),
)
cdk.CfnOutput(self, "RedshiftSecretArn", value=secret.secret_arn)
cdk.CfnOutput(self,
"RedshiftIdentifier",
value=redshift_cluster.cluster_name)
cdk.CfnOutput(
self,
"RedshiftAddress",
value=redshift_cluster.cluster_endpoint.hostname,
)
cdk.CfnOutput(self, "RedshiftPort", value=str(port))
cdk.CfnOutput(self, "RedshiftDatabase", value=database)
cdk.CfnOutput(self, "RedshiftSchema", value=schema)
cdk.CfnOutput(self, "RedshiftRole", value=redshift_role.role_arn)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.current_dir = os.path.dirname(__file__)
self.quicksight_migration_source_assume_role = iam.Role(
self, 'quicksight-migration-source-assume-role',
description='Role for the Quicksight dashboard migration Lambdas to assume',
role_name='quicksight-migration-source-assume-role',
max_session_duration=core.Duration.seconds(3600),
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
inline_policies={
'AllowAccess': iam.PolicyDocument(
statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"quicksight:*",
],
resources=["*"]
),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"ssm:GetParameter",
],
resources=["arn:aws:ssm:*:*:parameter/infra/config"]
)
]
)
}
)
self.quicksight_migration_source_assume_role.assume_role_policy.add_statements(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=['sts:AssumeRole'],
principals=[iam.AccountPrincipal("499080683179")]
)
)
self.quicksight_migration_target_assume_role = iam.Role(
self, 'quicksight-migration-target-assume-role',
description='Role for the Quicksight dashboard migration Lambdas to assume',
role_name='quicksight-migration-target-assume-role',
max_session_duration=core.Duration.seconds(3600),
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
inline_policies={
'AllowAccess': iam.PolicyDocument(
statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"quicksight:*",
],
resources=["*"]
),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"ssm:GetParameter",
],
resources=["arn:aws:ssm:*:*:parameter/infra/config"]
)
]
)
}
)
self.quicksight_migration_target_assume_role.assume_role_policy.add_statements(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=['sts:AssumeRole'],
principals=[iam.AccountPrincipal("499080683179")]
)
)
self.vpc = ec2.Vpc(self, "VPC",
cidr="10.0.0.0/21",
max_azs=3,
subnet_configuration=[
ec2.SubnetConfiguration(
cidr_mask=28,
name="Database",
subnet_type=ec2.SubnetType.ISOLATED,
)
]
)
self.vpc.add_interface_endpoint("redshift_endpoint",
service=ec2.InterfaceVpcEndpointAwsService("redshift")
)
self.vpc.add_interface_endpoint("rds_endpoint",
service=ec2.InterfaceVpcEndpointAwsService("rds")
)
self.redshift_secret = secrets.Secret(self,'redshift-admin',
secret_name='redshift-admin',
description="This secret has generated admin secret password for Redshift cluster",
generate_secret_string=secrets.SecretStringGenerator(
secret_string_template='{"username": "******"}',
generate_string_key='password',
password_length=32,
exclude_characters='"@\\\/',
exclude_punctuation=True
)
)
self.redshift_cluster = redshift.Cluster(self, "datasource-redshift",
master_user=redshift.Login(
master_username="******",
master_password=self.redshift_secret.secret_value_from_json('password')
),
vpc=self.vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.ISOLATED
)
)
self.rds_secret = secrets.Secret(self,'rds-admin',
secret_name='rds-admin',
description="This secret has generated admin secret password for RDS cluster",
generate_secret_string=secrets.SecretStringGenerator(
secret_string_template='{"username": "******"}',
generate_string_key='password',
password_length=32,
exclude_characters='"@\\\/',
exclude_punctuation=True
)
)
self.rds_cluster = rds.DatabaseCluster(self, "datasource-rds",
engine=rds.DatabaseClusterEngine.aurora_mysql(
version=rds.AuroraMysqlEngineVersion.VER_2_08_1),
instance_props={
"vpc_subnets": {
"subnet_type": ec2.SubnetType.ISOLATED
},
"vpc": self.vpc
},
credentials=rds.Credentials.from_secret(self.rds_secret)
)
ssm.StringParameter(self, 'InfraConfigParam',
parameter_name='/infra/config',
string_value=json.dumps(self.to_dict()))
