def __init__(self, scope: core.Construct, common_stack: CommonResourcesStack, **kwargs): self.deploy_env = active_environment self.common_stack = common_stack # Security Group self.dms_sg = ec2.SecurityGroup( scope=scope, id=f"dms-{self.deploy_env.value}-sg", vpc=self.common_stack.custom_vpc, security_group_name=f"dms-{self.deploy_env.value}-sg", ) # Subnet Group self.dms_subnet_group = dms.CfnReplicationSubnetGroup( scope=scope, id=f"dms-{self.deploy_env.value}-replication-subnet", replication_subnet_group_description= "dms replication instance subnet group", subnet_ids=[ subnet.subnet_id for subnet in self.common_stack.custom_vpc. public_subnets # use public subnet to avoid costs ], replication_subnet_group_identifier= f"dms-{self.deploy_env.value}-replication-subnet", ) # Replication Instance self.instance = dms.CfnReplicationInstance( scope=scope, id=f"dms-{self.deploy_env.value}-replication-instance", allocated_storage=100, publicly_accessible=False, engine_version="3.4.4", replication_instance_class="dms.t2.small", replication_instance_identifier= f"dms-{self.deploy_env.value}-replication-instance", vpc_security_group_ids=[self.dms_sg.security_group_id], replication_subnet_group_identifier=self.dms_subnet_group. replication_subnet_group_identifier, ) self.instance.node.add_dependency(self.dms_sg) self.instance.node.add_dependency(self.dms_subnet_group)
def __init__(self, scope: core.Construct, id: str, vpc: aws_ec2.Vpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) aws_ec2.SecurityGroup(self, 'dms-securitygroup', vpc=vpc) _subnets = [] _subnets.append( aws_ec2.Subnet(self, 'sbn-dms-1', availability_zone=vpc.availability_zones[0], vpc_id=vpc.vpc_id, cidr_block='10.0.3.0/25').subnet_id) _subnets.append( aws_ec2.Subnet(self, 'sbn-dms-2', availability_zone=vpc.availability_zones[1], vpc_id=vpc.vpc_id, cidr_block='10.0.3.128/25').subnet_id) _subnet_group_id = 'deta-pipeline-dms-subnet' dms.CfnReplicationSubnetGroup( self, 'dms-subnet-group', replication_subnet_group_description='subnet group for dms', subnet_ids=_subnets, replication_subnet_group_identifier=_subnet_group_id) dms.CfnReplicationInstance( self, 'dms-demo', replication_instance_class='dms.t3.medium', allocated_storage=50, allow_major_version_upgrade=False, auto_minor_version_upgrade=False, publicly_accessible=False, multi_az=False, replication_subnet_group_identifier=_subnet_group_id)
def __init__( self, scope: core.Construct, common_stack: CommonStack, data_lake_raw_bucket: BaseDataLakeBucket, **kwargs, ) -> None: self.data_lake_raw_bucket = data_lake_raw_bucket self.common_stack = common_stack self.deploy_env = scope.deploy_env self.rds_endpoint = dms.CfnEndpoint( scope, f"dms-{self.deploy_env.value}-orders-rds-endpoint", endpoint_type="source", endpoint_identifier=f"dms-source-{self.deploy_env.value}-orders-rds-endpoint", engine_name="postgres", password=core.CfnDynamicReference( core.CfnDynamicReferenceService.SECRETS_MANAGER, key=f"{self.common_stack.orders_rds.secret.secret_arn}:SecretString:password", ).to_string(), username=core.CfnDynamicReference( core.CfnDynamicReferenceService.SECRETS_MANAGER, key=f"{self.common_stack.orders_rds.secret.secret_arn}:SecretString:username", ).to_string(), database_name=core.CfnDynamicReference( core.CfnDynamicReferenceService.SECRETS_MANAGER, key=f"{self.common_stack.orders_rds.secret.secret_arn}:SecretString:dbname", ).to_string(), port=5432, server_name=self.common_stack.orders_rds.db_instance_endpoint_address, extra_connection_attributes="captureDDLs=Y", ) self.s3_endpoint = dms.CfnEndpoint( scope, f"dms-{self.deploy_env.value}-orders-s3-endpoint", endpoint_type="target", engine_name="s3", endpoint_identifier=f"dms-target-{self.deploy_env.value}-orders-s3-endpoint", extra_connection_attributes="DataFormat=parquet;maxFileSize=131072;timestampColumnName=extracted_at;includeOpForFullLoad=true;cdcMaxBatchInterval=120", s3_settings=dms.CfnEndpoint.S3SettingsProperty( bucket_name=self.data_lake_raw_bucket.bucket_name, bucket_folder="orders", compression_type="gzip", csv_delimiter=",", csv_row_delimiter="\n", service_access_role_arn=RawDMSRole( scope, self.data_lake_raw_bucket ).role_arn, ), ) self.dms_sg = ec2.SecurityGroup( scope, f"dms-{self.deploy_env.value}-sg", vpc=self.common_stack.custom_vpc, security_group_name=f"dms-{self.deploy_env.value}-sg", ) self.dms_subnet_group = dms.CfnReplicationSubnetGroup( scope, f"dms-{self.deploy_env.value}-replication-subnet", replication_subnet_group_description="dms replication instance subnet group", subnet_ids=[ subnet.subnet_id for subnet in self.common_stack.custom_vpc.private_subnets ], replication_subnet_group_identifier=f"dms-{self.deploy_env.value}-replication-subnet", ) self.instance = dms.CfnReplicationInstance( scope, f"dms-replication-instance-{self.deploy_env.value}", allocated_storage=100, publicly_accessible=False, engine_version="3.4.4", replication_instance_class="dms.t2.small", replication_instance_identifier=f"dms-{self.deploy_env.value}-replication-instance", vpc_security_group_ids=[self.dms_sg.security_group_id], replication_subnet_group_identifier=self.dms_subnet_group.replication_subnet_group_identifier, ) self.instance.node.add_dependency(self.dms_subnet_group) self.instance.node.add_dependency(self.dms_sg) super().__init__( scope, f"{self.deploy_env.value}-dms-task-orders-rds", migration_type="full-load-and-cdc", replication_task_identifier=f"{self.deploy_env.value}-dms-task-orders-rds", replication_instance_arn=self.instance.ref, source_endpoint_arn=self.rds_endpoint.ref, target_endpoint_arn=self.s3_endpoint.ref, table_mappings=json.dumps( { "rules": [ { "rule-type": "selection", "rule-id": "1", "rule-name": "1", "object-locator": { "schema-name": "%", "table-name": "%", }, "rule-action": "include", "filters": [], } ] } ), )
def __init__(self, scope: core.Construct, common: Common, data_lake: DataLake, **kwargs) -> None: self.rds_endpoint = dms.CfnEndpoint( scope, f'dms-{common.env}-orders-rds-endpoint', endpoint_type='source', endpoint_identifier=f'dms-source-{common.env}-orders-rds-endpoint', engine_name='postgres', password=core.CfnDynamicReference( core.CfnDynamicReferenceService.SECRETS_MANAGER, key= f'{common.orders_rds.secret.secret_arn}:SecretString:password' ).to_string(), username=core.CfnDynamicReference( core.CfnDynamicReferenceService.SECRETS_MANAGER, key= f'{common.orders_rds.secret.secret_arn}:SecretString:username' ).to_string(), database_name=core.CfnDynamicReference( core.CfnDynamicReferenceService.SECRETS_MANAGER, key=f'{common.orders_rds.secret.secret_arn}:SecretString:dbname' ).to_string(), port=5432, server_name=common.orders_rds.db_instance_endpoint_address, ) self.s3_endpoint = dms.CfnEndpoint( scope, f'dms-{common.env}-orders-s3-endpoint', endpoint_type='target', engine_name='s3', endpoint_identifier=f'dms-target-{common.env}-orders-s3-endpoint', extra_connection_attributes= "DataFormat=parquet;maxFileSize=131072;timestampColumnName=extracted_at;includeOpForFullLoad=true;cdcInsertsAndUpdates=true", s3_settings=dms.CfnEndpoint.S3SettingsProperty( bucket_name=data_lake.data_lake_raw_bucket.bucket_name, bucket_folder='orders', compression_type='gzip', csv_delimiter=',', csv_row_delimiter='\n', service_access_role_arn=RawDMSRole( scope, common.env, data_lake.data_lake_raw_bucket).role_arn)) self.dms_sg = ec2.SecurityGroup( scope, f'dms-{common.env}-sg', vpc=common.custom_vpc, security_group_name=f'dms-{common.env}-sg', ) self.dms_subnet_group = dms.CfnReplicationSubnetGroup( scope, f'dms-{common.env}-replication-subnet', replication_subnet_group_description= 'dms replication instance subnet group', subnet_ids=[ subnet.subnet_id for subnet in common.custom_vpc.private_subnets ], replication_subnet_group_identifier= f'dms-{common.env}-replication-subnet') self.instance = dms.CfnReplicationInstance( scope, f'dms-replication-instance-{common.env}', allocated_storage=100, publicly_accessible=False, engine_version='3.3.2', replication_instance_class='dms.t2.small', replication_instance_identifier= f'dms-{common.env}-replication-instance', vpc_security_group_ids=[self.dms_sg.security_group_id], replication_subnet_group_identifier=self.dms_subnet_group. replication_subnet_group_identifier) super().__init__( scope, f'{common.env}-dms-task-orders-rds', migration_type='full-load-and-cdc', replication_task_identifier=f'{common.env}-dms-task-orders-rds', replication_instance_arn=self.instance.ref, source_endpoint_arn=self.rds_endpoint.ref, target_endpoint_arn=self.s3_endpoint.ref, table_mappings=json.dumps({ "rules": [{ "rule-type": "selection", "rule-id": "1", "rule-name": "1", "object-locator": { "schema-name": "%", "table-name": "%", }, "rule-action": "include", "filters": [] }] }))