def __init__(self, scope: core.Construct,
common_stack: CommonResourcesStack, **kwargs):
self.deploy_env = active_environment
self.common_stack = common_stack
# Security Group
self.dms_sg = ec2.SecurityGroup(
scope=scope,
id=f"dms-{self.deploy_env.value}-sg",
vpc=self.common_stack.custom_vpc,
security_group_name=f"dms-{self.deploy_env.value}-sg",
)
# Subnet Group
self.dms_subnet_group = dms.CfnReplicationSubnetGroup(
scope=scope,
id=f"dms-{self.deploy_env.value}-replication-subnet",
replication_subnet_group_description=
"dms replication instance subnet group",
subnet_ids=[
subnet.subnet_id for subnet in self.common_stack.custom_vpc.
public_subnets # use public subnet to avoid costs
],
replication_subnet_group_identifier=
f"dms-{self.deploy_env.value}-replication-subnet",
)
# Replication Instance
self.instance = dms.CfnReplicationInstance(
scope=scope,
id=f"dms-{self.deploy_env.value}-replication-instance",
allocated_storage=100,
publicly_accessible=False,
engine_version="3.4.4",
replication_instance_class="dms.t2.small",
replication_instance_identifier=
f"dms-{self.deploy_env.value}-replication-instance",
vpc_security_group_ids=[self.dms_sg.security_group_id],
replication_subnet_group_identifier=self.dms_subnet_group.
replication_subnet_group_identifier,
)
self.instance.node.add_dependency(self.dms_sg)
self.instance.node.add_dependency(self.dms_subnet_group)
def __init__(self, scope: core.Construct, id: str, vpc: aws_ec2.Vpc,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
aws_ec2.SecurityGroup(self, 'dms-securitygroup', vpc=vpc)
_subnets = []
_subnets.append(
aws_ec2.Subnet(self,
'sbn-dms-1',
availability_zone=vpc.availability_zones[0],
vpc_id=vpc.vpc_id,
cidr_block='10.0.3.0/25').subnet_id)
_subnets.append(
aws_ec2.Subnet(self,
'sbn-dms-2',
availability_zone=vpc.availability_zones[1],
vpc_id=vpc.vpc_id,
cidr_block='10.0.3.128/25').subnet_id)
_subnet_group_id = 'deta-pipeline-dms-subnet'
dms.CfnReplicationSubnetGroup(
self,
'dms-subnet-group',
replication_subnet_group_description='subnet group for dms',
subnet_ids=_subnets,
replication_subnet_group_identifier=_subnet_group_id)
dms.CfnReplicationInstance(
self,
'dms-demo',
replication_instance_class='dms.t3.medium',
allocated_storage=50,
allow_major_version_upgrade=False,
auto_minor_version_upgrade=False,
publicly_accessible=False,
multi_az=False,
replication_subnet_group_identifier=_subnet_group_id)
def __init__(
self,
scope: core.Construct,
common_stack: CommonStack,
data_lake_raw_bucket: BaseDataLakeBucket,
**kwargs,
) -> None:
self.data_lake_raw_bucket = data_lake_raw_bucket
self.common_stack = common_stack
self.deploy_env = scope.deploy_env
self.rds_endpoint = dms.CfnEndpoint(
scope,
f"dms-{self.deploy_env.value}-orders-rds-endpoint",
endpoint_type="source",
endpoint_identifier=f"dms-source-{self.deploy_env.value}-orders-rds-endpoint",
engine_name="postgres",
password=core.CfnDynamicReference(
core.CfnDynamicReferenceService.SECRETS_MANAGER,
key=f"{self.common_stack.orders_rds.secret.secret_arn}:SecretString:password",
).to_string(),
username=core.CfnDynamicReference(
core.CfnDynamicReferenceService.SECRETS_MANAGER,
key=f"{self.common_stack.orders_rds.secret.secret_arn}:SecretString:username",
).to_string(),
database_name=core.CfnDynamicReference(
core.CfnDynamicReferenceService.SECRETS_MANAGER,
key=f"{self.common_stack.orders_rds.secret.secret_arn}:SecretString:dbname",
).to_string(),
port=5432,
server_name=self.common_stack.orders_rds.db_instance_endpoint_address,
extra_connection_attributes="captureDDLs=Y",
)
self.s3_endpoint = dms.CfnEndpoint(
scope,
f"dms-{self.deploy_env.value}-orders-s3-endpoint",
endpoint_type="target",
engine_name="s3",
endpoint_identifier=f"dms-target-{self.deploy_env.value}-orders-s3-endpoint",
extra_connection_attributes="DataFormat=parquet;maxFileSize=131072;timestampColumnName=extracted_at;includeOpForFullLoad=true;cdcMaxBatchInterval=120",
s3_settings=dms.CfnEndpoint.S3SettingsProperty(
bucket_name=self.data_lake_raw_bucket.bucket_name,
bucket_folder="orders",
compression_type="gzip",
csv_delimiter=",",
csv_row_delimiter="\n",
service_access_role_arn=RawDMSRole(
scope, self.data_lake_raw_bucket
).role_arn,
),
)
self.dms_sg = ec2.SecurityGroup(
scope,
f"dms-{self.deploy_env.value}-sg",
vpc=self.common_stack.custom_vpc,
security_group_name=f"dms-{self.deploy_env.value}-sg",
)
self.dms_subnet_group = dms.CfnReplicationSubnetGroup(
scope,
f"dms-{self.deploy_env.value}-replication-subnet",
replication_subnet_group_description="dms replication instance subnet group",
subnet_ids=[
subnet.subnet_id
for subnet in self.common_stack.custom_vpc.private_subnets
],
replication_subnet_group_identifier=f"dms-{self.deploy_env.value}-replication-subnet",
)
self.instance = dms.CfnReplicationInstance(
scope,
f"dms-replication-instance-{self.deploy_env.value}",
allocated_storage=100,
publicly_accessible=False,
engine_version="3.4.4",
replication_instance_class="dms.t2.small",
replication_instance_identifier=f"dms-{self.deploy_env.value}-replication-instance",
vpc_security_group_ids=[self.dms_sg.security_group_id],
replication_subnet_group_identifier=self.dms_subnet_group.replication_subnet_group_identifier,
)
self.instance.node.add_dependency(self.dms_subnet_group)
self.instance.node.add_dependency(self.dms_sg)
super().__init__(
scope,
f"{self.deploy_env.value}-dms-task-orders-rds",
migration_type="full-load-and-cdc",
replication_task_identifier=f"{self.deploy_env.value}-dms-task-orders-rds",
replication_instance_arn=self.instance.ref,
source_endpoint_arn=self.rds_endpoint.ref,
target_endpoint_arn=self.s3_endpoint.ref,
table_mappings=json.dumps(
{
"rules": [
{
"rule-type": "selection",
"rule-id": "1",
"rule-name": "1",
"object-locator": {
"schema-name": "%",
"table-name": "%",
},
"rule-action": "include",
"filters": [],
}
]
}
),
)
def __init__(self, scope: core.Construct, common: Common,
data_lake: DataLake, **kwargs) -> None:
self.rds_endpoint = dms.CfnEndpoint(
scope,
f'dms-{common.env}-orders-rds-endpoint',
endpoint_type='source',
endpoint_identifier=f'dms-source-{common.env}-orders-rds-endpoint',
engine_name='postgres',
password=core.CfnDynamicReference(
core.CfnDynamicReferenceService.SECRETS_MANAGER,
key=
f'{common.orders_rds.secret.secret_arn}:SecretString:password'
).to_string(),
username=core.CfnDynamicReference(
core.CfnDynamicReferenceService.SECRETS_MANAGER,
key=
f'{common.orders_rds.secret.secret_arn}:SecretString:username'
).to_string(),
database_name=core.CfnDynamicReference(
core.CfnDynamicReferenceService.SECRETS_MANAGER,
key=f'{common.orders_rds.secret.secret_arn}:SecretString:dbname'
).to_string(),
port=5432,
server_name=common.orders_rds.db_instance_endpoint_address,
)
self.s3_endpoint = dms.CfnEndpoint(
scope,
f'dms-{common.env}-orders-s3-endpoint',
endpoint_type='target',
engine_name='s3',
endpoint_identifier=f'dms-target-{common.env}-orders-s3-endpoint',
extra_connection_attributes=
"DataFormat=parquet;maxFileSize=131072;timestampColumnName=extracted_at;includeOpForFullLoad=true;cdcInsertsAndUpdates=true",
s3_settings=dms.CfnEndpoint.S3SettingsProperty(
bucket_name=data_lake.data_lake_raw_bucket.bucket_name,
bucket_folder='orders',
compression_type='gzip',
csv_delimiter=',',
csv_row_delimiter='\n',
service_access_role_arn=RawDMSRole(
scope, common.env,
data_lake.data_lake_raw_bucket).role_arn))
self.dms_sg = ec2.SecurityGroup(
scope,
f'dms-{common.env}-sg',
vpc=common.custom_vpc,
security_group_name=f'dms-{common.env}-sg',
)
self.dms_subnet_group = dms.CfnReplicationSubnetGroup(
scope,
f'dms-{common.env}-replication-subnet',
replication_subnet_group_description=
'dms replication instance subnet group',
subnet_ids=[
subnet.subnet_id
for subnet in common.custom_vpc.private_subnets
],
replication_subnet_group_identifier=
f'dms-{common.env}-replication-subnet')
self.instance = dms.CfnReplicationInstance(
scope,
f'dms-replication-instance-{common.env}',
allocated_storage=100,
publicly_accessible=False,
engine_version='3.3.2',
replication_instance_class='dms.t2.small',
replication_instance_identifier=
f'dms-{common.env}-replication-instance',
vpc_security_group_ids=[self.dms_sg.security_group_id],
replication_subnet_group_identifier=self.dms_subnet_group.
replication_subnet_group_identifier)
super().__init__(
scope,
f'{common.env}-dms-task-orders-rds',
migration_type='full-load-and-cdc',
replication_task_identifier=f'{common.env}-dms-task-orders-rds',
replication_instance_arn=self.instance.ref,
source_endpoint_arn=self.rds_endpoint.ref,
target_endpoint_arn=self.s3_endpoint.ref,
table_mappings=json.dumps({
"rules": [{
"rule-type": "selection",
"rule-id": "1",
"rule-name": "1",
"object-locator": {
"schema-name": "%",
"table-name": "%",
},
"rule-action": "include",
"filters": []
}]
}))
