def __init__(
self, scope: core.Construct, construct_id: str, iamRole: iam.Role,
kdaAppName: str, logGroup: log.LogGroup, logStream: log.LogStream,
appConfig: kda.CfnApplicationV2.ApplicationConfigurationProperty,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
core.CfnOutput(self, "RoleNameFlinkIsUsing", value=iamRole.role_name)
core.CfnOutput(self, "KDA App Name", value=kdaAppName)
# The code that defines your stack goes here
kdaApp = kda.CfnApplicationV2(
self,
"kda-flink-application",
runtime_environment="FLINK-1_11",
service_execution_role=iamRole.role_arn,
application_description="KDA Flink app for BikeRides",
application_name=kdaAppName,
application_configuration=appConfig,
)
kda.CfnApplicationCloudWatchLoggingOptionV2(
self,
"FlinkLogging",
application_name=kdaApp.ref,
cloud_watch_logging_option=kda.
CfnApplicationCloudWatchLoggingOptionV2.
CloudWatchLoggingOptionProperty(
log_stream_arn=
"arn:aws:logs:{0}:{1}:log-group:{2}:log-stream:{3}".format(
core.Aws.REGION, core.Aws.ACCOUNT_ID,
logGroup.log_group_name, logStream.log_stream_name)))
def __init__(self, scope: core.Construct, construct_id: str,
stream: kinesis.IStream, db_name: str, table_name: str,
kda_role: iam.IRole, log_group_name: str,
log_stream_name: str, asset: assets.Asset, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
batch_size_param = core.CfnParameter(
self,
"batchSizeParam",
type="Number",
min_value=1,
max_value=100,
default=75,
description=
"Number of records ingested from stream before flushing to "
"Timestream database")
kda_application = kda.CfnApplicationV2(
self,
"KdaApplication",
runtime_environment="FLINK-1_11",
service_execution_role=kda_role.role_arn,
application_name=core.Aws.STACK_NAME,
application_configuration=kda.CfnApplicationV2.
ApplicationConfigurationProperty(
application_code_configuration=kda.CfnApplicationV2.
ApplicationCodeConfigurationProperty(
code_content=kda.CfnApplicationV2.CodeContentProperty(
s3_content_location=kda.CfnApplicationV2.
S3ContentLocationProperty(
bucket_arn=asset.bucket.bucket_arn,
file_key=asset.s3_object_key)),
code_content_type="ZIPFILE"),
environment_properties=kda.CfnApplicationV2.
EnvironmentPropertiesProperty(property_groups=[
kda.CfnApplicationV2.PropertyGroupProperty(
property_group_id="FlinkApplicationProperties",
property_map={
"InputStreamName":
stream.stream_name,
"Region":
core.Aws.REGION,
"TimestreamDbName":
db_name,
"TimestreamTableName":
table_name,
"TimestreamIngestBatchSize":
batch_size_param.value_as_number
})
]),
application_snapshot_configuration=kda.CfnApplicationV2.
ApplicationSnapshotConfigurationProperty(
snapshots_enabled=False),
flink_application_configuration=kda.CfnApplicationV2.
FlinkApplicationConfigurationProperty(
monitoring_configuration=kda.CfnApplicationV2.
MonitoringConfigurationProperty(
configuration_type="CUSTOM",
log_level="INFO",
metrics_level="TASK"),
parallelism_configuration=kda.CfnApplicationV2.
ParallelismConfigurationProperty(
configuration_type="CUSTOM",
auto_scaling_enabled=False,
parallelism=1,
parallelism_per_kpu=1),
checkpoint_configuration=kda.CfnApplicationV2.
CheckpointConfigurationProperty(
configuration_type="CUSTOM",
# the properties below are optional
checkpointing_enabled=True,
checkpoint_interval=60_000,
min_pause_between_checkpoints=60_000))))
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# The code that defines your stack goes here
#获取vpc
#vpc = ec2.Vpc.from_lookup(self, 'default',is_default=True,vpc_name='default')
vpc = ec2.Vpc.from_lookup(self,
'dms-vpc',
vpc_id='vpc-08b56fb6053ca2c75')
#创建RDS参数组
db_parameter = rds.ParameterGroup(
self,
'dms-param-mysql5.7',
engine=rds.DatabaseInstanceEngine.mysql(
version=rds.MysqlEngineVersion.VER_5_7),
parameters={"binlog_format": "ROW"})
# sourceDB = rds.DatabaseInstanceFromSnapshot(
# self,'dms-rds-soruce',
# snapshot_identifier= 'tickets-mysql57',
# engine=rds.DatabaseInstanceEngine.MYSQL,
# instance_type=ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE3,ec2.InstanceSize.MEDIUM),
# vpc=vpc,
# parameter_group=db_parameter
# )
# sourceDB = rds.DatabaseInstance(
# self,'dms-rds-soruce',
# #instance_identifier='dms-rds-soruce',
# engine=rds.DatabaseInstanceEngine.mysql(
# version=rds.MysqlEngineVersion.VER_5_7
# ),
# instance_type=ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE3,ec2.InstanceSize.MEDIUM),
# vpc=vpc,
# parameter_group=db_parameter,
# #credentials=rdsPasswordSecret
# )
# sourceDB.connections.allow_default_port_internally()
dms_rep = dms.CfnReplicationInstance(
self,
'dms-replication',
replication_instance_class='dms.c5.large',
engine_version='3.4.0')
stream = kinesis.Stream(self, 'dms-steam')
streamWriteRole = iam.Role(
self,
'dms-stream-role',
assumed_by=iam.ServicePrincipal('dms.amazonaws.com'))
streamWriteRole.add_to_policy(
iam.PolicyStatement(resources=[stream.stream_arn],
actions=[
'kinesis:DescribeStream',
'kinesis:PutRecord', 'kinesis:PutRecords'
]))
source = dms.CfnEndpoint(
self,
'dms-source',
endpoint_type='source',
engine_name='mysql',
username='******',
password='******',
server_name=
"dms-rdssource.c7iucbqgd2xo.us-east-1.rds.amazonaws.com",
port=3306)
target = dms.CfnEndpoint(self,
'dms-target',
endpoint_type='target',
engine_name='kinesis',
kinesis_settings={
"messageFormat":
"JSON",
'streamArn':
stream.stream_arn,
"serviceAccessRoleArn":
streamWriteRole.role_arn
})
dmsTableMappings = {
"rules": [{
"rule-type": "selection",
"rule-id": "1",
"rule-name": "1",
"object-locator": {
"schema-name": "dms_sample",
"table-name": "t_log_levelup"
},
"rule-action": "include"
}]
}
dms.CfnReplicationTask(self,
'dms-stream-repTask',
replication_instance_arn=dms_rep.ref,
migration_type='full-load-and-cdc',
source_endpoint_arn=source.ref,
target_endpoint_arn=target.ref,
table_mappings=json.dumps(dmsTableMappings))
analyticsRole = iam.Role(
self,
'KinesisAnalyticsRole',
assumed_by=iam.ServicePrincipal('kinesisanalytics.amazonaws.com'))
kinesisanalytics.CfnApplicationV2(
self,
'KinesisAnalytics',
application_name='dms-stream-anlytics',
service_execution_role=analyticsRole.role_arn,
runtime_environment='SQL-1_0',
application_configuration={
'sqlApplicationConfiguration': {
'inputs': [{
'namePrefix': "exampleNamePrefix",
'inputSchema': {
'recordColumns': [{
'name': "example",
'sqlType': "VARCHAR(16)",
'mapping': "$.example"
}],
'recordFormat': {
'recordFormatType': "JSON",
'mappingParameters': {
'jsonMappingParameters': {
'recordRowPath': "$"
}
}
}
},
'kinesisStreamsInput': {
'resourceArn': stream.stream_arn
}
}]
},
'applicationCodeConfiguration': {
'codeContent': {
'textContent': "Example Application Code"
},
'codeContentType': "PLAINTEXT"
}
})
def __init__(self, scope: core.Construct, id: str, stream: kinesis.Stream,
stream_key: kms.Key, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
fat_jar = "payment-digestor-1.0-SNAPSHOT-all.jar"
path_to_fat_jar = f"../payment-digestor/build/libs/"
flink_log_group = logs.LogGroup(
self,
"FlinkLogGroup",
log_group_name="/aws/kinesis-analytics/FlinkLogGroup",
removal_policy=core.RemovalPolicy.DESTROY,
retention=logs.RetentionDays.THREE_DAYS)
flink_log_stream = flink_log_group.add_stream(
"FlinkLogStream", log_stream_name="FlinkLogStream")
flink_app_bucket = s3.Bucket(self, "FlinkAppBucket")
flink_fatjar_deployment = s3_deploy.BucketDeployment(
self,
"fat-jar-upload",
destination_bucket=flink_app_bucket,
sources=[s3_deploy.Source.asset(path_to_fat_jar)])
flink_fatjar_deployment.node.add_dependency(flink_app_bucket)
flink_role = iam.Role(
self,
"FlinkRole",
assumed_by=iam.ServicePrincipal(
service="kinesisanalytics.amazonaws.com"),
inline_policies={
"flinkPolicy":
iam.PolicyDocument(statements=[
iam.PolicyStatement(
sid="ReadCode",
effect=iam.Effect.ALLOW,
actions=["s3:GetObject", "s3:GetObjectVersion"],
resources=[f"{flink_app_bucket.bucket_arn}/*"]),
iam.PolicyStatement(
sid="ListCloudwatchLogGroups",
effect=iam.Effect.ALLOW,
actions=["logs:DescribeLogGroups"],
resources=[
f"arn:aws:logs:{self.region}:{self.account}:log-group:*"
]),
iam.PolicyStatement(
sid="ListCloudwatchLogStreams",
effect=iam.Effect.ALLOW,
actions=["logs:DescribeLogStreams"],
resources=[
f"arn:aws:logs:{self.region}:{self.account}:log-group:{flink_log_group.log_group_name}:log-stream:*"
]),
iam.PolicyStatement(
sid="PutCloudwatchLogs",
effect=iam.Effect.ALLOW,
actions=["logs:PutLogEvents", "logs:PutMetricData"],
resources=[
f"arn:aws:logs:{self.region}:{self.account}:log-group:{flink_log_group.log_group_name}:log-stream:{flink_log_stream.log_stream_name}"
]),
iam.PolicyStatement(sid="DecryptStream",
effect=iam.Effect.ALLOW,
actions=["kms:*"],
resources=[stream_key.key_arn]),
iam.PolicyStatement(
sid="ReadWriteInputStream",
effect=iam.Effect.ALLOW,
actions=[
"kinesis:DescribeStream",
"kinesis:GetShardIterator", "kinesis:GetRecords",
"kinesis:ListShards", "kinesis:PutRecords"
],
resources=[stream.stream_arn])
])
})
flink_app = kinesis_analytics.CfnApplicationV2(
self,
id,
application_name='PaymentDigestor',
runtime_environment="FLINK-1_8",
service_execution_role=flink_role.role_arn,
application_configuration={
"applicationCodeConfiguration": {
"codeContent": {
"s3ContentLocation": {
"bucketArn": flink_app_bucket.bucket_arn,
"fileKey": fat_jar
}
},
"codeContentType": "ZIPFILE"
},
"flinkApplicationConfiguration": {
"monitoringConfiguration": {
"configurationType": "CUSTOM",
"logLevel": "INFO",
"metricsLevel": "TASK"
}
},
"applicationSnapshotConfiguration": {
"snapshotsEnabled": True
}
})
flink_app.node.add_dependency(flink_fatjar_deployment)
flink_log_group_arn = f"arn:aws:logs:{self.region}:{self.account}:log-group:{flink_log_group.log_group_name}:log-stream:{flink_log_stream.log_stream_name}"
flink_log_group = kinesis_analytics.CfnApplicationCloudWatchLoggingOptionV2(
self,
"flinkLogGroup",
application_name=flink_app.ref,
cloud_watch_logging_option={"logStreamArn": flink_log_group_arn})
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# The code that defines your stack goes here
vpc_name = self.node.try_get_context('vpc_name')
vpc = aws_ec2.Vpc.from_lookup(self,
'ExistingVPC',
is_default=True,
vpc_name=vpc_name)
s3_bucket_name = self.node.try_get_context('s3_bucket_name')
s3_bucket = s3.Bucket.from_bucket_name(self, 'S3KdaFlinkCodeLocation',
s3_bucket_name)
s3_path_to_flink_app_code = self.node.try_get_context(
's3_path_to_flink_app_code')
KDA_APP_NAME = 'KdaMskReplcation'
kda_exec_role_policy_doc = aws_iam.PolicyDocument()
kda_exec_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"sid":
"ReadCode",
"effect":
aws_iam.Effect.ALLOW,
"resources": [
"{}/{}".format(s3_bucket.bucket_arn,
s3_path_to_flink_app_code)
],
"actions": ["s3:GetObject", "s3:GetObjectVersion"]
}))
kda_exec_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"sid":
"ListCloudwatchLogGroups",
"effect":
aws_iam.Effect.ALLOW,
#XXX: The ARN will be formatted as follows:
# arn:{partition}:{service}:{region}:{account}:{resource}{sep}{resource-name}
"resources": [
self.format_arn(
service="logs",
resource="log-group",
resource_name=
"/aws/kinesis-analytics/{}:log-stream:*".format(
KDA_APP_NAME),
arn_format=cdk.ArnFormat.COLON_RESOURCE_NAME)
],
"actions": ["logs:DescribeLogGroups"]
}))
kda_exec_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"sid":
"ListCloudwatchLogStreams",
"effect":
aws_iam.Effect.ALLOW,
#XXX: The ARN will be formatted as follows:
# arn:{partition}:{service}:{region}:{account}:{resource}{sep}{resource-name}
"resources": [
self.format_arn(
service="logs",
resource="log-group",
resource_name=
"/aws/kinesis-analytics/{}:log-stream:kinesis-analytics-log-stream"
.format(KDA_APP_NAME),
arn_format=cdk.ArnFormat.COLON_RESOURCE_NAME)
],
"actions": ["logs:DescribeLogStreams"]
}))
kda_exec_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"sid":
"PutCloudwatchLogs",
"effect":
aws_iam.Effect.ALLOW,
#XXX: The ARN will be formatted as follows:
# arn:{partition}:{service}:{region}:{account}:{resource}{sep}{resource-name}
"resources": [
self.format_arn(
service="logs",
resource="log-group",
resource_name=
"/aws/kinesis-analytics/{}:log-stream:kinesis-analytics-log-stream"
.format(KDA_APP_NAME),
arn_format=cdk.ArnFormat.COLON_RESOURCE_NAME)
],
"actions": ["logs:PutLogEvents"]
}))
kda_exec_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"sid":
"ENIReadWritePermissions",
"effect":
aws_iam.Effect.ALLOW,
"resources": ["*"],
"actions": [
"ec2:CreateNetworkInterface",
"ec2:CreateNetworkInterfacePermission",
"ec2:DescribeNetworkInterfaces",
"ec2:DeleteNetworkInterface"
]
}))
kda_exec_role_policy_name = "kinesis-analytics-service-{kda_app_name}-{region}".format(
region=cdk.Aws.REGION, kda_app_name=KDA_APP_NAME),
kda_execution_role = aws_iam.Role(
self,
'KdaExecutionRole',
role_name='kinesis-analytics-{kda_app_name}-{region}'.format(
region=cdk.Aws.REGION, kda_app_name=KDA_APP_NAME),
assumed_by=aws_iam.ServicePrincipal(
'kinesisanalytics.amazonaws.com'),
path='/service-role/',
inline_policies={
'kinesis-analytics-service': kda_exec_role_policy_doc
},
managed_policies=[
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonVPCReadOnlyAccess'),
])
kda_flink_code_content = aws_kda_flink.CfnApplicationV2.CodeContentProperty(
s3_content_location=aws_kda_flink.CfnApplicationV2.
S3ContentLocationProperty(bucket_arn=s3_bucket.bucket_arn,
file_key=s3_path_to_flink_app_code))
kda_flink_code_config = aws_kda_flink.CfnApplicationV2.ApplicationCodeConfigurationProperty(
code_content=kda_flink_code_content, code_content_type='ZIPFILE')
kda_flink_property_groups = self.node.try_get_context(
'kda_flink_property_groups')
_property_groups = [
aws_kda_flink.CfnApplicationV2.PropertyGroupProperty(**elem)
for elem in kda_flink_property_groups
]
kda_flink_env_props = aws_kda_flink.CfnApplicationV2.EnvironmentPropertiesProperty(
property_groups=_property_groups)
flink_app_config = aws_kda_flink.CfnApplicationV2.FlinkApplicationConfigurationProperty(
checkpoint_configuration=aws_kda_flink.CfnApplicationV2.
CheckpointConfigurationProperty(
configuration_type='CUSTOM',
checkpointing_enabled=True,
checkpoint_interval=60000,
min_pause_between_checkpoints=60000),
monitoring_configuration=aws_kda_flink.CfnApplicationV2.
MonitoringConfigurationProperty(configuration_type='CUSTOM',
log_level='INFO',
metrics_level='TASK'),
parallelism_configuration=aws_kda_flink.CfnApplicationV2.
ParallelismConfigurationProperty(
configuration_type='CUSTOM',
auto_scaling_enabled=False,
parallelism=1,
parallelism_per_kpu=1,
))
kda_flink_app_config = aws_kda_flink.CfnApplicationV2.ApplicationConfigurationProperty(
application_code_configuration=kda_flink_code_config,
application_snapshot_configuration=aws_kda_flink.CfnApplicationV2.
ApplicationSnapshotConfigurationProperty(snapshots_enabled=False),
environment_properties=kda_flink_env_props,
flink_application_configuration=flink_app_config)
kda_app = aws_kda_flink.CfnApplicationV2(
self,
'KdaMskReplication',
runtime_environment='FLINK-1_11',
service_execution_role=kda_execution_role.role_arn,
application_configuration=kda_flink_app_config,
application_description=
'A Kinesis Data Analytics application that reads from one Amazon MSK topic and writes to another',
application_name=KDA_APP_NAME)
kda_app_log_group = aws_logs.LogGroup(
self,
'KdaMskReplicationLogGroup',
log_group_name='/aws/kinesis-analytics/{}'.format(KDA_APP_NAME),
retention=aws_logs.RetentionDays.THREE_DAYS,
removal_policy=cdk.RemovalPolicy.DESTROY)
kda_app_log_stream = aws_logs.LogStream(
self,
'KdaMskReplicationLogStream',
log_group=kda_app_log_group,
log_stream_name='kinesis-analytics-log-stream',
removal_policy=cdk.RemovalPolicy.DESTROY)
#XXX: The ARN will be formatted as follows:
# arn:{partition}:{service}:{region}:{account}:{resource}{sep}{resource-name}
kda_app_log_stream_arn = self.format_arn(
service="logs",
resource="log-group",
resource_name=
"/aws/kinesis-analytics/{}:log-stream:kinesis-analytics-log-stream"
.format(KDA_APP_NAME),
arn_format=cdk.ArnFormat.COLON_RESOURCE_NAME)
kda_app_cw_log = aws_kda_flink.CfnApplicationCloudWatchLoggingOptionV2(
self,
'KdaMskReplicationCWLog',
application_name=kda_app.application_name,
cloud_watch_logging_option=aws_kda_flink.
CfnApplicationCloudWatchLoggingOptionV2.
CloudWatchLoggingOptionProperty(
log_stream_arn=kda_app_log_stream_arn))
kda_app_cw_log.add_depends_on(kda_app)