def create_log_group_and_stream(self) -> aws_logs.LogGroup: log_group = aws_logs.LogGroup( self, "integ_test_firehose_delivery_log_group", log_group_name=FirehoseStack.LOG_GROUP_NAME, removal_policy=core.RemovalPolicy.DESTROY, retention=aws_logs.RetentionDays.FIVE_DAYS, ) aws_logs.LogStream( self, "integ_test_firehose_delivery_log_stream", log_group=log_group, log_stream_name=FirehoseStack.LOG_STREAM_NAME, removal_policy=core.RemovalPolicy.DESTROY, ) return log_group
def __init__(self, scope: core.Stack, id: str, vpc, cluster, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.vpc = vpc self.cluster = cluster # Building a custom image for jenkins leader. self.container_image = ecr.DockerImageAsset( self, "JenkinsWorkerDockerImage", directory='./docker/worker/') # Security group to connect workers to leader self.worker_security_group = ec2.SecurityGroup( self, "WorkerSecurityGroup", vpc=self.vpc, description="Jenkins Worker access to Jenkins leader", ) # IAM execution role for the workers to pull from ECR and push to CloudWatch logs self.worker_execution_role = iam.Role( self, "WorkerExecutionRole", assumed_by=iam.ServicePrincipal('ecs-tasks.amazonaws.com'), ) self.worker_execution_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonECSTaskExecutionRolePolicy')) # Task role for worker containers - add to this role for any aws resources that jenkins requires access to self.worker_task_role = iam.Role( self, "WorkerTaskRole", assumed_by=iam.ServicePrincipal('ecs-tasks.amazonaws.com'), ) # Create log group for workers to log self.worker_logs_group = logs.LogGroup( self, "WorkerLogGroup", retention=logs.RetentionDays.ONE_DAY) # Create log stream for worker log group self.worker_log_stream = logs.LogStream( self, "WorkerLogStream", log_group=self.worker_logs_group)
def __init__(self, scope: Construct, construct_id: str, **kwargs: str) -> None: super().__init__(scope, construct_id, **kwargs) self.vpc = ec2.Vpc( self, "aws-data-wrangler-vpc", cidr="11.19.224.0/19", enable_dns_hostnames=True, enable_dns_support=True, ) Tags.of(self.vpc).add("Name", "aws-data-wrangler") self.key = kms.Key( self, id="aws-data-wrangler-key", description="Aws Data Wrangler Test Key.", policy=iam.PolicyDocument(statements=[ iam.PolicyStatement( sid="Enable IAM User Permissions", effect=iam.Effect.ALLOW, actions=["kms:*"], principals=[iam.AccountRootPrincipal()], resources=["*"], ) ]), ) kms.Alias( self, "aws-data-wrangler-key-alias", alias_name="alias/aws-data-wrangler-key", target_key=self.key, ) self.bucket = s3.Bucket( self, id="aws-data-wrangler", block_public_access=s3.BlockPublicAccess( block_public_acls=True, block_public_policy=True, ignore_public_acls=True, restrict_public_buckets=True, ), lifecycle_rules=[ s3.LifecycleRule( id="CleaningUp", enabled=True, expiration=Duration.days(1), abort_incomplete_multipart_upload_after=Duration.days(1), ), ], versioned=True, ) glue_db = glue.Database( self, id="aws_data_wrangler_glue_database", database_name="aws_data_wrangler", location_uri=f"s3://{self.bucket.bucket_name}", ) log_group = logs.LogGroup( self, id="aws_data_wrangler_log_group", retention=logs.RetentionDays.ONE_MONTH, ) log_stream = logs.LogStream( self, id="aws_data_wrangler_log_stream", log_group=log_group, ) CfnOutput(self, "Region", value=self.region) CfnOutput( self, "VPC", value=self.vpc.vpc_id, export_name="aws-data-wrangler-base-VPC", ) CfnOutput( self, "PublicSubnet1", value=self.vpc.public_subnets[0].subnet_id, export_name="aws-data-wrangler-base-PublicSubnet1", ) CfnOutput( self, "PublicSubnet2", value=self.vpc.public_subnets[1].subnet_id, export_name="aws-data-wrangler-base-PublicSubnet2", ) CfnOutput( self, "PrivateSubnet", value=self.vpc.private_subnets[0].subnet_id, export_name="aws-data-wrangler-base-PrivateSubnet", ) CfnOutput( self, "KmsKeyArn", value=self.key.key_arn, export_name="aws-data-wrangler-base-KmsKeyArn", ) CfnOutput( self, "BucketName", value=self.bucket.bucket_name, export_name="aws-data-wrangler-base-BucketName", ) CfnOutput(self, "GlueDatabaseName", value=glue_db.database_name) CfnOutput(self, "LogGroupName", value=log_group.log_group_name) CfnOutput(self, "LogStream", value=log_stream.log_stream_name)
def _build_firehose_delivery_stream(self, *, stack, vpc_db_instance): self.kfh_log_group = logs.LogGroup( stack, "exampledeliverystreamloggroup", log_group_name="/aws/kinesisfirehose/exampledeliverystream") self.kfh_es_log_stream = logs.LogStream(stack, "deliverytoeslogstream", log_stream_name="deliverytoes", log_group=self.kfh_log_group) self.kfh_instance = kfh.CfnDeliveryStream( stack, 'exampledeliverystream', delivery_stream_type='DirectPut', elasticsearch_destination_configuration={ "indexName": "webappclickstream", "cloudwatch_logging_options": { "Enabled": True, "LogGroupName": "exampledeliverystream", "LogStreamName": "deliverytoes" }, "roleArn": self.firehose_role.role_arn, "s3Configuration": { "bucketArn": self.firehose_bucket.bucket_arn, "roleArn": self.firehose_role.role_arn }, "domainArn": self.elastic_search.attr_arn, "vpcConfiguration": { "roleArn": self.firehose_role.role_arn, "securityGroupIds": [self.kfh_security_group.security_group_id], "subnetIds": [ vpc_db_instance.vpc.select_subnets( subnet_type=ec2.SubnetType.PRIVATE).subnet_ids[0] ] }, "bufferingHints": { "intervalInSeconds": 60, "sizeInMBs": 1 }, "ProcessingConfiguration": { "enabled": True, "Processors": [] }, "ProcessingConfiguration": { "Enabled": "true", "Processors": [{ "Parameters": [{ "ParameterName": "LambdaArn", "ParameterValue": self.lambda_transform_fn.function_arn }], "Type": "Lambda" }] } })
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # this could be parameter as well kdaAppName = "kda-flink-application" # Our Flink Jar File kdaApplicationJar = core.CfnParameter( self, "kdaApplicationJar", type="String", description="The name of the Flink JAR File.", default="kda-flink-app-1.0-SNAPSHOT.jar") bucketNameParm = core.CfnParameter( self, "bucketNameParm", type="String", description= "The name of the Amazon S3 bucket where Flink JAR File is.") inputStream = core.CfnParameter( self, "inputStream", type="String", description="The name of the Kinesis data stream for Input.", default="ProducerStream") outputStream = core.CfnParameter( self, "outputStream", type="String", description="The name of the Kinesis Firehose delivery steram.", default="AnalyticsOutput") region = core.CfnOutput(self, "region", value=self.region) account = core.CfnOutput(self, "account", value=self.account) # s3 bucket where JAR file is s3Bucket = s3.Bucket.from_bucket_name( self, "sourceBucket", bucket_name=bucketNameParm.value_as_string) bucketPolicy = iam.PolicyStatement( sid="GrantS3Access", actions=["s3:*"], resources=[s3Bucket.bucket_arn, s3Bucket.bucket_arn + "/*"]) core.CfnOutput(self, "KDASourceBucketARN", value=s3Bucket.bucket_arn) core.CfnOutput(self, "KDASourceBucketName", value=s3Bucket.bucket_name) # KDA Role - will be assumed by KDA to load JAR as well as to read from input or write to sink kdaIAMRole = iam.Role( self, "kdaIAMRole", assumed_by=iam.ServicePrincipal('kinesisanalytics.amazonaws.com'), description="Kinesis Analytics role for application " + kdaAppName) kdaIAMRole.add_to_policy(statement=bucketPolicy) # input Kinesis Data Stream kdsStream = kds.Stream.from_stream_arn( self, id="kdsinputstream", stream_arn="arn:aws:kinesis:" + region.value + ":" + account.value + ":stream/" + inputStream.value_as_string) # output (sink) Firehose Delivery Stream fhDeliveryARN = "arn:aws:firehose:" + region.value + ":" + account.value + ":deliverystream/" + outputStream.value_as_string # Logs and CWatch logGroup = log.LogGroup(self, kdaAppName + "LogGroup", retention=log.RetentionDays.ONE_DAY) core.CfnOutput(self, "LogGroupName", value=logGroup.log_group_name) logStream = log.LogStream(self, kdaAppName + "LogStream", log_group=logGroup) # grant permissions to KDA IAM role s3Bucket.grant_read(identity=kdaIAMRole) kdsStream.grant_read(grantee=kdaIAMRole) cwatch.Metric.grant_put_metric_data(grantee=kdaIAMRole) logGroup.grant_write(grantee=kdaIAMRole) kdaIAMRole.add_to_policy( iam.PolicyStatement(sid="DescribeLog", resources=[ "arn:aws:logs:{0}:{1}:log-group:*".format( core.Aws.REGION, core.Aws.ACCOUNT_ID) ], actions=["logs:DescribeLog*"])) kdaIAMRole.add_to_policy( iam.PolicyStatement(sid="FullAccessToJARFile", resources=[ s3Bucket.bucket_arn + "/" + kdaApplicationJar.value_as_string ], actions=['s3:*'])) kdaIAMRole.add_to_policy( iam.PolicyStatement(sid="WriteToFirehose", resources=[fhDeliveryARN], actions=["firehose:*"])) # KDA Flink application Configuration snapshots = kda.CfnApplicationV2.ApplicationSnapshotConfigurationProperty( snapshots_enabled=False) codeContent = kda.CfnApplicationV2.ApplicationCodeConfigurationProperty( code_content=kda.CfnApplicationV2.CodeContentProperty( s3_content_location=kda.CfnApplicationV2. S3ContentLocationProperty( bucket_arn=s3Bucket.bucket_arn, file_key=kdaApplicationJar.value_as_string)), code_content_type="ZIPFILE") appProperties = kda.CfnApplicationV2.EnvironmentPropertiesProperty( property_groups=[ kda.CfnApplicationV2.PropertyGroupProperty( property_group_id="ConsumerConfigProperties", property_map=({ "AWS_REGION": region.value, "INPUT_STREAM": inputStream.value_as_string, "flink.inputstream.initpos": "LATEST" })), kda.CfnApplicationV2.PropertyGroupProperty( property_group_id="OutputConfigProperties", property_map=({ "AWS_REGION": region.value, "OUTPUT_KDF": outputStream.value_as_string })) ]) appConfig = kda.CfnApplicationV2.ApplicationConfigurationProperty( application_code_configuration=codeContent, application_snapshot_configuration=snapshots, environment_properties=appProperties) # call KDA APP stack to create KDA Flink Application KdaAppStack(self, "app-stack", iamRole=kdaIAMRole, kdaAppName=kdaAppName, logGroup=logGroup, logStream=logStream, appConfig=appConfig)
def __init__(self, scope: core.Stack, id: str, **kwargs): super().__init__(scope, id, **kwargs) self.output_bucket = aws_s3.Bucket( self, "BucketTwitterStreamOutput", bucket_name = self.stack_name, ) self.bucket_url = self.output_bucket.bucket_regional_domain_name # Because kinesis firehose bindings are to direct CF, we have to create IAM policy/role and attach on our own self.iam_role = aws_iam.Role( self, "IAMRoleTwitterStreamKinesisFHToS3", role_name="KinesisFirehoseToS3-{}".format(self.stack_name), assumed_by=aws_iam.ServicePrincipal(service='firehose.amazonaws.com'), ) # S3 bucket actions self.s3_iam_policy_statement = aws_iam.PolicyStatement() actions = ["s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject"] for action in actions: self.s3_iam_policy_statement.add_actions(action) self.s3_iam_policy_statement.add_resources(self.output_bucket.bucket_arn) self.s3_iam_policy_statement.add_resources(self.output_bucket.bucket_arn + "/*") # CW error log setup self.s3_error_logs_group = aws_logs.LogGroup( self, "S3ErrorLogsGroup", log_group_name="{}-s3-errors".format(self.stack_name) ) self.s3_error_logs_stream = aws_logs.LogStream( self, "S3ErrorLogsStream", log_group=self.s3_error_logs_group, log_stream_name='s3Backup' ) self.firehose = aws_kinesisfirehose.CfnDeliveryStream( self, "FirehoseTwitterStream", delivery_stream_name = "{}-raw".format(self.stack_name), delivery_stream_type = "DirectPut", s3_destination_configuration={ 'bucketArn': self.output_bucket.bucket_arn, 'bufferingHints': { 'intervalInSeconds': 120, 'sizeInMBs': 10 }, 'compressionFormat': 'UNCOMPRESSED', 'roleArn': self.iam_role.role_arn, 'cloudWatchLoggingOptions': { 'enabled': True, 'logGroupName': "{}-raw".format(self.stack_name), 'logStreamName': 's3BackupRaw' }, 'prefix': 'twitter-raw/' }, ) # TODO: Only attach what's needed for this policy, right now i'm lazy and attaching all policies self.iam_policy = aws_iam.Policy( self, "IAMPolicyTwitterStreamKinesisFHToS3", policy_name="KinesisFirehoseToS3-{}".format(self.stack_name), statements=[self.s3_iam_policy_statement], ) self.iam_policy.attach_to_role(self.iam_role) # Because kinesis firehose bindings are to direct CF, we have to create IAM policy/role and attach on our own self.curator_firehose = aws_kinesisfirehose.CfnDeliveryStream( self, "CuratorFirehoseStream", delivery_stream_name = "{}-curator".format(self.stack_name), delivery_stream_type = "DirectPut", s3_destination_configuration={ 'bucketArn': self.output_bucket.bucket_arn, 'bufferingHints': { 'intervalInSeconds': 120, 'sizeInMBs': 10 }, 'compressionFormat': 'UNCOMPRESSED', 'roleArn': self.iam_role.role_arn, 'cloudWatchLoggingOptions': { 'enabled': True, 'logGroupName': "{}-curator".format(self.stack_name), 'logStreamName': 's3BackupCurator' }, 'prefix': 'twitter-curated/' }, ) def zip_package(): cwd = os.getcwd() file_name = 'curator-lambda.zip' zip_file = cwd + '/' + file_name os.chdir('src/') sh.zip('-r9', zip_file, '.') os.chdir(cwd) return file_name, zip_file _, zip_file = zip_package() self.twitter_stream_curator_lambda_function = aws_lambda.Function( self, "TwitterStreamCuratorLambdaFunction", function_name="{}-curator".format(self.stack_name), code=aws_lambda.AssetCode(zip_file), handler="sentiment_analysis.lambda_handler", runtime=aws_lambda.Runtime.PYTHON_3_7, tracing=aws_lambda.Tracing.ACTIVE, description="Triggers from S3 PUT event for twitter stream data and transorms it to clean json syntax with sentiment analysis attached", environment={ "STACK_NAME": self.stack_name, "FIREHOSE_STREAM": self.curator_firehose.delivery_stream_name }, memory_size=128, timeout=core.Duration.seconds(120), log_retention=aws_logs.RetentionDays.ONE_WEEK, ) # Permission to talk to comprehend for sentiment analysis self.comprehend_iam_policy_statement = aws_iam.PolicyStatement() self.comprehend_iam_policy_statement.add_actions('comprehend:*') self.comprehend_iam_policy_statement.add_all_resources() self.twitter_stream_curator_lambda_function.add_to_role_policy(self.comprehend_iam_policy_statement) # Permission to put in kinesis firehose self.curator_firehose_iam_policy_statement = aws_iam.PolicyStatement() self.curator_firehose_iam_policy_statement.add_actions('firehose:Put*') self.curator_firehose_iam_policy_statement.add_resources(self.curator_firehose.attr_arn) self.twitter_stream_curator_lambda_function.add_to_role_policy(self.curator_firehose_iam_policy_statement) # Attaching the policy to the IAM role for KFH self.output_bucket.grant_read(self.twitter_stream_curator_lambda_function) self.twitter_stream_curator_lambda_function.add_event_source( aws_lambda_event_sources.S3EventSource( bucket=self.output_bucket, events=[ aws_s3.EventType.OBJECT_CREATED ], filters=[ aws_s3.NotificationKeyFilter( prefix="twitter-raw/" ) ] ) )
def __init__(self, scope: core.Construct, id: str, es_domain: CfnDomain, kda_role: iam.Role, source_bucket: s3.Bucket, dest_bucket: s3.Bucket, **kwargs): super().__init__(scope, id, **kwargs) stack = Stack.of(self) kda_role.add_to_policy(PolicyStatement(actions=['cloudwatch:PutMetricData'], resources=['*'])) artifacts_bucket_arn = 'arn:aws:s3:::' + _config.ARA_BUCKET.replace("s3://", "") kda_role.add_to_policy(PolicyStatement(actions=['s3:GetObject', 's3:GetObjectVersion'], resources=[artifacts_bucket_arn, artifacts_bucket_arn + '/binaries/*'])) log_group = logs.LogGroup(scope=self, id='KdaLogGroup', retention=logs.RetentionDays.ONE_WEEK, removal_policy=RemovalPolicy.DESTROY) log_stream = logs.LogStream(scope=self, id='KdaLogStream', log_group=log_group, removal_policy=RemovalPolicy.DESTROY) log_stream_arn = stack.format_arn(service='logs', resource='log-group', resource_name=log_group.log_group_name + ':log-stream:' + log_stream.log_stream_name, sep=':') # TODO: restrict kda_role.add_to_policy(PolicyStatement(actions=['logs:*'], resources=[stack.format_arn(service='logs', resource='*')])) kda_role.add_to_policy(PolicyStatement(actions=['logs:DescribeLogStreams', 'logs:DescribeLogGroups'], resources=[log_group.log_group_arn, stack.format_arn(service='logs', resource='log-group', resource_name='*')])) kda_role.add_to_policy(PolicyStatement(actions=['logs:PutLogEvents'], resources=[log_stream_arn])) kda_role.add_to_policy(PolicyStatement(actions=['es:ESHttp*'], resources=[stack.format_arn(service='es', resource='domain', resource_name=es_domain.domain_name + '/*')])) # TODO: restrict kda_role.add_to_policy(PolicyStatement(actions=['s3:*'], resources=['arn:aws:s3::::*'])) # Define delivery stream # delivery_stream_name = 'clean_delivery_stream' # # s3_configuration = { # 'bucketArn': '', # 'compressionFormat': 'Snappy', # 'dataFormatConversionConfiguration': { # 'enabled': True, # 'inputFormatConfiguration': {'deserializer': }, # 'outputFormatConfiguration': {'serializer': {'parquetSerDe': }}, # 'schemaConfiguration': {} # }, # 'prefix': 'streaming' # } # # delivery_stream = CfnDeliveryStream(scope=self, # id='Firehose Delivery Stream', # delivery_stream_name=delivery_stream_name, # delivery_stream_type='DirectPut', # extended_s3_destination_configuration=s3_configuration # ) # Define KDA application application_configuration = { 'environmentProperties': { 'propertyGroups': [ { 'propertyGroupId': 'ConsumerConfigProperties', 'propertyMap': { 'CustomerStream': scope.customer_stream.stream_name, 'AddressStream': scope.address_stream.stream_name, 'SaleStream': scope.sale_stream.stream_name, 'PromoDataPath': source_bucket.s3_url_for_object('promo'), 'ItemDataPath': source_bucket.s3_url_for_object('item'), 'aws.region': scope.region } }, { 'propertyGroupId': 'ProducerConfigProperties', 'propertyMap': { 'ElasticsearchHost': 'https://' + es_domain.attr_domain_endpoint + ':443', 'Region': scope.region, 'DenormalizedSalesS3Path': dest_bucket.s3_url_for_object() + '/', 'IndexName': 'ara-write' } } ] }, 'applicationCodeConfiguration': { 'codeContent': { 's3ContentLocation': { 'bucketArn': artifacts_bucket_arn, 'fileKey': 'binaries/stream-processing-1.1.jar' } }, 'codeContentType': 'ZIPFILE' }, 'flinkApplicationConfiguration': { 'parallelismConfiguration': { 'configurationType': 'DEFAULT' }, 'checkpointConfiguration': { 'configurationType': 'DEFAULT' }, 'monitoringConfiguration': { 'logLevel': 'DEBUG', 'metricsLevel': 'TASK', 'configurationType': 'CUSTOM' } }, 'applicationSnapshotConfiguration': { 'snapshotsEnabled': False } } self.__app = CfnApplicationV2(scope=self, id='KDA application', runtime_environment='FLINK-1_11', application_name='KDA-application', service_execution_role=kda_role.role_arn, application_configuration=application_configuration) logging = CfnApplicationCloudWatchLoggingOptionV2(scope=self, id='KDA application logging', application_name=self.__app.ref, cloud_watch_logging_option={'logStreamArn': log_stream_arn}) logging.apply_removal_policy(policy=RemovalPolicy.RETAIN, apply_to_update_replace_policy=True, default=RemovalPolicy.RETAIN) # Use a custom resource to start the application create_params = { 'ApplicationName': self.__app.ref, 'RunConfiguration': { 'ApplicationRestoreConfiguration': { 'ApplicationRestoreType': 'SKIP_RESTORE_FROM_SNAPSHOT' }, 'FlinkRunConfiguration': { 'AllowNonRestoredState': True } } } # See https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/ for service name, actions and parameters create_action = AwsSdkCall(service='KinesisAnalyticsV2', action='startApplication', parameters=create_params, physical_resource_id=PhysicalResourceId.of(self.__app.ref + '-start')) delete_action = AwsSdkCall(service='KinesisAnalyticsV2', action='stopApplication', parameters={'ApplicationName': self.__app.ref, 'Force': True}) custom_resource = AwsCustomResource(scope=self, id='KdaStartAndStop', on_create=create_action, on_delete=delete_action, policy=AwsCustomResourcePolicy.from_statements([PolicyStatement( actions=['kinesisanalytics:StartApplication', 'kinesisanalytics:StopApplication', 'kinesisanalytics:DescribeApplication', 'kinesisanalytics:UpdateApplication'], resources=[ stack.format_arn(service='kinesisanalytics', resource='application', resource_name=self.app.application_name)])])) custom_resource.node.add_dependency(self.app)
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # The code that defines your stack goes here vpc_name = self.node.try_get_context('vpc_name') vpc = aws_ec2.Vpc.from_lookup(self, 'ExistingVPC', is_default=True, vpc_name=vpc_name) s3_bucket_name = self.node.try_get_context('s3_bucket_name') s3_bucket = s3.Bucket.from_bucket_name(self, 'S3KdaFlinkCodeLocation', s3_bucket_name) s3_path_to_flink_app_code = self.node.try_get_context( 's3_path_to_flink_app_code') KDA_APP_NAME = 'KdaMskReplcation' kda_exec_role_policy_doc = aws_iam.PolicyDocument() kda_exec_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "sid": "ReadCode", "effect": aws_iam.Effect.ALLOW, "resources": [ "{}/{}".format(s3_bucket.bucket_arn, s3_path_to_flink_app_code) ], "actions": ["s3:GetObject", "s3:GetObjectVersion"] })) kda_exec_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "sid": "ListCloudwatchLogGroups", "effect": aws_iam.Effect.ALLOW, #XXX: The ARN will be formatted as follows: # arn:{partition}:{service}:{region}:{account}:{resource}{sep}{resource-name} "resources": [ self.format_arn( service="logs", resource="log-group", resource_name= "/aws/kinesis-analytics/{}:log-stream:*".format( KDA_APP_NAME), arn_format=cdk.ArnFormat.COLON_RESOURCE_NAME) ], "actions": ["logs:DescribeLogGroups"] })) kda_exec_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "sid": "ListCloudwatchLogStreams", "effect": aws_iam.Effect.ALLOW, #XXX: The ARN will be formatted as follows: # arn:{partition}:{service}:{region}:{account}:{resource}{sep}{resource-name} "resources": [ self.format_arn( service="logs", resource="log-group", resource_name= "/aws/kinesis-analytics/{}:log-stream:kinesis-analytics-log-stream" .format(KDA_APP_NAME), arn_format=cdk.ArnFormat.COLON_RESOURCE_NAME) ], "actions": ["logs:DescribeLogStreams"] })) kda_exec_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "sid": "PutCloudwatchLogs", "effect": aws_iam.Effect.ALLOW, #XXX: The ARN will be formatted as follows: # arn:{partition}:{service}:{region}:{account}:{resource}{sep}{resource-name} "resources": [ self.format_arn( service="logs", resource="log-group", resource_name= "/aws/kinesis-analytics/{}:log-stream:kinesis-analytics-log-stream" .format(KDA_APP_NAME), arn_format=cdk.ArnFormat.COLON_RESOURCE_NAME) ], "actions": ["logs:PutLogEvents"] })) kda_exec_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "sid": "ENIReadWritePermissions", "effect": aws_iam.Effect.ALLOW, "resources": ["*"], "actions": [ "ec2:CreateNetworkInterface", "ec2:CreateNetworkInterfacePermission", "ec2:DescribeNetworkInterfaces", "ec2:DeleteNetworkInterface" ] })) kda_exec_role_policy_name = "kinesis-analytics-service-{kda_app_name}-{region}".format( region=cdk.Aws.REGION, kda_app_name=KDA_APP_NAME), kda_execution_role = aws_iam.Role( self, 'KdaExecutionRole', role_name='kinesis-analytics-{kda_app_name}-{region}'.format( region=cdk.Aws.REGION, kda_app_name=KDA_APP_NAME), assumed_by=aws_iam.ServicePrincipal( 'kinesisanalytics.amazonaws.com'), path='/service-role/', inline_policies={ 'kinesis-analytics-service': kda_exec_role_policy_doc }, managed_policies=[ aws_iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonVPCReadOnlyAccess'), ]) kda_flink_code_content = aws_kda_flink.CfnApplicationV2.CodeContentProperty( s3_content_location=aws_kda_flink.CfnApplicationV2. S3ContentLocationProperty(bucket_arn=s3_bucket.bucket_arn, file_key=s3_path_to_flink_app_code)) kda_flink_code_config = aws_kda_flink.CfnApplicationV2.ApplicationCodeConfigurationProperty( code_content=kda_flink_code_content, code_content_type='ZIPFILE') kda_flink_property_groups = self.node.try_get_context( 'kda_flink_property_groups') _property_groups = [ aws_kda_flink.CfnApplicationV2.PropertyGroupProperty(**elem) for elem in kda_flink_property_groups ] kda_flink_env_props = aws_kda_flink.CfnApplicationV2.EnvironmentPropertiesProperty( property_groups=_property_groups) flink_app_config = aws_kda_flink.CfnApplicationV2.FlinkApplicationConfigurationProperty( checkpoint_configuration=aws_kda_flink.CfnApplicationV2. CheckpointConfigurationProperty( configuration_type='CUSTOM', checkpointing_enabled=True, checkpoint_interval=60000, min_pause_between_checkpoints=60000), monitoring_configuration=aws_kda_flink.CfnApplicationV2. MonitoringConfigurationProperty(configuration_type='CUSTOM', log_level='INFO', metrics_level='TASK'), parallelism_configuration=aws_kda_flink.CfnApplicationV2. ParallelismConfigurationProperty( configuration_type='CUSTOM', auto_scaling_enabled=False, parallelism=1, parallelism_per_kpu=1, )) kda_flink_app_config = aws_kda_flink.CfnApplicationV2.ApplicationConfigurationProperty( application_code_configuration=kda_flink_code_config, application_snapshot_configuration=aws_kda_flink.CfnApplicationV2. ApplicationSnapshotConfigurationProperty(snapshots_enabled=False), environment_properties=kda_flink_env_props, flink_application_configuration=flink_app_config) kda_app = aws_kda_flink.CfnApplicationV2( self, 'KdaMskReplication', runtime_environment='FLINK-1_11', service_execution_role=kda_execution_role.role_arn, application_configuration=kda_flink_app_config, application_description= 'A Kinesis Data Analytics application that reads from one Amazon MSK topic and writes to another', application_name=KDA_APP_NAME) kda_app_log_group = aws_logs.LogGroup( self, 'KdaMskReplicationLogGroup', log_group_name='/aws/kinesis-analytics/{}'.format(KDA_APP_NAME), retention=aws_logs.RetentionDays.THREE_DAYS, removal_policy=cdk.RemovalPolicy.DESTROY) kda_app_log_stream = aws_logs.LogStream( self, 'KdaMskReplicationLogStream', log_group=kda_app_log_group, log_stream_name='kinesis-analytics-log-stream', removal_policy=cdk.RemovalPolicy.DESTROY) #XXX: The ARN will be formatted as follows: # arn:{partition}:{service}:{region}:{account}:{resource}{sep}{resource-name} kda_app_log_stream_arn = self.format_arn( service="logs", resource="log-group", resource_name= "/aws/kinesis-analytics/{}:log-stream:kinesis-analytics-log-stream" .format(KDA_APP_NAME), arn_format=cdk.ArnFormat.COLON_RESOURCE_NAME) kda_app_cw_log = aws_kda_flink.CfnApplicationCloudWatchLoggingOptionV2( self, 'KdaMskReplicationCWLog', application_name=kda_app.application_name, cloud_watch_logging_option=aws_kda_flink. CfnApplicationCloudWatchLoggingOptionV2. CloudWatchLoggingOptionProperty( log_stream_arn=kda_app_log_stream_arn)) kda_app_cw_log.add_depends_on(kda_app)
def __init__(self, scope: core.Construct, id: str, vpc: ec2.IVpc, directory: mad.CfnMicrosoftAD, **kwargs) -> None: super().__init__(scope, id, **kwargs) log_group = logs.LogGroup(self, 'LogGroup', log_group_name='/homenet/vpn', removal_policy=core.RemovalPolicy.DESTROY, retention=logs.RetentionDays.ONE_MONTH) log_stream = logs.LogStream(self, 'LogStream', log_group=log_group, log_stream_name='vpn-connection-logs') security_group = ec2.SecurityGroup(self, 'SecurityGroup', vpc=vpc, allow_all_outbound=True, description='VPN clients') endpoint = ec2.CfnClientVpnEndpoint( self, 'VpnEp', vpc_id=vpc.vpc_id, vpn_port=443, self_service_portal='enabled', split_tunnel=True, client_cidr_block='10.1.8.0/22', server_certificate_arn= 'arn:aws:acm:us-east-1:581361757134:certificate/14e094b5-fd1d-4031-b0cc-4be1b77e5955', description='HomeNet vpc:endpoint', security_group_ids=[security_group.security_group_id], authentication_options=[ ec2.CfnClientVpnEndpoint.ClientAuthenticationRequestProperty( type='directory-service-authentication', active_directory=ec2.CfnClientVpnEndpoint. DirectoryServiceAuthenticationRequestProperty( directory_id=directory.ref)), ], connection_log_options=ec2.CfnClientVpnEndpoint. ConnectionLogOptionsProperty( enabled=True, cloudwatch_log_group=log_group.log_group_name, cloudwatch_log_stream=log_stream.log_stream_name)) count = 0 for net in vpc.select_subnets(subnet_group_name='Vpn-Clients').subnets: count += 1 ec2.CfnClientVpnTargetNetworkAssociation( self, 'Network-' + str(count), client_vpn_endpoint_id=endpoint.ref, subnet_id=net.subnet_id) # ec2.CfnClientVpnTargetNetworkAssociation(self,'NetworkAssociation', # client_vpn_endpoint_id=endpoint.ref, # subnet_id= 'subnet-07f0e80d0ed1c1a27') ec2.CfnClientVpnAuthorizationRule( self, 'ClientAuthorization', authorize_all_groups=True, target_network_cidr='10.0.0.0/8', client_vpn_endpoint_id=endpoint.ref, description='Allow everyone/everywhere')