def __configure_base(self) -> None: """ Setup base dependencies """ self.__quotes_stream = k.Stream(self, 'QuoteStream', encryption=k.StreamEncryption.MANAGED, retention_period=core.Duration.days(7), shard_count=1, stream_name='finsurf-incoming-quotes') self.__fundamental_stream = k.Stream( self, 'FundamentalStream', encryption=k.StreamEncryption.MANAGED, retention_period=core.Duration.days(7), shard_count=1, stream_name='finsurf-incoming-fundamentals') self.pm_compute_cluster = ecs.Cluster(self, 'Cluster', vpc=self.vpc) self.security_group = ec2.SecurityGroup( self, 'CollectorComponents', vpc=self.vpc, allow_all_outbound=True, description='Security Group for the CollectorLayer')
def __configure_ingestion(self) -> None: self.__updates_stream = k.Stream( self, 'PortfolioUpdates', encryption=k.StreamEncryption.MANAGED, retention_period=core.Duration.days(7), shard_count=1, stream_name='portfolio-updates') self.__updates_handler = PythonLambda( self, 'UpdatesHandler', build_prefix='artifacts/FinSurf-PortfolioMgmt-UpdatesHandler', handler='updates_handler.lambda_handler', subnet_group_name='PortfolioMgmt', context=self.context, securityGroups=[self.security_group]).function self.updates_handler.add_event_source(source=evt.KinesisEventSource( stream=self.updates_stream, starting_position=lambda_.StartingPosition.LATEST)) # Configure writing to neptune self.updates_handler.add_environment( key='NEPTUNE_ENDPOINT', value=self.neptune_cluster.attr_endpoint)
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) s3bucket = s3.Bucket(self, 'vika-yy') kds = data_stream.Stream(self, 'data_stream', shard_count=1) delivery_stream_role = iam.Role( self, 'kdfdelivery_stream_role_role', assumed_by=iam.ServicePrincipal('firehose.amazonaws.com')) delivery_stream_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonKinesisFullAccess')) delivery_stream_role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[s3bucket.bucket_arn], actions=["s3:*"])) #s3bucket = s3.Bucket(self, 'vika-yy',bucket_name='yellowtaxicdk-input') s3_dest_config = delivery_stream.CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty( bucket_arn=s3bucket.bucket_arn, buffering_hints=delivery_stream.CfnDeliveryStream. BufferingHintsProperty(interval_in_seconds=60, size_in_m_bs=128), role_arn=delivery_stream_role.role_arn, compression_format='UNCOMPRESSED', s3_backup_mode='Disabled') stream_source_config = delivery_stream.CfnDeliveryStream.KinesisStreamSourceConfigurationProperty( kinesis_stream_arn=kds.stream_arn, role_arn=delivery_stream_role.role_arn) kfirehose = delivery_stream.CfnDeliveryStream( self, 'kfirehose', delivery_stream_name='deliverystream', delivery_stream_type='KinesisStreamAsSource', extended_s3_destination_configuration=s3_dest_config, kinesis_stream_source_configuration=stream_source_config) glue_role = iam.Role( self, 'glue_role', assumed_by=iam.ServicePrincipal('glue.amazonaws.com')) glue_role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[s3bucket.bucket_arn], actions=["s3:*"])) glue_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSGlueServiceRole')) bucket_name = s3bucket.bucket_name glue_crawler = glue.CfnCrawler( self, 'glue_crawler', database_name='yellow-taxis', role=glue_role.role_arn, #targets={"s3Targets": [{"path": f'{BUCKET}/input/'}]} targets={"s3Targets": [{ "path": f'{bucket_name}/input/' }]})
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) my_table = _dynamodb.Table(self, id='dynamoTable', table_name='testcdktable', partition_key=_dynamodb.Attribute( name='lastname', type=_dynamodb.AttributeType.STRING)) my_stream = _kinesis.Stream(self, id='kinesistream', stream_name='cdkkinesisstream') my_bucket = _s3.Bucket(self, id='s3bucket', bucket_name='rajcdkbucket') my_lambda = _lambda.Function(self, id='lambdafunction', runtime=_lambda.Runtime.PYTHON_3_7, handler='hello.handler', code=_lambda.Code.asset('lambdacode')) my_api = _apigateway.LambdaRestApi(self, id='lambdaapi', rest_api_name='cdkapi', handler=my_lambda)
def __init__(self, scope: core.Construct, id: str, shard_count: int, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here): ############################################## ####### CDK STABILITY EXPERIMENTAL ####### ############################################## self.kinesis_data_pipe = _kinesis.Stream( self, "dataPipe", # retention_period_hours=24, # retention_period=core.Duration.minutes(300), shard_count=1, stream_name="data_pipe") self.data_pipe_ssm_param = _ssm.StringParameter( self, "dataPipeParameter", description="Kinesis Stream Name", parameter_name= f"/{global_args.REPO_NAME}/streams/data_pipe/stream_name", string_value=f"{self.kinesis_data_pipe.stream_name}") output_0 = core.CfnOutput( self, "AutomationFrom", value=f"{global_args.SOURCE_INFO}", description= "To know more about this automation stack, check out our github page." )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) kms_policy = iam.PolicyDocument() kms_policy.add_statements( iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=["kms:*"], resources=['*'], principals=[iam.AccountPrincipal(account_id=self.account), iam.ServicePrincipal(service="lambda.amazonaws.com")] ) ) self._kinesis_key = kms.Key(self, "volumeKey", enable_key_rotation=True, policy=kms_policy, removal_policy=core.RemovalPolicy.RETAIN ) self._kinesis_stream = kinesis.Stream(self, id, encryption_key=self.kinesis_key, retention_period=core.Duration.hours(24), shard_count=1, stream_name="PaymentStream" )
def create_kinesis_stream(self, stream_name): kinesis_stream = kinesis.Stream( self, "stock_stream", stream_name=stream_name ) return kinesis_stream
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) stream = kds.Stream(self, "InputStream", shard_count=1, retention_period=Duration.hours(24)) self._stream = stream
def __init__(self, scope: core.Construct, _id: str, **kwargs) -> None: new_kwargs = {'env': kwargs['env']} super().__init__(scope, _id, **new_kwargs) # Create staging Kinesis Data Stream, set to 1 shard in this sample code kinesis_stream = kinesis.Stream(self, STREAM_NAME, stream_name=STREAM_NAME, shard_count=1) # Create replicator lambda function that consumes the Kinesis stream and writes to target DDB table target_table_name = kwargs['target_table_name'] dlq_sqs = sqs.Queue(self, 'replicator_failure_Q') replicator_lambda = lambda_.Function( self, 'replicator_kinesis', code=lambda_.Code.asset("../lambda_replicator"), runtime=lambda_.Runtime.PYTHON_3_7, handler='replicator_kinesis.lambda_handler', timeout=core.Duration.seconds(60), environment={'TARGET_TABLE': target_table_name}) kinesis_stream.grant_read(replicator_lambda) replicator_lambda.add_event_source( KinesisEventSource( stream=kinesis_stream, starting_position=lambda_.StartingPosition.LATEST, batch_size=500, retry_attempts=100, parallelization_factor=10, on_failure=SqsDlq(dlq_sqs))) target_table = ddb.Table.from_table_name(self, target_table_name, target_table_name) target_table.grant_read_write_data(replicator_lambda) # The replicator lambda will put metrics to Cloudwatch put_metrics_policy = PolicyStatement( actions=['cloudwatch:PutMetricData'], effect=Effect.ALLOW, resources=['*']) replicator_lambda.add_to_role_policy(put_metrics_policy) # Create replicator-stats table for statistics of replicator replicator_stats_table = ddb.Table( self, REPLICATOR_STATS_TABLE_NAME, table_name=REPLICATOR_STATS_TABLE_NAME, partition_key=ddb.Attribute(name="PK", type=ddb.AttributeType.STRING), billing_mode=ddb.BillingMode.PAY_PER_REQUEST, removal_policy=RemovalPolicy.DESTROY) replicator_stats_table.grant_read_write_data(replicator_lambda) core.CfnOutput(self, "replicator_stats_table", value=replicator_stats_table.table_name)
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # TO-DO: Resolve how to surface this through a property rather than directly. self.kinesis_stream = kinesis.Stream( self, "kinesisStream", retention_period=core.Duration.days(3), stream_name=core.PhysicalName.GENERATE_IF_NEEDED, ) core.CfnOutput( self, "kinesisDataStreamArn", value=self.kinesis_stream.stream_arn, ) bucket = s3.Bucket.from_bucket_name(self, "s3Bucket", bucket_name=OUTPUT_BUCKET) kf_role = iam.Role( self, "kinesisFirehoseRole", assumed_by=iam.ServicePrincipal("firehose.amazonaws.com"), ) kf_props = kf.CfnDeliveryStreamProps( extended_s3_destination_configuration=kf.CfnDeliveryStream. ExtendedS3DestinationConfigurationProperty( role_arn=kf_role.role_arn, bucket_arn=bucket.bucket_arn, prefix="twitter_data_", error_output_prefix="failures", )) data_pipeline = ( kinesis_data_pipeline.KinesisStreamsToKinesisFirehoseToS3( self, "dataPipeline", existing_stream_obj=self.kinesis_stream, existing_bucket_obj=bucket, kinesis_firehose_props=kf_props, )) self.kinesis_firehose = data_pipeline.kinesis_firehose bucket.grant_write(kf_role)
def __init__(self, scope: core.Construct, id: str, ctx: object, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Kinesis Data Stream for telemetry buffer self.kinesis_stream = ks.Stream( scope=self, id="logstash_queue", shard_count=ctx.queue.kinesis_shard_count) self.state_table = ddb.Table( scope=self, id="logstash_state", partition_key=ddb.Attribute(name="leaseKey", type=ddb.AttributeType.STRING), billing_mode=ddb.BillingMode.PAY_PER_REQUEST, removal_policy=core.RemovalPolicy.DESTROY)
def __init__(self, app: core.App, id: str) -> None: super().__init__(app, id) with open("lambda-handler.py", encoding="utf8") as fp: handler_code = fp.read() Kstream = kinesis_.Stream(self, "KinesisSagemakerInference", encryption=None, encryption_key=None, retention_period_hours=24, shard_count=1) lambdaFn = lambda_.Function( self, "KinesisSMLambda", code=lambda_.InlineCode(handler_code), handler="index.lambda_handler", timeout=core.Duration.seconds(300), runtime=lambda_.Runtime.PYTHON_3_7, environment={ "endpoint_name": endpoint_name, # CHANGE TO YOUR ENDPOINT NAME!! "content_type": "text/csv", "input_data": input_data, "bucket": bucket, "key": key }) lambdaFn.add_to_role_policy( aws_iam.PolicyStatement( actions=[ 'sagemaker:InvokeEndpoint', ], resources=[ 'arn:aws:sagemaker:{}:{}:endpoint/{}'.format( my_region, my_acc_id, endpoint_name), ])) # Add the Kinesis stream as Lambda source lambdaFn.add_event_source( aws_lambda_event_sources.KinesisEventSource( Kstream, starting_position=lambda_.StartingPosition.LATEST)) # Add stream read permissions Kstream.grant_read(lambdaFn.role)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ZachStreamName = "ZachWang" ksStream = ks.Stream( self, id=ZachStreamName, stream_name=ZachStreamName, encryption=ks.StreamEncryption.KMS, encryption_key=self.GenerateKmsKey(ZachStreamName), shard_count=2, retention_period_hours=168) core.CfnOutput(self, ZachStreamName + "Stream-ARN", value=ksStream.stream_arn) core.CfnOutput(self, ZachStreamName + "Stream-Name", value=ksStream.stream_name)
def base_kinesis_stream(construct, **kwargs): """ Function that generates a Kinesis Data Stream. :param construct: Custom construct that will use this function. From the external construct is usually 'self'. :param kwargs: Consist of required 'stream_name'. :return: Kinesis Stream Construct. """ stream_name = construct.prefix + "_" + kwargs[ "stream_name"] + "_" + "stream" + "_" + construct.environment_ stream_retention_period = Duration.hours( kwargs["retention_period"]) if kwargs.get( "retention_period") is not None else None kinesis_stream = stream.Stream( construct, id=stream_name, stream_name=stream_name, shard_count=kwargs["shard_count"], retention_period=stream_retention_period, ) return kinesis_stream
def __init__(self, scope, id, **kwargs): super().__init__(scope, id, **kwargs) self.prefix = MY_PREFIX # Destination Bucket on S3 self.s3_bucket = _s3.Bucket( self, "storage", bucket_name=f"{self.prefix}-data-store", ) # Event Stream self.stream = _kns.Stream(self, "stream", stream_name=f"{self.prefix}-stream", shard_count=1) # API Endpoint self.create_api_endpoint() # Event Processing self.create_stream_processor()
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) my_table = _dynamodb.Table(self, id='dynamoTable', table_name='testcdktable', partition_key=_dynamodb.Attribute( name='lastname', type=_dynamodb.AttributeType.STRING)) my_stream = _kinesis.Stream(self, id='kinesistream', stream_name='cdkkinesisstream') my_bucket = _s3.Bucket(self, id='s3bucket', bucket_name='rajcdkbucket') my_lambda = _lambda.Function(self, id='lambdafunction', runtime=_lambda.Runtime.PYTHON_3_7, handler='hello.handler', code=_lambda.Code.asset('lambdacode')) my_api = _apigateway.LambdaRestApi(self, id='lambdaapi', rest_api_name='cdkapi', handler=my_lambda) api_with_method = _apigateway.RestApi(self, id='restapi', rest_api_name='cdkrestapi_music') #music = api_with_method.root.addResource('music') #music.addMethod('GET') music = api_with_method.root.add_resource('music') music.add_method('GET') music.add_method("DELETE", _apigateway.HttpIntegration("http://aws.amazon.com"))
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) #kinesis data stream kinesis_stream = aws_kinesis.Stream( self, "kinesisStream", retention_period=core.Duration.hours(24), shard_count=1, stream_name="kinesis_test_data_pipe") #s3 for store stream data events kinesis_s3_bucket = aws_s3.Bucket( self, "kinesisS3Bucket", removal_policy=core.RemovalPolicy.DESTROY) #Lambda functions #import function codes - data consume try: with open("deployments/functions/stream_data_get.py", mode="r") as file: function_body_get = file.read() except OSError: print('File can not read') #consume function stream_get_function = aws_lambda.Function( self, "consumeFunction", function_name="StreamConsumeFunction", description="Process Data Streams and store to s3", runtime=aws_lambda.Runtime.PYTHON_3_6, handler="index.lambda_handler", code=aws_lambda.InlineCode(function_body_get), timeout=core.Duration.seconds(5), reserved_concurrent_executions=1, environment={ "LOG_LEVEL": "INFO", "BUCKET_NAME": f"{kinesis_s3_bucket.bucket_name}" }) #permision to use stream by lambda kinesis_stream.grant_read(stream_get_function) #s3 permision lambdas3Permision = aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, resources=[f"{kinesis_s3_bucket.bucket_arn}/*"], actions=["s3:PutObject"]) lambdas3Permision.sid = "S3writePermisionToLambda" stream_get_function.add_to_role_policy(lambdas3Permision) #logs stream_concume_logs = aws_logs.LogGroup( self, "StreamConcumeLogs", log_group_name=f"/aws/lambda/{stream_get_function.function_name}", removal_policy=core.RemovalPolicy.DESTROY, retention=aws_logs.RetentionDays.ONE_DAY) #kinesis event source kinesis_event_sources = aws_lambda_es.KinesisEventSource( stream=kinesis_stream, starting_position=aws_lambda.StartingPosition.LATEST, batch_size=1) #attached kinesis to lambda stream_get_function.add_event_source(kinesis_event_sources) #generate stream lambda function##### #import function codes - data consume try: with open("deployments/functions/stream_data_gen.py", mode="r") as file: function_body_gen = file.read() except OSError: print('File can not read') #stream generate function stream_gen_function = aws_lambda.Function( self, "GenarateFunction", function_name="StreamGenerateFunction", description="Generate Data Streams", runtime=aws_lambda.Runtime.PYTHON_3_6, handler="index.lambda_handler", code=aws_lambda.InlineCode(function_body_gen), timeout=core.Duration.seconds(60), reserved_concurrent_executions=1, environment={ "LOG_LEVEL": "INFO", "STREAM_NAME": f"{kinesis_stream.stream_name}" }) #permision to lambda to write into kinesis kinesis_stream.grant_read_write(stream_gen_function) #logs stream_generate_logs = aws_logs.LogGroup( self, "StreamGenerateLogs", log_group_name=f"/aws/lambda/{stream_gen_function.function_name}", removal_policy=core.RemovalPolicy.DESTROY, retention=aws_logs.RetentionDays.ONE_DAY)
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) kda_src_bucket_name = core.CfnParameter( self, "kda_src_bucket_name", type="String", description= "The name of the Amazon S3 bucket where uploaded files will be stored." ) kda_output_bucket_name = core.CfnParameter( self, "kda_output_bucket_name", type="String", description= "The name of the Amazon S3 bucket KDA output via Firehose will be stored." ) sourceStreamName = core.CfnParameter( self, "sourceStreamName", type="String", description="The name of the Kinesis Data Stream.", default="BikeRideGenerator") deliveryStreamName = core.CfnParameter( self, "deliveryStreamName", type="String", description="The name of the Kinesis Firehose output stream.", default="BikeAnalyticsOutput") # Create S3 buckets kda_src_bucket = s3.Bucket( self, "kda_src_bucket", bucket_name=kda_src_bucket_name.value_as_string, versioned=False, removal_policy=core.RemovalPolicy.DESTROY) kda_output_bucket = s3.Bucket( self, "kda_output_bucket", bucket_name=kda_output_bucket_name.value_as_string, versioned=False, removal_policy=core.RemovalPolicy.DESTROY) # create Kinesis Source Stream sourceStream = kds.Stream(self, "sourceStream", stream_name=sourceStreamName.value_as_string, shard_count=10) # Firehose Role aws_cdk.aws_iam.CfnRole fhIAMRole = iam.Role( self, "fhIAMRole", assumed_by=iam.ServicePrincipal('firehose.amazonaws.com'), role_name="BikeRideFirehoseDeliveryRole", description="FireHose Delivery S3 Role") fhIAMRole.add_to_policy( iam.PolicyStatement(resources=[kda_output_bucket.bucket_arn], actions=['s3:*'])) # create Firehose delivery stream fhS3Delivery = fh.CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty( bucket_arn=kda_output_bucket.bucket_arn, role_arn=fhIAMRole.role_arn) deliveryStream = fh.CfnDeliveryStream( self, "deliveryStream", delivery_stream_name=deliveryStreamName.value_as_string, extended_s3_destination_configuration=fhS3Delivery) # ec2 instance # VPC vpc = ec2.Vpc(self, "KDA-VPC", nat_gateways=0, subnet_configuration=[ ec2.SubnetConfiguration( name="public", subnet_type=ec2.SubnetType.PUBLIC) ]) # AMI amzn_linux = ec2.MachineImage.latest_amazon_linux( generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2, edition=ec2.AmazonLinuxEdition.STANDARD, virtualization=ec2.AmazonLinuxVirt.HVM, storage=ec2.AmazonLinuxStorage.GENERAL_PURPOSE) # Instance Role and SSM Managed Policy ec2role = iam.Role( self, "InstanceSSM", assumed_by=iam.ServicePrincipal("ec2.amazonaws.com")) ec2role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AmazonEC2RoleforSSM")) ec2role.add_to_policy( iam.PolicyStatement(resources=[sourceStream.stream_arn], actions=['kinesis:*'])) user_data = "#!/bin/bash\n" user_data += "echo export KINESIS_STREAM=" + sourceStreamName.value_as_string + " | sudo tee -a /etc/profile\n" user_data += "source /etc/profile\n" user_data += user_data_file # Instance instance = ec2.Instance(self, "Instance", instance_type=ec2.InstanceType("t3.small"), machine_image=amzn_linux, vpc=vpc, role=ec2role, user_data=ec2.UserData.custom(user_data))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here self._wellness_kns_stream = kinesis.Stream( self, 'WellnessKnsStream', retention_period=core.Duration.hours(24), shard_count=1, stream_name='event.member.appointment.devInfo')
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) self._table = ddb.Table(self, 'AirTicketOrder', partition_key={ 'name': 'customer_id', 'type': ddb.AttributeType.STRING }, stream=ddb.StreamViewType.NEW_AND_OLD_IMAGES, removal_policy=core.RemovalPolicy.DESTROY) self.lambda_cmd = _lambda.Function( self, 'CommandDDBSaver', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset("./lambda/cmd/"), handler="cmd.lambda_handler", environment={ "ORDER_TABLE_NAME": self._table.table_name, }) self._table.grant_read_write_data(self.lambda_cmd) # Allow Command lambda to invoke other lambda self.lambda_cmd.add_to_role_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=["*"], actions=["lambda:InvokeFunction"])) api = apigw.LambdaRestApi( self, "CommandEndPoint", handler=self.lambda_cmd, ) # TODO: 因为2个AZ,可以只生命一个公网和一个私网,这样X2 AZ就会2 pub + 2 pri # Lambda access RDS Aurora MySQL requires VPC for security and perf vpc = ec2.Vpc( self, 'air-ticket', cidr="10.125.0.0/16", max_azs=2, nat_gateways=1, subnet_configuration=[ ec2.SubnetConfiguration(name="public1", cidr_mask=24, subnet_type=ec2.SubnetType.PUBLIC), ec2.SubnetConfiguration(name="public2", cidr_mask=24, subnet_type=ec2.SubnetType.PUBLIC), ec2.SubnetConfiguration(name="private1", cidr_mask=24, subnet_type=ec2.SubnetType.PRIVATE), ec2.SubnetConfiguration(name="private2", cidr_mask=24, subnet_type=ec2.SubnetType.PRIVATE) ]) query_lambda_sg = ec2.SecurityGroup( self, 'Query-Lambda-SG', vpc=vpc, description="Allows DB connections from Query Lambda SG", ) sink_lambda_sg = ec2.SecurityGroup( self, 'RDS-Sink-Lambda-SG', vpc=vpc, description="Allows DB connections from Sink Lambda SG", ) db_name = "Demo" db_user_name = 'admin' db_user_passowrd = 'password' parameter_group = rds.ParameterGroup(self, "ParameterGroup", family="mysql5.7", parameters={}) aurora_db = rds.DatabaseInstance( self, "air-ticket-db", master_user_password=core.SecretValue.ssm_secure( 'AirTicket.AdminPass', version='1'), master_username=db_user_name, engine=rds.DatabaseInstanceEngine.MYSQL, engine_version="5.7", parameter_group=parameter_group, vpc=vpc, # Disable deletion protection for auto deletion deletion_protection=False, instance_class=ec2.InstanceType.of(ec2.InstanceClass.MEMORY5, ec2.InstanceSize.XLARGE), removal_policy=core.RemovalPolicy.DESTROY) self._query_handler = _lambda.Function( self, 'QueryHandler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset("./lambda/query/"), handler="query.lambda_handler", timeout=core.Duration.seconds(60), vpc=vpc, security_group=query_lambda_sg, environment={ "AuroraEndpoint": aurora_db.db_instance_endpoint_address, "dbName": db_name, "dbPassword": db_user_passowrd, "dbUser": db_user_name }) query_api = apigw.LambdaRestApi( self, "Query", handler=self._query_handler, ) # Init DB Lambda self.lambda_init = _lambda.Function( self, 'InitDBHandler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset("./lambda/initdb/"), handler="init.lambda_handler", timeout=core.Duration.seconds(60), vpc=vpc, security_group=query_lambda_sg, environment={ "AuroraEndpoint": aurora_db.db_instance_endpoint_address, "dbName": db_name, "dbPassword": db_user_passowrd, "dbUser": db_user_name }) self.lambda_cmd.add_environment('INITDB_LAMBDA_NAME', self.lambda_init.function_name) # Create stream for fan-out stream_name = 'kinesis-stream-for-fanout' # Sync DDB stream delta to RDS Lambda self.lambda_sync = _lambda.Function( self, 'SyncHandler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset("./lambda/sync/"), handler="sync.lambda_handler", timeout=core.Duration.seconds(60), vpc=vpc, security_group=query_lambda_sg, environment={"streamName": stream_name}) # Add DDB stream trigger to sync lambda self.lambda_sync.add_event_source( event_sources.DynamoEventSource( self._table, starting_position=_lambda.StartingPosition.TRIM_HORIZON)) self._table.grant_stream_read(self.lambda_sync) # Allow init/sync lambda access MySQL aurora_db.connections.allow_from( query_lambda_sg, ec2.Port.tcp(3306), "Allow MySQL access from Query Lambda (because Aurora actually exposes PostgreSQL/MySQL on port 3306)", ) aurora_db.connections.allow_from( sink_lambda_sg, ec2.Port.tcp(3306), "Allow MySQL access from Sink Lambda (because Aurora actually exposes PostgreSQL/MySQL on port 3306)", ) strm = kinesis.Stream(self, 'kinesis-stream-for-fanout', stream_name=stream_name) # Create RDS Sink Lambda self.lambda_rds_sink = _lambda.Function( self, 'RDS_SINK_1', handler='rds_sinker.lambda_handler', code=_lambda.Code.asset("./lambda/sink/"), runtime=_lambda.Runtime.PYTHON_3_7, timeout=core.Duration.seconds(300), vpc=vpc, security_group=sink_lambda_sg, environment={ "AuroraEndpoint": aurora_db.db_instance_endpoint_address, "dbName": db_name, "dbPassword": db_user_passowrd, "dbUser": db_user_name }) # Update Lambda Permissions To Use Stream strm.grant_read_write(self.lambda_sync) strm.grant_read(self.lambda_rds_sink) stream_consumer = kinesis.CfnStreamConsumer( self, 'lambda-efo-consumer-id', consumer_name='lambda-efo-consumer', stream_arn=strm.stream_arn) e_s_mappnig = _lambda.EventSourceMapping( self, 'lambda-efo-consumer-event-source-mapping', target=self.lambda_rds_sink, event_source_arn=stream_consumer.stream_arn, batch_size=1, starting_position=_lambda.StartingPosition.TRIM_HORIZON, ) # self.lambda_rds_sink.add_event_source_mapping(e_s_mappnig) # CDK below create lambda as a standand Kinesis consumer instead of EFO # # # Create New Kinesis Event Source # kinesis_stream_event_source = event_sources.KinesisEventSource( # stream=strm, # starting_position=_lambda.StartingPosition.TRIM_HORIZON, # batch_size=1 # ) # # Attach New Event Source To Lambda # self.lambda_rds_sink.add_event_source(kinesis_stream_event_source) # Create dead letter queue and grant send permission to sync/sink lambda self._queue = sqs.Queue( self, "DeadLetterQueue", #Amazon SQS sets a visibility timeout, a period of time during which Amazon # SQS prevents other consumers from receiving and processing the message. # The default visibility timeout for a message is 30 seconds. # The minimum is 0 seconds. The maximum is 12 hours. visibility_timeout=core.Duration.seconds(300), ) self._queue.grant_send_messages(self.lambda_sync) self._queue.grant_send_messages(self.lambda_rds_sink) self.lambda_sync.add_environment("DLQ_NAME", self._queue.queue_name)
def __init__(self, scope: cdk.Construct, construct_id: str, stack_log_level: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # Add your stack resources below): # Create Kinesis Data Stream self.data_pipe_stream = _kinesis.Stream( self, "dataPipeStream", retention_period=cdk.Duration.hours(24), shard_count=1, stream_name=f"data_pipe_{construct_id}", ) ######################################## ####### ####### ####### Stream Data Producer ####### ####### ####### ######################################## # Read Lambda Code try: with open( "stacks/back_end/serverless_kinesis_producer_stack/lambda_src/stream_data_producer.py", encoding="utf-8", mode="r", ) as f: data_producer_fn_code = f.read() except OSError: print("Unable to read Lambda Function Code") raise data_producer_fn = _lambda.Function( self, "streamDataProducerFn", function_name=f"data_producer_{construct_id}", description= "Produce streaming data events and push to Kinesis stream", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(data_producer_fn_code), handler="index.lambda_handler", timeout=cdk.Duration.seconds(2), reserved_concurrent_executions=1, environment={ "LOG_LEVEL": "INFO", "APP_ENV": "Production", "MAX_MSGS_TO_PRODUCE": "5", "STREAM_NAME": f"{self.data_pipe_stream.stream_name}", "STREAM_AWS_REGION": f"{cdk.Aws.REGION}", }, ) # Grant our Lambda Producer privileges to write to Kinesis Data Stream self.data_pipe_stream.grant_read_write(data_producer_fn) data_producer_fn_version = data_producer_fn.latest_version data_producer_fn_version_alias = _lambda.Alias( self, "streamDataProducerFnAlias", alias_name="MystiqueAutomation", version=data_producer_fn_version, ) # Create Custom Loggroup for Producer data_producer_lg = _logs.LogGroup( self, "streamDataProducerFnLogGroup", log_group_name=f"/aws/lambda/{data_producer_fn.function_name}", removal_policy=cdk.RemovalPolicy.DESTROY, retention=_logs.RetentionDays.ONE_DAY, ) # Restrict Produce Lambda to be invoked only from the stack owner account data_producer_fn.add_permission( "restrictLambdaInvocationToOwnAccount", principal=_iam.AccountRootPrincipal(), action="lambda:InvokeFunction", source_account=cdk.Aws.ACCOUNT_ID, ) ########################################### ################# OUTPUTS ################# ########################################### output_0 = cdk.CfnOutput( self, "AutomationFrom", value=f"{GlobalArgs.SOURCE_INFO}", description= "To know more about this automation stack, check out our github page.", ) output_1 = cdk.CfnOutput( self, "StoreOrdersEventsProducer", value= f"https://console.aws.amazon.com/lambda/home?region={cdk.Aws.REGION}#/functions/{data_producer_fn.function_name}", description= "Produce streaming data events and push to Kinesis stream.", )
def __init__(self, scope: core.Construct, id: str, vpc: ec2.IVpc, runnerrole: iam.IRole, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here clusterAdmin = iam.Role(self, "AdminRole", assumed_by=iam.AccountRootPrincipal()) cluster = eks.Cluster(self, 'ekscdkdemo', vpc=vpc, default_capacity=0) asg_worker_nodes = cluster.add_capacity( 'eksspot-cdkdemo', spot_price="0.0544", instance_type=ec2.InstanceType('t3.medium'), desired_capacity=2, bootstrap_options=eks.BootstrapOptions( docker_config_json=read_docker_daemon_resource( 'eksbaseresource/docker-daemon.json'))) alb_rbac = eks.KubernetesResource( self, 'alb-rbac', cluster=cluster, manifest=read_k8s_resource('eksbaseresource/alb-rbac.yml')) asg_worker_nodes.add_to_role_policy(iampolicy) cluster.aws_auth.add_masters_role(clusterAdmin) cluster.aws_auth.add_masters_role(runnerrole) service_account = cluster.add_service_account("external-dns-sa", name='external-dns-sa') wellnessuser_irsa = cluster.add_service_account("wellnessuser", name='wellnessuser') service_account.add_to_principal_policy(dnspolicy) deployment = { "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "labels": { "app.kubernetes.io/name": "alb-ingress-controller" }, "name": "alb-ingress-controller", "namespace": "kube-system" }, "spec": { "selector": { "matchLabels": { "app.kubernetes.io/name": "alb-ingress-controller" } }, "template": { "metadata": { "labels": { "app.kubernetes.io/name": "alb-ingress-controller" } }, "spec": { "containers": [{ "name": "alb-ingress-controller", "args": [ "--ingress-class=alb", "--cluster-name=" + cluster.cluster_name ], "image": "docker.io/amazon/aws-alb-ingress-controller:v1.1.8" }], "serviceAccountName": "alb-ingress-controller" } } } } alb_service = cluster.add_resource('alb-ingress-controller', deployment) external_dns = eks.KubernetesResource( self, 'external-dns', cluster=cluster, manifest=read_k8s_resource('eksbaseresource/external-dns.yml')) alb_service.node.add_dependency(alb_rbac) external_dns.node.add_dependency(service_account) core.CfnOutput(self, 'ClusterAdmin_Role_ARN', value=clusterAdmin.role_arn) core.CfnOutput( self, 'Getupdateeks', value="aws eks update-kubeconfig --name " + cluster.cluster_name + " --region ap-northeast-1 --role-arn " + clusterAdmin.role_arn) wellness_kns_stream = kinesis.Stream( self, 'WellnessKnsStream', retention_period=core.Duration.hours(24), shard_count=1, stream_name='event.member.appointment.devInfo') wellness_kns_stream.grant_read_write(wellnessuser_irsa) core.CfnOutput(self, 'kinesis_stream_arn', value=wellness_kns_stream.stream_arn) core.CfnOutput(self, 'kinesis_stream_name', value=wellness_kns_stream.stream_name)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) output_bucket = s3.Bucket(self, 'outputBucket', removal_policy=core.RemovalPolicy.DESTROY) input_data_stream = kinesis.Stream( self, 'inputDataStream', stream_name='anomaly-detection-data-streams-input-data-stream') delivery_stream_role = iam.Role( self, 'deliveryStreamRole', assumed_by=iam.ServicePrincipal('firehose.amazonaws.com'), inline_policies=[ iam.PolicyDocument(statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['kinesis:DescribeStream'], resources=[input_data_stream.stream_arn]) ]) ]) input_data_stream.grant_read(delivery_stream_role) output_bucket.grant_write(delivery_stream_role) anomaly_topic = sns.Topic(self, 'anomalyDetectionTopic') data_processing_function = _lambda.Function( self, "dataProcessingFunction", runtime=_lambda.Runtime.PYTHON_3_7, handler="lambda-handler.main", code=_lambda.Code.asset("./dataProcessingFunction"), environment={'TOPIC_ARN': anomaly_topic.topic_arn}) delivery_stream = firehose.CfnDeliveryStream( self, 'deliveryStream', s3_destination_configuration=firehose.CfnDeliveryStream. S3DestinationConfigurationProperty( bucket_arn=output_bucket.bucket_arn, role_arn=delivery_stream_role.role_arn), kinesis_stream_source_configuration=firehose.CfnDeliveryStream. KinesisStreamSourceConfigurationProperty( kinesis_stream_arn=input_data_stream.stream_arn, role_arn=delivery_stream_role.role_arn), delivery_stream_type='KinesisStreamAsSource') anomaly_topic.grant_publish(data_processing_function) anomalie_detection_app_role = iam.Role( self, 'anomalieDetectionAppRole', assumed_by=iam.ServicePrincipal('kinesisanalytics.amazonaws.com'), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonKinesisReadOnlyAccess') ], inline_policies=[ iam.PolicyDocument(statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ data_processing_function.function_arn, ], actions=[ "lambda:GetFunctionConfiguration", "lambda:InvokeFunction", ]) ]) ]) # Load Application Code with open('anomalie_detection_sql/application.sql', 'r') as file: app_code = file.read() anomalie_detection_app = kinesis_analytics.CfnApplication( self, 'anomalieDetectionApp', inputs=[ kinesis_analytics.CfnApplication.InputProperty( name_prefix='SOURCE_SQL_STREAM', kinesis_streams_input=kinesis_analytics.CfnApplication. KinesisStreamsInputProperty( resource_arn=input_data_stream.stream_arn, role_arn=anomalie_detection_app_role.role_arn), input_schema=kinesis_analytics.CfnApplication. InputSchemaProperty( record_columns=[ kinesis_analytics.CfnApplication. RecordColumnProperty(name="sensor_id", sql_type="CHAR(30)", mapping="$.sensor_id"), kinesis_analytics.CfnApplication. RecordColumnProperty(name="temperature", sql_type="DOUBLE", mapping="$.temperature"), kinesis_analytics.CfnApplication. RecordColumnProperty(name="rpm", sql_type="DOUBLE", mapping="$.rpm"), kinesis_analytics.CfnApplication. RecordColumnProperty(name="in_service", sql_type="BOOLEAN", mapping="$.in_service"), ], record_format=kinesis_analytics.CfnApplication. RecordFormatProperty( record_format_type='JSON', mapping_parameters=kinesis_analytics. CfnApplication.MappingParametersProperty( json_mapping_parameters=kinesis_analytics. CfnApplication.JSONMappingParametersProperty( record_row_path='$'))))) ], application_code=app_code) anomalie_detection_app_output_lambda = kinesis_analytics.CfnApplicationOutput( self, 'anomalieDetectionAppOutputLambda', application_name=core.Fn.ref(anomalie_detection_app.logical_id), output=kinesis_analytics.CfnApplicationOutput.OutputProperty( lambda_output=kinesis_analytics.CfnApplicationOutput. LambdaOutputProperty( resource_arn=data_processing_function.function_arn, role_arn=anomalie_detection_app_role.role_arn), destination_schema=kinesis_analytics.CfnApplicationOutput. DestinationSchemaProperty(record_format_type='JSON'), name='PROCESS_STREAM')) vpc = ec2.Vpc( self, "VPC", nat_gateways=1, subnet_configuration=[ ec2.SubnetConfiguration(name="private", subnet_type=ec2.SubnetType.PRIVATE), ec2.SubnetConfiguration(name="public", subnet_type=ec2.SubnetType.PUBLIC), ]) # AMI amzn_linux = ec2.MachineImage.latest_amazon_linux( generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2, edition=ec2.AmazonLinuxEdition.STANDARD, virtualization=ec2.AmazonLinuxVirt.HVM, storage=ec2.AmazonLinuxStorage.GENERAL_PURPOSE) producer_role = iam.Role( self, "producerInstanceRole", assumed_by=iam.ServicePrincipal("ec2.amazonaws.com")) producer_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonSSMManagedInstanceCore")) input_data_stream.grant_write(producer_role) user_data = '#!/bin/bash\n yum update -y\n yum install python3 -y\n sudo yum install -y jq\n pip3 install boto3 --user\n pip3 install numpy --user\n curl https://bigdatainsider-anomalydetection-article-fra.s3.eu-central-1.amazonaws.com/producer/producer.py -o /tmp/producer.py' producer_instance = ec2.Instance( self, 'producerInstance', instance_type=ec2.InstanceType('t2.micro'), machine_image=amzn_linux, vpc=vpc, role=producer_role, user_data=ec2.UserData.custom(user_data))
def __init__(self, scope: core.Construct, construct_id: str, stack_log_level: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # Add your stack resources below): # Create Kinesis Data Stream self.data_pipe_stream = _kinesis.Stream( self, "dataPipeStream", retention_period=core.Duration.hours(24), shard_count=1, stream_name="data_pipe") ######################################## ####### ####### ####### Stream Data Producer ####### ####### ####### ######################################## # Read Lambda Code try: with open( "tumbling_window_stream_analytics/stacks/back_end/lambda_src/stream_data_producer.py", encoding="utf-8", mode="r") as f: data_producer_fn_code = f.read() except OSError: print("Unable to read Lambda Function Code") raise data_producer_fn = _lambda.Function( self, "streamDataProducerFn", function_name=f"data_producer_fn", description= "Produce streaming data events and push to Kinesis stream", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(data_producer_fn_code), handler="index.lambda_handler", timeout=core.Duration.seconds(60), reserved_concurrent_executions=1, environment={ "LOG_LEVEL": "INFO", "STREAM_NAME": f"{self.data_pipe_stream.stream_name}", "APP_ENV": "Production", "STREAM_AWS_REGION": f"{core.Aws.REGION}" }) # Grant our Lambda Producer privileges to write to Kinesis Data Stream self.data_pipe_stream.grant_read_write(data_producer_fn) # Create Custom Loggroup for Producer data_producer_lg = _logs.LogGroup( self, "dataProducerLogGroup", log_group_name=f"/aws/lambda/{data_producer_fn.function_name}", removal_policy=core.RemovalPolicy.DESTROY, retention=_logs.RetentionDays.ONE_DAY) ########################################### ################# OUTPUTS ################# ########################################### output_0 = core.CfnOutput( self, "AutomationFrom", value=f"{GlobalArgs.SOURCE_INFO}", description= "To know more about this automation stack, check out our github page." ) output_1 = core.CfnOutput( self, "streamDataProducer", value= f"https://console.aws.amazon.com/lambda/home?region={core.Aws.REGION}#/functions/{data_producer_fn.function_name}?tab=code", description= "Produce streaming data events and push to Kinesis stream.")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # TODO: add resources for Part 2, 3 of blog post. # Kinesis Data Streams kds = kinesis.Stream(self, "KinesisTweets", stream_name="kinesis-tweets", shard_count=5, retention_period=Duration.hours(48)) # Fargate Task Role task_role = iam.Role( self, 'task_role', assumed_by=iam.ServicePrincipal('ecs-tasks.amazonaws.com')) # Policy to allow task to put records into Kinessis task_role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=[ 'kinesis:PutRecord', 'kinesis:PutRecords', 'kinesis:DescribeStream' ], resources=[kds.stream_arn])) # Policy to get secret from SecretsManager task_role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=[ 'secretsmanager:GetResourcePolicy', 'secretsmanager:GetSecretValue', 'secretsmanager:DescribeSecret', 'secretsmanager:ListSecretVersionIds' ], resources=['*'])) # VPC vpc = ec2.Vpc( self, 'FargateVPC', max_azs=2 # Default is all AZs in the region ) # ECS Cluster cluster = ecs.Cluster(self, 'EcsCluster', vpc=vpc) # Fargate Task Definition task_definition = ecs.FargateTaskDefinition(self, 'ServiceTaskDefinition', cpu=256, memory_limit_mib=512, task_role=task_role) # Fargate log driver fargate_logger = ecs.AwsLogDriver(stream_prefix='fargate_twitter_logs') # Container task_definition.add_container( 'ServiceContainer', image=ecs.ContainerImage.from_asset('./ECSContainerFiles'), environment={ 'KINESIS_STREAM_NAME': kds.stream_name, 'REGION_NAME': self.region, 'KEYWORD': 'trump', 'SECRETS_NAME': 'TwitterAPISecrets' }, logging=fargate_logger) # Fargate Service service = ecs.FargateService(self, 'ServiceFargateService', task_definition=task_definition, assign_public_ip=True, cluster=cluster)
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # The code that defines your stack goes here #获取vpc #vpc = ec2.Vpc.from_lookup(self, 'default',is_default=True,vpc_name='default') vpc = ec2.Vpc.from_lookup(self, 'dms-vpc', vpc_id='vpc-08b56fb6053ca2c75') #创建RDS参数组 db_parameter = rds.ParameterGroup( self, 'dms-param-mysql5.7', engine=rds.DatabaseInstanceEngine.mysql( version=rds.MysqlEngineVersion.VER_5_7), parameters={"binlog_format": "ROW"}) # sourceDB = rds.DatabaseInstanceFromSnapshot( # self,'dms-rds-soruce', # snapshot_identifier= 'tickets-mysql57', # engine=rds.DatabaseInstanceEngine.MYSQL, # instance_type=ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE3,ec2.InstanceSize.MEDIUM), # vpc=vpc, # parameter_group=db_parameter # ) # sourceDB = rds.DatabaseInstance( # self,'dms-rds-soruce', # #instance_identifier='dms-rds-soruce', # engine=rds.DatabaseInstanceEngine.mysql( # version=rds.MysqlEngineVersion.VER_5_7 # ), # instance_type=ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE3,ec2.InstanceSize.MEDIUM), # vpc=vpc, # parameter_group=db_parameter, # #credentials=rdsPasswordSecret # ) # sourceDB.connections.allow_default_port_internally() dms_rep = dms.CfnReplicationInstance( self, 'dms-replication', replication_instance_class='dms.c5.large', engine_version='3.4.0') stream = kinesis.Stream(self, 'dms-steam') streamWriteRole = iam.Role( self, 'dms-stream-role', assumed_by=iam.ServicePrincipal('dms.amazonaws.com')) streamWriteRole.add_to_policy( iam.PolicyStatement(resources=[stream.stream_arn], actions=[ 'kinesis:DescribeStream', 'kinesis:PutRecord', 'kinesis:PutRecords' ])) source = dms.CfnEndpoint( self, 'dms-source', endpoint_type='source', engine_name='mysql', username='******', password='******', server_name= "dms-rdssource.c7iucbqgd2xo.us-east-1.rds.amazonaws.com", port=3306) target = dms.CfnEndpoint(self, 'dms-target', endpoint_type='target', engine_name='kinesis', kinesis_settings={ "messageFormat": "JSON", 'streamArn': stream.stream_arn, "serviceAccessRoleArn": streamWriteRole.role_arn }) dmsTableMappings = { "rules": [{ "rule-type": "selection", "rule-id": "1", "rule-name": "1", "object-locator": { "schema-name": "dms_sample", "table-name": "t_log_levelup" }, "rule-action": "include" }] } dms.CfnReplicationTask(self, 'dms-stream-repTask', replication_instance_arn=dms_rep.ref, migration_type='full-load-and-cdc', source_endpoint_arn=source.ref, target_endpoint_arn=target.ref, table_mappings=json.dumps(dmsTableMappings)) analyticsRole = iam.Role( self, 'KinesisAnalyticsRole', assumed_by=iam.ServicePrincipal('kinesisanalytics.amazonaws.com')) kinesisanalytics.CfnApplicationV2( self, 'KinesisAnalytics', application_name='dms-stream-anlytics', service_execution_role=analyticsRole.role_arn, runtime_environment='SQL-1_0', application_configuration={ 'sqlApplicationConfiguration': { 'inputs': [{ 'namePrefix': "exampleNamePrefix", 'inputSchema': { 'recordColumns': [{ 'name': "example", 'sqlType': "VARCHAR(16)", 'mapping': "$.example" }], 'recordFormat': { 'recordFormatType': "JSON", 'mappingParameters': { 'jsonMappingParameters': { 'recordRowPath': "$" } } } }, 'kinesisStreamsInput': { 'resourceArn': stream.stream_arn } }] }, 'applicationCodeConfiguration': { 'codeContent': { 'textContent': "Example Application Code" }, 'codeContentType': "PLAINTEXT" } })
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) test_queue = sqs.Queue(self, 'test-queue', queue_name='test1') test_topic = sns.Topic(self, 'test-topic') sns.Subscription(self, 'test-subscription', topic=test_topic, endpoint=test_queue.queue_arn, protocol=sns.SubscriptionProtocol.SQS) kinesis.Stream(self, 'test-stream', stream_name='donut-sales', shard_count=2) create_order = step.Pass(self, 'create-order', result=step.Result.from_object({ "Order": { "Customer": "Alice", "Product": "Coffee", "Billing": { "Price": 10.0, "Quantity": 4.0 } } })) calculate_amount = step.Pass(self, 'calculate-amount', result=step.Result.from_number(40.0), result_path='$.Order.Billing.Amount', output_path='$.Order.Billing') order_definition = create_order.next(calculate_amount) step.StateMachine(self, 'test-state-machine', state_machine_name='order-machine', definition=order_definition) make_tea = step.Choice( self, 'make-tea', comment='Input should look like {"tea":"green"}') green = step.Pass(self, 'green', result=step.Result.from_string('Green tea')) make_tea.when(step.Condition.string_equals('$.tea', 'green'), green) black = step.Pass(self, 'black', result=step.Result.from_string('Black tea')) make_tea.when(step.Condition.string_equals('$.tea', 'black'), black) orange = step.Pass(self, 'orange', result=step.Result.from_string('Black tea')) make_tea.when(step.Condition.string_equals('$.tea', 'orange'), orange) error = step.Pass(self, 'error', result=step.Result.from_string('Bad input')) make_tea.otherwise(error) step.StateMachine(self, 'test-state-machine-2', state_machine_name='tea-machine', definition=make_tea)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.kinesis = ks.Stream(self, 'MailStream', encryption=ks.StreamEncryption.MANAGED)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) vpc = aws_ec2.Vpc( self, "OctemberVPC", max_azs=2, # subnet_configuration=[{ # "cidrMask": 24, # "name": "Public", # "subnetType": aws_ec2.SubnetType.PUBLIC, # }, # { # "cidrMask": 24, # "name": "Private", # "subnetType": aws_ec2.SubnetType.PRIVATE # }, # { # "cidrMask": 28, # "name": "Isolated", # "subnetType": aws_ec2.SubnetType.ISOLATED, # "reserved": True # } # ], gateway_endpoints={ "S3": aws_ec2.GatewayVpcEndpointOptions( service=aws_ec2.GatewayVpcEndpointAwsService.S3) }) dynamo_db_endpoint = vpc.add_gateway_endpoint( "DynamoDbEndpoint", service=aws_ec2.GatewayVpcEndpointAwsService.DYNAMODB) s3_bucket = s3.Bucket( self, "s3bucket", bucket_name="octember-bizcard-{region}-{account}".format( region=core.Aws.REGION, account=core.Aws.ACCOUNT_ID)) api = apigw.RestApi( self, "BizcardImageUploader", rest_api_name="BizcardImageUploader", description="This service serves uploading bizcard images into s3.", endpoint_types=[apigw.EndpointType.REGIONAL], binary_media_types=["image/png", "image/jpg"], deploy=True, deploy_options=apigw.StageOptions(stage_name="v1")) rest_api_role = aws_iam.Role( self, "ApiGatewayRoleForS3", role_name="ApiGatewayRoleForS3FullAccess", assumed_by=aws_iam.ServicePrincipal("apigateway.amazonaws.com"), managed_policies=[ aws_iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonS3FullAccess") ]) list_objects_responses = [ apigw.IntegrationResponse( status_code="200", #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationResponse.html#aws_cdk.aws_apigateway.IntegrationResponse.response_parameters # The response parameters from the backend response that API Gateway sends to the method response. # Use the destination as the key and the source as the value: # - The destination must be an existing response parameter in the MethodResponse property. # - The source must be an existing method request parameter or a static value. response_parameters={ 'method.response.header.Timestamp': 'integration.response.header.Date', 'method.response.header.Content-Length': 'integration.response.header.Content-Length', 'method.response.header.Content-Type': 'integration.response.header.Content-Type' }), apigw.IntegrationResponse(status_code="400", selection_pattern="4\d{2}"), apigw.IntegrationResponse(status_code="500", selection_pattern="5\d{2}") ] list_objects_integration_options = apigw.IntegrationOptions( credentials_role=rest_api_role, integration_responses=list_objects_responses) get_s3_integration = apigw.AwsIntegration( service="s3", integration_http_method="GET", path='/', options=list_objects_integration_options) api.root.add_method( "GET", get_s3_integration, authorization_type=apigw.AuthorizationType.IAM, api_key_required=False, method_responses=[ apigw.MethodResponse( status_code="200", response_parameters={ 'method.response.header.Timestamp': False, 'method.response.header.Content-Length': False, 'method.response.header.Content-Type': False }, response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ], request_parameters={'method.request.header.Content-Type': False}) get_s3_folder_integration_options = apigw.IntegrationOptions( credentials_role=rest_api_role, integration_responses=list_objects_responses, #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationOptions.html#aws_cdk.aws_apigateway.IntegrationOptions.request_parameters # Specify request parameters as key-value pairs (string-to-string mappings), with a destination as the key and a source as the value. # The source must be an existing method request parameter or a static value. request_parameters={ "integration.request.path.bucket": "method.request.path.folder" }) get_s3_folder_integration = apigw.AwsIntegration( service="s3", integration_http_method="GET", path="{bucket}", options=get_s3_folder_integration_options) s3_folder = api.root.add_resource('{folder}') s3_folder.add_method( "GET", get_s3_folder_integration, authorization_type=apigw.AuthorizationType.IAM, api_key_required=False, method_responses=[ apigw.MethodResponse( status_code="200", response_parameters={ 'method.response.header.Timestamp': False, 'method.response.header.Content-Length': False, 'method.response.header.Content-Type': False }, response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ], request_parameters={ 'method.request.header.Content-Type': False, 'method.request.path.folder': True }) get_s3_item_integration_options = apigw.IntegrationOptions( credentials_role=rest_api_role, integration_responses=list_objects_responses, request_parameters={ "integration.request.path.bucket": "method.request.path.folder", "integration.request.path.object": "method.request.path.item" }) get_s3_item_integration = apigw.AwsIntegration( service="s3", integration_http_method="GET", path="{bucket}/{object}", options=get_s3_item_integration_options) s3_item = s3_folder.add_resource('{item}') s3_item.add_method( "GET", get_s3_item_integration, authorization_type=apigw.AuthorizationType.IAM, api_key_required=False, method_responses=[ apigw.MethodResponse( status_code="200", response_parameters={ 'method.response.header.Timestamp': False, 'method.response.header.Content-Length': False, 'method.response.header.Content-Type': False }, response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ], request_parameters={ 'method.request.header.Content-Type': False, 'method.request.path.folder': True, 'method.request.path.item': True }) put_s3_item_integration_options = apigw.IntegrationOptions( credentials_role=rest_api_role, integration_responses=[ apigw.IntegrationResponse(status_code="200"), apigw.IntegrationResponse(status_code="400", selection_pattern="4\d{2}"), apigw.IntegrationResponse(status_code="500", selection_pattern="5\d{2}") ], request_parameters={ "integration.request.header.Content-Type": "method.request.header.Content-Type", "integration.request.path.bucket": "method.request.path.folder", "integration.request.path.object": "method.request.path.item" }) put_s3_item_integration = apigw.AwsIntegration( service="s3", integration_http_method="PUT", path="{bucket}/{object}", options=put_s3_item_integration_options) s3_item.add_method( "PUT", put_s3_item_integration, authorization_type=apigw.AuthorizationType.IAM, api_key_required=False, method_responses=[ apigw.MethodResponse( status_code="200", response_parameters={ 'method.response.header.Content-Type': False }, response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ], request_parameters={ 'method.request.header.Content-Type': False, 'method.request.path.folder': True, 'method.request.path.item': True }) ddb_table = dynamodb.Table( self, "BizcardImageMetaInfoDdbTable", table_name="OctemberBizcardImgMeta", partition_key=dynamodb.Attribute( name="image_id", type=dynamodb.AttributeType.STRING), billing_mode=dynamodb.BillingMode.PROVISIONED, read_capacity=15, write_capacity=5) img_kinesis_stream = kinesis.Stream( self, "BizcardImagePath", stream_name="octember-bizcard-image") # create lambda function trigger_textract_lambda_fn = _lambda.Function( self, "TriggerTextExtractorFromImage", runtime=_lambda.Runtime.PYTHON_3_7, function_name="TriggerTextExtractorFromImage", handler="trigger_text_extract_from_s3_image.lambda_handler", description="Trigger to extract text from an image in S3", code=_lambda.Code.asset( "./src/main/python/TriggerTextExtractFromS3Image"), environment={ 'REGION_NAME': core.Aws.REGION, 'DDB_TABLE_NAME': ddb_table.table_name, 'KINESIS_STREAM_NAME': img_kinesis_stream.stream_name }, timeout=core.Duration.minutes(5)) ddb_table_rw_policy_statement = aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, resources=[ddb_table.table_arn], actions=[ "dynamodb:BatchGetItem", "dynamodb:Describe*", "dynamodb:List*", "dynamodb:GetItem", "dynamodb:Query", "dynamodb:Scan", "dynamodb:BatchWriteItem", "dynamodb:DeleteItem", "dynamodb:PutItem", "dynamodb:UpdateItem", "dax:Describe*", "dax:List*", "dax:GetItem", "dax:BatchGetItem", "dax:Query", "dax:Scan", "dax:BatchWriteItem", "dax:DeleteItem", "dax:PutItem", "dax:UpdateItem" ]) trigger_textract_lambda_fn.add_to_role_policy( ddb_table_rw_policy_statement) trigger_textract_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=[img_kinesis_stream.stream_arn], actions=[ "kinesis:Get*", "kinesis:List*", "kinesis:Describe*", "kinesis:PutRecord", "kinesis:PutRecords" ])) # assign notification for the s3 event type (ex: OBJECT_CREATED) s3_event_filter = s3.NotificationKeyFilter(prefix="bizcard-raw-img/", suffix=".jpg") s3_event_source = S3EventSource(s3_bucket, events=[s3.EventType.OBJECT_CREATED], filters=[s3_event_filter]) trigger_textract_lambda_fn.add_event_source(s3_event_source) #XXX: https://github.com/aws/aws-cdk/issues/2240 # To avoid to create extra Lambda Functions with names like LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a # if log_retention=aws_logs.RetentionDays.THREE_DAYS is added to the constructor props log_group = aws_logs.LogGroup( self, "TriggerTextractLogGroup", log_group_name="/aws/lambda/TriggerTextExtractorFromImage", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(trigger_textract_lambda_fn) text_kinesis_stream = kinesis.Stream( self, "BizcardTextData", stream_name="octember-bizcard-txt") textract_lambda_fn = _lambda.Function( self, "GetTextFromImage", runtime=_lambda.Runtime.PYTHON_3_7, function_name="GetTextFromImage", handler="get_text_from_s3_image.lambda_handler", description="extract text from an image in S3", code=_lambda.Code.asset("./src/main/python/GetTextFromS3Image"), environment={ 'REGION_NAME': core.Aws.REGION, 'DDB_TABLE_NAME': ddb_table.table_name, 'KINESIS_STREAM_NAME': text_kinesis_stream.stream_name }, timeout=core.Duration.minutes(5)) textract_lambda_fn.add_to_role_policy(ddb_table_rw_policy_statement) textract_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=[text_kinesis_stream.stream_arn], actions=[ "kinesis:Get*", "kinesis:List*", "kinesis:Describe*", "kinesis:PutRecord", "kinesis:PutRecords" ])) textract_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ s3_bucket.bucket_arn, "{}/*".format( s3_bucket.bucket_arn) ], "actions": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject" ] })) textract_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=["textract:*"])) img_kinesis_event_source = KinesisEventSource( img_kinesis_stream, batch_size=100, starting_position=_lambda.StartingPosition.LATEST) textract_lambda_fn.add_event_source(img_kinesis_event_source) log_group = aws_logs.LogGroup( self, "GetTextFromImageLogGroup", log_group_name="/aws/lambda/GetTextFromImage", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(textract_lambda_fn) sg_use_bizcard_es = aws_ec2.SecurityGroup( self, "BizcardSearchClientSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard elasticsearch client', security_group_name='use-octember-bizcard-es') core.Tags.of(sg_use_bizcard_es).add('Name', 'use-octember-bizcard-es') sg_bizcard_es = aws_ec2.SecurityGroup( self, "BizcardSearchSG", vpc=vpc, allow_all_outbound=True, description='security group for octember bizcard elasticsearch', security_group_name='octember-bizcard-es') core.Tags.of(sg_bizcard_es).add('Name', 'octember-bizcard-es') sg_bizcard_es.add_ingress_rule(peer=sg_bizcard_es, connection=aws_ec2.Port.all_tcp(), description='octember-bizcard-es') sg_bizcard_es.add_ingress_rule(peer=sg_use_bizcard_es, connection=aws_ec2.Port.all_tcp(), description='use-octember-bizcard-es') sg_ssh_access = aws_ec2.SecurityGroup( self, "BastionHostSG", vpc=vpc, allow_all_outbound=True, description='security group for bastion host', security_group_name='octember-bastion-host-sg') core.Tags.of(sg_ssh_access).add('Name', 'octember-bastion-host') sg_ssh_access.add_ingress_rule(peer=aws_ec2.Peer.any_ipv4(), connection=aws_ec2.Port.tcp(22), description='ssh access') bastion_host = aws_ec2.BastionHostLinux( self, "BastionHost", vpc=vpc, instance_type=aws_ec2.InstanceType('t3.nano'), security_group=sg_ssh_access, subnet_selection=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC)) bastion_host.instance.add_security_group(sg_use_bizcard_es) #XXX: aws cdk elastsearch example - https://github.com/aws/aws-cdk/issues/2873 es_cfn_domain = aws_elasticsearch.CfnDomain( self, 'BizcardSearch', elasticsearch_cluster_config={ "dedicatedMasterCount": 3, "dedicatedMasterEnabled": True, "dedicatedMasterType": "t2.medium.elasticsearch", "instanceCount": 2, "instanceType": "t2.medium.elasticsearch", "zoneAwarenessEnabled": True }, ebs_options={ "ebsEnabled": True, "volumeSize": 10, "volumeType": "gp2" }, domain_name="octember-bizcard", elasticsearch_version="7.9", encryption_at_rest_options={"enabled": False}, access_policies={ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "AWS": "*" }, "Action": ["es:Describe*", "es:List*", "es:Get*", "es:ESHttp*"], "Resource": self.format_arn(service="es", resource="domain", resource_name="octember-bizcard/*") }] }, snapshot_options={"automatedSnapshotStartHour": 17}, vpc_options={ "securityGroupIds": [sg_bizcard_es.security_group_id], "subnetIds": vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids }) core.Tags.of(es_cfn_domain).add('Name', 'octember-bizcard-es') s3_lib_bucket_name = self.node.try_get_context("lib_bucket_name") #XXX: https://github.com/aws/aws-cdk/issues/1342 s3_lib_bucket = s3.Bucket.from_bucket_name(self, id, s3_lib_bucket_name) es_lib_layer = _lambda.LayerVersion( self, "ESLib", layer_version_name="es-lib", compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], code=_lambda.Code.from_bucket(s3_lib_bucket, "var/octember-es-lib.zip")) redis_lib_layer = _lambda.LayerVersion( self, "RedisLib", layer_version_name="redis-lib", compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], code=_lambda.Code.from_bucket(s3_lib_bucket, "var/octember-redis-lib.zip")) #XXX: Deploy lambda in VPC - https://github.com/aws/aws-cdk/issues/1342 upsert_to_es_lambda_fn = _lambda.Function( self, "UpsertBizcardToES", runtime=_lambda.Runtime.PYTHON_3_7, function_name="UpsertBizcardToElasticSearch", handler="upsert_bizcard_to_es.lambda_handler", description="Upsert bizcard text into elasticsearch", code=_lambda.Code.asset("./src/main/python/UpsertBizcardToES"), environment={ 'ES_HOST': es_cfn_domain.attr_domain_endpoint, 'ES_INDEX': 'octember_bizcard', 'ES_TYPE': 'bizcard' }, timeout=core.Duration.minutes(5), layers=[es_lib_layer], security_groups=[sg_use_bizcard_es], vpc=vpc) text_kinesis_event_source = KinesisEventSource( text_kinesis_stream, batch_size=99, starting_position=_lambda.StartingPosition.LATEST) upsert_to_es_lambda_fn.add_event_source(text_kinesis_event_source) log_group = aws_logs.LogGroup( self, "UpsertBizcardToESLogGroup", log_group_name="/aws/lambda/UpsertBizcardToElasticSearch", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(upsert_to_es_lambda_fn) firehose_role_policy_doc = aws_iam.PolicyDocument() firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ s3_bucket.bucket_arn, "{}/*".format( s3_bucket.bucket_arn) ], "actions": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject" ] })) firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=[ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions" ])) firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=[text_kinesis_stream.stream_arn], actions=[ "kinesis:DescribeStream", "kinesis:GetShardIterator", "kinesis:GetRecords" ])) firehose_log_group_name = "/aws/kinesisfirehose/octember-bizcard-txt-to-s3" firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, #XXX: The ARN will be formatted as follows: # arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name} resources=[ self.format_arn(service="logs", resource="log-group", resource_name="{}:log-stream:*".format( firehose_log_group_name), sep=":") ], actions=["logs:PutLogEvents"])) firehose_role = aws_iam.Role( self, "FirehoseDeliveryRole", role_name="FirehoseDeliveryRole", assumed_by=aws_iam.ServicePrincipal("firehose.amazonaws.com"), #XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221 inline_policies={"firehose_role_policy": firehose_role_policy_doc}) bizcard_text_to_s3_delivery_stream = aws_kinesisfirehose.CfnDeliveryStream( self, "BizcardTextToS3", delivery_stream_name="octember-bizcard-txt-to-s3", delivery_stream_type="KinesisStreamAsSource", kinesis_stream_source_configuration={ "kinesisStreamArn": text_kinesis_stream.stream_arn, "roleArn": firehose_role.role_arn }, extended_s3_destination_configuration={ "bucketArn": s3_bucket.bucket_arn, "bufferingHints": { "intervalInSeconds": 60, "sizeInMBs": 1 }, "cloudWatchLoggingOptions": { "enabled": True, "logGroupName": firehose_log_group_name, "logStreamName": "S3Delivery" }, "compressionFormat": "GZIP", "prefix": "bizcard-text/", "roleArn": firehose_role.role_arn }) sg_use_bizcard_es_cache = aws_ec2.SecurityGroup( self, "BizcardSearchCacheClientSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard search query cache client', security_group_name='use-octember-bizcard-es-cache') core.Tags.of(sg_use_bizcard_es_cache).add( 'Name', 'use-octember-bizcard-es-cache') sg_bizcard_es_cache = aws_ec2.SecurityGroup( self, "BizcardSearchCacheSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard search query cache', security_group_name='octember-bizcard-es-cache') core.Tags.of(sg_bizcard_es_cache).add('Name', 'octember-bizcard-es-cache') sg_bizcard_es_cache.add_ingress_rule( peer=sg_use_bizcard_es_cache, connection=aws_ec2.Port.tcp(6379), description='use-octember-bizcard-es-cache') es_query_cache_subnet_group = aws_elasticache.CfnSubnetGroup( self, "QueryCacheSubnetGroup", description="subnet group for octember-bizcard-es-cache", subnet_ids=vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids, cache_subnet_group_name='octember-bizcard-es-cache') es_query_cache = aws_elasticache.CfnCacheCluster( self, "BizcardSearchQueryCache", cache_node_type="cache.t3.small", num_cache_nodes=1, engine="redis", engine_version="5.0.5", auto_minor_version_upgrade=False, cluster_name="octember-bizcard-es-cache", snapshot_retention_limit=3, snapshot_window="17:00-19:00", preferred_maintenance_window="mon:19:00-mon:20:30", #XXX: Do not use referece for "cache_subnet_group_name" - https://github.com/aws/aws-cdk/issues/3098 #cache_subnet_group_name=es_query_cache_subnet_group.cache_subnet_group_name, # Redis cluster goes to wrong VPC cache_subnet_group_name='octember-bizcard-es-cache', vpc_security_group_ids=[sg_bizcard_es_cache.security_group_id]) #XXX: If you're going to launch your cluster in an Amazon VPC, you need to create a subnet group before you start creating a cluster. # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticache-cache-cluster.html#cfn-elasticache-cachecluster-cachesubnetgroupname es_query_cache.add_depends_on(es_query_cache_subnet_group) #XXX: add more than 2 security groups # https://github.com/aws/aws-cdk/blob/ea10f0d141a48819ec0000cd7905feda993870a9/packages/%40aws-cdk/aws-lambda/lib/function.ts#L387 # https://github.com/aws/aws-cdk/issues/1555 # https://github.com/aws/aws-cdk/pull/5049 bizcard_search_lambda_fn = _lambda.Function( self, "BizcardSearchServer", runtime=_lambda.Runtime.PYTHON_3_7, function_name="BizcardSearchProxy", handler="es_search_bizcard.lambda_handler", description="Proxy server to search bizcard text", code=_lambda.Code.asset("./src/main/python/SearchBizcard"), environment={ 'ES_HOST': es_cfn_domain.attr_domain_endpoint, 'ES_INDEX': 'octember_bizcard', 'ES_TYPE': 'bizcard', 'ELASTICACHE_HOST': es_query_cache.attr_redis_endpoint_address }, timeout=core.Duration.minutes(1), layers=[es_lib_layer, redis_lib_layer], security_groups=[sg_use_bizcard_es, sg_use_bizcard_es_cache], vpc=vpc) #XXX: create API Gateway + LambdaProxy search_api = apigw.LambdaRestApi( self, "BizcardSearchAPI", handler=bizcard_search_lambda_fn, proxy=False, rest_api_name="BizcardSearch", description="This service serves searching bizcard text.", endpoint_types=[apigw.EndpointType.REGIONAL], deploy=True, deploy_options=apigw.StageOptions(stage_name="v1")) bizcard_search = search_api.root.add_resource('search') bizcard_search.add_method( "GET", method_responses=[ apigw.MethodResponse( status_code="200", response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ]) sg_use_bizcard_graph_db = aws_ec2.SecurityGroup( self, "BizcardGraphDbClientSG", vpc=vpc, allow_all_outbound=True, description='security group for octember bizcard graph db client', security_group_name='use-octember-bizcard-neptune') core.Tags.of(sg_use_bizcard_graph_db).add( 'Name', 'use-octember-bizcard-neptune') sg_bizcard_graph_db = aws_ec2.SecurityGroup( self, "BizcardGraphDbSG", vpc=vpc, allow_all_outbound=True, description='security group for octember bizcard graph db', security_group_name='octember-bizcard-neptune') core.Tags.of(sg_bizcard_graph_db).add('Name', 'octember-bizcard-neptune') sg_bizcard_graph_db.add_ingress_rule( peer=sg_bizcard_graph_db, connection=aws_ec2.Port.tcp(8182), description='octember-bizcard-neptune') sg_bizcard_graph_db.add_ingress_rule( peer=sg_use_bizcard_graph_db, connection=aws_ec2.Port.tcp(8182), description='use-octember-bizcard-neptune') bizcard_graph_db_subnet_group = aws_neptune.CfnDBSubnetGroup( self, "NeptuneSubnetGroup", db_subnet_group_description= "subnet group for octember-bizcard-neptune", subnet_ids=vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids, db_subnet_group_name='octember-bizcard-neptune') bizcard_graph_db = aws_neptune.CfnDBCluster( self, "BizcardGraphDB", availability_zones=vpc.availability_zones, db_subnet_group_name=bizcard_graph_db_subnet_group. db_subnet_group_name, db_cluster_identifier="octember-bizcard", backup_retention_period=1, preferred_backup_window="08:45-09:15", preferred_maintenance_window="sun:18:00-sun:18:30", vpc_security_group_ids=[sg_bizcard_graph_db.security_group_id]) bizcard_graph_db.add_depends_on(bizcard_graph_db_subnet_group) bizcard_graph_db_instance = aws_neptune.CfnDBInstance( self, "BizcardGraphDBInstance", db_instance_class="db.r5.large", allow_major_version_upgrade=False, auto_minor_version_upgrade=False, availability_zone=vpc.availability_zones[0], db_cluster_identifier=bizcard_graph_db.db_cluster_identifier, db_instance_identifier="octember-bizcard", preferred_maintenance_window="sun:18:00-sun:18:30") bizcard_graph_db_instance.add_depends_on(bizcard_graph_db) bizcard_graph_db_replica_instance = aws_neptune.CfnDBInstance( self, "BizcardGraphDBReplicaInstance", db_instance_class="db.r5.large", allow_major_version_upgrade=False, auto_minor_version_upgrade=False, availability_zone=vpc.availability_zones[-1], db_cluster_identifier=bizcard_graph_db.db_cluster_identifier, db_instance_identifier="octember-bizcard-replica", preferred_maintenance_window="sun:18:00-sun:18:30") bizcard_graph_db_replica_instance.add_depends_on(bizcard_graph_db) bizcard_graph_db_replica_instance.add_depends_on( bizcard_graph_db_instance) gremlinpython_lib_layer = _lambda.LayerVersion( self, "GremlinPythonLib", layer_version_name="gremlinpython-lib", compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], code=_lambda.Code.from_bucket( s3_lib_bucket, "var/octember-gremlinpython-lib.zip")) #XXX: https://github.com/aws/aws-cdk/issues/1342 upsert_to_neptune_lambda_fn = _lambda.Function( self, "UpsertBizcardToGraphDB", runtime=_lambda.Runtime.PYTHON_3_7, function_name="UpsertBizcardToNeptune", handler="upsert_bizcard_to_graph_db.lambda_handler", description="Upsert bizcard into neptune", code=_lambda.Code.asset( "./src/main/python/UpsertBizcardToGraphDB"), environment={ 'REGION_NAME': core.Aws.REGION, 'NEPTUNE_ENDPOINT': bizcard_graph_db.attr_endpoint, 'NEPTUNE_PORT': bizcard_graph_db.attr_port }, timeout=core.Duration.minutes(5), layers=[gremlinpython_lib_layer], security_groups=[sg_use_bizcard_graph_db], vpc=vpc) upsert_to_neptune_lambda_fn.add_event_source(text_kinesis_event_source) log_group = aws_logs.LogGroup( self, "UpsertBizcardToGraphDBLogGroup", log_group_name="/aws/lambda/UpsertBizcardToNeptune", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(upsert_to_neptune_lambda_fn) sg_use_bizcard_neptune_cache = aws_ec2.SecurityGroup( self, "BizcardNeptuneCacheClientSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard recommendation query cache client', security_group_name='use-octember-bizcard-neptune-cache') core.Tags.of(sg_use_bizcard_neptune_cache).add( 'Name', 'use-octember-bizcard-es-cache') sg_bizcard_neptune_cache = aws_ec2.SecurityGroup( self, "BizcardNeptuneCacheSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard recommendation query cache', security_group_name='octember-bizcard-neptune-cache') core.Tags.of(sg_bizcard_neptune_cache).add( 'Name', 'octember-bizcard-neptune-cache') sg_bizcard_neptune_cache.add_ingress_rule( peer=sg_use_bizcard_neptune_cache, connection=aws_ec2.Port.tcp(6379), description='use-octember-bizcard-neptune-cache') recomm_query_cache_subnet_group = aws_elasticache.CfnSubnetGroup( self, "RecommQueryCacheSubnetGroup", description="subnet group for octember-bizcard-neptune-cache", subnet_ids=vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids, cache_subnet_group_name='octember-bizcard-neptune-cache') recomm_query_cache = aws_elasticache.CfnCacheCluster( self, "BizcardRecommQueryCache", cache_node_type="cache.t3.small", num_cache_nodes=1, engine="redis", engine_version="5.0.5", auto_minor_version_upgrade=False, cluster_name="octember-bizcard-neptune-cache", snapshot_retention_limit=3, snapshot_window="17:00-19:00", preferred_maintenance_window="mon:19:00-mon:20:30", #XXX: Do not use referece for "cache_subnet_group_name" - https://github.com/aws/aws-cdk/issues/3098 #cache_subnet_group_name=recomm_query_cache_subnet_group.cache_subnet_group_name, # Redis cluster goes to wrong VPC cache_subnet_group_name='octember-bizcard-neptune-cache', vpc_security_group_ids=[ sg_bizcard_neptune_cache.security_group_id ]) recomm_query_cache.add_depends_on(recomm_query_cache_subnet_group) bizcard_recomm_lambda_fn = _lambda.Function( self, "BizcardRecommender", runtime=_lambda.Runtime.PYTHON_3_7, function_name="BizcardRecommender", handler="neptune_recommend_bizcard.lambda_handler", description="This service serves PYMK(People You May Know).", code=_lambda.Code.asset("./src/main/python/RecommendBizcard"), environment={ 'REGION_NAME': core.Aws.REGION, 'NEPTUNE_ENDPOINT': bizcard_graph_db.attr_read_endpoint, 'NEPTUNE_PORT': bizcard_graph_db.attr_port, 'ELASTICACHE_HOST': recomm_query_cache.attr_redis_endpoint_address }, timeout=core.Duration.minutes(1), layers=[gremlinpython_lib_layer, redis_lib_layer], security_groups=[ sg_use_bizcard_graph_db, sg_use_bizcard_neptune_cache ], vpc=vpc) #XXX: create API Gateway + LambdaProxy recomm_api = apigw.LambdaRestApi( self, "BizcardRecommendAPI", handler=bizcard_recomm_lambda_fn, proxy=False, rest_api_name="BizcardRecommend", description="This service serves PYMK(People You May Know).", endpoint_types=[apigw.EndpointType.REGIONAL], deploy=True, deploy_options=apigw.StageOptions(stage_name="v1")) bizcard_recomm = recomm_api.root.add_resource('pymk') bizcard_recomm.add_method( "GET", method_responses=[ apigw.MethodResponse( status_code="200", response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ]) sagemaker_notebook_role_policy_doc = aws_iam.PolicyDocument() sagemaker_notebook_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ "arn:aws:s3:::aws-neptune-notebook", "arn:aws:s3:::aws-neptune-notebook/*" ], "actions": ["s3:GetObject", "s3:ListBucket"] })) sagemaker_notebook_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ "arn:aws:neptune-db:{region}:{account}:{cluster_id}/*". format(region=core.Aws.REGION, account=core.Aws.ACCOUNT_ID, cluster_id=bizcard_graph_db. attr_cluster_resource_id) ], "actions": ["neptune-db:connect"] })) sagemaker_notebook_role = aws_iam.Role( self, 'SageMakerNotebookForNeptuneWorkbenchRole', role_name='AWSNeptuneNotebookRole-OctemberBizcard', assumed_by=aws_iam.ServicePrincipal('sagemaker.amazonaws.com'), #XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221 inline_policies={ 'AWSNeptuneNotebook': sagemaker_notebook_role_policy_doc }) neptune_wb_lifecycle_content = '''#!/bin/bash sudo -u ec2-user -i <<'EOF' echo "export GRAPH_NOTEBOOK_AUTH_MODE=DEFAULT" >> ~/.bashrc echo "export GRAPH_NOTEBOOK_HOST={NeptuneClusterEndpoint}" >> ~/.bashrc echo "export GRAPH_NOTEBOOK_PORT={NeptuneClusterPort}" >> ~/.bashrc echo "export NEPTUNE_LOAD_FROM_S3_ROLE_ARN=''" >> ~/.bashrc echo "export AWS_REGION={AWS_Region}" >> ~/.bashrc aws s3 cp s3://aws-neptune-notebook/graph_notebook.tar.gz /tmp/graph_notebook.tar.gz rm -rf /tmp/graph_notebook tar -zxvf /tmp/graph_notebook.tar.gz -C /tmp /tmp/graph_notebook/install.sh EOF '''.format(NeptuneClusterEndpoint=bizcard_graph_db.attr_endpoint, NeptuneClusterPort=bizcard_graph_db.attr_port, AWS_Region=core.Aws.REGION) neptune_wb_lifecycle_config_prop = aws_sagemaker.CfnNotebookInstanceLifecycleConfig.NotebookInstanceLifecycleHookProperty( content=core.Fn.base64(neptune_wb_lifecycle_content)) neptune_wb_lifecycle_config = aws_sagemaker.CfnNotebookInstanceLifecycleConfig( self, 'NpetuneWorkbenchLifeCycleConfig', notebook_instance_lifecycle_config_name= 'AWSNeptuneWorkbenchOctemberBizcardLCConfig', on_start=[neptune_wb_lifecycle_config_prop]) neptune_workbench = aws_sagemaker.CfnNotebookInstance( self, 'NeptuneWorkbench', instance_type='ml.t2.medium', role_arn=sagemaker_notebook_role.role_arn, lifecycle_config_name=neptune_wb_lifecycle_config. notebook_instance_lifecycle_config_name, notebook_instance_name='OctemberBizcard-NeptuneWorkbench', root_access='Disabled', security_group_ids=[sg_use_bizcard_graph_db.security_group_name], subnet_id=bizcard_graph_db_subnet_group.subnet_ids[0])
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here table = aws_dynamodb.Table(self, "DashboardModel", partition_key=aws_dynamodb.Attribute(name="Pk", type=aws_dynamodb.AttributeType.STRING), sort_key=aws_dynamodb.Attribute(name="Sk", type=aws_dynamodb.AttributeType.STRING), billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST) kds_input_stream = aws_kinesis.Stream(self, "kds_dashboard_input_stream", shard_count=1, stream_name="kds_dashboard_input_stream") kds_output_stream = aws_kinesis.Stream(self, "kds_dashboard_output_stream", shard_count=1, stream_name="kds_dashboard_output_stream") # Creating a ingest bucket for this stack ingest_bucket = aws_s3.Bucket(self,'dreis_dboard_ingest_bucket') kfh_service_role = aws_iam.Role(self, 'KFH_Dashboard_Role', assumed_by=aws_iam.ServicePrincipal('firehose.amazonaws.com') ) kfh_policy_stmt = aws_iam.PolicyStatement( actions=["*"], resources=["*"] ) kfh_service_role.add_to_policy(kfh_policy_stmt) #Creating firehose for this stack kfh_source = aws_kinesisfirehose.CfnDeliveryStream.KinesisStreamSourceConfigurationProperty( kinesis_stream_arn=kds_input_stream.stream_arn, role_arn=kfh_service_role.role_arn ) kfh_datalake = aws_kinesisfirehose.CfnDeliveryStream(self, "kfh_datalake", s3_destination_configuration=aws_kinesisfirehose.CfnDeliveryStream.S3DestinationConfigurationProperty( bucket_arn=ingest_bucket.bucket_arn, buffering_hints=aws_kinesisfirehose.CfnDeliveryStream.BufferingHintsProperty( interval_in_seconds=60, size_in_m_bs=5), compression_format="UNCOMPRESSED", role_arn=kfh_service_role.role_arn ), delivery_stream_type="KinesisStreamAsSource", kinesis_stream_source_configuration=kfh_source ) kda_service_role = aws_iam.Role(self, 'KDA_Dashboard_Role', assumed_by=aws_iam.ServicePrincipal('kinesisanalytics.amazonaws.com') ) kda_policy_stmt = aws_iam.PolicyStatement( actions=["*"], resources=["*"] ) kda_service_role.add_to_policy(kda_policy_stmt) # KA doesn't like - (dash) in names col1 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty( name="state", sql_type="VARCHAR(2)", mapping="$.state" ) col2 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty( name="event_time", sql_type="TIMESTAMP", mapping="$.event-time" ) col3 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty( name="region", sql_type="VARCHAR(12)", mapping="$.region" ) col4 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty( name="store_id", sql_type="INTEGER", mapping="$.store-id" ) col5 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty( name="kpi_1", sql_type="INTEGER", mapping="$.kpi-1" ) col6 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty( name="kpi_2", sql_type="INTEGER", mapping="$.kpi-2" ) col7 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty( name="kpi_3", sql_type="INTEGER", mapping="$.kpi-3" ) col8 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty( name="kpi_4", sql_type="INTEGER", mapping="$.kpi-4" ) col9 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty( name="kpi_5", sql_type="INTEGER", mapping="$.kpi-5" ) schema = aws_kinesisanalytics.CfnApplication.InputSchemaProperty( record_columns=[col2, col1, col3, col4, col5, col6, col7, col8, col9], record_encoding="UTF-8", record_format=aws_kinesisanalytics.CfnApplication.RecordFormatProperty( record_format_type="JSON", mapping_parameters=aws_kinesisanalytics.CfnApplication.MappingParametersProperty( json_mapping_parameters=aws_kinesisanalytics.CfnApplication.JSONMappingParametersProperty( record_row_path="$" ) ) ) ) kda_is = aws_kinesisanalytics.CfnApplication.KinesisStreamsInputProperty( resource_arn=kds_input_stream.stream_arn, role_arn=kda_service_role.role_arn ) ip = aws_kinesisanalytics.CfnApplication.InputProperty( name_prefix="SOURCE_SQL_STREAM", input_schema=schema, kinesis_streams_input=kda_is ) application_code = "CREATE OR REPLACE STREAM \"DESTINATION_SQL_STREAM_BY_STORE\" (\"region\" VARCHAR(10), \"state\" VARCHAR(2), \"store-id\" INTEGER, kpi_1_sum INTEGER, kpi_2_sum INTEGER, ingest_time TIMESTAMP);" + \ "CREATE OR REPLACE STREAM \"DESTINATION_SQL_STREAM_BY_STATE\" (\"region\" VARCHAR(10), \"state\" VARCHAR(2), kpi_1_sum INTEGER, kpi_2_sum INTEGER, ingest_time TIMESTAMP);" + \ "CREATE OR REPLACE STREAM \"DESTINATION_SQL_STREAM_BY_REGION\" (\"region\" VARCHAR(10), kpi_1_sum INTEGER, kpi_2_sum INTEGER, ingest_time TIMESTAMP);" + \ "CREATE OR REPLACE PUMP \"STREAM_PUMP\" AS INSERT INTO \"DESTINATION_SQL_STREAM_BY_STORE\"" + \ "SELECT STREAM \"region\", \"state\", \"store-id\", SUM(\"kpi-1\") AS kpi_1_sum, SUM(\"kpi-2\") AS kpi_2_sum, FLOOR(\"SOURCE_SQL_STREAM_001\".APPROXIMATE_ARRIVAL_TIME TO MINUTE) as ingest_time" + \ "FROM \"SOURCE_SQL_STREAM_001\"" + \ "GROUP BY \"region\", \"state\", \"store-id\", FLOOR(\"SOURCE_SQL_STREAM_001\".APPROXIMATE_ARRIVAL_TIME TO MINUTE), FLOOR((\"SOURCE_SQL_STREAM_001\".ROWTIME - TIMESTAMP '1970-01-01 00:00:00') SECOND / 10 TO SECOND);" + \ "CREATE OR REPLACE PUMP \"STREAM_PUMP\" AS INSERT INTO \"DESTINATION_SQL_STREAM_BY_STATE\"" + \ "SELECT STREAM \"region\", \"state\", SUM(\"kpi-1\") AS kpi_1_sum, SUM(\"kpi-2\") AS kpi_2_sum, FLOOR(\"SOURCE_SQL_STREAM_001\".APPROXIMATE_ARRIVAL_TIME TO MINUTE) as ingest_time" + \ "FROM \"SOURCE_SQL_STREAM_001\"" + \ "GROUP BY \"region\", \"state\", FLOOR(\"SOURCE_SQL_STREAM_001\".APPROXIMATE_ARRIVAL_TIME TO MINUTE), FLOOR((\"SOURCE_SQL_STREAM_001\".ROWTIME - TIMESTAMP '1970-01-01 00:00:00') SECOND / 10 TO SECOND);" + \ "CREATE OR REPLACE PUMP \"STREAM_PUMP\" AS INSERT INTO \"DESTINATION_SQL_STREAM_BY_REGION\"" + \ "SELECT STREAM \"region\", SUM(\"kpi-1\") AS kpi_1_sum, SUM(\"kpi-2\") AS kpi_2_sum, FLOOR(\"SOURCE_SQL_STREAM_001\".APPROXIMATE_ARRIVAL_TIME TO MINUTE) as ingest_time" + \ "FROM \"SOURCE_SQL_STREAM_001\"" + \ "GROUP BY \"region\", FLOOR(\"SOURCE_SQL_STREAM_001\".APPROXIMATE_ARRIVAL_TIME TO MINUTE), FLOOR((\"SOURCE_SQL_STREAM_001\".ROWTIME - TIMESTAMP '1970-01-01 00:00:00') SECOND / 10 TO SECOND);" kda_app = aws_kinesisanalytics.CfnApplication(self, "kda_agg", inputs=[ip], #kda_inputs, application_code=application_code, application_description="Aggregating data", application_name="DashboardMetricsAggregator" ) kda_output_prop = aws_kinesisanalytics.CfnApplicationOutput.KinesisStreamsOutputProperty( resource_arn=kds_output_stream.stream_arn, role_arn=kda_service_role.role_arn ) kda_dest_schema = aws_kinesisanalytics.CfnApplicationOutput.DestinationSchemaProperty( record_format_type="JSON" ) kda_output_prop_by_store = aws_kinesisanalytics.CfnApplicationOutput.OutputProperty( destination_schema=kda_dest_schema, kinesis_streams_output=kda_output_prop, name="DESTINATION_SQL_STREAM_BY_STORE" ) kda_output_prop_by_state = aws_kinesisanalytics.CfnApplicationOutput.OutputProperty( destination_schema=kda_dest_schema, kinesis_streams_output=kda_output_prop, name="DESTINATION_SQL_STREAM_BY_STATE" ) kda_output_prop_by_region = aws_kinesisanalytics.CfnApplicationOutput.OutputProperty( destination_schema=kda_dest_schema, kinesis_streams_output=kda_output_prop, name="DESTINATION_SQL_STREAM_BY_REGION" ) kda_app_output_prop = aws_kinesisanalytics.CfnApplicationOutput(self, "kda_agg_output_store", application_name="DashboardMetricsAggregator", output=kda_output_prop_by_store ) kda_app_output_prop = aws_kinesisanalytics.CfnApplicationOutput(self, "kda_agg_output_state", application_name="DashboardMetricsAggregator", output=kda_output_prop_by_state ) kda_app_output_prop = aws_kinesisanalytics.CfnApplicationOutput(self, "kda_agg_output_region", application_name="DashboardMetricsAggregator", output=kda_output_prop_by_region ) lambda_agg_function = aws_lambda.Function(self, "AggDataLambda", runtime=aws_lambda.Runtime.PYTHON_3_7, handler="lambda_function.lambda_handler", code=aws_lambda.Code.asset("../models/dashboard/lambdas/aggregate_data_lambda"), timeout=Duration.minutes(5)) lambda_agg_function.add_environment("DDB_TABLE_DASHBOARD", table.table_name) lambda_agg_function.add_to_role_policy(aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, actions=[ "kinesis:*" ], resources=["*"] )) table.grant_read_write_data(lambda_agg_function) kes = aws_lambda_event_sources.KinesisEventSource(kds_output_stream, starting_position=aws_lambda.StartingPosition.TRIM_HORIZON, batch_size=50, #max_batching_window=100 ) lambda_agg_function.add_event_source(kes) core.CfnOutput( self, "TableName_Dashboard", description="Table name for Dashboard", value=table.table_name ) core.CfnOutput( self, "BucketName_Dashboard", description="Bucket name", value=ingest_bucket.bucket_arn ) core.CfnOutput( self, "KinesisInputStream_Dashboard", description="Kinesis input for Dashboard", value=kds_input_stream.stream_name ) core.CfnOutput( self, "KinesisOutputStream_Dashboard", description="Kinesis output for Dashboard", value=kds_output_stream.stream_name )