def __init__(self, scope: Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # create new IAM group and user group = iam.Group(self, "RekGroup") user = iam.User(self, "RekUser") # add IAM user to the new group user.add_to_group(group) # create S3 bucket to hold images # give new user access to the bucket bucket = s3.Bucket(self, 'Bucket') bucket.grant_read_write(user) # create DynamoDB table to hold Rekognition results table = ddb.Table(self, 'Classifications', partition_key={ 'name': 'image_name', 'type': ddb.AttributeType.STRING }) # create Lambda function lambda_function = _lambda.Function( self, 'RekFunction', runtime=_lambda.Runtime.PYTHON_3_8, handler='rekfunction.handler', code=_lambda.Code.from_asset( 'rekognition_lambda_s3_trigger/lambda'), environment={ 'BUCKET_NAME': bucket.bucket_name, 'TABLE_NAME': table.table_name }) # add Rekognition permissions for Lambda function statement = iam.PolicyStatement() statement.add_actions("rekognition:DetectLabels") statement.add_resources("*") lambda_function.add_to_role_policy(statement) # create trigger for Lambda function with image type suffixes notification = s3_notifications.LambdaDestination(lambda_function) notification.bind(self, bucket) bucket.add_object_created_notification( notification, s3.NotificationKeyFilter(suffix='.jpg')) bucket.add_object_created_notification( notification, s3.NotificationKeyFilter(suffix='.jpeg')) bucket.add_object_created_notification( notification, s3.NotificationKeyFilter(suffix='.png')) # grant permissions for lambda to read/write to DynamoDB table and bucket table.grant_read_write_data(lambda_function) bucket.grant_read_write(lambda_function)
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) shoe_company_ingestion_bucket_name = 'devassoc-shoe-company-ingestion' shoe_company_ingestion_bucket = s3.Bucket( self, 'shoe-company-ingestion-bucket', bucket_name=shoe_company_ingestion_bucket_name, access_control=s3.BucketAccessControl.PRIVATE, removal_policy=core.RemovalPolicy.DESTROY, auto_delete_objects=True) core.CfnOutput(self, 'new-ingestion-bucket', value=shoe_company_ingestion_bucket.bucket_name) shoe_company_json_bucket_name = 'devassoc-shoe-company-json' shoe_company_json_bucket = s3.Bucket( self, 'shoe-company-json-bucket', bucket_name=shoe_company_json_bucket_name, access_control=s3.BucketAccessControl.PRIVATE, removal_policy=core.RemovalPolicy.DESTROY, auto_delete_objects=True) core.CfnOutput(self, 'new-json-bucket', value=shoe_company_json_bucket.bucket_name) lambda_role = iam.Role( self, 'lambda-role', role_name='PayrollProcessingLambdaRole', description= 'Provides lambda with access to s3 and cloudwatch to execute payroll processing', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com')) lambda_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name('AWSLambdaExecute')) this_dir = path.dirname(__file__) conversion_function = lam.Function( self, 'conversion-function', function_name='PayrollProcessing', runtime=lam.Runtime.PYTHON_3_7, handler='conversion.lambda_handler', code=lam.Code.from_asset(path.join(this_dir, 'lambda')), role=lambda_role, description= 'Converts payroll csvs to json and puts results in s3 bucket', timeout=core.Duration.minutes(3), memory_size=128) conversion_function.add_permission( 'lambdas3permission', principal=iam.ServicePrincipal('s3.amazonaws.com'), action='lambda:InvokeFunction', source_arn=shoe_company_ingestion_bucket.bucket_arn, source_account=kwargs.get('env')['account']) shoe_company_ingestion_bucket.add_object_created_notification( s3n.LambdaDestination(conversion_function), s3.NotificationKeyFilter(suffix='.csv'))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here lambdaFn = _lambda.Function(self, "SampleLambdaFunction", code=_lambda.Code.from_asset('function/'), runtime=_lambda.Runtime.PYTHON_3_7, handler="index.lambda_handler", function_name="sample_lambda_function" ) # 環境変数を追加 lambdaFn.add_environment(key="STAGE", value="DEV") # s3バケットを作成し、通知イベントを設定 bucket = _s3.Bucket(self, "SampleBucket", bucket_name="kimi-first-cdk-bucket") notification = aws_s3_notifications.LambdaDestination(lambdaFn) bucket.add_event_notification(_s3.EventType.OBJECT_CREATED, notification, _s3.NotificationKeyFilter(prefix="hoge", suffix=".csv")) # 定期的に実行するイベントを設定 rule = _events.Rule(self, "SampleEventRule", rule_name="schedule_trigger_event", schedule=_events.Schedule.expression("cron(10 * * * ? *)") ) rule.add_target(_targets.LambdaFunction(lambdaFn))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here lambda_layer = aws_lambda.LayerVersion( self, "pillow_layer", code=aws_lambda.Code.asset("./python-pillow-6.2.1.zip"), compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_7], description="Pillow_upload_from_cdk") # aws_lambda.LayerVersion.from_layer_version_arn(self, "pillow_layer", # layer_version_arn="arn:aws-cn:lambda:cn-northwest-1:313497334385:layer:pillow:4" # ) # The code that defines your stack goes here mylambda = aws_lambda.Function( self, "myfunction_id", description="lambda trigger by S3 to convert image", # function_name="img-process-cdk-deploy", # the name will be auto create code=aws_lambda.Code.asset("./lambda"), handler="lambda_function.lambda_handler", runtime=aws_lambda.Runtime.PYTHON_3_7, timeout=core.Duration.seconds(60), memory_size=256, reserved_concurrent_executions=10, layers=[lambda_layer], dead_letter_queue_enabled=True, # cdk will create a new SQS for this # role = aws_iam.Role.from_role_arn(self, "mylambda_role", # role_arn="arn:aws-cn:iam::313497334385:role/Lambda-access-img-process-S3bucket" # ) # cdk will create a role for this ) with open('./env.js', 'r') as f: env = json.load(f) for (k, v) in env.items(): mylambda.add_environment(k, v) #create a new bucket, Import bucket can not add event trigger s3_bucket = aws_s3.Bucket( self, "mybucket", # bucket name will be auto created or define here # bucket_name="img-process-cdk-created" ) mylambda.add_event_source( aws_lambda_event_sources.S3EventSource( s3_bucket, events=[aws_s3.EventType.OBJECT_CREATED], filters=[aws_s3.NotificationKeyFilter(prefix='input/')])) s3_bucket.grant_read_write(mylambda)
def __init__(self, scope: core.Construct, id: str, sqsCfn, s3_loc_up, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.PREFIX = id self.sqs_queue = sqs.Queue.from_queue_arn( self, "QueuefromCfn", f"arn:aws:sqs:us-east-1:{core.Aws.ACCOUNT_ID}:{sqsCfn.queue_name}") s3_loc_up.add_object_created_notification( aws_s3_notifications.SqsDestination(self.sqs_queue), _s3.NotificationKeyFilter( prefix='stdized-data/comprehend_results/csv/', suffix='.csv'))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) #This will create the s3 bucket in AWS bucket = s3.Bucket(self, "ssl_s3_bucket_raw") #This will create the sqs in AWS queue = sqs.Queue(self, "ssl_sqs_event_queue") #Create S3 notification object which points to SQS. notification = aws_s3_notifications.SqsDestination(queue) filter1 = s3.NotificationKeyFilter(prefix="home/") #Attach notificaton event to S3 bucket. bucket.add_event_notification(s3.EventType.OBJECT_CREATED, notification, filter1)
def __init__( self, scope: core.Construct, construct_id: str, elastic_domain: aes.Domain, **kwargs, ) -> None: super().__init__(scope, construct_id, **kwargs) indexing_lambda = _lambda.Function( self, "IndexingHandler", runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(LAMBDA_PATH), handler="lambda_function.lambda_handler", environment={ "EMBEDDER_IP": config.get_embedder_ip(), "ES_URL": elastic_domain.domain_endpoint, "ES_USER": config.get_es_credentials()[0], "ES_PASSWORD": config.get_es_credentials()[1], "INDEX_NAME": config.get_es_index(), }, ) notification = s3n.LambdaDestination(indexing_lambda) block_public_access = s3.BlockPublicAccess( block_public_acls=True, block_public_policy=True, ignore_public_acls=True, restrict_public_buckets=True) bucket = s3.Bucket(self, "DocsDestination", block_public_access=block_public_access, removal_policy=core.RemovalPolicy.DESTROY) bucket.grant_read(indexing_lambda) bucket.add_event_notification( s3.EventType.OBJECT_CREATED, notification, s3.NotificationKeyFilter(prefix="wikipages/"), ) core.Tags.of(indexing_lambda).add("system-id", config.get_system_id()) core.Tags.of(bucket).add("system-id", config.get_system_id()) core.CfnOutput(self, "S3BucketName", value=bucket.bucket_name) core.CfnOutput(self, "IndexingLambdaName", value=indexing_lambda.function_name)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Create Primary S3 Bucket main_bucket = aws_s3.Bucket(self, "multipagepdfa2i", removal_policy=core.RemovalPolicy.DESTROY) # Create sqs queue page_sqs = aws_sqs.Queue( self, "multipagepdfa2i_page_sqs", queue_name = "multipagepdfa2i_page_sqs", visibility_timeout=core.Duration.minutes(3) ) # Create all of the Lambda Functions lambda_functions = self.create_lambda_functions(page_sqs) # Create notification that triggers kick off lambda on pdf being uploaded to kickoff kickoff_notification = aws_s3_notifications.LambdaDestination(lambda_functions["kickoff"]) lambda_functions["analyzepdf"].add_event_source(aws_lambda_event_sources.SqsEventSource(page_sqs, batch_size=3)) main_bucket.add_event_notification( aws_s3.EventType.OBJECT_CREATED, kickoff_notification, aws_s3.NotificationKeyFilter(prefix="uploads/", suffix="pdf") ) self.configure_dynamo_table("multia2ipdf_callback", "jobid", "callback_token") self.configure_dynamo_table("multipagepdfa2i_upload_ids", "id", "key") self.create_state_machine(lambda_functions, page_sqs) human_complete_target = aws_events_targets.LambdaFunction(lambda_functions["humancomplete"]) human_review_event_pattern = aws_events.EventPattern( source=["aws.sagemaker"], detail_type=["SageMaker A2I HumanLoop Status Change"] ) aws_events.Rule(self, "multipadepdfa2i_HumanReviewComplete", event_pattern=human_review_event_pattern, targets=[human_complete_target] )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Define S3 bucket my_bucket = s3.Bucket(self, "ssl-s3-sns-event-raw") #Add Filters if required filter1 = s3.NotificationKeyFilter(prefix="home/") #sns Topic my_sns = sns.Topic(self, id="my-sns-topic", display_name="my-sns-topic") #Create the s3 notification objects which points to Lambda notification = notifications.SnsDestination(my_sns) #link s3 and sns my_bucket.add_event_notification(s3.EventType.OBJECT_CREATED, notification, filter1) #create sqs queue my_sqs = sqs.Queue(self, id="my-queue") #create sqs / sns subcription subscription = aws_sns_subscriptions.SqsSubscription(my_sqs) #add subscription to sns. my_sns.add_subscription(subscription) #create lambda function my_lambda = _lambda.Function(self, "HelloHandler", runtime=_lambda.Runtime.PYTHON_3_7, handler="hello.handler", code=_lambda.Code.asset('lambda')) #create sns/lambda subscription subscription = aws_sns_subscriptions.LambdaSubscription(my_lambda) #add lambda subscription to sns my_sns.add_subscription(subscription)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) #Create Role S3SqsKmsSampleStack.__Role = IamService.create_role(self) #get KMS policy Document kms_policy_document = IamService.get_kms_policy_documents(self) kms_key = kms.Key(self, id='ssl_s3_sqs_kms_key', alias='sslS3SqsKmsKey', description='This is kms key', enabled=True, enable_key_rotation=True, policy=kms_policy_document) #This will create the s3 bucket in AWS bucket = s3.Bucket(self, "ssl_s3_bucket_raw_kms", bucket_name="ssl-s3-bucket-kms-raw", encryption=s3.BucketEncryption.KMS, encryption_key=kms_key) #This will create the sqs in AWS queue = sqs.Queue(self, "ssl_sqs_event_queue", queue_name="ssl-sqs-kms-event-queue", encryption=sqs.QueueEncryption.KMS, encryption_master_key=kms_key) #queue.node.add_dependency(kms_key) bucket.node.add_dependency(queue, kms_key) # #Create S3 notification object which points to SQS. notification = aws_s3_notifications.SqsDestination(queue) filter1 = s3.NotificationKeyFilter(prefix="home/") # #Attach notificaton event to S3 bucket. bucket.add_event_notification(s3.EventType.OBJECT_CREATED, notification, filter1)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Define aws lambda function my_lambda = _lambda.Function(self, "HelloHandler", runtime=_lambda.Runtime.PYTHON_3_7, handler="hello.handler", code=_lambda.Code.asset('lambda')) # Define S3 bucket my_bucket = s3.Bucket(self, "ssl-s3-lambda-event-raw") #Create the s3 notification objects which points to Lambda notification = aws_s3_notifications.LambdaDestination(my_lambda) #Add Filters if required filter1 = s3.NotificationKeyFilter(prefix="home/") #Add event trigger from s3 to lambda my_bucket.add_event_notification(s3.EventType.OBJECT_CREATED, notification, filter1)
def create_events(self, services): # kickoff_notification = aws_s3_notifications.LambdaDestination(services["lambda"]["kickoff"]) extensions = [ "pdf", "pDf", "pDF", "pdF", "PDF", "Pdf", "png", "pNg", "pNG", "pnG", "PNG", "Png", "jpg", "jPg", "jPG", "jpG", "JPG", "Jpg" ] for extension in extensions: services["main_s3_bucket"].add_event_notification( aws_s3.EventType.OBJECT_CREATED, aws_s3_notifications.SqsDestination(services["sf_sqs"]), aws_s3.NotificationKeyFilter(prefix="uploads/", suffix=extension) ) services["lambda"]["kickoff"].add_event_source( aws_lambda_event_sources.SqsEventSource( services["sf_sqs"], batch_size=1 ) ) services["lambda"]["analyzepdf"].add_event_source( aws_lambda_event_sources.SqsEventSource( services["textract_sqs"], batch_size=1 ) ) human_complete_target = aws_events_targets.LambdaFunction(services["lambda"]["humancomplete"]) human_review_event_pattern = aws_events.EventPattern( source=["aws.sagemaker"], detail_type=["SageMaker A2I HumanLoop Status Change"] ) aws_events.Rule(self, "multipadepdfa2i_HumanReviewComplete", event_pattern=human_review_event_pattern, targets=[human_complete_target] )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Create the S3 bucket my_bucket = s3.Bucket(self,"my_s3_bucket_raw") #create dynamo db table self._table = ddb.Table( self, 'Employee', partition_key={'name': 'Emp_Id', 'type': ddb.AttributeType.NUMBER} ) #Create lambda function my_lambda = _lambda.Function( self, id="EventFunction", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset("lambda"), handler="event.handler" , environment={ "Table_Name":self._table.table_name } ) #Add Filters filter1=s3.NotificationKeyFilter(prefix="home/",suffix=".json") #Create Notification s3_lambda_notification = notification.LambdaDestination(my_lambda) #link s3 and lambda my_bucket.add_event_notification(s3.EventType.OBJECT_CREATED,s3_lambda_notification,filter1) #grant lambda read/write access to lambda function self._table.grant_read_write_data(my_lambda) _lambda.IFunction.grant_invoke(self,my_lambda) #Grant s3 read access to lambda function my_bucket.grant_read(my_lambda)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # 创建sts lambda访问角色 # action -> statement -> policy -> role -> attach lambda actions = [ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", "sts:AssumeRole" ] policyStatement = PolicyStatement(actions=actions, effect=Effect.ALLOW) policyStatement.add_all_resources() policy_name = "{}-policy".format(Constant.PROJECT_NAME) sts_policy = Policy(self, policy_name, policy_name=policy_name) sts_policy.add_statements(policyStatement) role_name = "{}-role".format(Constant.PROJECT_NAME) access_role = Role(self, role_name, role_name=role_name, assumed_by=ServicePrincipal('lambda.amazonaws.com')) sts_policy.attach_to_role(access_role) # 创建thum lambda访问角色 # action -> statement -> policy -> role thum_actions = [ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", "s3:PutObject" ] thum_policyStatement = PolicyStatement(actions=thum_actions, effect=Effect.ALLOW) thum_policyStatement.add_all_resources() thum_policy_name = "{}-policy-thum".format(Constant.PROJECT_NAME) thum_policy = Policy(self, thum_policy_name, policy_name=thum_policy_name) thum_policy.add_statements(thum_policyStatement) thum_role_name = "{}-role-thum".format(Constant.PROJECT_NAME) thum_access_role = Role( self, thum_role_name, role_name=thum_role_name, assumed_by=ServicePrincipal('lambda.amazonaws.com')) thum_policy.attach_to_role(thum_access_role) # 创建S3 put的角色 # action -> statement -> policy -> role s3_actions = [ "s3:PutObject", "s3:GetObject", "s3:ListBucket", "s3:PutObjectTagging", "s3:PutObjectAcl", ] s3_policyStatement = PolicyStatement(actions=s3_actions, effect=Effect.ALLOW) s3_policyStatement.add_all_resources() s3_policy_name = "{}-policy-s3".format(Constant.PROJECT_NAME) s3_policy = Policy(self, s3_policy_name, policy_name=s3_policy_name) s3_policy.add_statements(s3_policyStatement) s3_role_name = "{}-role-s3".format(Constant.PROJECT_NAME) s3_access_role = Role(self, s3_role_name, role_name=s3_role_name, assumed_by=ArnPrincipal(access_role.role_arn)) s3_policy.attach_to_role(s3_access_role) # 创建STS lambda sts_lambda = _lambda.Function( self, 'sts', function_name='sts', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset('lambda'), handler='auth.handler', timeout=Duration.minutes(1), role=access_role, ) sts_lambda.add_environment("role_to_assume_arn", s3_access_role.role_arn) base_api = apigw.RestApi( self, 'Endpoint', endpoint_types=[EndpointType.REGIONAL], ) example_entity = base_api.root.add_resource('auth') example_entity_lambda_integration = apigw.LambdaIntegration( sts_lambda, proxy=False, integration_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': "'*'", } }]) example_entity.add_method( 'GET', example_entity_lambda_integration, method_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': True, } }]) self.add_cors_options(example_entity) # 创建缩略图lambda layer_cv2 = _lambda.LayerVersion( self, 'cv2', code=_lambda.Code.from_bucket( s3.Bucket.from_bucket_name(self, "cdk-data-layer", bucket_name='nowfox'), 'cdk-data/cv2.zip'), compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], ) lambda_thum = _lambda.Function( self, 'thum', function_name='thum', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset('lambda'), handler='thum.handler', timeout=Duration.minutes(1), role=thum_access_role, ) lambda_thum.add_environment("frame_second", "3") lambda_thum.add_layers(layer_cv2) # 创建存储上传图片的bucket s3_bucket_name = "{}-{}".format("upload", self._get_UUID(4)) s3_upload = s3.Bucket( self, id=s3_bucket_name, bucket_name=s3_bucket_name, # access_control=s3.BucketAccessControl.PUBLIC_READ,#不建议使用这个,这个会有list权限,下面这个没有list权限 public_read_access=True, removal_policy=core.RemovalPolicy. DESTROY, # TODO: destroy for test # removal_policy=core.RemovalPolicy.RETAIN ) notification = aws_s3_notifications.LambdaDestination(lambda_thum) s3_filter = s3.NotificationKeyFilter(suffix=".mp4") s3_upload.add_event_notification(s3.EventType.OBJECT_CREATED_PUT, notification, s3_filter) s3_upload.add_cors_rule(allowed_methods=[ HttpMethods.POST, HttpMethods.PUT, HttpMethods.GET ], allowed_origins=["*"], allowed_headers=["*"]) ''' #创建上传lambda lambda_upload = _lambda.Function( self, 'upload',function_name='upload', runtime=_lambda.Runtime.JAVA_8, code=_lambda.Code.from_bucket(s3.Bucket.from_bucket_name(self, "cdk-data-upload-jar", bucket_name='nowfox'), 'cdk-data/fileupload.jar'), handler='fileupload.FileUploadFunctionHandler::handleRequest', timeout=Duration.minutes(1), memory_size=512, role=access_role, ) lambda_upload.add_environment("bucket", s3_bucket_name) lambda_upload.add_environment("region", "cn-northwest-1") ''' core.CfnOutput(self, "authUrl", value=base_api.url + "auth", description="authUrl") core.CfnOutput(self, "S3BucketName", value=s3_bucket_name, description="S3BucketName")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # ------ Necessary Roles ------ roles = IamRole( self, 'IamRoles' ) # ------ S3 Buckets ------ # Create Athena bucket athena_bucket = _s3.Bucket(self, "AthenaBucket", removal_policy=core.RemovalPolicy.DESTROY ) # Create Forecast bucket forecast_bucket = _s3.Bucket(self, "FoecastBucket", removal_policy=core.RemovalPolicy.DESTROY ) # ------ Athena ------ # Config Athena query result output location workgroup_prop = _athena.CfnWorkGroup.WorkGroupConfigurationProperty( result_configuration=_athena.CfnWorkGroup.ResultConfigurationProperty( output_location="s3://"+athena_bucket.bucket_name ) ) # Create Athena workgroup athena_workgroup = _athena.CfnWorkGroup( self, 'ForecastGroup', name='ForecastGroup', recursive_delete_option=True, state='ENABLED', work_group_configuration=workgroup_prop ) # ------ SNS Topic ------ topic = sns.Topic( self, 'NotificationTopic', display_name='StepsTopic' ) # SNS email subscription. Get the email address from context value(cdk.json) topic.add_subscription(subs.EmailSubscription(self.node.try_get_context('my_email'))) # ------ Layers ------ shared_layer = _lambda.LayerVersion( self, 'LambdaLayer', layer_version_name='testfolderlayer', code=_lambda.AssetCode('shared/') ) # ------ Lambdas for stepfuctions------ create_dataset_lambda = _lambda.Function( self, 'CreateDataset', function_name='CreateDataset', code=_lambda.Code.asset('lambdas/createdataset/'), handler='dataset.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, timeout=core.Duration.seconds(30), layers=[shared_layer] ) create_dataset_group_lambda = _lambda.Function( self, 'CreateDatasetGroup', function_name='CreateDatasetGroup', code = _lambda.Code.asset('lambdas/createdatasetgroup/'), handler = 'datasetgroup.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) import_data_lambda = _lambda.Function( self, 'CreateDatasetImportJob', function_name='CreateDatasetImportJob', code = _lambda.Code.asset('lambdas/createdatasetimportjob/'), handler = 'datasetimport.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, environment= { 'FORECAST_ROLE': roles.forecast_role.role_arn }, layers=[shared_layer] ) create_predictor_lambda = _lambda.Function( self, 'CreatePredictor', function_name='CreatePredictor', code = _lambda.Code.asset('lambdas/createpredictor/'), handler = 'predictor.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) create_forecast_lambda = _lambda.Function( self, 'CreateForecast', function_name='CreateForecast', code = _lambda.Code.asset('lambdas/createforecast/'), handler = 'forecast.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, environment= { 'EXPORT_ROLE': roles.forecast_role.role_arn }, layers=[shared_layer], timeout=core.Duration.seconds(30) ) # Deploy lambda with python dependencies from requirements.txt update_resources_lambda = _lambda_python.PythonFunction( self, 'UpdateResources', function_name='UpdateResources', entry='lambdas/updateresources/', index='update.py', handler='lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.update_role, environment= { 'ATHENA_WORKGROUP': athena_workgroup.name, 'ATHENA_BUCKET' : athena_bucket.bucket_name }, layers=[shared_layer], timeout=core.Duration.seconds(900) ) notify_lambda = _lambda.Function( self, 'NotifyTopic', function_name='NotifyTopic', code = _lambda.Code.asset('lambdas/notify/'), handler = 'notify.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, environment= { 'SNS_TOPIC_ARN': topic.topic_arn }, layers=[shared_layer] ) delete_forecast_lambda = _lambda.Function( self, 'DeleteForecast', function_name='DeleteForecast', code = _lambda.Code.asset('lambdas/deleteforecast/'), handler = 'deleteforecast.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) delete_predctor_lambda = _lambda.Function( self, 'DeletePredictor', function_name='DeletePredictor', code = _lambda.Code.asset('lambdas/deletepredictor/'), handler = 'deletepredictor.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) delete_importjob_lambda = _lambda.Function( self, 'DeleteImportJob', function_name='DeleteImportJob', code = _lambda.Code.asset('lambdas/deletedatasetimport/'), handler = 'deletedataset.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) # ------ StepFunctions ------ strategy_choice = sfn.Choice( self, 'Strategy-Choice' ) success_state = sfn.Succeed( self, 'SuccessState' ) failed = sfn_tasks.LambdaInvoke( self, 'Failed', lambda_function = notify_lambda, result_path=None ).next(strategy_choice) create_dataset_job = sfn_tasks.LambdaInvoke( self, 'Create-Dataset', lambda_function = create_dataset_lambda, retry_on_service_exceptions=True, payload_response_only=True ) self.add_retry_n_catch(create_dataset_job, failed) create_dataset_group_job = sfn_tasks.LambdaInvoke( self, 'Create-DatasetGroup', lambda_function = create_dataset_group_lambda, payload_response_only=True ) self.add_retry_n_catch(create_dataset_group_job, failed) import_data_job = sfn_tasks.LambdaInvoke( self, 'Import-Data', lambda_function = import_data_lambda, payload_response_only=True ) self.add_retry_n_catch(import_data_job, failed) create_predictor_job = sfn_tasks.LambdaInvoke( self, 'Create-Predictor', lambda_function = create_predictor_lambda, payload_response_only=True ) self.add_retry_n_catch(create_predictor_job, failed) create_forecast_job = sfn_tasks.LambdaInvoke( self, 'Create-Forecast', lambda_function = create_forecast_lambda, payload_response_only=True ) self.add_retry_n_catch(create_forecast_job, failed) update_resources_job = sfn_tasks.LambdaInvoke( self, 'Update-Resources', lambda_function = update_resources_lambda, payload_response_only=True ) self.add_retry_n_catch(update_resources_job, failed) notify_success = sfn_tasks.LambdaInvoke( self, 'Notify-Success', lambda_function = notify_lambda, payload_response_only=True ) delete_forecast_job = sfn_tasks.LambdaInvoke( self, 'Delete-Forecast', lambda_function = delete_forecast_lambda, payload_response_only=True ) self.delete_retry(delete_forecast_job) delete_predictor_job = sfn_tasks.LambdaInvoke( self, 'Delete-Predictor', lambda_function = delete_predctor_lambda, payload_response_only=True ) self.delete_retry(delete_predictor_job) delete_import_job = sfn_tasks.LambdaInvoke( self, 'Delete-ImportJob', lambda_function = delete_importjob_lambda, payload_response_only=True ) self.delete_retry(delete_import_job) definition = create_dataset_job\ .next(create_dataset_group_job)\ .next(import_data_job)\ .next(create_predictor_job)\ .next(create_forecast_job)\ .next(update_resources_job)\ .next(notify_success)\ .next(strategy_choice.when(sfn.Condition.boolean_equals('$.params.PerformDelete', False), success_state)\ .otherwise(delete_forecast_job).afterwards())\ .next(delete_predictor_job)\ .next(delete_import_job) deployt_state_machine = sfn.StateMachine( self, 'StateMachine', definition = definition # role=roles.states_execution_role ) # S3 event trigger lambda s3_lambda = _lambda.Function( self, 'S3Lambda', function_name='S3Lambda', code=_lambda.Code.asset('lambdas/s3lambda/'), handler='parse.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, role=roles.trigger_role, environment= { 'STEP_FUNCTIONS_ARN': deployt_state_machine.state_machine_arn, 'PARAMS_FILE': self.node.try_get_context('parameter_file') } ) s3_lambda.add_event_source( event_src.S3EventSource( bucket=forecast_bucket, events=[_s3.EventType.OBJECT_CREATED], filters=[_s3.NotificationKeyFilter( prefix='train/', suffix='.csv' )] ) ) # CloudFormation output core.CfnOutput( self, 'StepFunctionsName', description='Step Functions Name', value=deployt_state_machine.state_machine_name ) core.CfnOutput( self, 'ForecastBucketName', description='Forecast bucket name to drop you files', value=forecast_bucket.bucket_name ) core.CfnOutput( self, 'AthenaBucketName', description='Athena bucket name to drop your files', value=athena_bucket.bucket_name )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) if self.node.try_get_context('vpc_type'): validate_cdk_json(self) ES_LOADER_TIMEOUT = 600 ###################################################################### # REGION mapping / ELB & Lambda Arch ###################################################################### elb_id_temp = region_info.FactName.ELBV2_ACCOUNT elb_map_temp = region_info.RegionInfo.region_map(elb_id_temp) region_dict = {} for region in elb_map_temp: # ELB account ID region_dict[region] = {'ElbV2AccountId': elb_map_temp[region]} # Lambda Arch if region in ('us-east-1', 'us-east-2', 'us-west-2', 'ap-south-1', 'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1', 'eu-central-1', 'eu-west-1', 'eu-west-2'): region_dict[region]['LambdaArch'] = ( aws_lambda.Architecture.ARM_64.name) else: region_dict[region]['LambdaArch'] = ( aws_lambda.Architecture.X86_64.name) region_mapping = core.CfnMapping( scope=self, id='RegionMap', mapping=region_dict) ###################################################################### # get params ###################################################################### allow_source_address = core.CfnParameter( self, 'AllowedSourceIpAddresses', allowed_pattern=r'^[0-9./\s]*', description='Space-delimited list of CIDR blocks', default='10.0.0.0/8 172.16.0.0/12 192.168.0.0/16') sns_email = core.CfnParameter( self, 'SnsEmail', allowed_pattern=r'^[0-9a-zA-Z@_\-\+\.]*', description=('Input your email as SNS topic, where Amazon ' 'OpenSearch Service will send alerts to'), default='*****@*****.**') geoip_license_key = core.CfnParameter( self, 'GeoLite2LicenseKey', allowed_pattern=r'^[0-9a-zA-Z]{16}$', default='xxxxxxxxxxxxxxxx', description=("If you wolud like to enrich geoip locaiton such as " "IP address's country, get a license key form MaxMind" " and input the key. If you not, keep " "xxxxxxxxxxxxxxxx")) reserved_concurrency = core.CfnParameter( self, 'ReservedConcurrency', default=10, type='Number', description=('Input reserved concurrency. Increase this value if ' 'there are steady logs delay despite no errors')) aes_domain_name = self.node.try_get_context('aes_domain_name') bucket = f'{aes_domain_name}-{core.Aws.ACCOUNT_ID}' s3bucket_name_geo = f'{bucket}-geo' s3bucket_name_log = f'{bucket}-log' s3bucket_name_snapshot = f'{bucket}-snapshot' # organizations / multiaccount org_id = self.node.try_get_context('organizations').get('org_id') org_mgmt_id = self.node.try_get_context( 'organizations').get('management_id') org_member_ids = self.node.try_get_context( 'organizations').get('member_ids') no_org_ids = self.node.try_get_context( 'no_organizations').get('aws_accounts') # Overwrite default S3 bucket name as customer name temp_geo = self.node.try_get_context('s3_bucket_name').get('geo') if temp_geo: s3bucket_name_geo = temp_geo else: print('Using default bucket names') temp_log = self.node.try_get_context('s3_bucket_name').get('log') if temp_log: s3bucket_name_log = temp_log elif org_id or no_org_ids: s3bucket_name_log = f'{aes_domain_name}-{self.account}-log' else: print('Using default bucket names') temp_snap = self.node.try_get_context('s3_bucket_name').get('snapshot') if temp_snap: s3bucket_name_snapshot = temp_snap else: print('Using default bucket names') kms_cmk_alias = self.node.try_get_context('kms_cmk_alias') if not kms_cmk_alias: kms_cmk_alias = 'aes-siem-key' print('Using default key alais') ###################################################################### # deploy VPC when context is defined as using VPC ###################################################################### # vpc_type is 'new' or 'import' or None vpc_type = self.node.try_get_context('vpc_type') if vpc_type == 'new': is_vpc = True vpc_cidr = self.node.try_get_context('new_vpc_nw_cidr_block') subnet_cidr_mask = int( self.node.try_get_context('new_vpc_subnet_cidr_mask')) is_vpc = True # VPC vpc_aes_siem = aws_ec2.Vpc( self, 'VpcAesSiem', cidr=vpc_cidr, max_azs=3, nat_gateways=0, subnet_configuration=[ aws_ec2.SubnetConfiguration( subnet_type=aws_ec2.SubnetType.ISOLATED, name='aes-siem-subnet', cidr_mask=subnet_cidr_mask)]) subnet1 = vpc_aes_siem.isolated_subnets[0] subnets = [{'subnet_type': aws_ec2.SubnetType.ISOLATED}] vpc_subnets = aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.ISOLATED) vpc_aes_siem_opt = vpc_aes_siem.node.default_child.cfn_options vpc_aes_siem_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN for subnet in vpc_aes_siem.isolated_subnets: subnet_opt = subnet.node.default_child.cfn_options subnet_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN elif vpc_type == 'import': vpc_id = self.node.try_get_context('imported_vpc_id') vpc_aes_siem = aws_ec2.Vpc.from_lookup( self, 'VpcAesSiem', vpc_id=vpc_id) subnet_ids = get_subnet_ids(self) subnets = [] for number, subnet_id in enumerate(subnet_ids, 1): obj_id = 'Subenet' + str(number) subnet = aws_ec2.Subnet.from_subnet_id(self, obj_id, subnet_id) subnets.append(subnet) subnet1 = subnets[0] vpc_subnets = aws_ec2.SubnetSelection(subnets=subnets) if vpc_type: is_vpc = True # Security Group sg_vpc_noinbound_aes_siem = aws_ec2.SecurityGroup( self, 'AesSiemVpcNoinboundSecurityGroup', security_group_name='aes-siem-noinbound-vpc-sg', vpc=vpc_aes_siem) sg_vpc_aes_siem = aws_ec2.SecurityGroup( self, 'AesSiemVpcSecurityGroup', security_group_name='aes-siem-vpc-sg', vpc=vpc_aes_siem) sg_vpc_aes_siem.add_ingress_rule( peer=aws_ec2.Peer.ipv4(vpc_aes_siem.vpc_cidr_block), connection=aws_ec2.Port.tcp(443),) sg_vpc_opt = sg_vpc_aes_siem.node.default_child.cfn_options sg_vpc_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN # VPC Endpoint vpc_aes_siem.add_gateway_endpoint( 'S3Endpoint', service=aws_ec2.GatewayVpcEndpointAwsService.S3, subnets=subnets) vpc_aes_siem.add_interface_endpoint( 'SQSEndpoint', security_groups=[sg_vpc_aes_siem], service=aws_ec2.InterfaceVpcEndpointAwsService.SQS,) vpc_aes_siem.add_interface_endpoint( 'KMSEndpoint', security_groups=[sg_vpc_aes_siem], service=aws_ec2.InterfaceVpcEndpointAwsService.KMS,) else: is_vpc = False is_vpc = core.CfnCondition( self, 'IsVpc', expression=core.Fn.condition_equals(is_vpc, True)) """ CloudFormation実行時の条件式の書き方 ClassのBasesが aws_cdk.core.Resource の時は、 node.default_child.cfn_options.condition = is_vpc ClassのBasesが aws_cdk.core.CfnResource の時は、 cfn_options.condition = is_vpc """ ###################################################################### # create cmk of KMS to encrypt S3 bucket ###################################################################### kms_aes_siem = aws_kms.Key( self, 'KmsAesSiemLog', description='CMK for SIEM solution', removal_policy=core.RemovalPolicy.RETAIN) aws_kms.Alias( self, 'KmsAesSiemLogAlias', alias_name=kms_cmk_alias, target_key=kms_aes_siem, removal_policy=core.RemovalPolicy.RETAIN) kms_aes_siem.add_to_resource_policy( aws_iam.PolicyStatement( sid='Allow GuardDuty to use the key', actions=['kms:GenerateDataKey'], principals=[aws_iam.ServicePrincipal( 'guardduty.amazonaws.com')], resources=['*'],),) kms_aes_siem.add_to_resource_policy( aws_iam.PolicyStatement( sid='Allow VPC Flow Logs to use the key', actions=['kms:Encrypt', 'kms:Decrypt', 'kms:ReEncrypt*', 'kms:GenerateDataKey*', 'kms:DescribeKey'], principals=[aws_iam.ServicePrincipal( 'delivery.logs.amazonaws.com')], resources=['*'],),) # basic policy key_policy_basic1 = aws_iam.PolicyStatement( sid='Allow principals in the account to decrypt log files', actions=['kms:DescribeKey', 'kms:ReEncryptFrom'], principals=[aws_iam.AccountPrincipal( account_id=core.Aws.ACCOUNT_ID)], resources=['*'],) kms_aes_siem.add_to_resource_policy(key_policy_basic1) # for Athena key_policy_athena = aws_iam.PolicyStatement( sid='Allow Athena to query s3 objects with this key', actions=['kms:Decrypt', 'kms:DescribeKey', 'kms:Encrypt', 'kms:GenerateDataKey*', 'kms:ReEncrypt*'], principals=[aws_iam.AccountPrincipal( account_id=core.Aws.ACCOUNT_ID)], resources=['*'], conditions={'ForAnyValue:StringEquals': { 'aws:CalledVia': 'athena.amazonaws.com'}}) kms_aes_siem.add_to_resource_policy(key_policy_athena) # for CloudTrail key_policy_trail1 = aws_iam.PolicyStatement( sid='Allow CloudTrail to describe key', actions=['kms:DescribeKey'], principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')], resources=['*'],) kms_aes_siem.add_to_resource_policy(key_policy_trail1) key_policy_trail2 = aws_iam.PolicyStatement( sid=('Allow CloudTrail to encrypt logs'), actions=['kms:GenerateDataKey*'], principals=[aws_iam.ServicePrincipal( 'cloudtrail.amazonaws.com')], resources=['*'], conditions={'StringLike': { 'kms:EncryptionContext:aws:cloudtrail:arn': [ f'arn:aws:cloudtrail:*:{core.Aws.ACCOUNT_ID}:trail/*']}}) kms_aes_siem.add_to_resource_policy(key_policy_trail2) ###################################################################### # create s3 bucket ###################################################################### block_pub = aws_s3.BlockPublicAccess( block_public_acls=True, ignore_public_acls=True, block_public_policy=True, restrict_public_buckets=True ) s3_geo = aws_s3.Bucket( self, 'S3BucketForGeoip', block_public_access=block_pub, bucket_name=s3bucket_name_geo, # removal_policy=core.RemovalPolicy.DESTROY, ) # create s3 bucket for log collector s3_log = aws_s3.Bucket( self, 'S3BucketForLog', block_public_access=block_pub, bucket_name=s3bucket_name_log, versioned=True, encryption=aws_s3.BucketEncryption.S3_MANAGED, # removal_policy=core.RemovalPolicy.DESTROY, ) # create s3 bucket for aes snapshot s3_snapshot = aws_s3.Bucket( self, 'S3BucketForSnapshot', block_public_access=block_pub, bucket_name=s3bucket_name_snapshot, # removal_policy=core.RemovalPolicy.DESTROY, ) ###################################################################### # IAM Role ###################################################################### # delopyment policy for lambda deploy-aes arn_prefix = f'arn:aws:logs:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}' loggroup_aes = f'log-group:/aws/aes/domains/{aes_domain_name}/*' loggroup_opensearch = ( f'log-group:/aws/OpenSearchService/domains/{aes_domain_name}/*') loggroup_lambda = 'log-group:/aws/lambda/aes-siem-*' policydoc_create_loggroup = aws_iam.PolicyDocument( statements=[ aws_iam.PolicyStatement( actions=[ 'logs:PutResourcePolicy', 'logs:DescribeLogGroups', 'logs:DescribeLogStreams' ], resources=[f'{arn_prefix}:*', ] ), aws_iam.PolicyStatement( actions=[ 'logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents', 'logs:PutRetentionPolicy'], resources=[ f'{arn_prefix}:{loggroup_aes}', f'{arn_prefix}:{loggroup_opensearch}', f'{arn_prefix}:{loggroup_lambda}', ], ) ] ) policydoc_crhelper = aws_iam.PolicyDocument( statements=[ aws_iam.PolicyStatement( actions=[ 'lambda:AddPermission', 'lambda:RemovePermission', 'events:ListRules', 'events:PutRule', 'events:DeleteRule', 'events:PutTargets', 'events:RemoveTargets'], resources=['*'] ) ] ) # snaphot rule for AES policydoc_snapshot = aws_iam.PolicyDocument( statements=[ aws_iam.PolicyStatement( actions=['s3:ListBucket'], resources=[s3_snapshot.bucket_arn] ), aws_iam.PolicyStatement( actions=['s3:GetObject', 's3:PutObject', 's3:DeleteObject'], resources=[s3_snapshot.bucket_arn + '/*'] ) ] ) aes_siem_snapshot_role = aws_iam.Role( self, 'AesSiemSnapshotRole', role_name='aes-siem-snapshot-role', inline_policies=[policydoc_snapshot, ], assumed_by=aws_iam.ServicePrincipal('es.amazonaws.com') ) policydoc_assume_snapshrole = aws_iam.PolicyDocument( statements=[ aws_iam.PolicyStatement( actions=['iam:PassRole'], resources=[aes_siem_snapshot_role.role_arn] ), ] ) aes_siem_deploy_role_for_lambda = aws_iam.Role( self, 'AesSiemDeployRoleForLambda', role_name='aes-siem-deploy-role-for-lambda', managed_policies=[ aws_iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonOpenSearchServiceFullAccess'), aws_iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaBasicExecutionRole'), ], inline_policies=[policydoc_assume_snapshrole, policydoc_snapshot, policydoc_create_loggroup, policydoc_crhelper], assumed_by=aws_iam.ServicePrincipal('lambda.amazonaws.com') ) if vpc_type: aes_siem_deploy_role_for_lambda.add_managed_policy( aws_iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaVPCAccessExecutionRole') ) # for alert from Amazon OpenSearch Service aes_siem_sns_role = aws_iam.Role( self, 'AesSiemSnsRole', role_name='aes-siem-sns-role', assumed_by=aws_iam.ServicePrincipal('es.amazonaws.com') ) # EC2 role aes_siem_es_loader_ec2_role = aws_iam.Role( self, 'AesSiemEsLoaderEC2Role', role_name='aes-siem-es-loader-for-ec2', assumed_by=aws_iam.ServicePrincipal('ec2.amazonaws.com'), ) aws_iam.CfnInstanceProfile( self, 'AesSiemEsLoaderEC2InstanceProfile', instance_profile_name=aes_siem_es_loader_ec2_role.role_name, roles=[aes_siem_es_loader_ec2_role.role_name] ) ###################################################################### # in VPC ###################################################################### aes_role_exist = check_iam_role('/aws-service-role/es.amazonaws.com/') if vpc_type and not aes_role_exist: slr_aes = aws_iam.CfnServiceLinkedRole( self, 'AWSServiceRoleForAmazonOpenSearchService', aws_service_name='es.amazonaws.com', description='Created by cloudformation of siem stack' ) slr_aes.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN ###################################################################### # SQS for es-laoder's DLQ ###################################################################### sqs_aes_siem_dlq = aws_sqs.Queue( self, 'AesSiemDlq', queue_name='aes-siem-dlq', retention_period=core.Duration.days(14)) sqs_aes_siem_splitted_logs = aws_sqs.Queue( self, 'AesSiemSqsSplitLogs', queue_name='aes-siem-sqs-splitted-logs', dead_letter_queue=aws_sqs.DeadLetterQueue( max_receive_count=2, queue=sqs_aes_siem_dlq), visibility_timeout=core.Duration.seconds(ES_LOADER_TIMEOUT), retention_period=core.Duration.days(14)) ###################################################################### # Setup Lambda ###################################################################### # setup lambda of es_loader lambda_es_loader_vpc_kwargs = {} if vpc_type: lambda_es_loader_vpc_kwargs = { 'security_group': sg_vpc_noinbound_aes_siem, 'vpc': vpc_aes_siem, 'vpc_subnets': vpc_subnets, } lambda_es_loader = aws_lambda.Function( self, 'LambdaEsLoader', **lambda_es_loader_vpc_kwargs, function_name='aes-siem-es-loader', description=f'{SOLUTION_NAME} / es-loader', runtime=aws_lambda.Runtime.PYTHON_3_8, architecture=aws_lambda.Architecture.X86_64, # architecture=region_mapping.find_in_map( # core.Aws.REGION, 'LambdaArm'), # code=aws_lambda.Code.asset('../lambda/es_loader.zip'), code=aws_lambda.Code.asset('../lambda/es_loader'), handler='index.lambda_handler', memory_size=2048, timeout=core.Duration.seconds(ES_LOADER_TIMEOUT), reserved_concurrent_executions=( reserved_concurrency.value_as_number), dead_letter_queue_enabled=True, dead_letter_queue=sqs_aes_siem_dlq, environment={ 'GEOIP_BUCKET': s3bucket_name_geo, 'LOG_LEVEL': 'info', 'POWERTOOLS_LOGGER_LOG_EVENT': 'false', 'POWERTOOLS_SERVICE_NAME': 'es-loader', 'POWERTOOLS_METRICS_NAMESPACE': 'SIEM'}) es_loader_newver = lambda_es_loader.add_version( name=__version__, description=__version__) es_loader_opt = es_loader_newver.node.default_child.cfn_options es_loader_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN # send only # sqs_aes_siem_dlq.grant(lambda_es_loader, 'sqs:SendMessage') # send and reieve. but it must be loop sqs_aes_siem_dlq.grant( lambda_es_loader, 'sqs:SendMessage', 'sqs:ReceiveMessage', 'sqs:DeleteMessage', 'sqs:GetQueueAttributes') sqs_aes_siem_splitted_logs.grant( lambda_es_loader, 'sqs:SendMessage', 'sqs:ReceiveMessage', 'sqs:DeleteMessage', 'sqs:GetQueueAttributes') lambda_es_loader.add_event_source( aws_lambda_event_sources.SqsEventSource( sqs_aes_siem_splitted_logs, batch_size=1)) # es-loaer on EC2 role sqs_aes_siem_dlq.grant( aes_siem_es_loader_ec2_role, 'sqs:GetQueue*', 'sqs:ListQueues*', 'sqs:ReceiveMessage*', 'sqs:DeleteMessage*') lambda_geo = aws_lambda.Function( self, 'LambdaGeoipDownloader', function_name='aes-siem-geoip-downloader', description=f'{SOLUTION_NAME} / geoip-downloader', runtime=aws_lambda.Runtime.PYTHON_3_8, architecture=aws_lambda.Architecture.X86_64, # architecture=region_mapping.find_in_map( # core.Aws.REGION, 'LambdaArm'), code=aws_lambda.Code.asset('../lambda/geoip_downloader'), handler='index.lambda_handler', memory_size=320, timeout=core.Duration.seconds(300), environment={ 's3bucket_name': s3bucket_name_geo, 'license_key': geoip_license_key.value_as_string, } ) lambda_geo_newver = lambda_geo.add_version( name=__version__, description=__version__) lamba_geo_opt = lambda_geo_newver.node.default_child.cfn_options lamba_geo_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN ###################################################################### # setup OpenSearch Service ###################################################################### lambda_deploy_es = aws_lambda.Function( self, 'LambdaDeployAES', function_name='aes-siem-deploy-aes', description=f'{SOLUTION_NAME} / opensearch domain deployment', runtime=aws_lambda.Runtime.PYTHON_3_8, architecture=aws_lambda.Architecture.X86_64, # architecture=region_mapping.find_in_map( # core.Aws.REGION, 'LambdaArm'), # code=aws_lambda.Code.asset('../lambda/deploy_es.zip'), code=aws_lambda.Code.asset('../lambda/deploy_es'), handler='index.aes_domain_handler', memory_size=128, timeout=core.Duration.seconds(300), environment={ 'accountid': core.Aws.ACCOUNT_ID, 'aes_domain_name': aes_domain_name, 'aes_admin_role': aes_siem_deploy_role_for_lambda.role_arn, 'es_loader_role': lambda_es_loader.role.role_arn, 'allow_source_address': allow_source_address.value_as_string, }, role=aes_siem_deploy_role_for_lambda, ) lambda_deploy_es.add_environment( 's3_snapshot', s3_snapshot.bucket_name) if vpc_type: lambda_deploy_es.add_environment( 'vpc_subnet_id', subnet1.subnet_id) lambda_deploy_es.add_environment( 'security_group_id', sg_vpc_aes_siem.security_group_id) else: lambda_deploy_es.add_environment('vpc_subnet_id', 'None') lambda_deploy_es.add_environment('security_group_id', 'None') deploy_es_newver = lambda_deploy_es.add_version( name=__version__, description=__version__) deploy_es_opt = deploy_es_newver.node.default_child.cfn_options deploy_es_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN # execute lambda_deploy_es to deploy Amaozon ES Domain aes_domain = aws_cloudformation.CfnCustomResource( self, 'AesSiemDomainDeployedR2', service_token=lambda_deploy_es.function_arn,) aes_domain.add_override('Properties.ConfigVersion', __version__) es_endpoint = aes_domain.get_att('es_endpoint').to_string() lambda_es_loader.add_environment('ES_ENDPOINT', es_endpoint) lambda_es_loader.add_environment( 'SQS_SPLITTED_LOGS_URL', sqs_aes_siem_splitted_logs.queue_url) lambda_configure_es_vpc_kwargs = {} if vpc_type: lambda_configure_es_vpc_kwargs = { 'security_group': sg_vpc_noinbound_aes_siem, 'vpc': vpc_aes_siem, 'vpc_subnets': aws_ec2.SubnetSelection(subnets=[subnet1, ]), } lambda_configure_es = aws_lambda.Function( self, 'LambdaConfigureAES', **lambda_configure_es_vpc_kwargs, function_name='aes-siem-configure-aes', description=f'{SOLUTION_NAME} / opensearch configuration', runtime=aws_lambda.Runtime.PYTHON_3_8, architecture=aws_lambda.Architecture.X86_64, # architecture=region_mapping.find_in_map( # core.Aws.REGION, 'LambdaArm'), code=aws_lambda.Code.asset('../lambda/deploy_es'), handler='index.aes_config_handler', memory_size=128, timeout=core.Duration.seconds(300), environment={ 'accountid': core.Aws.ACCOUNT_ID, 'aes_domain_name': aes_domain_name, 'aes_admin_role': aes_siem_deploy_role_for_lambda.role_arn, 'es_loader_role': lambda_es_loader.role.role_arn, 'allow_source_address': allow_source_address.value_as_string, 'es_endpoint': es_endpoint, }, role=aes_siem_deploy_role_for_lambda, ) lambda_configure_es.add_environment( 's3_snapshot', s3_snapshot.bucket_name) if vpc_type: lambda_configure_es.add_environment( 'vpc_subnet_id', subnet1.subnet_id) lambda_configure_es.add_environment( 'security_group_id', sg_vpc_aes_siem.security_group_id) else: lambda_configure_es.add_environment('vpc_subnet_id', 'None') lambda_configure_es.add_environment('security_group_id', 'None') configure_es_newver = lambda_configure_es.add_version( name=__version__, description=__version__) configure_es_opt = configure_es_newver.node.default_child.cfn_options configure_es_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN aes_config = aws_cloudformation.CfnCustomResource( self, 'AesSiemDomainConfiguredR2', service_token=lambda_configure_es.function_arn,) aes_config.add_override('Properties.ConfigVersion', __version__) aes_config.add_depends_on(aes_domain) aes_config.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN es_arn = (f'arn:aws:es:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}' f':domain/{aes_domain_name}') # grant permission to es_loader role inline_policy_to_load_entries_into_es = aws_iam.Policy( self, 'aes-siem-policy-to-load-entries-to-es', policy_name='aes-siem-policy-to-load-entries-to-es', statements=[ aws_iam.PolicyStatement( actions=['es:*'], resources=[es_arn + '/*', ]), ] ) lambda_es_loader.role.attach_inline_policy( inline_policy_to_load_entries_into_es) aes_siem_es_loader_ec2_role.attach_inline_policy( inline_policy_to_load_entries_into_es) # grant additional permission to es_loader role additional_kms_cmks = self.node.try_get_context('additional_kms_cmks') if additional_kms_cmks: inline_policy_access_to_additional_cmks = aws_iam.Policy( self, 'access_to_additional_cmks', policy_name='access_to_additional_cmks', statements=[ aws_iam.PolicyStatement( actions=['kms:Decrypt'], resources=sorted(set(additional_kms_cmks)) ) ] ) lambda_es_loader.role.attach_inline_policy( inline_policy_access_to_additional_cmks) aes_siem_es_loader_ec2_role.attach_inline_policy( inline_policy_access_to_additional_cmks) additional_buckets = self.node.try_get_context('additional_s3_buckets') if additional_buckets: buckets_list = [] for bucket in additional_buckets: buckets_list.append(f'arn:aws:s3:::{bucket}') buckets_list.append(f'arn:aws:s3:::{bucket}/*') inline_policy_access_to_additional_buckets = aws_iam.Policy( self, 'access_to_additional_buckets', policy_name='access_to_additional_buckets', statements=[ aws_iam.PolicyStatement( actions=['s3:GetObject*', 's3:GetBucket*', 's3:List*'], resources=sorted(set(buckets_list)) ) ] ) lambda_es_loader.role.attach_inline_policy( inline_policy_access_to_additional_buckets) aes_siem_es_loader_ec2_role.attach_inline_policy( inline_policy_access_to_additional_buckets) kms_aes_siem.grant_decrypt(lambda_es_loader) kms_aes_siem.grant_decrypt(aes_siem_es_loader_ec2_role) ###################################################################### # s3 notification and grant permisssion ###################################################################### s3_geo.grant_read_write(lambda_geo) s3_geo.grant_read(lambda_es_loader) s3_geo.grant_read(aes_siem_es_loader_ec2_role) s3_log.grant_read(lambda_es_loader) s3_log.grant_read(aes_siem_es_loader_ec2_role) # create s3 notification for es_loader notification = aws_s3_notifications.LambdaDestination(lambda_es_loader) # assign notification for the s3 PUT event type # most log system use PUT, but also CLB use POST & Multipart Upload s3_log.add_event_notification( aws_s3.EventType.OBJECT_CREATED, notification, aws_s3.NotificationKeyFilter(prefix='AWSLogs/')) # For user logs, not AWS logs s3_log.add_event_notification( aws_s3.EventType.OBJECT_CREATED, notification, aws_s3.NotificationKeyFilter(prefix='UserLogs/')) # Download geoip to S3 once by executing lambda_geo get_geodb = aws_cloudformation.CfnCustomResource( self, 'ExecLambdaGeoipDownloader', service_token=lambda_geo.function_arn,) get_geodb.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN # Download geoip every day at 6PM UTC rule = aws_events.Rule( self, 'CwlRuleLambdaGeoipDownloaderDilly', schedule=aws_events.Schedule.rate(core.Duration.hours(12))) rule.add_target(aws_events_targets.LambdaFunction(lambda_geo)) ###################################################################### # bucket policy ###################################################################### s3_awspath = s3_log.bucket_arn + '/AWSLogs/' + core.Aws.ACCOUNT_ID bucket_policy_common1 = aws_iam.PolicyStatement( sid='ELB Policy', principals=[aws_iam.AccountPrincipal( account_id=region_mapping.find_in_map( core.Aws.REGION, 'ElbV2AccountId'))], actions=['s3:PutObject'], resources=[s3_awspath + '/*'],) # NLB / ALB / R53resolver / VPC Flow Logs bucket_policy_elb1 = aws_iam.PolicyStatement( sid='AWSLogDeliveryAclCheck For ALB NLB R53Resolver Flowlogs', principals=[aws_iam.ServicePrincipal( 'delivery.logs.amazonaws.com')], actions=['s3:GetBucketAcl', 's3:ListBucket'], resources=[s3_log.bucket_arn],) bucket_policy_elb2 = aws_iam.PolicyStatement( sid='AWSLogDeliveryWrite For ALB NLB R53Resolver Flowlogs', principals=[aws_iam.ServicePrincipal( 'delivery.logs.amazonaws.com')], actions=['s3:PutObject'], resources=[s3_awspath + '/*'], conditions={ 'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}}) s3_log.add_to_resource_policy(bucket_policy_common1) s3_log.add_to_resource_policy(bucket_policy_elb1) s3_log.add_to_resource_policy(bucket_policy_elb2) # CloudTrail bucket_policy_trail1 = aws_iam.PolicyStatement( sid='AWSLogDeliveryAclCheck For Cloudtrail', principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')], actions=['s3:GetBucketAcl'], resources=[s3_log.bucket_arn],) bucket_policy_trail2 = aws_iam.PolicyStatement( sid='AWSLogDeliveryWrite For CloudTrail', principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')], actions=['s3:PutObject'], resources=[s3_awspath + '/*'], conditions={ 'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}}) s3_log.add_to_resource_policy(bucket_policy_trail1) s3_log.add_to_resource_policy(bucket_policy_trail2) # GuardDuty bucket_policy_gd1 = aws_iam.PolicyStatement( sid='Allow GuardDuty to use the getBucketLocation operation', principals=[aws_iam.ServicePrincipal('guardduty.amazonaws.com')], actions=['s3:GetBucketLocation'], resources=[s3_log.bucket_arn],) bucket_policy_gd2 = aws_iam.PolicyStatement( sid='Allow GuardDuty to upload objects to the bucket', principals=[aws_iam.ServicePrincipal('guardduty.amazonaws.com')], actions=['s3:PutObject'], resources=[s3_log.bucket_arn + '/*'],) bucket_policy_gd5 = aws_iam.PolicyStatement( sid='Deny non-HTTPS access', effect=aws_iam.Effect.DENY, actions=['s3:*'], resources=[s3_log.bucket_arn + '/*'], conditions={'Bool': {'aws:SecureTransport': 'false'}}) bucket_policy_gd5.add_any_principal() s3_log.add_to_resource_policy(bucket_policy_gd1) s3_log.add_to_resource_policy(bucket_policy_gd2) s3_log.add_to_resource_policy(bucket_policy_gd5) # Config bucket_policy_config1 = aws_iam.PolicyStatement( sid='AWSConfig BucketPermissionsCheck and BucketExistenceCheck', principals=[aws_iam.ServicePrincipal('config.amazonaws.com')], actions=['s3:GetBucketAcl', 's3:ListBucket'], resources=[s3_log.bucket_arn],) bucket_policy_config2 = aws_iam.PolicyStatement( sid='AWSConfigBucketDelivery', principals=[aws_iam.ServicePrincipal('config.amazonaws.com')], actions=['s3:PutObject'], resources=[s3_awspath + '/Config/*'], conditions={ 'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}}) s3_log.add_to_resource_policy(bucket_policy_config1) s3_log.add_to_resource_policy(bucket_policy_config2) # geoip bucket_policy_geo1 = aws_iam.PolicyStatement( sid='Allow geoip downloader and es-loader to read/write', principals=[lambda_es_loader.role, lambda_geo.role], actions=['s3:PutObject', 's3:GetObject', 's3:DeleteObject'], resources=[s3_geo.bucket_arn + '/*'],) s3_geo.add_to_resource_policy(bucket_policy_geo1) # ES Snapshot bucket_policy_snapshot = aws_iam.PolicyStatement( sid='Allow ES to store snapshot', principals=[aes_siem_snapshot_role], actions=['s3:PutObject', 's3:GetObject', 's3:DeleteObject'], resources=[s3_snapshot.bucket_arn + '/*'],) s3_snapshot.add_to_resource_policy(bucket_policy_snapshot) ###################################################################### # for multiaccount / organizaitons ###################################################################### if org_id or no_org_ids: ################################################################## # KMS key policy for multiaccount / organizaitons ################################################################## # for CloudTrail cond_tail2 = self.make_resource_list( path='arn:aws:cloudtrail:*:', tail=':trail/*', keys=self.list_without_none(org_mgmt_id, no_org_ids)) key_policy_mul_trail2 = aws_iam.PolicyStatement( sid=('Allow CloudTrail to encrypt logs for multiaccounts'), actions=['kms:GenerateDataKey*'], principals=[aws_iam.ServicePrincipal( 'cloudtrail.amazonaws.com')], resources=['*'], conditions={'StringLike': { 'kms:EncryptionContext:aws:cloudtrail:arn': cond_tail2}}) kms_aes_siem.add_to_resource_policy(key_policy_mul_trail2) # for replicaiton key_policy_rep1 = aws_iam.PolicyStatement( sid=('Enable cross account encrypt access for S3 Cross Region ' 'Replication'), actions=['kms:Encrypt'], principals=self.make_account_principals( org_mgmt_id, org_member_ids, no_org_ids), resources=['*'],) kms_aes_siem.add_to_resource_policy(key_policy_rep1) ################################################################## # Buckdet Policy for multiaccount / organizaitons ################################################################## s3_log_bucket_arn = 'arn:aws:s3:::' + s3bucket_name_log # for CloudTrail s3_mulpaths = self.make_resource_list( path=f'{s3_log_bucket_arn}/AWSLogs/', tail='/*', keys=self.list_without_none(org_id, org_mgmt_id, no_org_ids)) bucket_policy_org_trail = aws_iam.PolicyStatement( sid='AWSCloudTrailWrite for Multiaccounts / Organizations', principals=[ aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')], actions=['s3:PutObject'], resources=s3_mulpaths, conditions={'StringEquals': { 's3:x-amz-acl': 'bucket-owner-full-control'}}) s3_log.add_to_resource_policy(bucket_policy_org_trail) # config s3_conf_multpaths = self.make_resource_list( path=f'{s3_log_bucket_arn}/AWSLogs/', tail='/Config/*', keys=self.list_without_none(org_id, org_mgmt_id, no_org_ids)) bucket_policy_mul_config2 = aws_iam.PolicyStatement( sid='AWSConfigBucketDelivery', principals=[aws_iam.ServicePrincipal('config.amazonaws.com')], actions=['s3:PutObject'], resources=s3_conf_multpaths, conditions={'StringEquals': { 's3:x-amz-acl': 'bucket-owner-full-control'}}) s3_log.add_to_resource_policy(bucket_policy_mul_config2) # for replication bucket_policy_rep1 = aws_iam.PolicyStatement( sid='PolicyForDestinationBucket / Permissions on objects', principals=self.make_account_principals( org_mgmt_id, org_member_ids, no_org_ids), actions=['s3:ReplicateDelete', 's3:ReplicateObject', 's3:ReplicateTags', 's3:GetObjectVersionTagging', 's3:ObjectOwnerOverrideToBucketOwner'], resources=[f'{s3_log_bucket_arn}/*']) bucket_policy_rep2 = aws_iam.PolicyStatement( sid='PolicyForDestinationBucket / Permissions on bucket', principals=self.make_account_principals( org_mgmt_id, org_member_ids, no_org_ids), actions=['s3:List*', 's3:GetBucketVersioning', 's3:PutBucketVersioning'], resources=[f'{s3_log_bucket_arn}']) s3_log.add_to_resource_policy(bucket_policy_rep1) s3_log.add_to_resource_policy(bucket_policy_rep2) ###################################################################### # SNS topic for Amazon OpenSearch Service Alert ###################################################################### sns_topic = aws_sns.Topic( self, 'SnsTopic', topic_name='aes-siem-alert', display_name='AES SIEM') sns_topic.add_subscription(aws_sns_subscriptions.EmailSubscription( email_address=sns_email.value_as_string)) sns_topic.grant_publish(aes_siem_sns_role) ###################################################################### # output of CFn ###################################################################### kibanaurl = f'https://{es_endpoint}/_dashboards/' kibanaadmin = aes_domain.get_att('kibanaadmin').to_string() kibanapass = aes_domain.get_att('kibanapass').to_string() core.CfnOutput(self, 'RoleDeploy', export_name='role-deploy', value=aes_siem_deploy_role_for_lambda.role_arn) core.CfnOutput(self, 'DashboardsUrl', export_name='dashboards-url', value=kibanaurl) core.CfnOutput(self, 'DashboardsPassword', export_name='dashboards-pass', value=kibanapass, description=('Please change the password in OpenSearch ' 'Dashboards ASAP')) core.CfnOutput(self, 'DashboardsAdminID', export_name='dashboards-admin', value=kibanaadmin)
def __init__(self, scope: core.Construct, id: str, infra: RtspBaseResourcesConstruct, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Create the inventory bucket... self.inventories = s3.Bucket( self, 'InventoryBucket', bucket_name='homenet-{}.rtsp-inventories.{}.virtual.world'.format( infra.landing_zone.zone_name, core.Stack.of(self).region).lower(), removal_policy=core.RemovalPolicy.DESTROY, cors=[ s3.CorsRule(allowed_methods=[s3.HttpMethods.GET], allowed_origins=['*']) ], lifecycle_rules=[ s3.LifecycleRule( id='Transition-to-IA-after-30D', prefix='eufy/', abort_incomplete_multipart_upload_after=core.Duration.days( 7), transitions=[ s3.Transition( storage_class=s3.StorageClass.INFREQUENT_ACCESS, transition_after=core.Duration.days(30)) ]) ]) # Create inventory collections for the Eufy Homebases... infra.bucket.add_inventory( objects_prefix='eufy/', inventory_id='{}-InventoryReport'.format('EufyFull'), format=s3.InventoryFormat.CSV, frequency=s3.InventoryFrequency.DAILY, include_object_versions=s3.InventoryObjectVersion.CURRENT, destination=s3.InventoryDestination( bucket=self.inventories, bucket_owner=core.Aws.ACCOUNT_ID, prefix=None)) for base_name in ['Moonbase', 'Starbase']: prefix = 'eufy/{}.cameras.real.world/'.format(base_name).lower() infra.bucket.add_inventory( objects_prefix=prefix, inventory_id='{}-InventoryReport'.format(base_name), format=s3.InventoryFormat.CSV, frequency=s3.InventoryFrequency.DAILY, include_object_versions=s3.InventoryObjectVersion.CURRENT, destination=s3.InventoryDestination( bucket=self.inventories, bucket_owner=core.Aws.ACCOUNT_ID, prefix=None)) # Broadcast inventory creation events... self.inventoryAvailable = sns.Topic( self, 'InventoryAvailable', display_name='HomeNet-{}-Rtsp-InventoryAvailable'.format( infra.landing_zone.zone_name), topic_name='HomeNet-{}-Rtsp-InventoryAvailable'.format( infra.landing_zone.zone_name)) self.inventories.add_event_notification( s3.EventType.OBJECT_CREATED, s3n.SnsDestination(topic=self.inventoryAvailable), s3.NotificationKeyFilter(suffix='manifest.json')) # Attach debug queue to the notification self.inventoryAvailable.add_subscription( subs.SqsSubscription( raw_message_delivery=True, queue=sqs.Queue( self, 'InventoryDebugQueue', removal_policy=core.RemovalPolicy.DESTROY, retention_period=core.Duration.days(7), queue_name='HomeNet-{}-RtspInventoryAvailable_Debug'. format(infra.landing_zone.zone_name)))) # Subscribe the GroundTruth Manifest Generator groundtruth = RtspGroundTruthManifestGenerationFunction( self, 'GroundTruthManifest', infra=infra, topic=self.inventoryAvailable) self.inventories.grant_read_write(groundtruth.function.role) # Create the RtspNormalizeImage S3 Object Lambda RtspNormalizeImageAccessPoint(scope=self, id='NormalizedImage', infra=infra)
def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # Image Bucket image_bucket = s3.Bucket(self, IMG_BUCKET_NAME, removal_policy=cdk.RemovalPolicy.DESTROY) cdk.CfnOutput(self, "imageBucket", value=image_bucket.bucket_name) image_bucket.add_cors_rule( allowed_methods=[s3.HttpMethods.GET, s3.HttpMethods.PUT], allowed_origins=["*"], allowed_headers=["*"], max_age=3000, ) # Thumbnail Bucket resized_image_bucket = s3.Bucket( self, RESIZED_IMG_BUCKET_NAME, removal_policy=cdk.RemovalPolicy.DESTROY) cdk.CfnOutput(self, "resizedBucket", value=resized_image_bucket.bucket_name) resized_image_bucket.add_cors_rule( allowed_methods=[s3.HttpMethods.GET, s3.HttpMethods.PUT], allowed_origins=["*"], allowed_headers=["*"], max_age=3000, ) # S3 Static bucket for website code web_bucket = s3.Bucket( self, WEBSITE_BUCKET_NAME, website_index_document="index.html", website_error_document="index.html", removal_policy=cdk.RemovalPolicy.DESTROY, # uncomment this and delete the policy statement below to allow public access to our # static website # public_read_access=true ) web_policy_statement = iam.PolicyStatement( actions=["s3:GetObject"], resources=[web_bucket.arn_for_objects("*")], principals=[iam.AnyPrincipal()], conditions={"IpAddress": { "aws:SourceIp": ["139.138.203.36"] }}, ) web_bucket.add_to_resource_policy(web_policy_statement) cdk.CfnOutput(self, "bucketURL", value=web_bucket.bucket_website_domain_name) # Deploy site contents to S3 Bucket s3_dep.BucketDeployment( self, "DeployWebsite", sources=[s3_dep.Source.asset("./public")], destination_bucket=web_bucket, ) # DynamoDB to store image labels partition_key = dynamodb.Attribute(name="image", type=dynamodb.AttributeType.STRING) table = dynamodb.Table( self, "ImageLabels", partition_key=partition_key, removal_policy=cdk.RemovalPolicy.DESTROY, ) cdk.CfnOutput(self, "ddbTable", value=table.table_name) # Lambda layer for Pillow library layer = lb.LayerVersion( self, "pil", code=lb.Code.from_asset("reklayer"), compatible_runtimes=[lb.Runtime.PYTHON_3_7], license="Apache-2.0", description= "A layer to enable the PIL library in our Rekognition Lambda", ) # Lambda function rek_fn = lb.Function( self, "rekognitionFunction", code=lb.Code.from_asset("rekognitionFunction"), runtime=lb.Runtime.PYTHON_3_7, handler="index.handler", timeout=cdk.Duration.seconds(30), memory_size=1024, layers=[layer], environment={ "TABLE": table.table_name, "BUCKET": image_bucket.bucket_name, "THUMBBUCKET": resized_image_bucket.bucket_name, }, ) image_bucket.grant_read(rek_fn) resized_image_bucket.grant_write(rek_fn) table.grant_write_data(rek_fn) rek_fn.add_to_role_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=["rekognition:DetectLabels"], resources=["*"])) # Lambda for Synchronous front end serviceFn = lb.Function( self, "serviceFunction", code=lb.Code.from_asset("servicelambda"), runtime=lb.Runtime.PYTHON_3_7, handler="index.handler", environment={ "TABLE": table.table_name, "BUCKET": image_bucket.bucket_name, "RESIZEDBUCKET": resized_image_bucket.bucket_name, }, ) image_bucket.grant_write(serviceFn) resized_image_bucket.grant_write(serviceFn) table.grant_read_write_data(serviceFn) # Cognito User Pool Auth auto_verified_attrs = cognito.AutoVerifiedAttrs(email=True) sign_in_aliases = cognito.SignInAliases(email=True, username=True) user_pool = cognito.UserPool( self, "UserPool", self_sign_up_enabled=True, auto_verify=auto_verified_attrs, sign_in_aliases=sign_in_aliases, ) user_pool_client = cognito.UserPoolClient(self, "UserPoolClient", user_pool=user_pool, generate_secret=False) identity_pool = cognito.CfnIdentityPool( self, "ImageRekognitionIdentityPool", allow_unauthenticated_identities=False, cognito_identity_providers=[{ "clientId": user_pool_client.user_pool_client_id, "providerName": user_pool.user_pool_provider_name, }], ) # API Gateway cors_options = apigw.CorsOptions(allow_origins=apigw.Cors.ALL_ORIGINS, allow_methods=apigw.Cors.ALL_METHODS) api = apigw.LambdaRestApi( self, "imageAPI", default_cors_preflight_options=cors_options, handler=serviceFn, proxy=False, ) auth = apigw.CfnAuthorizer( self, "ApiGatewayAuthorizer", name="customer-authorizer", identity_source="method.request.header.Authorization", provider_arns=[user_pool.user_pool_arn], rest_api_id=api.rest_api_id, # type=apigw.AuthorizationType.COGNITO, type="COGNITO_USER_POOLS", ) assumed_by = iam.FederatedPrincipal( "cognito-identity.amazon.com", conditions={ "StringEquals": { "cognito-identity.amazonaws.com:aud": identity_pool.ref }, "ForAnyValue:StringLike": { "cognito-identity.amazonaws.com:amr": "authenticated" }, }, assume_role_action="sts:AssumeRoleWithWebIdentity", ) authenticated_role = iam.Role( self, "ImageRekognitionAuthenticatedRole", assumed_by=assumed_by, ) # IAM policy granting users permission to get and put their pictures policy_statement = iam.PolicyStatement( actions=["s3:GetObject", "s3:PutObject"], effect=iam.Effect.ALLOW, resources=[ image_bucket.bucket_arn + "/private/${cognito-identity.amazonaws.com:sub}/*", image_bucket.bucket_arn + "/private/${cognito-identity.amazonaws.com:sub}/", resized_image_bucket.bucket_arn + "/private/${cognito-identity.amazonaws.com:sub}/*", resized_image_bucket.bucket_arn + "/private/${cognito-identity.amazonaws.com:sub}/", ], ) # IAM policy granting users permission to list their pictures list_policy_statement = iam.PolicyStatement( actions=["s3:ListBucket"], effect=iam.Effect.ALLOW, resources=[ image_bucket.bucket_arn, resized_image_bucket.bucket_arn ], conditions={ "StringLike": { "s3:prefix": ["private/${cognito-identity.amazonaws.com:sub}/*"] } }, ) authenticated_role.add_to_policy(policy_statement) authenticated_role.add_to_policy(list_policy_statement) # Attach role to our Identity Pool cognito.CfnIdentityPoolRoleAttachment( self, "IdentityPoolRoleAttachment", identity_pool_id=identity_pool.ref, roles={"authenticated": authenticated_role.role_arn}, ) # Get some outputs from cognito cdk.CfnOutput(self, "UserPoolId", value=user_pool.user_pool_id) cdk.CfnOutput(self, "AppClientId", value=user_pool_client.user_pool_client_id) cdk.CfnOutput(self, "IdentityPoolId", value=identity_pool.ref) # New Amazon API Gateway with AWS Lambda Integration success_response = apigw.IntegrationResponse( status_code="200", response_parameters={ "method.response.header.Access-Control-Allow-Origin": "'*'" }, ) error_response = apigw.IntegrationResponse( selection_pattern="(\n|.)+", status_code="500", response_parameters={ "method.response.header.Access-Control-Allow-Origin": "'*'" }, ) request_template = json.dumps({ "action": "$util.escapeJavaScript($input.params('action'))", "key": "$util.escapeJavaScript($input.params('key'))", }) lambda_integration = apigw.LambdaIntegration( serviceFn, proxy=False, request_parameters={ "integration.request.querystring.action": "method.request.querystring.action", "integration.request.querystring.key": "method.request.querystring.key", }, request_templates={"application/json": request_template}, passthrough_behavior=apigw.PassthroughBehavior.WHEN_NO_TEMPLATES, integration_responses=[success_response, error_response], ) imageAPI = api.root.add_resource("images") success_resp = apigw.MethodResponse( status_code="200", response_parameters={ "method.response.header.Access-Control-Allow-Origin": True }, ) error_resp = apigw.MethodResponse( status_code="500", response_parameters={ "method.response.header.Access-Control-Allow-Origin": True }, ) # GET /images get_method = imageAPI.add_method( "GET", lambda_integration, authorization_type=apigw.AuthorizationType.COGNITO, request_parameters={ "method.request.querystring.action": True, "method.request.querystring.key": True, }, method_responses=[success_resp, error_resp], ) # DELETE /images delete_method = imageAPI.add_method( "DELETE", lambda_integration, authorization_type=apigw.AuthorizationType.COGNITO, request_parameters={ "method.request.querystring.action": True, "method.request.querystring.key": True, }, method_responses=[success_resp, error_resp], ) # Override the authorizer id because it doesn't work when defininting it as a param # in add_method get_method_resource = get_method.node.find_child("Resource") get_method_resource.add_property_override("AuthorizerId", auth.ref) delete_method_resource = delete_method.node.find_child("Resource") delete_method_resource.add_property_override("AuthorizerId", auth.ref) # Building SQS queue and DeadLetter Queue dl_queue = sqs.Queue( self, "ImageDLQueue", queue_name="ImageDLQueue", ) dl_queue_opts = sqs.DeadLetterQueue(max_receive_count=2, queue=dl_queue) queue = sqs.Queue( self, "ImageQueue", queue_name="ImageQueue", visibility_timeout=cdk.Duration.seconds(30), receive_message_wait_time=cdk.Duration.seconds(20), dead_letter_queue=dl_queue_opts, ) # S3 Bucket Create Notification to SQS # Whenever an image is uploaded add it to the queue image_bucket.add_object_created_notification( s3n.SqsDestination(queue), s3.NotificationKeyFilter(prefix="private/"))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) vpc = aws_ec2.Vpc( self, "OctemberVPC", max_azs=2, # subnet_configuration=[{ # "cidrMask": 24, # "name": "Public", # "subnetType": aws_ec2.SubnetType.PUBLIC, # }, # { # "cidrMask": 24, # "name": "Private", # "subnetType": aws_ec2.SubnetType.PRIVATE # }, # { # "cidrMask": 28, # "name": "Isolated", # "subnetType": aws_ec2.SubnetType.ISOLATED, # "reserved": True # } # ], gateway_endpoints={ "S3": aws_ec2.GatewayVpcEndpointOptions( service=aws_ec2.GatewayVpcEndpointAwsService.S3) }) dynamo_db_endpoint = vpc.add_gateway_endpoint( "DynamoDbEndpoint", service=aws_ec2.GatewayVpcEndpointAwsService.DYNAMODB) s3_bucket = s3.Bucket( self, "s3bucket", bucket_name="octember-bizcard-{region}-{account}".format( region=core.Aws.REGION, account=core.Aws.ACCOUNT_ID)) api = apigw.RestApi( self, "BizcardImageUploader", rest_api_name="BizcardImageUploader", description="This service serves uploading bizcard images into s3.", endpoint_types=[apigw.EndpointType.REGIONAL], binary_media_types=["image/png", "image/jpg"], deploy=True, deploy_options=apigw.StageOptions(stage_name="v1")) rest_api_role = aws_iam.Role( self, "ApiGatewayRoleForS3", role_name="ApiGatewayRoleForS3FullAccess", assumed_by=aws_iam.ServicePrincipal("apigateway.amazonaws.com"), managed_policies=[ aws_iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonS3FullAccess") ]) list_objects_responses = [ apigw.IntegrationResponse( status_code="200", #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationResponse.html#aws_cdk.aws_apigateway.IntegrationResponse.response_parameters # The response parameters from the backend response that API Gateway sends to the method response. # Use the destination as the key and the source as the value: # - The destination must be an existing response parameter in the MethodResponse property. # - The source must be an existing method request parameter or a static value. response_parameters={ 'method.response.header.Timestamp': 'integration.response.header.Date', 'method.response.header.Content-Length': 'integration.response.header.Content-Length', 'method.response.header.Content-Type': 'integration.response.header.Content-Type' }), apigw.IntegrationResponse(status_code="400", selection_pattern="4\d{2}"), apigw.IntegrationResponse(status_code="500", selection_pattern="5\d{2}") ] list_objects_integration_options = apigw.IntegrationOptions( credentials_role=rest_api_role, integration_responses=list_objects_responses) get_s3_integration = apigw.AwsIntegration( service="s3", integration_http_method="GET", path='/', options=list_objects_integration_options) api.root.add_method( "GET", get_s3_integration, authorization_type=apigw.AuthorizationType.IAM, api_key_required=False, method_responses=[ apigw.MethodResponse( status_code="200", response_parameters={ 'method.response.header.Timestamp': False, 'method.response.header.Content-Length': False, 'method.response.header.Content-Type': False }, response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ], request_parameters={'method.request.header.Content-Type': False}) get_s3_folder_integration_options = apigw.IntegrationOptions( credentials_role=rest_api_role, integration_responses=list_objects_responses, #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationOptions.html#aws_cdk.aws_apigateway.IntegrationOptions.request_parameters # Specify request parameters as key-value pairs (string-to-string mappings), with a destination as the key and a source as the value. # The source must be an existing method request parameter or a static value. request_parameters={ "integration.request.path.bucket": "method.request.path.folder" }) get_s3_folder_integration = apigw.AwsIntegration( service="s3", integration_http_method="GET", path="{bucket}", options=get_s3_folder_integration_options) s3_folder = api.root.add_resource('{folder}') s3_folder.add_method( "GET", get_s3_folder_integration, authorization_type=apigw.AuthorizationType.IAM, api_key_required=False, method_responses=[ apigw.MethodResponse( status_code="200", response_parameters={ 'method.response.header.Timestamp': False, 'method.response.header.Content-Length': False, 'method.response.header.Content-Type': False }, response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ], request_parameters={ 'method.request.header.Content-Type': False, 'method.request.path.folder': True }) get_s3_item_integration_options = apigw.IntegrationOptions( credentials_role=rest_api_role, integration_responses=list_objects_responses, request_parameters={ "integration.request.path.bucket": "method.request.path.folder", "integration.request.path.object": "method.request.path.item" }) get_s3_item_integration = apigw.AwsIntegration( service="s3", integration_http_method="GET", path="{bucket}/{object}", options=get_s3_item_integration_options) s3_item = s3_folder.add_resource('{item}') s3_item.add_method( "GET", get_s3_item_integration, authorization_type=apigw.AuthorizationType.IAM, api_key_required=False, method_responses=[ apigw.MethodResponse( status_code="200", response_parameters={ 'method.response.header.Timestamp': False, 'method.response.header.Content-Length': False, 'method.response.header.Content-Type': False }, response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ], request_parameters={ 'method.request.header.Content-Type': False, 'method.request.path.folder': True, 'method.request.path.item': True }) put_s3_item_integration_options = apigw.IntegrationOptions( credentials_role=rest_api_role, integration_responses=[ apigw.IntegrationResponse(status_code="200"), apigw.IntegrationResponse(status_code="400", selection_pattern="4\d{2}"), apigw.IntegrationResponse(status_code="500", selection_pattern="5\d{2}") ], request_parameters={ "integration.request.header.Content-Type": "method.request.header.Content-Type", "integration.request.path.bucket": "method.request.path.folder", "integration.request.path.object": "method.request.path.item" }) put_s3_item_integration = apigw.AwsIntegration( service="s3", integration_http_method="PUT", path="{bucket}/{object}", options=put_s3_item_integration_options) s3_item.add_method( "PUT", put_s3_item_integration, authorization_type=apigw.AuthorizationType.IAM, api_key_required=False, method_responses=[ apigw.MethodResponse( status_code="200", response_parameters={ 'method.response.header.Content-Type': False }, response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ], request_parameters={ 'method.request.header.Content-Type': False, 'method.request.path.folder': True, 'method.request.path.item': True }) ddb_table = dynamodb.Table( self, "BizcardImageMetaInfoDdbTable", table_name="OctemberBizcardImgMeta", partition_key=dynamodb.Attribute( name="image_id", type=dynamodb.AttributeType.STRING), billing_mode=dynamodb.BillingMode.PROVISIONED, read_capacity=15, write_capacity=5) img_kinesis_stream = kinesis.Stream( self, "BizcardImagePath", stream_name="octember-bizcard-image") # create lambda function trigger_textract_lambda_fn = _lambda.Function( self, "TriggerTextExtractorFromImage", runtime=_lambda.Runtime.PYTHON_3_7, function_name="TriggerTextExtractorFromImage", handler="trigger_text_extract_from_s3_image.lambda_handler", description="Trigger to extract text from an image in S3", code=_lambda.Code.asset( "./src/main/python/TriggerTextExtractFromS3Image"), environment={ 'REGION_NAME': core.Aws.REGION, 'DDB_TABLE_NAME': ddb_table.table_name, 'KINESIS_STREAM_NAME': img_kinesis_stream.stream_name }, timeout=core.Duration.minutes(5)) ddb_table_rw_policy_statement = aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, resources=[ddb_table.table_arn], actions=[ "dynamodb:BatchGetItem", "dynamodb:Describe*", "dynamodb:List*", "dynamodb:GetItem", "dynamodb:Query", "dynamodb:Scan", "dynamodb:BatchWriteItem", "dynamodb:DeleteItem", "dynamodb:PutItem", "dynamodb:UpdateItem", "dax:Describe*", "dax:List*", "dax:GetItem", "dax:BatchGetItem", "dax:Query", "dax:Scan", "dax:BatchWriteItem", "dax:DeleteItem", "dax:PutItem", "dax:UpdateItem" ]) trigger_textract_lambda_fn.add_to_role_policy( ddb_table_rw_policy_statement) trigger_textract_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=[img_kinesis_stream.stream_arn], actions=[ "kinesis:Get*", "kinesis:List*", "kinesis:Describe*", "kinesis:PutRecord", "kinesis:PutRecords" ])) # assign notification for the s3 event type (ex: OBJECT_CREATED) s3_event_filter = s3.NotificationKeyFilter(prefix="bizcard-raw-img/", suffix=".jpg") s3_event_source = S3EventSource(s3_bucket, events=[s3.EventType.OBJECT_CREATED], filters=[s3_event_filter]) trigger_textract_lambda_fn.add_event_source(s3_event_source) #XXX: https://github.com/aws/aws-cdk/issues/2240 # To avoid to create extra Lambda Functions with names like LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a # if log_retention=aws_logs.RetentionDays.THREE_DAYS is added to the constructor props log_group = aws_logs.LogGroup( self, "TriggerTextractLogGroup", log_group_name="/aws/lambda/TriggerTextExtractorFromImage", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(trigger_textract_lambda_fn) text_kinesis_stream = kinesis.Stream( self, "BizcardTextData", stream_name="octember-bizcard-txt") textract_lambda_fn = _lambda.Function( self, "GetTextFromImage", runtime=_lambda.Runtime.PYTHON_3_7, function_name="GetTextFromImage", handler="get_text_from_s3_image.lambda_handler", description="extract text from an image in S3", code=_lambda.Code.asset("./src/main/python/GetTextFromS3Image"), environment={ 'REGION_NAME': core.Aws.REGION, 'DDB_TABLE_NAME': ddb_table.table_name, 'KINESIS_STREAM_NAME': text_kinesis_stream.stream_name }, timeout=core.Duration.minutes(5)) textract_lambda_fn.add_to_role_policy(ddb_table_rw_policy_statement) textract_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=[text_kinesis_stream.stream_arn], actions=[ "kinesis:Get*", "kinesis:List*", "kinesis:Describe*", "kinesis:PutRecord", "kinesis:PutRecords" ])) textract_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ s3_bucket.bucket_arn, "{}/*".format( s3_bucket.bucket_arn) ], "actions": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject" ] })) textract_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=["textract:*"])) img_kinesis_event_source = KinesisEventSource( img_kinesis_stream, batch_size=100, starting_position=_lambda.StartingPosition.LATEST) textract_lambda_fn.add_event_source(img_kinesis_event_source) log_group = aws_logs.LogGroup( self, "GetTextFromImageLogGroup", log_group_name="/aws/lambda/GetTextFromImage", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(textract_lambda_fn) sg_use_bizcard_es = aws_ec2.SecurityGroup( self, "BizcardSearchClientSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard elasticsearch client', security_group_name='use-octember-bizcard-es') core.Tags.of(sg_use_bizcard_es).add('Name', 'use-octember-bizcard-es') sg_bizcard_es = aws_ec2.SecurityGroup( self, "BizcardSearchSG", vpc=vpc, allow_all_outbound=True, description='security group for octember bizcard elasticsearch', security_group_name='octember-bizcard-es') core.Tags.of(sg_bizcard_es).add('Name', 'octember-bizcard-es') sg_bizcard_es.add_ingress_rule(peer=sg_bizcard_es, connection=aws_ec2.Port.all_tcp(), description='octember-bizcard-es') sg_bizcard_es.add_ingress_rule(peer=sg_use_bizcard_es, connection=aws_ec2.Port.all_tcp(), description='use-octember-bizcard-es') sg_ssh_access = aws_ec2.SecurityGroup( self, "BastionHostSG", vpc=vpc, allow_all_outbound=True, description='security group for bastion host', security_group_name='octember-bastion-host-sg') core.Tags.of(sg_ssh_access).add('Name', 'octember-bastion-host') sg_ssh_access.add_ingress_rule(peer=aws_ec2.Peer.any_ipv4(), connection=aws_ec2.Port.tcp(22), description='ssh access') bastion_host = aws_ec2.BastionHostLinux( self, "BastionHost", vpc=vpc, instance_type=aws_ec2.InstanceType('t3.nano'), security_group=sg_ssh_access, subnet_selection=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC)) bastion_host.instance.add_security_group(sg_use_bizcard_es) #XXX: aws cdk elastsearch example - https://github.com/aws/aws-cdk/issues/2873 es_cfn_domain = aws_elasticsearch.CfnDomain( self, 'BizcardSearch', elasticsearch_cluster_config={ "dedicatedMasterCount": 3, "dedicatedMasterEnabled": True, "dedicatedMasterType": "t2.medium.elasticsearch", "instanceCount": 2, "instanceType": "t2.medium.elasticsearch", "zoneAwarenessEnabled": True }, ebs_options={ "ebsEnabled": True, "volumeSize": 10, "volumeType": "gp2" }, domain_name="octember-bizcard", elasticsearch_version="7.9", encryption_at_rest_options={"enabled": False}, access_policies={ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "AWS": "*" }, "Action": ["es:Describe*", "es:List*", "es:Get*", "es:ESHttp*"], "Resource": self.format_arn(service="es", resource="domain", resource_name="octember-bizcard/*") }] }, snapshot_options={"automatedSnapshotStartHour": 17}, vpc_options={ "securityGroupIds": [sg_bizcard_es.security_group_id], "subnetIds": vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids }) core.Tags.of(es_cfn_domain).add('Name', 'octember-bizcard-es') s3_lib_bucket_name = self.node.try_get_context("lib_bucket_name") #XXX: https://github.com/aws/aws-cdk/issues/1342 s3_lib_bucket = s3.Bucket.from_bucket_name(self, id, s3_lib_bucket_name) es_lib_layer = _lambda.LayerVersion( self, "ESLib", layer_version_name="es-lib", compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], code=_lambda.Code.from_bucket(s3_lib_bucket, "var/octember-es-lib.zip")) redis_lib_layer = _lambda.LayerVersion( self, "RedisLib", layer_version_name="redis-lib", compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], code=_lambda.Code.from_bucket(s3_lib_bucket, "var/octember-redis-lib.zip")) #XXX: Deploy lambda in VPC - https://github.com/aws/aws-cdk/issues/1342 upsert_to_es_lambda_fn = _lambda.Function( self, "UpsertBizcardToES", runtime=_lambda.Runtime.PYTHON_3_7, function_name="UpsertBizcardToElasticSearch", handler="upsert_bizcard_to_es.lambda_handler", description="Upsert bizcard text into elasticsearch", code=_lambda.Code.asset("./src/main/python/UpsertBizcardToES"), environment={ 'ES_HOST': es_cfn_domain.attr_domain_endpoint, 'ES_INDEX': 'octember_bizcard', 'ES_TYPE': 'bizcard' }, timeout=core.Duration.minutes(5), layers=[es_lib_layer], security_groups=[sg_use_bizcard_es], vpc=vpc) text_kinesis_event_source = KinesisEventSource( text_kinesis_stream, batch_size=99, starting_position=_lambda.StartingPosition.LATEST) upsert_to_es_lambda_fn.add_event_source(text_kinesis_event_source) log_group = aws_logs.LogGroup( self, "UpsertBizcardToESLogGroup", log_group_name="/aws/lambda/UpsertBizcardToElasticSearch", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(upsert_to_es_lambda_fn) firehose_role_policy_doc = aws_iam.PolicyDocument() firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ s3_bucket.bucket_arn, "{}/*".format( s3_bucket.bucket_arn) ], "actions": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject" ] })) firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=[ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions" ])) firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=[text_kinesis_stream.stream_arn], actions=[ "kinesis:DescribeStream", "kinesis:GetShardIterator", "kinesis:GetRecords" ])) firehose_log_group_name = "/aws/kinesisfirehose/octember-bizcard-txt-to-s3" firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, #XXX: The ARN will be formatted as follows: # arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name} resources=[ self.format_arn(service="logs", resource="log-group", resource_name="{}:log-stream:*".format( firehose_log_group_name), sep=":") ], actions=["logs:PutLogEvents"])) firehose_role = aws_iam.Role( self, "FirehoseDeliveryRole", role_name="FirehoseDeliveryRole", assumed_by=aws_iam.ServicePrincipal("firehose.amazonaws.com"), #XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221 inline_policies={"firehose_role_policy": firehose_role_policy_doc}) bizcard_text_to_s3_delivery_stream = aws_kinesisfirehose.CfnDeliveryStream( self, "BizcardTextToS3", delivery_stream_name="octember-bizcard-txt-to-s3", delivery_stream_type="KinesisStreamAsSource", kinesis_stream_source_configuration={ "kinesisStreamArn": text_kinesis_stream.stream_arn, "roleArn": firehose_role.role_arn }, extended_s3_destination_configuration={ "bucketArn": s3_bucket.bucket_arn, "bufferingHints": { "intervalInSeconds": 60, "sizeInMBs": 1 }, "cloudWatchLoggingOptions": { "enabled": True, "logGroupName": firehose_log_group_name, "logStreamName": "S3Delivery" }, "compressionFormat": "GZIP", "prefix": "bizcard-text/", "roleArn": firehose_role.role_arn }) sg_use_bizcard_es_cache = aws_ec2.SecurityGroup( self, "BizcardSearchCacheClientSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard search query cache client', security_group_name='use-octember-bizcard-es-cache') core.Tags.of(sg_use_bizcard_es_cache).add( 'Name', 'use-octember-bizcard-es-cache') sg_bizcard_es_cache = aws_ec2.SecurityGroup( self, "BizcardSearchCacheSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard search query cache', security_group_name='octember-bizcard-es-cache') core.Tags.of(sg_bizcard_es_cache).add('Name', 'octember-bizcard-es-cache') sg_bizcard_es_cache.add_ingress_rule( peer=sg_use_bizcard_es_cache, connection=aws_ec2.Port.tcp(6379), description='use-octember-bizcard-es-cache') es_query_cache_subnet_group = aws_elasticache.CfnSubnetGroup( self, "QueryCacheSubnetGroup", description="subnet group for octember-bizcard-es-cache", subnet_ids=vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids, cache_subnet_group_name='octember-bizcard-es-cache') es_query_cache = aws_elasticache.CfnCacheCluster( self, "BizcardSearchQueryCache", cache_node_type="cache.t3.small", num_cache_nodes=1, engine="redis", engine_version="5.0.5", auto_minor_version_upgrade=False, cluster_name="octember-bizcard-es-cache", snapshot_retention_limit=3, snapshot_window="17:00-19:00", preferred_maintenance_window="mon:19:00-mon:20:30", #XXX: Do not use referece for "cache_subnet_group_name" - https://github.com/aws/aws-cdk/issues/3098 #cache_subnet_group_name=es_query_cache_subnet_group.cache_subnet_group_name, # Redis cluster goes to wrong VPC cache_subnet_group_name='octember-bizcard-es-cache', vpc_security_group_ids=[sg_bizcard_es_cache.security_group_id]) #XXX: If you're going to launch your cluster in an Amazon VPC, you need to create a subnet group before you start creating a cluster. # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticache-cache-cluster.html#cfn-elasticache-cachecluster-cachesubnetgroupname es_query_cache.add_depends_on(es_query_cache_subnet_group) #XXX: add more than 2 security groups # https://github.com/aws/aws-cdk/blob/ea10f0d141a48819ec0000cd7905feda993870a9/packages/%40aws-cdk/aws-lambda/lib/function.ts#L387 # https://github.com/aws/aws-cdk/issues/1555 # https://github.com/aws/aws-cdk/pull/5049 bizcard_search_lambda_fn = _lambda.Function( self, "BizcardSearchServer", runtime=_lambda.Runtime.PYTHON_3_7, function_name="BizcardSearchProxy", handler="es_search_bizcard.lambda_handler", description="Proxy server to search bizcard text", code=_lambda.Code.asset("./src/main/python/SearchBizcard"), environment={ 'ES_HOST': es_cfn_domain.attr_domain_endpoint, 'ES_INDEX': 'octember_bizcard', 'ES_TYPE': 'bizcard', 'ELASTICACHE_HOST': es_query_cache.attr_redis_endpoint_address }, timeout=core.Duration.minutes(1), layers=[es_lib_layer, redis_lib_layer], security_groups=[sg_use_bizcard_es, sg_use_bizcard_es_cache], vpc=vpc) #XXX: create API Gateway + LambdaProxy search_api = apigw.LambdaRestApi( self, "BizcardSearchAPI", handler=bizcard_search_lambda_fn, proxy=False, rest_api_name="BizcardSearch", description="This service serves searching bizcard text.", endpoint_types=[apigw.EndpointType.REGIONAL], deploy=True, deploy_options=apigw.StageOptions(stage_name="v1")) bizcard_search = search_api.root.add_resource('search') bizcard_search.add_method( "GET", method_responses=[ apigw.MethodResponse( status_code="200", response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ]) sg_use_bizcard_graph_db = aws_ec2.SecurityGroup( self, "BizcardGraphDbClientSG", vpc=vpc, allow_all_outbound=True, description='security group for octember bizcard graph db client', security_group_name='use-octember-bizcard-neptune') core.Tags.of(sg_use_bizcard_graph_db).add( 'Name', 'use-octember-bizcard-neptune') sg_bizcard_graph_db = aws_ec2.SecurityGroup( self, "BizcardGraphDbSG", vpc=vpc, allow_all_outbound=True, description='security group for octember bizcard graph db', security_group_name='octember-bizcard-neptune') core.Tags.of(sg_bizcard_graph_db).add('Name', 'octember-bizcard-neptune') sg_bizcard_graph_db.add_ingress_rule( peer=sg_bizcard_graph_db, connection=aws_ec2.Port.tcp(8182), description='octember-bizcard-neptune') sg_bizcard_graph_db.add_ingress_rule( peer=sg_use_bizcard_graph_db, connection=aws_ec2.Port.tcp(8182), description='use-octember-bizcard-neptune') bizcard_graph_db_subnet_group = aws_neptune.CfnDBSubnetGroup( self, "NeptuneSubnetGroup", db_subnet_group_description= "subnet group for octember-bizcard-neptune", subnet_ids=vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids, db_subnet_group_name='octember-bizcard-neptune') bizcard_graph_db = aws_neptune.CfnDBCluster( self, "BizcardGraphDB", availability_zones=vpc.availability_zones, db_subnet_group_name=bizcard_graph_db_subnet_group. db_subnet_group_name, db_cluster_identifier="octember-bizcard", backup_retention_period=1, preferred_backup_window="08:45-09:15", preferred_maintenance_window="sun:18:00-sun:18:30", vpc_security_group_ids=[sg_bizcard_graph_db.security_group_id]) bizcard_graph_db.add_depends_on(bizcard_graph_db_subnet_group) bizcard_graph_db_instance = aws_neptune.CfnDBInstance( self, "BizcardGraphDBInstance", db_instance_class="db.r5.large", allow_major_version_upgrade=False, auto_minor_version_upgrade=False, availability_zone=vpc.availability_zones[0], db_cluster_identifier=bizcard_graph_db.db_cluster_identifier, db_instance_identifier="octember-bizcard", preferred_maintenance_window="sun:18:00-sun:18:30") bizcard_graph_db_instance.add_depends_on(bizcard_graph_db) bizcard_graph_db_replica_instance = aws_neptune.CfnDBInstance( self, "BizcardGraphDBReplicaInstance", db_instance_class="db.r5.large", allow_major_version_upgrade=False, auto_minor_version_upgrade=False, availability_zone=vpc.availability_zones[-1], db_cluster_identifier=bizcard_graph_db.db_cluster_identifier, db_instance_identifier="octember-bizcard-replica", preferred_maintenance_window="sun:18:00-sun:18:30") bizcard_graph_db_replica_instance.add_depends_on(bizcard_graph_db) bizcard_graph_db_replica_instance.add_depends_on( bizcard_graph_db_instance) gremlinpython_lib_layer = _lambda.LayerVersion( self, "GremlinPythonLib", layer_version_name="gremlinpython-lib", compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], code=_lambda.Code.from_bucket( s3_lib_bucket, "var/octember-gremlinpython-lib.zip")) #XXX: https://github.com/aws/aws-cdk/issues/1342 upsert_to_neptune_lambda_fn = _lambda.Function( self, "UpsertBizcardToGraphDB", runtime=_lambda.Runtime.PYTHON_3_7, function_name="UpsertBizcardToNeptune", handler="upsert_bizcard_to_graph_db.lambda_handler", description="Upsert bizcard into neptune", code=_lambda.Code.asset( "./src/main/python/UpsertBizcardToGraphDB"), environment={ 'REGION_NAME': core.Aws.REGION, 'NEPTUNE_ENDPOINT': bizcard_graph_db.attr_endpoint, 'NEPTUNE_PORT': bizcard_graph_db.attr_port }, timeout=core.Duration.minutes(5), layers=[gremlinpython_lib_layer], security_groups=[sg_use_bizcard_graph_db], vpc=vpc) upsert_to_neptune_lambda_fn.add_event_source(text_kinesis_event_source) log_group = aws_logs.LogGroup( self, "UpsertBizcardToGraphDBLogGroup", log_group_name="/aws/lambda/UpsertBizcardToNeptune", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(upsert_to_neptune_lambda_fn) sg_use_bizcard_neptune_cache = aws_ec2.SecurityGroup( self, "BizcardNeptuneCacheClientSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard recommendation query cache client', security_group_name='use-octember-bizcard-neptune-cache') core.Tags.of(sg_use_bizcard_neptune_cache).add( 'Name', 'use-octember-bizcard-es-cache') sg_bizcard_neptune_cache = aws_ec2.SecurityGroup( self, "BizcardNeptuneCacheSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard recommendation query cache', security_group_name='octember-bizcard-neptune-cache') core.Tags.of(sg_bizcard_neptune_cache).add( 'Name', 'octember-bizcard-neptune-cache') sg_bizcard_neptune_cache.add_ingress_rule( peer=sg_use_bizcard_neptune_cache, connection=aws_ec2.Port.tcp(6379), description='use-octember-bizcard-neptune-cache') recomm_query_cache_subnet_group = aws_elasticache.CfnSubnetGroup( self, "RecommQueryCacheSubnetGroup", description="subnet group for octember-bizcard-neptune-cache", subnet_ids=vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids, cache_subnet_group_name='octember-bizcard-neptune-cache') recomm_query_cache = aws_elasticache.CfnCacheCluster( self, "BizcardRecommQueryCache", cache_node_type="cache.t3.small", num_cache_nodes=1, engine="redis", engine_version="5.0.5", auto_minor_version_upgrade=False, cluster_name="octember-bizcard-neptune-cache", snapshot_retention_limit=3, snapshot_window="17:00-19:00", preferred_maintenance_window="mon:19:00-mon:20:30", #XXX: Do not use referece for "cache_subnet_group_name" - https://github.com/aws/aws-cdk/issues/3098 #cache_subnet_group_name=recomm_query_cache_subnet_group.cache_subnet_group_name, # Redis cluster goes to wrong VPC cache_subnet_group_name='octember-bizcard-neptune-cache', vpc_security_group_ids=[ sg_bizcard_neptune_cache.security_group_id ]) recomm_query_cache.add_depends_on(recomm_query_cache_subnet_group) bizcard_recomm_lambda_fn = _lambda.Function( self, "BizcardRecommender", runtime=_lambda.Runtime.PYTHON_3_7, function_name="BizcardRecommender", handler="neptune_recommend_bizcard.lambda_handler", description="This service serves PYMK(People You May Know).", code=_lambda.Code.asset("./src/main/python/RecommendBizcard"), environment={ 'REGION_NAME': core.Aws.REGION, 'NEPTUNE_ENDPOINT': bizcard_graph_db.attr_read_endpoint, 'NEPTUNE_PORT': bizcard_graph_db.attr_port, 'ELASTICACHE_HOST': recomm_query_cache.attr_redis_endpoint_address }, timeout=core.Duration.minutes(1), layers=[gremlinpython_lib_layer, redis_lib_layer], security_groups=[ sg_use_bizcard_graph_db, sg_use_bizcard_neptune_cache ], vpc=vpc) #XXX: create API Gateway + LambdaProxy recomm_api = apigw.LambdaRestApi( self, "BizcardRecommendAPI", handler=bizcard_recomm_lambda_fn, proxy=False, rest_api_name="BizcardRecommend", description="This service serves PYMK(People You May Know).", endpoint_types=[apigw.EndpointType.REGIONAL], deploy=True, deploy_options=apigw.StageOptions(stage_name="v1")) bizcard_recomm = recomm_api.root.add_resource('pymk') bizcard_recomm.add_method( "GET", method_responses=[ apigw.MethodResponse( status_code="200", response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ]) sagemaker_notebook_role_policy_doc = aws_iam.PolicyDocument() sagemaker_notebook_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ "arn:aws:s3:::aws-neptune-notebook", "arn:aws:s3:::aws-neptune-notebook/*" ], "actions": ["s3:GetObject", "s3:ListBucket"] })) sagemaker_notebook_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ "arn:aws:neptune-db:{region}:{account}:{cluster_id}/*". format(region=core.Aws.REGION, account=core.Aws.ACCOUNT_ID, cluster_id=bizcard_graph_db. attr_cluster_resource_id) ], "actions": ["neptune-db:connect"] })) sagemaker_notebook_role = aws_iam.Role( self, 'SageMakerNotebookForNeptuneWorkbenchRole', role_name='AWSNeptuneNotebookRole-OctemberBizcard', assumed_by=aws_iam.ServicePrincipal('sagemaker.amazonaws.com'), #XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221 inline_policies={ 'AWSNeptuneNotebook': sagemaker_notebook_role_policy_doc }) neptune_wb_lifecycle_content = '''#!/bin/bash sudo -u ec2-user -i <<'EOF' echo "export GRAPH_NOTEBOOK_AUTH_MODE=DEFAULT" >> ~/.bashrc echo "export GRAPH_NOTEBOOK_HOST={NeptuneClusterEndpoint}" >> ~/.bashrc echo "export GRAPH_NOTEBOOK_PORT={NeptuneClusterPort}" >> ~/.bashrc echo "export NEPTUNE_LOAD_FROM_S3_ROLE_ARN=''" >> ~/.bashrc echo "export AWS_REGION={AWS_Region}" >> ~/.bashrc aws s3 cp s3://aws-neptune-notebook/graph_notebook.tar.gz /tmp/graph_notebook.tar.gz rm -rf /tmp/graph_notebook tar -zxvf /tmp/graph_notebook.tar.gz -C /tmp /tmp/graph_notebook/install.sh EOF '''.format(NeptuneClusterEndpoint=bizcard_graph_db.attr_endpoint, NeptuneClusterPort=bizcard_graph_db.attr_port, AWS_Region=core.Aws.REGION) neptune_wb_lifecycle_config_prop = aws_sagemaker.CfnNotebookInstanceLifecycleConfig.NotebookInstanceLifecycleHookProperty( content=core.Fn.base64(neptune_wb_lifecycle_content)) neptune_wb_lifecycle_config = aws_sagemaker.CfnNotebookInstanceLifecycleConfig( self, 'NpetuneWorkbenchLifeCycleConfig', notebook_instance_lifecycle_config_name= 'AWSNeptuneWorkbenchOctemberBizcardLCConfig', on_start=[neptune_wb_lifecycle_config_prop]) neptune_workbench = aws_sagemaker.CfnNotebookInstance( self, 'NeptuneWorkbench', instance_type='ml.t2.medium', role_arn=sagemaker_notebook_role.role_arn, lifecycle_config_name=neptune_wb_lifecycle_config. notebook_instance_lifecycle_config_name, notebook_instance_name='OctemberBizcard-NeptuneWorkbench', root_access='Disabled', security_group_ids=[sg_use_bizcard_graph_db.security_group_name], subnet_id=bizcard_graph_db_subnet_group.subnet_ids[0])
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: """ initialize function for CDK """ super().__init__(scope, construct_id, **kwargs) # ------------------------------- # S3 Bucket for Manifests # ------------------------------- qs_gov_bucket = s3.Bucket( self, id=f"{cf.PROJECT}-Bucket", ) bucket_name = qs_gov_bucket.bucket_name # ------------------------------- # IAM # ------------------------------- list_roles_policy = iam.ManagedPolicy( self, id=f"{cf.PROJECT}-ListRolesPolicy", managed_policy_name=f"{cf.PROJECT}-ListRolesPolicy", description=None, path="/", statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=["iam:ListRoles", "iam:ListAccountAliases"], ) ], ) federated_quicksight_policy = iam.ManagedPolicy( self, id=f"{cf.PROJECT}-FederatedQuickSightPolicy", managed_policy_name=f"{cf.PROJECT}-FederatedQuickSightPolicy", statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:iam::{cf.ACCOUNT}:saml-provider/{cf.OKTA_IDP_NAME}" ], actions=["sts:AssumeRoleWithSAML"], conditions={ "StringEquals": { "saml:aud": "https://signin.aws.amazon.com/saml" } }, ) ], ) okta_federated_principal = iam.FederatedPrincipal( federated= f"arn:aws:iam::{cf.ACCOUNT}:saml-provider/{cf.OKTA_IDP_NAME}", assume_role_action="sts:AssumeRoleWithSAML", conditions={ "StringEquals": { "SAML:aud": "https://signin.aws.amazon.com/saml" } }, ) federated_quicksight_role = iam.Role( self, id=f"{cf.PROJECT}-{cf.QS_FEDERATED_ROLE_NAME}", role_name=f"{cf.QS_FEDERATED_ROLE_NAME}", assumed_by=okta_federated_principal, description= "Allow Okta to Federate Login & User Creation to QuickSight", managed_policies=[federated_quicksight_policy], ) iam.User( self, id=f"{cf.PROJECT}-OktaSSOUser", user_name=f"{cf.PROJECT}-OktaSSOUser", managed_policies=[list_roles_policy], ) # ------------------------------- # Lambda Layers # ------------------------------- path_src = os.path.join(cf.PATH_SRC, "") sp.call(["make", "bundle"], cwd=path_src) requests_layer = _lambda.LayerVersion( self, f"{cf.PROJECT}-requests-layer", code=_lambda.Code.from_asset(os.path.join(path_src, "requests.zip"))) sp.call(["make", "clean"], cwd=path_src) # ------------------------------- # Lambda Functions # ------------------------------- # iam role for Lambdas qs_governance_policy = iam.ManagedPolicy( self, id=f"{cf.PROJECT}-LambdaPolicy", managed_policy_name=f"{cf.PROJECT}-LambdaPolicy", statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=["ses:SendEmail", "ses:SendRawEmail"], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:secretsmanager:{cf.REGION}:{cf.ACCOUNT}:secret:{cf.OKTA_SECRET_ID}*", f"arn:aws:secretsmanager:{cf.REGION}:{cf.ACCOUNT}:secret:{cf.SLACK_SECRET_ID}*" ], actions=[ "secretsmanager:GetSecretValue", ], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:iam::{cf.ACCOUNT}:policy/{cf.QS_PREFIX}*" ], actions=[ "iam:CreatePolicy", "iam:GetPolicy", "iam:DeletePolicy", ], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=["iam:ListPolicies"], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=[ "glue:GetDatabase", "glue:GetDatabases", "glue:GetTable", "glue:GetPartitions", "glue:GetPartition", "glue:GetTables", ], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=["quicksight:*", "ds:*"], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:s3:::{bucket_name}/*", f"arn:aws:s3:::{bucket_name}*" ], actions=["s3:*"], ), ], ) qs_governance_role = iam.Role( self, id=f"{cf.PROJECT}-QuickSightPermissionMappingRole", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaBasicExecutionRole"), qs_governance_policy, ], ) # Lambdas get_okta_information_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-GetOktaInfo", function_name=f"{cf.PROJECT}-GetOktaInformation", handler="get_okta_information.handler", role=qs_governance_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), environment={ "OKTA_SECRET_ID": cf.OKTA_SECRET_ID, "QS_PREFIX": cf.QS_PREFIX, "QS_GOVERNANCE_BUCKET": bucket_name, "QS_FEDERATED_ROLE_NAME": f"{cf.QS_FEDERATED_ROLE_NAME}", "S3_PREFIX_USERS": cf.S3_PREFIX_USERS, "SLACK_SECRET_ID": cf.SLACK_SECRET_ID }, memory_size=256, timeout=core.Duration.seconds(180), layers=[requests_layer], ) # Lamda Okta to QuickSight Mappers qs_user_governance_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-QSUserGovernance", function_name=f"{cf.PROJECT}-QSUserGovernance", handler="qs_user_governance.handler", role=qs_governance_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), environment={ "QS_FEDERATED_ROLE_NAME": f"{cf.QS_FEDERATED_ROLE_NAME}", "QS_PREFIX": cf.QS_PREFIX, "QS_ADMIN_OKTA_GROUP": cf.QS_ADMIN_OKTA_GROUP, "QS_AUTHOR_OKTA_GROUP": cf.QS_AUTHOR_OKTA_GROUP, "QS_READER_OKTA_GROUP": cf.QS_READER_OKTA_GROUP, "SLACK_SECRET_ID": cf.SLACK_SECRET_ID }, memory_size=256, timeout=core.Duration.seconds(180), layers=[requests_layer], ) qs_asset_governance_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-QSAssetGovernance", function_name=f"{cf.PROJECT}-QSAssetGovernance", handler="qs_asset_governance.handler", role=qs_governance_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), memory_size=256, timeout=core.Duration.seconds(180), layers=[requests_layer], environment={ "QS_GOVERNANCE_BUCKET": f"{bucket_name}", "S3_PREFIX_USERS": cf.S3_PREFIX_USERS, "S3_PREFIX_POLICIES": cf.S3_PREFIX_POLICIES, "STAGE": cf.DEPLOYMENT_STAGE, "SLACK_SECRET_ID": cf.SLACK_SECRET_ID, "VERIFIED_EMAIL": cf.VERIFIED_EMAIL }) qs_policy_governance_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-QSPolicyGovernance", function_name=f"{cf.PROJECT}-QSPolicyGovernance", handler="qs_policy_governance.handler", role=qs_governance_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), environment={ "QS_ADMIN_OKTA_GROUP": cf.QS_ADMIN_OKTA_GROUP, "QS_AUTHOR_OKTA_GROUP": cf.QS_AUTHOR_OKTA_GROUP, "QS_AUTHOR_BASE_POLICY": cf.QS_AUTHOR_BASE_POLICY, "QS_PREFIX": cf.QS_PREFIX, "QS_GOVERNANCE_BUCKET": f"{bucket_name}", "S3_PREFIX_POLICIES": cf.S3_PREFIX_POLICIES, "QS_SUPERUSER": cf.QS_SUPERUSER, "DEPLOYMENT_STAGE": cf.DEPLOYMENT_STAGE, "SLACK_SECRET_ID": cf.SLACK_SECRET_ID }, memory_size=256, timeout=core.Duration.seconds(180), layers=[requests_layer]) qs_resource_cleanup_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-QSResourceCleanup", function_name=f"{cf.PROJECT}-QSResourceCleanup", handler="qs_resource_cleanup.handler", role=qs_governance_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), memory_size=256, timeout=core.Duration.seconds(180), layers=[requests_layer], environment={ "QS_PREFIX": cf.QS_PREFIX, "QS_SUPERUSER": cf.QS_SUPERUSER, "QS_GOVERNANCE_BUCKET": f"{bucket_name}", "S3_PREFIX_ASSETS": cf.S3_PREFIX_ASSETS, "S3_PREFIX_USERS": cf.S3_PREFIX_USERS, "S3_PREFIX_POLICIES": cf.S3_PREFIX_POLICIES, "SLACK_SECRET_ID": cf.SLACK_SECRET_ID }) # ------------------------------- # S3 Event Triggers # ------------------------------- qs_user_governance_lambda.add_event_source( lambda_event_sources.S3EventSource( bucket=qs_gov_bucket, events=[s3.EventType.OBJECT_CREATED], filters=[s3.NotificationKeyFilter(prefix=cf.S3_PREFIX_USERS)], )) qs_policy_governance_lambda.add_event_source( lambda_event_sources.S3EventSource( bucket=qs_gov_bucket, events=[s3.EventType.OBJECT_CREATED], filters=[ s3.NotificationKeyFilter(prefix=cf.S3_PREFIX_POLICIES) ], )) # ------------------------------- # CloudWatch Event Rules (30 minutes) # ------------------------------- lambda_schedule = events.Schedule.rate(core.Duration.minutes(30)) get_okta_info_target = events_targets.LambdaFunction( handler=get_okta_information_lambda) events.Rule( self, id=f"{cf.PROJECT}-GetOktaInfoScheduledEvent", description="The scheduled CloudWatch event trigger for the Lambda", enabled=True, schedule=lambda_schedule, targets=[get_okta_info_target], ) qs_asset_gov_target = events_targets.LambdaFunction( handler=qs_asset_governance_lambda) events.Rule( self, id=f"{cf.PROJECT}-QSAssetGovScheduledEvent", description="The scheduled CloudWatch event trigger for the Lambda", enabled=True, schedule=lambda_schedule, targets=[qs_asset_gov_target], ) qs_cleanup_target = events_targets.LambdaFunction( handler=qs_resource_cleanup_lambda) events.Rule( self, id=f"{cf.PROJECT}-QSCleanupScheduledEvent", description="The scheduled CloudWatch event trigger for the Lambda", enabled=True, schedule=lambda_schedule, targets=[qs_cleanup_target], ) # ------------------------------- # Athena WorkGroup # ------------------------------- workgroup_encryption = athena.CfnWorkGroup.EncryptionConfigurationProperty( encryption_option='SSE_S3') workgroup_output = athena.CfnWorkGroup.ResultConfigurationProperty( output_location="s3://" + cf.ATHENA_OUTPUT_LOC, encryption_configuration=workgroup_encryption) quicksight_athena_workgroup = athena.CfnWorkGroup( self, id=f"{cf.PROJECT}-workgroup", name=f"{cf.QS_PREFIX}-workgroup", description="workgroup for QuickSight Data Source operations", recursive_delete_option=True, work_group_configuration=athena.CfnWorkGroup. WorkGroupConfigurationProperty( result_configuration=workgroup_output)) workgroup_name = quicksight_athena_workgroup.name # ------------------------------- # S3 Object Deployments # ------------------------------- policy_manifest_deploy = s3_deploy.BucketDeployment( self, id=f"{cf.PROJECT}-PolicyManifestDeploy", sources=[ s3_deploy.Source.asset( os.path.join(cf.PATH_ROOT, "qs_config/policies")) ], destination_bucket=qs_gov_bucket, destination_key_prefix=cf.S3_PREFIX_POLICIES) # ------------------------------- # QuickSight IAM Policy Assignments # ------------------------------- qs_author_base_policy = iam.ManagedPolicy( self, id=f"{cf.PROJECT}-{cf.QS_AUTHOR_BASE_POLICY}", managed_policy_name=f"{cf.QS_AUTHOR_BASE_POLICY}", statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:athena:{cf.REGION}:{cf.ACCOUNT}:workgroup/{workgroup_name}" ], actions=[ "athena:GetNamedQuery", "athena:CancelQueryExecution", "athena:CreateNamedQuery", "athena:DeleteNamedQuery", "athena:StartQueryExecution", "athena:StopQueryExecution", "athena:GetWorkGroup", "athena:GetNamedQuery", "athena:GetQueryResults", "athena:GetQueryExecution", "athena:BatchGetQueryExecution", "athena:BatchGetNamedQuery", "athena:ListNamedQueries", "athena:ListQueryExecutions", "athena:GetQueryResultsStream" ]), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=[ "athena:ListWorkGroups", "athena:ListDataCatalogs", "athena:StartQueryExecution", "athena:GetQueryExecution", "athena:GetQueryResultsStream", "athena:ListTableMetadata", "athena:GetTableMetadata" ]), iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[ "arn:aws:s3:::ast-datalake-*", "arn:aws:s3:::aws-athena-query-*", f"arn:aws:s3:::{bucket_name}*", f"arn:aws:s3:::{cf.ATHENA_OUTPUT_LOC}*" ], actions=[ "s3:List*", "s3:Get*", ]), iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=["*"], actions=["s3:ListAllMyBuckets"]), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ "arn:aws:s3:::aws-athena-query-*", f"arn:aws:s3:::{bucket_name}/{cf.QS_PREFIX}/athena-results*" ], actions=["s3:Put*"]), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:athena:{cf.REGION}:{cf.ACCOUNT}:datacatalog/AwsDataCatalog" ], actions=["athena:ListDatabases"]), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=["kms:Decrypt", "kms:GenerateDataKey"]), ], )
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id) lambda_dir = kwargs["lambda_dir"] # Note: A dead-letter queue is optional but it helps capture any failed messages dlq = sqs.Queue(self, id="dead_letter_queue_id", retention_period=Duration.days(7)) dead_letter_queue = sqs.DeadLetterQueue(max_receive_count=1, queue=dlq) upload_queue = sqs.Queue(self, id="sample_queue_id", visibility_timeout=Duration.seconds(30), dead_letter_queue=dead_letter_queue) sqs_subscription = sns_subs.SqsSubscription(upload_queue, raw_message_delivery=True) upload_event_topic = sns.Topic(self, id="sample_sns_topic_id") # This binds the SNS Topic to the SQS Queue upload_event_topic.add_subscription(sqs_subscription) # Note: Lifecycle Rules are optional but are included here to keep costs # low by cleaning up old files or moving them to lower cost storage options s3_bucket = s3.Bucket( self, id="sample_bucket_id", block_public_access=s3.BlockPublicAccess.BLOCK_ALL, versioned=True, lifecycle_rules=[ s3.LifecycleRule( enabled=True, expiration=Duration.days(365), transitions=[ s3.Transition( storage_class=s3.StorageClass.INFREQUENT_ACCESS, transition_after=Duration.days(30)), s3.Transition(storage_class=s3.StorageClass.GLACIER, transition_after=Duration.days(90)), ]) ]) # Note: If you don't specify a filter all uploads will trigger an event. # Also, modifying the event type will handle other object operations # This binds the S3 bucket to the SNS Topic s3_bucket.add_event_notification( s3.EventType.OBJECT_CREATED_PUT, s3n.SnsDestination(upload_event_topic), s3.NotificationKeyFilter(prefix="uploads", suffix=".csv")) function = _lambda.Function( self, "lambda_function", runtime=_lambda.Runtime.PYTHON_3_9, handler="lambda_function.handler", code=_lambda.Code.from_asset(path=lambda_dir)) # This binds the lambda to the SQS Queue invoke_event_source = lambda_events.SqsEventSource(upload_queue) function.add_event_source(invoke_event_source) # Examples of CloudFormation outputs CfnOutput( self, "UploadFileToS3Example", value="aws s3 cp <local-path-to-file> s3://{}/".format( s3_bucket.bucket_name), description= "Upload a file to S3 (using AWS CLI) to trigger the SQS chain", ) CfnOutput( self, "UploadSqsQueueUrl", value=upload_queue.queue_url, description="Link to the SQS Queue triggered on S3 uploads", ) CfnOutput( self, "LambdaFunctionName", value=function.function_name, ) CfnOutput( self, "LambdaFunctionLogGroupName", value=function.log_group.log_group_name, )
def __init__(self, scope: core.Construct, id: str, instance_id: str, contact_flow_id: str, source_phone_number: str, timeout: int, **kwargs): super().__init__(scope, id, **kwargs) web_bucket = _s3.Bucket(self, "StaticWebBucket", website_index_document="index.html", website_error_document="index.html", removal_policy=core.RemovalPolicy.DESTROY, public_read_access=True) core.CfnOutput(self, 'WebBucketUrl', value=web_bucket.bucket_domain_name) web_distribution = _clf.CloudFrontWebDistribution( self, 'StaticWebDistribution', origin_configs=[ _clf.SourceConfiguration( s3_origin_source=_clf.S3OriginConfig( s3_bucket_source=web_bucket), behaviors=[_clf.Behavior(is_default_behavior=True)]) ], viewer_protocol_policy=_clf.ViewerProtocolPolicy.REDIRECT_TO_HTTPS) _s3d.BucketDeployment( self, "S3StaticWebContentDeploymentWithInvalidation", sources=[ _s3d.Source.asset( f"{pathlib.Path(__file__).parent.absolute()}/site-content/build" ) ], destination_bucket=web_bucket, distribution=web_distribution, distribution_paths=["/*"]) file_bucket = _s3.Bucket(self, "FileBucket", removal_policy=core.RemovalPolicy.DESTROY) call_dead_letter_queue = _sqs.Queue(self, "CallDeadLetterQueue", fifo=True, content_based_deduplication=True) call_sqs_queue = _sqs.Queue( self, "CallSqsQueue", fifo=True, content_based_deduplication=True, visibility_timeout=core.Duration.seconds(120), dead_letter_queue=_sqs.DeadLetterQueue( max_receive_count=1, queue=call_dead_letter_queue)) async_call_dead_letter_queue = _sqs.Queue( self, "AsyncCallDeadLetterQueue", fifo=True, content_based_deduplication=True) async_callout_queue = _sqs.Queue( self, "AsyncCalloutQueue", fifo=True, content_based_deduplication=True, visibility_timeout=core.Duration.seconds(120), dead_letter_queue=_sqs.DeadLetterQueue( max_receive_count=1, queue=async_call_dead_letter_queue)) call_job_complete_sns_topic = _sns.Topic( self, "CallJobCompleteSnsTopic", display_name="CallJobCompletion") call_result_table = _dynamodb.Table( self, "CallResultDynamodbTable", billing_mode=_dynamodb.BillingMode.PAY_PER_REQUEST, partition_key=_dynamodb.Attribute( name="task_id", type=_dynamodb.AttributeType.STRING), sort_key=_dynamodb.Attribute(name="receiver_id", type=_dynamodb.AttributeType.STRING), removal_policy=core.RemovalPolicy.DESTROY) callout_record_table = _dynamodb.Table( self, "CallTaskDynamodbTable", billing_mode=_dynamodb.BillingMode.PAY_PER_REQUEST, partition_key=_dynamodb.Attribute( name="task_id", type=_dynamodb.AttributeType.STRING), sort_key=_dynamodb.Attribute(name="created_at", type=_dynamodb.AttributeType.NUMBER), removal_policy=core.RemovalPolicy.DESTROY) callout_record_table.add_global_secondary_index( partition_key=_dynamodb.Attribute( name='call_type', type=_dynamodb.AttributeType.STRING), sort_key=_dynamodb.Attribute(name='created_at', type=_dynamodb.AttributeType.NUMBER), index_name='CallTypeCreatedAtGlobalIndex', projection_type=_dynamodb.ProjectionType.ALL) python_function_layer = _lambda.LayerVersion( self, "LambdaPythonFunctionLayer", code=_lambda.Code.asset("aws_callouts_cdk/layer/_python"), compatible_runtimes=[ _lambda.Runtime.PYTHON_3_7, _lambda.Runtime.PYTHON_3_8 ], license="Available under the MIT-0 license") nodejs_function_layer = _lambda.LayerVersion( self, "LambdaNodeJsFunctionLayer", code=_lambda.Code.asset("aws_callouts_cdk/layer/_nodejs"), compatible_runtimes=[ _lambda.Runtime.NODEJS_10_X, _lambda.Runtime.NODEJS_12_X ], license="Available under the MIT-0 license") global_python_function_arguments = { "code": _lambda.Code.asset("aws_callouts_cdk/src/python"), "layers": [python_function_layer], "runtime": _lambda.Runtime.PYTHON_3_7 } global_nodeje_function_arguments = { "code": _lambda.Code.asset("aws_callouts_cdk/src/nodejs"), "layers": [nodejs_function_layer], "runtime": _lambda.Runtime.NODEJS_12_X } get_callout_job_function = _lambda.Function( self, "GetCalloutJobFunction", handler="get_call_job.lambda_handler", **global_python_function_arguments) get_callout_job_function.add_environment(key="S3Bucket", value=file_bucket.bucket_name) file_bucket.grant_read(get_callout_job_function) callout_function = _lambda.Function(self, "CalloutFunction", handler="send_call.lambda_handler", **global_python_function_arguments) callout_function.add_environment( key="ContactFlowArn", value= f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}/contact-flow/{contact_flow_id}" ) callout_function.add_environment(key="SourcePhoneNumber", value=source_phone_number) callout_function.add_environment(key="ExcelFileBucket", value=file_bucket.bucket_name) callout_function.add_environment(key="AsynCalloutQueueUrl", value=async_callout_queue.queue_url) callout_function.add_to_role_policy(statement=_iam.PolicyStatement( resources=[ f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}/*" ], actions=["connect:StartOutboundVoiceContact"])) callout_function.add_event_source(source=_les.SqsEventSource( queue=async_callout_queue, batch_size=1)) 'arn:aws:connect:751225572132:ap-southeast-2:instance/9d0c7cc5-7d2a-42e4-a3dd-70f402e0d040' file_bucket.grant_read_write(callout_function) response_handler_function = _lambda.Function( self, "ResponseHandlerFunction", handler="response_handler.lambda_handler", **global_python_function_arguments) response_handler_function.add_permission( id="ResponseHandlerFunctionLambdaInvokePermission", principal=_iam.ServicePrincipal(service="connect.amazonaws.com"), action="lambda:InvokeFunction", source_account=self.account, source_arn= f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}" ) send_task_success_function = _lambda.Function( self, "SendTaskSuccessFunction", handler="send_task_success.lambda_handler", **global_python_function_arguments) send_task_success_function.add_permission( id="SendTaskSuccessFunctionLambdaInvokePermission", principal=_iam.ServicePrincipal(service="connect.amazonaws.com"), action="lambda:InvokeFunction", source_account=self.account, source_arn= f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}" ) get_call_result_function = _lambda.Function( self, "GetCallResultFunction", handler="get_call_result.lambda_handler", memory_size=512, **global_python_function_arguments) get_call_result_function.add_environment( key="CallResultDynamoDBTable", value=call_result_table.table_name) get_call_result_function.add_environment(key="S3Bucket", value=file_bucket.bucket_name) call_result_table.grant_read_data(grantee=get_call_result_function) file_bucket.grant_read_write(get_call_result_function) iterator_function = _lambda.Function( self, "IteratorFunction", handler="iterator.lambda_handler", **global_python_function_arguments) iterator_function.add_permission( id="IteratorFunctionLambdaInvokePermission", principal=_iam.ServicePrincipal(service="connect.amazonaws.com"), action="lambda:InvokeFunction", source_account=self.account, source_arn= f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}" ) create_appsync_call_task_function = _lambda.Function( self, "CreateAppSyncCallTaskFunction", handler="create_appsync_call_task.lambda_handler", **global_nodeje_function_arguments) create_appsync_call_task_function.add_environment( key="CallSqsQueueUrl", value=call_sqs_queue.queue_url) create_appsync_call_task_function.add_environment( key="CallRecordTableName", value=callout_record_table.table_name) call_sqs_queue.grant_send_messages(create_appsync_call_task_function) callout_record_table.grant_write_data( create_appsync_call_task_function) create_call_report_record_function = _lambda.Function( self, "CreateCallReportRecordFunction", handler="create_call_report_record.lambda_handler", **global_nodeje_function_arguments) create_excel_call_task_function = _lambda.Function( self, "CreateExcelCallTaskFunction", handler="create_excel_call_task.lambda_handler", **global_python_function_arguments) create_excel_call_task_function.add_environment( key="CallSqsQueueUrl", value=call_sqs_queue.queue_url) call_sqs_queue.grant_send_messages(create_excel_call_task_function) create_excel_call_task_function.add_event_source( source=_les.S3EventSource(bucket=file_bucket, events=[_s3.EventType.OBJECT_CREATED], filters=[ _s3.NotificationKeyFilter( prefix="call_task", suffix=".xlsx") ])) start_callout_flow_function = _lambda.Function( self, "StartCalloutFlowFunction", handler="start_call_out_flow.lambda_handler", reserved_concurrent_executions=1, **global_python_function_arguments) start_callout_flow_function.add_environment( key="CallSqsQueueUrl", value=call_sqs_queue.queue_url) start_callout_flow_function.add_environment( key="ResponseHandlerFunctionArn", value=response_handler_function.function_arn) start_callout_flow_function.add_environment( key="IteratorFunctionArn", value=iterator_function.function_arn) start_callout_flow_function.add_environment( key="SendTaskSuccessFunctionArn", value=send_task_success_function.function_arn) start_callout_flow_function.add_environment( key="S3Bucket", value=file_bucket.bucket_name) start_callout_flow_function.add_event_source( source=_les.SqsEventSource(queue=call_sqs_queue, batch_size=1)) file_bucket.grant_read_write(start_callout_flow_function) call_state_machine_definition = { "Comment": "Reading messages from an SQS queue and iteratively processing each message.", "StartAt": "Start", "States": { "Start": { "Type": "Pass", "Next": "Process Call Messages" }, "Process Call Messages": { "Type": "Map", "Next": "Get Call Result", "InputPath": "$", "ItemsPath": "$", "OutputPath": "$.[0]", "Iterator": { "StartAt": "Get Call out job", "States": { "Get Call out job": { "Type": "Task", "Resource": get_callout_job_function.function_arn, "Next": "Callout with AWS Connect" }, "Callout with AWS Connect": { "Type": "Task", "Resource": "arn:aws:states:::sqs:sendMessage.waitForTaskToken", "TimeoutSeconds": timeout, "Parameters": { "QueueUrl": async_callout_queue.queue_url, "MessageGroupId": "1", "MessageBody": { "Message.$": "$", "TaskToken.$": "$$.Task.Token" } }, "Catch": [{ "ErrorEquals": ["States.Timeout"], "ResultPath": None, "Next": "Call Timeout" }], "Next": "Save call result" }, "Call Timeout": { "Type": "Pass", "ResultPath": None, "Next": "Save call result" }, "Save call result": { "Type": "Task", "Resource": "arn:aws:states:::dynamodb:putItem", "Parameters": { "TableName": call_result_table.table_name, "Item": { "receiver_id": { "S.$": "$.receiver_id" }, "task_id": { "S.$": "$.task_id" }, "username": { "S.$": "$.username" }, "phone_number": { "S.$": "$.phone_number" }, "status": { "S.$": "$.status" }, "answers": { "S.$": "$.answers" }, "error": { "S.$": "$.error" }, "call_at": { "S.$": "$.call_at" } } }, "ResultPath": "$.Result", "OutputPath": "$.task_id", "End": True } } } }, "Get Call Result": { "Type": "Task", "Resource": get_call_result_function.function_arn, "Next": "Create Call Report Record" }, "Create Call Report Record": { "Type": "Task", "Resource": create_call_report_record_function.function_arn, "Next": "Send Completion message to SNS" }, "Send Completion message to SNS": { "Type": "Task", "Resource": "arn:aws:states:::sns:publish", "Parameters": { "TopicArn": call_job_complete_sns_topic.topic_arn, "Message.$": "$" }, "Next": "Finish" }, "Finish": { "Type": "Succeed" } } } callout_state_machine_role = _iam.Role( self, "CalloutStatesExecutionRole", assumed_by=_iam.ServicePrincipal( f"states.{self.region}.amazonaws.com")) callout_state_machine_role.add_to_policy( _iam.PolicyStatement( actions=[ "sqs:SendMessage", "dynamodb:PutItem", "lambda:InvokeFunction", "SNS:Publish" ], resources=[ async_callout_queue.queue_arn, call_result_table.table_arn, get_callout_job_function.function_arn, get_call_result_function.function_arn, call_job_complete_sns_topic.topic_arn, create_appsync_call_task_function.function_arn, create_call_report_record_function.function_arn ])) callout_state_machine = _sfn.CfnStateMachine( self, "CalloutStateMachine", role_arn=callout_state_machine_role.role_arn, definition_string=json.dumps(call_state_machine_definition)) send_task_success_function.add_to_role_policy( _iam.PolicyStatement(actions=["states:SendTaskSuccess"], resources=[callout_state_machine.ref])) start_callout_flow_function.add_environment( key="CalloutStateMachineArn", value=callout_state_machine.ref) start_callout_flow_function.add_to_role_policy( _iam.PolicyStatement(effect=_iam.Effect.ALLOW, resources=[callout_state_machine.ref], actions=['states:StartExecution'])) user_pool = _cognito.UserPool( self, "UserPool", sign_in_type=_cognito.SignInType.USERNAME) user_pool_client = _cognito.UserPoolClient(self, "UserPoolClient", user_pool=user_pool) appsync_api = _appsync.GraphQLApi( self, "AppSyncApi", name="AWSCalloutApi", user_pool_config=_appsync.UserPoolConfig( user_pool=user_pool, default_action=_appsync.UserPoolDefaultAction.ALLOW), log_config=_appsync.LogConfig( field_log_level=_appsync.FieldLogLevel.ALL), schema_definition_file= f"{pathlib.Path(__file__).parent.absolute()}/schema.graphql") callout_record_ddb_ds = appsync_api.add_dynamo_db_data_source( name="CalloutRecordDdb", description="Callout Record DynamoDB Data Source", table=callout_record_table) callout_record_ddb_ds.create_resolver( type_name="Query", field_name="getLatestCallTaskRecords", request_mapping_template=_appsync.MappingTemplate.from_string( '{"version":"2017-02-28","operation":"Query","index":"CallTypeCreatedAtGlobalIndex","query":{"expression":"call_type = :call_type","expressionValues":{":call_type":{"S":"TASK"}}},"scanIndexForward":false,"limit":${ctx.args.limit}}' ), response_mapping_template=_appsync.MappingTemplate. dynamo_db_result_list()) callout_record_ddb_ds.create_resolver( type_name="Query", field_name="getLatestCallReportRecords", request_mapping_template=_appsync.MappingTemplate.from_string( '{"version":"2017-02-28","operation":"Query","index":"CallTypeCreatedAtGlobalIndex","query":{"expression":"call_type = :call_type","expressionValues":{":call_type":{"S":"REPORT"}}},"scanIndexForward":false,"limit":${ctx.args.limit}}' ), response_mapping_template=_appsync.MappingTemplate. dynamo_db_result_list()) callout_record_ddb_ds.create_resolver( type_name="Mutation", field_name="createCallReport", request_mapping_template=_appsync.MappingTemplate.from_string( '{"version":"2017-02-28","operation":"PutItem","key":{"task_id":{"S":"${ctx.args.report.task_id}"},"created_at":{"N":"${ctx.args.report.created_at}"}},"attributeValues":$util.dynamodb.toMapValuesJson($ctx.args.report)}' ), response_mapping_template=_appsync.MappingTemplate. dynamo_db_result_item()) call_task_lambda_ds = appsync_api.add_lambda_data_source( name="CallTaskLambda", description="Call Task Lambda Data Source", lambda_function=create_appsync_call_task_function) call_task_lambda_ds.create_resolver( type_name="Mutation", field_name="createCallTask", request_mapping_template=_appsync.MappingTemplate.lambda_request( "$utils.toJson($ctx.args)"), response_mapping_template=_appsync.MappingTemplate.lambda_result()) create_call_report_record_function.add_environment( value=appsync_api.graph_ql_url, key="AppSyncGraphQlApiUrl") create_call_report_record_function.add_to_role_policy( statement=_iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=['appsync:GraphQL'], resources=[ f"{appsync_api.arn}/types/Mutation/fields/createCallReport" ])) core.CfnOutput(self, id="OutputCallSqsQueue", value=call_sqs_queue.queue_arn) core.CfnOutput(self, id="OutputCallJobCompletionSNSTopic", value=call_job_complete_sns_topic.topic_arn) core.CfnOutput(self, id="OutputExcelFileS3Bucket", value=file_bucket.bucket_name) core.CfnOutput(self, id="OutputStaticWebS3Bucket", value=web_bucket.bucket_name) core.CfnOutput(self, id="OutputStaticWebUrl", value=web_bucket.bucket_website_url) identity_pool = _cognito.CfnIdentityPool( self, "IdentityPool", allow_unauthenticated_identities=True, cognito_identity_providers=[ _cognito.CfnIdentityPool.CognitoIdentityProviderProperty( provider_name=user_pool.user_pool_provider_name, client_id=user_pool_client.user_pool_client_id) ]) identity_pool_unauthorized_role = _iam.Role( self, 'IdentityPoolUnAuthorizedRole', assumed_by=_iam.FederatedPrincipal( federated="cognito-identity.amazonaws.com", assume_role_action="sts:AssumeRoleWithWebIdentity", conditions={ "StringEquals": { "cognito-identity.amazonaws.com:aud": identity_pool.ref }, "ForAnyValue:StringLike": { "cognito-identity.amazonaws.com:amr": "unauthenticated" } })) identity_pool_unauthorized_role.add_to_policy( _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=["appsync:GraphQL"], resources=[ f"{appsync_api.arn}/types/*", # f"{appsync_api.arn}/types/Query/fields/getLatestCallTaskRecords", # f"{appsync_api.arn}/types/Query/fields/getLatestCallReportRecords", # f"{appsync_api.arn}/types/Mutation/fields/createCallRecord", # f"{appsync_api.arn}/types/Subscription/fields/createCallTask", # f"{appsync_api.arn}/types/Subscription/fields/createCallReport" ])) _cognito.CfnIdentityPoolRoleAttachment( self, "CognitoIdentityPoolRoleAttachment", identity_pool_id=identity_pool.ref, roles={ "unauthenticated": identity_pool_unauthorized_role.role_arn }) core.CfnOutput(self, id="UserPoolId", value=user_pool.user_pool_id) core.CfnOutput(self, id="UserPoolClientId", value=user_pool_client.user_pool_client_id) core.CfnOutput(self, id="IdentityPoolId", value=identity_pool.ref)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) if kwargs['env'].account == '828661178764': gbc_environment = 'devl' else: raise ValueError('Account not mapped!') # Define location of lambda deployment packages: bucket_source = s3.Bucket.from_bucket_name(self, 'bucket_source_id', bucket_name='gbc-lambda') generic_loader = _lambda.S3Code( bucket_source, key='generic-loader/generic-loader-test2.zip') # Define attributes for the networking part: _vpc = ec2.Vpc.from_vpc_attributes( self, 'myVPC', vpc_id='vpc-042f6b22897562107', availability_zones=['ca-central-1a', 'ca-central-1b']) _subnets = [ ec2.Subnet.from_subnet_id(self, 'subnet1', subnet_id='subnet-0579afb06d9cec8ed'), ec2.Subnet.from_subnet_id(self, 'subnet2', subnet_id='subnet-07a0e458dc7ea0228') ] _security_group = [ ec2.SecurityGroup.from_security_group_id( self, 'mySG', security_group_id='sg-0264ea677ccfef4ff') ] my_lambda = _lambda.Function( self, 'redshift-generic-loader', runtime=_lambda.Runtime.PYTHON_3_6, code=generic_loader, timeout=core.Duration.seconds(30), handler='lambda_function.lambda_handler', role=iam.Role.from_role_arn( self, 'myRole', role_arn='arn:aws:iam::828661178764:role/lambda-data-analytics', mutable=False), vpc=_vpc, vpc_subnets=ec2.SubnetSelection(subnets=_subnets), security_groups=_security_group) # Create main bucket for pipelines and register notifications: main_bucket = s3.Bucket( self, 'new_bucket_id', bucket_name=f'gbc-analytics-prototype-1-cdk-{gbc_environment}', versioned=True, encryption=s3.BucketEncryption.S3_MANAGED) target_lambda = s3_notif.LambdaDestination(my_lambda) key_filter_1 = s3.NotificationKeyFilter(prefix='cdk-test/') main_bucket.add_event_notification(s3.EventType.OBJECT_CREATED, target_lambda, key_filter_1)
def __init__(self, scope: core.Construct, id: str, log_bucket: _s3.Bucket, config_table: _dynamodb.Table, tshirt_size: str, sink_bucket: _s3.Bucket, web_sale_stream: str, web_customer_stream: str, web_customer_address_stream: str, kinesis_key: _kms.Key, vpc: _ec2.Vpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) stack = core.Stack.of(self) stream_source_bucket = AutoEmptyBucket( self, 'StreamSource', bucket_name='ara-stream-source-' + core.Aws.ACCOUNT_ID, uuid='95505f50-0276-11eb-adc1-0242ac120002') service_role = _iam.Role( self, 'StreamEmrServiceRole', assumed_by=_iam.ServicePrincipal('elasticmapreduce.amazonaws.com')) service_role.add_managed_policy( _iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonElasticMapReduceRole')) cluster_role = _iam.Role( self, 'StreamEmrClusterRole', assumed_by=_iam.ServicePrincipal("ec2.amazonaws.com")) _iam.Policy( self, 'StreamEmrClusterPolicy', statements=[ _iam.PolicyStatement(actions=[ "glue:CreateDatabase", "glue:UpdateDatabase", "glue:DeleteDatabase", "glue:GetDatabase", "glue:GetDatabases", "glue:CreateTable", "glue:UpdateTable", "glue:DeleteTable", "glue:GetTable", "glue:GetTables", "glue:GetTableVersions", "glue:CreatePartition", "glue:BatchCreatePartition", "glue:UpdatePartition", "glue:DeletePartition", "glue:BatchDeletePartition", "glue:GetPartition", "glue:GetPartitions", "glue:BatchGetPartition", "glue:CreateUserDefinedFunction", "glue:UpdateUserDefinedFunction", "glue:DeleteUserDefinedFunction", "glue:GetUserDefinedFunction", "glue:GetUserDefinedFunctions", "cloudwatch:PutMetricData", "dynamodb:ListTables", "s3:HeadBucket", "ec2:Describe*", ], resources=['*']), _iam.PolicyStatement( actions=['s3:GetObject'], resources=[ 'arn:aws:s3:::' + ARA_BUCKET_NAME + BINARIES + DataGenConfig.DSDGEN_INSTALL_SCRIPT, 'arn:aws:s3:::' + ARA_BUCKET_NAME + BINARIES + DataGenConfig.JAR_FILE ]), _iam.PolicyStatement( actions=['s3:PutObject'], resources=[log_bucket.bucket_arn + "/data-generator/*"]), _iam.PolicyStatement( actions=[ "s3:AbortMultipartUpload", "s3:CreateBucket", "s3:DeleteObject", "s3:GetBucketVersioning", "s3:GetObject", "s3:GetObjectTagging", "s3:GetObjectVersion", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:ListBucketVersions", "s3:ListMultipartUploadParts", "s3:PutBucketVersioning", "s3:PutObject", "s3:PutObjectTagging" ], resources=[ sink_bucket.bucket_arn + '/*', sink_bucket.bucket_arn, stream_source_bucket.bucket.bucket_arn + '/*', stream_source_bucket.bucket.bucket_arn ]) ], roles=[cluster_role]) cluster_role.add_managed_policy( _iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonSSMManagedInstanceCore')) _iam.CfnInstanceProfile(self, 'StreamEmrClusterInstanceProfile', roles=[cluster_role.role_name], instance_profile_name=cluster_role.role_name) # Security Groups for the EMR cluster (private subnet) # https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-man-sec-groups.html#emr-sg-elasticmapreduce-master-private master_sg = _ec2.SecurityGroup(self, 'ElasticMapReduce-Master-Private', vpc=vpc) slave_sg = _ec2.SecurityGroup(self, 'ElasticMapReduce-Slave-Private', vpc=vpc) service_sg = _ec2.SecurityGroup(self, 'ElasticMapReduce-ServiceAccess', vpc=vpc, allow_all_outbound=False) # Service SG used by the proxy instance service_sg.add_ingress_rule(master_sg, _ec2.Port.tcp(9443)) service_sg.add_egress_rule(master_sg, _ec2.Port.tcp(8443)) service_sg.add_egress_rule(slave_sg, _ec2.Port.tcp(8443)) # EMR Master master_sg.add_ingress_rule(master_sg, _ec2.Port.all_icmp()) master_sg.add_ingress_rule(master_sg, _ec2.Port.all_tcp()) master_sg.add_ingress_rule(master_sg, _ec2.Port.all_udp()) master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_icmp()) master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_tcp()) master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_udp()) master_sg.add_ingress_rule(service_sg, _ec2.Port.tcp(8443)) # EMR Slave slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_icmp()) slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_tcp()) slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_udp()) slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_icmp()) slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_tcp()) slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_udp()) slave_sg.add_ingress_rule(service_sg, _ec2.Port.tcp(8443)) with open('common/common_cdk/lambda/datagen_config.py', 'r') as f: lambda_source = f.read() configure_datagen_function = _lambda.SingletonFunction( self, 'StreamConfigureDatagenLambda', uuid="a9904dec-01cf-11eb-adc1-0242ac120002", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.inline(lambda_source), handler='index.handler', function_name='stream-datagen-config', environment={ 'TABLE_NAME': config_table.table_name, 'JAR_LOCATION': BINARIES_LOCATION + DataGenConfig.JAR_FILE, }, timeout=core.Duration.seconds(10)) configure_datagen_function.role.add_to_policy( _iam.PolicyStatement(actions=[ 'dynamodb:GetItem', 'dynamodb:PutItem', ], resources=[config_table.table_arn])) emr_cluster = _emr.CfnCluster( self, 'StreamEmrCluster', name="StreamDatagenCluster", job_flow_role=cluster_role.role_name, service_role=service_role.role_name, release_label='emr-5.30.1', visible_to_all_users=True, log_uri=log_bucket.s3_url_for_object() + "/data-generator", applications=[ _emr.CfnCluster.ApplicationProperty(name='hadoop'), _emr.CfnCluster.ApplicationProperty(name='spark') ], bootstrap_actions=[ _emr.CfnCluster.BootstrapActionConfigProperty( name="dsdgen-install", script_bootstrap_action=_emr.CfnCluster. ScriptBootstrapActionConfigProperty( path=BINARIES_LOCATION + DataGenConfig.DSDGEN_INSTALL_SCRIPT)) ], instances=_emr.CfnCluster.JobFlowInstancesConfigProperty( emr_managed_master_security_group=master_sg.security_group_id, emr_managed_slave_security_group=slave_sg.security_group_id, service_access_security_group=service_sg.security_group_id, ec2_subnet_id=vpc.private_subnets[0].subnet_id, core_instance_group=_emr.CfnCluster. InstanceGroupConfigProperty(instance_count=DataGenConfig. BATCH_CLUSTER_SIZE[tshirt_size], instance_type='m5.xlarge'), master_instance_group=_emr.CfnCluster. InstanceGroupConfigProperty(instance_count=1, instance_type='m4.large'))) configure_datagen = _sfn_tasks.LambdaInvoke( self, "ConfigureDatagenTask", lambda_function=configure_datagen_function, payload=_sfn.TaskInput.from_text( '{' '"Param": "stream_iterator",' '"Module": "stream",' '"SinkBucket": "' + sink_bucket.s3_url_for_object() + '",' '"Parallelism": "' + str(int(DataGenConfig.STREAM_DATA_SIZE[tshirt_size]) * 2) + '",' '"DataSize": "' + DataGenConfig.STREAM_DATA_SIZE[tshirt_size] + '",' '"TmpBucket": "' + str(stream_source_bucket.bucket.s3_url_for_object()) + '"' '}'), result_path='$.Config') add_datagen_step = _sfn.CustomState( self, 'StreamAddDataGenStep', state_json={ "Type": "Task", "Resource": "arn:aws:states:::elasticmapreduce:addStep.sync", "Parameters": { "ClusterId.$": "$.Emr.Cluster.Id", "Step": { "Name": "DatagenStep", "ActionOnFailure": "CONTINUE", "HadoopJarStep": { "Jar": "command-runner.jar", "Args.$": "$.Config.Payload.StepParam" } } }, "ResultPath": "$.Step", "Next": "StreamUpdateIterator" }) update_iterator = _sfn_tasks.DynamoUpdateItem( self, 'StreamUpdateIterator', table=config_table, key={ 'param': _sfn_tasks.DynamoAttributeValue.from_string('stream_iterator') }, update_expression= 'SET iterator = if_not_exists(iterator, :start) + :inc', expression_attribute_values={ ":inc": _sfn_tasks.DynamoAttributeValue.from_number(1), ":start": _sfn_tasks.DynamoAttributeValue.from_number(0) }, result_path=_sfn.JsonPath.DISCARD) definition = configure_datagen \ .next(add_datagen_step) \ .next(update_iterator) datagen_stepfunctions = _sfn.StateMachine( self, "StreamDataGenStepFunctions", definition=definition, timeout=core.Duration.minutes(30)) datagen_stepfunctions.add_to_role_policy( _iam.PolicyStatement(actions=[ 'elasticmapreduce:AddJobFlowSteps', 'elasticmapreduce:DescribeStep' ], resources=['*'])) step_trigger = _events.Rule(self, 'StreamStepTrigger', schedule=_events.Schedule.cron( minute='0/10', hour='*', month='*', week_day='*', year='*')) step_trigger.add_target( _events_targets.SfnStateMachine( machine=datagen_stepfunctions, input=_events.RuleTargetInput.from_object({ "Emr": { "Cluster": { "Id": core.Fn.ref(emr_cluster.logical_id) } } }))) with open('common/common_cdk/lambda/stepfunctions_trigger.py', 'r') as f: lambda_source = f.read() stepfunctions_trigger_lambda = _lambda.SingletonFunction( self, 'StreamStepFunctionsTriggerLambda', uuid="cf042246-01d0-11eb-adc1-0242ac120002", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.inline(lambda_source), handler='index.handler', function_name='stepfunctions-stream-datagen-trigger') stepfunctions_trigger_lambda.role.add_to_policy( _iam.PolicyStatement(actions=["states:StartExecution"], resources=['*'])) trigger_step_lambda_provider = _custom_resources.Provider( self, 'StreamStepFunctionsTriggerLambdaProvider', on_event_handler=stepfunctions_trigger_lambda) core.CustomResource( self, 'StreamStepFunctionsTrigger', service_token=trigger_step_lambda_provider.service_token, properties={"stepArn": datagen_stepfunctions.state_machine_arn}) with open('common/common_cdk/lambda/stream_generator.py', 'r') as f: lambda_source = f.read() sale_stream_generator_lambda = _lambda.Function( scope=self, id='WebSaleStreamGenerator', runtime=_lambda.Runtime.PYTHON_3_7, memory_size=2048, timeout=core.Duration.minutes(15), code=_lambda.Code.inline(lambda_source), handler='index.lambda_handler', environment={ 'REGION': core.Aws.REGION, 'STREAM_NAME': web_sale_stream }) stream_source_bucket.bucket.add_event_notification( _s3.EventType.OBJECT_CREATED, _s3_notifications.LambdaDestination(sale_stream_generator_lambda), _s3.NotificationKeyFilter(prefix='sale', suffix='csv')) sale_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement( actions=[ "s3:DeleteObject", "s3:GetObject", "s3:ListBucket", ], resources=[ stream_source_bucket.bucket.bucket_arn + '/*', stream_source_bucket.bucket.bucket_arn ])) sale_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement(actions=["kinesis:PutRecords"], resources=[ stack.format_arn( service='kinesis', resource='stream', resource_name=web_sale_stream) ])) sale_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement(actions=['kms:GenerateDataKey'], resources=[ stack.format_arn( service='kms', resource='key', sep='/', resource_name=kinesis_key.key_id) ])) customer_stream_generator_lambda = _lambda.Function( scope=self, id='WebCustomerStreamGenerator', runtime=_lambda.Runtime.PYTHON_3_7, memory_size=2048, timeout=core.Duration.minutes(15), code=_lambda.Code.inline(lambda_source), handler='index.lambda_handler', environment={ 'REGION': core.Aws.REGION, 'STREAM_NAME': web_customer_stream }) stream_source_bucket.bucket.add_event_notification( _s3.EventType.OBJECT_CREATED, _s3_notifications.LambdaDestination( customer_stream_generator_lambda), _s3.NotificationKeyFilter(prefix='customer', suffix='csv')) customer_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement( actions=[ "s3:DeleteObject", "s3:GetObject", "s3:ListBucket", ], resources=[ stream_source_bucket.bucket.bucket_arn + '/*', stream_source_bucket.bucket.bucket_arn ])) customer_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement(actions=["kinesis:PutRecords"], resources=[ stack.format_arn( service='kinesis', resource='stream', resource_name=web_customer_stream) ])) customer_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement(actions=['kms:GenerateDataKey'], resources=[ stack.format_arn( service='kms', resource='key', sep='/', resource_name=kinesis_key.key_id) ])) address_stream_generator_lambda = _lambda.Function( scope=self, id='WebCustomerAddressStreamGenerator', runtime=_lambda.Runtime.PYTHON_3_7, memory_size=2048, timeout=core.Duration.minutes(15), code=_lambda.Code.inline(lambda_source), handler='index.lambda_handler', environment={ 'REGION': core.Aws.REGION, 'STREAM_NAME': web_customer_address_stream }) stream_source_bucket.bucket.add_event_notification( _s3.EventType.OBJECT_CREATED, _s3_notifications.LambdaDestination( address_stream_generator_lambda), _s3.NotificationKeyFilter(prefix='address', suffix='csv')) address_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement( actions=[ "s3:DeleteObject", "s3:GetObject", "s3:ListBucket", ], resources=[ stream_source_bucket.bucket.bucket_arn + '/*', stream_source_bucket.bucket.bucket_arn ])) address_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement( actions=["kinesis:PutRecords"], resources=[ stack.format_arn(service='kinesis', resource='stream', resource_name=web_customer_address_stream) ])) address_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement(actions=['kms:GenerateDataKey'], resources=[ stack.format_arn( service='kms', resource='key', sep='/', resource_name=kinesis_key.key_id) ]))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Tag all constructs with the project for easy billing drilldown, # filtering, and organization. core.Tags.of(self).add('project', 'MediaTranscription') # Media files bucket media_bucket = s3.Bucket( self, 'media-transcription-bucket', encryption=s3.BucketEncryption.S3_MANAGED, ) # SQS queue for media files bucket event notifications media_bucket_event_queue = sqs.Queue( self, 'media-transcription-event-notification-queue', queue_name='media-transcription-event-notification-queue', visibility_timeout=core.Duration.seconds(60), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=3, queue=sqs.Queue( self, 'media-transcription-event-notifications-dlq', queue_name='media-transcription-event-notifications-dlq', )), ) # S3 object created notifications sent to SQS queue media_bucket.add_event_notification( s3.EventType.OBJECT_CREATED, s3n.SqsDestination(media_bucket_event_queue), *[s3.NotificationKeyFilter(prefix='media-input/')], ) # Lambda function to create/submit Transcribe jobs transcribe_job_init_fn = lambda_.Function( self, 'transcribe-job-init-fn', runtime=lambda_.Runtime.PYTHON_3_8, code=lambda_.Code.from_asset( '../lambdas/transcribe-job-init-fn', # The following is just dumb. # The Lambda runtime doesn't use the latest boto3 by default. # In order to use the latest boto3, we have to pip install # and bundle locally using Docker. # Q: Why need the latest boto3? # A: https://github.com/boto/boto3/issues/2630 # I'll have to delete the ECR containers to avoid cost. # TODO: Revert back to normal in like a month I guess. bundling={ 'image': lambda_.Runtime.PYTHON_3_8.bundling_docker_image, 'command': [ 'bash', '-c', '\n pip install -r requirements.txt -t /asset-output &&\n cp -au . /asset-output\n ' ] }), handler='fn.handler', reserved_concurrent_executions=1, # Effectively single-threaded ) # Triggered by SQS messages created for media file puts transcribe_job_init_fn.add_event_source( les.SqsEventSource( queue=media_bucket_event_queue, batch_size=5, enabled=True, )) # Grant access to start transcription jobs transcribe_job_init_fn.add_to_role_policy( statement=iam.PolicyStatement( actions=[ 'transcribe:StartTranscriptionJob', ], resources=['*'], effect=iam.Effect.ALLOW, )) # Grant Lambda role to read and write to input and output portions of # the S3 bucket. # Q: Why grant Lambda the permissions instead of Transcribe service? # A: Two-fold: # - i) https://amzn.to/321Nx5I # - ii) Granting just to this Lambda means other Transcribe jobs # across the account cannot use this bucket (least privilege). media_bucket.grant_read( identity=transcribe_job_init_fn.grant_principal, objects_key_pattern='media-input/*') # Cannot specify a prefix for writes as Transcribe will not accept # a job unless it has write permission on the whole bucket. # Edit: The above statement was when I had to use '*' for writes. But # now, I granted access to that .write_access_check_file.temp # file and it seems to all work now? media_bucket.grant_write( identity=transcribe_job_init_fn.grant_principal, objects_key_pattern='transcribe-output-raw/*') # This is just as frustrating to you as it is to me. media_bucket.grant_write( identity=transcribe_job_init_fn.grant_principal, objects_key_pattern='.write_access_check_file.temp') # DynamoDB table for Jobs metadata jobs_metadata_table = ddb.Table( self, 'MediaTranscription-TranscriptionJobs', table_name='MediaTranscription-TranscriptionJobs', partition_key=ddb.Attribute( name='Bucket-Key-ETag', type=ddb.AttributeType.STRING, ), billing_mode=ddb.BillingMode.PAY_PER_REQUEST, ) jobs_metadata_table.grant(transcribe_job_init_fn.grant_principal, *[ 'dynamodb:GetItem', 'dynamodb:PutItem', ]) # Create IAM Group with read/write permissions to S3 bucket # TODO: Make this more federated and robust console_users_group = iam.Group(self, 'MediaTranscriptionConsoleUsers') console_users_group.attach_inline_policy(policy=iam.Policy( self, 'MediaTranscriptionConsoleUserS3Access', statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ 's3:ListBucket', ], resources=[ media_bucket.bucket_arn, ], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ 's3:GetObject', 's3:PutObject', ], resources=[ media_bucket.arn_for_objects('media-input/*'), ], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ 's3:GetObject', ], resources=[ media_bucket.arn_for_objects( 'transcribe-output-raw/*'), ], ), ], ))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) with open("stack/config.yml", 'r') as stream: configs = yaml.safe_load(stream) ### S3 core images_S3_bucket = _s3.Bucket(self, "ICS_IMAGES") images_S3_bucket.add_cors_rule( allowed_methods=[_s3.HttpMethods.POST], allowed_origins=["*"] # add API gateway web resource URL ) ### SQS core image_deadletter_queue = _sqs.Queue(self, "ICS_IMAGES_DEADLETTER_QUEUE") image_queue = _sqs.Queue(self, "ICS_IMAGES_QUEUE", dead_letter_queue={ "max_receive_count": configs["DeadLetterQueue"]["MaxReceiveCount"], "queue": image_deadletter_queue }) ### api gateway core api_gateway = RestApi(self, 'ICS_API_GATEWAY', rest_api_name='ImageContentSearchApiGateway') api_gateway_resource = api_gateway.root.add_resource(configs["ProjectName"]) api_gateway_landing_page_resource = api_gateway_resource.add_resource('web') api_gateway_get_signedurl_resource = api_gateway_resource.add_resource('signedUrl') api_gateway_image_search_resource = api_gateway_resource.add_resource('search') ### landing page function get_landing_page_function = Function(self, "ICS_GET_LANDING_PAGE", function_name="ICS_GET_LANDING_PAGE", runtime=Runtime.PYTHON_3_7, handler="main.handler", code=Code.asset("./src/landingPage")) get_landing_page_integration = LambdaIntegration( get_landing_page_function, proxy=True, integration_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': "'*'", } }]) api_gateway_landing_page_resource.add_method('GET', get_landing_page_integration, method_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': True, } }]) ### cognito required_attribute = _cognito.StandardAttribute(required=True) users_pool = _cognito.UserPool(self, "ICS_USERS_POOL", auto_verify=_cognito.AutoVerifiedAttrs(email=True), #required for self sign-up standard_attributes=_cognito.StandardAttributes(email=required_attribute), #required for self sign-up self_sign_up_enabled=configs["Cognito"]["SelfSignUp"]) user_pool_app_client = _cognito.CfnUserPoolClient(self, "ICS_USERS_POOL_APP_CLIENT", supported_identity_providers=["COGNITO"], allowed_o_auth_flows=["implicit"], allowed_o_auth_scopes=configs["Cognito"]["AllowedOAuthScopes"], user_pool_id=users_pool.user_pool_id, callback_ur_ls=[api_gateway_landing_page_resource.url], allowed_o_auth_flows_user_pool_client=True, explicit_auth_flows=["ALLOW_REFRESH_TOKEN_AUTH"]) user_pool_domain = _cognito.UserPoolDomain(self, "ICS_USERS_POOL_DOMAIN", user_pool=users_pool, cognito_domain=_cognito.CognitoDomainOptions(domain_prefix=configs["Cognito"]["DomainPrefix"])) ### get signed URL function get_signedurl_function = Function(self, "ICS_GET_SIGNED_URL", function_name="ICS_GET_SIGNED_URL", environment={ "ICS_IMAGES_BUCKET": images_S3_bucket.bucket_name, "DEFAULT_SIGNEDURL_EXPIRY_SECONDS": configs["Functions"]["DefaultSignedUrlExpirySeconds"] }, runtime=Runtime.PYTHON_3_7, handler="main.handler", code=Code.asset("./src/getSignedUrl")) get_signedurl_integration = LambdaIntegration( get_signedurl_function, proxy=True, integration_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': "'*'", } }]) api_gateway_get_signedurl_authorizer = CfnAuthorizer(self, "ICS_API_GATEWAY_GET_SIGNED_URL_AUTHORIZER", rest_api_id=api_gateway_get_signedurl_resource.rest_api.rest_api_id, name="ICS_API_GATEWAY_GET_SIGNED_URL_AUTHORIZER", type="COGNITO_USER_POOLS", identity_source="method.request.header.Authorization", provider_arns=[users_pool.user_pool_arn]) api_gateway_get_signedurl_resource.add_method('GET', get_signedurl_integration, authorization_type=AuthorizationType.COGNITO, method_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': True, } }] ).node.find_child('Resource').add_property_override('AuthorizerId', api_gateway_get_signedurl_authorizer.ref) images_S3_bucket.grant_put(get_signedurl_function, objects_key_pattern="new/*") ### image massage function image_massage_function = Function(self, "ICS_IMAGE_MASSAGE", function_name="ICS_IMAGE_MASSAGE", timeout=core.Duration.seconds(6), runtime=Runtime.PYTHON_3_7, environment={"ICS_IMAGE_MASSAGE": image_queue.queue_name}, handler="main.handler", code=Code.asset("./src/imageMassage")) images_S3_bucket.grant_write(image_massage_function, "processed/*") images_S3_bucket.grant_delete(image_massage_function, "new/*") images_S3_bucket.grant_read(image_massage_function, "new/*") new_image_added_notification = _s3notification.LambdaDestination(image_massage_function) images_S3_bucket.add_event_notification(_s3.EventType.OBJECT_CREATED, new_image_added_notification, _s3.NotificationKeyFilter(prefix="new/") ) image_queue.grant_send_messages(image_massage_function) ### image analyzer function image_analyzer_function = Function(self, "ICS_IMAGE_ANALYSIS", function_name="ICS_IMAGE_ANALYSIS", runtime=Runtime.PYTHON_3_7, timeout=core.Duration.seconds(10), environment={ "ICS_IMAGES_BUCKET": images_S3_bucket.bucket_name, "DEFAULT_MAX_CALL_ATTEMPTS": configs["Functions"]["DefaultMaxApiCallAttempts"], "REGION": core.Aws.REGION, }, handler="main.handler", code=Code.asset("./src/imageAnalysis")) image_analyzer_function.add_event_source(_lambda_event_source.SqsEventSource(queue=image_queue, batch_size=10)) image_queue.grant_consume_messages(image_massage_function) lambda_rekognition_access = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=["rekognition:DetectLabels", "rekognition:DetectModerationLabels"], resources=["*"] ) image_analyzer_function.add_to_role_policy(lambda_rekognition_access) images_S3_bucket.grant_read(image_analyzer_function, "processed/*") ### API gateway finalizing self.add_cors_options(api_gateway_get_signedurl_resource) self.add_cors_options(api_gateway_landing_page_resource) self.add_cors_options(api_gateway_image_search_resource) ### database database_secret = _secrets_manager.Secret(self, "ICS_DATABASE_SECRET", secret_name="rds-db-credentials/image-content-search-rds-secret", generate_secret_string=_secrets_manager.SecretStringGenerator( generate_string_key='password', secret_string_template='{"username": "******"}', exclude_punctuation=True, exclude_characters='/@\" \\\'', require_each_included_type=True ) ) database = _rds.CfnDBCluster(self, "ICS_DATABASE", engine=_rds.DatabaseClusterEngine.aurora_mysql(version=_rds.AuroraMysqlEngineVersion.VER_5_7_12).engine_type, engine_mode="serverless", database_name=configs["Database"]["Name"], enable_http_endpoint=True, deletion_protection=configs["Database"]["DeletionProtection"], master_username=database_secret.secret_value_from_json("username").to_string(), master_user_password=database_secret.secret_value_from_json("password").to_string(), scaling_configuration=_rds.CfnDBCluster.ScalingConfigurationProperty( auto_pause=configs["Database"]["Scaling"]["AutoPause"], min_capacity=configs["Database"]["Scaling"]["Min"], max_capacity=configs["Database"]["Scaling"]["Max"], seconds_until_auto_pause=configs["Database"]["Scaling"]["SecondsToAutoPause"] ), ) database_cluster_arn = "arn:aws:rds:{}:{}:cluster:{}".format(core.Aws.REGION, core.Aws.ACCOUNT_ID, database.ref) secret_target = _secrets_manager.CfnSecretTargetAttachment(self,"ICS_DATABASE_SECRET_TARGET", target_type="AWS::RDS::DBCluster", target_id=database.ref, secret_id=database_secret.secret_arn ) secret_target.node.add_dependency(database) ### database function image_data_function_role = _iam.Role(self, "ICS_IMAGE_DATA_FUNCTION_ROLE", role_name="ICS_IMAGE_DATA_FUNCTION_ROLE", assumed_by=_iam.ServicePrincipal("lambda.amazonaws.com"), managed_policies=[ _iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaVPCAccessExecutionRole"), _iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole"), _iam.ManagedPolicy.from_aws_managed_policy_name("AmazonRDSDataFullAccess") ] ) image_data_function = Function(self, "ICS_IMAGE_DATA", function_name="ICS_IMAGE_DATA", runtime=Runtime.PYTHON_3_7, timeout=core.Duration.seconds(5), role=image_data_function_role, environment={ "DEFAULT_MAX_CALL_ATTEMPTS": configs["Functions"]["DefaultMaxApiCallAttempts"], "CLUSTER_ARN": database_cluster_arn, "CREDENTIALS_ARN": database_secret.secret_arn, "DB_NAME": database.database_name, "REGION": core.Aws.REGION }, handler="main.handler", code=Code.asset("./src/imageData") ) image_search_integration = LambdaIntegration( image_data_function, proxy=True, integration_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': "'*'", } }]) api_gateway_image_search_authorizer = CfnAuthorizer(self, "ICS_API_GATEWAY_IMAGE_SEARCH_AUTHORIZER", rest_api_id=api_gateway_image_search_resource.rest_api.rest_api_id, name="ICS_API_GATEWAY_IMAGE_SEARCH_AUTHORIZER", type="COGNITO_USER_POOLS", identity_source="method.request.header.Authorization", provider_arns=[users_pool.user_pool_arn]) api_gateway_image_search_resource.add_method('POST', image_search_integration, authorization_type=AuthorizationType.COGNITO, method_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': True, } }] ).node.find_child('Resource').add_property_override('AuthorizerId', api_gateway_image_search_authorizer.ref) lambda_access_search = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=["translate:TranslateText"], resources=["*"] ) image_data_function.add_to_role_policy(lambda_access_search) ### custom resource lambda_provider = Provider(self, 'ICS_IMAGE_DATA_PROVIDER', on_event_handler=image_data_function ) core.CustomResource(self, 'ICS_IMAGE_DATA_RESOURCE', service_token=lambda_provider.service_token, pascal_case_properties=False, resource_type="Custom::SchemaCreation", properties={ "source": "Cloudformation" } ) ### event bridge event_bus = _events.EventBus(self, "ICS_IMAGE_CONTENT_BUS") event_rule = _events.Rule(self, "ICS_IMAGE_CONTENT_RULE", rule_name="ICS_IMAGE_CONTENT_RULE", description="The event from image analyzer to store the data", event_bus=event_bus, event_pattern=_events.EventPattern(resources=[image_analyzer_function.function_arn]), ) event_rule.add_target(_event_targets.LambdaFunction(image_data_function)) event_bus.grant_put_events(image_analyzer_function) image_analyzer_function.add_environment("EVENT_BUS", event_bus.event_bus_name) ### outputs core.CfnOutput(self, 'CognitoHostedUILogin', value='https://{}.auth.{}.amazoncognito.com/login?client_id={}&response_type=token&scope={}&redirect_uri={}'.format(user_pool_domain.domain_name, core.Aws.REGION, user_pool_app_client.ref, '+'.join(user_pool_app_client.allowed_o_auth_scopes), api_gateway_landing_page_resource.url), description='The Cognito Hosted UI Login Page' )
def __init__(self, scope: core.Construct, id: str, config_dict, **kwargs) -> None: super().__init__(scope, id, **kwargs) """ Get VPC details """ Ivpc = ec2.Vpc.from_lookup(self, "VPC", vpc_id=config_dict['vpc_id']) """ Get sunet seclection context created """ subnet_1 = ec2.Subnet.from_subnet_attributes( self, "subnet_1", subnet_id=config_dict['SubnetIds'].split(",")[0], availability_zone=config_dict['AvailabilityZones'].split(",")[0]) subnet_2 = ec2.Subnet.from_subnet_attributes( self, "subnet_2", subnet_id=config_dict['SubnetIds'].split(",")[1], availability_zone=config_dict['AvailabilityZones'].split(",")[1]) """ Create Security Group for Lambda Functions """ lambda_security_group = "datalake-lambda-sg" createLambdaSecurityGroup = ec2.SecurityGroup( self, "createLambdaSecurityGroup", vpc=Ivpc, allow_all_outbound=True, description="This security group will be used for Lambda Funcs", security_group_name=lambda_security_group) """ Create the Datalake Bucket """ createDatalakeBucket = s3.Bucket( self, "createCompRegBucket", bucket_name=config_dict['datalake_bucket_name'], block_public_access=s3.BlockPublicAccess( block_public_acls=True, block_public_policy=True, ignore_public_acls=True, restrict_public_buckets=True)) core.CfnOutput(self, "createCompRegBucketName", value=createDatalakeBucket.bucket_name) """ Create Comp Reg Lambda Function """ createCompRegLambda = _lambda.Function( self, "createCompRegLambda", function_name="datalake-comp-reg-trigger", description= "This lambda function will trigger the compound reg pipeline.", runtime=_lambda.Runtime.PYTHON_3_7, handler="trigger_compound_reg_pipeline.lambda_handler", code=_lambda.Code.asset('lambdas'), timeout=core.Duration.seconds(90), vpc=Ivpc, vpc_subnets=ec2.SubnetSelection(subnets=[subnet_1, subnet_2]), security_group=createLambdaSecurityGroup, initial_policy=[ iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=["s3:*", "batch:*"], resources=["*"]) ]) """ Add s3 event trigger to above lambda function """ createCompRegLambda.add_event_source( S3EventSource(createDatalakeBucket, events=[s3.EventType.OBJECT_CREATED], filters=[ s3.NotificationKeyFilter( prefix="compound_reg/triggers/", suffix=".trigger") ]))
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # Get the DynamoDB environment variables from context DYNAMODB_ENV = self.node.try_get_context('dynamodb') # Create lambda function. It gets the code from the ./assets folder from_s3_to_dynamo_db_function = _lambda.Function( self, "FromS3ToDynamoDBFunction", function_name="from-s3-to-dynamodb", runtime=_lambda.Runtime.PYTHON_3_7, handler="s3_trigger.lambda_handler", code=_lambda.Code.asset("./cdk/assets"), timeout=core.Duration.seconds(3), reserved_concurrent_executions=1, environment={ 'LOG_LEVEL': 'INFO', 'TABLE_NAME': DYNAMODB_ENV['TABLE_NAME'] }) # create log group to manage it with cdk _logs.LogGroup( self, "FromS3ToDynamoDBLogGroup", log_group_name= f"/aws/lambda/{from_s3_to_dynamo_db_function.function_name}", removal_policy=core.RemovalPolicy.DESTROY) # create s3 bucket with encryption, versioning and blocked public access s3_to_dynamo_db_bucket = _s3.Bucket( self, "S3ToDynamoDBBucket", bucket_name="s3-to-dynamo-db-workshop", encryption=_s3.BucketEncryption.S3_MANAGED, versioned=True, block_public_access=_s3.BlockPublicAccess.BLOCK_ALL) # Grant read access to Lambda Function to access S3 Bucket s3_to_dynamo_db_bucket.grant_read(from_s3_to_dynamo_db_function) # Create s3 notification for lambda function notification = _s3_notifications.LambdaDestination( from_s3_to_dynamo_db_function) # Assign notification for the s3 event type s3_to_dynamo_db_bucket.add_event_notification( _s3.EventType.OBJECT_CREATED, notification, _s3.NotificationKeyFilter(suffix='.csv')) # Create DynamoDb table dynamo_db = _dynamodb.Table( self, 'FromS3ToDynamoDBTable', table_name=DYNAMODB_ENV['TABLE_NAME'], billing_mode=_dynamodb.BillingMode.PAY_PER_REQUEST, partition_key=_dynamodb.Attribute( name=DYNAMODB_ENV['PARTITION_KEY_NAME'], type=_dynamodb.AttributeType.STRING)) # Grant full access to Lambda Function to access DynamoDb dynamo_db.grant_full_access(from_s3_to_dynamo_db_function)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ################################################ # DynamoDB ################################################ ddb_sims_tbl = ddb.Table(self, 'RecommendationSIMS', table_name='RecommendationSIMS', removal_policy=core.RemovalPolicy.DESTROY, partition_key={ 'name': 'itemid', 'type': ddb.AttributeType.STRING }, billing_mode=ddb.BillingMode.PROVISIONED, read_capacity=5, write_capacity=50) ddb_sims_tbl_rw_policy_statement = iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ddb_sims_tbl.table_arn], actions=[ "dynamodb:BatchGetItem", "dynamodb:Describe*", "dynamodb:List*", "dynamodb:GetItem", "dynamodb:Query", "dynamodb:Scan", "dynamodb:BatchWriteItem", "dynamodb:DeleteItem", "dynamodb:PutItem", "dynamodb:UpdateItem" ]) ################################################ # Lambda + S3 ################################################ ddb_lambda_fn = _lambda.Function( self, 'UpsertManhwakyungToDynmodbDB', function_name='UpsertManhwakyungToDynmodbDB', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset('src/lambda/UpsertManhwakyungToDynmodbDB'), handler='manhwakyung_s3_to_dynamodb.lambda_handler', description= 'Store recommendation results in dynamoDB (Recipe: SIMS)', environment={ 'REGION_NAME': kwargs['env'].region, 'DDB_SIMS_TABLE': ddb_sims_tbl.table_name }, timeout=core.Duration.minutes(1)) s3_bucket = s3.Bucket( self, "data", #removal_policy=core.RemovalPolicy.DESTROY ) # https://stackoverflow.com/questions/60087302/how-to-add-resource-policy-to-existing-s3-bucket-with-cdk-in-javascript # https://stackoverflow.com/questions/60282173/lookup-s3-bucket-and-add-a-trigger-to-invoke-a-lambda #s3_bucket = s3.Bucket.from_bucket_name(self, "ExistingBucket", # bucket_name="team3-recommendation-system-personalize-data-jhhwang-cdk") s3_event_filter = s3.NotificationKeyFilter( prefix="results/by-title-id/", suffix=".out") s3_event_source = S3EventSource( s3_bucket, events=[s3.EventType.OBJECT_CREATED_PUT], filters=[s3_event_filter]) ddb_lambda_fn.add_event_source(s3_event_source) s3_read_statement = iam.PolicyStatement( effect=iam.Effect.ALLOW, #resources = [s3_bucket.bucket_arn, "{}/*".format(s3_bucket.bucket_arn)], resources=["*"], actions=["s3:Get*", "s3:List*"]) ddb_lambda_fn.add_to_role_policy(s3_read_statement) ddb_lambda_fn.add_to_role_policy(ddb_sims_tbl_rw_policy_statement) ############################################################## # APIGW + Lambda ############################################################## sims_title_lambda_fn = _lambda.Function( self, 'RecommendSimsByTitleAPI', function_name='RecommendSimsByTitleAPI', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset('src/lambda/RecommendSimsByTitleAPI'), handler='recommend_sims_by_title.lambda_handler', description='API to provide recommended results (Recipe: SIMS)', environment={ 'REGION_NAME': kwargs['env'].region, 'DDB_SIMS_TABLE': ddb_sims_tbl.table_name, 'DEF_ITEM_LIMIT': "10", }, timeout=core.Duration.minutes(1)) ddb_sims_tbl.grant_read_write_data(sims_title_lambda_fn) sims_title_api = apigw.LambdaRestApi( self, "RecommendSimsByTitle", handler=sims_title_lambda_fn, proxy=False, rest_api_name="RecommendSimsByTitle", description= 'Webtoon is recommended based on webtoon id. (Recipe: SIMS)', endpoint_types=[apigw.EndpointType.REGIONAL], deploy=True, deploy_options=apigw.StageOptions(stage_name="v1"), ) recomm_titles = sims_title_api.root.add_resource('recommended-titles') by_title = recomm_titles.add_resource('by-title') by_title.add_method("GET") by_title_id = by_title.add_resource('{id}') by_title_id.add_method("GET") by_title_id.add_cors_preflight(allow_origins=["*"], allow_methods=["*"])
def __init__(self, scope: core.Stack, id: str, **kwargs): super().__init__(scope, id, **kwargs) self.output_bucket = aws_s3.Bucket( self, "BucketTwitterStreamOutput", bucket_name = self.stack_name, ) self.bucket_url = self.output_bucket.bucket_regional_domain_name # Because kinesis firehose bindings are to direct CF, we have to create IAM policy/role and attach on our own self.iam_role = aws_iam.Role( self, "IAMRoleTwitterStreamKinesisFHToS3", role_name="KinesisFirehoseToS3-{}".format(self.stack_name), assumed_by=aws_iam.ServicePrincipal(service='firehose.amazonaws.com'), ) # S3 bucket actions self.s3_iam_policy_statement = aws_iam.PolicyStatement() actions = ["s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject"] for action in actions: self.s3_iam_policy_statement.add_actions(action) self.s3_iam_policy_statement.add_resources(self.output_bucket.bucket_arn) self.s3_iam_policy_statement.add_resources(self.output_bucket.bucket_arn + "/*") # CW error log setup self.s3_error_logs_group = aws_logs.LogGroup( self, "S3ErrorLogsGroup", log_group_name="{}-s3-errors".format(self.stack_name) ) self.s3_error_logs_stream = aws_logs.LogStream( self, "S3ErrorLogsStream", log_group=self.s3_error_logs_group, log_stream_name='s3Backup' ) self.firehose = aws_kinesisfirehose.CfnDeliveryStream( self, "FirehoseTwitterStream", delivery_stream_name = "{}-raw".format(self.stack_name), delivery_stream_type = "DirectPut", s3_destination_configuration={ 'bucketArn': self.output_bucket.bucket_arn, 'bufferingHints': { 'intervalInSeconds': 120, 'sizeInMBs': 10 }, 'compressionFormat': 'UNCOMPRESSED', 'roleArn': self.iam_role.role_arn, 'cloudWatchLoggingOptions': { 'enabled': True, 'logGroupName': "{}-raw".format(self.stack_name), 'logStreamName': 's3BackupRaw' }, 'prefix': 'twitter-raw/' }, ) # TODO: Only attach what's needed for this policy, right now i'm lazy and attaching all policies self.iam_policy = aws_iam.Policy( self, "IAMPolicyTwitterStreamKinesisFHToS3", policy_name="KinesisFirehoseToS3-{}".format(self.stack_name), statements=[self.s3_iam_policy_statement], ) self.iam_policy.attach_to_role(self.iam_role) # Because kinesis firehose bindings are to direct CF, we have to create IAM policy/role and attach on our own self.curator_firehose = aws_kinesisfirehose.CfnDeliveryStream( self, "CuratorFirehoseStream", delivery_stream_name = "{}-curator".format(self.stack_name), delivery_stream_type = "DirectPut", s3_destination_configuration={ 'bucketArn': self.output_bucket.bucket_arn, 'bufferingHints': { 'intervalInSeconds': 120, 'sizeInMBs': 10 }, 'compressionFormat': 'UNCOMPRESSED', 'roleArn': self.iam_role.role_arn, 'cloudWatchLoggingOptions': { 'enabled': True, 'logGroupName': "{}-curator".format(self.stack_name), 'logStreamName': 's3BackupCurator' }, 'prefix': 'twitter-curated/' }, ) def zip_package(): cwd = os.getcwd() file_name = 'curator-lambda.zip' zip_file = cwd + '/' + file_name os.chdir('src/') sh.zip('-r9', zip_file, '.') os.chdir(cwd) return file_name, zip_file _, zip_file = zip_package() self.twitter_stream_curator_lambda_function = aws_lambda.Function( self, "TwitterStreamCuratorLambdaFunction", function_name="{}-curator".format(self.stack_name), code=aws_lambda.AssetCode(zip_file), handler="sentiment_analysis.lambda_handler", runtime=aws_lambda.Runtime.PYTHON_3_7, tracing=aws_lambda.Tracing.ACTIVE, description="Triggers from S3 PUT event for twitter stream data and transorms it to clean json syntax with sentiment analysis attached", environment={ "STACK_NAME": self.stack_name, "FIREHOSE_STREAM": self.curator_firehose.delivery_stream_name }, memory_size=128, timeout=core.Duration.seconds(120), log_retention=aws_logs.RetentionDays.ONE_WEEK, ) # Permission to talk to comprehend for sentiment analysis self.comprehend_iam_policy_statement = aws_iam.PolicyStatement() self.comprehend_iam_policy_statement.add_actions('comprehend:*') self.comprehend_iam_policy_statement.add_all_resources() self.twitter_stream_curator_lambda_function.add_to_role_policy(self.comprehend_iam_policy_statement) # Permission to put in kinesis firehose self.curator_firehose_iam_policy_statement = aws_iam.PolicyStatement() self.curator_firehose_iam_policy_statement.add_actions('firehose:Put*') self.curator_firehose_iam_policy_statement.add_resources(self.curator_firehose.attr_arn) self.twitter_stream_curator_lambda_function.add_to_role_policy(self.curator_firehose_iam_policy_statement) # Attaching the policy to the IAM role for KFH self.output_bucket.grant_read(self.twitter_stream_curator_lambda_function) self.twitter_stream_curator_lambda_function.add_event_source( aws_lambda_event_sources.S3EventSource( bucket=self.output_bucket, events=[ aws_s3.EventType.OBJECT_CREATED ], filters=[ aws_s3.NotificationKeyFilter( prefix="twitter-raw/" ) ] ) )