def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) bucket = s3.Bucket(self, "BucketOne", bucket_name="jc-100-test-bucket") another_bucket = s3.Bucket(self, "BucketTwo", bucket_name="jc-200-test-bucket") lambda_function = lam.Function( self, "test-copy-bucket-cdk", runtime=lam.Runtime.PYTHON_3_7, events=[ lam_events.S3EventSource( bucket, events=[ s3.EventType.OBJECT_CREATED_PUT, s3.EventType.OBJECT_CREATED_COPY, ], ) ], handler="index.handle", code=lam.Code.from_asset("lambda"), environment={"DESTINATION_BUCKET": another_bucket.bucket_name}, initial_policy=[ iam.PolicyStatement( actions=["s3:PutObject", "s3:GetObject"], resources=[ bucket.bucket_arn, bucket.arn_for_objects("*"), another_bucket.bucket_arn, another_bucket.arn_for_objects("*"), ], ) ], )
def __init__(self, scope: core.Construct, id: str, *, prefix: str, environment: str, configuration, **kwargs): """ :param scope: Stack class, used by CDK. :param id: ID of the construct, used by CDK. :param prefix: Prefix of the construct, used for naming purposes. :param environment: Environment of the construct, used for naming purposes. :param configuration: Configuration of the construct. In this case SNS_CONFIG_SCHEMA. :param kwargs: Other parameters that could be used by the construct. """ super().__init__(scope, id, **kwargs) self.prefix = prefix self.environment_ = environment self._configuration = configuration # Validating that the payload passed is correct validate_configuration(configuration_schema=S3_LAMBDA_CONFIG_SCHEMA, configuration_received=self._configuration) # Defining S3 Bucket bucket_data = deepcopy(self._configuration["bucket"]) self._s3_bucket = base_bucket(self, **bucket_data) # Validating Lambda Function Runtime functions_data = self._configuration["lambda_handler"] self._lambda_function = base_lambda_function(self, **functions_data) # Defining the Lambda subscription to the specified S3 Bucket in cdk.json file. s3_events = self._configuration["events"] event_list = enum_parsing(source_list=s3_events, target_enum=s3.EventType) s3_subscription = events.S3EventSource(bucket=self._s3_bucket, events=event_list) self._lambda_function.add_event_source(source=s3_subscription)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # S3 bucket used for Lambda function code lambda_code_bucket = s3.Bucket(self, "IntelixLambda") # S3 bucket used by application - this will contain objects to be scanned input_bucket = s3.Bucket(self, "IntelixLambdaInput") # S3 bucket used by application output - this will contain objects that have been scanned and pass as non-malware output_bucket = s3.Bucket(self, "IntelixLambdaOutput") # Core Lambda function handler = lambda_.Function( self, "IntelixLambdaHandler", runtime=lambda_.Runtime.PYTHON_3_7, code=lambda_.Code.from_asset("resources"), handler="intelix_file_check.file_check_handler", timeout=core.Duration.seconds(900), environment=dict(BUCKET=lambda_code_bucket.bucket_name, INTELIX_CREDENTIALS= "<YOUR BASE64 CODED CLIENT_ID:CLIENT_SECRET>", OUTPUT_BUCKET=output_bucket.bucket_name)) lambda_code_bucket.grant_read_write(handler) input_bucket.grant_read_write(handler) output_bucket.grant_read_write(handler) # Event Trigger - upon object created in input_bucket call the lambda handler.add_event_source( aws_lambda_event_sources.S3EventSource( input_bucket, events=[s3.EventType.OBJECT_CREATED]))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Create S3 bucket here. bucket = aws_s3.Bucket(self, image_bucket_name) # CFN output for S3 bucket creation. core.CfnOutput( self, "image-bucket", value=bucket.bucket_name, description="Bucket for uploading images", ) # Create dynamodb table for storing image labels here. table = aws_dynamodb.Table( self, "image-lables", partition_key=aws_dynamodb.Attribute( name="image", type=aws_dynamodb.AttributeType.STRING ), ) # CFN output for dynamodb table creation. core.CfnOutput( self, "image-lables-ddb-table", value=table.table_name, description="DDB table for storing image lables.", ) function = aws_lambda.Function( self, "rekognitionFunction", runtime=aws_lambda.Runtime.PYTHON_3_7, handler="handler.main", code=aws_lambda.Code.asset("./rekognitionLambda"), timeout=core.Duration.seconds(30), memory_size=1024, ) function.add_environment("TABLE", table.table_name) function.add_environment("BUCKET", bucket.bucket_name) function.add_event_source( aws_lambda_event_sources.S3EventSource( bucket=bucket, events=[aws_s3.EventType.OBJECT_CREATED] ) ) bucket.grant_read(function) table.grant_write_data(function) function.add_to_role_policy( statement=aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, actions=["rekognition:DetectLabels"], resources=["*"], ) )
def create_lambda_trigger_task_custom(self, vpc, **kwargs): #################### # Unpack Value app_name = kwargs['function'].replace("_", "-") lambda_name = "{}-lambda".format(app_name) code_name = "{}".format(app_name) trigger_bucket_name = "{}-bucket-event".format(app_name) train_bucket_name = "{}-bucket-model".format(app_name) job_name = "{}-job".format(app_name) task = "{}-task".format(app_name) instance = kwargs['instance'] image_uri = kwargs['image_uri'] # Config role lambda_base_role = iam.Role( self, "gw_lambda_train_graph_role", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com")) #lambda_base_role.add_managed_policy(iam.ManagedPolicy.from_managed_policy_name("AWSLambdaBasicExectutionRole", "arn:aws:iam:aws:policy/service-role/AWSLambdaBasicExecutionRole")) #lambda_base_role.add_managed_policy(iam.ManagedPolicy.from_managed_policy_name("AWSLambdaVPCAcessExecutionRole", "arn:aws:iam:aws:policy/service-role/AWSLambdaVPCAccessExecutionRole")) #lambda_base_role.add_managed_policy(iam.ManagedPolicy.from_managed_policy_name("AmazonSageMakerFullAccess", "arn:aws:iam:aws:policy/AmazonSageMakerFullAccess")) lambda_base_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaBasicExecutionRole")) lambda_base_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaVPCAccessExecutionRole")) lambda_base_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonSageMakerFullAccess")) # Create Lambda lambda_app = _lambda.Function(self, lambda_name, handler='{}.handler'.format(code_name), runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset('lambda'), role=lambda_base_role, environment={ 'BUCKET': train_bucket_name, 'JOB_NAME': job_name, 'INSTANCE': instance, 'IMAGE_URI': image_uri, 'TASK': task, "REDIS_URL": self.redis_host, "REDIS_PORT": self.redis_port }) # Create an S3 event soruce for Lambda bucket = s3.Bucket(self, trigger_bucket_name) s3_event_source = lambda_event_source.S3EventSource( bucket, events=[s3.EventType.OBJECT_CREATED]) lambda_app.add_event_source(s3_event_source) return lambda_app
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here lambda_layer = aws_lambda.LayerVersion( self, "pillow_layer", code=aws_lambda.Code.asset("./python-pillow-6.2.1.zip"), compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_7], description="Pillow_upload_from_cdk") # aws_lambda.LayerVersion.from_layer_version_arn(self, "pillow_layer", # layer_version_arn="arn:aws-cn:lambda:cn-northwest-1:313497334385:layer:pillow:4" # ) # The code that defines your stack goes here mylambda = aws_lambda.Function( self, "myfunction_id", description="lambda trigger by S3 to convert image", # function_name="img-process-cdk-deploy", # the name will be auto create code=aws_lambda.Code.asset("./lambda"), handler="lambda_function.lambda_handler", runtime=aws_lambda.Runtime.PYTHON_3_7, timeout=core.Duration.seconds(60), memory_size=256, reserved_concurrent_executions=10, layers=[lambda_layer], dead_letter_queue_enabled=True, # cdk will create a new SQS for this # role = aws_iam.Role.from_role_arn(self, "mylambda_role", # role_arn="arn:aws-cn:iam::313497334385:role/Lambda-access-img-process-S3bucket" # ) # cdk will create a role for this ) with open('./env.js', 'r') as f: env = json.load(f) for (k, v) in env.items(): mylambda.add_environment(k, v) #create a new bucket, Import bucket can not add event trigger s3_bucket = aws_s3.Bucket( self, "mybucket", # bucket name will be auto created or define here # bucket_name="img-process-cdk-created" ) mylambda.add_event_source( aws_lambda_event_sources.S3EventSource( s3_bucket, events=[aws_s3.EventType.OBJECT_CREATED], filters=[aws_s3.NotificationKeyFilter(prefix='input/')])) s3_bucket.grant_read_write(mylambda)
def create_trigger_training_task(self, **kwargs): #################### # Unpack Value name = kwargs['name'].replace("_", "-") lambda_name = "{}-lambda".format(name) code_name = "{}".format(name) job_name = "{}-job".format(name) task = "{}-task".format(name) instance = kwargs['instance'] image_uri = kwargs['image_uri'] trigger_bucket = kwargs['trigger_bucket'] input_train_bucket = kwargs['input_train_bucket'] input_validation_bucket = kwargs['input_validation_bucket'] # hparams = kwargs['hparams'] output_bucket = kwargs['output_bucket'] lambda_train_role = kwargs['lambda_role'] sagemaker_train_role = kwargs['sagemaker_role'].role_arn # Create Lambda lambda_app = _lambda.Function( self, lambda_name, handler='{}.handler'.format(code_name), runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset('lambda'), role=lambda_train_role, environment={ 'INPUT_TRAIN_BUCKET': input_train_bucket, 'INPUT_VALIDATION_BUCKET': input_validation_bucket, 'OUTPUT_BUCKET': output_bucket, 'NAME': name, 'IMAGE_URI': image_uri, 'SAGEMAKER_ROLE': sagemaker_train_role, # 'HPARAMS': hparams, 'INSTANCE': instance }) # Create an S3 event soruce for Lambda bucket = s3.Bucket(self, trigger_bucket) s3_event_source = lambda_event_source.S3EventSource( bucket, events=[s3.EventType.OBJECT_CREATED]) lambda_app.add_event_source(s3_event_source) return lambda_app
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) my_s3_bucket = _s3.Bucket(self, id='myS3Bucket') my_sns_topic = _sns.Topic(\ self, id = 'demoTopic' ) my_sns_sub = _sns_subscriptions.EmailSubscription( "*****@*****.**") my_sns_sub.bind(my_sns_topic) my_function = _lambda.Function(self, id='demoFunction', code=_lambda.Code.asset(r'../src'), handler='fun01.handler') trigger_event = _event.S3EventSource( bucket=my_s3_bucket, events=[_s3.EventType.OBJECT_CREATED])
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) input_bucket = s3.Bucket( self, "input-bucket" ) metadata_table = dynamodb.Table( self, "metadata-table", billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST, partition_key=dynamodb.Attribute(name="PK",type=dynamodb.AttributeType.STRING) ) dependency_layer_path = build_layer_package( requirement_files=[os.path.join(os.path.dirname(__file__), "..", "src", "requirements.txt")] ) dependency_layer = _lambda.LayerVersion( self, "dependency-layer", code=_lambda.Code.from_asset(path=dependency_layer_path), compatible_runtimes=[_lambda.Runtime.PYTHON_3_7, _lambda.Runtime.PYTHON_3_8] ) lambda_package_path = build_lambda_package( code_directories=[os.path.join(os.path.dirname(__file__), "..", "src")], ) processing_lambda = _lambda.Function( self, "processing-lambda", code=_lambda.Code.from_asset(path=lambda_package_path), handler="src.processing_lambda.lambda_handler", runtime=_lambda.Runtime.PYTHON_3_8, timeout=core.Duration.seconds(30), environment={ "METADATA_TABLE_NAME": metadata_table.table_name }, layers=[dependency_layer], description="Triggers object recognition on an S3 object and stores the metadata", tracing=_lambda.Tracing.ACTIVE ) # Allow Lambda to talk to Rekognition processing_lambda.add_to_role_policy( iam.PolicyStatement( actions=["rekognition:DetectLabels"], resources=["*"] ) ) # Create the S3 Event Trigger for Lambda processing_lambda.add_event_source( lambda_event_sources.S3EventSource( input_bucket, events=[s3.EventType.OBJECT_CREATED] ) ) input_bucket.grant_read(processing_lambda) metadata_table.grant_read_write_data(processing_lambda)
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: """ initialize function for CDK """ super().__init__(scope, construct_id, **kwargs) # ------------------------------- # S3 Bucket for Manifests # ------------------------------- qs_gov_bucket = s3.Bucket( self, id=f"{cf.PROJECT}-ManifestBucket", ) bucket_name = qs_gov_bucket.bucket_name # ------------------------------- # IAM # ------------------------------- list_roles_policy = iam.ManagedPolicy( self, id=f"{cf.PROJECT}-ListRolesPolicy", description=None, managed_policy_name=None, path="/", statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=["iam:ListRoles", "iam:ListAccountAliases"], ) ], ) federated_quicksight_policy = iam.ManagedPolicy( self, id=f"{cf.PROJECT}-FederatedQuickSightPolicy", managed_policy_name=f"{cf.PROJECT}-FederatedQuickSightPolicy", statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:iam::{cf.ACCOUNT}:saml-provider/{cf.OKTA_IDP_NAME}" ], actions=["sts:AssumeRoleWithSAML"], conditions={ "StringEquals": { "saml:aud": "https://signin.aws.amazon.com/saml" } }, ) ], ) okta_federated_principal = iam.FederatedPrincipal( federated= f"arn:aws:iam::{cf.ACCOUNT}:saml-provider/{cf.OKTA_IDP_NAME}", assume_role_action="sts:AssumeRoleWithSAML", conditions={ "StringEquals": { "SAML:aud": "https://signin.aws.amazon.com/saml" } }, ) federated_quicksight_role = iam.Role( self, id=f"{cf.PROJECT}-{cf.OKTA_ROLE_NAME}", role_name=f"{cf.PROJECT}-{cf.OKTA_ROLE_NAME}", assumed_by=okta_federated_principal, description= "Allow Okta to Federate Login & User Creation to QuickSight", managed_policies=[federated_quicksight_policy], ) iam.User( self, id=f"{cf.PROJECT}-OktaSSOUser", user_name=f"{cf.PROJECT}-OktaSSOUser", managed_policies=[list_roles_policy], ) # ------------------------------- # Lambda Functions # ------------------------------- # iam role for Lambdas qs_governance_policy = iam.ManagedPolicy( self, id=f"{cf.PROJECT}-QuickSightGovernancePolicy", managed_policy_name=f"{cf.PROJECT}-QuickSightGovernancePolicy", statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:secretsmanager:{cf.REGION}:{cf.ACCOUNT}:secret:{cf.OKTA_SECRET}*" ], actions=[ "secretsmanager:GetSecretValue", ], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=["quicksight:*", "ds:*"], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[f"arn:aws:s3:::{bucket_name}/*"], actions=["s3:Get*", "s3:Put*"], ), ], ) quicksight_permission_mapping_role = iam.Role( self, id=f"{cf.PROJECT}-QuickSightPermissionMappingRole", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaBasicExecutionRole"), qs_governance_policy, ], ) # Lambdas get_okta_info_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-GetOktaInfo", handler="get_okta_info.handler", role=quicksight_permission_mapping_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), function_name=f"{cf.PROJECT}-GetOktaInfo", environment={ "OKTA_SECRET": cf.OKTA_SECRET, "OKTA_ROLE_NAME": cf.OKTA_ROLE_NAME, "QS_GOVERNANCE_BUCKET": bucket_name, "QS_USER_GOVERNANCE_KEY": cf.QS_USER_GOVERNANCE_KEY, }, memory_size=256, timeout=core.Duration.seconds(180), ) # Lamda Okta to QuickSight Mappers qs_user_governance_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-QSUserGovernance", handler="qs_user_gov.handler", role=quicksight_permission_mapping_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), function_name=f"{cf.PROJECT}-QSUserGovernance", environment={ "OKTA_ROLE_NAME": f"{cf.PROJECT}-{cf.OKTA_ROLE_NAME}", "QS_GOVERNANCE_BUCKET": bucket_name, "QS_USER_GOVERNANCE_KEY": cf.QS_USER_GOVERNANCE_KEY, "OKTA_GROUP_QS_PREFIX": cf.OKTA_GROUP_QS_PREFIX, "QS_ADMIN_OKTA_GROUP": cf.QS_ADMIN_OKTA_GROUP, "QS_AUTHOR_OKTA_GROUP": cf.QS_AUTHOR_OKTA_GROUP, "QS_READER_OKTA_GROUP": cf.QS_READER_OKTA_GROUP }, memory_size=256, timeout=core.Duration.seconds(180), ) qs_asset_governance_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-QSAssetGovernance", handler="qs_asset_gov.handler", role=quicksight_permission_mapping_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), function_name=f"{cf.PROJECT}-QSAssetGovernance", environment={ "QS_GOVERNANCE_BUCKET": bucket_name, "QS_ASSET_GOVERNANCE_KEY": cf.QS_ASSET_GOVERNANCE_KEY, }, memory_size=256, timeout=core.Duration.seconds(180), ) # ------------------------------- # Events # ------------------------------- qs_user_governance_lambda.add_event_source( lambda_event_sources.S3EventSource( bucket=qs_gov_bucket, events=[s3.EventType.OBJECT_CREATED], filters=[ s3.NotificationKeyFilter(prefix=cf.QS_USER_GOVERNANCE_KEY) ], )) qs_asset_governance_lambda.add_event_source( lambda_event_sources.S3EventSource( bucket=qs_gov_bucket, events=[s3.EventType.OBJECT_CREATED], filters=[ s3.NotificationKeyFilter(prefix=cf.QS_ASSET_GOVERNANCE_KEY) ], )) lambda_schedule = events.Schedule.rate(core.Duration.days(1)) get_okta_info_target = events_targets.LambdaFunction( handler=get_okta_info_lambda) events.Rule( self, id=f"{cf.PROJECT}-GetOktaInfoScheduledEvent", description= "The once per day CloudWatch event trigger for the Lambda", enabled=True, schedule=lambda_schedule, targets=[get_okta_info_target], ) # ------------------------------- # S3 Object Deployment - QS Asset Manifest # ------------------------------- asset_manifest_deploy = s3_deploy.BucketDeployment( self, id=f"{cf.PROJECT}-AssetManifestDeploy", sources=[ s3_deploy.Source.asset(os.path.join(cf.PATH_ROOT, 'qs_config')) ], destination_bucket=qs_gov_bucket)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # ------ Necessary Roles ------ roles = IamRole( self, 'IamRoles' ) # ------ S3 Buckets ------ # Create Athena bucket athena_bucket = _s3.Bucket(self, "AthenaBucket", removal_policy=core.RemovalPolicy.DESTROY ) # Create Forecast bucket forecast_bucket = _s3.Bucket(self, "FoecastBucket", removal_policy=core.RemovalPolicy.DESTROY ) # ------ Athena ------ # Config Athena query result output location workgroup_prop = _athena.CfnWorkGroup.WorkGroupConfigurationProperty( result_configuration=_athena.CfnWorkGroup.ResultConfigurationProperty( output_location="s3://"+athena_bucket.bucket_name ) ) # Create Athena workgroup athena_workgroup = _athena.CfnWorkGroup( self, 'ForecastGroup', name='ForecastGroup', recursive_delete_option=True, state='ENABLED', work_group_configuration=workgroup_prop ) # ------ SNS Topic ------ topic = sns.Topic( self, 'NotificationTopic', display_name='StepsTopic' ) # SNS email subscription. Get the email address from context value(cdk.json) topic.add_subscription(subs.EmailSubscription(self.node.try_get_context('my_email'))) # ------ Layers ------ shared_layer = _lambda.LayerVersion( self, 'LambdaLayer', layer_version_name='testfolderlayer', code=_lambda.AssetCode('shared/') ) # ------ Lambdas for stepfuctions------ create_dataset_lambda = _lambda.Function( self, 'CreateDataset', function_name='CreateDataset', code=_lambda.Code.asset('lambdas/createdataset/'), handler='dataset.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, timeout=core.Duration.seconds(30), layers=[shared_layer] ) create_dataset_group_lambda = _lambda.Function( self, 'CreateDatasetGroup', function_name='CreateDatasetGroup', code = _lambda.Code.asset('lambdas/createdatasetgroup/'), handler = 'datasetgroup.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) import_data_lambda = _lambda.Function( self, 'CreateDatasetImportJob', function_name='CreateDatasetImportJob', code = _lambda.Code.asset('lambdas/createdatasetimportjob/'), handler = 'datasetimport.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, environment= { 'FORECAST_ROLE': roles.forecast_role.role_arn }, layers=[shared_layer] ) create_predictor_lambda = _lambda.Function( self, 'CreatePredictor', function_name='CreatePredictor', code = _lambda.Code.asset('lambdas/createpredictor/'), handler = 'predictor.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) create_forecast_lambda = _lambda.Function( self, 'CreateForecast', function_name='CreateForecast', code = _lambda.Code.asset('lambdas/createforecast/'), handler = 'forecast.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, environment= { 'EXPORT_ROLE': roles.forecast_role.role_arn }, layers=[shared_layer], timeout=core.Duration.seconds(30) ) # Deploy lambda with python dependencies from requirements.txt update_resources_lambda = _lambda_python.PythonFunction( self, 'UpdateResources', function_name='UpdateResources', entry='lambdas/updateresources/', index='update.py', handler='lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.update_role, environment= { 'ATHENA_WORKGROUP': athena_workgroup.name, 'ATHENA_BUCKET' : athena_bucket.bucket_name }, layers=[shared_layer], timeout=core.Duration.seconds(900) ) notify_lambda = _lambda.Function( self, 'NotifyTopic', function_name='NotifyTopic', code = _lambda.Code.asset('lambdas/notify/'), handler = 'notify.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, environment= { 'SNS_TOPIC_ARN': topic.topic_arn }, layers=[shared_layer] ) delete_forecast_lambda = _lambda.Function( self, 'DeleteForecast', function_name='DeleteForecast', code = _lambda.Code.asset('lambdas/deleteforecast/'), handler = 'deleteforecast.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) delete_predctor_lambda = _lambda.Function( self, 'DeletePredictor', function_name='DeletePredictor', code = _lambda.Code.asset('lambdas/deletepredictor/'), handler = 'deletepredictor.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) delete_importjob_lambda = _lambda.Function( self, 'DeleteImportJob', function_name='DeleteImportJob', code = _lambda.Code.asset('lambdas/deletedatasetimport/'), handler = 'deletedataset.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) # ------ StepFunctions ------ strategy_choice = sfn.Choice( self, 'Strategy-Choice' ) success_state = sfn.Succeed( self, 'SuccessState' ) failed = sfn_tasks.LambdaInvoke( self, 'Failed', lambda_function = notify_lambda, result_path=None ).next(strategy_choice) create_dataset_job = sfn_tasks.LambdaInvoke( self, 'Create-Dataset', lambda_function = create_dataset_lambda, retry_on_service_exceptions=True, payload_response_only=True ) self.add_retry_n_catch(create_dataset_job, failed) create_dataset_group_job = sfn_tasks.LambdaInvoke( self, 'Create-DatasetGroup', lambda_function = create_dataset_group_lambda, payload_response_only=True ) self.add_retry_n_catch(create_dataset_group_job, failed) import_data_job = sfn_tasks.LambdaInvoke( self, 'Import-Data', lambda_function = import_data_lambda, payload_response_only=True ) self.add_retry_n_catch(import_data_job, failed) create_predictor_job = sfn_tasks.LambdaInvoke( self, 'Create-Predictor', lambda_function = create_predictor_lambda, payload_response_only=True ) self.add_retry_n_catch(create_predictor_job, failed) create_forecast_job = sfn_tasks.LambdaInvoke( self, 'Create-Forecast', lambda_function = create_forecast_lambda, payload_response_only=True ) self.add_retry_n_catch(create_forecast_job, failed) update_resources_job = sfn_tasks.LambdaInvoke( self, 'Update-Resources', lambda_function = update_resources_lambda, payload_response_only=True ) self.add_retry_n_catch(update_resources_job, failed) notify_success = sfn_tasks.LambdaInvoke( self, 'Notify-Success', lambda_function = notify_lambda, payload_response_only=True ) delete_forecast_job = sfn_tasks.LambdaInvoke( self, 'Delete-Forecast', lambda_function = delete_forecast_lambda, payload_response_only=True ) self.delete_retry(delete_forecast_job) delete_predictor_job = sfn_tasks.LambdaInvoke( self, 'Delete-Predictor', lambda_function = delete_predctor_lambda, payload_response_only=True ) self.delete_retry(delete_predictor_job) delete_import_job = sfn_tasks.LambdaInvoke( self, 'Delete-ImportJob', lambda_function = delete_importjob_lambda, payload_response_only=True ) self.delete_retry(delete_import_job) definition = create_dataset_job\ .next(create_dataset_group_job)\ .next(import_data_job)\ .next(create_predictor_job)\ .next(create_forecast_job)\ .next(update_resources_job)\ .next(notify_success)\ .next(strategy_choice.when(sfn.Condition.boolean_equals('$.params.PerformDelete', False), success_state)\ .otherwise(delete_forecast_job).afterwards())\ .next(delete_predictor_job)\ .next(delete_import_job) deployt_state_machine = sfn.StateMachine( self, 'StateMachine', definition = definition # role=roles.states_execution_role ) # S3 event trigger lambda s3_lambda = _lambda.Function( self, 'S3Lambda', function_name='S3Lambda', code=_lambda.Code.asset('lambdas/s3lambda/'), handler='parse.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, role=roles.trigger_role, environment= { 'STEP_FUNCTIONS_ARN': deployt_state_machine.state_machine_arn, 'PARAMS_FILE': self.node.try_get_context('parameter_file') } ) s3_lambda.add_event_source( event_src.S3EventSource( bucket=forecast_bucket, events=[_s3.EventType.OBJECT_CREATED], filters=[_s3.NotificationKeyFilter( prefix='train/', suffix='.csv' )] ) ) # CloudFormation output core.CfnOutput( self, 'StepFunctionsName', description='Step Functions Name', value=deployt_state_machine.state_machine_name ) core.CfnOutput( self, 'ForecastBucketName', description='Forecast bucket name to drop you files', value=forecast_bucket.bucket_name ) core.CfnOutput( self, 'AthenaBucketName', description='Athena bucket name to drop your files', value=athena_bucket.bucket_name )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) stack_role = iam.Role( self, "SimulationServiceRole", assumed_by=iam.ServicePrincipal("batch.amazonaws.com"), ) stack_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "AdministratorAccess")) job_role = iam.Role( self, "SimulationJobServiceRole", assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"), ) job_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "AdministratorAccess")) lambda_role = iam.Role( self, "SimulationLambdaServiceRole", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), ) lambda_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "AdministratorAccess")) # Create Input S3 input_bucket = s3.Bucket(self, "InputS3Bucket") # Create Output S3 output_bucket = s3.Bucket(self, "OutputS3Bucket") # admin_policy = iam.from_policy_name("AdministratorAccess", "AdministratorAccess") job_table = aws_dynamodb.Table( self, id="JobTable", partition_key=aws_dynamodb.Attribute( name="PK", type=aws_dynamodb.AttributeType.STRING), stream=aws_dynamodb.StreamViewType.NEW_AND_OLD_IMAGES, removal_policy=core.RemovalPolicy.DESTROY, ) orchestration_handler_lambda = aws_lambda.Function( self, id="JobOrchestrationHandler", runtime=aws_lambda.Runtime.PYTHON_3_7, handler="orchestration_handler_lambda.handler", code=aws_lambda.Code.asset("./simulations_service/functions/"), ) # Give only write access to the post handler job_table.grant_write_data(orchestration_handler_lambda) # Pass table_name as env variable orchestration_handler_lambda.add_environment("TABLE_NAME", job_table.table_name) # Create lambda function for processing dynamodb streams dynamodb_streams_processor_lambda = aws_lambda.Function( self, id="JobsDynamoDBStreamsProcessor", runtime=aws_lambda.Runtime.PYTHON_3_7, handler="dynamodb_streams_processor_lambda.handler", code=aws_lambda.Code.asset("./simulations_service/functions/"), role=lambda_role, ) # Add dynamo db as lambda event source dynamodb_streams_processor_lambda.add_event_source( aws_lambda_event_sources.DynamoEventSource( job_table, starting_position=aws_lambda.StartingPosition.LATEST, batch_size=1, )) dynamodb_streams_processor_lambda.add_environment( "S3_OUTPUT_BUCKET", output_bucket.bucket_name) dynamodb_streams_processor_lambda.add_environment( "TABLE_NAME", job_table.table_name) vpc = ec2.Vpc(self, "VPC") spot_environment = batch.ComputeEnvironment( self, "MyComputedEnvironment", compute_resources={ "vpc": vpc, }, service_role=stack_role.without_policy_updates(), ) job_queue = batch.JobQueue( self, "JobQueue", compute_environments=[ batch.JobQueueComputeEnvironment( compute_environment=spot_environment, order=1) ], ) dynamodb_streams_processor_lambda.add_environment( "JOB_QUEUE", job_queue.job_queue_name) job_definition = batch.JobDefinition( self, "batch-job-def-from-local", container={ "image": ecs.ContainerImage.from_asset("./simulations_service/job/"), "memory_limit_mib": 500, "privileged": True, "job_role": job_role, }, ) dynamodb_streams_processor_lambda.add_environment( "JOB_DEFINITION", job_definition.job_definition_name) orchestration_handler_lambda.add_event_source( aws_lambda_event_sources.S3EventSource( bucket=input_bucket, events=[s3.EventType.OBJECT_CREATED], ))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) bucket_with_sns = s3.Bucket(self, "bucket-with-sns-integration", removal_policy=core.RemovalPolicy.DESTROY) bucket_with_lambda = s3.Bucket( self, "bucket-with-lambda-integration", removal_policy=core.RemovalPolicy.DESTROY) exchange_topic = sns.Topic(self, "lambda-info-topic") bucket_with_sns.add_event_notification( event=s3.EventType.OBJECT_CREATED, dest=s3_notifications.SnsDestination(exchange_topic)) measurement_table = dynamodb.Table( self, "measurement-table", partition_key=dynamodb.Attribute( name="PK", type=dynamodb.AttributeType.STRING), sort_key=dynamodb.Attribute(name="SK", type=dynamodb.AttributeType.STRING), billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST, removal_policy=core.RemovalPolicy.DESTROY) s3_event_generator = _lambda.Function( self, "s3-event-generator", code=_lambda.Code.from_asset( path=os.path.join(os.path.dirname(__file__), "..", "src")), environment={ "BUCKET_WITH_LAMBDA": bucket_with_lambda.bucket_name, "BUCKET_WITH_SNS": bucket_with_sns.bucket_name, }, handler="s3_event_generator.lambda_handler", runtime=_lambda.Runtime.PYTHON_3_8, timeout=core.Duration.seconds(300), memory_size=1024, ) bucket_with_lambda.grant_write(s3_event_generator) bucket_with_sns.grant_write(s3_event_generator) measure_lambda = _lambda.Function( self, "measure-lambda", code=_lambda.Code.from_asset( path=os.path.join(os.path.dirname(__file__), "..", "src")), environment={"MEASUREMENT_TABLE": measurement_table.table_name}, handler="measure_lambda.lambda_handler", runtime=_lambda.Runtime.PYTHON_3_8, timeout=core.Duration.seconds(50), memory_size=1024, ) measurement_table.grant_read_write_data(measure_lambda) measure_lambda.add_event_source( lambda_event_sources.SnsEventSource(exchange_topic)) measure_lambda.add_event_source( lambda_event_sources.S3EventSource( bucket=bucket_with_lambda, events=[s3.EventType.OBJECT_CREATED])) ssm.StringParameter(self, "bucket-with-sns-parameter", string_value=bucket_with_sns.bucket_name, parameter_name=BUCKET_WITH_SNS_PARAMETER) ssm.StringParameter(self, "bucket-with-lambda-parameter", string_value=bucket_with_lambda.bucket_name, parameter_name=BUCKET_WITH_LAMBDA_PARAMETER) ssm.StringParameter(self, "measurement-table-parameter", string_value=measurement_table.table_name, parameter_name=MEASUREMENT_TABLE_PARAMETER) ssm.StringParameter(self, "generator-function-name-parameter", string_value=s3_event_generator.function_name, parameter_name=GENERATOR_FUNCTION_NAME_PARAMETER)
def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # TODO should this stuff be passed as inputs to the stack ? source_code_directory = "/opt/python" # https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/Cluster.html#aws_cdk.aws_ecs.Cluster.add_capacity asg_parameters = { "instance_type": ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE2, ec2.InstanceSize.MICRO), "machine_image": ecs.EcsOptimizedImage.amazon_linux2(), "desired_capacity": 0, "max_capacity": 5, "min_capacity": 0, } # https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/Ec2TaskDefinition.html#aws_cdk.aws_ecs.Ec2TaskDefinition.add_container container_settings = { "memory_limit_mib": 300, "logging": ecs.AwsLogDriver(stream_prefix="ecslogs"), } input_bucket_name = "cdkdemoinput" output_bucket_name = "cdkdemooutput" # Create an Docker image from an given directory, which will be later published to Amazon ECR # TODO can this be cleanup on destroy as well ? container_image = ecs.ContainerImage.from_asset( directory=source_code_directory) # Create an Amazon ECS cluster cluster = ecs.Cluster(self, "ecscluster") cluster.apply_removal_policy(cdk.RemovalPolicy.DESTROY) # Create an auto scaling group for the ECS cluster asg = cluster.add_capacity("ecsautoscalinggroup", **asg_parameters) # TODO check if needed asg.apply_removal_policy(cdk.RemovalPolicy.DESTROY) # Create a capacity provider for the ECS cluster based on the auto scaling group capacity_provider = ecs.CfnCapacityProvider( self, "ecscapacityprovider", # Name can't start with ecs... name="capacityproviderecs", auto_scaling_group_provider=ecs.CfnCapacityProvider. AutoScalingGroupProviderProperty( auto_scaling_group_arn=asg.auto_scaling_group_name, managed_scaling=ecs.CfnCapacityProvider.ManagedScalingProperty( status="ENABLED"), # TODO investigate this better managed_termination_protection="DISABLED", ), ) capacity_provider.apply_removal_policy(cdk.RemovalPolicy.DESTROY) # Currently the CDK checks if the string is FARGATE or FARGATE_SPOT and errors out # cluster.add_capacity_provider(capacity_provider.name) lame_hack = cr.AwsCustomResource( self, "lamehack", on_create={ "service": "ECS", "action": "putClusterCapacityProviders", "parameters": { "cluster": cluster.cluster_arn, "capacityProviders": [capacity_provider.name], "defaultCapacityProviderStrategy": [], }, "physical_resource_id": cr.PhysicalResourceId.of(str(int(time.time()))), }, on_delete={ "service": "ECS", "action": "putClusterCapacityProviders", "parameters": { "cluster": cluster.cluster_arn, "capacityProviders": [], "defaultCapacityProviderStrategy": [], }, }, # TODO lower this permissions policy=cr.AwsCustomResourcePolicy.from_sdk_calls( resources=cr.AwsCustomResourcePolicy.ANY_RESOURCE), ) lame_hack.node.add_dependency(capacity_provider) lame_hack.node.add_dependency(cluster) # Create an ECS task definition with our Docker image task_definition = ecs.Ec2TaskDefinition(self, "ecstaskdefinition") container_definition = task_definition.add_container( "ecscontainer", image=container_image, **container_settings) # TODO lower this permissions task_definition.task_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonS3FullAccess")) task_definition.apply_removal_policy(cdk.RemovalPolicy.DESTROY) # Create the Amazon S3 input and output buckets input_bucket = s3.Bucket( self, "bucketinput", bucket_name=input_bucket_name, versioned=False, removal_policy=cdk.RemovalPolicy.DESTROY, auto_delete_objects=True, ) output_bucket = s3.Bucket( self, "bucketoutput", bucket_name=output_bucket_name, versioned=False, removal_policy=cdk.RemovalPolicy.DESTROY, auto_delete_objects=True, ) # Create the Amazon Lambda function for transforming the input from bucket information to the container inputs function = lambda_.Function( self, "inputlambda", code=lambda_.Code.from_inline(lambda_function_code), handler="index.lambda_handler", runtime=lambda_.Runtime.PYTHON_3_8, environment={ "CAPACITY_PROVIDER_NAME": capacity_provider.name, "CLUSTER_NAME": cluster.cluster_arn, "CONTAINER_NAME": container_definition.container_name, "REGION_NAME": self.region, # TODO flaky, why can't we pass the ARN directly ? "OUTPUT_BUCKET_NAME": output_bucket.bucket_name, "TASK_DEFINITION": task_definition.task_definition_arn, }, ) # Add an S3 object creation trigger for the function function.add_event_source( lambda_event_sources.S3EventSource( input_bucket, events=[s3.EventType.OBJECT_CREATED])) # TODO fix this for less permissions function.role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonECS_FullAccess")) function.apply_removal_policy(cdk.RemovalPolicy.DESTROY)
def __init__(self, scope: core.Construct, id: str, instance_id: str, contact_flow_id: str, source_phone_number: str, timeout: int, **kwargs): super().__init__(scope, id, **kwargs) web_bucket = _s3.Bucket(self, "StaticWebBucket", website_index_document="index.html", website_error_document="index.html", removal_policy=core.RemovalPolicy.DESTROY, public_read_access=True) core.CfnOutput(self, 'WebBucketUrl', value=web_bucket.bucket_domain_name) web_distribution = _clf.CloudFrontWebDistribution( self, 'StaticWebDistribution', origin_configs=[ _clf.SourceConfiguration( s3_origin_source=_clf.S3OriginConfig( s3_bucket_source=web_bucket), behaviors=[_clf.Behavior(is_default_behavior=True)]) ], viewer_protocol_policy=_clf.ViewerProtocolPolicy.REDIRECT_TO_HTTPS) _s3d.BucketDeployment( self, "S3StaticWebContentDeploymentWithInvalidation", sources=[ _s3d.Source.asset( f"{pathlib.Path(__file__).parent.absolute()}/site-content/build" ) ], destination_bucket=web_bucket, distribution=web_distribution, distribution_paths=["/*"]) file_bucket = _s3.Bucket(self, "FileBucket", removal_policy=core.RemovalPolicy.DESTROY) call_dead_letter_queue = _sqs.Queue(self, "CallDeadLetterQueue", fifo=True, content_based_deduplication=True) call_sqs_queue = _sqs.Queue( self, "CallSqsQueue", fifo=True, content_based_deduplication=True, visibility_timeout=core.Duration.seconds(120), dead_letter_queue=_sqs.DeadLetterQueue( max_receive_count=1, queue=call_dead_letter_queue)) async_call_dead_letter_queue = _sqs.Queue( self, "AsyncCallDeadLetterQueue", fifo=True, content_based_deduplication=True) async_callout_queue = _sqs.Queue( self, "AsyncCalloutQueue", fifo=True, content_based_deduplication=True, visibility_timeout=core.Duration.seconds(120), dead_letter_queue=_sqs.DeadLetterQueue( max_receive_count=1, queue=async_call_dead_letter_queue)) call_job_complete_sns_topic = _sns.Topic( self, "CallJobCompleteSnsTopic", display_name="CallJobCompletion") call_result_table = _dynamodb.Table( self, "CallResultDynamodbTable", billing_mode=_dynamodb.BillingMode.PAY_PER_REQUEST, partition_key=_dynamodb.Attribute( name="task_id", type=_dynamodb.AttributeType.STRING), sort_key=_dynamodb.Attribute(name="receiver_id", type=_dynamodb.AttributeType.STRING), removal_policy=core.RemovalPolicy.DESTROY) callout_record_table = _dynamodb.Table( self, "CallTaskDynamodbTable", billing_mode=_dynamodb.BillingMode.PAY_PER_REQUEST, partition_key=_dynamodb.Attribute( name="task_id", type=_dynamodb.AttributeType.STRING), sort_key=_dynamodb.Attribute(name="created_at", type=_dynamodb.AttributeType.NUMBER), removal_policy=core.RemovalPolicy.DESTROY) callout_record_table.add_global_secondary_index( partition_key=_dynamodb.Attribute( name='call_type', type=_dynamodb.AttributeType.STRING), sort_key=_dynamodb.Attribute(name='created_at', type=_dynamodb.AttributeType.NUMBER), index_name='CallTypeCreatedAtGlobalIndex', projection_type=_dynamodb.ProjectionType.ALL) python_function_layer = _lambda.LayerVersion( self, "LambdaPythonFunctionLayer", code=_lambda.Code.asset("aws_callouts_cdk/layer/_python"), compatible_runtimes=[ _lambda.Runtime.PYTHON_3_7, _lambda.Runtime.PYTHON_3_8 ], license="Available under the MIT-0 license") nodejs_function_layer = _lambda.LayerVersion( self, "LambdaNodeJsFunctionLayer", code=_lambda.Code.asset("aws_callouts_cdk/layer/_nodejs"), compatible_runtimes=[ _lambda.Runtime.NODEJS_10_X, _lambda.Runtime.NODEJS_12_X ], license="Available under the MIT-0 license") global_python_function_arguments = { "code": _lambda.Code.asset("aws_callouts_cdk/src/python"), "layers": [python_function_layer], "runtime": _lambda.Runtime.PYTHON_3_7 } global_nodeje_function_arguments = { "code": _lambda.Code.asset("aws_callouts_cdk/src/nodejs"), "layers": [nodejs_function_layer], "runtime": _lambda.Runtime.NODEJS_12_X } get_callout_job_function = _lambda.Function( self, "GetCalloutJobFunction", handler="get_call_job.lambda_handler", **global_python_function_arguments) get_callout_job_function.add_environment(key="S3Bucket", value=file_bucket.bucket_name) file_bucket.grant_read(get_callout_job_function) callout_function = _lambda.Function(self, "CalloutFunction", handler="send_call.lambda_handler", **global_python_function_arguments) callout_function.add_environment( key="ContactFlowArn", value= f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}/contact-flow/{contact_flow_id}" ) callout_function.add_environment(key="SourcePhoneNumber", value=source_phone_number) callout_function.add_environment(key="ExcelFileBucket", value=file_bucket.bucket_name) callout_function.add_environment(key="AsynCalloutQueueUrl", value=async_callout_queue.queue_url) callout_function.add_to_role_policy(statement=_iam.PolicyStatement( resources=[ f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}/*" ], actions=["connect:StartOutboundVoiceContact"])) callout_function.add_event_source(source=_les.SqsEventSource( queue=async_callout_queue, batch_size=1)) 'arn:aws:connect:751225572132:ap-southeast-2:instance/9d0c7cc5-7d2a-42e4-a3dd-70f402e0d040' file_bucket.grant_read_write(callout_function) response_handler_function = _lambda.Function( self, "ResponseHandlerFunction", handler="response_handler.lambda_handler", **global_python_function_arguments) response_handler_function.add_permission( id="ResponseHandlerFunctionLambdaInvokePermission", principal=_iam.ServicePrincipal(service="connect.amazonaws.com"), action="lambda:InvokeFunction", source_account=self.account, source_arn= f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}" ) send_task_success_function = _lambda.Function( self, "SendTaskSuccessFunction", handler="send_task_success.lambda_handler", **global_python_function_arguments) send_task_success_function.add_permission( id="SendTaskSuccessFunctionLambdaInvokePermission", principal=_iam.ServicePrincipal(service="connect.amazonaws.com"), action="lambda:InvokeFunction", source_account=self.account, source_arn= f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}" ) get_call_result_function = _lambda.Function( self, "GetCallResultFunction", handler="get_call_result.lambda_handler", memory_size=512, **global_python_function_arguments) get_call_result_function.add_environment( key="CallResultDynamoDBTable", value=call_result_table.table_name) get_call_result_function.add_environment(key="S3Bucket", value=file_bucket.bucket_name) call_result_table.grant_read_data(grantee=get_call_result_function) file_bucket.grant_read_write(get_call_result_function) iterator_function = _lambda.Function( self, "IteratorFunction", handler="iterator.lambda_handler", **global_python_function_arguments) iterator_function.add_permission( id="IteratorFunctionLambdaInvokePermission", principal=_iam.ServicePrincipal(service="connect.amazonaws.com"), action="lambda:InvokeFunction", source_account=self.account, source_arn= f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}" ) create_appsync_call_task_function = _lambda.Function( self, "CreateAppSyncCallTaskFunction", handler="create_appsync_call_task.lambda_handler", **global_nodeje_function_arguments) create_appsync_call_task_function.add_environment( key="CallSqsQueueUrl", value=call_sqs_queue.queue_url) create_appsync_call_task_function.add_environment( key="CallRecordTableName", value=callout_record_table.table_name) call_sqs_queue.grant_send_messages(create_appsync_call_task_function) callout_record_table.grant_write_data( create_appsync_call_task_function) create_call_report_record_function = _lambda.Function( self, "CreateCallReportRecordFunction", handler="create_call_report_record.lambda_handler", **global_nodeje_function_arguments) create_excel_call_task_function = _lambda.Function( self, "CreateExcelCallTaskFunction", handler="create_excel_call_task.lambda_handler", **global_python_function_arguments) create_excel_call_task_function.add_environment( key="CallSqsQueueUrl", value=call_sqs_queue.queue_url) call_sqs_queue.grant_send_messages(create_excel_call_task_function) create_excel_call_task_function.add_event_source( source=_les.S3EventSource(bucket=file_bucket, events=[_s3.EventType.OBJECT_CREATED], filters=[ _s3.NotificationKeyFilter( prefix="call_task", suffix=".xlsx") ])) start_callout_flow_function = _lambda.Function( self, "StartCalloutFlowFunction", handler="start_call_out_flow.lambda_handler", reserved_concurrent_executions=1, **global_python_function_arguments) start_callout_flow_function.add_environment( key="CallSqsQueueUrl", value=call_sqs_queue.queue_url) start_callout_flow_function.add_environment( key="ResponseHandlerFunctionArn", value=response_handler_function.function_arn) start_callout_flow_function.add_environment( key="IteratorFunctionArn", value=iterator_function.function_arn) start_callout_flow_function.add_environment( key="SendTaskSuccessFunctionArn", value=send_task_success_function.function_arn) start_callout_flow_function.add_environment( key="S3Bucket", value=file_bucket.bucket_name) start_callout_flow_function.add_event_source( source=_les.SqsEventSource(queue=call_sqs_queue, batch_size=1)) file_bucket.grant_read_write(start_callout_flow_function) call_state_machine_definition = { "Comment": "Reading messages from an SQS queue and iteratively processing each message.", "StartAt": "Start", "States": { "Start": { "Type": "Pass", "Next": "Process Call Messages" }, "Process Call Messages": { "Type": "Map", "Next": "Get Call Result", "InputPath": "$", "ItemsPath": "$", "OutputPath": "$.[0]", "Iterator": { "StartAt": "Get Call out job", "States": { "Get Call out job": { "Type": "Task", "Resource": get_callout_job_function.function_arn, "Next": "Callout with AWS Connect" }, "Callout with AWS Connect": { "Type": "Task", "Resource": "arn:aws:states:::sqs:sendMessage.waitForTaskToken", "TimeoutSeconds": timeout, "Parameters": { "QueueUrl": async_callout_queue.queue_url, "MessageGroupId": "1", "MessageBody": { "Message.$": "$", "TaskToken.$": "$$.Task.Token" } }, "Catch": [{ "ErrorEquals": ["States.Timeout"], "ResultPath": None, "Next": "Call Timeout" }], "Next": "Save call result" }, "Call Timeout": { "Type": "Pass", "ResultPath": None, "Next": "Save call result" }, "Save call result": { "Type": "Task", "Resource": "arn:aws:states:::dynamodb:putItem", "Parameters": { "TableName": call_result_table.table_name, "Item": { "receiver_id": { "S.$": "$.receiver_id" }, "task_id": { "S.$": "$.task_id" }, "username": { "S.$": "$.username" }, "phone_number": { "S.$": "$.phone_number" }, "status": { "S.$": "$.status" }, "answers": { "S.$": "$.answers" }, "error": { "S.$": "$.error" }, "call_at": { "S.$": "$.call_at" } } }, "ResultPath": "$.Result", "OutputPath": "$.task_id", "End": True } } } }, "Get Call Result": { "Type": "Task", "Resource": get_call_result_function.function_arn, "Next": "Create Call Report Record" }, "Create Call Report Record": { "Type": "Task", "Resource": create_call_report_record_function.function_arn, "Next": "Send Completion message to SNS" }, "Send Completion message to SNS": { "Type": "Task", "Resource": "arn:aws:states:::sns:publish", "Parameters": { "TopicArn": call_job_complete_sns_topic.topic_arn, "Message.$": "$" }, "Next": "Finish" }, "Finish": { "Type": "Succeed" } } } callout_state_machine_role = _iam.Role( self, "CalloutStatesExecutionRole", assumed_by=_iam.ServicePrincipal( f"states.{self.region}.amazonaws.com")) callout_state_machine_role.add_to_policy( _iam.PolicyStatement( actions=[ "sqs:SendMessage", "dynamodb:PutItem", "lambda:InvokeFunction", "SNS:Publish" ], resources=[ async_callout_queue.queue_arn, call_result_table.table_arn, get_callout_job_function.function_arn, get_call_result_function.function_arn, call_job_complete_sns_topic.topic_arn, create_appsync_call_task_function.function_arn, create_call_report_record_function.function_arn ])) callout_state_machine = _sfn.CfnStateMachine( self, "CalloutStateMachine", role_arn=callout_state_machine_role.role_arn, definition_string=json.dumps(call_state_machine_definition)) send_task_success_function.add_to_role_policy( _iam.PolicyStatement(actions=["states:SendTaskSuccess"], resources=[callout_state_machine.ref])) start_callout_flow_function.add_environment( key="CalloutStateMachineArn", value=callout_state_machine.ref) start_callout_flow_function.add_to_role_policy( _iam.PolicyStatement(effect=_iam.Effect.ALLOW, resources=[callout_state_machine.ref], actions=['states:StartExecution'])) user_pool = _cognito.UserPool( self, "UserPool", sign_in_type=_cognito.SignInType.USERNAME) user_pool_client = _cognito.UserPoolClient(self, "UserPoolClient", user_pool=user_pool) appsync_api = _appsync.GraphQLApi( self, "AppSyncApi", name="AWSCalloutApi", user_pool_config=_appsync.UserPoolConfig( user_pool=user_pool, default_action=_appsync.UserPoolDefaultAction.ALLOW), log_config=_appsync.LogConfig( field_log_level=_appsync.FieldLogLevel.ALL), schema_definition_file= f"{pathlib.Path(__file__).parent.absolute()}/schema.graphql") callout_record_ddb_ds = appsync_api.add_dynamo_db_data_source( name="CalloutRecordDdb", description="Callout Record DynamoDB Data Source", table=callout_record_table) callout_record_ddb_ds.create_resolver( type_name="Query", field_name="getLatestCallTaskRecords", request_mapping_template=_appsync.MappingTemplate.from_string( '{"version":"2017-02-28","operation":"Query","index":"CallTypeCreatedAtGlobalIndex","query":{"expression":"call_type = :call_type","expressionValues":{":call_type":{"S":"TASK"}}},"scanIndexForward":false,"limit":${ctx.args.limit}}' ), response_mapping_template=_appsync.MappingTemplate. dynamo_db_result_list()) callout_record_ddb_ds.create_resolver( type_name="Query", field_name="getLatestCallReportRecords", request_mapping_template=_appsync.MappingTemplate.from_string( '{"version":"2017-02-28","operation":"Query","index":"CallTypeCreatedAtGlobalIndex","query":{"expression":"call_type = :call_type","expressionValues":{":call_type":{"S":"REPORT"}}},"scanIndexForward":false,"limit":${ctx.args.limit}}' ), response_mapping_template=_appsync.MappingTemplate. dynamo_db_result_list()) callout_record_ddb_ds.create_resolver( type_name="Mutation", field_name="createCallReport", request_mapping_template=_appsync.MappingTemplate.from_string( '{"version":"2017-02-28","operation":"PutItem","key":{"task_id":{"S":"${ctx.args.report.task_id}"},"created_at":{"N":"${ctx.args.report.created_at}"}},"attributeValues":$util.dynamodb.toMapValuesJson($ctx.args.report)}' ), response_mapping_template=_appsync.MappingTemplate. dynamo_db_result_item()) call_task_lambda_ds = appsync_api.add_lambda_data_source( name="CallTaskLambda", description="Call Task Lambda Data Source", lambda_function=create_appsync_call_task_function) call_task_lambda_ds.create_resolver( type_name="Mutation", field_name="createCallTask", request_mapping_template=_appsync.MappingTemplate.lambda_request( "$utils.toJson($ctx.args)"), response_mapping_template=_appsync.MappingTemplate.lambda_result()) create_call_report_record_function.add_environment( value=appsync_api.graph_ql_url, key="AppSyncGraphQlApiUrl") create_call_report_record_function.add_to_role_policy( statement=_iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=['appsync:GraphQL'], resources=[ f"{appsync_api.arn}/types/Mutation/fields/createCallReport" ])) core.CfnOutput(self, id="OutputCallSqsQueue", value=call_sqs_queue.queue_arn) core.CfnOutput(self, id="OutputCallJobCompletionSNSTopic", value=call_job_complete_sns_topic.topic_arn) core.CfnOutput(self, id="OutputExcelFileS3Bucket", value=file_bucket.bucket_name) core.CfnOutput(self, id="OutputStaticWebS3Bucket", value=web_bucket.bucket_name) core.CfnOutput(self, id="OutputStaticWebUrl", value=web_bucket.bucket_website_url) identity_pool = _cognito.CfnIdentityPool( self, "IdentityPool", allow_unauthenticated_identities=True, cognito_identity_providers=[ _cognito.CfnIdentityPool.CognitoIdentityProviderProperty( provider_name=user_pool.user_pool_provider_name, client_id=user_pool_client.user_pool_client_id) ]) identity_pool_unauthorized_role = _iam.Role( self, 'IdentityPoolUnAuthorizedRole', assumed_by=_iam.FederatedPrincipal( federated="cognito-identity.amazonaws.com", assume_role_action="sts:AssumeRoleWithWebIdentity", conditions={ "StringEquals": { "cognito-identity.amazonaws.com:aud": identity_pool.ref }, "ForAnyValue:StringLike": { "cognito-identity.amazonaws.com:amr": "unauthenticated" } })) identity_pool_unauthorized_role.add_to_policy( _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=["appsync:GraphQL"], resources=[ f"{appsync_api.arn}/types/*", # f"{appsync_api.arn}/types/Query/fields/getLatestCallTaskRecords", # f"{appsync_api.arn}/types/Query/fields/getLatestCallReportRecords", # f"{appsync_api.arn}/types/Mutation/fields/createCallRecord", # f"{appsync_api.arn}/types/Subscription/fields/createCallTask", # f"{appsync_api.arn}/types/Subscription/fields/createCallReport" ])) _cognito.CfnIdentityPoolRoleAttachment( self, "CognitoIdentityPoolRoleAttachment", identity_pool_id=identity_pool.ref, roles={ "unauthenticated": identity_pool_unauthorized_role.role_arn }) core.CfnOutput(self, id="UserPoolId", value=user_pool.user_pool_id) core.CfnOutput(self, id="UserPoolClientId", value=user_pool_client.user_pool_client_id) core.CfnOutput(self, id="IdentityPoolId", value=identity_pool.ref)
def __init__(self, scope: core.Stack, id: str, **kwargs): super().__init__(scope, id, **kwargs) self.output_bucket = aws_s3.Bucket( self, "BucketTwitterStreamOutput", bucket_name = self.stack_name, ) self.bucket_url = self.output_bucket.bucket_regional_domain_name # Because kinesis firehose bindings are to direct CF, we have to create IAM policy/role and attach on our own self.iam_role = aws_iam.Role( self, "IAMRoleTwitterStreamKinesisFHToS3", role_name="KinesisFirehoseToS3-{}".format(self.stack_name), assumed_by=aws_iam.ServicePrincipal(service='firehose.amazonaws.com'), ) # S3 bucket actions self.s3_iam_policy_statement = aws_iam.PolicyStatement() actions = ["s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject"] for action in actions: self.s3_iam_policy_statement.add_actions(action) self.s3_iam_policy_statement.add_resources(self.output_bucket.bucket_arn) self.s3_iam_policy_statement.add_resources(self.output_bucket.bucket_arn + "/*") # CW error log setup self.s3_error_logs_group = aws_logs.LogGroup( self, "S3ErrorLogsGroup", log_group_name="{}-s3-errors".format(self.stack_name) ) self.s3_error_logs_stream = aws_logs.LogStream( self, "S3ErrorLogsStream", log_group=self.s3_error_logs_group, log_stream_name='s3Backup' ) self.firehose = aws_kinesisfirehose.CfnDeliveryStream( self, "FirehoseTwitterStream", delivery_stream_name = "{}-raw".format(self.stack_name), delivery_stream_type = "DirectPut", s3_destination_configuration={ 'bucketArn': self.output_bucket.bucket_arn, 'bufferingHints': { 'intervalInSeconds': 120, 'sizeInMBs': 10 }, 'compressionFormat': 'UNCOMPRESSED', 'roleArn': self.iam_role.role_arn, 'cloudWatchLoggingOptions': { 'enabled': True, 'logGroupName': "{}-raw".format(self.stack_name), 'logStreamName': 's3BackupRaw' }, 'prefix': 'twitter-raw/' }, ) # TODO: Only attach what's needed for this policy, right now i'm lazy and attaching all policies self.iam_policy = aws_iam.Policy( self, "IAMPolicyTwitterStreamKinesisFHToS3", policy_name="KinesisFirehoseToS3-{}".format(self.stack_name), statements=[self.s3_iam_policy_statement], ) self.iam_policy.attach_to_role(self.iam_role) # Because kinesis firehose bindings are to direct CF, we have to create IAM policy/role and attach on our own self.curator_firehose = aws_kinesisfirehose.CfnDeliveryStream( self, "CuratorFirehoseStream", delivery_stream_name = "{}-curator".format(self.stack_name), delivery_stream_type = "DirectPut", s3_destination_configuration={ 'bucketArn': self.output_bucket.bucket_arn, 'bufferingHints': { 'intervalInSeconds': 120, 'sizeInMBs': 10 }, 'compressionFormat': 'UNCOMPRESSED', 'roleArn': self.iam_role.role_arn, 'cloudWatchLoggingOptions': { 'enabled': True, 'logGroupName': "{}-curator".format(self.stack_name), 'logStreamName': 's3BackupCurator' }, 'prefix': 'twitter-curated/' }, ) def zip_package(): cwd = os.getcwd() file_name = 'curator-lambda.zip' zip_file = cwd + '/' + file_name os.chdir('src/') sh.zip('-r9', zip_file, '.') os.chdir(cwd) return file_name, zip_file _, zip_file = zip_package() self.twitter_stream_curator_lambda_function = aws_lambda.Function( self, "TwitterStreamCuratorLambdaFunction", function_name="{}-curator".format(self.stack_name), code=aws_lambda.AssetCode(zip_file), handler="sentiment_analysis.lambda_handler", runtime=aws_lambda.Runtime.PYTHON_3_7, tracing=aws_lambda.Tracing.ACTIVE, description="Triggers from S3 PUT event for twitter stream data and transorms it to clean json syntax with sentiment analysis attached", environment={ "STACK_NAME": self.stack_name, "FIREHOSE_STREAM": self.curator_firehose.delivery_stream_name }, memory_size=128, timeout=core.Duration.seconds(120), log_retention=aws_logs.RetentionDays.ONE_WEEK, ) # Permission to talk to comprehend for sentiment analysis self.comprehend_iam_policy_statement = aws_iam.PolicyStatement() self.comprehend_iam_policy_statement.add_actions('comprehend:*') self.comprehend_iam_policy_statement.add_all_resources() self.twitter_stream_curator_lambda_function.add_to_role_policy(self.comprehend_iam_policy_statement) # Permission to put in kinesis firehose self.curator_firehose_iam_policy_statement = aws_iam.PolicyStatement() self.curator_firehose_iam_policy_statement.add_actions('firehose:Put*') self.curator_firehose_iam_policy_statement.add_resources(self.curator_firehose.attr_arn) self.twitter_stream_curator_lambda_function.add_to_role_policy(self.curator_firehose_iam_policy_statement) # Attaching the policy to the IAM role for KFH self.output_bucket.grant_read(self.twitter_stream_curator_lambda_function) self.twitter_stream_curator_lambda_function.add_event_source( aws_lambda_event_sources.S3EventSource( bucket=self.output_bucket, events=[ aws_s3.EventType.OBJECT_CREATED ], filters=[ aws_s3.NotificationKeyFilter( prefix="twitter-raw/" ) ] ) )
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # Change pillow arn to latest version in your region. See https://github.com/keithrozario/Klayers/tree/master/deployments/python3.8/arns layerpillow = lambda_.LayerVersion.from_layer_version_arn( self, 'pillowlayerversion', 'Your_Region_Pillow_ARN') # -------------------------------------------------------- # Create buckets for thumbnail # -------------------------------------------------------- bucket = s3.Bucket(self, id='SourceBucket', block_public_access=s3.BlockPublicAccess.BLOCK_ALL, bucket_name='<your-name>-thumbnail-upload-372') ResizedBucket = s3.Bucket( self, id='ResizedBucket', block_public_access=s3.BlockPublicAccess.BLOCK_ALL, bucket_name='<your-name>-thumbnail-upload-372-resized') # ------------------------------------------------------------------------------------- # Create IAM role for lambda invocation, write to X-Ray and Get and Put objects into s3 # ------------------------------------------------------------------------------------- Role = iam.Role( self, "Lambda-role", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonS3ReadOnlyAccess"), iam.ManagedPolicy.from_aws_managed_policy_name( "AWSXrayWriteOnlyAccess") ]) Role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[bucket.bucket_arn + "/*"], actions=["s3:GetObject"])) Role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[ResizedBucket.bucket_arn + "/*"], actions=["s3:PutObject"])) # ------------------------------------------ # Create a lambda layer for botos3 and X-Ray # ------------------------------------------ layerxray = lambda_.LayerVersion( self, 'xray_layer', code=lambda_.Code.from_asset( os.path.join(os.getcwd(), "lambda_layer_x_ray_stack/layers/xray")), description="Lambda Layer containing Xray SDK Python Library", compatible_runtimes=[ lambda_.Runtime.PYTHON_3_7, lambda_.Runtime.PYTHON_3_8, ], removal_policy=RemovalPolicy.DESTROY, ) # ------------------------------------------- # Create Lambda Function # ------------------------------------------- xray_trace_lambda = lambda_.Function( self, id='xray-sample-app', function_name='xray-handler', code=lambda_.Code.from_asset( os.path.join(os.getcwd(), "lambda_code")), runtime=lambda_.Runtime.PYTHON_3_8, handler="lambda-handler.lambda_handler", layers=[layerxray, layerpillow], role=Role, tracing=lambda_.Tracing.ACTIVE) xray_trace_lambda.add_event_source( eventsources.S3EventSource( bucket, events=[s3.EventType.OBJECT_CREATED_PUT], ))
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: """ initialize function for CDK """ super().__init__(scope, construct_id, **kwargs) # ------------------------------- # S3 Bucket for Manifests # ------------------------------- qs_gov_bucket = s3.Bucket( self, id=f"{cf.PROJECT}-Bucket", ) bucket_name = qs_gov_bucket.bucket_name # ------------------------------- # IAM # ------------------------------- list_roles_policy = iam.ManagedPolicy( self, id=f"{cf.PROJECT}-ListRolesPolicy", managed_policy_name=f"{cf.PROJECT}-ListRolesPolicy", description=None, path="/", statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=["iam:ListRoles", "iam:ListAccountAliases"], ) ], ) federated_quicksight_policy = iam.ManagedPolicy( self, id=f"{cf.PROJECT}-FederatedQuickSightPolicy", managed_policy_name=f"{cf.PROJECT}-FederatedQuickSightPolicy", statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:iam::{cf.ACCOUNT}:saml-provider/{cf.OKTA_IDP_NAME}" ], actions=["sts:AssumeRoleWithSAML"], conditions={ "StringEquals": { "saml:aud": "https://signin.aws.amazon.com/saml" } }, ) ], ) okta_federated_principal = iam.FederatedPrincipal( federated= f"arn:aws:iam::{cf.ACCOUNT}:saml-provider/{cf.OKTA_IDP_NAME}", assume_role_action="sts:AssumeRoleWithSAML", conditions={ "StringEquals": { "SAML:aud": "https://signin.aws.amazon.com/saml" } }, ) federated_quicksight_role = iam.Role( self, id=f"{cf.PROJECT}-{cf.QS_FEDERATED_ROLE_NAME}", role_name=f"{cf.QS_FEDERATED_ROLE_NAME}", assumed_by=okta_federated_principal, description= "Allow Okta to Federate Login & User Creation to QuickSight", managed_policies=[federated_quicksight_policy], ) iam.User( self, id=f"{cf.PROJECT}-OktaSSOUser", user_name=f"{cf.PROJECT}-OktaSSOUser", managed_policies=[list_roles_policy], ) # ------------------------------- # Lambda Layers # ------------------------------- path_src = os.path.join(cf.PATH_SRC, "") sp.call(["make", "bundle"], cwd=path_src) requests_layer = _lambda.LayerVersion( self, f"{cf.PROJECT}-requests-layer", code=_lambda.Code.from_asset(os.path.join(path_src, "requests.zip"))) sp.call(["make", "clean"], cwd=path_src) # ------------------------------- # Lambda Functions # ------------------------------- # iam role for Lambdas qs_governance_policy = iam.ManagedPolicy( self, id=f"{cf.PROJECT}-LambdaPolicy", managed_policy_name=f"{cf.PROJECT}-LambdaPolicy", statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=["ses:SendEmail", "ses:SendRawEmail"], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:secretsmanager:{cf.REGION}:{cf.ACCOUNT}:secret:{cf.OKTA_SECRET_ID}*", f"arn:aws:secretsmanager:{cf.REGION}:{cf.ACCOUNT}:secret:{cf.SLACK_SECRET_ID}*" ], actions=[ "secretsmanager:GetSecretValue", ], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:iam::{cf.ACCOUNT}:policy/{cf.QS_PREFIX}*" ], actions=[ "iam:CreatePolicy", "iam:GetPolicy", "iam:DeletePolicy", ], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=["iam:ListPolicies"], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=[ "glue:GetDatabase", "glue:GetDatabases", "glue:GetTable", "glue:GetPartitions", "glue:GetPartition", "glue:GetTables", ], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=["quicksight:*", "ds:*"], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:s3:::{bucket_name}/*", f"arn:aws:s3:::{bucket_name}*" ], actions=["s3:*"], ), ], ) qs_governance_role = iam.Role( self, id=f"{cf.PROJECT}-QuickSightPermissionMappingRole", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaBasicExecutionRole"), qs_governance_policy, ], ) # Lambdas get_okta_information_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-GetOktaInfo", function_name=f"{cf.PROJECT}-GetOktaInformation", handler="get_okta_information.handler", role=qs_governance_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), environment={ "OKTA_SECRET_ID": cf.OKTA_SECRET_ID, "QS_PREFIX": cf.QS_PREFIX, "QS_GOVERNANCE_BUCKET": bucket_name, "QS_FEDERATED_ROLE_NAME": f"{cf.QS_FEDERATED_ROLE_NAME}", "S3_PREFIX_USERS": cf.S3_PREFIX_USERS, "SLACK_SECRET_ID": cf.SLACK_SECRET_ID }, memory_size=256, timeout=core.Duration.seconds(180), layers=[requests_layer], ) # Lamda Okta to QuickSight Mappers qs_user_governance_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-QSUserGovernance", function_name=f"{cf.PROJECT}-QSUserGovernance", handler="qs_user_governance.handler", role=qs_governance_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), environment={ "QS_FEDERATED_ROLE_NAME": f"{cf.QS_FEDERATED_ROLE_NAME}", "QS_PREFIX": cf.QS_PREFIX, "QS_ADMIN_OKTA_GROUP": cf.QS_ADMIN_OKTA_GROUP, "QS_AUTHOR_OKTA_GROUP": cf.QS_AUTHOR_OKTA_GROUP, "QS_READER_OKTA_GROUP": cf.QS_READER_OKTA_GROUP, "SLACK_SECRET_ID": cf.SLACK_SECRET_ID }, memory_size=256, timeout=core.Duration.seconds(180), layers=[requests_layer], ) qs_asset_governance_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-QSAssetGovernance", function_name=f"{cf.PROJECT}-QSAssetGovernance", handler="qs_asset_governance.handler", role=qs_governance_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), memory_size=256, timeout=core.Duration.seconds(180), layers=[requests_layer], environment={ "QS_GOVERNANCE_BUCKET": f"{bucket_name}", "S3_PREFIX_USERS": cf.S3_PREFIX_USERS, "S3_PREFIX_POLICIES": cf.S3_PREFIX_POLICIES, "STAGE": cf.DEPLOYMENT_STAGE, "SLACK_SECRET_ID": cf.SLACK_SECRET_ID, "VERIFIED_EMAIL": cf.VERIFIED_EMAIL }) qs_policy_governance_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-QSPolicyGovernance", function_name=f"{cf.PROJECT}-QSPolicyGovernance", handler="qs_policy_governance.handler", role=qs_governance_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), environment={ "QS_ADMIN_OKTA_GROUP": cf.QS_ADMIN_OKTA_GROUP, "QS_AUTHOR_OKTA_GROUP": cf.QS_AUTHOR_OKTA_GROUP, "QS_AUTHOR_BASE_POLICY": cf.QS_AUTHOR_BASE_POLICY, "QS_PREFIX": cf.QS_PREFIX, "QS_GOVERNANCE_BUCKET": f"{bucket_name}", "S3_PREFIX_POLICIES": cf.S3_PREFIX_POLICIES, "QS_SUPERUSER": cf.QS_SUPERUSER, "DEPLOYMENT_STAGE": cf.DEPLOYMENT_STAGE, "SLACK_SECRET_ID": cf.SLACK_SECRET_ID }, memory_size=256, timeout=core.Duration.seconds(180), layers=[requests_layer]) qs_resource_cleanup_lambda = _lambda.Function( self, id=f"{cf.PROJECT}-QSResourceCleanup", function_name=f"{cf.PROJECT}-QSResourceCleanup", handler="qs_resource_cleanup.handler", role=qs_governance_role, runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset(os.path.join(cf.PATH_SRC, "pkg")), memory_size=256, timeout=core.Duration.seconds(180), layers=[requests_layer], environment={ "QS_PREFIX": cf.QS_PREFIX, "QS_SUPERUSER": cf.QS_SUPERUSER, "QS_GOVERNANCE_BUCKET": f"{bucket_name}", "S3_PREFIX_ASSETS": cf.S3_PREFIX_ASSETS, "S3_PREFIX_USERS": cf.S3_PREFIX_USERS, "S3_PREFIX_POLICIES": cf.S3_PREFIX_POLICIES, "SLACK_SECRET_ID": cf.SLACK_SECRET_ID }) # ------------------------------- # S3 Event Triggers # ------------------------------- qs_user_governance_lambda.add_event_source( lambda_event_sources.S3EventSource( bucket=qs_gov_bucket, events=[s3.EventType.OBJECT_CREATED], filters=[s3.NotificationKeyFilter(prefix=cf.S3_PREFIX_USERS)], )) qs_policy_governance_lambda.add_event_source( lambda_event_sources.S3EventSource( bucket=qs_gov_bucket, events=[s3.EventType.OBJECT_CREATED], filters=[ s3.NotificationKeyFilter(prefix=cf.S3_PREFIX_POLICIES) ], )) # ------------------------------- # CloudWatch Event Rules (30 minutes) # ------------------------------- lambda_schedule = events.Schedule.rate(core.Duration.minutes(30)) get_okta_info_target = events_targets.LambdaFunction( handler=get_okta_information_lambda) events.Rule( self, id=f"{cf.PROJECT}-GetOktaInfoScheduledEvent", description="The scheduled CloudWatch event trigger for the Lambda", enabled=True, schedule=lambda_schedule, targets=[get_okta_info_target], ) qs_asset_gov_target = events_targets.LambdaFunction( handler=qs_asset_governance_lambda) events.Rule( self, id=f"{cf.PROJECT}-QSAssetGovScheduledEvent", description="The scheduled CloudWatch event trigger for the Lambda", enabled=True, schedule=lambda_schedule, targets=[qs_asset_gov_target], ) qs_cleanup_target = events_targets.LambdaFunction( handler=qs_resource_cleanup_lambda) events.Rule( self, id=f"{cf.PROJECT}-QSCleanupScheduledEvent", description="The scheduled CloudWatch event trigger for the Lambda", enabled=True, schedule=lambda_schedule, targets=[qs_cleanup_target], ) # ------------------------------- # Athena WorkGroup # ------------------------------- workgroup_encryption = athena.CfnWorkGroup.EncryptionConfigurationProperty( encryption_option='SSE_S3') workgroup_output = athena.CfnWorkGroup.ResultConfigurationProperty( output_location="s3://" + cf.ATHENA_OUTPUT_LOC, encryption_configuration=workgroup_encryption) quicksight_athena_workgroup = athena.CfnWorkGroup( self, id=f"{cf.PROJECT}-workgroup", name=f"{cf.QS_PREFIX}-workgroup", description="workgroup for QuickSight Data Source operations", recursive_delete_option=True, work_group_configuration=athena.CfnWorkGroup. WorkGroupConfigurationProperty( result_configuration=workgroup_output)) workgroup_name = quicksight_athena_workgroup.name # ------------------------------- # S3 Object Deployments # ------------------------------- policy_manifest_deploy = s3_deploy.BucketDeployment( self, id=f"{cf.PROJECT}-PolicyManifestDeploy", sources=[ s3_deploy.Source.asset( os.path.join(cf.PATH_ROOT, "qs_config/policies")) ], destination_bucket=qs_gov_bucket, destination_key_prefix=cf.S3_PREFIX_POLICIES) # ------------------------------- # QuickSight IAM Policy Assignments # ------------------------------- qs_author_base_policy = iam.ManagedPolicy( self, id=f"{cf.PROJECT}-{cf.QS_AUTHOR_BASE_POLICY}", managed_policy_name=f"{cf.QS_AUTHOR_BASE_POLICY}", statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:athena:{cf.REGION}:{cf.ACCOUNT}:workgroup/{workgroup_name}" ], actions=[ "athena:GetNamedQuery", "athena:CancelQueryExecution", "athena:CreateNamedQuery", "athena:DeleteNamedQuery", "athena:StartQueryExecution", "athena:StopQueryExecution", "athena:GetWorkGroup", "athena:GetNamedQuery", "athena:GetQueryResults", "athena:GetQueryExecution", "athena:BatchGetQueryExecution", "athena:BatchGetNamedQuery", "athena:ListNamedQueries", "athena:ListQueryExecutions", "athena:GetQueryResultsStream" ]), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=[ "athena:ListWorkGroups", "athena:ListDataCatalogs", "athena:StartQueryExecution", "athena:GetQueryExecution", "athena:GetQueryResultsStream", "athena:ListTableMetadata", "athena:GetTableMetadata" ]), iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[ "arn:aws:s3:::ast-datalake-*", "arn:aws:s3:::aws-athena-query-*", f"arn:aws:s3:::{bucket_name}*", f"arn:aws:s3:::{cf.ATHENA_OUTPUT_LOC}*" ], actions=[ "s3:List*", "s3:Get*", ]), iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=["*"], actions=["s3:ListAllMyBuckets"]), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ "arn:aws:s3:::aws-athena-query-*", f"arn:aws:s3:::{bucket_name}/{cf.QS_PREFIX}/athena-results*" ], actions=["s3:Put*"]), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ f"arn:aws:athena:{cf.REGION}:{cf.ACCOUNT}:datacatalog/AwsDataCatalog" ], actions=["athena:ListDatabases"]), iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=["kms:Decrypt", "kms:GenerateDataKey"]), ], )