def api_lambda_function(scope, name, handler, apigw, path, method, layer, tables, code="./backend"): _lambda = Function( scope, name, handler=handler, runtime=Runtime.PYTHON_3_8, code=Code.asset(code), tracing=Tracing.ACTIVE, layers=layer, ) _lambda.add_environment("POLL_TABLE", tables[0].table_name) _lambda.add_environment("MAIN_PAGE_GSI", "main_page_gsi") apigw.add_routes( path=path, methods=[method], integration=LambdaProxyIntegration(handler=_lambda), ) return _lambda
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) with open("stack/config.yml", 'r') as stream: configs = yaml.safe_load(stream) ### S3 core images_S3_bucket = _s3.Bucket(self, "ICS_IMAGES") images_S3_bucket.add_cors_rule( allowed_methods=[_s3.HttpMethods.POST], allowed_origins=["*"] # add API gateway web resource URL ) ### SQS core image_deadletter_queue = _sqs.Queue(self, "ICS_IMAGES_DEADLETTER_QUEUE") image_queue = _sqs.Queue(self, "ICS_IMAGES_QUEUE", dead_letter_queue={ "max_receive_count": configs["DeadLetterQueue"]["MaxReceiveCount"], "queue": image_deadletter_queue }) ### api gateway core api_gateway = RestApi(self, 'ICS_API_GATEWAY', rest_api_name='ImageContentSearchApiGateway') api_gateway_resource = api_gateway.root.add_resource(configs["ProjectName"]) api_gateway_landing_page_resource = api_gateway_resource.add_resource('web') api_gateway_get_signedurl_resource = api_gateway_resource.add_resource('signedUrl') api_gateway_image_search_resource = api_gateway_resource.add_resource('search') ### landing page function get_landing_page_function = Function(self, "ICS_GET_LANDING_PAGE", function_name="ICS_GET_LANDING_PAGE", runtime=Runtime.PYTHON_3_7, handler="main.handler", code=Code.asset("./src/landingPage")) get_landing_page_integration = LambdaIntegration( get_landing_page_function, proxy=True, integration_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': "'*'", } }]) api_gateway_landing_page_resource.add_method('GET', get_landing_page_integration, method_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': True, } }]) ### cognito required_attribute = _cognito.StandardAttribute(required=True) users_pool = _cognito.UserPool(self, "ICS_USERS_POOL", auto_verify=_cognito.AutoVerifiedAttrs(email=True), #required for self sign-up standard_attributes=_cognito.StandardAttributes(email=required_attribute), #required for self sign-up self_sign_up_enabled=configs["Cognito"]["SelfSignUp"]) user_pool_app_client = _cognito.CfnUserPoolClient(self, "ICS_USERS_POOL_APP_CLIENT", supported_identity_providers=["COGNITO"], allowed_o_auth_flows=["implicit"], allowed_o_auth_scopes=configs["Cognito"]["AllowedOAuthScopes"], user_pool_id=users_pool.user_pool_id, callback_ur_ls=[api_gateway_landing_page_resource.url], allowed_o_auth_flows_user_pool_client=True, explicit_auth_flows=["ALLOW_REFRESH_TOKEN_AUTH"]) user_pool_domain = _cognito.UserPoolDomain(self, "ICS_USERS_POOL_DOMAIN", user_pool=users_pool, cognito_domain=_cognito.CognitoDomainOptions(domain_prefix=configs["Cognito"]["DomainPrefix"])) ### get signed URL function get_signedurl_function = Function(self, "ICS_GET_SIGNED_URL", function_name="ICS_GET_SIGNED_URL", environment={ "ICS_IMAGES_BUCKET": images_S3_bucket.bucket_name, "DEFAULT_SIGNEDURL_EXPIRY_SECONDS": configs["Functions"]["DefaultSignedUrlExpirySeconds"] }, runtime=Runtime.PYTHON_3_7, handler="main.handler", code=Code.asset("./src/getSignedUrl")) get_signedurl_integration = LambdaIntegration( get_signedurl_function, proxy=True, integration_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': "'*'", } }]) api_gateway_get_signedurl_authorizer = CfnAuthorizer(self, "ICS_API_GATEWAY_GET_SIGNED_URL_AUTHORIZER", rest_api_id=api_gateway_get_signedurl_resource.rest_api.rest_api_id, name="ICS_API_GATEWAY_GET_SIGNED_URL_AUTHORIZER", type="COGNITO_USER_POOLS", identity_source="method.request.header.Authorization", provider_arns=[users_pool.user_pool_arn]) api_gateway_get_signedurl_resource.add_method('GET', get_signedurl_integration, authorization_type=AuthorizationType.COGNITO, method_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': True, } }] ).node.find_child('Resource').add_property_override('AuthorizerId', api_gateway_get_signedurl_authorizer.ref) images_S3_bucket.grant_put(get_signedurl_function, objects_key_pattern="new/*") ### image massage function image_massage_function = Function(self, "ICS_IMAGE_MASSAGE", function_name="ICS_IMAGE_MASSAGE", timeout=core.Duration.seconds(6), runtime=Runtime.PYTHON_3_7, environment={"ICS_IMAGE_MASSAGE": image_queue.queue_name}, handler="main.handler", code=Code.asset("./src/imageMassage")) images_S3_bucket.grant_write(image_massage_function, "processed/*") images_S3_bucket.grant_delete(image_massage_function, "new/*") images_S3_bucket.grant_read(image_massage_function, "new/*") new_image_added_notification = _s3notification.LambdaDestination(image_massage_function) images_S3_bucket.add_event_notification(_s3.EventType.OBJECT_CREATED, new_image_added_notification, _s3.NotificationKeyFilter(prefix="new/") ) image_queue.grant_send_messages(image_massage_function) ### image analyzer function image_analyzer_function = Function(self, "ICS_IMAGE_ANALYSIS", function_name="ICS_IMAGE_ANALYSIS", runtime=Runtime.PYTHON_3_7, timeout=core.Duration.seconds(10), environment={ "ICS_IMAGES_BUCKET": images_S3_bucket.bucket_name, "DEFAULT_MAX_CALL_ATTEMPTS": configs["Functions"]["DefaultMaxApiCallAttempts"], "REGION": core.Aws.REGION, }, handler="main.handler", code=Code.asset("./src/imageAnalysis")) image_analyzer_function.add_event_source(_lambda_event_source.SqsEventSource(queue=image_queue, batch_size=10)) image_queue.grant_consume_messages(image_massage_function) lambda_rekognition_access = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=["rekognition:DetectLabels", "rekognition:DetectModerationLabels"], resources=["*"] ) image_analyzer_function.add_to_role_policy(lambda_rekognition_access) images_S3_bucket.grant_read(image_analyzer_function, "processed/*") ### API gateway finalizing self.add_cors_options(api_gateway_get_signedurl_resource) self.add_cors_options(api_gateway_landing_page_resource) self.add_cors_options(api_gateway_image_search_resource) ### database database_secret = _secrets_manager.Secret(self, "ICS_DATABASE_SECRET", secret_name="rds-db-credentials/image-content-search-rds-secret", generate_secret_string=_secrets_manager.SecretStringGenerator( generate_string_key='password', secret_string_template='{"username": "******"}', exclude_punctuation=True, exclude_characters='/@\" \\\'', require_each_included_type=True ) ) database = _rds.CfnDBCluster(self, "ICS_DATABASE", engine=_rds.DatabaseClusterEngine.aurora_mysql(version=_rds.AuroraMysqlEngineVersion.VER_5_7_12).engine_type, engine_mode="serverless", database_name=configs["Database"]["Name"], enable_http_endpoint=True, deletion_protection=configs["Database"]["DeletionProtection"], master_username=database_secret.secret_value_from_json("username").to_string(), master_user_password=database_secret.secret_value_from_json("password").to_string(), scaling_configuration=_rds.CfnDBCluster.ScalingConfigurationProperty( auto_pause=configs["Database"]["Scaling"]["AutoPause"], min_capacity=configs["Database"]["Scaling"]["Min"], max_capacity=configs["Database"]["Scaling"]["Max"], seconds_until_auto_pause=configs["Database"]["Scaling"]["SecondsToAutoPause"] ), ) database_cluster_arn = "arn:aws:rds:{}:{}:cluster:{}".format(core.Aws.REGION, core.Aws.ACCOUNT_ID, database.ref) secret_target = _secrets_manager.CfnSecretTargetAttachment(self,"ICS_DATABASE_SECRET_TARGET", target_type="AWS::RDS::DBCluster", target_id=database.ref, secret_id=database_secret.secret_arn ) secret_target.node.add_dependency(database) ### database function image_data_function_role = _iam.Role(self, "ICS_IMAGE_DATA_FUNCTION_ROLE", role_name="ICS_IMAGE_DATA_FUNCTION_ROLE", assumed_by=_iam.ServicePrincipal("lambda.amazonaws.com"), managed_policies=[ _iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaVPCAccessExecutionRole"), _iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole"), _iam.ManagedPolicy.from_aws_managed_policy_name("AmazonRDSDataFullAccess") ] ) image_data_function = Function(self, "ICS_IMAGE_DATA", function_name="ICS_IMAGE_DATA", runtime=Runtime.PYTHON_3_7, timeout=core.Duration.seconds(5), role=image_data_function_role, environment={ "DEFAULT_MAX_CALL_ATTEMPTS": configs["Functions"]["DefaultMaxApiCallAttempts"], "CLUSTER_ARN": database_cluster_arn, "CREDENTIALS_ARN": database_secret.secret_arn, "DB_NAME": database.database_name, "REGION": core.Aws.REGION }, handler="main.handler", code=Code.asset("./src/imageData") ) image_search_integration = LambdaIntegration( image_data_function, proxy=True, integration_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': "'*'", } }]) api_gateway_image_search_authorizer = CfnAuthorizer(self, "ICS_API_GATEWAY_IMAGE_SEARCH_AUTHORIZER", rest_api_id=api_gateway_image_search_resource.rest_api.rest_api_id, name="ICS_API_GATEWAY_IMAGE_SEARCH_AUTHORIZER", type="COGNITO_USER_POOLS", identity_source="method.request.header.Authorization", provider_arns=[users_pool.user_pool_arn]) api_gateway_image_search_resource.add_method('POST', image_search_integration, authorization_type=AuthorizationType.COGNITO, method_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': True, } }] ).node.find_child('Resource').add_property_override('AuthorizerId', api_gateway_image_search_authorizer.ref) lambda_access_search = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=["translate:TranslateText"], resources=["*"] ) image_data_function.add_to_role_policy(lambda_access_search) ### custom resource lambda_provider = Provider(self, 'ICS_IMAGE_DATA_PROVIDER', on_event_handler=image_data_function ) core.CustomResource(self, 'ICS_IMAGE_DATA_RESOURCE', service_token=lambda_provider.service_token, pascal_case_properties=False, resource_type="Custom::SchemaCreation", properties={ "source": "Cloudformation" } ) ### event bridge event_bus = _events.EventBus(self, "ICS_IMAGE_CONTENT_BUS") event_rule = _events.Rule(self, "ICS_IMAGE_CONTENT_RULE", rule_name="ICS_IMAGE_CONTENT_RULE", description="The event from image analyzer to store the data", event_bus=event_bus, event_pattern=_events.EventPattern(resources=[image_analyzer_function.function_arn]), ) event_rule.add_target(_event_targets.LambdaFunction(image_data_function)) event_bus.grant_put_events(image_analyzer_function) image_analyzer_function.add_environment("EVENT_BUS", event_bus.event_bus_name) ### outputs core.CfnOutput(self, 'CognitoHostedUILogin', value='https://{}.auth.{}.amazoncognito.com/login?client_id={}&response_type=token&scope={}&redirect_uri={}'.format(user_pool_domain.domain_name, core.Aws.REGION, user_pool_app_client.ref, '+'.join(user_pool_app_client.allowed_o_auth_scopes), api_gateway_landing_page_resource.url), description='The Cognito Hosted UI Login Page' )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # SQS queue state_change_sqs = Queue( self, "state_change_sqs", visibility_timeout=core.Duration.seconds(60) ) # Dynamodb Tables # EC2 state changes tb_states = Table( self, "ec2_states", partition_key=Attribute(name="instance-id", type=AttributeType.STRING), sort_key=Attribute( name="time", type=AttributeType.STRING ), billing_mode=BillingMode.PAY_PER_REQUEST, removal_policy=core.RemovalPolicy.DESTROY, stream=StreamViewType.NEW_IMAGE) # EC2 inventory tb_inventory = Table( self, "ec2_inventory", partition_key=Attribute(name="instance-id", type=AttributeType.STRING), sort_key=Attribute( name="time", type=AttributeType.STRING ), billing_mode=BillingMode.PAY_PER_REQUEST, removal_policy=core.RemovalPolicy.DESTROY, stream=StreamViewType.KEYS_ONLY) # IAM policies - AWS managed basic_exec = ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole") sqs_access = ManagedPolicy(self, "LambdaSQSExecution", statements=[ PolicyStatement( effect=Effect.ALLOW, actions=[ "sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:GetQueueAttributes" ], resources=[state_change_sqs.queue_arn] )]) # IAM Policies pol_ec2_states_ro = ManagedPolicy(self, "pol_EC2StatesReadOnly", statements=[ PolicyStatement( effect=Effect.ALLOW, actions=[ "dynamodb:DescribeStream", "dynamodb:GetRecords", "dynamodb:GetItem", "dynamodb:GetShardIterator", "dynamodb:ListStreams" ], resources=[tb_states.table_arn] )]) pol_ec2_states_rwd = ManagedPolicy( self, "pol_EC2StatesWriteDelete", statements=[ PolicyStatement( effect=Effect.ALLOW, actions=[ "dynamodb:DeleteItem", "dynamodb:DescribeTable", "dynamodb:PutItem", "dynamodb:Query", "dynamodb:UpdateItem" ], resources=[tb_states.table_arn] )]) pol_ec2_inventory_full = ManagedPolicy( self, "pol_EC2InventoryFullAccess", statements=[ PolicyStatement( effect=Effect.ALLOW, actions=[ "dynamodb:DeleteItem", "dynamodb:DescribeTable", "dynamodb:GetItem", "dynamodb:PutItem", "dynamodb:Query", "dynamodb:UpdateItem" ], resources=[tb_inventory.table_arn] )]) pol_lambda_describe_ec2 = ManagedPolicy( self, "pol_LambdaDescribeEC2", statements=[ PolicyStatement( effect=Effect.ALLOW, actions=[ "ec2:Describe*" ], resources=["*"] )]) # IAM Roles rl_event_capture = Role( self, 'rl_state_capture', assumed_by=ServicePrincipal('lambda.amazonaws.com'), managed_policies=[basic_exec, sqs_access, pol_ec2_states_rwd] ) rl_event_processor = Role( self, 'rl_state_processor', assumed_by=ServicePrincipal('lambda.amazonaws.com'), managed_policies=[ basic_exec, pol_ec2_states_ro, pol_ec2_states_rwd, pol_ec2_inventory_full, pol_lambda_describe_ec2]) # event capture lambda lambda_event_capture = Function( self, "lambda_event_capture", handler="event_capture.handler", runtime=Runtime.PYTHON_3_7, code=Code.asset('event_capture'), role=rl_event_capture, events=[SqsEventSource(state_change_sqs)], environment={"state_table": tb_states.table_name} ) # event processor lambda lambda_event_processor = Function( self, "lambda_event_processor", handler="event_processor.handler", runtime=Runtime.PYTHON_3_7, code=Code.asset('event_processor'), role=rl_event_processor, events=[ DynamoEventSource( tb_states, starting_position=StartingPosition.LATEST) ], environment={ "inventory_table": tb_inventory.table_name, } ) # Cloudwatch Event event_ec2_change = Rule( self, "ec2_state_change", description="trigger on ec2 start, stop and terminate instances", event_pattern=EventPattern( source=["aws.ec2"], detail_type=["EC2 Instance State-change Notification"], detail={ "state": [ "running", "stopped", "terminated"] } ), targets=[aws_events_targets.SqsQueue(state_change_sqs)] ) # Outputs core.CfnOutput(self, "rl_state_capture_arn", value=rl_event_capture.role_arn) core.CfnOutput(self, "rl_state_processor_arn", value=rl_event_processor.role_arn) core.CfnOutput(self, "tb_states_arn", value=tb_states.table_arn) core.CfnOutput(self, "tb_inventory_arn", value=tb_inventory.table_arn) core.CfnOutput(self, "sqs_state_change", value=state_change_sqs.queue_arn)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) """ Create Lambda Layer The packages should be stored in `python/lib/python3.7/site-packages` which translates to `/opt/python/lib/python3.7/site-packages` in AWS Lambda Refer here: https://stackoverflow.com/a/58702328/7999204 """ python_deps_layer = LayerVersion( self, "PythonDepsLayer", code=Code.from_asset("./python-deps-layer"), compatible_runtimes=[PYTHON_RUNTIME], description="A layer that contains Python Dependencies", ) """ Create DynamoDB Tables """ poll_table = Table( self, "PollTable", partition_key=Attribute(name="id", type=AttributeType.STRING), sort_key=Attribute(name="SK", type=AttributeType.STRING), read_capacity=10, write_capacity=10, stream=StreamViewType.NEW_IMAGE, ) # DynamoDB Lambda consumer worker aggregate_votes_function = Function( self, "AggregateVotesLambda", handler="ddb_stream.aggregate_vote_table", runtime=PYTHON_RUNTIME, code=Code.asset("./backend"), layers=[python_deps_layer], timeout=core.Duration.seconds(30), ) aggregate_votes_function.add_environment("POLL_TABLE", poll_table.table_name) # DynamoDB Stream (Lambda Event Source) poll_table.grant_stream_read(aggregate_votes_function) poll_table.grant_read_write_data(aggregate_votes_function) ddb_aggregate_votes_event_source = DynamoEventSource( poll_table, starting_position=StartingPosition.LATEST ) aggregate_votes_function.add_event_source(ddb_aggregate_votes_event_source) # DynamoDB main_page GSI poll_table.add_global_secondary_index( partition_key=Attribute(name="PK2", type=AttributeType.STRING), projection_type=ProjectionType.INCLUDE, index_name=MAIN_PAGE_GSI, non_key_attributes=["date", "question", "result"], ) """ Create AWS Cognito User Pool """ self.users = UserPool(self, "vote-user") """ HTTP API API Gateway with CORS """ api = HttpApi( self, "VoteHttpApi", cors_preflight={ "allow_headers": ["*"], "allow_methods": [ HttpMethod.GET, HttpMethod.HEAD, HttpMethod.OPTIONS, HttpMethod.POST, ], "allow_origins": ["*"], "max_age": core.Duration.days(10), }, ) """ HTTP API Lambda functions """ get_all_votes_function = api_lambda_function( self, "GetAllVoteLambda", "api.get_all_votes", api, "/vote", GET, [python_deps_layer], [poll_table], ) poll_table.grant_read_data(get_all_votes_function) get_vote_function = api_lambda_function( self, "GetVoteLambda", "api.get_vote_by_id", api, "/vote/{vote_id}", GET, [python_deps_layer], [poll_table], ) poll_table.grant_read_data(get_vote_function) create_poll_function = api_lambda_function( self, "CreatePollLambda", "api.create_poll", api, "/vote", POST, [python_deps_layer], [poll_table], ) poll_table.grant_write_data(create_poll_function) post_vote_function = api_lambda_function( self, "PostVoteLambda", "api.vote", api, "/vote/{vote_id}", POST, [python_deps_layer], [poll_table], ) """ Create SQS Queues """ voting_queue = Queue(self, "voting-queue") # SQS Consumer worker voting_to_ddb_function = Function( self, "VotingToDDBLambda", handler="sqs_worker.insert_to_vote_db_table", runtime=PYTHON_RUNTIME, code=Code.asset("./backend"), layers=[python_deps_layer], ) voting_to_ddb_function.add_environment("POLL_TABLE", poll_table.table_name) # SQS Queue to Lambda trigger mapping voting_to_ddb_event_source = SqsEventSource(voting_queue) voting_to_ddb_function.add_event_source(voting_to_ddb_event_source) poll_table.grant_read_write_data(voting_to_ddb_function) voting_queue.grant_send_messages(post_vote_function) post_vote_function.add_environment("VOTING_QUEUE_URL", voting_queue.queue_url) core.CfnOutput(self, "api-domain", value=api.url)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) graphql_api = CfnGraphQLApi(self, 'WeatherApi', name='weather-api', authentication_type='API_KEY') CfnApiKey(self, 'WeatherApiKey', api_id=graphql_api.attr_api_id) api_schema = CfnGraphQLSchema(self, 'WeatherSchema', api_id=graphql_api.attr_api_id, definition=""" type Destination { id: ID! description: String! state: String! city: String! zip: String! conditions: Weather! } type Mutation { addDestination( id: ID, description: String!, state: String!, city: String!, zip: String! ): Destination! } type Query { getWeather(city: String!): Weather # Get a single value of type 'Post' by primary key. getDestination(id: ID!, zip: String): Destination getAllDestinations: [Destination] getDestinationsByState(state: String!): [Destination] } type Subscription { newDestination: Destination @aws_subscribe(mutations: ["addDestination"]) } type Weather { description: String current: String maxTemp: String minTemp: String } schema { query: Query mutation: Mutation subscription: Subscription } """) table_name = 'destinations' table = Table(self, 'DestinationsTable', table_name=table_name, partition_key=Attribute( name="id", type=AttributeType.STRING, ), billing_mode=BillingMode.PAY_PER_REQUEST, stream=StreamViewType.NEW_IMAGE) table_role = Role(self, 'DestinationsDynamoDBRole', assumed_by=ServicePrincipal('appsync.amazonaws.com')) table_role.add_managed_policy( ManagedPolicy.from_aws_managed_policy_name( 'AmazonDynamoDBFullAccess')) data_source = CfnDataSource( self, 'DestinationsDataSource', api_id=graphql_api.attr_api_id, name='DestinationsDynamoDataSource', type='AMAZON_DYNAMODB', dynamo_db_config=CfnDataSource.DynamoDBConfigProperty( table_name=table.table_name, aws_region=self.region), service_role_arn=table_role.role_arn) lambdaFn = Function(self, "GetWeather", code=Code.asset(os.getcwd() + "/lambdas/weather/"), handler="weather.get", timeout=core.Duration.seconds(900), memory_size=128, runtime=Runtime.NODEJS_10_X, environment={'APPID': os.getenv('APPID')}) lambda_role = Role( self, 'WeatherLambdaRole', assumed_by=ServicePrincipal('appsync.amazonaws.com')) lambda_role.add_managed_policy( ManagedPolicy.from_aws_managed_policy_name('AWSLambdaFullAccess')) lambda_source = CfnDataSource( self, 'WeatherDataSource', api_id=graphql_api.attr_api_id, name='WeatherCondition', type='AWS_LAMBDA', lambda_config=CfnDataSource.LambdaConfigProperty( lambda_function_arn=lambdaFn.function_arn), service_role_arn=lambda_role.role_arn) self.add_resolvers(graphql_api, api_schema, data_source=data_source, lambda_source=lambda_source)