def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) email_subscription_parameter = core.CfnParameter( self, "EmailSubscriptionParameter", description="Email Address for Notification Subscription") email_subscription = email_subscription_parameter.value_as_string ar1 = accessanalyzer.CfnAnalyzer.ArchiveRuleProperty( rule_name="test", filter=[ accessanalyzer.CfnAnalyzer.FilterProperty( property="principal.AWS", eq=["123456789123"]) ]) analyzer = accessanalyzer.CfnAnalyzer( self, id="accessanalyzer", type="ACCOUNT", tags=[core.CfnTag(key="AccessAnalyzerType", value="ACCOUNT")], archive_rules=[ar1]) runtime = aws_lambda.Runtime.PYTHON_3_8 boto3_lambda_layer = aws_lambda.LayerVersion( self, "Boto3LambdaLayer", code=aws_lambda.AssetCode("./layers/boto3"), compatible_runtimes=[runtime], description="Boto3 Lambda Layer") context_enrichment = aws_lambda.Function( self, "context_enrichment", runtime=runtime, handler="app.handler", code=aws_lambda.AssetCode("./functions/context-enrichment"), layers=[boto3_lambda_layer]) handler_statement = iam.PolicyStatement(actions=[ "iam:ListRoleTags", "s3:GetBucketTagging", "lambda:ListTags", "sqs:ListQueueTags", "kms:ListAliases", "kms:ListResourceTags" ], effect=iam.Effect.ALLOW, resources=["*"]) context_enrichment.add_to_role_policy(handler_statement) cmk_key = kms.Key( self, "SNSEncryptionAtRestKey", description="SNS Encryption at rest key", alias="sns-encryption-at-rest", enable_key_rotation=True, ) email_topic = sns.Topic( self, "AccessAnalyzerNotificationTopic", display_name="Access Analyzer Finding Notification Topic", master_key=cmk_key) email_topic.add_subscription( subscriptions.EmailSubscription(email_subscription)) notification = aws_lambda.Function( self, "notification", runtime=runtime, handler="app.handler", code=aws_lambda.AssetCode("./functions/notification"), layers=[boto3_lambda_layer], environment={"SNS_TOPIC_ARN": email_topic.topic_arn}) notification_statement = iam.PolicyStatement(actions=[ "sns:Publish", ], effect=iam.Effect.ALLOW, resources=["*"]) notification.add_to_role_policy(notification_statement) cmk_key.grant_encrypt_decrypt(notification) archive_access_analyzer_finding = aws_lambda.Function( self, "archive-access-analyzer-finding", runtime=runtime, handler="app.handler", code=aws_lambda.AssetCode( "./functions/archive-access-analyzer-finding"), layers=[boto3_lambda_layer]) archive_statement = iam.PolicyStatement(actions=[ "access-analyzer:UpdateFindings", ], effect=iam.Effect.ALLOW, resources=["*"]) archive_access_analyzer_finding.add_to_role_policy(archive_statement) evaluate_access_analyzer_finding = aws_lambda.Function( self, "evaluate-access-analyzer-finding", runtime=runtime, handler="app.handler", code=aws_lambda.AssetCode( "./functions/evaluate-access-analyzer-finding"), layers=[boto3_lambda_layer]) #https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html access_analyzer_handler_task = sfn.Task( self, "Context Enrichment", task=sfn_tasks.InvokeFunction(context_enrichment), result_path="$.guid", ) notification_task = sfn.Task( self, "Send Notification", task=sfn_tasks.InvokeFunction(notification), result_path="$.guid", ) archive_task = sfn.Task( self, "Archive Finding", task=sfn_tasks.InvokeFunction(archive_access_analyzer_finding), result_path="$.guid", ) evaluate_task = sfn.Task( self, "Evaluate Risk Level", task=sfn_tasks.InvokeFunction(evaluate_access_analyzer_finding), result_path="$.guid", ) definition=access_analyzer_handler_task. \ next(evaluate_task). \ next(sfn.Choice(self, "Archive?"). \ when(sfn.Condition.string_equals("$.guid.status", "ARCHIVE"), archive_task). \ when(sfn.Condition.string_equals("$.guid.status", "NOTIFY"), notification_task) \ ) state_machine = sfn.StateMachine( self, "Access-Analyzer-Automatic-Finding-Archive-State-Machine", definition=definition, timeout=core.Duration.minutes(5), ) #https://docs.aws.amazon.com/step-functions/latest/dg/tutorial-cloudwatch-events-s3.html access_analyzer_finding_rule = aws_events.Rule( self, "AccessAnalzyerFindingActiveEventRule", description="Access Analyzer Finding Event Active", enabled=True, event_pattern=aws_events.EventPattern( source=["aws.access-analyzer"], detail_type=["Access Analyzer Finding"], detail={"status": ["ACTIVE"]}), targets=[ aws_events_targets.SfnStateMachine(state_machine), aws_events_targets.LambdaFunction(context_enrichment) ])
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # create s3 bucket for artifacts artifacts_bucket = aws_s3.Bucket( self, "artifacts-bucket", block_public_access=aws_s3.BlockPublicAccess.BLOCK_ALL, removal_policy=cdk.RemovalPolicy.DESTROY, encryption=aws_s3.BucketEncryption.S3_MANAGED) # create codebuild project that executes a long-running script build_project = aws_codebuild.Project( self, "long-running-script-build-project", environment_variables={ "S3_ARTIFACTS_BUCKET": { "value": artifacts_bucket.bucket_name }, "S3_ARTIFACTS_OBJECT": { "value": "script.py" } }, environment=aws_codebuild.BuildEnvironment( build_image=aws_codebuild.LinuxBuildImage.STANDARD_3_0, ), timeout=cdk.Duration.hours(1), build_spec=aws_codebuild.BuildSpec.from_object({ "version": "0.2", "phases": { "install": { "runtime-versions": { "python": 3.8 } }, "build": { "commands": [ "aws s3 cp s3://$S3_ARTIFACTS_BUCKET/$S3_ARTIFACTS_OBJECT $S3_ARTIFACTS_OBJECT", "python $S3_ARTIFACTS_OBJECT" ] } } })) # grant read access of the artifacts bucket to the codebuild role artifacts_bucket.grant_read(build_project.role) # create eventbridge rule to trigger codebuild project long_running_script_rule = aws_events.Rule( self, "long-running-script-build-trigger", schedule=aws_events.Schedule.rate(cdk.Duration.hours(1))) long_running_script_rule.add_target( targets.CodeBuildProject(build_project)) # create sns topic as part of downstream services after codebuild project completes sns_topic = aws_sns.Topic(self, "script-completes-topic") # create eventbridge rule to publish to sns topic once codebuild project finishes (either succeeded, failed or stopped) codebuild_completes_rule = aws_events.Rule( self, "codebuild-scripts-complete-rule", event_pattern=aws_events.EventPattern( source=["aws.codebuild"], detail_type=["CodeBuild Build State Change"], detail={ "build-status": ["SUCCEEDED", "FAILED", "STOPPED"], "project-name": [build_project.project_name] })) codebuild_completes_rule.add_target(targets.SnsTopic(sns_topic)) cdk.CfnOutput(self, "artifacts-bucket-output", value=artifacts_bucket.bucket_name) cdk.CfnOutput(self, "script-complete-topic-output", value=sns_topic.topic_arn)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) aws_region = os.environ.get("CDK_DEPLOY_REGION", os.environ["CDK_DEFAULT_REGION"]) account_id = os.environ.get("CDK_DEPLOY_ACCOUNT", os.environ["CDK_DEFAULT_ACCOUNT"]) ssm_client = boto3.client('ssm', aws_region) # Prepare pipeline config details in SSM parameters if prefix == 'us': self.qs_reports_env_config = {"Permissions": [{"Group_Name": "critical", "Reports": ["Sales Results - Critical"], "ns_name": "default"}, {"Group_Name": "highlyconfidential", "Reports": ["Field Operations Dashboard", "Sales Results - Highly Confidential" ], "ns_name": "default"}, {"Group_Name": "bi-developer", "Reports": ["all"], "ns_name": "default"}, {"Group_Name": "bi-admin", "Reports": ["all"], "ns_name": "default"}, {"Group_Name": "power-reader", "Reports": ["read-all"], "ns_name": "default"}, {"Group_Name": "3rd-party", "Reports": ["Marketing KPIs"], "ns_name": "3rd-party"}, {"Group_Name": "3rd-party-reader", "Reports": ["Marketing KPIs"], "ns_name": "3rd-party"} ] } if prefix == 'eu': self.qs_reports_env_config = {"Permissions": [{"Group_Name": "eu-critical", "Reports": ["EUResults - Critical"]}, {"Group_Name": "bi-developer", "Reports": ["all"]}, {"Group_Name": "bi-admin", "Reports": ["all"]}, {"Group_Name": "eu-highlyconfidential", "Reports": ["EUField Operations Dashboard", "EUResults - Highly Confidential"]}, {"Group_Name": "power-reader", "Reports": ["read-all"]}]} self.qs_reports_env_config_ssm = ssm.StringParameter( self, '/qs/config/access', string_value=json.dumps(self.qs_reports_env_config), parameter_name='/qs/config/access' ) #group-user mapping information is stored in s3 bucket. A ssm parameter stores the bucket name. self.qs_user_group_config = {'bucket-name':f'qs-granular-access-demo-{account_id}'} bucket = s3.Bucket(self, f'qs-granular-access-demo-{account_id}', bucket_name=f'qs-granular-access-demo-{account_id}', versioned=True, removal_policy=core.RemovalPolicy.DESTROY, auto_delete_objects=True) s3deploy.BucketDeployment(self, "DeployMembership", sources=[s3deploy.Source.asset('membership.zip')], destination_bucket=bucket, destination_key_prefix='membership', prune=False) self.qs_user_group_config_ssm = ssm.StringParameter( self, '/qs/config/groups', string_value=json.dumps(self.qs_user_group_config), parameter_name='/qs/config/groups' ) # group-role mapping information is stored in a ssm parameter. self.qs_role_config = {'default_bi-developer': 'AUTHOR', 'default_bi-admin': 'ADMIN', 'default_power-reader': 'AUTHOR', 'default_critical': 'READER', 'default_highlyconfidential': 'READER', 'default_marketing': 'AUTHOR', '3rd-party_3rd-party': 'AUTHOR', '3rd-party_3rd-party-reader': 'READER' } self.qs_role_config_ssm = ssm.StringParameter( self, '/qs/config/roles', string_value=json.dumps(self.qs_role_config), parameter_name='/qs/config/roles' ) # group-namespace mapping information is stored in a ssm parameter. self.qs_ns_config = {"ns":['default', '3rd-party']} self.qs_ns_config_ssm = ssm.StringParameter( self, '/qs/config/ns', string_value=json.dumps(self.qs_ns_config), parameter_name='/qs/config/ns' ) lambda_role = iam.Role( self, id='lambda-role', description='Role for the quicksight lambda', role_name=f'{aws_region}-role-quicksight-lambda', max_session_duration=core.Duration.seconds(3600), assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'), inline_policies={ 'AllowS3Access': iam.PolicyDocument( statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=["kms:GetParametersForImport", "kms:GetPublicKey", "kms:ListKeyPolicies", "kms:ListRetirableGrants", "kms:GetKeyPolicy", "kms:ListResourceTags", "kms:ListGrants", "kms:GetParametersForImport", "kms:GetKeyRotationStatus", "kms:DescribeKey", "kms:CreateGrant", "kms:ListAliases", "kms:ListKeys", "kms:DescribeCustomKeyStores", "ssm:GetParameters", "ssm:GetParameter", "ssm:GetParametersByPath" ], resources=['*'] ), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=["lambda:InvokeFunction", "logs:CreateLogStream", "logs:CreateLogGroup", "logs:PutLogEvents", "quicksight:*", "s3:HeadBucket", "s3:ListAllMyBuckets", "s3:PutObject", "s3:GetObject", "s3:ListBucket", "s3:GetObjectVersionForReplication", "s3:GetBucketPolicy", "s3:GetObjectVersion", "cloudwatch:PutMetricData", "sts:GetCallerIdentity"], resources=['*'] ) ] ) } ) user_init = _lambda.Function(self, 'user_init', handler='user_init.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset(os.path.join(current_dir, '../lambda_functions/user_init/')), function_name='user_init', role=lambda_role, timeout=core.Duration.minutes(15), memory_size=512 ) check_team_members = _lambda.Function(self, 'check_team_members', handler='check_team_members.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset(os.path.join(current_dir, '../lambda_functions/check_team_members/')), function_name='check_team_members', role=lambda_role, timeout=core.Duration.minutes(15), memory_size=512, environment={'aws_region': f'{core.Aws.REGION}'} ) downgrade_user = _lambda.Function(self, 'downgrade_user', handler='downgrade_user.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.from_asset(os.path.join(current_dir, '../lambda_functions/downgrade_user/')), function_name='downgrade_user', role=lambda_role, timeout=core.Duration.minutes(15), memory_size=2048, environment={'aws_region': f'{core.Aws.REGION}'} ) granular_user_govenance = _lambda.Function(self, 'granular_user_govenance', handler='granular_user_govenance.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset(os.path.join(current_dir, '../lambda_functions/granular_user_govenance')), function_name='granular_user_govenance', role=lambda_role, timeout=core.Duration.minutes(15), memory_size=2048, environment={'aws_region': f'{core.Aws.REGION}'} ) granular_access_assets_govenance = _lambda.Function(self, 'granular_access_assets_govenance', handler='granular_access_assets_govenance.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset(os.path.join(current_dir, '../lambda_functions/granular_access_assets_govenance')), function_name='granular_access_assets_govenance', role=lambda_role, timeout=core.Duration.minutes(15), memory_size=2048, environment={'aws_region': f'{core.Aws.REGION}'} ) quicksight_event_rule = events.Rule(self, 'QuickSightCWEventRule', description='CloudWatch rule to detect new QuickSight user creation', rule_name='qs-gc-user-creation', targets=[targets.LambdaFunction(user_init)], event_pattern=events.EventPattern(source=['aws.quicksight'], detail_type=[ 'AWS Service Event via CloudTrail'], detail={ "eventSource": [ "quicksight.amazonaws.com"], "eventName": ["CreateUser"] } ) ) quicksight_schedule_rule = events.Rule(self, 'quicksight_schedule_rule', description='CloudWatch rule to run QS objects/groups assignment every hour', rule_name='qs-gc-every-hour', schedule=events.Schedule.cron(minute="0"), targets=[targets.LambdaFunction(granular_user_govenance)] ) quicksight_assume_condition_object = {"StringEquals": { "SAML:aud": "https://signin.aws.amazon.com/saml"}} quicksight_federated_prin_with_conditionb_obj = iam.FederatedPrincipal( f'arn:aws:iam::{core.Aws.ACCOUNT_ID}:saml-provider/saml', quicksight_assume_condition_object, 'sts:AssumeRoleWithSAML') quicksight_resource_scope = '${aws:userid}' quicksight_reader_saml_inline_policies = { 'AllowQuicksightAccessSAML': iam.PolicyDocument( statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['quicksight:CreateReader'], resources=[ f'arn:aws:quicksight::{core.Aws.ACCOUNT_ID}:user/{quicksight_resource_scope}'] ) ] ) } quicksight_users = iam.Role( self, id=f"quicksight-fed-{prefix}-users", # this is the default group with no access description='Role for the quicksight reader SAML', role_name=f"quicksight-fed-{prefix}-users", max_session_duration=core.Duration.seconds(3600), assumed_by=quicksight_federated_prin_with_conditionb_obj, inline_policies=quicksight_reader_saml_inline_policies )
def __init__(self, scope: core.Construct, construct_id: str, stack_log_level: str, orders_bus, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # Add your stack resources below) # Read Lambda Code try: with open( "stacks/back_end/serverless_eventbridge_consumer_stack/lambda_src/eventbridge_data_consumer.py", encoding="utf-8", mode="r") as f: msg_consumer_fn_code = f.read() except OSError: print("Unable to read Lambda Function Code") raise msg_consumer_fn = _lambda.Function( self, "msgConsumerFn", function_name=f"events_consumer_fn", description="Process messages in EventBridge queue", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(msg_consumer_fn_code), handler="index.lambda_handler", timeout=core.Duration.seconds(3), reserved_concurrent_executions=1, environment={ "LOG_LEVEL": f"{stack_log_level}", "APP_ENV": "Production" }) msg_consumer_fn_version = msg_consumer_fn.latest_version msg_consumer_fn_version_alias = _lambda.Alias( self, "msgConsumerFnAlias", alias_name="MystiqueAutomation", version=msg_consumer_fn_version) # Create Custom Loggroup for Producer msg_consumer_fn_lg = _logs.LogGroup( self, "msgConsumerFnLogGroup", log_group_name=f"/aws/lambda/{msg_consumer_fn.function_name}", removal_policy=core.RemovalPolicy.DESTROY, retention=_logs.RetentionDays.ONE_DAY) # Restrict Produce Lambda to be invoked only from the stack owner account msg_consumer_fn.add_permission("restrictLambdaInvocationToOwnAccount", principal=_iam.AccountRootPrincipal(), action="lambda:InvokeFunction", source_account=core.Aws.ACCOUNT_ID, source_arn=orders_bus.event_bus_arn) # Event Pattern self.orders_pattern = _evnts.EventPattern(detail_type=["sales-events"]) # EventBridge Routing Rule self.orders_routing = _evnts.Rule( self, f"ordersEventRoutingRule01", description="A simple events routing rule", enabled=True, event_bus=orders_bus, event_pattern=self.orders_pattern, rule_name="orders_routing_to_consumer", targets=[_evnts_tgt.LambdaFunction(handler=msg_consumer_fn)]) self.orders_routing.apply_removal_policy(core.RemovalPolicy.DESTROY) # Restrict Produce Lambda to be invoked only from the stack owner account data_producer_fn.add_permission( "restrictLambdaInvocationToFhInOwnAccount", principal=_iam.AccountRootPrincipal(), action="lambda:InvokeFunction", source_account=core.Aws.ACCOUNT_ID) ########################################### ################# OUTPUTS ################# ########################################### output_0 = core.CfnOutput( self, "AutomationFrom", value=f"{GlobalArgs.SOURCE_INFO}", description= "To know more about this automation stack, check out our github page." ) output_2 = core.CfnOutput( self, "msgConsumer", value= f"https://console.aws.amazon.com/lambda/home?region={core.Aws.REGION}#/functions/{msg_consumer_fn.function_name}", description="Process events received from eventbridge event bus")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) eventTargets = [] policyStatement = _iam.PolicyStatement( resources = ['*'], actions = [ "cloudwatch:PutMetricAlarm", "cloudwatch:ListMetrics", "cloudwatch:DeleteAlarms", "ec2:CreateTags", "ec2:Describe*", "ec2:Attach*", "elasticloadbalancing:Describe*", "elasticloadbalancing:Create*", "elasticloadbalancing:AddTags" ], effect = _iam.Effect.ALLOW ) glom_layer = _lambda.LayerVersion.from_layer_version_attributes( self, "glom_api_layer", layer_version_arn="arn:aws:lambda:us-east-1:<AWS ACCOUNT>:layer:python-glom-layer:1", compatible_runtimes=[ _lambda.Runtime.PYTHON_3_6, _lambda.Runtime.PYTHON_3_7 ] ) eventHandler = _lambda.Function( self, 'resourceTagger', runtime = _lambda.Runtime.PYTHON_3_7, code = _lambda.Code.asset('lambda'), handler = 'auto_tag.handler', layers=[glom_layer] ) eventHandler.add_to_role_policy(policyStatement) eventTargets.append(_targets.LambdaFunction(handler = eventHandler)) pattern = _events.EventPattern( source = ['aws.ec2', 'aws.elasticloadbalancing'], detail_type = [ "AWS API Call via CloudTrail"], detail = { "eventSource": [ "ec2.amazonaws.com", "elasticloadbalancing.amazonaws.com" ], "eventName": [ "RunInstances", "CreateSnapshot", "CreateVolume", "CreateImage", "CreateLoadBalancer", "AttachNetworkInterface" ] } ) _events.Rule( scope = self, id = 'AutoTagsRule', description = 'Monitor EC2 and ELB events', rule_name = 'AutoTagsRule', event_pattern = pattern, targets = eventTargets )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # DynamoDB Table # This will store our error records # TTL Docs - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/time-to-live-ttl-how-to.html table = dynamo_db.Table(self, "CircuitBreaker", partition_key=dynamo_db.Attribute( name="RequestID", type=dynamo_db.AttributeType.STRING), sort_key=dynamo_db.Attribute( name="ExpirationTime", type=dynamo_db.AttributeType.NUMBER), time_to_live_attribute='ExpirationTime') # Add an index that lets us query on site url and Expiration Time table.add_global_secondary_index( index_name='UrlIndex', partition_key=dynamo_db.Attribute( name="SiteUrl", type=dynamo_db.AttributeType.STRING), sort_key=dynamo_db.Attribute(name="ExpirationTime", type=dynamo_db.AttributeType.NUMBER)) # defines an Integration Lambda to call our failing web service integration_lambda = _lambda.Function( self, "WebserviceIntegrationLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="lambda.handler", code=_lambda.Code.from_asset("lambda_fns/webservice"), timeout=core.Duration.seconds(20), environment=dict(TABLE_NAME=table.table_name)) # grant the lambda role read/write permissions to our table table.grant_read_data(integration_lambda) # We need to give your lambda permission to put events on our EventBridge event_policy = iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=['*'], actions=['events:PutEvents']) integration_lambda.add_to_role_policy(event_policy) # defines a lambda to insert errors into dynamoDB error_lambda = _lambda.Function( self, "ErrorLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="lambda.handler", code=_lambda.Code.from_asset("lambda_fns/error"), timeout=core.Duration.seconds(3), environment=dict(TABLE_NAME=table.table_name)) table.grant_write_data(error_lambda) # Create EventBridge rule to route failures error_rule = events.Rule( self, 'webserviceErrorRule', description='Failed Webservice Call', event_pattern=events.EventPattern( source=['cdkpatterns.eventbridge.circuitbreaker'], detail_type=['httpcall'], detail={"status": ["fail"]})) error_rule.add_target(targets.LambdaFunction(handler=error_lambda)) # defines an API Gateway REST API resource backed by our "integration_lambda" function api_gw.LambdaRestApi(self, 'CircuitBreakerGateway', handler=integration_lambda)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) table = ddb.Table(self, 'StripeSampleCustomers', partition_key={ 'name': 'CustomerID', 'type': ddb.AttributeType.STRING }) bus = events.EventBus(self, 'stripeAppEventBus', event_bus_name='stripeAppEventBus') lambda_role_for_go = iam.Role( self, "Role", role_name='stripeAppRole', assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaBasicExecutionRole"), iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonEventBridgeFullAccess"), iam.ManagedPolicy.from_aws_managed_policy_name( "SecretsManagerReadWrite") ]) customer_created_handler = _lambda.Function( self, "createStripeCustomerHandler", runtime=_lambda.Runtime.GO_1_X, code=_lambda.Code.asset('lambda/stripe-create-customer'), handler='createCustomerHandler', timeout=core.Duration.seconds(8), role=lambda_role_for_go, environment={ 'CUSTOMER_TABLE_NAME': table.table_name, }) table.grant_read_write_data(customer_created_handler) go_lambda = _lambda.Function( self, "stripeWebhookEventHandler", runtime=_lambda.Runtime.GO_1_X, code=_lambda.Code.asset('lambda/stripe-webhook-handler'), handler='stripeWebhookHandler', timeout=core.Duration.seconds(8), role=lambda_role_for_go, ) _apigw.LambdaRestApi(self, "stripeWebhookAPI", handler=go_lambda) customer_created_handler.add_permission( "createStripeCustomerHandlerPermission", principal=iam.ServicePrincipal("events.amazonaws.com"), action='lambda:InvokeFunction', source_arn=go_lambda.function_arn) go_lambda.add_permission( "stripeWebhookHandlerPermission", principal=iam.ServicePrincipal("lambda.amazonaws.com"), action='lambda:InvokeFunction', source_arn=customer_created_handler.function_arn) event = events.Rule( self, 'stripeWebhookEventRule', rule_name='stripeWebhookEventRule', enabled=True, event_bus=bus, description= 'all success events are caught here and logged centrally', event_pattern=events.EventPattern( detail={"stripeEvent": ["customer.subscription.created"]}, source=["stripeWebHookHandler.lambda"])) event.add_target(targets.LambdaFunction(customer_created_handler))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here pvt_bkt = _s3.Bucket(self, "s3bucket") core.Tag.add(pvt_bkt, key="isMonitoredBucket", value="True") # Lets create a cloudtrail to track s3 data events s3_data_event_trail = _cloudtrail.Trail( self, "s3DataEventTrailId", is_multi_region_trail=False, include_global_service_events=False, enable_file_validation=True) # Lets capture S3 Data Events only for our bucket- TO REDUCE COST s3_data_event_trail.add_s3_event_selector( prefixes=[f"{pvt_bkt.bucket_arn}/"], include_management_events=True, read_write_type=_cloudtrail.ReadWriteType.ALL) # Defines an AWS Lambda resource """ with open("lambda_src/make_object_private.py", encoding="utf8") as fp: make_object_private_fn_handler_code = fp.read() remediate_object_acl_fn = _lambda.Function( self, id='remediateObjAclFn', function_name="remediate_object_acl_fn", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(make_object_private_fn_handler_code), handler='index.lambda_handler', timeout=core.Duration.seconds(10) ) # Lets add the necessary permission for the lambda function remediate_object_acl_fn_perms=_iam.PolicyStatement( effect=_iam.Effect.ALLOW, resources=[ "arn:aws:s3:::*", ], actions=[ "s3:GetObjectAcl", "s3:PutObjectAcl" ] ) remediate_object_acl_fn_perms.sid="PutBucketPolicy" remediate_object_acl_fn.add_to_role_policy( remediate_object_acl_fn_perms ) """ with open("lambda_src/is_object_private.py", encoding="utf8") as fp: is_object_private_fn_handler_code = fp.read() is_object_private_fn = _lambda.Function( self, id='isObjPrivateFn', function_name="is_object_private_fn", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(is_object_private_fn_handler_code), handler='index.lambda_handler', timeout=core.Duration.seconds(3)) # Lets add the necessary permission for the lambda function is_object_private_fn_perms = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, resources=[ "arn:aws:s3:::*", ], actions=["s3:GetObjectAcl"]) is_object_private_fn.sid = "CheckObjectAcl" is_object_private_fn.add_to_role_policy(is_object_private_fn_perms) with open("lambda_src/make_object_private.py", encoding="utf8") as fp: make_object_private_fn_handler_code = fp.read() remediate_object_acl_fn = _lambda.Function( self, id='remediateObjAclFn', function_name="remediate_object_acl_fn", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(make_object_private_fn_handler_code), handler='index.lambda_handler', timeout=core.Duration.seconds(10)) # Lets add the necessary permission for the lambda function remediate_object_acl_fn_perms = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, resources=[ "arn:aws:s3:::*", ], actions=["s3:PutObjectAcl"]) remediate_object_acl_fn_perms.sid = "PutObjectAcl" remediate_object_acl_fn.add_to_role_policy( remediate_object_acl_fn_perms) info_sec_ops_topic = _sns.Topic(self, "infoSecOpsTopicId", display_name="InfoSecTopic", topic_name="InfoSecOpsTopic") # Subscribe InfoSecOps Email to topic info_sec_ops_topic.add_subscription( _subs.EmailSubscription(global_args.INFO_SEC_OPS_EMAIL)) # Grant Lambda permission to publish to topic # info_sec_ops_topic.grant_publish(lambda_notifier) # State Machine for notifying failed ACLs # Ref: https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html ############################################################################### ################# STEP FUNCTIONS EXPERIMENTAL CODE - UNSTABLE ################# ############################################################################### is_object_private_task = _sfn.Task( self, "isObjectPrivate?", task=_tasks.InvokeFunction(is_object_private_fn), result_path="$", output_path="$") remediate_object_acl_task = _sfn.Task( self, "RemediateObjectAcl", task=_tasks.InvokeFunction(remediate_object_acl_fn), result_path="$", output_path="$") notify_secops_task = _sfn.Task( self, "Notify InfoSecOps", task=_tasks.PublishToTopic( info_sec_ops_topic, integration_pattern=_sfn.ServiceIntegrationPattern. FIRE_AND_FORGET, message=_sfn.TaskInput.from_data_at("$.sns_message"), subject="Object Acl Remediation")) acl_remediation_failed_task = _sfn.Fail(self, "Acl Remediation Failed", cause="Acl Remediation Failed", error="Check Logs") acl_compliant_task = _sfn.Succeed(self, "Object Acl Compliant", comment="Object Acl is Compliant") remediate_object_acl_sfn_definition = is_object_private_task\ .next(_sfn.Choice(self, "Is Object Private?")\ .when(_sfn.Condition.boolean_equals("$.is_private", True), acl_compliant_task)\ .when(_sfn.Condition.boolean_equals("$.is_private", False), remediate_object_acl_task\ .next(_sfn.Choice(self, "Object Remediation Complete?")\ .when(_sfn.Condition.boolean_equals("$.status", True),acl_compliant_task)\ .when(_sfn.Condition.boolean_equals("$.status", False), notify_secops_task.next(acl_remediation_failed_task))\ .otherwise(acl_remediation_failed_task)\ ) ) .otherwise(acl_remediation_failed_task) ) remediate_object_acl_statemachine = _sfn.StateMachine( self, "stateMachineId", definition=remediate_object_acl_sfn_definition, timeout=core.Duration.minutes(3)) # Cloudwatch Event Triggers put_object_acl_event_targets = [] """ put_object_acl_event_targets.append( _targets.LambdaFunction( handler=remediate_object_acl_fn ) ) """ put_object_acl_event_targets.append( _targets.SfnStateMachine( machine=remediate_object_acl_statemachine)) put_object_acl_event_pattern = _events.EventPattern( source=["aws.s3"], detail_type=["AWS API Call via CloudTrail"], detail={ "eventSource": ["s3.amazonaws.com"], "eventName": ["PutObjectAcl", "PutObject"], "requestParameters": { "bucketName": [f"{pvt_bkt.bucket_name}"] } }) put_object_acl_event_pattern_rule = _events.Rule( self, "putObjectAclEventId", event_pattern=put_object_acl_event_pattern, rule_name=f"put_s3_policy_event_{global_args.OWNER}", enabled=True, description="Trigger an event for S3 PutObjectAcl or PutObject", targets=put_object_acl_event_targets) ########################################### ################# OUTPUTS ################# ########################################### output0 = core.CfnOutput( self, "SecuirtyAutomationFrom", value=f"{global_args.SOURCE_INFO}", description= "To know more about this automation stack, check out our github page." ) output1 = core.CfnOutput( self, "MonitoredS3Bucket", value=(f"https://console.aws.amazon.com/s3/buckets/" f"{pvt_bkt.bucket_name}"), description=f"S3 Bucket for testing purposes") output2 = core.CfnOutput( self, "Helpercommands", value= (f"aws s3api get-object-acl --bucket ${pvt_bkt.bucket_name} --key OBJECT-KEY-NAME" ), description= f"Commands to set object to public, Update OBJECT-KEY-NAME to your needs" )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Parameteres # notification_email = create_notification_email_parameter(self) template_zip_name = create_template_zip_name_parameter(self) template_file_name = create_template_file_name_parameter(self) dev_params_file_name = create_stage_params_file_name_parameter(self, "DEV_PARAMS_NAME", "development") staging_params_file_name = create_stage_params_file_name_parameter(self, "STAGING_PARAMS_NAME", "staging") prod_params_file_name = create_stage_params_file_name_parameter(self, "PROD_PARAMS_NAME", "production") # create development parameters account_type = "development" dev_account_id = create_account_id_parameter(self, "DEV_ACCOUNT_ID", account_type) dev_org_id = create_org_id_parameter(self, "DEV_ORG_ID", account_type) # create staging parameters account_type = "staging" staging_account_id = create_account_id_parameter(self, "STAGING_ACCOUNT_ID", account_type) staging_org_id = create_org_id_parameter(self, "STAGING_ORG_ID", account_type) # create production parameters account_type = "production" prod_account_id = create_account_id_parameter(self, "PROD_ACCOUNT_ID", account_type) prod_org_id = create_org_id_parameter(self, "PROD_ORG_ID", account_type) # assets parameters blueprint_bucket_name = create_blueprint_bucket_name_parameter(self) assets_bucket_name = create_assets_bucket_name_parameter(self) stack_name = create_stack_name_parameter(self) # Resources # assets_bucket = s3.Bucket.from_bucket_name(self, "AssetsBucket", assets_bucket_name.value_as_string) # getting blueprint bucket object from its name - will be used later in the stack blueprint_bucket = s3.Bucket.from_bucket_name(self, "BlueprintBucket", blueprint_bucket_name.value_as_string) # create sns topic and subscription pipeline_notification_topic = sns.Topic( self, "PipelineNotification", ) pipeline_notification_topic.node.default_child.cfn_options.metadata = suppress_sns() pipeline_notification_topic.add_subscription( subscriptions.EmailSubscription(email_address=notification_email.value_as_string) ) # Defining pipeline stages # source stage source_output, source_action_definition = source_action_template(template_zip_name, assets_bucket) # DeployDev stage dev_deploy_lambda_arn, dev_stackset_action = create_stackset_action( self, "DeployDevStackSet", blueprint_bucket, source_output, "Artifact_Source_S3Source", template_file_name.value_as_string, dev_params_file_name.value_as_string, [dev_account_id.value_as_string], [dev_org_id.value_as_string], [core.Aws.REGION], assets_bucket, f"{stack_name.value_as_string}-dev", ) # DeployStaging manual approval deploy_staging_approval = approval_action( "DeployStaging", pipeline_notification_topic, [notification_email.value_as_string], "Please approve to deploy to staging account", ) # DeployStaging stage staging_deploy_lambda_arn, staging_stackset_action = create_stackset_action( self, "DeployStagingStackSet", blueprint_bucket, source_output, "Artifact_Source_S3Source", template_file_name.value_as_string, staging_params_file_name.value_as_string, [staging_account_id.value_as_string], [staging_org_id.value_as_string], [core.Aws.REGION], assets_bucket, f"{stack_name.value_as_string}-staging", ) # DeployProd manual approval deploy_prod_approval = approval_action( "DeployProd", pipeline_notification_topic, [notification_email.value_as_string], "Please approve to deploy to production account", ) # DeployProd stage prod_deploy_lambda_arn, prod_stackset_action = create_stackset_action( self, "DeployProdStackSet", blueprint_bucket, source_output, "Artifact_Source_S3Source", template_file_name.value_as_string, prod_params_file_name.value_as_string, [prod_account_id.value_as_string], [prod_org_id.value_as_string], [core.Aws.REGION], assets_bucket, f"{stack_name.value_as_string}-prod", ) # create invoking lambda policy invoke_lambdas_policy = iam.PolicyStatement( actions=[ "lambda:InvokeFunction", ], resources=[dev_deploy_lambda_arn, staging_deploy_lambda_arn, prod_deploy_lambda_arn], ) # createing pipeline stages source_stage = codepipeline.StageProps(stage_name="Source", actions=[source_action_definition]) deploy_dev_stage = codepipeline.StageProps( stage_name="DeployDev", actions=[dev_stackset_action, deploy_staging_approval], ) deploy_staging_stage = codepipeline.StageProps( stage_name="DeployStaging", actions=[staging_stackset_action, deploy_prod_approval], ) deploy_prod_stage = codepipeline.StageProps( stage_name="DeployProd", actions=[prod_stackset_action], ) # constructing multi-account pipeline multi_account_pipeline = codepipeline.Pipeline( self, "MultiAccountPipeline", stages=[source_stage, deploy_dev_stage, deploy_staging_stage, deploy_prod_stage], cross_account_keys=False, ) # add notification to the development stackset action dev_stackset_action.on_state_change( "NotifyUserDevDeployment", description="Notify user of the outcome of the DeployDev action", target=targets.SnsTopic( pipeline_notification_topic, message=events.RuleTargetInput.from_text( ( f"DeployDev action {events.EventField.from_path('$.detail.action')} in the Pipeline " f"{events.EventField.from_path('$.detail.pipeline')} finished executing. " f"Action execution result is {events.EventField.from_path('$.detail.state')}" ) ), ), event_pattern=events.EventPattern(detail={"state": ["SUCCEEDED", "FAILED"]}), ) # add notification to the staging stackset action staging_stackset_action.on_state_change( "NotifyUserStagingDeployment", description="Notify user of the outcome of the DeployStaging action", target=targets.SnsTopic( pipeline_notification_topic, message=events.RuleTargetInput.from_text( ( f"DeployStaging action {events.EventField.from_path('$.detail.action')} in the Pipeline " f"{events.EventField.from_path('$.detail.pipeline')} finished executing. " f"Action execution result is {events.EventField.from_path('$.detail.state')}" ) ), ), event_pattern=events.EventPattern(detail={"state": ["SUCCEEDED", "FAILED"]}), ) # add notification to the production stackset action prod_stackset_action.on_state_change( "NotifyUserProdDeployment", description="Notify user of the outcome of the DeployProd action", target=targets.SnsTopic( pipeline_notification_topic, message=events.RuleTargetInput.from_text( ( f"DeployProd action {events.EventField.from_path('$.detail.action')} in the Pipeline " f"{events.EventField.from_path('$.detail.pipeline')} finished executing. " f"Action execution result is {events.EventField.from_path('$.detail.state')}" ) ), ), event_pattern=events.EventPattern(detail={"state": ["SUCCEEDED", "FAILED"]}), ) # add notification to the multi-account pipeline multi_account_pipeline.on_state_change( "NotifyUser", description="Notify user of the outcome of the pipeline", target=targets.SnsTopic( pipeline_notification_topic, message=events.RuleTargetInput.from_text( ( f"Pipeline {events.EventField.from_path('$.detail.pipeline')} finished executing. " f"Pipeline execution result is {events.EventField.from_path('$.detail.state')}" ) ), ), event_pattern=events.EventPattern(detail={"state": ["SUCCEEDED", "FAILED"]}), ) multi_account_pipeline.add_to_role_policy( iam.PolicyStatement( actions=["events:PutEvents"], resources=[ f"arn:{core.Aws.PARTITION}:events:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:event-bus/*", ], ) ) # add lambda permissons multi_account_pipeline.add_to_role_policy(invoke_lambdas_policy) # add cfn supressions pipeline_child_nodes = multi_account_pipeline.node.find_all() pipeline_child_nodes[1].node.default_child.cfn_options.metadata = suppress_pipeline_bucket() pipeline_child_nodes[6].node.default_child.cfn_options.metadata = suppress_iam_complex() pipeline_child_nodes[19].node.default_child.cfn_options.metadata = suppress_list_function_policy() pipeline_child_nodes[32].node.default_child.cfn_options.metadata = suppress_list_function_policy() pipeline_child_nodes[45].node.default_child.cfn_options.metadata = suppress_list_function_policy() # attaching iam permissions to the pipelines pipeline_permissions(multi_account_pipeline, assets_bucket) # Outputs # core.CfnOutput( self, id="Pipelines", value=( f"https://console.aws.amazon.com/codesuite/codepipeline/pipelines/" f"{multi_account_pipeline.pipeline_name}/view?region={core.Aws.REGION}" ), )
def __init__(self, scope: Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) with open("stack/config.yml", 'r') as stream: configs = yaml.safe_load(stream) ### S3 core images_S3_bucket = _s3.Bucket(self, "ICS_IMAGES") images_S3_bucket.add_cors_rule( allowed_methods=[_s3.HttpMethods.POST], allowed_origins=["*"] # add API gateway web resource URL ) ### SQS core image_deadletter_queue = _sqs.Queue(self, "ICS_IMAGES_DEADLETTER_QUEUE") image_queue = _sqs.Queue( self, "ICS_IMAGES_QUEUE", dead_letter_queue={ "max_receive_count": configs["DeadLetterQueue"]["MaxReceiveCount"], "queue": image_deadletter_queue }) ### api gateway core api_gateway = RestApi(self, 'ICS_API_GATEWAY', rest_api_name='ImageContentSearchApiGateway') api_gateway_resource = api_gateway.root.add_resource( configs["ProjectName"]) api_gateway_landing_page_resource = api_gateway_resource.add_resource( 'web') api_gateway_get_signedurl_resource = api_gateway_resource.add_resource( 'signedUrl') api_gateway_image_search_resource = api_gateway_resource.add_resource( 'search') ### landing page function get_landing_page_function = Function( self, "ICS_GET_LANDING_PAGE", function_name="ICS_GET_LANDING_PAGE", runtime=Runtime.PYTHON_3_7, handler="main.handler", code=Code.from_asset("./src/landingPage")) get_landing_page_integration = LambdaIntegration( get_landing_page_function, proxy=True, integration_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': "'*'", } }]) api_gateway_landing_page_resource.add_method( 'GET', get_landing_page_integration, method_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': True, } }]) ### cognito required_attribute = _cognito.StandardAttribute(required=True) users_pool = _cognito.UserPool( self, "ICS_USERS_POOL", auto_verify=_cognito.AutoVerifiedAttrs( email=True), #required for self sign-up standard_attributes=_cognito.StandardAttributes( email=required_attribute), #required for self sign-up self_sign_up_enabled=configs["Cognito"]["SelfSignUp"]) user_pool_app_client = _cognito.CfnUserPoolClient( self, "ICS_USERS_POOL_APP_CLIENT", supported_identity_providers=["COGNITO"], allowed_o_auth_flows=["implicit"], allowed_o_auth_scopes=configs["Cognito"]["AllowedOAuthScopes"], user_pool_id=users_pool.user_pool_id, callback_ur_ls=[api_gateway.url_for_path('/web')], allowed_o_auth_flows_user_pool_client=True, explicit_auth_flows=["ALLOW_REFRESH_TOKEN_AUTH"]) user_pool_domain = _cognito.UserPoolDomain( self, "ICS_USERS_POOL_DOMAIN", user_pool=users_pool, cognito_domain=_cognito.CognitoDomainOptions( domain_prefix=configs["Cognito"]["DomainPrefix"])) ### get signed URL function get_signedurl_function = Function( self, "ICS_GET_SIGNED_URL", function_name="ICS_GET_SIGNED_URL", environment={ "ICS_IMAGES_BUCKET": images_S3_bucket.bucket_name, "DEFAULT_SIGNEDURL_EXPIRY_SECONDS": configs["Functions"]["DefaultSignedUrlExpirySeconds"] }, runtime=Runtime.PYTHON_3_7, handler="main.handler", code=Code.from_asset("./src/getSignedUrl")) get_signedurl_integration = LambdaIntegration( get_signedurl_function, proxy=True, integration_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': "'*'", } }]) api_gateway_get_signedurl_authorizer = CfnAuthorizer( self, "ICS_API_GATEWAY_GET_SIGNED_URL_AUTHORIZER", rest_api_id=api_gateway_get_signedurl_resource.api.rest_api_id, name="ICS_API_GATEWAY_GET_SIGNED_URL_AUTHORIZER", type="COGNITO_USER_POOLS", identity_source="method.request.header.Authorization", provider_arns=[users_pool.user_pool_arn]) api_gateway_get_signedurl_resource.add_method( 'GET', get_signedurl_integration, authorization_type=AuthorizationType.COGNITO, method_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': True, } }]).node.find_child('Resource').add_property_override( 'AuthorizerId', api_gateway_get_signedurl_authorizer.ref) images_S3_bucket.grant_put(get_signedurl_function, objects_key_pattern="new/*") ### image massage function image_massage_function = Function( self, "ICS_IMAGE_MASSAGE", function_name="ICS_IMAGE_MASSAGE", timeout=Duration.seconds(6), runtime=Runtime.PYTHON_3_7, environment={"ICS_IMAGE_MASSAGE": image_queue.queue_name}, handler="main.handler", code=Code.from_asset("./src/imageMassage")) images_S3_bucket.grant_write(image_massage_function, "processed/*") images_S3_bucket.grant_delete(image_massage_function, "new/*") images_S3_bucket.grant_read(image_massage_function, "new/*") new_image_added_notification = _s3notification.LambdaDestination( image_massage_function) images_S3_bucket.add_event_notification( _s3.EventType.OBJECT_CREATED, new_image_added_notification, _s3.NotificationKeyFilter(prefix="new/")) image_queue.grant_send_messages(image_massage_function) ### image analyzer function image_analyzer_function = Function( self, "ICS_IMAGE_ANALYSIS", function_name="ICS_IMAGE_ANALYSIS", runtime=Runtime.PYTHON_3_7, timeout=Duration.seconds(10), environment={ "ICS_IMAGES_BUCKET": images_S3_bucket.bucket_name, "DEFAULT_MAX_CALL_ATTEMPTS": configs["Functions"]["DefaultMaxApiCallAttempts"], "REGION": Aws.REGION, }, handler="main.handler", code=Code.from_asset("./src/imageAnalysis")) image_analyzer_function.add_event_source( _lambda_event_source.SqsEventSource(queue=image_queue, batch_size=10)) image_queue.grant_consume_messages(image_massage_function) lambda_rekognition_access = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=[ "rekognition:DetectLabels", "rekognition:DetectModerationLabels" ], resources=["*"]) image_analyzer_function.add_to_role_policy(lambda_rekognition_access) images_S3_bucket.grant_read(image_analyzer_function, "processed/*") ### API gateway finalizing self.add_cors_options(api_gateway_get_signedurl_resource) self.add_cors_options(api_gateway_landing_page_resource) self.add_cors_options(api_gateway_image_search_resource) ### database database_secret = _secrets_manager.Secret( self, "ICS_DATABASE_SECRET", secret_name="rds-db-credentials/image-content-search-rds-secret", generate_secret_string=_secrets_manager.SecretStringGenerator( generate_string_key='password', secret_string_template='{"username": "******"}', exclude_punctuation=True, exclude_characters='/@\" \\\'', require_each_included_type=True)) database = _rds.CfnDBCluster( self, "ICS_DATABASE", engine=_rds.DatabaseClusterEngine.aurora_mysql( version=_rds.AuroraMysqlEngineVersion.VER_5_7_12).engine_type, engine_mode="serverless", database_name=configs["Database"]["Name"], enable_http_endpoint=True, deletion_protection=configs["Database"]["DeletionProtection"], master_username=database_secret.secret_value_from_json( "username").to_string(), master_user_password=database_secret.secret_value_from_json( "password").to_string(), scaling_configuration=_rds.CfnDBCluster. ScalingConfigurationProperty( auto_pause=configs["Database"]["Scaling"]["AutoPause"], min_capacity=configs["Database"]["Scaling"]["Min"], max_capacity=configs["Database"]["Scaling"]["Max"], seconds_until_auto_pause=configs["Database"]["Scaling"] ["SecondsToAutoPause"]), ) database_cluster_arn = "arn:aws:rds:{}:{}:cluster:{}".format( Aws.REGION, Aws.ACCOUNT_ID, database.ref) secret_target = _secrets_manager.CfnSecretTargetAttachment( self, "ICS_DATABASE_SECRET_TARGET", target_type="AWS::RDS::DBCluster", target_id=database.ref, secret_id=database_secret.secret_arn) secret_target.node.add_dependency(database) ### database function image_data_function_role = _iam.Role( self, "ICS_IMAGE_DATA_FUNCTION_ROLE", role_name="ICS_IMAGE_DATA_FUNCTION_ROLE", assumed_by=_iam.ServicePrincipal("lambda.amazonaws.com"), managed_policies=[ _iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaVPCAccessExecutionRole"), _iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaBasicExecutionRole"), _iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonRDSDataFullAccess") ]) image_data_function = Function( self, "ICS_IMAGE_DATA", function_name="ICS_IMAGE_DATA", runtime=Runtime.PYTHON_3_7, timeout=Duration.seconds(5), role=image_data_function_role, environment={ "DEFAULT_MAX_CALL_ATTEMPTS": configs["Functions"]["DefaultMaxApiCallAttempts"], "CLUSTER_ARN": database_cluster_arn, "CREDENTIALS_ARN": database_secret.secret_arn, "DB_NAME": database.database_name, "REGION": Aws.REGION }, handler="main.handler", code=Code.from_asset("./src/imageData")) image_search_integration = LambdaIntegration( image_data_function, proxy=True, integration_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': "'*'", } }]) api_gateway_image_search_authorizer = CfnAuthorizer( self, "ICS_API_GATEWAY_IMAGE_SEARCH_AUTHORIZER", rest_api_id=api_gateway_image_search_resource.api.rest_api_id, name="ICS_API_GATEWAY_IMAGE_SEARCH_AUTHORIZER", type="COGNITO_USER_POOLS", identity_source="method.request.header.Authorization", provider_arns=[users_pool.user_pool_arn]) api_gateway_image_search_resource.add_method( 'POST', image_search_integration, authorization_type=AuthorizationType.COGNITO, method_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': True, } }]).node.find_child('Resource').add_property_override( 'AuthorizerId', api_gateway_image_search_authorizer.ref) lambda_access_search = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=["translate:TranslateText"], resources=["*"]) image_data_function.add_to_role_policy(lambda_access_search) ### custom resource lambda_provider = Provider(self, 'ICS_IMAGE_DATA_PROVIDER', on_event_handler=image_data_function) CustomResource(self, 'ICS_IMAGE_DATA_RESOURCE', service_token=lambda_provider.service_token, pascal_case_properties=False, resource_type="Custom::SchemaCreation", properties={"source": "Cloudformation"}) ### event bridge event_bus = _events.EventBus(self, "ICS_IMAGE_CONTENT_BUS", event_bus_name="ImageContentBus") event_rule = _events.Rule( self, "ICS_IMAGE_CONTENT_RULE", rule_name="ICS_IMAGE_CONTENT_RULE", description="The event from image analyzer to store the data", event_bus=event_bus, event_pattern=_events.EventPattern( resources=[image_analyzer_function.function_arn]), ) event_rule.add_target( _event_targets.LambdaFunction(image_data_function)) event_bus.grant_all_put_events(image_analyzer_function) image_analyzer_function.add_environment("EVENT_BUS", event_bus.event_bus_name) ### outputs CfnOutput( self, 'CognitoHostedUILogin', value= 'https://{}.auth.{}.amazoncognito.com/login?client_id={}&response_type=token&scope={}&redirect_uri={}' .format(user_pool_domain.domain_name, Aws.REGION, user_pool_app_client.ref, '+'.join(user_pool_app_client.allowed_o_auth_scopes), api_gateway.url_for_path('/web')), description='The Cognito Hosted UI Login Page')
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_iam/Role.html iamRole = aws_iam.Role( self, 'aws-synctags-role', role_name=RESOURCE_NAME_PREFIX + '-role', assumed_by=aws_iam.ServicePrincipal('lambda.amazonaws.com'), path='/service-role/') #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_lambda/Function.html lambdaHandler = _lambda.Function(self, 'aws-synctags-event-handler', function_name=RESOURCE_NAME_PREFIX + '-event-handler', code=_lambda.Code.asset('./lambda'), handler='synctags.lambda_handler', timeout=core.Duration.seconds(30), runtime=_lambda.Runtime.PYTHON_3_7, role=iamRole) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_sns/Topic.html snsTopic = aws_sns.Topic(self, 'aws-synctags-topic', topic_name=RESOURCE_NAME_PREFIX + '-topic', display_name=RESOURCE_NAME_PREFIX + '-topic') #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_sns_subscriptions.html snsTopic.add_subscription( aws_sns_subscriptions.EmailSubscription(NOTIFY_EMAIL)) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_iam/Role.html #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_iam/PolicyStatement.html iamRole.attach_inline_policy( aws_iam.Policy( self, 'aws-synctags-policy', policy_name=RESOURCE_NAME_PREFIX + '-role-policy', statements=[ aws_iam.PolicyStatement(resources=[ self.format_arn(service='logs', resource='*') ], actions=['logs:CreateLogGroup']), aws_iam.PolicyStatement( resources=[ self.format_arn(service='logs', resource='log-group', sep=':', resource_name='/aws/lambda/' + RESOURCE_NAME_PREFIX + '-event-handler:*') ], actions=['logs:CreateLogStream', 'logs:PutLogEvents']), aws_iam.PolicyStatement( resources=['*'], actions=[ 'ec2:DescribeInstances', 'ec2:DescribeVolumes', 'ec2:CreateTags', 'ec2:DeleteTags', 'ec2:DescribeNetworkInterfaces', 'ec2:DescribeTags', 'ec2:DescribeSnapshots', 'ec2:DescribeAddresses' ]), aws_iam.PolicyStatement( resources=[snsTopic.topic_arn], actions=['sns:Publish', 'sns:Subscribe']), aws_iam.PolicyStatement(resources=['*'], actions=['iam:ListAccountAliases']) ])) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_events.html eventRule = aws_events.Rule( self, 'aws-synctags-event-rule', rule_name=RESOURCE_NAME_PREFIX + '-event-rule', enabled=True, event_pattern=aws_events.EventPattern( source=['aws.tag'], detail_type=['Tag Change on Resource'], detail={ 'service': ['ec2'], 'resource-type': ['instance'] }), targets=[aws_events_targets.LambdaFunction(lambdaHandler)]) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_lambda/Function.html#aws_cdk.aws_lambda.Function.add_permission #lambdaHandler.add_permission('aws-synctags-lambda-permission', #principal=aws_iam.ServicePrincipal('events.amazonaws.com'), #action='lambda:InvokeFunction', #source_arn=eventRule.rule_arn) lambdaHandler.add_environment('SyncTagKeys', SYNCTAG_KEYS) lambdaHandler.add_environment('NotifyTopicArn', snsTopic.topic_arn)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ### # Let's create our own Event Bus for this rather than using default ### bus = events.EventBus(self, 'DestinedEventBus', event_bus_name='the-destined-lambda') ### # Destinations need invoked Asynchronously so let's use SNS ### topic = sns.Topic(self, 'theDestinedLambdaTopic', display_name='The Destined Lambda CDK Pattern Topic') ### # Lambda configured with success and failure destinations # Note the actual lambda has no EventBridge code inside it ### destined_lambda = _lambda.Function( self, "destinedLambda", runtime=_lambda.Runtime.NODEJS_12_X, handler="destinedLambda.handler", code=_lambda.Code.from_asset("lambdas"), retry_attempts=0, on_success=destinations.EventBridgeDestination(event_bus=bus), on_failure=destinations.EventBridgeDestination(event_bus=bus)) topic.add_subscription( subscriptions.LambdaSubscription(destined_lambda)) ### # This is a lambda that will be called by onSuccess for destinedLambda # It simply prints the event it receives to the cloudwatch logs ### success_lambda = _lambda.Function( self, "successLambda", runtime=_lambda.Runtime.NODEJS_12_X, handler="success.handler", code=_lambda.Code.from_asset("lambdas"), timeout=core.Duration.seconds(3)) ### # EventBridge Rule to send events to our success lambda # Notice how we can still do event filtering based on the json payload returned by the destined lambda ### success_rule = events.Rule( self, 'successRule', event_bus=bus, description= 'all success events are caught here and logged centrally', event_pattern=events.EventPattern( detail={ "requestContext": { "condition": ["Success"] }, "responsePayload": { "source": ["cdkpatterns.the-destined-lambda"], "action": ["message"] } })) success_rule.add_target(targets.LambdaFunction(success_lambda)) ### # This is a lambda that will be called by onFailure for destinedLambda # It simply prints the event it receives to the cloudwatch logs. # Notice how it includes the message that came into destined lambda to make it fail so you have # everything you need to do retries or manually investigate ### failure_lambda = _lambda.Function( self, "failureLambda", runtime=_lambda.Runtime.NODEJS_12_X, handler="failure.handler", code=_lambda.Code.from_asset("lambdas"), timeout=core.Duration.seconds(3)) ### # EventBridge Rule to send events to our failure lambda ### failure_rule = events.Rule( self, 'failureRule', event_bus=bus, description= 'all failure events are caught here and logged centrally', event_pattern=events.EventPattern( detail={"responsePayload": { "errorType": ["Error"] }})) failure_rule.add_target(targets.LambdaFunction(failure_lambda)) ### # API Gateway Creation # This is complicated because it transforms the incoming json payload into a query string url # this url is used to post the payload to sns without a lambda inbetween ### gateway = api_gw.RestApi( self, 'theDestinedLambdaAPI', deploy_options=api_gw.StageOptions( metrics_enabled=True, logging_level=api_gw.MethodLoggingLevel.INFO, data_trace_enabled=True, stage_name='prod')) # Give our gateway permissions to interact with SNS api_gw_sns_role = iam.Role( self, 'ApiGatewaySNSRole', assumed_by=iam.ServicePrincipal('apigateway.amazonaws.com')) topic.grant_publish(api_gw_sns_role) # shortening the lines of later code schema = api_gw.JsonSchema schema_type = api_gw.JsonSchemaType # Because this isn't a proxy integration, we need to define our response model response_model = gateway.add_model( 'ResponseModel', content_type='application/json', model_name='ResponseModel', schema=schema( schema=api_gw.JsonSchemaVersion.DRAFT4, title='pollResponse', type=schema_type.OBJECT, properties={'message': schema(type=schema_type.STRING)})) error_response_model = gateway.add_model( 'ErrorResponseModel', content_type='application/json', model_name='ErrorResponseModel', schema=schema(schema=api_gw.JsonSchemaVersion.DRAFT4, title='errorResponse', type=schema_type.OBJECT, properties={ 'state': schema(type=schema_type.STRING), 'message': schema(type=schema_type.STRING) })) request_template = "Action=Publish&" + \ "TargetArn=$util.urlEncode('" + topic.topic_arn + "')&" + \ "Message=please $input.params().querystring.get('mode')&" + \ "Version=2010-03-31" # This is the VTL to transform the error response error_template = { "state": 'error', "message": "$util.escapeJavaScript($input.path('$.errorMessage'))" } error_template_string = json.dumps(error_template, separators=(',', ':')) # This is how our gateway chooses what response to send based on selection_pattern integration_options = api_gw.IntegrationOptions( credentials_role=api_gw_sns_role, request_parameters={ 'integration.request.header.Content-Type': "'application/x-www-form-urlencoded'" }, request_templates={"application/json": request_template}, passthrough_behavior=api_gw.PassthroughBehavior.NEVER, integration_responses=[ api_gw.IntegrationResponse( status_code='200', response_templates={ "application/json": json.dumps({"message": 'Message added to SNS topic'}) }), api_gw.IntegrationResponse( selection_pattern="^\[Error\].*", status_code='400', response_templates={ "application/json": error_template_string }, response_parameters={ 'method.response.header.Content-Type': "'application/json'", 'method.response.header.Access-Control-Allow-Origin': "'*'", 'method.response.header.Access-Control-Allow-Credentials': "'true'" }) ]) # Add an SendEvent endpoint onto the gateway gateway.root.add_resource('SendEvent') \ .add_method('GET', api_gw.Integration(type=api_gw.IntegrationType.AWS, integration_http_method='POST', uri='arn:aws:apigateway:us-east-1:sns:path//', options=integration_options ), method_responses=[ api_gw.MethodResponse(status_code='200', response_parameters={ 'method.response.header.Content-Type': True, 'method.response.header.Access-Control-Allow-Origin': True, 'method.response.header.Access-Control-Allow-Credentials': True }, response_models={ 'application/json': response_model }), api_gw.MethodResponse(status_code='400', response_parameters={ 'method.response.header.Content-Type': True, 'method.response.header.Access-Control-Allow-Origin': True, 'method.response.header.Access-Control-Allow-Credentials': True }, response_models={ 'application/json': error_response_model }), ] )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Model all required resources ''' Define IAM role that will be used for AWS Lambda Functions ''' lambda_role = _iam.Role( self, id='lab1-bdc-lambda-role', assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com')) ''' Define Amazon EventBridge Construct and also pattern to be included later. ''' eb = _eb.EventBus(self, id="lab1-bdc-eventbus", event_bus_name="lab1-bdc-eventbus") eb_pattern = _eb.EventPattern(detail_type=["message-received"], ) ''' These lines below define construct for our AWS Lambda Functions. There are 2 Lambda functions that we need to create: dispatch and consume. As the dispatch function need to add environment variable, noticed that we add an env_var into AWS Lambda Function that later could be retrieved within the function. ''' fnLambda_dispatch = _lambda.Function( self, "lab1-bdc-function-dispatch", code=_lambda.AssetCode("../lambda-functions/dispatch-function"), handler="app.handler", timeout=core.Duration.seconds(60), role=lambda_role, runtime=_lambda.Runtime.PYTHON_3_8) fnLambda_dispatch.add_environment("EVENT_BUS_NAME", eb.event_bus_name) fnLambda_consume = _lambda.Function( self, "lab1-bdc-function-consume", code=_lambda.AssetCode("../lambda-functions/consume-function"), handler="app.handler", role=lambda_role, timeout=core.Duration.seconds(60), runtime=_lambda.Runtime.PYTHON_3_8) cw_policy_statement = _iam.PolicyStatement(effect=_iam.Effect.ALLOW) cw_policy_statement.add_actions("logs:CreateLogGroup") cw_policy_statement.add_actions("logs:CreateLogStream") cw_policy_statement.add_actions("logs:PutLogEvents") cw_policy_statement.add_actions("logs:DescribeLogStreams") cw_policy_statement.add_resources("*") lambda_role.add_to_policy(cw_policy_statement) eb_policy_statement = _iam.PolicyStatement(effect=_iam.Effect.ALLOW) eb_policy_statement.add_actions("events:PutEvents") eb_policy_statement.add_resources(eb.event_bus_arn) lambda_role.add_to_policy(eb_policy_statement) _eb.Rule(self, id="lab1-bdc-eventRule", description="A basic rule sample", enabled=True, event_bus=eb, event_pattern=eb_pattern, rule_name="BDC-BasicDispatchConsume", targets=[_ebt.LambdaFunction(handler=fnLambda_consume)])
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # If left unchecked this pattern could "fan out" on the transform and load # lambdas to the point that it consumes all resources on the account. This is # why we are limiting concurrency to 2 on all 3 lambdas. Feel free to raise this. lambda_throttle_size = 2 #### # DynamoDB Table # This is where our transformed data ends up #### table = dynamo_db.Table(self, "TransformedData", partition_key=dynamo_db.Attribute( name="id", type=dynamo_db.AttributeType.STRING)) #### # S3 Landing Bucket # This is where the user uploads the file to be transformed #### bucket = s3.Bucket(self, "LandingBucket") #### # Queue that listens for S3 Bucket events #### queue = sqs.Queue(self, 'newObjectInLandingBucketEventQueue', visibility_timeout=core.Duration.seconds(300)) bucket.add_event_notification(s3.EventType.OBJECT_CREATED, s3n.SqsDestination(queue)) # EventBridge Permissions event_bridge_put_policy = iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=['*'], actions=['events:PutEvents']) #### # Fargate ECS Task Creation to pull data from S3 # # Fargate is used here because if you had a seriously large file, # you could stream the data to fargate for as long as needed before # putting the data onto eventbridge or up the memory/storage to # download the whole file. Lambda has limitations on runtime and # memory/storage #### vpc = ec2.Vpc(self, "Vpc", max_azs=2) logging = ecs.AwsLogDriver(stream_prefix='TheEventBridgeETL', log_retention=logs.RetentionDays.ONE_WEEK) cluster = ecs.Cluster(self, 'Ec2Cluster', vpc=vpc) task_definition = ecs.TaskDefinition( self, 'FargateTaskDefinition', memory_mib="512", cpu="256", compatibility=ecs.Compatibility.FARGATE) # We need to give our fargate container permission to put events on our EventBridge task_definition.add_to_task_role_policy(event_bridge_put_policy) # Grant fargate container access to the object that was uploaded to s3 bucket.grant_read(task_definition.task_role) container = task_definition.add_container( 'AppContainer', image=ecs.ContainerImage.from_asset( 'container/s3DataExtractionTask'), logging=logging, environment={ 'S3_BUCKET_NAME': bucket.bucket_name, 'S3_OBJECT_KEY': '' }) #### # Lambdas # # These are used for 4 phases: # # Extract - kicks of ecs fargate task to download data and splinter to eventbridge events # Transform - takes the two comma separated strings and produces a json object # Load - inserts the data into dynamodb # Observe - This is a lambda that subscribes to all events and logs them centrally #### subnet_ids = [] for subnet in vpc.private_subnets: subnet_ids.append(subnet.subnet_id) #### # Extract # defines an AWS Lambda resource to trigger our fargate ecs task #### extract_lambda = _lambda.Function( self, "extractLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="s3SqsEventConsumer.handler", code=_lambda.Code.from_asset("lambdas/extract"), reserved_concurrent_executions=lambda_throttle_size, environment={ "CLUSTER_NAME": cluster.cluster_name, "TASK_DEFINITION": task_definition.task_definition_arn, "SUBNETS": json.dumps(subnet_ids), "CONTAINER_NAME": container.container_name }) queue.grant_consume_messages(extract_lambda) extract_lambda.add_event_source(_event.SqsEventSource(queue=queue)) extract_lambda.add_to_role_policy(event_bridge_put_policy) run_task_policy_statement = iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[task_definition.task_definition_arn], actions=['ecs:RunTask']) extract_lambda.add_to_role_policy(run_task_policy_statement) task_execution_role_policy_statement = iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ task_definition.obtain_execution_role().role_arn, task_definition.task_role.role_arn ], actions=['iam:PassRole']) extract_lambda.add_to_role_policy(task_execution_role_policy_statement) #### # Transform # defines a lambda to transform the data that was extracted from s3 #### transform_lambda = _lambda.Function( self, "TransformLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="transform.handler", code=_lambda.Code.from_asset("lambdas/transform"), reserved_concurrent_executions=lambda_throttle_size, timeout=core.Duration.seconds(3)) transform_lambda.add_to_role_policy(event_bridge_put_policy) # Create EventBridge rule to route extraction events transform_rule = events.Rule( self, 'transformRule', description='Data extracted from S3, Needs transformed', event_pattern=events.EventPattern( source=['cdkpatterns.the-eventbridge-etl'], detail_type=['s3RecordExtraction'], detail={"status": ["extracted"]})) transform_rule.add_target( targets.LambdaFunction(handler=transform_lambda)) #### # Load # load the transformed data in dynamodb #### load_lambda = _lambda.Function( self, "LoadLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="load.handler", code=_lambda.Code.from_asset("lambdas/load"), reserved_concurrent_executions=lambda_throttle_size, timeout=core.Duration.seconds(3), environment={"TABLE_NAME": table.table_name}) load_lambda.add_to_role_policy(event_bridge_put_policy) table.grant_read_write_data(load_lambda) load_rule = events.Rule( self, 'loadRule', description='Data transformed, Needs loaded into dynamodb', event_pattern=events.EventPattern( source=['cdkpatterns.the-eventbridge-etl'], detail_type=['transform'], detail={"status": ["transformed"]})) load_rule.add_target(targets.LambdaFunction(handler=load_lambda)) #### # Observe # Watch for all cdkpatterns.the-eventbridge-etl events and log them centrally #### observe_lambda = _lambda.Function( self, "ObserveLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="observe.handler", code=_lambda.Code.from_asset("lambdas/observe"), reserved_concurrent_executions=lambda_throttle_size, timeout=core.Duration.seconds(3)) observe_rule = events.Rule( self, 'observeRule', description='all events are caught here and logged centrally', event_pattern=events.EventPattern( source=['cdkpatterns.the-eventbridge-etl'])) observe_rule.add_target(targets.LambdaFunction(handler=observe_lambda))
def __init__( self, scope: core.Construct, construct_id: str, # deployment_asset: s3_assets.Asset, **kwargs, ) -> None: super().__init__(scope, construct_id, **kwargs) # Create Required parameters for sagemaker projects # see: https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-projects-templates-custom.html # see also: # https://docs.aws.amazon.com/cdk/latest/guide/parameters.html project_name = core.CfnParameter( self, "SageMakerProjectName", type="String", description="The name of the SageMaker project.", min_length=1, max_length=32, ) project_id = core.CfnParameter( self, "SageMakerProjectId", type="String", min_length=1, max_length=16, description="Service generated Id of the project.", ) stage_name = core.CfnParameter( self, "StageName", type="String", min_length=1, max_length=8, description="The stage name.", default="dev", ) seed_bucket = core.CfnParameter( self, "CodeCommitSeedBucket", type="String", description="The optional s3 seed bucket", min_length=1, ) seed_key = core.CfnParameter( self, "CodeCommitSeedKey", type="String", description="The optional s3 seed key", min_length=1, ) register_lambda = core.CfnParameter( self, "RegisterLambda", type="String", description="The AWS Lambda to invoke when registering this model", min_length=1, ) # Get the service catalog role for all permssions (if None CDK will create new roles) # CodeBuild and CodePipeline resources need to start with "sagemaker-" to be within default policy service_catalog_role = aws_iam.Role.from_role_arn( self, "PipelineRole", f"arn:{self.partition}:iam::{self.account}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", ) # Define the repository name and branch branch_name = "main" # Create source repo from seed bucket/key repo = codecommit.CfnRepository( self, "CodeRepo", repository_name="sagemaker-{}-repo".format( project_name.value_as_string), repository_description="Amazon SageMaker A/B testing pipeline", code=codecommit.CfnRepository.CodeProperty( s3=codecommit.CfnRepository.S3Property( bucket=seed_bucket.value_as_string, key=seed_key.value_as_string, object_version=None, ), branch_name=branch_name, ), tags=[ core.CfnTag(key="sagemaker:deployment-stage", value=stage_name.value_as_string), core.CfnTag(key="sagemaker:project-id", value=project_id.value_as_string), core.CfnTag(key="sagemaker:project-name", value=project_name.value_as_string), ], ) # Reference the newly created repository code = codecommit.Repository.from_repository_name( self, "ImportedRepo", repo.attr_name) cdk_build = codebuild.PipelineProject( self, "CdkBuild", project_name="sagemaker-{}-cdk-{}".format( project_name.value_as_string, stage_name.value_as_string), role=service_catalog_role, build_spec=codebuild.BuildSpec.from_object( dict( version="0.2", phases=dict( install=dict(commands=[ "npm install aws-cdk", "npm update", "python -m pip install -r requirements.txt", ]), build=dict(commands=[ "npx cdk synth -o dist --path-metadata false", ]), ), artifacts={ "base-directory": "dist", "files": ["*.template.json"], }, environment=dict( buildImage=codebuild.LinuxBuildImage.AMAZON_LINUX_2_3, ), )), environment_variables={ "SAGEMAKER_PROJECT_NAME": codebuild.BuildEnvironmentVariable( value=project_name.value_as_string), "SAGEMAKER_PROJECT_ID": codebuild.BuildEnvironmentVariable( value=project_id.value_as_string), "STAGE_NAME": codebuild.BuildEnvironmentVariable( value=stage_name.value_as_string), }, ) register_build = codebuild.PipelineProject( self, "RegisterBuild", project_name="sagemaker-{}-register-{}".format( project_name.value_as_string, stage_name.value_as_string), role=service_catalog_role, build_spec=codebuild.BuildSpec.from_object( dict( version="0.2", phases=dict(build=dict(commands=[ "python register.py > output.txt", ]), ), artifacts={ "files": ["output.txt"], }, environment=dict( buildImage=codebuild.LinuxBuildImage.AMAZON_LINUX_2_3, ), )), environment_variables={ "SAGEMAKER_PROJECT_NAME": codebuild.BuildEnvironmentVariable( value=project_name.value_as_string), "STAGE_NAME": codebuild.BuildEnvironmentVariable( value=stage_name.value_as_string), "REGISTER_LAMBDA": codebuild.BuildEnvironmentVariable( value=register_lambda.value_as_string), }, ) source_output = codepipeline.Artifact() cdk_build_output = codepipeline.Artifact() register_build_output = codepipeline.Artifact() # Create the s3 artifact (name must be < 63 chars) s3_artifact = s3.Bucket( self, "S3Artifact", bucket_name="sagemaker-{}-artifact-{}-{}".format( project_id.value_as_string, stage_name.value_as_string, self.region), removal_policy=core.RemovalPolicy.DESTROY, ) deploy_pipeline = codepipeline.Pipeline( self, "Pipeline", role=service_catalog_role, artifact_bucket=s3_artifact, pipeline_name="sagemaker-{}-pipeline-{}".format( project_name.value_as_string, stage_name.value_as_string), stages=[ codepipeline.StageProps( stage_name="Source", actions=[ codepipeline_actions.CodeCommitSourceAction( action_name="CodeCommit_Source", repository=code, trigger=codepipeline_actions.CodeCommitTrigger. NONE, # Created below event_role=service_catalog_role, output=source_output, branch=branch_name, role=service_catalog_role, ) ], ), codepipeline.StageProps( stage_name="Build", actions=[ codepipeline_actions.CodeBuildAction( action_name="CDK_Build", project=cdk_build, input=source_output, outputs=[ cdk_build_output, ], role=service_catalog_role, ), ], ), codepipeline.StageProps( stage_name="Deploy", actions=[ codepipeline_actions. CloudFormationCreateUpdateStackAction( action_name="SageMaker_CFN_Deploy", run_order=1, template_path=cdk_build_output.at_path( "ab-testing-sagemaker.template.json"), stack_name="sagemaker-{}-deploy-{}".format( project_name.value_as_string, stage_name.value_as_string), admin_permissions=False, role=service_catalog_role, deployment_role=service_catalog_role, replace_on_failure=True, ), ], ), codepipeline.StageProps( stage_name="Register", actions=[ codepipeline_actions.CodeBuildAction( action_name="Register_Build", project=register_build, input=source_output, outputs=[ register_build_output, ], role=service_catalog_role, ), ], ), ], ) # Add deploy role to target the code pipeline when model package is approved deploy_rule = events.Rule( self, "DeployRule", rule_name="sagemaker-{}-model-{}".format( project_name.value_as_string, stage_name.value_as_string), description= "Rule to trigger a deployment when SageMaker Model registry is updated with a new model package. For example, a new model package is registered with Registry", event_pattern=events.EventPattern( source=["aws.sagemaker"], detail_type=["SageMaker Model Package State Change"], detail={ "ModelPackageGroupName": [ f"{project_name.value_as_string}-champion", f"{project_name.value_as_string}-challenger", ] }, ), targets=[ targets.CodePipeline( pipeline=deploy_pipeline, event_role=service_catalog_role, ) ], ) code_rule = events.Rule( self, "CodeRule", rule_name="sagemaker-{}-code-{}".format( project_name.value_as_string, stage_name.value_as_string), description= "Rule to trigger a deployment when SageMaker Model registry is updated with a new model package. For example, a new model package is registered with Registry", event_pattern=events.EventPattern( source=["aws.codecommit"], detail_type=["CodeCommit Repository State Change"], detail={ "event": ["referenceCreated", "referenceUpdated"], "referenceType": ["branch"], "referenceName": [branch_name], }, resources=[code.repository_arn], ), targets=[ targets.CodePipeline( pipeline=deploy_pipeline, event_role=service_catalog_role, ) ], )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) dynamodb_table = _ddb.Table( self, id="lab2-cm-ddb", table_name="lab2-cm-order-status", partition_key=Attribute(name='ID', type=AttributeType.STRING), removal_policy=core.RemovalPolicy.DESTROY # NOT for production ) eb = _eb.EventBus(self, id="lab2-cm-eventbus", event_bus_name="lab2-cm-eventbus") lambda_role = _iam.Role( self, id='lab2-cm-role', assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com')) dynamodb_policy_statement = _iam.PolicyStatement( effect=_iam.Effect.ALLOW) dynamodb_policy_statement.add_actions("dynamodb:*") dynamodb_policy_statement.add_resources(dynamodb_table.table_arn) lambda_role.add_to_policy(dynamodb_policy_statement) eventbridge_policy_statement = _iam.PolicyStatement( effect=_iam.Effect.ALLOW) eventbridge_policy_statement.add_actions("events:*") eventbridge_policy_statement.add_resources(eb.event_bus_arn) lambda_role.add_to_policy(eventbridge_policy_statement) cloudwatch_policy_statement = _iam.PolicyStatement( effect=_iam.Effect.ALLOW) cloudwatch_policy_statement.add_actions("logs:CreateLogGroup") cloudwatch_policy_statement.add_actions("logs:CreateLogStream") cloudwatch_policy_statement.add_actions("logs:PutLogEvents") cloudwatch_policy_statement.add_actions("logs:DescribeLogStreams") cloudwatch_policy_statement.add_resources("*") lambda_role.add_to_policy(cloudwatch_policy_statement) fn_lambda_invoice_service = aws_lambda.Function( self, "lab2-cm-invoiceService", code=aws_lambda.AssetCode("../lambda-functions/invoice-service/"), handler="app.lambda_handler", tracing=aws_lambda.Tracing.ACTIVE, timeout=core.Duration.seconds(30), role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) fn_lambda_invoice_service.add_environment("TABLE_NAME", dynamodb_table.table_name) fn_lambda_fulfilment_service = aws_lambda.Function( self, "lab2-cm-fulfilmentService", code=aws_lambda.AssetCode( "../lambda-functions/fulfilment-service/"), handler="app.lambda_handler", tracing=aws_lambda.Tracing.ACTIVE, timeout=core.Duration.seconds(30), role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) fn_lambda_fulfilment_service.add_environment("TABLE_NAME", dynamodb_table.table_name) fn_lambda_fulfilment_service.add_environment("EVENTBUS_NAME", eb.event_bus_name) fn_lambda_forecasting_service = aws_lambda.Function( self, "lab2-cm-forecastingService", code=aws_lambda.AssetCode( "../lambda-functions/forecasting-service/"), handler="app.lambda_handler", tracing=aws_lambda.Tracing.ACTIVE, timeout=core.Duration.seconds(30), role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) fn_lambda_forecasting_service.add_environment( "TABLE_NAME", dynamodb_table.table_name) fn_lambda_order_service = aws_lambda.Function( self, "lab2-cm-orderService", code=aws_lambda.AssetCode("../lambda-functions/order-service/"), handler="app.lambda_handler", timeout=core.Duration.seconds(30), tracing=aws_lambda.Tracing.ACTIVE, role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) fn_lambda_order_service.add_environment("TABLE_NAME", dynamodb_table.table_name) fn_lambda_order_service.add_environment("EVENTBUS_NAME", eb.event_bus_name) fn_lambda_logistic_service = aws_lambda.Function( self, "lab2-cm-logisticService", code=aws_lambda.AssetCode("../lambda-functions/logistic-service/"), handler="app.lambda_handler", timeout=core.Duration.seconds(30), tracing=aws_lambda.Tracing.ACTIVE, role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) fn_lambda_logistic_service.add_environment("TABLE_NAME", dynamodb_table.table_name) eb_order_created_pattern = _eb.EventPattern( detail_type=["order_created"], ) eb_fulfilment_completed_pattern = _eb.EventPattern( detail_type=["fulfilment_completed"], ) eb_order_created_rule = _eb.Rule( self, id="lab2-cm-eventRule-order-created", description="Order created event", enabled=True, event_bus=eb, event_pattern=eb_order_created_pattern, rule_name="lab2-OrderCreated", targets=[ _ebt.LambdaFunction(handler=fn_lambda_invoice_service), _ebt.LambdaFunction(handler=fn_lambda_fulfilment_service), _ebt.LambdaFunction(handler=fn_lambda_forecasting_service) ]) eb_fulfilment_completed_rule = _eb.Rule( self, id="lab2-cm-eventRule-fulfilment-completed", description="Fulfilment completedevent", enabled=True, event_bus=eb, event_pattern=eb_fulfilment_completed_pattern, rule_name="lab2-FulfilmentCompleted", targets=[_ebt.LambdaFunction(handler=fn_lambda_logistic_service)]) api = _ag.RestApi( self, id='lab2-cm-api-gateway', ) api_lambda_integration = _ag.LambdaIntegration(fn_lambda_order_service) api.root.add_resource('order').add_method('GET', api_lambda_integration)
def __init__(self, scope: core.Construct, id: str, params: dict, **kwargs): # pylint: disable=W0622 super().__init__(scope, id, **kwargs) # pylint: disable=no-value-for-parameter stack = core.Stack.of(self) _pipeline = _codepipeline.Pipeline.from_pipeline_arn( self, 'pipeline', params["pipeline"]) _source_account = params.get('source', {}).get('account_id') _provider = params.get('source', {}).get('provider') _add_trigger_on_changes = ( _provider == 'codecommit' and _source_account and params.get('source', {}).get('trigger_on_changes') and not params.get('source', {}).get('poll_for_changes')) name = params.get('name') account_id = params['source']['account_id'] repo_name = params['source']['repo_name'] if _add_trigger_on_changes: _event = _events.Rule( self, f'trigger_{name}', description= f'Triggers {name} on changes in source CodeCommit repository', event_pattern=_events. EventPattern(resources=[ f'arn:{stack.partition}:codecommit:{ADF_DEPLOYMENT_REGION}:{account_id}:{repo_name}' ], source=["aws.codecommit"], detail_type=[ 'CodeCommit Repository State Change' ], detail={ "event": ["referenceCreated", "referenceUpdated"], "referenceType": ["branch"], "referenceName": [params['source']['branch']] })) _event.add_target(_targets.CodePipeline(pipeline=_pipeline)) if params.get('topic_arn'): # pylint: disable=no-value-for-parameter _topic = _sns.Topic.from_topic_arn(self, 'topic_arn', params["topic_arn"]) _event = _events.Rule( self, f'pipeline_state_{name}', description= f"{name} | Trigger notifications based on pipeline state changes", enabled=True, event_pattern=_events.EventPattern( detail={ "state": ["FAILED", "STARTED", "SUCCEEDED"], "pipeline": [ f"{ADF_PIPELINE_PREFIX}{name}", ] }, detail_type=[ "CodePipeline Pipeline Execution State Change" ], source=["aws.codepipeline"])) _event.add_target( _targets.SnsTopic( topic=_topic, message=_events.RuleTargetInput.from_text( # Need to parse and get the pipeline: "$.detail.pipeline" state: "$.detail.state" f"The pipeline {_events.EventField.from_path('$.detail.pipeline')} " f"from account {_events.EventField.account} " f"has {_events.EventField.from_path('$.detail.state')} " f"at {_events.EventField.time}."))) if params.get('completion_trigger'): # There might be other types of completion triggers later, eg lambda.. for index, pipeline in enumerate(params['completion_trigger'].get( 'pipelines', [])): _event = _events.Rule( self, f'completion_{pipeline}', description= "Triggers {pipeline} on completion of {params['pipeline']}", enabled=True, event_pattern=_events.EventPattern( detail={ "state": ["SUCCEEDED"], "pipeline": [ f"{ADF_PIPELINE_PREFIX}{name}", ] }, detail_type=[ "CodePipeline Pipeline Execution State Change" ], source=["aws.codepipeline"])) # pylint: disable=no-value-for-parameter _completion_pipeline = _codepipeline.Pipeline.from_pipeline_arn( self, f'pipeline-{index}', f'arn:{stack.partition}:codepipeline:' f'{ADF_DEPLOYMENT_REGION}:{ADF_DEPLOYMENT_ACCOUNT_ID}:' f'{ADF_PIPELINE_PREFIX}{pipeline}') _event.add_target( _targets.CodePipeline(pipeline=_completion_pipeline)) if params.get('schedule'): _event = _events.Rule( self, f'schedule_{params["name"]}', description= f"Triggers {params['name']} on a schedule of {params['schedule']}", enabled=True, # pylint: disable=no-value-for-parameter schedule=_events.Schedule.expression(params['schedule'])) _target_pipeline = _targets.CodePipeline(pipeline=_pipeline) _event.add_target(_target_pipeline)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Parameteres # notification_email = create_notification_email_parameter(self) assets_bucket_name = create_assets_bucket_name_parameter(self) custom_container = create_custom_container_parameter(self) ecr_repo_name = create_ecr_repo_name_parameter(self) image_tag = create_image_tag_parameter(self) # Resources # assets_bucket = s3.Bucket.from_bucket_name( self, "AssetsBucket", assets_bucket_name.value_as_string) # Defining pipeline stages # source stage source_output, source_action_definition = source_action_custom( assets_bucket, custom_container) # build stage build_action_definition, container_uri = build_action( self, ecr_repo_name.value_as_string, image_tag.value_as_string, source_output) pipeline_notification_topic = sns.Topic( self, "PipelineNotification", ) pipeline_notification_topic.node.default_child.cfn_options.metadata = suppress_sns( ) pipeline_notification_topic.add_subscription( subscriptions.EmailSubscription( email_address=notification_email.value_as_string)) # createing pipeline stages source_stage = codepipeline.StageProps( stage_name="Source", actions=[source_action_definition]) build_stage = codepipeline.StageProps( stage_name="Build", actions=[build_action_definition]) image_builder_pipeline = codepipeline.Pipeline( self, "BYOMPipelineReatimeBuild", stages=[source_stage, build_stage], cross_account_keys=False, ) image_builder_pipeline.on_state_change( "NotifyUser", description="Notify user of the outcome of the pipeline", target=targets.SnsTopic( pipeline_notification_topic, message=events.RuleTargetInput.from_text(( f"Pipeline {events.EventField.from_path('$.detail.pipeline')} finished executing. " f"Pipeline execution result is {events.EventField.from_path('$.detail.state')}" )), ), event_pattern=events.EventPattern( detail={"state": ["SUCCEEDED", "FAILED"]}), ) image_builder_pipeline.add_to_role_policy( iam.PolicyStatement( actions=["events:PutEvents"], resources=[ f"arn:{core.Aws.PARTITION}:events:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:event-bus/*", ], )) # add cfn nag supressions pipeline_child_nodes = image_builder_pipeline.node.find_all() pipeline_child_nodes[ 1].node.default_child.cfn_options.metadata = suppress_pipeline_bucket( ) pipeline_child_nodes[ 6].node.default_child.cfn_options.metadata = suppress_iam_complex( ) # attaching iam permissions to the pipelines pipeline_permissions(image_builder_pipeline, assets_bucket) # Outputs # core.CfnOutput( self, id="Pipelines", value= (f"https://console.aws.amazon.com/codesuite/codepipeline/pipelines/" f"{image_builder_pipeline.pipeline_name}/view?region={core.Aws.REGION}" ), ) core.CfnOutput( self, id="CustomAlgorithmImageURI", value=container_uri, )
def __init__(self, scope: core.Construct, id: str, group_name: str, minute_duration: int, **kwargs) -> None: super().__init__(scope, id, **kwargs) # TODO: Setup alerting of failure to an SNS # TODO: Failure is not the same as a student not in a group # TODO: Streamline input data so that lambda's only get the info they really need # TODO: Comment # TODO: Need to separate unexpected errors from regular errors # Setting up monitoring schedule_stop = lambda_.Function( self, id="ScheduleStopLambda", runtime=lambda_.Runtime.PYTHON_3_7, code=lambda_.Code.from_inline( open("./resources/schedule-termination.py", 'r').read()), handler="index.handler", log_retention=logs.RetentionDays.ONE_DAY, environment=dict(GROUP_NAME=group_name), timeout=core.Duration.seconds(30)) schedule_stop.add_to_role_policy( statement=iam.PolicyStatement(actions=[ "ec2:Describe*", "iam:ListGroupsForUser", "iam:ListUsers" ], effect=iam.Effect.ALLOW, resources=["*"])) terminate_ec2 = lambda_.Function( self, id="TerminateEC2", runtime=lambda_.Runtime.PYTHON_3_7, code=lambda_.Code.from_inline( open("./resources/terminate-ec2.py", 'r').read()), handler="index.handler", log_retention=logs.RetentionDays.ONE_DAY, timeout=core.Duration.seconds(30)) terminate_ec2.add_to_role_policy( statement=iam.PolicyStatement(actions=[ "ec2:DescribeInstance*", "ec2:TerminateInstances", ], effect=iam.Effect.ALLOW, resources=["*"])) # The lambda object that will see if we should schedule. schedule_stop_task = tasks.LambdaInvoke( self, id='schedule stop', lambda_function=schedule_stop, input_path="$.detail.userIdentity", result_path="$.Payload", ) # TODO: Need to change this based on the configuration info above # Wait state to try and delete # wait_x = sfn.Wait(self, 'Wait x minutes', time=sfn.WaitTime.seconds_path("10")) wait_x = sfn.Wait(self, id='Wait x minutes', time=sfn.WaitTime.duration( core.Duration.minutes(minute_duration))) job_failed = sfn.Fail(self, id="Failed Job", cause="Error in the input", error="Error") job_finished = sfn.Succeed(self, id="Job Finished") choice = sfn.Choice(self, 'Can I delete') choice.when(sfn.Condition.boolean_equals('$.Payload.Payload', False), job_finished) choice.otherwise(wait_x) terminate_ec2_task = tasks.LambdaInvoke( self, 'terminate', lambda_function=terminate_ec2, input_path="$.detail.responseElements.instancesSet") wait_x.next(terminate_ec2_task).next(job_finished) state_definition = schedule_stop_task \ .next(choice) terminate_machine = sfn.StateMachine(self, id="State Machine", definition=state_definition) cloudwatch.Alarm(self, "EC2ScheduleAlarm", metric=terminate_machine.metric_failed(), threshold=1, evaluation_periods=1) # TODO Build Rule that monitors for EC2 creation # Any new creation, the EC2 will have to be destroyed. Including # other things? create_event = events.Rule( self, id='detect-ec2-start', description="Detects if an EC2 is created", enabled=True, event_pattern=events.EventPattern( detail_type=["AWS API Call via CloudTrail"], source=["aws.ec2"], detail={ "eventName": ["RunInstances"], "eventSource": ["ec2.amazonaws.com"] }), targets=[targets.SfnStateMachine(terminate_machine)])
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Parameteres # notification_email = core.CfnParameter( self, "NOTIFICATION_EMAIL", type="String", description="email for pipeline outcome notifications", allowed_pattern="^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$", constraint_description= "Please enter an email address with correct format ([email protected])", min_length=5, max_length=320, ) blueprint_bucket_name = core.CfnParameter( self, "BLUEPRINT_BUCKET", type="String", description= "Bucket name for blueprints of different types of ML Pipelines.", min_length=3, ) assets_bucket_name = core.CfnParameter( self, "ASSETS_BUCKET", type="String", description="Bucket name for access logs.", min_length=3) custom_container = core.CfnParameter( self, "CUSTOM_CONTAINER", default="", type="String", description= ("Should point to a zip file containing dockerfile and assets for building a custom model. " "If empty it will beusing containers from SageMaker Registry"), ) model_framework = core.CfnParameter( self, "MODEL_FRAMEWORK", default="", type="String", description= "The ML framework which is used for training the model. E.g., xgboost, kmeans, etc.", ) model_framework_version = core.CfnParameter( self, "MODEL_FRAMEWORK_VERSION", default="", type="String", description= "The version of the ML framework which is used for training the model. E.g., 1.1-2", ) model_name = core.CfnParameter( self, "MODEL_NAME", type="String", description="An arbitrary name for the model.", min_length=1) model_artifact_location = core.CfnParameter( self, "MODEL_ARTIFACT_LOCATION", type="String", description="Path to model artifact inside assets bucket.", ) inference_instance = core.CfnParameter( self, "INFERENCE_INSTANCE", type="String", description= "Inference instance that inference requests will be running on. E.g., ml.m5.large", allowed_pattern="^[a-zA-Z0-9_.+-]+\.[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$", min_length=7, ) inference_type = core.CfnParameter( self, "INFERENCE_TYPE", type="String", allowed_values=["batch", "realtime"], default="realtime", description="Type of inference. Possible values: batch | realtime", ) batch_inference_data = core.CfnParameter( self, "BATCH_INFERENCE_DATA", type="String", default="", description= ("Location of batch inference data if inference type is set to batch. Otherwise, can be left empty." ), ) # Resources # assets_bucket = s3.Bucket.from_bucket_name( self, "AssetsBucket", assets_bucket_name.value_as_string) # getting blueprint bucket object from its name - will be used later in the stack blueprint_bucket = s3.Bucket.from_bucket_name( self, "BlueprintBucket", blueprint_bucket_name.value_as_string) # Defining pipeline stages # source stage source_output, source_action_definition = source_action_custom( model_artifact_location, assets_bucket, custom_container) # build stage build_action_definition, container_uri = build_action( self, source_output) # deploy stage sm_layer = sagemaker_layer(self, blueprint_bucket) # creating a sagemaker model model_lambda_arn, create_model_definition = create_model( self, blueprint_bucket, assets_bucket, model_name, model_artifact_location, custom_container, model_framework, model_framework_version, container_uri, sm_layer, ) # creating a batch transform job batch_lambda_arn, batch_transform_definition = batch_transform( self, blueprint_bucket, assets_bucket, model_name, inference_instance, batch_inference_data, sm_layer, ) # create invoking lambda policy invoke_lambdas_policy = iam.PolicyStatement( actions=[ "lambda:InvokeFunction", ], resources=[model_lambda_arn, batch_lambda_arn], ) pipeline_notification_topic = sns.Topic( self, "PipelineNotification", ) pipeline_notification_topic.node.default_child.cfn_options.metadata = suppress_sns( ) pipeline_notification_topic.add_subscription( subscriptions.EmailSubscription( email_address=notification_email.value_as_string)) # createing pipeline stages source_stage = codepipeline.StageProps( stage_name="Source", actions=[source_action_definition]) build_stage = codepipeline.StageProps( stage_name="Build", actions=[build_action_definition]) deploy_stage_batch = codepipeline.StageProps( stage_name="Deploy", actions=[create_model_definition, batch_transform_definition], ) batch_build_pipeline = codepipeline.Pipeline( self, "BYOMPipelineBatchBuild", stages=[source_stage, build_stage, deploy_stage_batch], cross_account_keys=False, ) batch_build_pipeline.on_state_change( "NotifyUser", description="Notify user of the outcome of the pipeline", target=targets.SnsTopic( pipeline_notification_topic, message=events.RuleTargetInput.from_text(( f"Pipeline {events.EventField.from_path('$.detail.pipeline')} finished executing. " f"Pipeline execution result is {events.EventField.from_path('$.detail.state')}" )), ), event_pattern=events.EventPattern( detail={"state": ["SUCCEEDED", "FAILED"]}), ) batch_build_pipeline.add_to_role_policy( iam.PolicyStatement( actions=["events:PutEvents"], resources=[ f"arn:{core.Aws.PARTITION}:events:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:event-bus/*", ], )) # add lambda permissons batch_build_pipeline.add_to_role_policy(invoke_lambdas_policy) # Enhancement: This is to find CDK object nodes so that unnecessary cfn-nag warnings can be suppressed # There is room for improving the method in future versions to find CDK nodes without having to use # hardocded index numbers pipeline_child_nodes = batch_build_pipeline.node.find_all() pipeline_child_nodes[ 1].node.default_child.cfn_options.metadata = suppress_pipeline_bucket( ) pipeline_child_nodes[ 6].node.default_child.cfn_options.metadata = suppress_iam_complex( ) pipeline_child_nodes[ 13].node.default_child.cfn_options.metadata = suppress_list_function_policy( ) pipeline_child_nodes[ 19].node.default_child.cfn_options.metadata = suppress_list_function_policy( ) pipeline_child_nodes[ 25].node.default_child.cfn_options.metadata = suppress_list_function_policy( ) pipeline_child_nodes[ 30].node.default_child.cfn_options.metadata = suppress_list_function_policy( ) # attaching iam permissions to the pipelines pipeline_permissions(batch_build_pipeline, assets_bucket) core.CfnOutput( self, id="Pipelines", value= (f"https://console.aws.amazon.com/codesuite/codepipeline/pipelines/" f"{batch_build_pipeline.pipeline_name}/view?region={core.Aws.REGION}" ), ) core.CfnOutput( self, id="BatchTransformOutputLocation", value= f"https://s3.console.aws.amazon.com/s3/buckets/{assets_bucket.bucket_name}/batch_transform/output", description= "Output location of the batch transform. Output will be saved under the job name", )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Parameteres # notification_email = create_notification_email_parameter(self) template_zip_name = create_template_zip_name_parameter(self) template_file_name = create_template_file_name_parameter(self) template_params_file_name = create_stage_params_file_name_parameter( self, "TEMPLATE_PARAMS_NAME", "main") assets_bucket_name = create_assets_bucket_name_parameter(self) stack_name = create_stack_name_parameter(self) # Resources # assets_bucket = s3.Bucket.from_bucket_name( self, "AssetsBucket", assets_bucket_name.value_as_string) # create sns topic and subscription pipeline_notification_topic = sns.Topic( self, "SinglePipelineNotification", ) pipeline_notification_topic.node.default_child.cfn_options.metadata = suppress_sns( ) pipeline_notification_topic.add_subscription( subscriptions.EmailSubscription( email_address=notification_email.value_as_string)) # Defining pipeline stages # source stage source_output, source_action_definition = source_action_template( template_zip_name, assets_bucket) # create cloudformation action cloudformation_action = create_cloudformation_action( self, "deploy_stack", stack_name.value_as_string, source_output, template_file_name.value_as_string, template_params_file_name.value_as_string, ) source_stage = codepipeline.StageProps( stage_name="Source", actions=[source_action_definition]) deploy = codepipeline.StageProps( stage_name="DeployCloudFormation", actions=[cloudformation_action], ) single_account_pipeline = codepipeline.Pipeline( self, "SingleAccountPipeline", stages=[source_stage, deploy], cross_account_keys=False, ) # Add CF suppressions to the action deployment_policy = cloudformation_action.deployment_role.node.find_all( )[2] deployment_policy.node.default_child.cfn_options.metadata = suppress_cloudformation_action( ) # add notification to the single-account pipeline single_account_pipeline.on_state_change( "NotifyUser", description="Notify user of the outcome of the pipeline", target=targets.SnsTopic( pipeline_notification_topic, message=events.RuleTargetInput.from_text(( f"Pipeline {events.EventField.from_path('$.detail.pipeline')} finished executing. " f"Pipeline execution result is {events.EventField.from_path('$.detail.state')}" )), ), event_pattern=events.EventPattern( detail={"state": ["SUCCEEDED", "FAILED"]}), ) single_account_pipeline.add_to_role_policy( iam.PolicyStatement( actions=["events:PutEvents"], resources=[ f"arn:{core.Aws.PARTITION}:events:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:event-bus/*", ], )) # add cfn supressions pipeline_child_nodes = single_account_pipeline.node.find_all() pipeline_child_nodes[ 1].node.default_child.cfn_options.metadata = suppress_pipeline_bucket( ) pipeline_child_nodes[ 6].node.default_child.cfn_options.metadata = suppress_iam_complex( ) # attaching iam permissions to the pipelines pipeline_permissions(single_account_pipeline, assets_bucket) # Outputs # core.CfnOutput( self, id="Pipelines", value= (f"https://console.aws.amazon.com/codesuite/codepipeline/pipelines/" f"{single_account_pipeline.pipeline_name}/view?region={core.Aws.REGION}" ), )