def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ### VPC # create a new VPC with 2 AZ's and two NAT gateways # TODO - include VPC SQS endpoint so the NAT gateway isn't needed anymore vpc = aws_ec2.Vpc(self, "Vpc", max_azs=2, nat_gateways=2, subnet_configuration=[ aws_ec2.SubnetConfiguration( name="private", cidr_mask=24, subnet_type=aws_ec2.SubnetType.PRIVATE), aws_ec2.SubnetConfiguration( name="public", cidr_mask=28, subnet_type=aws_ec2.SubnetType.PUBLIC) ]) # create a new ECS cluster cluster = aws_ecs.Cluster(self, "FargateSQS", vpc=vpc) ### SQS # create a new SQS queue msg_queue = aws_sqs.Queue(self, "SQSQueue", visibility_timeout=core.Duration.seconds(0), retention_period=core.Duration.minutes(30)) ### FARGATE # build the docker image from local "./docker" directory sqscontainer = aws_ecs.ContainerImage.from_asset(directory="docker") # add the aws-xray-daemon as a sidecar running on UDP/2000 xraycontainer = aws_ecs.ContainerImage.from_registry( "amazon/aws-xray-daemon") # create the queue processing service on fargate with a locally built container # the pattern automatically adds an environment variable with the queue name for the container to read fargate_service = aws_ecs_patterns.QueueProcessingFargateService( self, "Service", cluster=cluster, memory_limit_mib=512, cpu=256, image=sqscontainer, enable_logging=True, desired_task_count=0, max_scaling_capacity=5, scaling_steps= [{ "upper": 0, "change": -5 }, { "lower": 1, "change": +1 } # disabled metric based scaling to test scaling on cpu usage only # this may potentially lower cost as fargate will scale in smaller steps #{"lower": 50000, "change": +2}, #{"lower": 250000, "change": +4} ], queue=msg_queue, environment={"sqs_queue_url": msg_queue.queue_url}) # add the standard aws xray sidecar to the container task xray_sidecar = fargate_service.task_definition.add_container( "xraycontainer", image=xraycontainer, logging=fargate_service.log_driver) # expose the sidecar on port UDP/2000 xray_sidecar.add_port_mappings( aws_ecs.PortMapping(container_port=2000, protocol=aws_ecs.Protocol.UDP)) ### LAMBDA # build the go binary for the lambda SQS generator and retrieve the unix timestamp of when the file was modified # since CDK cannot natively build Go binaries yet, we need to do this manually through build_lambda_zip.py os.system("python loadgen/build_lambda_zip.py") filets = str(int(os.path.getctime("./loadgen/lambda.zip"))) # create a lambda function to generate load, using the filets value as a source hash for the zip sqs_lambda = aws_lambda.Function( self, "GenerateLoadSQS", runtime=aws_lambda.Runtime.GO_1_X, code=aws_lambda.Code.from_asset("./loadgen/lambda.zip", source_hash=filets), handler="loadgensqs", timeout=core.Duration.seconds(20), memory_size=128, retry_attempts=0, tracing=aws_lambda.Tracing.ACTIVE, environment={ "sqs_queue_url": msg_queue.queue_url, "total_message_count": "100" }) ### CLOUDWATCH RULE # create a new cloudwatch rule running every minute to trigger the lambda function eventRuleMinu = aws_events.Rule( self, "lambda-generator-minute-rule", enabled=True, schedule=aws_events.Schedule.cron(minute="*")) eventRuleMinu.add_target(aws_events_targets.LambdaFunction(sqs_lambda)) ### IAM # add the Lambda IAM permission to send SQS messages msg_queue.grant_send_messages(sqs_lambda) # add XRay permissions to Fargate task and Lambda xray_policy = PolicyStatement(resources=["*"], actions=[ "xray:GetGroup", "xray:GetGroups", "xray:GetSampling*", "xray:GetTime*", "xray:GetService*", "xray:PutTelemetryRecords", "xray:PutTraceSegments" ]) fargate_service.task_definition.add_to_task_role_policy(xray_policy) sqs_lambda.add_to_role_policy(xray_policy)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) with open("index.py", encoding="utf8") as fp: handler_code = fp.read() role = iam.Role( self, 'myappRole', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com')) role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=["*"], actions=['events:*'])) role.add_to_policy( iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["arn:aws:iam::*:role/AWS_Events_Invoke_Targets"], actions=['iam:PassRole'])) role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=["*"], actions=[ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents" ])) role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=["*"], actions=["s3:*"])) role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=["*"], actions=["lambda:*"])) role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=["*"], actions=["sns:*"])) lambdaFn = lambdas.Function(self, "Singleton", code=lambdas.InlineCode(handler_code), handler="index.lambda_handler", timeout=core.Duration.seconds(600), runtime=lambdas.Runtime.PYTHON_3_6, memory_size=512, environment=dict(PATH="/opt"), role=role) rule = events.Rule( self, "Rule", schedule=events.Schedule.cron(minute='59', hour='6-20/4', month='*', week_day='*', year='*'), ) rule.add_target(targets.LambdaFunction(lambdaFn)) ac = AssetCode("./python") layer = LayerVersion(self, "myapp1", code=ac, description="myapp1 layer", compatible_runtimes=[lambdas.Runtime.PYTHON_3_6], layer_version_name='myapp-layer') lambdaFn.add_layers(layer)
def __init__(self, scope: core.Construct, id: str, slack_channel: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) lambda_role = _iam.Role( self, 'SlackLambdaRole', assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com'), managed_policies=[ _iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonSSMReadOnlyAccess'), _iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaBasicExecutionRole') ]) function = _lambda.Function(self, 'BatchSlackLambda', handler='notify_slack.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset('lambdas/batch'), environment={ "SLACK_HOST": "hooks.slack.com", "SLACK_CHANNEL": slack_channel }, role=lambda_role) batch_job_queues = { 'AGHA': { 'name': 'agha-job-queue', 'enabled': True }, 'Umccrise': { 'name': 'cdk-umccrise_job_queue', 'enabled': True }, 'UmccriseDragen': { 'name': 'cdk-umccrise_job_queue-dragen-testing', 'enabled': False }, 'Nextflow': { 'name': 'nextflow-job-queue', 'enabled': False }, 'WtsReport': { 'name': 'wts_report_batch_queue_dev', 'enabled': True }, } for job_queue_id, job_queue_config in batch_job_queues.items(): job_queue_name = job_queue_config['name'] job_queue_arn = f'arn:aws:batch:{self.region}:{self.account}:job-queue/{job_queue_name}' _events.Rule( self, f'BatchEventToSlackLambda{job_queue_id}', enabled=job_queue_config['enabled'], event_pattern=_events.EventPattern( detail={ 'status': [ 'FAILED', 'SUCCEEDED', 'RUNNABLE', ], 'jobQueue': [job_queue_arn], }, detail_type=['Batch Job State Change'], source=['aws.batch'], ), rule_name=f'batch-slack-notifications-{job_queue_id.lower()}', targets=[_events_targets.LambdaFunction(handler=function)])
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: """Invoke the base class constructor via super with the received scope, id, and props Args: scope: Defines scope in which this custom construct stack is created. id (str): Defines local identity of the construct. Must be unique amongst constructs within the same scope, as it's used to formulate the cF logical id for each resource defined in this scope. kwargs: Lots of possibilities """ # example of passing app.py level params to stack class self.stage=kwargs['stage'] kwargs={} super().__init__(scope, id, **kwargs) # Resources to create s3_bucket = s3.Bucket( self, "Bucket", bucket_name=f"asteroids-{self.stage}", versioned=False, removal_policy=core.RemovalPolicy.DESTROY # NOT recommended for production code ) ddb_asteroids_table = ddb.Table( self, "Table", table_name="asteroids_table", partition_key={ "name": "id", "type": ddb.AttributeType.STRING }, removal_policy=core.RemovalPolicy.DESTROY # NOT recommended for production code ) # Lambdas and layers requests_layer = _lambda.LayerVersion( self, "requests", code=_lambda.AssetCode('layers/requests.zip') ) process_asteroid_data = _lambda.Function( self, "ProcessAsteroidsLambda", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.AssetCode("lambda"), handler="asteroids.handler", layers=[requests_layer], environment={ "S3_BUCKET": s3_bucket.bucket_name, "NASA_KEY": "<nasa_key_here>", } ) db_write = _lambda.Function( self, "DbWriteLambda", runtime=_lambda.Runtime.PYTHON_3_7, handler="dbwrite.handler", code=_lambda.Code.asset('lambda'), environment={ "ASTEROIDS_TABLE": ddb_asteroids_table.table_name, "S3_BUCKET": s3_bucket.bucket_name } ) # Rules and Events json_rule = events.Rule( self, "JSONRule", schedule=events.Schedule.cron( minute="15", hour="*", month="*", week_day="*", year="*" ) ) csv_rule = events.Rule( self, "CSVRule", schedule=events.Schedule.cron( minute="30", hour="*", month="*", week_day="*", year="*" ) ) # add lambda function target as well as custom trigger input to rules json_rule.add_target( targets.LambdaFunction( process_asteroid_data, event=events.RuleTargetInput.from_text("json") ) ) csv_rule.add_target( targets.LambdaFunction( process_asteroid_data, event=events.RuleTargetInput.from_text("csv") ) ) # create s3 notification for the db_write function notify_lambda = s3n.LambdaDestination(db_write) # assign 'notify_lambda' notification for 'OBJECT_CREATED' event type s3_bucket.add_event_notification(s3.EventType.OBJECT_CREATED, notify_lambda) # Permissions s3_bucket.grant_read_write(process_asteroid_data) s3_bucket.grant_read_write(db_write) ddb_asteroids_table.grant_read_write_data(db_write)
def __init__(self, scope: core.Construct, _id: str, **kwargs) -> None: super().__init__(scope, _id, **kwargs) # Setup SSM parameter of credentials, bucket_para, ignore_list ssm_credential_para = ssm.StringParameter.from_secure_string_parameter_attributes( self, "ssm_parameter_credentials", parameter_name=ssm_parameter_credentials, version=1) ssm_bucket_para = ssm.StringParameter(self, "s3bucket_serverless", string_value=json.dumps( bucket_para, indent=4)) ssm_parameter_ignore_list = ssm.StringParameter( self, "s3_migrate_ignore_list", string_value=ignore_list) # Setup DynamoDB ddb_file_list = ddb.Table(self, "s3migrate_serverless", partition_key=ddb.Attribute( name="Key", type=ddb.AttributeType.STRING), billing_mode=ddb.BillingMode.PAY_PER_REQUEST) # Setup SQS sqs_queue_DLQ = sqs.Queue(self, "s3migrate_serverless_Q_DLQ", visibility_timeout=core.Duration.minutes(15), retention_period=core.Duration.days(14)) sqs_queue = sqs.Queue(self, "s3migrate_serverless_Q", visibility_timeout=core.Duration.minutes(15), retention_period=core.Duration.days(14), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=3, queue=sqs_queue_DLQ)) # Setup API for Lambda to get IP address (for debug networking routing purpose) checkip = api.RestApi( self, "lambda-checkip-api", cloud_watch_role=True, deploy=True, description="For Lambda get IP address", default_integration=api.MockIntegration( integration_responses=[ api.IntegrationResponse(status_code="200", response_templates={ "application/json": "$context.identity.sourceIp" }) ], request_templates={"application/json": '{"statusCode": 200}'}), endpoint_types=[api.EndpointType.REGIONAL]) checkip.root.add_method("GET", method_responses=[ api.MethodResponse( status_code="200", response_models={ "application/json": api.Model.EMPTY_MODEL }) ]) # Setup Lambda functions handler = lam.Function(self, "s3-migrate-worker", code=lam.Code.asset("./lambda"), handler="lambda_function_worker.lambda_handler", runtime=lam.Runtime.PYTHON_3_8, memory_size=1024, timeout=core.Duration.minutes(15), tracing=lam.Tracing.ACTIVE, environment={ 'table_queue_name': ddb_file_list.table_name, 'Des_bucket_default': Des_bucket_default, 'Des_prefix_default': Des_prefix_default, 'StorageClass': StorageClass, 'checkip_url': checkip.url, 'ssm_parameter_credentials': ssm_parameter_credentials }) handler_jobsender = lam.Function( self, "s3-migrate-jobsender", code=lam.Code.asset("./lambda"), handler="lambda_function_jobsender.lambda_handler", runtime=lam.Runtime.PYTHON_3_8, memory_size=1024, timeout=core.Duration.minutes(15), tracing=lam.Tracing.ACTIVE, environment={ 'table_queue_name': ddb_file_list.table_name, 'StorageClass': StorageClass, 'checkip_url': checkip.url, 'sqs_queue': sqs_queue.queue_name, 'ssm_parameter_credentials': ssm_parameter_credentials, 'ssm_parameter_ignore_list': ssm_parameter_ignore_list.parameter_name, 'ssm_parameter_bucket': ssm_bucket_para.parameter_name }) # Allow lambda read/write DDB, SQS ddb_file_list.grant_read_write_data(handler) ddb_file_list.grant_read_write_data(handler_jobsender) sqs_queue.grant_send_messages(handler_jobsender) # SQS trigger Lambda worker handler.add_event_source(SqsEventSource(sqs_queue, batch_size=1)) # Option1: Create S3 Bucket, all new objects in this bucket will be transmitted by Lambda Worker s3bucket = s3.Bucket(self, "s3_new_migrate") s3bucket.grant_read(handler) s3bucket.add_event_notification(s3.EventType.OBJECT_CREATED, s3n.SqsDestination(sqs_queue)) # Option2: Allow Exist S3 Buckets to be read by Lambda functions. # Lambda Jobsender will scan and compare the these buckets and trigger Lambda Workers to transmit bucket_name = '' for b in bucket_para: if bucket_name != b['src_bucket']: # 如果列了多个相同的Bucket,就跳过 bucket_name = b['src_bucket'] s3exist_bucket = s3.Bucket.from_bucket_name( self, bucket_name, # 用这个做id bucket_name=bucket_name) s3exist_bucket.grant_read(handler_jobsender) s3exist_bucket.grant_read(handler) # Allow Lambda read ssm parameters ssm_bucket_para.grant_read(handler_jobsender) ssm_credential_para.grant_read(handler) ssm_credential_para.grant_read(handler_jobsender) ssm_parameter_ignore_list.grant_read(handler_jobsender) # Schedule cron event to trigger Lambda Jobsender per hour: event.Rule(self, 'cron_trigger_jobsender', schedule=event.Schedule.rate(core.Duration.hours(1)), targets=[target.LambdaFunction(handler_jobsender)]) # Create Lambda logs filter to create network traffic metric handler.log_group.add_metric_filter( "Complete-bytes", metric_name="Complete-bytes", metric_namespace="s3_migrate", metric_value="$bytes", filter_pattern=logs.FilterPattern.literal( '[info, date, sn, p="--->Complete", bytes, key]')) handler.log_group.add_metric_filter( "Uploading-bytes", metric_name="Uploading-bytes", metric_namespace="s3_migrate", metric_value="$bytes", filter_pattern=logs.FilterPattern.literal( '[info, date, sn, p="--->Uploading", bytes, key]')) handler.log_group.add_metric_filter( "Downloading-bytes", metric_name="Downloading-bytes", metric_namespace="s3_migrate", metric_value="$bytes", filter_pattern=logs.FilterPattern.literal( '[info, date, sn, p="--->Downloading", bytes, key]')) lambda_metric_Complete = cw.Metric(namespace="s3_migrate", metric_name="Complete-bytes", statistic="Sum", period=core.Duration.minutes(1)) lambda_metric_Upload = cw.Metric(namespace="s3_migrate", metric_name="Uploading-bytes", statistic="Sum", period=core.Duration.minutes(1)) lambda_metric_Download = cw.Metric(namespace="s3_migrate", metric_name="Downloading-bytes", statistic="Sum", period=core.Duration.minutes(1)) handler.log_group.add_metric_filter( "ERROR", metric_name="ERROR-Logs", metric_namespace="s3_migrate", metric_value="1", filter_pattern=logs.FilterPattern.literal('"ERROR"')) handler.log_group.add_metric_filter( "WARNING", metric_name="WARNING-Logs", metric_namespace="s3_migrate", metric_value="1", filter_pattern=logs.FilterPattern.literal('"WARNING"')) log_metric_ERROR = cw.Metric(namespace="s3_migrate", metric_name="ERROR-Logs", statistic="Sum", period=core.Duration.minutes(1)) log_metric_WARNING = cw.Metric(namespace="s3_migrate", metric_name="WARNING-Logs", statistic="Sum", period=core.Duration.minutes(1)) # Dashboard to monitor SQS and Lambda board = cw.Dashboard(self, "s3_migrate_serverless") board.add_widgets( cw.GraphWidget(title="Lambda-NETWORK", left=[ lambda_metric_Download, lambda_metric_Upload, lambda_metric_Complete ]), # TODO: here monitor all lambda concurrency not just the working one. Limitation from CDK # Lambda now supports monitor single lambda concurrency, will change this after CDK support cw.GraphWidget(title="Lambda-all-concurrent", left=[ handler.metric_all_concurrent_executions( period=core.Duration.minutes(1)) ]), cw.GraphWidget( title="Lambda-invocations/errors/throttles", left=[ handler.metric_invocations( period=core.Duration.minutes(1)), handler.metric_errors(period=core.Duration.minutes(1)), handler.metric_throttles(period=core.Duration.minutes(1)) ]), cw.GraphWidget( title="Lambda-duration", left=[ handler.metric_duration(period=core.Duration.minutes(1)) ]), ) board.add_widgets( cw.GraphWidget( title="SQS-Jobs", left=[ sqs_queue.metric_approximate_number_of_messages_visible( period=core.Duration.minutes(1)), sqs_queue. metric_approximate_number_of_messages_not_visible( period=core.Duration.minutes(1)) ]), cw.GraphWidget( title="SQS-DeadLetterQueue", left=[ sqs_queue_DLQ. metric_approximate_number_of_messages_visible( period=core.Duration.minutes(1)), sqs_queue_DLQ. metric_approximate_number_of_messages_not_visible( period=core.Duration.minutes(1)) ]), cw.GraphWidget(title="ERROR/WARNING Logs", left=[log_metric_ERROR], right=[log_metric_WARNING]), cw.SingleValueWidget( title="Running/Waiting and Dead Jobs", metrics=[ sqs_queue. metric_approximate_number_of_messages_not_visible( period=core.Duration.minutes(1)), sqs_queue.metric_approximate_number_of_messages_visible( period=core.Duration.minutes(1)), sqs_queue_DLQ. metric_approximate_number_of_messages_not_visible( period=core.Duration.minutes(1)), sqs_queue_DLQ. metric_approximate_number_of_messages_visible( period=core.Duration.minutes(1)) ], height=6)) # Alarm for queue - DLQ alarm_DLQ = cw.Alarm( self, "SQS_DLQ", metric=sqs_queue_DLQ.metric_approximate_number_of_messages_visible( ), threshold=0, comparison_operator=cw.ComparisonOperator.GREATER_THAN_THRESHOLD, evaluation_periods=1, datapoints_to_alarm=1) alarm_topic = sns.Topic(self, "SQS queue-DLQ has dead letter") alarm_topic.add_subscription( subscription=sub.EmailSubscription(alarm_email)) alarm_DLQ.add_alarm_action(action.SnsAction(alarm_topic)) core.CfnOutput(self, "Dashboard", value="CloudWatch Dashboard name s3_migrate_serverless")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ## create lambda function from inline function try: with open("./lambda_src/custom_lambda.py", "r") as f: custom_lambda_code = f.read() except OSError as e: raise Exception(f"fals to open lambda function code, error {e}") custom_lambda_fn = _lambda.Function( self, "customLambda", function_name="custom_lambda", runtime=_lambda.Runtime.PYTHON_3_7, handler="index.lambda_handler", code=_lambda.InlineCode(custom_lambda_code), timeout=core.Duration.seconds(3), reserved_concurrent_executions=1, environment={"LOG_LEVEL": "INFO"}, ) # create custom log group custom_loggroup = _logs.LogGroup( self, "customLogGroup", log_group_name=f"/aws/lambda/{custom_lambda_fn.function_name}", removal_policy=core.RemovalPolicy.DESTROY, ) ## create lambda function from s3 source_bucket = _s3.Bucket.from_bucket_name(self, "sourceBucket", "cdk-tutorials-resources") custom_lambda_s3 = _lambda.Function( self, "customLambdaS3", function_name="custom_lambda_s3", runtime=_lambda.Runtime.PYTHON_3_7, handler="custom_lambda.lambda_handler", code=_lambda.S3Code(bucket=source_bucket, key="custom_lambda.zip"), timeout=core.Duration.seconds(3), reserved_concurrent_executions=1, environment={"Log_Group": "INFO"}, ) _logs.LogGroup( self, "customLogGroupS3", log_group_name=f"/aws/lambda/{custom_lambda_s3.function_name}", removal_policy=core.RemovalPolicy.DESTROY, retention=_logs.RetentionDays.ONE_WEEK, ) ## cloudwatch rules # - every day at 10:00 UTC six_pm_cron = _events.Rule( self, "sixPmRule", schedule=_events.Schedule.cron( minute="0", hour="18", month="*", week_day="MON-FRI", year="*" ), ) # - every 3 minutes run_every_3_mins = _events.Rule( self, "every3Mins", schedule=_events.Schedule.rate(core.Duration.minutes(3)) ) # add lambda to cloudwatch event rules six_pm_cron.add_target(_targets.LambdaFunction(custom_lambda_fn)) run_every_3_mins.add_target(_targets.LambdaFunction(custom_lambda_s3))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # store dynamodb_table = dynamodb.Table( self, 'dynamodb_table', table_name=f'{PROJECT}_{STAGE}', partition_key=dynamodb.Attribute( name='date', type=dynamodb.AttributeType.STRING), billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST, point_in_time_recovery=False, removal_policy=core.RemovalPolicy.DESTROY, server_side_encryption=True, ) # public api public_api = appsync.CfnGraphQLApi( self, 'public_api', name=f'{PROJECT}_{STAGE}', authentication_type='API_KEY', ) now = time.localtime() epoch = time.mktime(now) public_api_key = appsync.CfnApiKey( self, 'public_api_key', api_id=public_api.attr_api_id, expires=epoch + core.Duration.days(90).to_seconds(), ) with open('schema.gql', mode='r') as f: graphql_schema = f.read() appsync.CfnGraphQLSchema(self, 'public_api_schema', api_id=public_api.attr_api_id, definition=graphql_schema) public_api_role = iam.Role( self, 'public_api_role', assumed_by=iam.ServicePrincipal('appsync.amazonaws.com'), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonDynamoDBFullAccess') ], ) public_api_datasource = appsync.CfnDataSource( self, 'public_api_datasource', api_id=public_api.attr_api_id, name=f'{PROJECT}_{STAGE}_dynamodb', type='AMAZON_DYNAMODB', dynamo_db_config={ 'awsRegion': 'us-east-1', 'tableName': dynamodb_table.table_name, }, service_role_arn=public_api_role.role_arn, ) with open('mapping_templates/get_holiday.json', mode='r') as f: get_holiday_json = f.read() appsync.CfnResolver( self, 'public_api_resolver_get_holiday', api_id=public_api.attr_api_id, type_name='Query', field_name='getHoliday', data_source_name=public_api_datasource.attr_name, kind='UNIT', request_mapping_template=get_holiday_json, response_mapping_template='$util.toJson($context.result)', ) with open('mapping_templates/list_holidays.json', mode='r') as f: list_holidays_json = f.read() appsync.CfnResolver( self, 'public_api_resolver_list_holidays', api_id=public_api.attr_api_id, type_name='Query', field_name='listHolidays', data_source_name=public_api_datasource.attr_name, kind='UNIT', request_mapping_template=list_holidays_json, response_mapping_template='$util.toJson($context.result)', ) # lambda source code upload to s3 lambda_assets = s3_assets.Asset(self, 'lambda_assets', path='./function/.artifact/') # update function func_api = lambda_.Function( self, f'{PROJECT}-{STAGE}-func', function_name=f'{PROJECT}-{STAGE}-func', code=lambda_.Code.from_bucket(bucket=lambda_assets.bucket, key=lambda_assets.s3_object_key), handler='app.handler', runtime=lambda_.Runtime.PYTHON_3_7, timeout=core.Duration.seconds(120), log_retention=logs.RetentionDays.SIX_MONTHS, memory_size=128, tracing=lambda_.Tracing.ACTIVE, ) func_api.add_environment('TABLE_NAME', dynamodb_table.table_name) func_api.add_environment('CSV_URL', CSV_URL) func_api.add_to_role_policy( iam.PolicyStatement( actions=[ 'dynamodb:Get*', 'dynamodb:Put*', 'dynamodb:Batch*', ], resources=[dynamodb_table.table_arn], )) # schedule execute events.Rule( self, f'{PROJECT}-{STAGE}-schedule', enabled=True, schedule=events.Schedule.rate(core.Duration.days(10)), targets=[events_targets.LambdaFunction(func_api)], ) # lambda@edge func_lambdaedge = lambda_.Function( self, f'{PROJECT}-{STAGE}-func-lambdaedge', function_name=f'{PROJECT}-{STAGE}-func-lambdaedge', code=lambda_.Code.from_inline( open('./function/src/lambdaedge.py').read().replace( '__X_API_KEY__', public_api_key.attr_api_key)), handler='index.handler', runtime=lambda_.Runtime.PYTHON_3_7, timeout=core.Duration.seconds(30), memory_size=128, role=iam.Role( self, f'{PROJECT}-{STAGE}-func-lambdaedge-role', assumed_by=iam.CompositePrincipal( iam.ServicePrincipal('edgelambda.amazonaws.com'), iam.ServicePrincipal('lambda.amazonaws.com'), ), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaBasicExecutionRole'), ], ), ) lambdaedge_version = func_lambdaedge.add_version( hashlib.sha256( open('./function/src/lambdaedge.py').read().replace( '__X_API_KEY__', public_api_key.attr_api_key).encode()).hexdigest()) # ACM certificates = acm.Certificate( self, 'certificates', domain_name=DOMAIN, validation_method=acm.ValidationMethod.DNS, ) # CDN cdn = cloudfront.CloudFrontWebDistribution( self, f'{PROJECT}-{STAGE}-cloudfront', origin_configs=[ cloudfront.SourceConfiguration( behaviors=[ # default behavior cloudfront.Behavior( allowed_methods=cloudfront. CloudFrontAllowedMethods.ALL, default_ttl=core.Duration.seconds(0), max_ttl=core.Duration.seconds(0), min_ttl=core.Duration.seconds(0), is_default_behavior=True, lambda_function_associations=[ cloudfront.LambdaFunctionAssociation( event_type=cloudfront.LambdaEdgeEventType. ORIGIN_REQUEST, lambda_function=lambdaedge_version, ), ]) ], custom_origin_source=cloudfront.CustomOriginConfig( domain_name=core.Fn.select( 2, core.Fn.split('/', public_api.attr_graph_ql_url)), ), ) ], alias_configuration=cloudfront.AliasConfiguration( acm_cert_ref=certificates.certificate_arn, names=[DOMAIN], security_policy=cloudfront.SecurityPolicyProtocol. TLS_V1_2_2018, ), price_class=cloudfront.PriceClass.PRICE_CLASS_ALL, ) core.CfnOutput( self, 'cloudfront-domain', value=cdn.domain_name, )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # create dynamo table demo_table = aws_dynamodb.Table( self, "demo_table", partition_key=aws_dynamodb.Attribute( name="id", type=aws_dynamodb.AttributeType.STRING ) ) # create a Cloudwatch Event rule one_day_rule = aws_events.Rule( self, "one_day_rule", schedule=aws_events.Schedule.rate(core.Duration.days(1)), ) # Lambda stuff self.lambda_code_etl = _lambda.Code.from_cfn_parameters() lambda_etl = _lambda.Function(self,'LambdaETL', handler='lambda-handler-etl.handler', timeout=core.Duration.seconds(120), runtime=_lambda.Runtime.PYTHON_3_7, code=self.lambda_code_etl, ) self.lambda_code_serve = _lambda.Code.from_cfn_parameters() lambda_serve = _lambda.Function(self,'LambdaServe', handler='lambda-handler-serve.handler', runtime=_lambda.Runtime.PYTHON_3_7, code=self.lambda_code_serve, ) # Add target to Cloudwatch Event one_day_rule.add_target(aws_events_targets.LambdaFunction(lambda_etl)) # grant permission to lambda to write to demo table lambda_etl.add_environment("TABLE_NAME", demo_table.table_name) demo_table.grant_write_data(lambda_etl) demo_table.grant_read_data(lambda_etl) lambda_serve.add_environment("TABLE_NAME", demo_table.table_name) demo_table.grant_write_data(lambda_serve) demo_table.grant_read_data(lambda_serve) # API Gateway stuff base_api = _apigw.RestApi(self, 'ApiGatewayWithCors', rest_api_name='ApiGatewayWithCors') entity = base_api.root.add_resource('api') entity_lambda_integration = _apigw.LambdaIntegration(lambda_serve,proxy=False, integration_responses=[ { 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': "'*'", } } ] ) entity.add_method('GET', entity_lambda_integration, method_responses=[{ 'statusCode': '200', 'responseParameters': { 'method.response.header.Access-Control-Allow-Origin': True, } } ] ) self.add_cors_options(entity) # make the SNS resource (called a topic) and give access to the ETL lambda sns_topic = aws_sns.Topic(self, 'PipelineTopic') lambda_etl.add_environment('SNS_TOPIC_ARN', sns_topic.topic_arn) sns_topic.grant_publish(lambda_etl) # make an email subscription subscription = aws_sns_subscriptions.EmailSubscription('*****@*****.**') # fake email because I don't want AWS spam sns_topic.add_subscription(subscription)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) eventTargets = [] policyStatement = _iam.PolicyStatement( resources = ['*'], actions = [ "cloudwatch:PutMetricAlarm", "cloudwatch:ListMetrics", "cloudwatch:DeleteAlarms", "ec2:CreateTags", "ec2:Describe*", "ec2:Attach*", "elasticloadbalancing:Describe*", "elasticloadbalancing:Create*", "elasticloadbalancing:AddTags" ], effect = _iam.Effect.ALLOW ) glom_layer = _lambda.LayerVersion.from_layer_version_attributes( self, "glom_api_layer", layer_version_arn="arn:aws:lambda:us-east-1:<AWS ACCOUNT>:layer:python-glom-layer:1", compatible_runtimes=[ _lambda.Runtime.PYTHON_3_6, _lambda.Runtime.PYTHON_3_7 ] ) eventHandler = _lambda.Function( self, 'resourceTagger', runtime = _lambda.Runtime.PYTHON_3_7, code = _lambda.Code.asset('lambda'), handler = 'auto_tag.handler', layers=[glom_layer] ) eventHandler.add_to_role_policy(policyStatement) eventTargets.append(_targets.LambdaFunction(handler = eventHandler)) pattern = _events.EventPattern( source = ['aws.ec2', 'aws.elasticloadbalancing'], detail_type = [ "AWS API Call via CloudTrail"], detail = { "eventSource": [ "ec2.amazonaws.com", "elasticloadbalancing.amazonaws.com" ], "eventName": [ "RunInstances", "CreateSnapshot", "CreateVolume", "CreateImage", "CreateLoadBalancer", "AttachNetworkInterface" ] } ) _events.Rule( scope = self, id = 'AutoTagsRule', description = 'Monitor EC2 and ELB events', rule_name = 'AutoTagsRule', event_pattern = pattern, targets = eventTargets )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # If left unchecked this pattern could "fan out" on the transform and load # lambdas to the point that it consumes all resources on the account. This is # why we are limiting concurrency to 2 on all 3 lambdas. Feel free to raise this. lambda_throttle_size = 2 #### # DynamoDB Table # This is where our transformed data ends up #### table = dynamo_db.Table(self, "TransformedData", partition_key=dynamo_db.Attribute( name="id", type=dynamo_db.AttributeType.STRING)) #### # S3 Landing Bucket # This is where the user uploads the file to be transformed #### bucket = s3.Bucket(self, "LandingBucket") #### # Queue that listens for S3 Bucket events #### queue = sqs.Queue(self, 'newObjectInLandingBucketEventQueue', visibility_timeout=core.Duration.seconds(300)) bucket.add_event_notification(s3.EventType.OBJECT_CREATED, s3n.SqsDestination(queue)) # EventBridge Permissions event_bridge_put_policy = iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=['*'], actions=['events:PutEvents']) #### # Fargate ECS Task Creation to pull data from S3 # # Fargate is used here because if you had a seriously large file, # you could stream the data to fargate for as long as needed before # putting the data onto eventbridge or up the memory/storage to # download the whole file. Lambda has limitations on runtime and # memory/storage #### vpc = ec2.Vpc(self, "Vpc", max_azs=2) logging = ecs.AwsLogDriver(stream_prefix='TheEventBridgeETL', log_retention=logs.RetentionDays.ONE_WEEK) cluster = ecs.Cluster(self, 'Ec2Cluster', vpc=vpc) task_definition = ecs.TaskDefinition( self, 'FargateTaskDefinition', memory_mib="512", cpu="256", compatibility=ecs.Compatibility.FARGATE) # We need to give our fargate container permission to put events on our EventBridge task_definition.add_to_task_role_policy(event_bridge_put_policy) # Grant fargate container access to the object that was uploaded to s3 bucket.grant_read(task_definition.task_role) container = task_definition.add_container( 'AppContainer', image=ecs.ContainerImage.from_asset( 'container/s3DataExtractionTask'), logging=logging, environment={ 'S3_BUCKET_NAME': bucket.bucket_name, 'S3_OBJECT_KEY': '' }) #### # Lambdas # # These are used for 4 phases: # # Extract - kicks of ecs fargate task to download data and splinter to eventbridge events # Transform - takes the two comma separated strings and produces a json object # Load - inserts the data into dynamodb # Observe - This is a lambda that subscribes to all events and logs them centrally #### subnet_ids = [] for subnet in vpc.private_subnets: subnet_ids.append(subnet.subnet_id) #### # Extract # defines an AWS Lambda resource to trigger our fargate ecs task #### extract_lambda = _lambda.Function( self, "extractLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="s3SqsEventConsumer.handler", code=_lambda.Code.from_asset("lambdas/extract"), reserved_concurrent_executions=lambda_throttle_size, environment={ "CLUSTER_NAME": cluster.cluster_name, "TASK_DEFINITION": task_definition.task_definition_arn, "SUBNETS": json.dumps(subnet_ids), "CONTAINER_NAME": container.container_name }) queue.grant_consume_messages(extract_lambda) extract_lambda.add_event_source(_event.SqsEventSource(queue=queue)) extract_lambda.add_to_role_policy(event_bridge_put_policy) run_task_policy_statement = iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[task_definition.task_definition_arn], actions=['ecs:RunTask']) extract_lambda.add_to_role_policy(run_task_policy_statement) task_execution_role_policy_statement = iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ task_definition.obtain_execution_role().role_arn, task_definition.task_role.role_arn ], actions=['iam:PassRole']) extract_lambda.add_to_role_policy(task_execution_role_policy_statement) #### # Transform # defines a lambda to transform the data that was extracted from s3 #### transform_lambda = _lambda.Function( self, "TransformLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="transform.handler", code=_lambda.Code.from_asset("lambdas/transform"), reserved_concurrent_executions=lambda_throttle_size, timeout=core.Duration.seconds(3)) transform_lambda.add_to_role_policy(event_bridge_put_policy) # Create EventBridge rule to route extraction events transform_rule = events.Rule( self, 'transformRule', description='Data extracted from S3, Needs transformed', event_pattern=events.EventPattern( source=['cdkpatterns.the-eventbridge-etl'], detail_type=['s3RecordExtraction'], detail={"status": ["extracted"]})) transform_rule.add_target( targets.LambdaFunction(handler=transform_lambda)) #### # Load # load the transformed data in dynamodb #### load_lambda = _lambda.Function( self, "LoadLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="load.handler", code=_lambda.Code.from_asset("lambdas/load"), reserved_concurrent_executions=lambda_throttle_size, timeout=core.Duration.seconds(3), environment={"TABLE_NAME": table.table_name}) load_lambda.add_to_role_policy(event_bridge_put_policy) table.grant_read_write_data(load_lambda) load_rule = events.Rule( self, 'loadRule', description='Data transformed, Needs loaded into dynamodb', event_pattern=events.EventPattern( source=['cdkpatterns.the-eventbridge-etl'], detail_type=['transform'], detail={"status": ["transformed"]})) load_rule.add_target(targets.LambdaFunction(handler=load_lambda)) #### # Observe # Watch for all cdkpatterns.the-eventbridge-etl events and log them centrally #### observe_lambda = _lambda.Function( self, "ObserveLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="observe.handler", code=_lambda.Code.from_asset("lambdas/observe"), reserved_concurrent_executions=lambda_throttle_size, timeout=core.Duration.seconds(3)) observe_rule = events.Rule( self, 'observeRule', description='all events are caught here and logged centrally', event_pattern=events.EventPattern( source=['cdkpatterns.the-eventbridge-etl'])) observe_rule.add_target(targets.LambdaFunction(handler=observe_lambda))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) dynamodb_table = _ddb.Table( self, id="lab2-cm-ddb", table_name="lab2-cm-order-status", partition_key=Attribute(name='ID', type=AttributeType.STRING), removal_policy=core.RemovalPolicy.DESTROY # NOT for production ) eb = _eb.EventBus(self, id="lab2-cm-eventbus", event_bus_name="lab2-cm-eventbus") lambda_role = _iam.Role( self, id='lab2-cm-role', assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com')) dynamodb_policy_statement = _iam.PolicyStatement( effect=_iam.Effect.ALLOW) dynamodb_policy_statement.add_actions("dynamodb:*") dynamodb_policy_statement.add_resources(dynamodb_table.table_arn) lambda_role.add_to_policy(dynamodb_policy_statement) eventbridge_policy_statement = _iam.PolicyStatement( effect=_iam.Effect.ALLOW) eventbridge_policy_statement.add_actions("events:*") eventbridge_policy_statement.add_resources(eb.event_bus_arn) lambda_role.add_to_policy(eventbridge_policy_statement) cloudwatch_policy_statement = _iam.PolicyStatement( effect=_iam.Effect.ALLOW) cloudwatch_policy_statement.add_actions("logs:CreateLogGroup") cloudwatch_policy_statement.add_actions("logs:CreateLogStream") cloudwatch_policy_statement.add_actions("logs:PutLogEvents") cloudwatch_policy_statement.add_actions("logs:DescribeLogStreams") cloudwatch_policy_statement.add_resources("*") lambda_role.add_to_policy(cloudwatch_policy_statement) fn_lambda_invoice_service = aws_lambda.Function( self, "lab2-cm-invoiceService", code=aws_lambda.AssetCode("../lambda-functions/invoice-service/"), handler="app.lambda_handler", tracing=aws_lambda.Tracing.ACTIVE, timeout=core.Duration.seconds(30), role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) fn_lambda_invoice_service.add_environment("TABLE_NAME", dynamodb_table.table_name) fn_lambda_fulfilment_service = aws_lambda.Function( self, "lab2-cm-fulfilmentService", code=aws_lambda.AssetCode( "../lambda-functions/fulfilment-service/"), handler="app.lambda_handler", tracing=aws_lambda.Tracing.ACTIVE, timeout=core.Duration.seconds(30), role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) fn_lambda_fulfilment_service.add_environment("TABLE_NAME", dynamodb_table.table_name) fn_lambda_fulfilment_service.add_environment("EVENTBUS_NAME", eb.event_bus_name) fn_lambda_forecasting_service = aws_lambda.Function( self, "lab2-cm-forecastingService", code=aws_lambda.AssetCode( "../lambda-functions/forecasting-service/"), handler="app.lambda_handler", tracing=aws_lambda.Tracing.ACTIVE, timeout=core.Duration.seconds(30), role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) fn_lambda_forecasting_service.add_environment( "TABLE_NAME", dynamodb_table.table_name) fn_lambda_order_service = aws_lambda.Function( self, "lab2-cm-orderService", code=aws_lambda.AssetCode("../lambda-functions/order-service/"), handler="app.lambda_handler", timeout=core.Duration.seconds(30), tracing=aws_lambda.Tracing.ACTIVE, role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) fn_lambda_order_service.add_environment("TABLE_NAME", dynamodb_table.table_name) fn_lambda_order_service.add_environment("EVENTBUS_NAME", eb.event_bus_name) fn_lambda_logistic_service = aws_lambda.Function( self, "lab2-cm-logisticService", code=aws_lambda.AssetCode("../lambda-functions/logistic-service/"), handler="app.lambda_handler", timeout=core.Duration.seconds(30), tracing=aws_lambda.Tracing.ACTIVE, role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) fn_lambda_logistic_service.add_environment("TABLE_NAME", dynamodb_table.table_name) eb_order_created_pattern = _eb.EventPattern( detail_type=["order_created"], ) eb_fulfilment_completed_pattern = _eb.EventPattern( detail_type=["fulfilment_completed"], ) eb_order_created_rule = _eb.Rule( self, id="lab2-cm-eventRule-order-created", description="Order created event", enabled=True, event_bus=eb, event_pattern=eb_order_created_pattern, rule_name="lab2-OrderCreated", targets=[ _ebt.LambdaFunction(handler=fn_lambda_invoice_service), _ebt.LambdaFunction(handler=fn_lambda_fulfilment_service), _ebt.LambdaFunction(handler=fn_lambda_forecasting_service) ]) eb_fulfilment_completed_rule = _eb.Rule( self, id="lab2-cm-eventRule-fulfilment-completed", description="Fulfilment completedevent", enabled=True, event_bus=eb, event_pattern=eb_fulfilment_completed_pattern, rule_name="lab2-FulfilmentCompleted", targets=[_ebt.LambdaFunction(handler=fn_lambda_logistic_service)]) api = _ag.RestApi( self, id='lab2-cm-api-gateway', ) api_lambda_integration = _ag.LambdaIntegration(fn_lambda_order_service) api.root.add_resource('order').add_method('GET', api_lambda_integration)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) email_subscription_parameter = core.CfnParameter( self, "EmailSubscriptionParameter", description="Email Address for Notification Subscription") email_subscription = email_subscription_parameter.value_as_string ar1 = accessanalyzer.CfnAnalyzer.ArchiveRuleProperty( rule_name="test", filter=[ accessanalyzer.CfnAnalyzer.FilterProperty( property="principal.AWS", eq=["123456789123"]) ]) analyzer = accessanalyzer.CfnAnalyzer( self, id="accessanalyzer", type="ACCOUNT", tags=[core.CfnTag(key="AccessAnalyzerType", value="ACCOUNT")], archive_rules=[ar1]) runtime = aws_lambda.Runtime.PYTHON_3_8 boto3_lambda_layer = aws_lambda.LayerVersion( self, "Boto3LambdaLayer", code=aws_lambda.AssetCode("./layers/boto3"), compatible_runtimes=[runtime], description="Boto3 Lambda Layer") context_enrichment = aws_lambda.Function( self, "context_enrichment", runtime=runtime, handler="app.handler", code=aws_lambda.AssetCode("./functions/context-enrichment"), layers=[boto3_lambda_layer]) handler_statement = iam.PolicyStatement(actions=[ "iam:ListRoleTags", "s3:GetBucketTagging", "lambda:ListTags", "sqs:ListQueueTags", "kms:ListAliases", "kms:ListResourceTags" ], effect=iam.Effect.ALLOW, resources=["*"]) context_enrichment.add_to_role_policy(handler_statement) cmk_key = kms.Key( self, "SNSEncryptionAtRestKey", description="SNS Encryption at rest key", alias="sns-encryption-at-rest", enable_key_rotation=True, ) email_topic = sns.Topic( self, "AccessAnalyzerNotificationTopic", display_name="Access Analyzer Finding Notification Topic", master_key=cmk_key) email_topic.add_subscription( subscriptions.EmailSubscription(email_subscription)) notification = aws_lambda.Function( self, "notification", runtime=runtime, handler="app.handler", code=aws_lambda.AssetCode("./functions/notification"), layers=[boto3_lambda_layer], environment={"SNS_TOPIC_ARN": email_topic.topic_arn}) notification_statement = iam.PolicyStatement(actions=[ "sns:Publish", ], effect=iam.Effect.ALLOW, resources=["*"]) notification.add_to_role_policy(notification_statement) cmk_key.grant_encrypt_decrypt(notification) archive_access_analyzer_finding = aws_lambda.Function( self, "archive-access-analyzer-finding", runtime=runtime, handler="app.handler", code=aws_lambda.AssetCode( "./functions/archive-access-analyzer-finding"), layers=[boto3_lambda_layer]) archive_statement = iam.PolicyStatement(actions=[ "access-analyzer:UpdateFindings", ], effect=iam.Effect.ALLOW, resources=["*"]) archive_access_analyzer_finding.add_to_role_policy(archive_statement) evaluate_access_analyzer_finding = aws_lambda.Function( self, "evaluate-access-analyzer-finding", runtime=runtime, handler="app.handler", code=aws_lambda.AssetCode( "./functions/evaluate-access-analyzer-finding"), layers=[boto3_lambda_layer]) #https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html access_analyzer_handler_task = sfn.Task( self, "Context Enrichment", task=sfn_tasks.InvokeFunction(context_enrichment), result_path="$.guid", ) notification_task = sfn.Task( self, "Send Notification", task=sfn_tasks.InvokeFunction(notification), result_path="$.guid", ) archive_task = sfn.Task( self, "Archive Finding", task=sfn_tasks.InvokeFunction(archive_access_analyzer_finding), result_path="$.guid", ) evaluate_task = sfn.Task( self, "Evaluate Risk Level", task=sfn_tasks.InvokeFunction(evaluate_access_analyzer_finding), result_path="$.guid", ) definition=access_analyzer_handler_task. \ next(evaluate_task). \ next(sfn.Choice(self, "Archive?"). \ when(sfn.Condition.string_equals("$.guid.status", "ARCHIVE"), archive_task). \ when(sfn.Condition.string_equals("$.guid.status", "NOTIFY"), notification_task) \ ) state_machine = sfn.StateMachine( self, "Access-Analyzer-Automatic-Finding-Archive-State-Machine", definition=definition, timeout=core.Duration.minutes(5), ) #https://docs.aws.amazon.com/step-functions/latest/dg/tutorial-cloudwatch-events-s3.html access_analyzer_finding_rule = aws_events.Rule( self, "AccessAnalzyerFindingActiveEventRule", description="Access Analyzer Finding Event Active", enabled=True, event_pattern=aws_events.EventPattern( source=["aws.access-analyzer"], detail_type=["Access Analyzer Finding"], detail={"status": ["ACTIVE"]}), targets=[ aws_events_targets.SfnStateMachine(state_machine), aws_events_targets.LambdaFunction(context_enrichment) ])
def __init__(self, scope: core.Construct, construct_id: str, stack_log_level: str, orders_bus, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # Add your stack resources below) # Read Lambda Code try: with open( "stacks/back_end/serverless_eventbridge_consumer_stack/lambda_src/eventbridge_data_consumer.py", encoding="utf-8", mode="r") as f: msg_consumer_fn_code = f.read() except OSError: print("Unable to read Lambda Function Code") raise msg_consumer_fn = _lambda.Function( self, "msgConsumerFn", function_name=f"events_consumer_fn", description="Process messages in EventBridge queue", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(msg_consumer_fn_code), handler="index.lambda_handler", timeout=core.Duration.seconds(3), reserved_concurrent_executions=1, environment={ "LOG_LEVEL": f"{stack_log_level}", "APP_ENV": "Production" }) msg_consumer_fn_version = msg_consumer_fn.latest_version msg_consumer_fn_version_alias = _lambda.Alias( self, "msgConsumerFnAlias", alias_name="MystiqueAutomation", version=msg_consumer_fn_version) # Create Custom Loggroup for Producer msg_consumer_fn_lg = _logs.LogGroup( self, "msgConsumerFnLogGroup", log_group_name=f"/aws/lambda/{msg_consumer_fn.function_name}", removal_policy=core.RemovalPolicy.DESTROY, retention=_logs.RetentionDays.ONE_DAY) # Restrict Produce Lambda to be invoked only from the stack owner account msg_consumer_fn.add_permission("restrictLambdaInvocationToOwnAccount", principal=_iam.AccountRootPrincipal(), action="lambda:InvokeFunction", source_account=core.Aws.ACCOUNT_ID, source_arn=orders_bus.event_bus_arn) # Event Pattern self.orders_pattern = _evnts.EventPattern(detail_type=["sales-events"]) # EventBridge Routing Rule self.orders_routing = _evnts.Rule( self, f"ordersEventRoutingRule01", description="A simple events routing rule", enabled=True, event_bus=orders_bus, event_pattern=self.orders_pattern, rule_name="orders_routing_to_consumer", targets=[_evnts_tgt.LambdaFunction(handler=msg_consumer_fn)]) self.orders_routing.apply_removal_policy(core.RemovalPolicy.DESTROY) # Restrict Produce Lambda to be invoked only from the stack owner account data_producer_fn.add_permission( "restrictLambdaInvocationToFhInOwnAccount", principal=_iam.AccountRootPrincipal(), action="lambda:InvokeFunction", source_account=core.Aws.ACCOUNT_ID) ########################################### ################# OUTPUTS ################# ########################################### output_0 = core.CfnOutput( self, "AutomationFrom", value=f"{GlobalArgs.SOURCE_INFO}", description= "To know more about this automation stack, check out our github page." ) output_2 = core.CfnOutput( self, "msgConsumer", value= f"https://console.aws.amazon.com/lambda/home?region={core.Aws.REGION}#/functions/{msg_consumer_fn.function_name}", description="Process events received from eventbridge event bus")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) load_dotenv(override=True) org_list = getenv("GITHUB_ORGS").split(",") # S3 bucket bucket_name = getenv("S3_ROOT_BUCKET") bucket = s3.Bucket( self, bucket_name, block_public_access=s3.BlockPublicAccess.BLOCK_ALL, bucket_name=bucket_name, encryption=s3.BucketEncryption.S3_MANAGED, versioned=False, ) # SQS queue sqs_queue = sqs.Queue( self, "GitHubDatastoreQueue", encryption=sqs.QueueEncryption.KMS_MANAGED, queue_name="GitHubDatastoreQueue", visibility_timeout=core.Duration.minutes(15), ) # SSM config config_manager = ssm.StringListParameter( self, "GitHubDatastoreOrgList", string_list_value=org_list, description="List of orgs to get data", parameter_name="GitHubDatastoreOrgList", ) # Lambda function role lambda_role = iam.Role( self, "GitHubDataLambdaRole", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name("AmazonS3FullAccess"), iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSQSFullAccess"), iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSSMFullAccess"), iam.ManagedPolicy.from_aws_managed_policy_name("AWSLambdaFullAccess"), iam.ManagedPolicy.from_aws_managed_policy_name( "SecretsManagerReadWrite" ), ], ) # Lambda function(s) oss_datastore_repo_lambda = _lambda.Function( self, "GitHubRepoAggregate", runtime=_lambda.Runtime.PYTHON_3_6, code=_lambda.Code.from_asset("lambda/package.zip"), handler="github-data-pull.github_repo_handler", role=lambda_role, timeout=core.Duration.minutes(15), ) # TODO: rate limit this as I am running out of tokens oss_datastore_pull_lambda = _lambda.Function( self, "GitHubDataHandler", runtime=_lambda.Runtime.PYTHON_3_6, code=_lambda.Code.from_asset("lambda/package.zip"), events=[lambda_events.SqsEventSource(sqs_queue)], handler="github-data-pull.github_data_handler", reserved_concurrent_executions=2, # To slow down the token drain role=lambda_role, timeout=core.Duration.minutes(15), ) # lambda scheduler lambda_rule = events.Rule( self, "Cron Rule", description="Setup cron schedule to get org data", schedule=events.Schedule.cron( # timezone is GMT and unconfigurable # adjust accordingly for your desired timezone minute="0", hour="7", month="*", year="*", week_day="*", ), targets=[targets.LambdaFunction(oss_datastore_repo_lambda)], )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # # Producer Lambda # event_producer_lambda = _lambda.Function( self, "eventProducerLambda", runtime=_lambda.Runtime.PYTHON_3_8, handler="event_producer_lambda.lambda_handler", code=_lambda.Code.from_asset("lambda")) event_policy = iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=['*'], actions=['events:PutEvents']) event_producer_lambda.add_to_role_policy(event_policy) # # Approved Consumer1 # event_consumer1_lambda = _lambda.Function( self, "eventConsumer1Lambda", runtime=_lambda.Runtime.PYTHON_3_8, handler="event_consumer_lambda.lambda_handler", code=_lambda.Code.from_asset("lambda")) event_consumer1_rule = events.Rule( self, 'eventConsumer1LambdaRule', description='Approved Transactions', event_pattern=events.EventPattern(source=['com.mycompany.myapp'])) event_consumer1_rule.add_target( targets.LambdaFunction(handler=event_consumer1_lambda)) # # Approved Consumer2 # event_consumer2_lambda = _lambda.Function( self, "eventConsumer2Lambda", runtime=_lambda.Runtime.PYTHON_3_8, handler="event_consumer_lambda.lambda_handler", code=_lambda.Code.from_asset("lambda")) event_consumer2_rule = events.Rule( self, 'eventConsumer2LambdaRule', description='Approved Transactions', event_pattern=events.EventPattern(source=['com.mycompany.myapp'])) event_consumer2_rule.add_target( targets.LambdaFunction(handler=event_consumer2_lambda)) # # Approved Consumer3 # # Create S3 bucket for KinesisFirehose destination ingest_bucket = s3.Bucket(self, 'test-ngest-bucket') # Create a Role for KinesisFirehose firehose_role = iam.Role( self, 'myRole', assumed_by=iam.ServicePrincipal('firehose.amazonaws.com')) # Create and attach policy that gives permissions to write in to the S3 bucket. iam.Policy( self, 's3_attr', policy_name='s3kinesis', statements=[ iam.PolicyStatement(actions=['s3:*'], resources=[ 'arn:aws:s3:::' + ingest_bucket.bucket_name + '/*' ]) ], # resources=['*'])], roles=[firehose_role], ) event_consumer3_kinesisfirehose = _firehose.CfnDeliveryStream( self, "consumer3-firehose", s3_destination_configuration=_firehose.CfnDeliveryStream. S3DestinationConfigurationProperty( bucket_arn=ingest_bucket.bucket_arn, buffering_hints=_firehose.CfnDeliveryStream. BufferingHintsProperty(interval_in_seconds=60), compression_format="UNCOMPRESSED", role_arn=firehose_role.role_arn)) event_consumer3_rule = events.Rule( self, 'eventConsumer3KinesisRule', description='Approved Transactions', event_pattern=events.EventPattern(source=['com.mycompany.myapp'])) event_consumer3_rule.add_target( targets.KinesisFirehoseStream( stream=event_consumer3_kinesisfirehose)) # defines an API Gateway REST API resource backed by our "atm_producer_lambda" function. api = api_gw.LambdaRestApi(self, 'SampleAPI-EventBridge-Multi-Consumer', handler=event_producer_lambda, proxy=False) items = api.root.add_resource("items") items.add_method("POST") # POST /items
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Kinesis to lambda self.stream_lambda = kinesis_lambda.KinesisStreamsToLambda( self, 'clickstream', lambda_function_props=_lambda.FunctionProps( runtime=_lambda.Runtime.PYTHON_3_7, handler='index.lambda_handler', code=_lambda.Code.inline( get_code('send_data_to_firehose.py'))), kinesis_stream_props=kinesis.StreamProps( stream_name='clickstream', retention_period=core.Duration.days(1), shard_count=4), kinesis_event_source_props=lambda_sources.KinesisEventSourceProps( starting_position=_lambda.StartingPosition.TRIM_HORIZON, batch_size=1)) # Lambda to produce data self.produce_fake_data = _lambda.Function( self, 'produce_data', runtime=_lambda.Runtime.PYTHON_3_7, timeout=core.Duration.seconds(90), handler='index.lambda_handler', code=_lambda.Code.inline(get_code('produce_data.py')), environment={ 'STREAM_NAME': self.stream_lambda.kinesis_stream.stream_name }) self.stream_lambda.kinesis_stream.grant_read_write( self.produce_fake_data) # EventBridge to activate my function above self.event_rule = events.Rule( self, 'scheduledRule', schedule=events.Schedule.expression('rate(1 minute)')) self.event_rule.add_target( targets.LambdaFunction(self.produce_fake_data)) # S3 Bucket self.bucket = s3.Bucket(self, 'data-clicks-lake', removal_policy=core.RemovalPolicy.DESTROY, auto_delete_objects=True) # Glue self.glue_db_analytical = glue.Database( self, 'analytic_clickstream', database_name='clickstream_db', location_uri=None, ) self.glue_table_analytical = glue.Table( self, 'analytical-table', table_name='analytical-table', columns=[ glue_column('custid', 'int'), glue_column('trafficfrom', 'string'), glue_column('url', 'string'), glue_column('device', 'string'), glue_column('touchproduct', 'int'), glue_column('trans_timestamp', 'string') ], database=self.glue_db_analytical, data_format=glue.DataFormat.PARQUET, bucket=self.bucket, s3_prefix='kinesis/', ) # Firehose iam_role_firehose_analytical = self.create_firehose_role() self.bucket.grant_read_write(iam_role_firehose_analytical) firehose_props = FirehoseProps( bucket=self.bucket, role=iam_role_firehose_analytical, stream=self.stream_lambda.kinesis_stream, glue_db=self.glue_db_analytical, glue_table=self.glue_table_analytical) self.firehose = FirehoseLib(self, 'firehose_clickstream', firehose_props) # Elasticsearh self.es_domain = ElasticsearchLib(self, 'ES-clickstream-domain').es_domain # Lambda to send data to Elasticsearch self.send_data_to_elasticsearch = lambda_python.PythonFunction( self, 'clickstream_to_es', entry='./analytics_ml_flow/lambda/lambda_with_requirements/', handler='handler', timeout=core.Duration.seconds(180), index='Kinesis_ES.py', environment={ 'ES_HOST_HTTP': self.es_domain.domain_endpoint, 'ES_INDEX': 'clickstream', 'ES_IND_TYPE': 'transactions', 'ES_REGION': 'us-west-2', }) self.es_domain.grant_index_read_write('clickstream', self.send_data_to_elasticsearch) self.es_domain.grant_read_write(self.send_data_to_elasticsearch) stream_source = lambda_sources.KinesisEventSource( self.stream_lambda.kinesis_stream, starting_position=_lambda.StartingPosition.TRIM_HORIZON, batch_size=1) self.stream_lambda.kinesis_stream.grant_read( self.send_data_to_elasticsearch) self.send_data_to_elasticsearch.add_event_source(stream_source) # Glue Crawler crawler_role = self.create_crawler_permissions() glue_props = GlueCrawlerProps(bucket=self.bucket, role=crawler_role) self.glue_crawler = GlueCrawlerLib(self, 'glueCrawler', glue_props)
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # Parameters notification_email_address = CfnParameter( self, "notification_email_address", type="String", min_length=7, description= "The E-mail address subscribed to notifications when an S3 bucket is detected as open to the public." ) profiling = CfnParameter( self, "profiling", type="String", allowed_values=["TRUE", "FALSE"], default="FALSE", description= "Enable Profiling on Lambda functions: TRUE or FALSE. Default: FALSE" ) tracing = CfnParameter( self, "tracing", type="String", allowed_values=["TRUE", "FALSE"], default="FALSE", description= "Enable tracing on Lambda functions: TRUE or FALSE. Default: FALSE" ) trusted_advisor_refresh_minutes = CfnParameter( self, "trusted_advisor_refresh_minutes", type="Number", default=6, min_value=5, max_value=1440, description= "Number of minutes to schedule a trusted advisor refresh. Default: 6" ) enable_profiling = profiling.value_as_string == 'TRUE' enable_tracing = aws_lambda.Tracing.ACTIVE if tracing.value_as_string != 'TRUE': enable_tracing = aws_lambda.Tracing.DISABLED # Layers dependencies_layer = aws_lambda.LayerVersion( self, "dependenciesLayer", code=aws_lambda.Code.from_asset( "lambda_functions/dependencies_layer/"), compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_8], ) # create SNS target email_notification_topic = sns.Topic( self, 'taEmailNotificationTopic', display_name='taEmailNotificationTopic', topic_name='taEmailNotificationTopic') # add subscription sns.Subscription(self, 'emailSubscription', protocol=sns.SubscriptionProtocol.EMAIL, endpoint=notification_email_address.value_as_string, topic=email_notification_topic) default_event_bus = events.EventBus.from_event_bus_name( self, 'default', 'default') ta_event_pattern = events.EventPattern( source=['aws.trustedadvisor'], detail_type=['Trusted Advisor Check Item Refresh Notification'], detail={ 'check-name': ['Amazon S3 Bucket Permissions'], 'status': ['WARN', 'ERROR'] }) # Lambda function to trigger when TA check flagged ta_check_s3_open_lambda_function_code = aws_lambda.AssetCode( 'lambda_functions/s3openbucket') ta_check_s3_open_lambda_function = aws_lambda.Function( self, 'ta_s3_open_bucket', code=ta_check_s3_open_lambda_function_code, runtime=aws_lambda.Runtime.PYTHON_3_8, handler='s3openbucket.lambda_handler', description='Function Triggered from Trusted Advisor ' 'to Block public access to an S3 Bucket', function_name='ta-check-s3-open-lambda-function', memory_size=128, profiling=enable_profiling, tracing=enable_tracing, log_retention=aws_logs.RetentionDays.ONE_WEEK, timeout=Duration.seconds(10), environment={'topic_arn': email_notification_topic.topic_arn}, initial_policy=[ aws_iam.PolicyStatement(actions=[ 's3:GetBucketPolicy', 's3:DeleteBucketPolicy', 's3:PutBucketPolicy', 's3:GetAccountPublicAccessBlock', 's3:GetBucketPublicAccessBlock', 's3:PutAccountPublicAccessBlock', 's3:PutBucketPublicAccessBlock', 's3:GetBucketAcl', 's3:GetObjectAcl', 's3:PutBucketAcl', 's3:PutObjectAcl' ], effect=aws_iam.Effect.ALLOW, resources=['*']), aws_iam.PolicyStatement( actions=['SNS:Publish'], effect=aws_iam.Effect.ALLOW, resources=[email_notification_topic.topic_arn]) ]) events.Rule( self, 's3PublicBucketRule', description= 'Blocks Public access on an S3 bucket once detected by Trusted Advisor', event_pattern=ta_event_pattern, event_bus=default_event_bus, targets=[targets.LambdaFunction(ta_check_s3_open_lambda_function)]) # Refresh TA check every X minutes # Lambda function to trigger when TA check flagged ta_refresh_lambda_function_code = aws_lambda.AssetCode( 'lambda_functions/refreshTrustedAdvisorCheck') ta_refresh_lambda_function = aws_lambda.Function( self, 'refresh_ta_check', code=ta_refresh_lambda_function_code, runtime=aws_lambda.Runtime.PYTHON_3_8, handler='refreshTrustedAdvisorCheck.lambda_handler', description='Refreshes Trusted Advisor checks', function_name='ta-refresh-ta-check-lambda-function', memory_size=128, profiling=enable_profiling, tracing=enable_tracing, log_retention=aws_logs.RetentionDays.ONE_WEEK, timeout=Duration.seconds(5), initial_policy=[ aws_iam.PolicyStatement(actions=[ 'support:DescribeTrustedAdvisorChecks', 'support:RefreshTrustedAdvisorCheck', 'support:DescribeTrustedAdvisorCheckResult' ], effect=aws_iam.Effect.ALLOW, resources=['*']) ]) ta_refresh_lambda_function.add_layers(dependencies_layer) events.Rule( self, 'refreshTAS3BucketPermissionsRule', schedule=events.Schedule.rate( Duration.minutes( trusted_advisor_refresh_minutes.value_as_number)), rule_name='refreshTAS3BucketPermissionsRule', targets=[targets.LambdaFunction(ta_refresh_lambda_function)])
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) replicator_role = iam.Role( scope=self, role_name='SecretsManagerRegionReplicatorRole', id='region-replicator-role', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'), inline_policies={ 'ReplicatorPermissions': iam.PolicyDocument(statements=[ iam.PolicyStatement(resources=['*'], actions=[ 'kms:Decrypt', 'kms:Encrypt', 'kms:GenerateDataKey', ]), iam.PolicyStatement(resources=[ 'arn:aws:secretsmanager:{region}:{account}:secret:*'. format(region=constants.ACCOUNT_REGION, account=constants.ACCOUNT_ID) ], actions=[ 'secretsmanager:DescribeSecret', 'secretsmanager:GetSecretValue' ]), iam.PolicyStatement( resources=[ 'arn:aws:secretsmanager:{region}:{account}:secret:*' .format(region=constants.TARGET_REGION_1, account=constants.ACCOUNT_ID) ], actions=[ 'secretsmanager:CreateSecret', 'secretsmanager:UpdateSecretVersionStage', 'secretsmanager:PutSecretValue', 'secretsmanager:DescribeSecret' ]), iam.PolicyStatement( resources=[ 'arn:aws:secretsmanager:{region}:{account}:secret:*' .format(region=constants.TARGET_REGION_2, account=constants.ACCOUNT_ID) ], actions=[ 'secretsmanager:CreateSecret', 'secretsmanager:UpdateSecretVersionStage', 'secretsmanager:PutSecretValue', 'secretsmanager:DescribeSecret' ]), ]) }, managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( managed_policy_name= 'service-role/AWSLambdaBasicExecutionRole'), ]) fn = _lambda.Function( scope=self, id='replicator-lambda', runtime=_lambda.Runtime.PYTHON_3_8, role=replicator_role, handler='index.handler', code=_lambda.Code.from_asset(path=path.join('lambda')), log_retention=logs.RetentionDays.ONE_WEEK, retry_attempts=0, environment={ 'TargetRegions': constants.TARGET_REGION_1 + ";" + constants.TARGET_REGION_2, }, ) rule = events.Rule( scope=self, id='event-rule', ) rule.add_target(target=targets.LambdaFunction(handler=fn)) rule.add_event_pattern(source=['aws.secretsmanager'], detail_type=['AWS API Call via CloudTrail'], region=[constants.ACCOUNT_REGION], detail={ "eventSource": ['secretsmanager.amazonaws.com'], "eventName": ['CreateSecret', 'PutSecretValue'] })
def __init__(self, scope: core.Construct, id: str, # pylint: disable=redefined-builtin lambda_notifications: aws_lambda.IFunction, **kwargs) -> None: super().__init__(scope, id, **kwargs) # CloudWatch LogGroup and Stream to store 'since' timestamp value since_log_group = aws_logs.LogGroup( self, f"{id}-log-group", log_group_name=f"{id}-timestamps", retention=DEFAULT_LOG_RETENTION, removal_policy=core.RemovalPolicy.DESTROY, ) since_log_group.add_stream( f"{id}-log-stream", log_stream_name=since_log_group.log_group_name, ) # Lambda shared code lambda_code = code_from_path(path=f"lib/stacks/{id}/lambdas") # Lambda create_doc (and layers): build document file and store to S3 bucket bucket = get_bucket(self, f"{id}-bucket") lambda_create_doc = get_lambda( self, id + "-create-document", code=lambda_code, handler="create_doc.handler", environment={ "DOCUMENT_BUCKET": bucket.bucket_name, }, layers=[get_layer(self, layer_name=layer, prefix=id) for layer in ("readability", "requests_oauthlib")], timeout=core.Duration.minutes(5), # pylint: disable=no-value-for-parameter ) bucket.grant_write(lambda_create_doc) # Lambda send_to_kindle: invoked when new documents dropped into S3 bucket, # deliver document as email attachment via lambda_notifications lambda_send_to_kindle = get_lambda( self, id + "-send-to-kindle", code=lambda_code, handler="send_to_kindle.handler", environment={ "KINDLE_EMAIL": env["KINDLE_EMAIL"], "LAMBDA_NOTIFICATIONS": lambda_notifications.function_name, "DOCUMENT_SRC_BUCKET": bucket.bucket_name, "POCKET_CONSUMER_KEY": env["POCKET_CONSUMER_KEY"], "POCKET_SECRET_TOKEN": env["POCKET_SECRET_TOKEN"], } ) bucket.add_event_notification( event=aws_s3.EventType.OBJECT_CREATED_PUT, dest=aws_s3_notifications.LambdaDestination(lambda_send_to_kindle), ) lambda_notifications.grant_invoke(lambda_send_to_kindle) aws_iam.Policy( self, f"{id}-mail-attachment-policy", roles=[lambda_notifications.role], statements=[ aws_iam.PolicyStatement( actions=["s3:GetObject"], resources=[f"{bucket.bucket_arn}/*"] ) ], ) # Lambda reader: fetch new articles from Pocket and fan-out trigger create_doc Lambda lambda_reader = get_lambda( self, id + "-reader", code=lambda_code, handler="reader.handler", environment={ "LAMBDA_PUBLISHER": lambda_create_doc.function_name, "POCKET_CONSUMER_KEY": env["POCKET_CONSUMER_KEY"], "POCKET_SECRET_TOKEN": env["POCKET_SECRET_TOKEN"], "SINCE_LOG_GROUP": since_log_group.log_group_name, }, ) since_log_group.grant( lambda_reader, "logs:GetLogEvents", "logs:PutLogEvents", ) lambda_create_doc.grant_invoke(lambda_reader) # Cloudwatch cronjob event to check for new articles every hour cronjob = aws_events.Rule( self, f"{id}-scheduled-event", enabled=True, schedule=aws_events.Schedule.cron(minute="0"), # pylint: disable=no-value-for-parameter ) cronjob.add_target(aws_events_targets.LambdaFunction(handler=lambda_reader))
def __init__(self, scope: core.Construct, id: str, *, email: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Code Asset lamba_code = lambda_.AssetCode("./assets/") # DynamoDB dynamo_store_db = dynamo.Table(self,"products_to_check_db", partition_key=dynamo.Attribute(name="ProductTs",type=dynamo.AttributeType.NUMBER)) # SNS Topics sns_input_topic = sns.Topic(self,"checker_url_topic") sns_output_topic = sns.Topic(self,"email_topic") # Lambda function that scrapes the pages & emails lambda_checker = lambda_.Function( self, "lambda_checker", code=lamba_code, handler="checker.handler", timeout=core.Duration.seconds(60), runtime=lambda_.Runtime.NODEJS_12_X, environment= { "TOPIC_ARN": sns_output_topic.topic_arn, "DYNAMO_TABLE": dynamo_store_db.table_name } ) # Subscribe to SNS sns_input_topic.add_subscription(subs.LambdaSubscription(lambda_checker)) sns_output_topic.add_subscription(subs.EmailSubscription(email)) # Lambda function that populates SNS lambda_invoker = lambda_.Function( self, "lambda_invoker", code=lamba_code, handler="invoker.handler", timeout=core.Duration.seconds(300), runtime=lambda_.Runtime.NODEJS_12_X, environment= { "TOPIC_ARN": sns_input_topic.topic_arn, "DYNAMO_TABLE": dynamo_store_db.table_name } ) # Grant access to publish on SNS topics sns_input_topic.grant_publish(lambda_invoker) sns_output_topic.grant_publish(lambda_checker) # Grant access to Dynamo for lambdas dynamo_store_db.grant_read_data(lambda_invoker) dynamo_store_db.grant_read_write_data(lambda_checker) # Run every day at 05:00 UTC # See https://docs.aws.amazon.com/lambda/latest/dg/tutorial-scheduled-events-schedule-expressions.html rule = events.Rule( self, "runEveryDayAt5AM", schedule=events.Schedule.cron( minute='0', hour='5', month='*', week_day='*', year='*'), ) rule.add_target(targets.LambdaFunction(lambda_invoker))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here lambda_policies = [iam.PolicyStatement( actions=[ "logs:CreateLogStream", "logs:PutLogEvents", "logs:CreateLogGroup"], effect=iam.Effect.ALLOW, resources=["arn:aws:logs:" + core.Aws.REGION + ":" + core.Aws.ACCOUNT_ID + ":*"] ), iam.PolicyStatement( actions=[ "dynamodb:*"], effect=iam.Effect.ALLOW, resources=["arn:aws:dynamodb:" + core.Aws.REGION + ":" + core.Aws.ACCOUNT_ID + ":*"] )] base_api = _apigw.RestApi(self, 'PetclinicApiGatewayWithCors', rest_api_name='PetclinicApiGatewayWithCors') api_resource = base_api.root.add_resource('api') website_bucket = _s3.Bucket(self, 'PetclinicWebsite', website_index_document='index.html', public_read_access=True, removal_policy=core.RemovalPolicy.DESTROY ) deployment = _s3deploy.BucketDeployment(self, 'PetclinicDeployWebsite', sources=[_s3deploy.Source.asset('./spring-petclinic-static')], destination_bucket=website_bucket, retain_on_delete=False #destination_key_prefix='web/static' ) # Modify the config.js with CF custome resource modify_policy = [iam.PolicyStatement( actions=[ "s3:PutObject","s3:PutObjectAcl","s3:PutObjectVersionAcl","s3:GetObject"], effect=iam.Effect.ALLOW, resources=[website_bucket.bucket_arn + "/*"] ),iam.PolicyStatement( actions=[ "s3:ListBucket"], effect=iam.Effect.ALLOW, resources=[website_bucket.bucket_arn] ),iam.PolicyStatement( actions=[ "dynamodb:*"], effect=iam.Effect.ALLOW, resources=["arn:aws:dynamodb:" + core.Aws.REGION + ":" + core.Aws.ACCOUNT_ID + ":*"] )] with open("custom-resource-code/init.py", encoding="utf-8") as fp: code_body = fp.read() dynamodb_tables = [] # Warm Lambda function Event rule event_rule = _event.Rule(self, 'PetclinicLambdaWarmRule', schedule=_event.Schedule.rate(core.Duration.minutes(3)) ) for service in ['customer', 'vet', 'visit']: table = _dynamodb.Table(self, service.capitalize() + 'Table', partition_key={ 'name': 'id', 'type': _dynamodb.AttributeType.STRING }, removal_policy=core.RemovalPolicy.DESTROY, read_capacity=5, write_capacity=5, ) dynamodb_tables.append(table.table_name) base_lambda = _lambda.Function(self,'ApiPetclinic' + service.capitalize() + 'Lambda', handler='org.springframework.samples.petclinic.' + service + 's.StreamLambdaHandler::handleRequest', runtime=_lambda.Runtime.JAVA_8, code=_lambda.Code.asset('./spring-petclinic-serverless/spring-petclinic-' + service +'s-serverless/target/spring-petclinic-' + service +'s-serverless-2.0.7.jar'), memory_size=1024, timeout=core.Duration.seconds(300), initial_policy=lambda_policies, environment={"DYNAMODB_TABLE_NAME":table.table_name, "SERVER_SERVLET_CONTEXT_PATH":"/api/" + service} ) entity = api_resource.add_resource(service) entity.add_proxy(default_integration=_apigw.LambdaIntegration(base_lambda)) self.add_cors_options(entity) event_rule.add_target(_target.LambdaFunction(handler=base_lambda)) resource = _cfn.CustomResource(self, "S3ModifyCustomResource", provider=_cfn.CustomResourceProvider.lambda_( _lambda.SingletonFunction( self, "CustomResourceSingleton", uuid="f7d4f730-4ee1-11e8-9c2d-fa7ae01bbebc", code=_lambda.InlineCode(code_body), handler="index.handler", timeout=core.Duration.seconds(300), runtime=_lambda.Runtime.PYTHON_3_7, initial_policy=modify_policy ) ), properties={"Bucket": website_bucket.bucket_name, "InvokeUrl":base_api.url, "DynamoDBTables": dynamodb_tables } ) core.CfnOutput(self,"PetclinicWebsiteUrl",export_name="PetclinicWebsiteUrl",value=website_bucket.bucket_website_url)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # # Producer Lambda # atm_producer_lambda = _lambda.Function( self, "atmProducerLambda", runtime=_lambda.Runtime.NODEJS_12_X, handler="handler.lambdaHandler", code=_lambda.Code.from_asset("lambda_fns/atmProducer")) event_policy = iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=['*'], actions=['events:PutEvents']) atm_producer_lambda.add_to_role_policy(event_policy) # # Approved Transaction Consumer # atm_consumer1_lambda = _lambda.Function( self, "atmConsumer1Lambda", runtime=_lambda.Runtime.NODEJS_12_X, handler="handler.case1Handler", code=_lambda.Code.from_asset("lambda_fns/atmConsumer")) atm_consumer1_rule = events.Rule(self, 'atmConsumer1LambdaRule', description='Approved Transactions', event_pattern=events.EventPattern( source=['custom.myATMapp'], detail_type=['transaction'], detail={"result": ["approved"]})) atm_consumer1_rule.add_target( targets.LambdaFunction(handler=atm_consumer1_lambda)) # # NY Prefix Consumer # atm_consumer2_lambda = _lambda.Function( self, "atmConsumer2Lambda", runtime=_lambda.Runtime.NODEJS_12_X, handler="handler.case2Handler", code=_lambda.Code.from_asset("lambda_fns/atmConsumer")) atm_consumer2_rule = events.Rule( self, 'atmConsumer2LambdaRule', event_pattern=events.EventPattern( source=['custom.myATMapp'], detail_type=['transaction'], detail={"location": [{ "prefix": "NY-" }]})) atm_consumer2_rule.add_target( targets.LambdaFunction(handler=atm_consumer2_lambda)) # # Not Approved Consumer # atm_consumer3_lambda = _lambda.Function( self, "atmConsumer3Lambda", runtime=_lambda.Runtime.NODEJS_12_X, handler="handler.case3Handler", code=_lambda.Code.from_asset("lambda_fns/atmConsumer")) atm_consumer3_rule = events.Rule( self, 'atmConsumer3LambdaRule', event_pattern=events.EventPattern( source=['custom.myATMapp'], detail_type=['transaction'], detail={"result": [{ "anything-but": "approved" }]})) atm_consumer3_rule.add_target( targets.LambdaFunction(handler=atm_consumer3_lambda)) # defines an API Gateway REST API resource backed by our "atm_producer_lambda" function. api_gw.LambdaRestApi(self, 'Endpoint', handler=atm_producer_lambda)
def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) codeLocation = 'lambdas' layerLocation = self.installRequirements(codeLocation) self.ip = self.getIp() self.vpc = self.createVpc() self.lambdaRole = self.createLambdaRole() self.lambdaCode = lambda_.Code.from_asset(codeLocation) self.lambdaLayer = lambda_.LayerVersion(self, 'lambdaLayer', code=lambda_.Code.from_asset(layerLocation), compatible_runtimes=[ lambda_.Runtime.PYTHON_3_8 ] ) self.statesRole = iam.Role(self, 'statesExecutionRole', assumed_by=iam.ServicePrincipal('states.amazonaws.com'), inline_policies={ 'StatesExecutionPolicy': iam.PolicyDocument( statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['lambda:InvokeFunction'], resources=['*'] ) ] ) } ) api = agw.RestApi(self, 'lntipbot', endpoint_types=[agw.EndpointType.REGIONAL], deploy_options=agw.StageOptions( metrics_enabled=True ) ) api.root.add_resource('info', default_integration=agw.MockIntegration( integration_responses=[ agw.IntegrationResponse( status_code='301', response_parameters={ 'method.response.header.Location': '\'https://www.reddit.com/r/LNTipBot2/wiki/index\'', 'method.response.header.Cache-Control': '\'max-age=300\'' } ) ], request_templates={ 'application/json': '{"statusCode": 301}' } )).add_method('GET', method_responses=[{ 'statusCode': '301', 'responseParameters': { 'method.response.header.Location': True, 'method.response.header.Cache-Control': True } }] ) api.root.add_resource('uri', default_integration=agw.LambdaIntegration( self.createLambda('invoiceUriFunction', 'getURI.getURI') )).add_method('GET') api.root.add_resource('qr', default_integration=agw.LambdaIntegration( self.createLambda('qrFunction', 'qrEncoder.qrEncoder') )).add_method('GET') events.Rule(self, 'oauthRefreshEvent', schedule=events.Schedule.rate(cdk.Duration.minutes(28)), targets=[eventsTargets.LambdaFunction( self.createLambda('oauthFunction', 'redditOAuthRequester.redditOAuthRequester') )] ) self.settledInvoiceHandler = self.createLambda('settledInvoiceHandler', 'settledInvoiceHandler.settledInvoiceHandler') self.createLambda('apiTest', 'lambda_function.lambda_handler') withdrawWorkflow = self.createWithdrawWorkflow() tipWorkflow = self.createTipWorkflow() events.Rule(self, 'redditCommentScannerEvent', schedule=events.Schedule.rate(cdk.Duration.minutes(1)), targets=[eventsTargets.LambdaFunction( lambda_.Function(self, 'redditCommentScanner', code=self.lambdaCode, runtime=lambda_.Runtime.PYTHON_3_8, handler='scanComments.scannerLoop', role=self.lambdaRole, layers=[self.lambdaLayer], timeout=cdk.Duration.seconds(55), reserved_concurrent_executions=1 ), event=events.RuleTargetInput.from_object({ 'tipWorkflowArn': tipWorkflow.state_machine_arn, 'withdrawWorkflowArn': withdrawWorkflow.state_machine_arn }) )] ) self.backupBucket = s3.Bucket(self, 'bitcoindBackups', block_public_access=s3.BlockPublicAccess.BLOCK_ALL, bucket_name='bitcoind-pruned-backups-lntipbot', ) self.serverRole = self.createServerRole() self.securityGroup = self.createSecurityGroup() self.createServer() self.createOps()
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Let us create the DENY ALL policy deny_iam_policy_statement = _iam.PolicyStatement( resources=['*'], actions=['iam:*'], effect=_iam.Effect.DENY, # sid='DenyIAMPermissions' ) deny_iam_policy_statement.sid = "DenyIAMPermissions" deny_iam_policy = _iam.ManagedPolicy( self, "deny_iam_policy", description="A policy to deny IAM permissions", managed_policy_name="deny_iam_privileges", statements=[deny_iam_policy_statement]) # Lambda Function that will quarantine the user try: with open("./lambda_src/revoke_iam_privileges.py", mode='r') as file: revoke_iam_privileges_fn_handler_code = file.read() except OSError as e: print(f'Unable to read file. ERROR:{str(e)}') revoke_iam_privileges_fn = _lambda.Function( self, id='revokeIamPrivilegesFnId', function_name="revoke_iam_privileges_fn", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(revoke_iam_privileges_fn_handler_code), handler='index.lambda_handler', timeout=core.Duration.seconds(5), environment={ "ADMIN_GROUP_NAME": global_args.ADMIN_GROUP_NAME, "DENY_IAM_POLICY_ARN": f"{deny_iam_policy.managed_policy_arn}" }) revoke_iam_privileges_fn_perms = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, resources=[ "*", ], actions=[ "iam:AttachRolePolicy", "iam:AttachUserPolicy", "iam:ListAttachedUserPolicies", "iam:ListGroupsForUser", "iam:PutUserPolicy", ]) revoke_iam_privileges_fn_perms.sid = "AllowLambdaToQuarantineUser" revoke_iam_privileges_fn.add_to_role_policy( revoke_iam_privileges_fn_perms) # Cloudwatch IAM Events Rule iam_evant_validator_targets = [] iam_evant_validator_targets.append( _targets.LambdaFunction(handler=revoke_iam_privileges_fn)) iam_event_pattern = _events.EventPattern( source=["aws.iam"], detail_type=["AWS API Call via CloudTrail"], detail={ "eventSource": ["iam.amazonaws.com"], "userIdentity": { "type": ["IAMUser"] } }) """ # Dedicted Event Bus for Security sec_event_bus = _events.EventBus( self, "securityEventBusId", event_bus_name=f"{global_args.OWNER}_security_event_bus" ) """ # Event Rule to trigger Lambda iam_event_rule = _events.Rule( self, "iamEventRuleId", event_pattern=iam_event_pattern, rule_name=f"iam_event_pattern_{global_args.OWNER}", # event_bus=sec_event_bus, enabled=True, description="Trigger an event for IAM Events", targets=iam_evant_validator_targets) # Lets create a cloudtrail to track API events _event_trail = _cloudtrail.Trail(self, "cloudEventTrailId", is_multi_region_trail=False, include_global_service_events=True, enable_file_validation=False, send_to_cloud_watch_logs=False) ########################################### ################# OUTPUTS ################# ########################################### output0 = core.CfnOutput( self, "SecuirtyAutomationBy", value=f"{global_args.SOURCE_INFO}", description= "To know more about this automation stack, check out our github page." ) output1 = core.CfnOutput( self, "LambdaFunction", value=(f"https://console.aws.amazon.com/lambda/home?region=" f"{core.Aws.REGION}" f"#functions/" f"{revoke_iam_privileges_fn.function_name}"), description="The Quarantine Lambda Function") output2 = core.CfnOutput( self, "CreateUser", value=(f"aws iam create-user --user-name Mystique$RANDOM"), description="command to create users") output3 = core.CfnOutput( self, "DeleteUser", value=(f"aws iam delete-user --user-name Mystique*"), description="command to delete users")
def __init__( self, scope: core.Construct, id: str, # pylint: disable=redefined-builtin lambda_notifications: aws_lambda.IFunction, **kwargs) -> None: super().__init__(scope, id, **kwargs) # CloudWatch LogGroup and Stream to store 'since' timestamp value since_log_group = aws_logs.LogGroup( self, f"{id}-log-group", log_group_name=f"{id}-timestamps", retention=DEFAULT_LOG_RETENTION, removal_policy=core.RemovalPolicy.DESTROY, ) since_log_group.add_stream( f"{id}-log-stream", log_stream_name=since_log_group.log_group_name, ) # Lambda shared code lambda_code = code_from_path(path=f"lib/stacks/{id}/lambdas") # Lambda create_epub (and layers): build epub file and store to S3 bucket epub_bucket = get_bucket(self, f"{id}-epub-bucket") lambda_create_epub = get_lambda( self, id + "-create-epub", code=lambda_code, handler="create_epub.handler", environment={ "EPUB_BUCKET": epub_bucket.bucket_name, }, layers=[ get_layer(self, layer_name=layer, prefix=id) for layer in ("pandoc", "html2text", "requests_oauthlib") ], timeout=core.Duration.minutes(5), # pylint: disable=no-value-for-parameter ) epub_bucket.grant_write(lambda_create_epub) # Lambda send_to_kindle: invoked when new MOBI dropped into S3 bucket, deliver MOBI as # email attachment via lambda_notifications mobi_bucket = get_bucket(self, f"{id}-mobi-bucket") lambda_send_to_kindle = get_lambda( self, id + "-send-to-kindle", code=lambda_code, handler="send_to_kindle.handler", environment={ "KINDLE_EMAIL": env["KINDLE_EMAIL"], "LAMBDA_NOTIFICATIONS": lambda_notifications.function_name, "MOBI_SRC_BUCKET": mobi_bucket.bucket_name, "POCKET_CONSUMER_KEY": env["POCKET_CONSUMER_KEY"], "POCKET_SECRET_TOKEN": env["POCKET_SECRET_TOKEN"], }) mobi_bucket.add_event_notification( event=aws_s3.EventType.OBJECT_CREATED_PUT, dest=aws_s3_notifications.LambdaDestination(lambda_send_to_kindle), ) lambda_notifications.grant_invoke(lambda_send_to_kindle) aws_iam.Policy( self, f"{id}-mail-attachment-policy", roles=[lambda_notifications.role], statements=[ aws_iam.PolicyStatement( actions=["s3:GetObject"], resources=[f"{mobi_bucket.bucket_arn}/*"]) ], ) # Lambda reader: fetch new articles from Pocket and fan-out trigger create_epub Lambda lambda_reader = get_lambda( self, id + "-reader", code=lambda_code, handler="reader.handler", environment={ "LAMBDA_PUBLISHER": lambda_create_epub.function_name, "POCKET_CONSUMER_KEY": env["POCKET_CONSUMER_KEY"], "POCKET_SECRET_TOKEN": env["POCKET_SECRET_TOKEN"], "SINCE_LOG_GROUP": since_log_group.log_group_name, }, ) since_log_group.grant( lambda_reader, "logs:GetLogEvents", "logs:PutLogEvents", ) lambda_create_epub.grant_invoke(lambda_reader) # Fargate task: run dockerized `kindlegen` to parse EPUB to MOBI, # triggered by trigger_ecs_task Lambda # https://medium.com/@piyalikamra/s3-event-based-trigger-mechanism-to-start-ecs-far-gate-tasks-without-lambda-32f57ed10b0d cluster, vpc = get_fargate_cluster(self, id) mem_limit = "512" task = get_fargate_task(self, id, mem_limit) aws_iam.Policy( self, f"{id}-bucket-policy", roles=[task.task_role], statements=[ aws_iam.PolicyStatement( actions=["s3:GetObject"], resources=[f"{epub_bucket.bucket_arn}/*"]), aws_iam.PolicyStatement( actions=["s3:PutObject"], resources=[f"{mobi_bucket.bucket_arn}/*"]), ], ) container = get_fargate_container(self, id, task, mem_limit) # Lambda trigger_ecs_task: trigger Fargate task when new EPUB file is dropped into epub_bucket lambda_trigger_ecs_task = get_lambda( self, f"{id}-trigger-ecs-task", code=lambda_code, handler="trigger_ecs_task.handler", environment={ "ECS_CLUSTER": cluster.cluster_arn, "ECS_CLUSTER_SECURITY_GROUP": vpc.vpc_default_security_group, "ECS_CLUSTER_SUBNET": vpc.public_subnets[0].subnet_id, "ECS_CONTAINER": container.container_name, "ECS_TASK": task.task_definition_arn, "MOBI_DEST_BUCKET": mobi_bucket.bucket_name, }, ) epub_bucket.add_event_notification( event=aws_s3.EventType.OBJECT_CREATED_PUT, dest=aws_s3_notifications.LambdaDestination( lambda_trigger_ecs_task), ) aws_iam.Policy( self, f"{id}-lambda-trigger-policy", roles=[lambda_trigger_ecs_task.role], statements=[ aws_iam.PolicyStatement( actions=["ecs:RunTask"], resources=[task.task_definition_arn], ), aws_iam.PolicyStatement( actions=["iam:PassRole"], resources=[ task.execution_role.role_arn, task.task_role.role_arn, ], ) ], ) # Cloudwatch cronjob event to check for new articles every hour cronjob = aws_events.Rule( self, f"{id}-scheduled-event", enabled=True, schedule=aws_events.Schedule.cron(minute="0"), # pylint: disable=no-value-for-parameter ) cronjob.add_target( aws_events_targets.LambdaFunction(handler=lambda_reader))
def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) log_bucket_name = cdk.Fn.import_value('sime-log-bucket-name') service_role_kdf_to_s3 = cdk.Fn.import_value( 'siem-kdf-to-s3-role-name') cwe_frequency = cdk.CfnParameter( self, 'cweRulesFrequency', type='Number', description=( 'How often do you get WorkSpaces Inventory? (every minutes)'), default=720) kdf_workspaces_name = cdk.CfnParameter( self, 'KdfWorkSpacesName', description=( 'Kinesis Data Firehose Name to deliver workspaces event'), default='siem-workspaces-event-to-s3', ) kdf_buffer_size = cdk.CfnParameter( self, 'KdfBufferSize', type='Number', description='Enter a buffer size between 1 - 128 (MiB)', default=1, min_value=1, max_value=128) kdf_buffer_interval = cdk.CfnParameter( self, 'KdfBufferInterval', type='Number', description='Enter a buffer interval between 60 - 900 (seconds.)', default=60, min_value=60, max_value=900) role_get_workspaces_inventory = aws_iam.Role( self, 'getWorkspacesInventoryRole', role_name='siem-get-workspaces-inventory-role', inline_policies={ 'describe-workspaces': aws_iam.PolicyDocument(statements=[ aws_iam.PolicyStatement( actions=['workspaces:Describe*'], resources=['*'], sid='DescribeWorkSpacesPolicyGeneratedBySeimCfn') ]), 'firehose-to-s3': aws_iam.PolicyDocument(statements=[ aws_iam.PolicyStatement( actions=['s3:PutObject'], resources=[f'arn:aws:s3:::{log_bucket_name}/*'], sid='FirehoseToS3PolicyGeneratedBySeimCfn') ]) }, managed_policies=[ aws_iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaBasicExecutionRole'), ], assumed_by=aws_iam.ServicePrincipal('lambda.amazonaws.com')) # Lambda Functions to get workspaces inventory lambda_func = aws_lambda.Function( self, 'lambdaGetWorkspacesInventory', runtime=aws_lambda.Runtime.PYTHON_3_8, code=aws_lambda.InlineCode(LAMBDA_GET_WORKSPACES_INVENTORY), function_name='siem-get-workspaces-inventory', description='SIEM: get workspaces inventory', handler='index.lambda_handler', timeout=cdk.Duration.seconds(300), role=role_get_workspaces_inventory, environment={'log_bucket_name': log_bucket_name}) rule = aws_events.Rule(self, 'eventBridgeRuleWorkSpaceInventory', rule_name='siem-workspaces-inventory-to-lambda', schedule=aws_events.Schedule.rate( cdk.Duration.minutes( cwe_frequency.value_as_number))) rule.add_target(aws_events_targets.LambdaFunction(lambda_func)) kdf_to_s3 = aws_kinesisfirehose.CfnDeliveryStream( self, "KDFForWorkSpacesEvent", delivery_stream_name=kdf_workspaces_name.value_as_string, s3_destination_configuration=CDS. S3DestinationConfigurationProperty( bucket_arn=f'arn:aws:s3:::{log_bucket_name}', prefix=f'AWSLogs/{cdk.Aws.ACCOUNT_ID}/WorkSpaces/Event/', compression_format='GZIP', buffering_hints=CDS.BufferingHintsProperty( interval_in_seconds=kdf_buffer_interval.value_as_number, size_in_m_bs=kdf_buffer_size.value_as_number), role_arn=(f'arn:aws:iam::{cdk.Aws.ACCOUNT_ID}:role/' f'service-role/{service_role_kdf_to_s3}'))) pattern = aws_events.EventPattern(detail_type=["WorkSpaces Access"], source=['aws.workspaces']) aws_events.Rule( self, 'eventBridgeRuleWorkSpacesEvent', event_pattern=pattern, rule_name='siem-workspaces-event-to-kdf', targets=[aws_events_targets.KinesisFirehoseStream(kdf_to_s3)])
def __init__(self, app: cdk.App, id: str) -> None: super().__init__(app, id) with open("lambda-handler.py", encoding="utf8") as fp: handler_code = fp.read() principals_actions_json = cdk.CfnParameter( self, "PrincipalsActionsJSON", type="String", default="{}", ) alert_notification_email = cdk.CfnParameter( self, "AlertNotificationEmail", type="String", default="*****@*****.**", ) role = iam.Role( self, "CheckPrincipalsActionsLambdaRole", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaBasicExecutionRole") ]) role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=["iam:SimulatePrincipalPolicy"], resources=["*"])) lambdaFn = lambda_.Function( self, "CheckPrincipalsActions", code=lambda_.InlineCode(handler_code), handler="index.lambda_handler", memory_size=128, timeout=cdk.Duration.seconds(10), runtime=lambda_.Runtime.PYTHON_3_8, environment={ "principals_actions_json": principals_actions_json.value_as_string }, description= "Check actions assigned to an IAM users, roles, or groups", role=role, ) rule = events.Rule( self, "CheckPrincipalsActionsEventScheduler", schedule=events.Schedule.rate(cdk.Duration.minutes(1)), ) rule.add_target(targets.LambdaFunction(lambdaFn)) topic = sns.Topic(self, "CheckPrincipalsActionsLambdaErrorTopic") topic.add_subscription( subscriptions.EmailSubscription( alert_notification_email.value_as_string)) metric = lambdaFn.metric("Errors").with_( period=cdk.Duration.seconds(60), statistic="Sum") alarm = metric.create_alarm( self, "CheckPrincipalsActionsAlarm", threshold=1, evaluation_periods=1, datapoints_to_alarm=1, ) alarm.add_alarm_action(actions.SnsAction(topic))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) env = kwargs['env'] work_dir = pathlib.Path(__file__).parents[1] # These below steps allows to reuse ecs cluster which is aleady creatd by shared stack # Get cluster name from ssm parameter cluster_name = ssm.StringParameter.from_string_parameter_name( self, "GetClusterName", string_parameter_name="/dev/compute/container/ecs-cluster-name" ).string_value vpc_az = ssm.StringListParameter.from_string_list_parameter_name( self, "GetVpcAz", string_list_parameter_name="/dev/network/vpc/vpc-az" ).string_list_value # using string instead of stringlist because of subnets parsing issue vpc_public_subnets_1 = ssm.StringParameter.from_string_parameter_name( self, "GetVpcPublicSubnets1", string_parameter_name="/dev/network/vpc/vpc-public-subnets-1" ).string_value vpc_public_subnets_2 = ssm.StringParameter.from_string_parameter_name( self, "GetVpcPublicSubnets2", string_parameter_name="/dev/network/vpc/vpc-public-subnets-2" ).string_value vpc_id = ssm.StringParameter.from_string_parameter_name( self, "GetVpcId", string_parameter_name="/dev/network/vpc/vpc-id").string_value ec2_vpc = ec2.Vpc.from_vpc_attributes( self, "GetVpc", availability_zones=vpc_az, vpc_id=vpc_id, public_subnet_ids=[vpc_public_subnets_1, vpc_public_subnets_2]) # Get security group id from ssm parameter security_group_id = ssm.StringParameter.from_string_parameter_name( self, "GetSgId", string_parameter_name="/dev/network/vpc/security-group-id" ).string_value # Get security group from lookup ec2_sgp = ec2.SecurityGroup.from_security_group_id( self, "GetSgp", security_group_id=security_group_id) # myDateTimeFunction lambda function my_datetime_lambda = _lambda.Function( self, "my-datetime", runtime=_lambda.Runtime.NODEJS_12_X, handler="myDateTimeFunction.handler", code=_lambda.Code.asset("./lambda"), current_version_options=_lambda.VersionOptions( removal_policy=core.RemovalPolicy.RETAIN, retry_attempts=1)) my_datetime_lambda.add_to_role_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=["lambda:InvokeFunction"], resources=["*"])) # beforeAllowTraffic lambda function pre_traffic_lambda = _lambda.Function( self, "pre-traffic", runtime=_lambda.Runtime.NODEJS_12_X, handler="beforeAllowTraffic.handler", code=_lambda.Code.asset("./lambda"), environment=dict( NewVersion=my_datetime_lambda.current_version.function_arn)) pre_traffic_lambda.add_to_role_policy( iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=["codedeploy:PutLifecycleEventHookExecutionStatus"], resources=["*"])) pre_traffic_lambda.add_to_role_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=["lambda:InvokeFunction"], resources=["*"])) # afterAllowTraffic lambda function post_traffic_lambda = _lambda.Function( self, "post-traffic", runtime=_lambda.Runtime.NODEJS_12_X, handler="afterAllowTraffic.handler", code=_lambda.Code.asset("./lambda"), environment=dict( NewVersion=my_datetime_lambda.current_version.function_arn)) post_traffic_lambda.add_to_role_policy( iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=["codedeploy:PutLifecycleEventHookExecutionStatus"], resources=["*"])) post_traffic_lambda.add_to_role_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=["lambda:InvokeFunction"], resources=["*"])) # create a cloudwatch event rule rule = events.Rule( self, "CanaryRule", schedule=events.Schedule.expression("rate(10 minutes)"), targets=[ events_targets.LambdaFunction( my_datetime_lambda.current_version) ], ) # create a cloudwatch alarm based on the lambda erros metrics alarm = cloudwatch.Alarm( self, "CanaryAlarm", metric=my_datetime_lambda.current_version.metric_invocations(), threshold=0, evaluation_periods=2, datapoints_to_alarm=2, treat_missing_data=cloudwatch.TreatMissingData.IGNORE, period=core.Duration.minutes(5), alarm_name="CanaryAlarm") lambda_deployment_group = codedeploy.LambdaDeploymentGroup( self, "datetime-lambda-deployment", alias=my_datetime_lambda.current_version.add_alias("live"), deployment_config=codedeploy.LambdaDeploymentConfig.ALL_AT_ONCE, alarms=[alarm], auto_rollback=codedeploy.AutoRollbackConfig( deployment_in_alarm=True), pre_hook=pre_traffic_lambda, post_hook=post_traffic_lambda) # Pass vpc, sgp and ecs cluster name to get ecs cluster info ecs_cluster = ecs.Cluster.from_cluster_attributes( self, "GetEcsCluster", cluster_name=cluster_name, vpc=ec2_vpc, security_groups=[ec2_sgp]) # Fargate Service task_definition = ecs.FargateTaskDefinition( self, "TaskDef", memory_limit_mib=512, cpu=256, ) container = task_definition.add_container( "web", image=ecs.ContainerImage.from_asset( os.path.join(work_dir, "container")), # Built custom health check for your application specific # and add them here. Ex: Pingcheck, Database etc. health_check=ecs.HealthCheck(command=["CMD-SHELL", "echo"]), # environment=dict(name="latest") ) port_mapping = ecs.PortMapping(container_port=8000, protocol=ecs.Protocol.TCP) container.add_port_mappings(port_mapping) # Create Fargate Service # Current limitation: Blue/Green deployment # https://github.com/aws/aws-cdk/issues/1559 service = ecs.FargateService( self, "Service", cluster=ecs_cluster, task_definition=task_definition, assign_public_ip=True, deployment_controller=ecs.DeploymentController( type=ecs.DeploymentControllerType.ECS), desired_count=2, min_healthy_percent=50) # Create Application LoadBalancer lb = elbv2.ApplicationLoadBalancer(self, "LB", vpc=ec2_vpc, internet_facing=True) # Add listener to the LB listener = lb.add_listener("Listener", port=80, open=True) # Default to Lambda listener.add_targets( "Lambda", targets=[elb_targets.LambdaTarget(my_datetime_lambda)]) # Additionally route to container listener.add_targets("Fargate", port=8000, path_pattern="/container", priority=10, targets=[service]) # add an output with a well-known name to read it from the integ tests self.load_balancer_dns_name = lb.load_balancer_dns_name
def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) """Default values if not specified via context variables from CLI logging_level = 'INFO' slack_webhook_secret_name = 'aws-to-slack/dev/webhooks' """ if self.node.try_get_context('logging_level') is None: LOGGING_LEVEL = 'INFO' else: LOGGING_LEVEL = self.node.try_get_context('logging_level') if self.node.try_get_context('slack_webhook_secret_name') is None: WEBHOOK_SECRET_NAME = 'aws-to-slack/dev/webhooks' else: WEBHOOK_SECRET_NAME = self.node.try_get_context( 'slack_webhook_secret_name') """Create CloudFormation parameters so we can easily use the template this CDK app generates and convert it to a SAM application. """ webhook_secret_name_param = cdk.CfnParameter( self, 'WebhookSecretName', description=('The name of the Secrets Manager secret ' 'which stores the Slack webhook URL'), type='String', default=WEBHOOK_SECRET_NAME, allowed_pattern='[a-zA-Z0-9/_+=.@-]+', min_length=1, max_length=512 ).value_as_string whats_new_rss_feed = cdk.CfnParameter( self, 'WhatsNewRSSFeed', description='The RSS feed of all AWS new releases', type='String', default=self.node.try_get_context( 'whats_new_rss_feed') ).value_as_string whats_new_search_api = cdk.CfnParameter( self, 'WhatsNewSearchAPI', description='The search API url of new releases', type='String', default=self.node.try_get_context( 'whats_new_search_api') ).value_as_string logging_level = cdk.CfnParameter( self, 'LoggingLevel', description='The verbosity of the logs in the Lambda function', type='String', allowed_values=['INFO', 'ERROR', 'DEBUG', 'WARN'], default=LOGGING_LEVEL, ).value_as_string """DynamoDB table which stores a history of messages sent""" ddb_table = dynamodb.Table( self, 'SlackMessageHistory', partition_key=dynamodb.Attribute( name='url', type=dynamodb.AttributeType.STRING), read_capacity=1, write_capacity=1 ) """Lambda function that queries the AWS What's New RSS feed and sends each release to Slack if it has not already been sent. """ new_release_function = lambda_python.PythonFunction( self, 'AWSReleasesFunction', entry='lambda', handler='main', index='new_releases.py', runtime=lambda_.Runtime.PYTHON_3_8, description='Queries https://aws.amazon.com/new/ and sends new release info to a Slack channel via AWS Chatbot', environment=dict( WHATS_NEW_RSS_FEED=whats_new_rss_feed, WHATS_NEW_SEARCH_API=whats_new_search_api, WEBHOOK_SECRET_NAME=webhook_secret_name_param, DDB_TABLE=ddb_table.table_name, LOG_LEVEL=logging_level, POWERTOOLS_SERVICE_NAME='aws-to-slack' ), memory_size=512, tracing=lambda_.Tracing.ACTIVE, timeout=cdk.Duration.seconds(30), log_retention=logs.RetentionDays.SIX_MONTHS ) """Imports the SecretsManager secret which contains the Slack webhook url(s) and adds read access to the Lambda execution role """ slack_webhook_urls = secretsmanager.Secret.from_secret_name_v2( self, "SlackWebhookURLSecrets", secret_name=webhook_secret_name_param ) slack_webhook_urls.grant_read(new_release_function.role) """Invoke this function every X minutes""" rule = events.Rule( self, 'AWSReleaseToSlackRule', description='Schedule to invoke Lambda function that sends new AWS releases to Slack', schedule=events.Schedule.rate(cdk.Duration.minutes(5)) ) rule.add_target(events_targets.LambdaFunction(new_release_function)) """Grant the Lambda function Query and PutItem access to the DDB table""" ddb_table.grant( new_release_function, 'dynamodb:Query', 'dynamodb:PutItem' )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) aws_region = os.environ.get("CDK_DEPLOY_REGION", os.environ["CDK_DEFAULT_REGION"]) account_id = os.environ.get("CDK_DEPLOY_ACCOUNT", os.environ["CDK_DEFAULT_ACCOUNT"]) ssm_client = boto3.client('ssm', aws_region) # Prepare pipeline config details in SSM parameters if prefix == 'us': self.qs_reports_env_config = {"Permissions": [{"Group_Name": "critical", "Reports": ["Sales Results - Critical"], "ns_name": "default"}, {"Group_Name": "highlyconfidential", "Reports": ["Field Operations Dashboard", "Sales Results - Highly Confidential" ], "ns_name": "default"}, {"Group_Name": "bi-developer", "Reports": ["all"], "ns_name": "default"}, {"Group_Name": "bi-admin", "Reports": ["all"], "ns_name": "default"}, {"Group_Name": "power-reader", "Reports": ["read-all"], "ns_name": "default"}, {"Group_Name": "3rd-party", "Reports": ["Marketing KPIs"], "ns_name": "3rd-party"}, {"Group_Name": "3rd-party-reader", "Reports": ["Marketing KPIs"], "ns_name": "3rd-party"} ] } if prefix == 'eu': self.qs_reports_env_config = {"Permissions": [{"Group_Name": "eu-critical", "Reports": ["EUResults - Critical"]}, {"Group_Name": "bi-developer", "Reports": ["all"]}, {"Group_Name": "bi-admin", "Reports": ["all"]}, {"Group_Name": "eu-highlyconfidential", "Reports": ["EUField Operations Dashboard", "EUResults - Highly Confidential"]}, {"Group_Name": "power-reader", "Reports": ["read-all"]}]} self.qs_reports_env_config_ssm = ssm.StringParameter( self, '/qs/config/access', string_value=json.dumps(self.qs_reports_env_config), parameter_name='/qs/config/access' ) #group-user mapping information is stored in s3 bucket. A ssm parameter stores the bucket name. self.qs_user_group_config = {'bucket-name':f'qs-granular-access-demo-{account_id}'} bucket = s3.Bucket(self, f'qs-granular-access-demo-{account_id}', bucket_name=f'qs-granular-access-demo-{account_id}', versioned=True, removal_policy=core.RemovalPolicy.DESTROY, auto_delete_objects=True) s3deploy.BucketDeployment(self, "DeployMembership", sources=[s3deploy.Source.asset('membership.zip')], destination_bucket=bucket, destination_key_prefix='membership', prune=False) self.qs_user_group_config_ssm = ssm.StringParameter( self, '/qs/config/groups', string_value=json.dumps(self.qs_user_group_config), parameter_name='/qs/config/groups' ) # group-role mapping information is stored in a ssm parameter. self.qs_role_config = {'default_bi-developer': 'AUTHOR', 'default_bi-admin': 'ADMIN', 'default_power-reader': 'AUTHOR', 'default_critical': 'READER', 'default_highlyconfidential': 'READER', 'default_marketing': 'AUTHOR', '3rd-party_3rd-party': 'AUTHOR', '3rd-party_3rd-party-reader': 'READER' } self.qs_role_config_ssm = ssm.StringParameter( self, '/qs/config/roles', string_value=json.dumps(self.qs_role_config), parameter_name='/qs/config/roles' ) # group-namespace mapping information is stored in a ssm parameter. self.qs_ns_config = {"ns":['default', '3rd-party']} self.qs_ns_config_ssm = ssm.StringParameter( self, '/qs/config/ns', string_value=json.dumps(self.qs_ns_config), parameter_name='/qs/config/ns' ) lambda_role = iam.Role( self, id='lambda-role', description='Role for the quicksight lambda', role_name=f'{aws_region}-role-quicksight-lambda', max_session_duration=core.Duration.seconds(3600), assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'), inline_policies={ 'AllowS3Access': iam.PolicyDocument( statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=["kms:GetParametersForImport", "kms:GetPublicKey", "kms:ListKeyPolicies", "kms:ListRetirableGrants", "kms:GetKeyPolicy", "kms:ListResourceTags", "kms:ListGrants", "kms:GetParametersForImport", "kms:GetKeyRotationStatus", "kms:DescribeKey", "kms:CreateGrant", "kms:ListAliases", "kms:ListKeys", "kms:DescribeCustomKeyStores", "ssm:GetParameters", "ssm:GetParameter", "ssm:GetParametersByPath" ], resources=['*'] ), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=["lambda:InvokeFunction", "logs:CreateLogStream", "logs:CreateLogGroup", "logs:PutLogEvents", "quicksight:*", "s3:HeadBucket", "s3:ListAllMyBuckets", "s3:PutObject", "s3:GetObject", "s3:ListBucket", "s3:GetObjectVersionForReplication", "s3:GetBucketPolicy", "s3:GetObjectVersion", "cloudwatch:PutMetricData", "sts:GetCallerIdentity"], resources=['*'] ) ] ) } ) user_init = _lambda.Function(self, 'user_init', handler='user_init.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset(os.path.join(current_dir, '../lambda_functions/user_init/')), function_name='user_init', role=lambda_role, timeout=core.Duration.minutes(15), memory_size=512 ) check_team_members = _lambda.Function(self, 'check_team_members', handler='check_team_members.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset(os.path.join(current_dir, '../lambda_functions/check_team_members/')), function_name='check_team_members', role=lambda_role, timeout=core.Duration.minutes(15), memory_size=512, environment={'aws_region': f'{core.Aws.REGION}'} ) downgrade_user = _lambda.Function(self, 'downgrade_user', handler='downgrade_user.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.from_asset(os.path.join(current_dir, '../lambda_functions/downgrade_user/')), function_name='downgrade_user', role=lambda_role, timeout=core.Duration.minutes(15), memory_size=2048, environment={'aws_region': f'{core.Aws.REGION}'} ) granular_user_govenance = _lambda.Function(self, 'granular_user_govenance', handler='granular_user_govenance.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset(os.path.join(current_dir, '../lambda_functions/granular_user_govenance')), function_name='granular_user_govenance', role=lambda_role, timeout=core.Duration.minutes(15), memory_size=2048, environment={'aws_region': f'{core.Aws.REGION}'} ) granular_access_assets_govenance = _lambda.Function(self, 'granular_access_assets_govenance', handler='granular_access_assets_govenance.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset(os.path.join(current_dir, '../lambda_functions/granular_access_assets_govenance')), function_name='granular_access_assets_govenance', role=lambda_role, timeout=core.Duration.minutes(15), memory_size=2048, environment={'aws_region': f'{core.Aws.REGION}'} ) quicksight_event_rule = events.Rule(self, 'QuickSightCWEventRule', description='CloudWatch rule to detect new QuickSight user creation', rule_name='qs-gc-user-creation', targets=[targets.LambdaFunction(user_init)], event_pattern=events.EventPattern(source=['aws.quicksight'], detail_type=[ 'AWS Service Event via CloudTrail'], detail={ "eventSource": [ "quicksight.amazonaws.com"], "eventName": ["CreateUser"] } ) ) quicksight_schedule_rule = events.Rule(self, 'quicksight_schedule_rule', description='CloudWatch rule to run QS objects/groups assignment every hour', rule_name='qs-gc-every-hour', schedule=events.Schedule.cron(minute="0"), targets=[targets.LambdaFunction(granular_user_govenance)] ) quicksight_assume_condition_object = {"StringEquals": { "SAML:aud": "https://signin.aws.amazon.com/saml"}} quicksight_federated_prin_with_conditionb_obj = iam.FederatedPrincipal( f'arn:aws:iam::{core.Aws.ACCOUNT_ID}:saml-provider/saml', quicksight_assume_condition_object, 'sts:AssumeRoleWithSAML') quicksight_resource_scope = '${aws:userid}' quicksight_reader_saml_inline_policies = { 'AllowQuicksightAccessSAML': iam.PolicyDocument( statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['quicksight:CreateReader'], resources=[ f'arn:aws:quicksight::{core.Aws.ACCOUNT_ID}:user/{quicksight_resource_scope}'] ) ] ) } quicksight_users = iam.Role( self, id=f"quicksight-fed-{prefix}-users", # this is the default group with no access description='Role for the quicksight reader SAML', role_name=f"quicksight-fed-{prefix}-users", max_session_duration=core.Duration.seconds(3600), assumed_by=quicksight_federated_prin_with_conditionb_obj, inline_policies=quicksight_reader_saml_inline_policies )