def setup_iot_endpoint_provider(self): describe_endpoint_policy = aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, actions=["iot:DescribeEndpoint"], resources=["*"], ) provider_lambda = aws_lambda.SingletonFunction( self, "iot_data_ats_endpoint_provider_lambda", uuid="iot_data_ats_endpoint_provider_lambda_20200507150213", runtime=aws_lambda.Runtime.PYTHON_3_7, code=aws_lambda.Code.asset("custom_resources/iot_endpoint"), handler="iot_endpoint_provider.on_event", description="Returns iot:Data-ATS endpoint for this account", current_version_options=aws_lambda.VersionOptions( removal_policy=core.RemovalPolicy.DESTROY), initial_policy=[describe_endpoint_policy], ) provider = custom_resources.Provider(self, "iot_data_ats_endpoint_provider", on_event_handler=provider_lambda) iot_endpoint = core.CustomResource( self, "iot_data_ats_endpoint", resource_type="Custom::IotDataAtsEndpoint", service_token=provider.service_token, ) endpoint_address = iot_endpoint.get_att("EndpointAddress").to_string() self._parameters_to_save["iot_endpoint_address"] = endpoint_address
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) on_event = _lambda.Function( self, 'ConnectHandler', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset('lambda'), handler='connect_create.handler', ) on_event.add_to_role_policy( iam.PolicyStatement(actions=[ "connect:CreateInstance", "connect:DeleteInstance", "ds:CreateAlias", "ds:AuthorizeApplication", "ds:UnauthorizeApplication", "ds:CreateIdentityPoolDirectory", "ds:CreateDirectory", "ds:DescribeDirectories", "ds:CheckAlias", "ds:DeleteDirectory", "iam:AttachRolePolicy", "iam:CreateServiceLinkedRole", "iam:PutRolePolicy" ], resources=["*"])) my_provider = cr.Provider( self, "MyProvider", on_event_handler=on_event, #is_complete_handler=is_complete, # optional async "waiter" log_retention=logs.RetentionDays.ONE_DAY) CustomResource(self, "Resource1", service_token=my_provider.service_token)
def wrapper(*args, **kwargs): hashed = hashlib.md5(bytes(json.dumps(args) + json.dumps(kwargs), 'UTF-8')) hashi = hashed.hexdigest()[:9] function_name = func.__name__ + hashi remote = lamb.Function(self._scope, f'LambdaFunction-{function_name}', code=lamb.Code.from_inline(self._create_function_code(func)), runtime=lamb.Runtime.PYTHON_3_7, handler='index.handler', timeout=cdk.Duration.minutes(15), memory_size=256, ) provider = cr.Provider(self._scope, f'CustomResourceProvider-{function_name}', on_event_handler=remote) # stack = cdk.Stack.of(self._scope) resource = cdk.CustomResource(self._scope, f'CustomResource-{function_name}', service_token=provider.service_token, properties={ # avoid any CFN type conversion quirks by serializing # on output and deserializing on input 'args': json.dumps(list(args)), 'kwargs': json.dumps(dict(kwargs)) }) return resource.get_att_string('Value')
def create_custom_authorizer_signing_key_generic( self, unique_id, description, token_value) -> core.CustomResource: """ Uses a Lambda to create an asymmetric key pair, since neither CFn nor CDK support that as of this writing (2020-05-09) https://github.com/aws-cloudformation/aws-cloudformation-coverage-roadmap/issues/337 After creating the key, it signs the token value using the private key, and stores all of `token_value`, `token_value`'s signature, and the public key in the stack's parameter store. :return: the CustomResource for the signing key """ create_authorizer_policy = aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, actions=[ "kms:CreateKey", "kms:GetPublicKey", "kms:ScheduleKeyDeletion", "kms:Sign" ], resources=["*"], ) provider_lambda = aws_lambda.SingletonFunction( self, f"iot_custom_authorizer_key_provider_lambda_{unique_id}", uuid= f"iot_custom_authorizer_key_provider_lambda_20200507150213_{unique_id}", runtime=aws_lambda.Runtime.PYTHON_3_7, code=aws_lambda.Code.asset( "custom_resources/iot_custom_authorizer_key_provider"), handler="iot_custom_authorizer_key_provider.on_event", description=description, current_version_options=aws_lambda.VersionOptions( removal_policy=core.RemovalPolicy.DESTROY), initial_policy=[create_authorizer_policy], ) provider = custom_resources.Provider( self, f"iot_custom_authorizer_key_provider_{unique_id}", on_event_handler=provider_lambda, ) iot_custom_authorizer_key = core.CustomResource( self, f"iot_custom_authorizer_key_{unique_id}", resource_type="Custom::IoTCustomAuthorizer", service_token=provider.service_token, properties={"token_value": token_value}, ) return iot_custom_authorizer_key
def __init__(self, scope: core.Construct, id: str, bucket_name: str, uuid: str, **kwargs): super().__init__(scope, id, **kwargs) bucket_storage = _s3.LifecycleRule(transitions=[ _s3.Transition(storage_class=_s3.StorageClass.INTELLIGENT_TIERING, transition_after=core.Duration.days(1)) ]) self.__bucket = _s3.Bucket(self, 'S3Bucket', bucket_name=bucket_name, removal_policy=core.RemovalPolicy.DESTROY, encryption=_s3.BucketEncryption.KMS_MANAGED, lifecycle_rules=[bucket_storage]) with open('common/common_cdk/lambda/empty_bucket.py', 'r') as f: lambda_source = f.read() empty_bucket_lambda = _lambda.SingletonFunction( self, 'EmptyBucketLambda', uuid=uuid, runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.inline(lambda_source), handler='index.handler', timeout=core.Duration.minutes(15)) empty_bucket_lambda.role.add_to_policy( _iam.PolicyStatement(actions=[ 's3:DeleteObject', 's3:DeleteObjectVersion', 's3:ListBucketVersions', 's3:ListBucket' ], resources=[ self.__bucket.bucket_arn + '/*', self.__bucket.bucket_arn ])) empty_bucket_lambda_provider = _custom_resources.Provider( self, 'EmptyBucketLambdaProvider', on_event_handler=empty_bucket_lambda) custom_resource = core.CustomResource( self, 'EmptyBucketCustomResource', service_token=empty_bucket_lambda_provider.service_token, properties={"bucket_name": self.__bucket.bucket_name}) custom_resource.node.add_dependency(self.__bucket)
def __init__(self, scope: cdk.Construct, construct_id: str, stage: str, table: _dynamo.Table, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) initial_data_role = iam.Role( self, "InitialDataRole", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaBasicExecutionRole") ], ) initial_data_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "AWSLambdaInvocation-DynamoDB")) initial_data_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonDynamoDBFullAccess")) on_event = _lambda.Function( self, "DataHandler", runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset("lambda"), handler="initial_data.lambda_handler", timeout=cdk.Duration.minutes(5), environment={ "TABLE_NAME": table.table_name, "STAGE": stage }, ) table.grant_full_access(on_event) initial_data_provider = _resources.Provider( self, "InitialDataProvider", on_event_handler=on_event, log_retention=logs.RetentionDays.ONE_DAY, role=initial_data_role, ) cdk.CustomResource( self, "InitialDataResource", service_token=initial_data_provider.service_token, )
def __init__( self, scope: core.Construct, id: str, policy_id: str, account_targets: List[str] = None, organization_unit_targets: List[str] = None, ) -> None: super().__init__(scope, id) on_event = _lambda.Function( self, "ON-SCP-ATTACHMENT-EVENT", runtime=_lambda.Runtime.PYTHON_3_8, handler="app.on_event", timeout=core.Duration.seconds(600), memory_size=128, code=_lambda.Code.asset(os.path.join(dirname, "attachment_lambda")), description="Service control policy attachment resource", ) on_event.add_to_role_policy( iam.PolicyStatement( actions=[ "organizations:CreatePolicy", "organizations:DeletePolicy", "organizations:AttachPolicy", "organizations:DetachPolicy", ], resources=["*"], )) attachment_provider = cr.Provider( self, "ON_EVENT_CUSTOM_RESOURCE_PROVIDER", on_event_handler=on_event, ) CustomResource( self, "scp-attachment-custom-resource", service_token=attachment_provider.service_token, properties={ "PolicyId": policy_id, "AccountTargets": account_targets, "OrganizationUnitTargets": organization_unit_targets, }, )
def __init__(self, scope: core.Construct, id: str, f_lambda, bot_locale) -> None: super().__init__(scope, id) on_event = _lambda.Function( self, "ON-EVENT", runtime=_lambda.Runtime.PYTHON_3_6, handler="lex-bot-provider.on_event", timeout=core.Duration.seconds(60), memory_size=256, code=_lambda.Code.asset("./custom_resource_lex_bot/lambda"), description='PROCESA EVENTOS CUSTOM RESOURCE', environment={ 'LAMBDA_ARN_FULLFILL': f_lambda.function_arn, 'BOT_LOCALE': bot_locale }) is_complete = _lambda.Function( self, "IS-COMPLETE", runtime=_lambda.Runtime.PYTHON_3_6, handler="lex-bot-provider.is_complete", timeout=core.Duration.seconds(60), memory_size=256, code=_lambda.Code.asset("./custom_resource_lex_bot/lambda"), description='IS COMPLETE HANDLER') on_event.add_to_role_policy( iam.PolicyStatement(actions=["lex:*"], resources=['*'])) is_complete.add_to_role_policy( iam.PolicyStatement(actions=["lex:*"], resources=['*'])) my_provider = cr.Provider( self, "ON_EVENT_CUSTOM_RESOURCE_PROVIDER", on_event_handler=on_event, is_complete_handler=is_complete, # optional async "waiter" log_retention=logs.RetentionDays.ONE_DAY) CustomResource(self, "lexbotcustom", service_token=my_provider.service_token)
def __init__(self, scope: core.Construct, id: str, secgroup_name: str, **kwargs): super().__init__(scope, id, **kwargs) with open('common/common_cdk/lambda/empty_security_group.py', 'r') as f: lambda_source = f.read() # lambda utils to empty security group before deletion empty_secgroup_lambda = _lambda.SingletonFunction(self, 'EmptySecurityGroupLambda', uuid="dfs3k8730-4ee1-11e8-9c2d-fdfs65dfsc", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.inline(lambda_source), handler='index.handler', function_name='ara-auto-empty-secgroup' ) empty_secgroup_lambda_role = _iam.Role( self, 'AutoEmptyBucketLambdaRole', assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com') ) empty_secgroup_lambda_role.add_to_policy( _iam.PolicyStatement( actions=[ 'ec2:RevokeSecurityGroupIngress', 'ec2:RevokeSecurityGroupEgress' ], resources=['arn:aws:ec2::'+core.Aws.ACCOUNT_ID+':security-group/'+secgroup_name] ) ) empty_secgroup_lambda_provider = _custom_resources.Provider( self, 'EmptyBucketLambdaProvider', on_event_handler=empty_secgroup_lambda ) core.CustomResource( self, 'EmptyBucketCustomResource', service_token=empty_secgroup_lambda_provider.service_token, properties={ "secgroup_name": secgroup_name } )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Create bucket and upload scrips bucket = s3.Bucket(self, "ScriptBucket") self.script_bucket = bucket s3deploy.BucketDeployment(self, "UploadScriptsToBucket", sources=[s3deploy.Source.asset(os.path.join(dirname, "scripts"))], destination_bucket=bucket ) # Greengrass Core Thing policy greengrass_core_policy = iot.CfnPolicy(self, 'GreenGrassCorePolicy', policy_name='greengrass-demo-policy', policy_document={ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "iot:Publish", "iot:Subscribe", "iot:Connect", "iot:Receive" ], "Resource": [ "*" ] }, { "Effect": "Allow", "Action": [ "iot:GetThingShadow", "iot:UpdateThingShadow", "iot:DeleteThingShadow" ], "Resource": [ "*" ] }, { "Effect": "Allow", "Action": [ "greengrass:*" ], "Resource": [ "*" ] } ] } ) self.core_policy_name = greengrass_core_policy.policy_name # Create a Greengrass group role greengrass_group_role = iam.Role(self, "GroupRole", assumed_by=iam.ServicePrincipal("greengrass.amazonaws.com") ) greengrass_group_role.add_to_policy(iam.PolicyStatement( resources=["arn:aws:logs:*:*:*"], actions=[ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents" ] )) greengrass_group_role.add_to_policy(iam.PolicyStatement( resources=["*"], actions=["iot:*"] )) self.greengrass_group_role_arn = greengrass_group_role.role_arn # A custom resource to verify that there is a service role for greengrass on the account greengrass_mgmt_function = awslambda.SingletonFunction( self, "MgmttHandler", uuid="58854ea2-0624-4ca5-b600-fa88d4b9164e", runtime=awslambda.Runtime.PYTHON_3_7, code=awslambda.Code.asset("custom_resources"), handler="greengrassmgmt.handler", ) greengrass_mgmt_function.add_to_role_policy( iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ 'greengrass:*', 'iot:*', 'iam:CreateRole', 'iam:AttachRolePolicy', 'iam:PassRole' ], resources=['*'] ) ) greengrass_mgmt_provider = cust_resource.Provider(self, "MgmtProvider", on_event_handler=greengrass_mgmt_function ) core.CustomResource(self, "MgmtCustResource", service_token=greengrass_mgmt_provider.service_token )
def setup_custom_authorizer(self): # These values are used in the custom authorizer setup, and exported to Parameter Store # for use by integration tests custom_authorizer_name = "iot_custom_authorizer" self._parameters_to_save[ "custom_authorizer_name"] = custom_authorizer_name # Note: "key" is a bit overloaded here. In the context of the custom authorizer, "key name" # refers to the HTTP header field that the custom authorizer looks for a token value in. # # In the case of the custom authorizer key provider, the "key" is the KMS asymmetric CMK # used to sign the token value passed in the `token_key_name` header. In order to keep the # terminology consistent between client integ tests that are expecting to pass something for # a "key name" field, we'll let the ambiguity stand. token_key_name = "iot_custom_authorizer_token" self._parameters_to_save[ "custom_authorizer_token_key_name"] = token_key_name token_value = "allow" self._parameters_to_save["custom_authorizer_token_value"] = token_value iot_custom_authorizer_key_resource = self.create_custom_authorizer_signing_key_generic( "1", "Manages an asymmetric CMK and token signature for iot custom authorizer.", token_value, ) custom_authorizer_token_signature = iot_custom_authorizer_key_resource.get_att( "custom_authorizer_token_signature").to_string() self._parameters_to_save[ "custom_authorizer_token_signature"] = custom_authorizer_token_signature authorizer_function_arn = self.setup_custom_authorizer_function( "1", "custom_resources/iot_custom_authorizer_function", "iot_custom_authorizer.handler", "Sample custom authorizer that allows or denies based on 'token' value", {}, self.region, ) create_authorizer_policy = aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, actions=[ "iot:CreateAuthorizer", "iot:UpdateAuthorizer", "iot:DeleteAuthorizer" ], resources=["*"], ) provider_lambda = aws_lambda.SingletonFunction( self, "iot_custom_authorizer_provider_lambda", uuid=self.custom_auth_user_pass_uuid, runtime=aws_lambda.Runtime.PYTHON_3_7, code=aws_lambda.Code.asset( "custom_resources/iot_custom_authorizer_provider"), handler="iot_custom_authorizer_provider.on_event", description="Sets up an IoT custom authorizer", current_version_options=aws_lambda.VersionOptions( removal_policy=core.RemovalPolicy.DESTROY), initial_policy=[create_authorizer_policy], ) provider = custom_resources.Provider(self, "iot_custom_authorizer_provider", on_event_handler=provider_lambda) public_key = iot_custom_authorizer_key_resource.get_att( "custom_authorizer_public_key").to_string() core.CustomResource( self, "iot_custom_authorizer", resource_type="Custom::IoTCustomAuthorizer", service_token=provider.service_token, properties={ "authorizer_function_arn": authorizer_function_arn, "authorizer_name": custom_authorizer_name, "public_key": public_key, "token_key_name": token_key_name, }, )
def setup_custom_authorizer_user_pass(self): custom_authorizer_name = self.custom_auth_user_pass_default_authorizer_name self._parameters_to_save[ "custom_authorizer_user_pass_name"] = custom_authorizer_name token_key_name = "IoTTokenKeyName" self._parameters_to_save[ "custom_authorizer_user_pass_token_key_name"] = token_key_name token_value = "allow" self._parameters_to_save[ "custom_authorizer_user_pass_token_value"] = token_value self._parameters_to_save[ "custom_authorizer_user_pass_username"] = self.custom_auth_user_pass_username self._parameters_to_save[ "custom_authorizer_user_pass_password"] = self.custom_auth_user_pass_password iot_custom_authorizer_key_resource = self.create_custom_authorizer_signing_key_generic( "2", "Manages an asymmetric CMK and token signature for iot custom authorizer with username and password.", token_value, ) custom_authorizer_token_signature = iot_custom_authorizer_key_resource.get_att( "custom_authorizer_token_signature").to_string() self._parameters_to_save[ "custom_authorizer_user_pass_token_signature"] = custom_authorizer_token_signature # TODO: remove forcing of us-east-1 when enhanced custom authorizers are available in all regions # Force region to 'us-east-1' due to enhanced custom authorizers only available in this region authorizer_function_arn = self.setup_custom_authorizer_function( "2", "custom_resources/iot_custom_authorizer_user_pass_function", "iot_custom_authorizer_user_pass.handler", "Sample custom authorizer that allows or denies based on username and password", { "custom_auth_user_pass_username": self.custom_auth_user_pass_username, "custom_auth_user_pass_password": self.custom_auth_user_pass_password }, "us-east-1", ) create_authorizer_policy = aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, actions=[ "iot:CreateAuthorizer", "iot:UpdateAuthorizer", "iot:DeleteAuthorizer", "iot:UpdateDomainConfiguration", "iot:CreateDomainConfiguration", "iot:DescribeDomainConfiguration", "iot:DeleteDomainConfiguration", ], resources=["*"], ) provider_lambda = aws_lambda.SingletonFunction( self, "iot_custom_authorizer_user_pass_provider_lambda", uuid= "iot_custom_authorizer_user_pass_provider_lambda_20200727123737", runtime=aws_lambda.Runtime.PYTHON_3_7, code=aws_lambda.Code.asset( "custom_resources/iot_custom_authorizer_user_pass_provider"), handler="iot_custom_authorizer_user_pass_provider.on_event", description= "Sets up an IoT custom authorizer for user password & required domain config due to beta status", environment={ "custom_auth_user_pass_uuid": self.custom_auth_user_pass_uuid, "custom_auth_user_pass_default_authorizer_name": self.custom_auth_user_pass_default_authorizer_name, "custom_auth_user_pass_domain_configuration_name": self.custom_auth_user_pass_domain_configuration_name }, current_version_options=aws_lambda.VersionOptions( removal_policy=core.RemovalPolicy.DESTROY), initial_policy=[create_authorizer_policy], ) provider = custom_resources.Provider( self, "iot_custom_authorizer_user_pass_provider", on_event_handler=provider_lambda) public_key = iot_custom_authorizer_key_resource.get_att( "custom_authorizer_public_key").to_string() iot_endpoint = core.CustomResource( self, "iot_custom_authorizer_user_pass", resource_type="Custom::IoTCustomAuthorizer", service_token=provider.service_token, properties={ "authorizer_function_arn": authorizer_function_arn, "authorizer_name": custom_authorizer_name, "public_key": public_key, "token_key_name": token_key_name, }, ) endpoint_address = iot_endpoint.get_att( "BetaEndpointAddress").to_string() self._parameters_to_save[ "iot_beta_endpoint_address"] = endpoint_address
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Add the VPC connection arn as an input parameter vpc_conn_arn = core.CfnParameter( self, "VpcConnectionArn", type="String", description="The Arn of the VPC connection to use for Redshift.") quicksight_group_arn = core.Fn.import_value('ara-QuickSight-Group-Arn') secret_arn = core.Fn.import_value('ara-QuickSight-Redshift-Secret-Arn') # Create the custom resource policy with the necessary permissions iam_policy = cr.AwsCustomResourcePolicy.from_statements([ iam.PolicyStatement(actions=cfg.CDK_POLICY_ACTIONS, resources=['*']), ]) redshift_datasource_lambda = lambda_.SingletonFunction( self, id='RedshiftDatasourceLambda', uuid='b438edeb-f5dc-486a-ac2d-bc0918b975b8', runtime=lambda_.Runtime.PYTHON_3_8, code=lambda_.Code.from_asset('dataviz_redshift/lambda'), handler='redshift_datasource.handler', function_name='ara_redshift_datasource') redshift_datasource_lambda.role.add_to_policy( iam.PolicyStatement(actions=['secretsmanager:GetSecretValue'], resources=[secret_arn])) redshift_datasource_lambda.role.add_to_policy( iam.PolicyStatement(actions=[ 'quicksight:CreateDataSource', 'quicksight:DeleteDataSource' ], resources=['*'])) lambda_provider = cr.Provider( self, id='LambdaProvider', on_event_handler=redshift_datasource_lambda) responseLamb = core.CustomResource( self, 'RedshiftDatasourceResource', service_token=lambda_provider.service_token, properties={ 'Secret_arn': secret_arn, 'Datasource_name': cfg.REDSHIFT_DATASOURCE_NAME, 'Aws_account_id': self.account, 'Quicksight_group_arn': quicksight_group_arn, 'Datasource_actions': cfg.DATASOURCE_ACTIONS, 'Vpc_conn_arn': vpc_conn_arn.value_as_string }) redshift_datasource_arn = responseLamb.get_att_string('datasource_arn') core.CfnOutput(self, "RedshiftDataSourceArn", description="Redshift Data Source Arn", value=redshift_datasource_arn) # Create a Redshift dataset with custom SQL redshift_dataset_arn = QuickSightRedshiftDataset( self, 'RedshiftDataset', iam_policy=iam_policy, quicksight_group_arn=quicksight_group_arn, redshift_datasource_arn=redshift_datasource_arn, redshift_dataset_name=cfg.REDSHIFT_DATASET_NAME, dataset_actions=cfg.DATASET_ACTIONS, redshift_custom_sql=cfg.REDSHIFT_CUSTOM_SQL, redshift_columns=cfg.REDSHIFT_COLUMNS, redshift_data_transformations=cfg.REDSHIFT_DATA_TRANSFORMATIONS ).redshift_dataset_arn QuickSightRedshiftAnalysis( self, 'RedshiftAnalysis', iam_policy=iam_policy, quicksight_group_arn=quicksight_group_arn, redshift_dataset_arn=redshift_dataset_arn, redshift_analysis_name=cfg.REDSHIFT_ANALYSIS_NAME, redshift_analysis_template_alias=cfg. REDSHIFT_ANALYSIS_TEMPLATE_ALIAS, analysis_actions=cfg.ANALYSIS_ACTIONS)
def __init__(self, scope: cdk.Construct, id: str, cognito_user_pool: cognito.UserPool, s3_bucket_name: str, create_configuration_lambda_role_arn: str, redis: ec.CfnCacheCluster, domain_name: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) config_yaml = yaml.load(open("config.yaml"), Loader=yaml.FullLoader) spoke_accounts = config_yaml.get("spoke_accounts", []) cognito_user_pool_client = cognito.UserPoolClient( self, "UserPoolClient", user_pool=cognito_user_pool, generate_secret=True, supported_identity_providers=[ cognito.UserPoolClientIdentityProvider.COGNITO ], prevent_user_existence_errors=True, o_auth=cognito.OAuthSettings( callback_urls=[ "https://" + domain_name + "/auth", "https://" + domain_name + "/oauth2/idpresponse", ], logout_urls=["https://" + domain_name + "/logout"], flows=cognito.OAuthFlows(authorization_code_grant=True, implicit_code_grant=True), scopes=[cognito.OAuthScope.OPENID, cognito.OAuthScope.EMAIL], ), auth_flows=cognito.AuthFlow(user_password=True, user_srp=True), ) describe_cognito_user_pool_client = cr.AwsCustomResource( self, "UserPoolClientIDResource", policy=cr.AwsCustomResourcePolicy.from_sdk_calls( resources=cr.AwsCustomResourcePolicy.ANY_RESOURCE), on_create=cr.AwsSdkCall( service="CognitoIdentityServiceProvider", action="describeUserPoolClient", parameters={ "UserPoolId": cognito_user_pool.user_pool_id, "ClientId": cognito_user_pool_client.user_pool_client_id, }, physical_resource_id=cr.PhysicalResourceId.of( cognito_user_pool_client.user_pool_client_id), ), install_latest_aws_sdk=True, log_retention=logs.RetentionDays.ONE_WEEK, ) cognito_user_pool_client_secret = ( describe_cognito_user_pool_client.get_response_field( "UserPoolClient.ClientSecret")) imported_create_configuration_lambda_role = iam.Role.from_role_arn( self, "ImportedCreateConfigurationFileLambdaRole", role_arn=create_configuration_lambda_role_arn, ) jwt_secret = config_yaml["jwt_secret"] config_secret_dict = { "oidc_secrets": { "client_id": cognito_user_pool_client.user_pool_client_id, "secret": cognito_user_pool_client_secret, "client_scope": ["email", "openid"], }, "jwt_secret": jwt_secret, } config_secret_yaml = yaml.dump( config_secret_dict, explicit_start=True, default_flow_style=False, ) config_secret = cr.AwsCustomResource( self, "ConfigSecretResource", policy=cr.AwsCustomResourcePolicy.from_sdk_calls( resources=cr.AwsCustomResourcePolicy.ANY_RESOURCE), on_update=cr.AwsSdkCall( service="SecretsManager", action="updateSecret", parameters={ "SecretId": CONFIG_SECRET_NAME, "SecretString": config_secret_yaml, }, physical_resource_id=cr.PhysicalResourceId.from_response( "Name"), ), on_create=cr.AwsSdkCall( service="SecretsManager", action="createSecret", parameters={ "Name": CONFIG_SECRET_NAME, "Description": "Sensitive configuration parameters for ConsoleMe", "SecretString": config_secret_yaml, }, physical_resource_id=cr.PhysicalResourceId.from_response( "Name"), ), on_delete=cr.AwsSdkCall( service="SecretsManager", action="deleteSecret", parameters={ "SecretId": CONFIG_SECRET_NAME, "ForceDeleteWithoutRecovery": True, }, ), install_latest_aws_sdk=True, log_retention=logs.RetentionDays.ONE_WEEK, ) create_configuration_lambda = lambda_.Function( self, "CreateConfigurationFileLambda", code=lambda_.Code.from_asset("resources/create_config_lambda"), handler="index.handler", timeout=cdk.Duration.seconds(30), layers=[create_dependencies_layer(self, "create_config_lambda")], runtime=lambda_.Runtime.PYTHON_3_8, role=imported_create_configuration_lambda_role, environment={ "DEPLOYMENT_BUCKET": s3_bucket_name, "OIDC_METADATA_URL": "https://cognito-idp." + self.region + ".amazonaws.com/" + cognito_user_pool.user_pool_id + "/.well-known/openid-configuration", "REDIS_HOST": redis.attr_redis_endpoint_address, "SES_IDENTITY_ARN": "arn:aws:ses:" + self.region + ":" + self.account + ":identity/" + domain_name, "SUPPORT_CHAT_URL": "https://discord.gg/nQVpNGGkYu", "APPLICATION_ADMIN": "consoleme_admin", "ACCOUNT_NUMBER": self.account, "ISSUER": domain_name, "SPOKE_ACCOUNTS": ",".join(spoke_accounts), "CONFIG_SECRET_NAME": CONFIG_SECRET_NAME, }, ) create_configuration_resource_provider = cr.Provider( self, "CreateConfigurationFileProvider", on_event_handler=create_configuration_lambda, log_retention=logs.RetentionDays.ONE_WEEK, ) create_configuration_lambda_resource = cdk.CustomResource( self, "CreateConfigurationFile", service_token=create_configuration_resource_provider.service_token, removal_policy=cdk.RemovalPolicy.DESTROY, properties={"UUID": str(uuid4())}, ) create_configuration_lambda_resource.node.add_dependency(config_secret)
def __init__(self, scope: core.Construct, id: str, cluster: eks.Cluster, kafka: msk.CfnCluster, vpc: ec2.Vpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) pip.main([ "install", "--system", "--target", "custom_resources/kafka/lib", "kafka-python" ]) arn = cr.AwsCustomResource( self, 'clusterArn', policy=cr.AwsCustomResourcePolicy.from_sdk_calls(resources=['*']), on_create=cr.AwsSdkCall( action='listClusters', service='Kafka', physical_resource_id=cr.PhysicalResourceId.of( "ClusterNameFilter"), parameters={ "ClusterNameFilter": kafka.cluster_name, "MaxResults": 1 }, ), ) bootstraps = cr.AwsCustomResource( self, 'clusterBootstraps', policy=cr.AwsCustomResourcePolicy.from_sdk_calls(resources=["*"]), on_create=cr.AwsSdkCall( action='getBootstrapBrokers', service='Kafka', physical_resource_id=cr.PhysicalResourceId.of("ClusterArn"), parameters={ "ClusterArn": arn.get_response_field("ClusterInfoList.0.ClusterArn") }, ), ) manifests = [] for namespace in self.node.try_get_context("kubernetes")['namespaces']: manifests.append({ "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": "kafka", "namespace": namespace }, "data": { "bootstrap": bootstraps.get_response_field('BootstrapBrokerStringTls'), } }) eks.KubernetesManifest(self, "kafka-config", cluster=cluster, manifest=manifests) function = lbd.SingletonFunction( self, "KafkaConfigFunction", uuid="b09329a3-5206-46f7-822f-337da714aeac", code=lbd.Code.from_asset("custom_resources/kafka/"), handler="config.handler", runtime=lbd.Runtime.PYTHON_3_7, function_name="kafkaConfig", log_retention=logs.RetentionDays.ONE_DAY, security_group=ec2.SecurityGroup.from_security_group_id( self, "lambdaKafkaVPC", vpc.vpc_default_security_group), timeout=core.Duration.seconds(30), vpc=vpc, vpc_subnets=ec2.SubnetSelection(one_per_az=True)) provider = cr.Provider(self, "KafkaConfigProvider", on_event_handler=function, log_retention=logs.RetentionDays.ONE_DAY) core.CustomResource( self, "KafkaLoadTopic", service_token=provider.service_token, properties={ "bootstrap": bootstraps.get_response_field('BootstrapBrokerStringTls'), "topic": "load", "partitions": 150, "replicas": 1 }) core.CustomResource( self, "KafkaGenerateTopic", service_token=provider.service_token, properties={ "bootstrap": bootstraps.get_response_field('BootstrapBrokerStringTls'), "topic": "generate", "partitions": 200, "replicas": 1 })
def __init__(self, scope: Construct, id: str, vpc: _ec2.IVpc, codebucket: IBucket, s3_deploy, metrics) -> None: super().__init__(scope, id) self._metrics_mapping = CfnMapping( self, 'AnonymousData', mapping={'SendAnonymousData': { 'Data': 'Yes' }}) self._metrics_condition = CfnCondition( self, 'AnonymousDatatoAWS', expression=Fn.condition_equals( self._metrics_mapping.find_in_map('SendAnonymousData', 'Data'), 'Yes')) self._helper_func = _lambda.SingletonFunction( self, 'SolutionHelper', uuid='75248a81-9138-468c-9ba1-bca6c7137599', runtime=_lambda.Runtime.PYTHON_3_8, handler='lambda_function.handler', description= 'This function generates UUID for each deployment and sends anonymous data to the AWS Solutions team', code=_lambda.Code.from_bucket(bucket=codebucket, key='app_code/solution_helper.zip'), vpc=vpc, timeout=Duration.seconds(30)) self._helper_func.add_dependency(s3_deploy) self._lambda_provider = _custom_resources.Provider( self, 'LambdaProvider', on_event_handler=self._helper_func, vpc=vpc) self._uuid = CustomResource( self, 'UUIDCustomResource', service_token=self._lambda_provider.service_token, properties={"Resource": "UUID"}, resource_type="Custom::CreateUUID", removal_policy=RemovalPolicy.DESTROY) self._send_data = CustomResource( self, 'SendDataCustomResource', service_token=self._lambda_provider.service_token, properties={ "Resource": "AnonymousMetric", "UUID": self._uuid.get_att_string("UUID"), "Solution": metrics["Solution"], "Data": metrics }, resource_type='Custom::AnonymousData', removal_policy=RemovalPolicy.DESTROY) self._send_data.node.add_dependency(self._uuid) Aspects.of(self._helper_func).add(Condition(self._metrics_condition)) Aspects.of(self._uuid).add(Condition(self._metrics_condition)) Aspects.of(self._send_data).add(Condition(self._metrics_condition))
def __init__(self, scope: core.Construct, id: str, props, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Validated require props. required_props_keys = ['CfOriginDomainName', 'Asg', 'HostedZoneName', 'WebsiteDns'] for k in required_props_keys: if k not in props or not props[k]: raise ValueError("Required prop %s is not present" % k) # Create a custom resource that returns the IP of the host behind the autoscaling group asg = props['Asg'] asg_ip_handler = lambda_.Function( self, 'GhostIpHandler', runtime=lambda_.Runtime.PYTHON_3_6, code=lambda_.Code.asset('lambda'), handler='ghost_ip.handler', ) asg_ip_handler.add_to_role_policy( statement=iam.PolicyStatement( actions=['autoscaling:DescribeAutoScalingGroups', 'ec2:DescribeInstances'], resources=['*', '*'], ) ) asg_ip_provider = cr.Provider( self, 'GhostIpProvider', on_event_handler=asg_ip_handler, ) asg_ip_resource = cfn.CustomResource( self, 'GhostIpResource', provider=asg_ip_provider, properties={ 'AsgName': asg.auto_scaling_group_name, 'ts': time.time(), # this makes sure the function is invoked for every CFN update } ) # Create R53 HZ and cf origin domain if 'ExistingHostedZoneId' in props and props['ExistingHostedZoneId']: hz = route53.HostedZone.from_hosted_zone_attributes( self, 'HostedZone', zone_name=props['HostedZoneName'], hosted_zone_id=props['ExistingHostedZoneId'], ) else: hz = route53.HostedZone( self, 'HostedZone', zone_name=props['HostedZoneName'] ) origin_rrset = route53.ARecord( self, 'OriginRecord', target=route53.RecordTarget.from_ip_addresses(asg_ip_resource.get_att_string('GhostIp')), record_name=props['CfOriginDomainName'], zone=hz, ) # Create a CF distro acm_cert = acm.DnsValidatedCertificate( self, 'GhostAcmCert', hosted_zone=hz, domain_name=props['WebsiteDns'], region='us-east-1', ) cf_distro = cf.CloudFrontWebDistribution( self, 'CfDistro', origin_configs=[cf.SourceConfiguration( custom_origin_source=cf.CustomOriginConfig( domain_name=props['CfOriginDomainName'], origin_protocol_policy=cf.OriginProtocolPolicy.HTTP_ONLY, ), behaviors=[cf.Behavior(is_default_behavior=True)], )], alias_configuration=cf.AliasConfiguration( names=[props['WebsiteDns']], acm_cert_ref=acm_cert.certificate_arn, ), default_root_object='', ) # Create the top level website DNS pointing to the CF distro ghost_rrset = route53.CnameRecord( self, 'GhostDns', domain_name=cf_distro.domain_name, zone=hz, record_name=props['WebsiteDns'], )
def __init__(self, scope: core.Construct, config: dict, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # create the securty group that will allow to connect to this instance # I am lazy and create only 1 SG that allows TCP 5432 from itself # database clients (lambda functions) will have TCP 5432 authorized for themselves too, # which is not necessary but harmless self.db_security_group = ec2.SecurityGroup(self, "Database Security Group", vpc=config['vpc']) self.db_security_group.add_ingress_rule(self.db_security_group, ec2.Port.tcp(5432)) self.cluster = rds.DatabaseCluster( self, config['rds']['name'], engine=rds.DatabaseClusterEngine.aurora_postgres( version=rds.AuroraPostgresEngineVersion.VER_11_7), default_database_name=config['rds']['databaseName'], master_user=rds.Login(username=config['rds']['masterUsername']), instance_props=rds.InstanceProps( vpc=config['vpc'], security_groups=[self.db_security_group])) # Add Secrets Manager Password rotation self.cluster.add_rotation_single_user() # aurora serverless is not yet support by CDK, https://github.com/aws/aws-cdk/issues/929 # escape hatch https://docs.aws.amazon.com/cdk/latest/guide/cfn_layer.html#cfn_layer_raw # cfn_aurora_cluster = cluster.node.default_child # cfn_aurora_cluster.add_override("Properties.EngineMode", "serverless") # cfn_aurora_cluster.add_override("Properties.EnableHttpEndpoint",True) # Enable Data API # cfn_aurora_cluster.add_override("Properties.ScalingConfiguration", { # 'AutoPause': True, # 'MaxCapacity': 4, # 'MinCapacity': 1, # 'SecondsUntilAutoPause': 600 # }) # cluster.node.try_remove_child('Instance1') # Remove 'Server' instance that isn't required for serverless Aurora # create a custom resource to initialize the data schema function = _lambda.Function( self, config['custom resource lambda']['name'], runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset('./custom_resources'), handler='app.on_event', vpc=config['vpc'], environment={ 'DB_SECRET_ARN': self.get_secret_arn(), 'PYTHON_LOGLEVEL': 'DEBUG' }, security_groups=[self.db_security_group]) # add permission to access the secret function.add_to_role_policy( iam.PolicyStatement(resources=[self.get_secret_arn()], actions=["secretsmanager:GetSecretValue"])) custom_resource_provider = cr.Provider(self, 'Custom Resource Provider', on_event_handler=function) custom_resource = core.CustomResource( self, 'Custom Resource', service_token=custom_resource_provider.service_token) # Tell CFN to wait for the database to be ready before ot create the custom resource custom_resource.node.add_dependency(self.cluster)
def __init__( self, scope: cdk.Construct, construct_id: str, vpc: ec2.Vpc, domain: sagemaker.CfnDomain, **kwargs, ) -> None: super().__init__(scope, construct_id, **kwargs) studio_domain_id = (domain.attr_domain_id ) # cdk.Fn.import_value("StudioDomainId") # Get the security group associated with the EFS volume managed by SageMaker Studio get_parameter = cr.AwsCustomResource( self, "GetEfsSgId", on_update={ # will also be called for a CREATE event "service": "EC2", "action": "describeSecurityGroups", "parameters": { "Filters": [ {"Name": "vpc-id", "Values": [vpc.vpc_id]}, { "Name": "group-name", "Values": [ f"security-group-for-inbound-nfs-{studio_domain_id}" ], }, ] }, "physical_resource_id": cr.PhysicalResourceId.of("GetEfsSgId"), }, policy=cr.AwsCustomResourcePolicy.from_sdk_calls( resources=cr.AwsCustomResourcePolicy.ANY_RESOURCE ), ) sg_name = get_parameter.get_response_field("SecurityGroups.0.GroupId") sg_efs = ec2.SecurityGroup.from_security_group_id( self, "SG", security_group_id=sg_name) # We can now retrive a handler for the EFS volume StudioDomainEfsId = cdk.Fn.import_value("StudioDomainEfsId") studio_efs = efs.FileSystem.from_file_system_attributes( self, "StudioEFS", file_system_id=StudioDomainEfsId, security_group=sg_efs) # Create EFS access point to enable the lambda fn to mount the EFS volume efs_ap = efs.AccessPoint( self, "EfsAccessPoint", file_system=studio_efs, posix_user=efs.PosixUser(gid="0", uid="0"), ) # Function that takes care of setting up the user environment self.lambda_fn = lambda_python.PythonFunction( self, "UserSetupLambdaFn", entry="populate_git_fn", index="populate_from_git.py", handler="on_event", vpc=vpc, layers=[ lambda_.LayerVersion.from_layer_version_arn( self, "GitLayer", layer_version_arn= f"arn:aws:lambda:{self.region}:553035198032:layer:git-lambda2:8", ), ], filesystem=lambda_.FileSystem.from_efs_access_point( efs_ap, "/mnt/efs"), timeout=cdk.Duration.seconds(300), initial_policy=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ "sagemaker:DescribeUserProfile", ], resources=["*"], ) ], ) provider = cr.Provider( self, "Provider", on_event_handler=self.lambda_fn, ) cdk.CfnOutput( self, "StudioUserProviderToken", value=provider.service_token, description="StudioUserProviderToken", export_name="StudioUserProviderToken", ) self.provider = provider
def __init__(self, scope: core.Construct, id: str, log_bucket: _s3.Bucket, config_table: _dynamodb.Table, tshirt_size: str, sink_bucket: _s3.Bucket, vpc: _ec2.Vpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) service_role = _iam.Role( self, 'BatchEmrServiceRole', assumed_by=_iam.ServicePrincipal('elasticmapreduce.amazonaws.com') ) service_role.add_managed_policy(_iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AmazonElasticMapReduceRole')) cluster_role = _iam.Role( self, 'BatchEmrClusterRole', assumed_by=_iam.ServicePrincipal("ec2.amazonaws.com") ) _iam.Policy( self, 'BatchEmrClusterPolicy', statements=[ _iam.PolicyStatement( actions=[ "glue:CreateDatabase", "glue:UpdateDatabase", "glue:DeleteDatabase", "glue:GetDatabase", "glue:GetDatabases", "glue:CreateTable", "glue:UpdateTable", "glue:DeleteTable", "glue:GetTable", "glue:GetTables", "glue:GetTableVersions", "glue:CreatePartition", "glue:BatchCreatePartition", "glue:UpdatePartition", "glue:DeletePartition", "glue:BatchDeletePartition", "glue:GetPartition", "glue:GetPartitions", "glue:BatchGetPartition", "glue:CreateUserDefinedFunction", "glue:UpdateUserDefinedFunction", "glue:DeleteUserDefinedFunction", "glue:GetUserDefinedFunction", "glue:GetUserDefinedFunctions", "cloudwatch:PutMetricData", "dynamodb:ListTables", "s3:HeadBucket", "ec2:Describe*", ], resources=['*'] ), _iam.PolicyStatement( actions=['s3:GetObject'], resources=[ 'arn:aws:s3:::' + ARA_BUCKET_NAME + BINARIES + DataGenConfig.DSDGEN_INSTALL_SCRIPT, 'arn:aws:s3:::' + ARA_BUCKET_NAME + BINARIES + DataGenConfig.JAR_FILE ] ), _iam.PolicyStatement( actions=['s3:PutObject'], resources=[log_bucket.bucket_arn + "/data-generator/*"] ), _iam.PolicyStatement( actions=[ "s3:AbortMultipartUpload", "s3:CreateBucket", "s3:DeleteObject", "s3:GetBucketVersioning", "s3:GetObject", "s3:GetObjectTagging", "s3:GetObjectVersion", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:ListBucketVersions", "s3:ListMultipartUploadParts", "s3:PutBucketVersioning", "s3:PutObject", "s3:PutObjectTagging" ], resources=[ sink_bucket.bucket_arn + '/*', sink_bucket.bucket_arn ] ) ], roles=[cluster_role] ) cluster_role.add_managed_policy(_iam.ManagedPolicy.from_aws_managed_policy_name('AmazonSSMManagedInstanceCore')) _iam.CfnInstanceProfile( self, 'BatchEmrClusterInstanceProfile', roles=[cluster_role.role_name], instance_profile_name=cluster_role.role_name ) # Security Groups for the EMR cluster (private subnet) # https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-man-sec-groups.html#emr-sg-elasticmapreduce-master-private master_sg = _ec2.SecurityGroup(self, 'ElasticMapReduce-Master-Private', vpc=vpc) slave_sg = _ec2.SecurityGroup(self, 'ElasticMapReduce-Slave-Private', vpc=vpc) service_sg = _ec2.SecurityGroup(self, 'ElasticMapReduce-ServiceAccess', vpc=vpc, allow_all_outbound=False) # Service SG used by the proxy instance service_sg.add_ingress_rule(master_sg, _ec2.Port.tcp(9443)) service_sg.add_egress_rule(master_sg, _ec2.Port.tcp(8443)) service_sg.add_egress_rule(slave_sg, _ec2.Port.tcp(8443)) # EMR Master master_sg.add_ingress_rule(master_sg, _ec2.Port.all_icmp()) master_sg.add_ingress_rule(master_sg, _ec2.Port.all_tcp()) master_sg.add_ingress_rule(master_sg, _ec2.Port.all_udp()) master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_icmp()) master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_tcp()) master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_udp()) master_sg.add_ingress_rule(service_sg, _ec2.Port.tcp(8443)) # EMR Slave slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_icmp()) slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_tcp()) slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_udp()) slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_icmp()) slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_tcp()) slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_udp()) slave_sg.add_ingress_rule(service_sg, _ec2.Port.tcp(8443)) with open('common/common_cdk/lambda/datagen_config.py', 'r') as f: lambda_source = f.read() configure_datagen_function = _lambda.SingletonFunction( self, 'BatchConfigureDatagenLambda', uuid="58a9a222-ff07-11ea-adc1-0242ac120002", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.inline(lambda_source), handler='index.handler', function_name='datagen-config', environment={ 'TABLE_NAME': config_table.table_name, 'JAR_LOCATION': BINARIES_LOCATION + DataGenConfig.JAR_FILE, }, timeout=core.Duration.seconds(10) ) configure_datagen_function.role.add_to_policy( _iam.PolicyStatement( actions=[ 'dynamodb:GetItem', 'dynamodb:PutItem', ], resources=[config_table.table_arn] ) ) terminate_cluster = _sfn_tasks.EmrTerminateCluster( self, 'BatchDeleteCluster', cluster_id=_sfn.TaskInput.from_data_at("$.Emr.Cluster.Id").value, integration_pattern=_sfn.IntegrationPattern.RUN_JOB, ) terminate_cluster_error = _sfn_tasks.EmrTerminateCluster( self, 'BatchDeleteClusterError', cluster_id=_sfn.TaskInput.from_data_at("$.Emr.Cluster.Id").value, integration_pattern=_sfn.IntegrationPattern.RUN_JOB, ).next(_sfn.Fail(self, 'StepFailure')) create_cluster = _sfn_tasks.EmrCreateCluster( self, "BatchCreateEMRCluster", name="BatchDatagenCluster", result_path="$.Emr", release_label='emr-5.30.1', log_uri=log_bucket.s3_url_for_object() + "/data-generator", cluster_role=cluster_role, service_role=service_role, bootstrap_actions=[ _sfn_tasks.EmrCreateCluster.BootstrapActionConfigProperty( name="dsdgen-install", script_bootstrap_action=_sfn_tasks.EmrCreateCluster.ScriptBootstrapActionConfigProperty( path=BINARIES_LOCATION + DataGenConfig.DSDGEN_INSTALL_SCRIPT, ) ) ], applications=[ _sfn_tasks.EmrCreateCluster.ApplicationConfigProperty( name="spark" ), _sfn_tasks.EmrCreateCluster.ApplicationConfigProperty( name="hadoop" ) ], instances=_sfn_tasks.EmrCreateCluster.InstancesConfigProperty( emr_managed_master_security_group=master_sg.security_group_id, emr_managed_slave_security_group=slave_sg.security_group_id, service_access_security_group=service_sg.security_group_id, ec2_subnet_ids=vpc.select_subnets().subnet_ids, instance_fleets=[ _sfn_tasks.EmrCreateCluster.InstanceFleetConfigProperty( instance_fleet_type=_sfn_tasks.EmrCreateCluster.InstanceRoleType.MASTER, instance_type_configs=[ _sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty( instance_type='m5.xlarge', weighted_capacity=1 ), _sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty( instance_type='m5a.xlarge', weighted_capacity=1 ), _sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty( instance_type='m4.xlarge', weighted_capacity=1 ), _sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty( instance_type='m5d.xlarge', weighted_capacity=1 ), ], launch_specifications=_sfn_tasks.EmrCreateCluster.InstanceFleetProvisioningSpecificationsProperty( spot_specification=_sfn_tasks.EmrCreateCluster.SpotProvisioningSpecificationProperty( timeout_action=_sfn_tasks.EmrCreateCluster.SpotTimeoutAction.SWITCH_TO_ON_DEMAND, timeout_duration_minutes=5 ) ), target_on_demand_capacity=0, target_spot_capacity=1 ), _sfn_tasks.EmrCreateCluster.InstanceFleetConfigProperty( instance_fleet_type=_sfn_tasks.EmrCreateCluster.InstanceRoleType.CORE, instance_type_configs=[ _sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty( instance_type='m5.xlarge', weighted_capacity=1 ), _sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty( instance_type='m5.2xlarge', weighted_capacity=2 ), _sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty( instance_type='m5a.xlarge', weighted_capacity=1 ), _sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty( instance_type='m5a.2xlarge', weighted_capacity=2 ), _sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty( instance_type='m4.xlarge', weighted_capacity=1 ) ], launch_specifications=_sfn_tasks.EmrCreateCluster.InstanceFleetProvisioningSpecificationsProperty( spot_specification=_sfn_tasks.EmrCreateCluster.SpotProvisioningSpecificationProperty( timeout_action=_sfn_tasks.EmrCreateCluster.SpotTimeoutAction.SWITCH_TO_ON_DEMAND, timeout_duration_minutes=5 ) ), target_on_demand_capacity=0, target_spot_capacity=DataGenConfig.BATCH_CLUSTER_SIZE[tshirt_size] ) ] ) ).add_catch(handler=terminate_cluster_error, result_path="$.error") configure_datagen = _sfn_tasks.LambdaInvoke( self, "BatchConfigureDatagenTask", lambda_function=configure_datagen_function, payload=_sfn.TaskInput.from_text('{' '"Param": "batch_iterator",' '"Module": "batch",' '"SinkBucket": "'+sink_bucket.s3_url_for_object()+'",' '"Parallelism": "'+str(int(DataGenConfig.BATCH_DATA_SIZE[tshirt_size])*2)+'",' '"DataSize": "'+DataGenConfig.BATCH_DATA_SIZE[tshirt_size]+'",' '"TmpBucket": "fake-bucket"' '}'), result_path='$.Config' ).add_catch(handler=terminate_cluster_error, result_path="$.error") add_datagen_step = _sfn.CustomState( self, 'BatchAddDataGenStep', state_json={ "Type": "Task", "Resource": "arn:aws:states:::elasticmapreduce:addStep.sync", "Parameters": { "ClusterId.$": "$.Emr.Cluster.Id", "Step": { "Name": "DatagenStep", "ActionOnFailure": "CONTINUE", "HadoopJarStep": { "Jar": "command-runner.jar", "Args.$": "$.Config.Payload.StepParam" } } }, "ResultPath": "$.Step", "Next": "BatchUpdateIterator", "Catch": [ { "ErrorEquals": ["States.ALL"], "Next": "BatchDeleteClusterError", "ResultPath": "$.error" } ] } ) update_iterator = _sfn_tasks.DynamoUpdateItem( self, 'BatchUpdateIterator', table=config_table, key={ 'param': _sfn_tasks.DynamoAttributeValue.from_string('batch_iterator') }, update_expression='SET iterator = if_not_exists(iterator, :start) + :inc', expression_attribute_values={ ":inc": _sfn_tasks.DynamoAttributeValue.from_number(1), ":start": _sfn_tasks.DynamoAttributeValue.from_number(0) }, result_path=_sfn.JsonPath.DISCARD ) definition = configure_datagen \ .next(create_cluster) \ .next(add_datagen_step) \ .next(update_iterator) \ .next(terminate_cluster) datagen_stepfunctions = _sfn.StateMachine( self, "BatchDataGenStepFunctions", definition=definition, timeout=core.Duration.minutes(30) ) datagen_stepfunctions.add_to_role_policy( _iam.PolicyStatement( actions=[ 'elasticmapreduce:AddJobFlowSteps', 'elasticmapreduce:DescribeStep' ], resources=['*'] ) ) datagen_stepfunctions.add_to_role_policy( _iam.PolicyStatement( actions= [ "iam:CreateServiceLinkedRole", "iam:PutRolePolicy" ], resources=["arn:aws:iam::*:role/aws-service-role/elasticmapreduce.amazonaws.com*/AWSServiceRoleForEMRCleanup*"], conditions= { "StringLike": { "iam:AWSServiceName": [ "elasticmapreduce.amazonaws.com", "elasticmapreduce.amazonaws.com.cn" ] } } ) ) step_trigger = _events.Rule( self, 'BatchSteptrigger', schedule=_events.Schedule.cron(minute='0/30', hour='*', month='*', week_day='*', year='*') ) step_trigger.add_target(_events_targets.SfnStateMachine(machine=datagen_stepfunctions)) with open('common/common_cdk/lambda/stepfunctions_trigger.py', 'r') as f: lambda_source = f.read() stepfunctions_trigger_lambda = _lambda.SingletonFunction( self, 'BatchStepFunctionsTriggerLambda', uuid="9597f6f2-f840-11ea-adc1-0242ac120002", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.inline(lambda_source), handler='index.handler', function_name='stepfunctions-batch-datagen-trigger' ) stepfunctions_trigger_lambda.role.add_to_policy( _iam.PolicyStatement( actions=["states:StartExecution"], resources=['*'] ) ) trigger_step_lambda_provider = _custom_resources.Provider( self, 'StepFunctionsTriggerLambdaProvider', on_event_handler=stepfunctions_trigger_lambda ) core.CustomResource( self, 'StepFunctionsTrigger', service_token=trigger_step_lambda_provider.service_token, properties={ "stepArn": datagen_stepfunctions.state_machine_arn } ) # terminate clusters with open('common/common_cdk/lambda/stepfunctions_terminate_emr.py', 'r') as f: lambda_source = f.read() sfn_terminate = _lambda.SingletonFunction( self, 'StepFuncTerminateBatch', uuid='58a9a422-ff07-11ea-adc1-0242ac120002', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.inline(lambda_source), handler='index.handler', timeout=core.Duration.minutes(5) ) sfn_terminate.role.add_to_policy( _iam.PolicyStatement( actions=[ 'elasticmapreduce:ListClusters', 'elasticmapreduce:TerminateJobFlows', 'states:ListStateMachines', 'states:ListExecutions', 'states:StopExecution' ], resources=['*'] ) ) sfn_terminate_provider = _custom_resources.Provider( self, 'StepFuncTerminateBatchLambdaProvider', on_event_handler=sfn_terminate ) core.CustomResource( self, 'StepFuncTerminateBatchCustomResource', service_token=sfn_terminate_provider.service_token, properties={ "state_machine": 'BatchDatagen' })
def __init__(self, scope: core.Construct, id: str, cluster: eks.Cluster, **kwargs) -> None: super().__init__(scope, id, **kwargs) maps= [] self.roles=[] ecr_policy = iam.PolicyStatement( actions=[ "ecr:DescribeImages", "ecr:ListImages", "ecr:BatchDeleteImage" ], effect=iam.Effect.ALLOW, resources=[ "arn:aws:ecr:%s:%s:repository/%s" % (core.Stack.of(self).region, core.Stack.of(self).account, namespace) for namespace in self.node.try_get_context("kubernetes")['namespaces'] ] ) function = lbd.SingletonFunction( self, "ECRDeleteImagesFunction", uuid="19411b0e-0e80-4ad4-a316-3235940775e4", code=lbd.Code.from_asset( "custom_resources/kubernetes/" ), handler="config.handler", runtime=lbd.Runtime.PYTHON_3_7, function_name="kubernetesConfig", initial_policy=[ecr_policy], log_retention=logs.RetentionDays.ONE_DAY, timeout=core.Duration.seconds(30) ) provider = cr.Provider( self, "ECRDeleteImagesFunctionProvider", on_event_handler=function, log_retention=logs.RetentionDays.ONE_DAY ) repositores = [] for namespace in self.node.try_get_context("kubernetes")['namespaces']: manifest = cluster.add_manifest( "eksConfigNamespace-%s" % namespace, { "apiVersion": "v1", "kind": "Namespace", "metadata": { "name": namespace } } ) sa = cluster.add_service_account( "service-account-%s" % namespace, name="statement-demo", namespace=namespace ) sa.node.add_dependency(manifest) self.roles.append(sa.role) repository = ecr.Repository( self, ("repository-%s" % namespace), removal_policy=core.RemovalPolicy.DESTROY, repository_name=namespace, lifecycle_rules=[ecr.LifecycleRule(max_image_count=1)] ) repositores.append(repository.repository_arn) maps.append({ "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": "application.properties", "namespace": namespace }, "data": { "application-aws.properties": Path("../%s/src/main/resources/application-aws.properties" % namespace).read_text() } }) core.CustomResource( self, "ECRDeleteImagesFunction-%s" % namespace, service_token=provider.service_token, properties={ "repository": namespace } ).node.add_dependency(repository) eks.KubernetesManifest( self, "eksConfigMaps", cluster=cluster, manifest=maps ) iam.Policy( self, "saPolicy", force=True, policy_name="EKSSAPolicy", roles=self.roles, statements=[ iam.PolicyStatement( actions=["cloudwatch:PutMetricData"], conditions={ "StringEquals": { "cloudwatch:namespace": "statement12" }, }, resources=["*"] ) ] )
def __init__(self, scope: core.Construct, id: str, elastic: Elastic, vpc: ec2.Vpc, roles: list, cluster: eks.Cluster, **kwargs) -> None: super().__init__(scope, id, **kwargs) sm_policy = iam.PolicyStatement( actions=["secretsmanager:GetSecretValue"], effect=iam.Effect.ALLOW, resources=[elastic.secret.secret_arn] ) es_policy = iam.PolicyStatement( actions=["es:DescribeElasticsearchDomain"], effect=iam.Effect.ALLOW, resources=[elastic.domain.domain_arn] ) function = lbd.SingletonFunction( self, "ElasticsearchConfigFunction", uuid="e579d5f9-1709-43ea-b75f-9d1452ca7690", code=lbd.Code.from_asset( "custom_resources/elasticsearch/" ), handler="config.handler", runtime=lbd.Runtime.PYTHON_3_7, function_name="elasticsearchConfig", initial_policy=[sm_policy,es_policy], log_retention=logs.RetentionDays.ONE_DAY, security_group=ec2.SecurityGroup.from_security_group_id(self, "lambdaVPC", vpc.vpc_default_security_group), timeout=core.Duration.seconds(30), vpc=vpc, vpc_subnets=ec2.SubnetSelection( one_per_az=True ) ) provider = cr.Provider( self, "ElasticsearchConfigProvider", on_event_handler=function, log_retention=logs.RetentionDays.ONE_DAY ) core.CustomResource( self, "ElasticSearchConfig", service_token=provider.service_token, properties={ "domain": elastic.domain.domain_name, "secret": elastic.secret.secret_arn, "roles": [role.role_arn for role in roles], "shards": self.node.try_get_context("elastic")['shards'], "user": boto3.client('sts').get_caller_identity().get('Arn'), "replicas": self.node.try_get_context("elastic")['replicas'] } ) manifests = [] for namespace in self.node.try_get_context("kubernetes")['namespaces']: manifests.append({ "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": "elasticsearch", "namespace": namespace }, "data": { "url": elastic.domain.domain_endpoint } }) eks.KubernetesManifest( self, "elastic-search-cm", cluster=cluster, manifest=manifests )
def __init__(self, scope: cdk.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ### Parameters bootstrap_script_args = cdk.CfnParameter(self, 'BootstrapScriptArgs', type='String', default='', description='Space seperated arguments passed to the bootstrap script.' ) # create a VPC vpc = ec2.Vpc(self, 'VPC', cidr='10.0.0.0/16', max_azs=99) # create a private and public subnet per vpc selection = vpc.select_subnets( subnet_type=ec2.SubnetType.PRIVATE ) # Output created subnets for i, public_subnet in enumerate(vpc.public_subnets): cdk.CfnOutput(self, 'PublicSubnet%i' % i, value=public_subnet.subnet_id) for i, private_subnet in enumerate(vpc.private_subnets): cdk.CfnOutput(self, 'PrivateSubnet%i' % i, value=private_subnet.subnet_id) cdk.CfnOutput(self, 'VPCId', value=vpc.vpc_id) # Create a Bucket bucket = s3.Bucket(self, "DataRepository") quickstart_bucket = s3.Bucket.from_bucket_name(self, 'QuickStartBucket', 'aws-quickstart') # Upload Bootstrap Script to that bucket bootstrap_script = assets.Asset(self, 'BootstrapScript', path='scripts/bootstrap.sh' ) # Upload parallel cluster post_install_script to that bucket pcluster_post_install_script = assets.Asset(self, 'PclusterPostInstallScript', path='scripts/post_install_script.sh' ) # Setup CloudTrail cloudtrail.Trail(self, 'CloudTrail', bucket=bucket) # Create a Cloud9 instance # Cloud9 doesn't have the ability to provide userdata # Because of this we need to use SSM run command cloud9_instance = cloud9.Ec2Environment(self, 'Cloud9Env', vpc=vpc, instance_type=ec2.InstanceType(instance_type_identifier='c5.large')) cdk.CfnOutput(self, 'URL', value=cloud9_instance.ide_url) # Create a keypair in lambda and store the private key in SecretsManager c9_createkeypair_role = iam.Role(self, 'Cloud9CreateKeypairRole', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com')) c9_createkeypair_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AWSLambdaBasicExecutionRole')) # Add IAM permissions to the lambda role c9_createkeypair_role.add_to_policy(iam.PolicyStatement( actions=[ 'ec2:CreateKeyPair', 'ec2:DeleteKeyPair' ], resources=['*'], )) # Lambda for Cloud9 keypair c9_createkeypair_lambda = _lambda.Function(self, 'C9CreateKeyPairLambda', runtime=_lambda.Runtime.PYTHON_3_6, handler='lambda_function.handler', timeout=cdk.Duration.seconds(300), role=c9_createkeypair_role, code=_lambda.Code.asset('functions/source/c9keypair'), # code=_lambda.Code.from_bucket( ) c9_createkeypair_provider = cr.Provider(self, "C9CreateKeyPairProvider", on_event_handler=c9_createkeypair_lambda) c9_createkeypair_cr = cfn.CustomResource(self, "C9CreateKeyPair", provider=c9_createkeypair_provider, properties={ 'ServiceToken': c9_createkeypair_lambda.function_arn } ) #c9_createkeypair_cr.node.add_dependency(instance_id) c9_ssh_private_key_secret = secretsmanager.CfnSecret(self, 'SshPrivateKeySecret', secret_string=c9_createkeypair_cr.get_att_string('PrivateKey') ) # The iam policy has a <REGION> parameter that needs to be replaced. # We do it programmatically so future versions of the synth'd stack # template include all regions. with open('iam/ParallelClusterUserPolicy.json') as json_file: data = json.load(json_file) for s in data['Statement']: if s['Sid'] == 'S3ParallelClusterReadOnly': s['Resource'] = [] for r in region_info.RegionInfo.regions: s['Resource'].append('arn:aws:s3:::{0}-aws-parallelcluster*'.format(r.name)) parallelcluster_user_policy = iam.CfnManagedPolicy(self, 'ParallelClusterUserPolicy', policy_document=iam.PolicyDocument.from_json(data)) # Cloud9 IAM Role cloud9_role = iam.Role(self, 'Cloud9Role', assumed_by=iam.ServicePrincipal('ec2.amazonaws.com')) cloud9_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name('AmazonSSMManagedInstanceCore')) cloud9_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name('AWSCloud9User')) cloud9_role.add_managed_policy(iam.ManagedPolicy.from_managed_policy_arn(self, 'AttachParallelClusterUserPolicy', parallelcluster_user_policy.ref)) cloud9_role.add_to_policy(iam.PolicyStatement( resources=['*'], actions=[ 'ec2:DescribeInstances', 'ec2:DescribeVolumes', 'ec2:ModifyVolume' ] )) cloud9_role.add_to_policy(iam.PolicyStatement( resources=[c9_ssh_private_key_secret.ref], actions=[ 'secretsmanager:GetSecretValue' ] )) bootstrap_script.grant_read(cloud9_role) pcluster_post_install_script.grant_read(cloud9_role) # Cloud9 User # user = iam.User(self, 'Cloud9User', password=cdk.SecretValue.plain_text('supersecretpassword'), password_reset_required=True) # Cloud9 Setup IAM Role cloud9_setup_role = iam.Role(self, 'Cloud9SetupRole', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com')) cloud9_setup_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AWSLambdaBasicExecutionRole')) # Allow pcluster to be run in bootstrap cloud9_setup_role.add_managed_policy(iam.ManagedPolicy.from_managed_policy_arn(self, 'AttachParallelClusterUserPolicySetup', parallelcluster_user_policy.ref)) # Add IAM permissions to the lambda role cloud9_setup_role.add_to_policy(iam.PolicyStatement( actions=[ 'cloudformation:DescribeStackResources', 'ec2:AssociateIamInstanceProfile', 'ec2:AuthorizeSecurityGroupIngress', 'ec2:DescribeInstances', 'ec2:DescribeInstanceStatus', 'ec2:DescribeInstanceAttribute', 'ec2:DescribeIamInstanceProfileAssociations', 'ec2:DescribeVolumes', 'ec2:DesctibeVolumeAttribute', 'ec2:DescribeVolumesModifications', 'ec2:DescribeVolumeStatus', 'ssm:DescribeInstanceInformation', 'ec2:ModifyVolume', 'ec2:ReplaceIamInstanceProfileAssociation', 'ec2:ReportInstanceStatus', 'ssm:SendCommand', 'ssm:GetCommandInvocation', 's3:GetObject', 'lambda:AddPermission', 'lambda:RemovePermission', 'events:PutRule', 'events:DeleteRule', 'events:PutTargets', 'events:RemoveTargets', ], resources=['*'], )) cloud9_setup_role.add_to_policy(iam.PolicyStatement( actions=['iam:PassRole'], resources=[cloud9_role.role_arn] )) cloud9_setup_role.add_to_policy(iam.PolicyStatement( actions=[ 'lambda:AddPermission', 'lambda:RemovePermission' ], resources=['*'] )) # Cloud9 Instance Profile c9_instance_profile = iam.CfnInstanceProfile(self, "Cloud9InstanceProfile", roles=[cloud9_role.role_name]) # Lambda to add Instance Profile to Cloud9 c9_instance_profile_lambda = _lambda.Function(self, 'C9InstanceProfileLambda', runtime=_lambda.Runtime.PYTHON_3_6, handler='lambda_function.handler', timeout=cdk.Duration.seconds(900), role=cloud9_setup_role, code=_lambda.Code.asset('functions/source/c9InstanceProfile'), ) c9_instance_profile_provider = cr.Provider(self, "C9InstanceProfileProvider", on_event_handler=c9_instance_profile_lambda, ) instance_id = cfn.CustomResource(self, "C9InstanceProfile", provider=c9_instance_profile_provider, properties={ 'InstanceProfile': c9_instance_profile.ref, 'Cloud9Environment': cloud9_instance.environment_id, } ) instance_id.node.add_dependency(cloud9_instance) # Lambda for Cloud9 Bootstrap c9_bootstrap_lambda = _lambda.Function(self, 'C9BootstrapLambda', runtime=_lambda.Runtime.PYTHON_3_6, handler='lambda_function.handler', timeout=cdk.Duration.seconds(900), role=cloud9_setup_role, code=_lambda.Code.asset('functions/source/c9bootstrap'), ) c9_bootstrap_provider = cr.Provider(self, "C9BootstrapProvider", on_event_handler=c9_bootstrap_lambda) c9_bootstrap_cr = cfn.CustomResource(self, "C9Bootstrap", provider=c9_bootstrap_provider, properties={ 'Cloud9Environment': cloud9_instance.environment_id, 'BootstrapPath': 's3://%s/%s' % (bootstrap_script.s3_bucket_name, bootstrap_script.s3_object_key), 'BootstrapArguments': bootstrap_script_args, 'VPCID': vpc.vpc_id, 'MasterSubnetID': vpc.public_subnets[0].subnet_id, 'ComputeSubnetID': vpc.private_subnets[0].subnet_id, 'PostInstallScriptS3Url': "".join( ['s3://', pcluster_post_install_script.s3_bucket_name, "/", pcluster_post_install_script.s3_object_key ] ), 'PostInstallScriptBucket': pcluster_post_install_script.s3_bucket_name, 'KeyPairId': c9_createkeypair_cr.ref, 'KeyPairSecretArn': c9_ssh_private_key_secret.ref } ) c9_bootstrap_cr.node.add_dependency(instance_id) c9_bootstrap_cr.node.add_dependency(c9_createkeypair_cr) c9_bootstrap_cr.node.add_dependency(c9_ssh_private_key_secret)
def __init__(self, scope: cdk.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Version of ParallelCluster for Cloud9. pcluster_version = cdk.CfnParameter( self, 'ParallelClusterVersion', description= 'Specify a custom parallelcluster version. See https://pypi.org/project/aws-parallelcluster/#history for options.', default='2.8.0', type='String', allowed_values=get_version_list('aws-parallelcluster')) # S3 URI for Config file config = cdk.CfnParameter( self, 'ConfigS3URI', description='Set a custom parallelcluster config file.', default= 'https://notearshpc-quickstart.s3.amazonaws.com/{0}/config.ini'. format(__version__)) # Password password = cdk.CfnParameter( self, 'UserPasswordParameter', description='Set a password for the hpc-quickstart user', no_echo=True) # create a VPC vpc = ec2.Vpc( self, 'VPC', cidr='10.0.0.0/16', gateway_endpoints={ "S3": ec2.GatewayVpcEndpointOptions( service=ec2.GatewayVpcEndpointAwsService.S3), "DynamoDB": ec2.GatewayVpcEndpointOptions( service=ec2.GatewayVpcEndpointAwsService.DYNAMODB) }, max_azs=99) # create a private and public subnet per vpc selection = vpc.select_subnets(subnet_type=ec2.SubnetType.PRIVATE) # Output created subnets for i, public_subnet in enumerate(vpc.public_subnets): cdk.CfnOutput(self, 'PublicSubnet%i' % i, value=public_subnet.subnet_id) for i, private_subnet in enumerate(vpc.private_subnets): cdk.CfnOutput(self, 'PrivateSubnet%i' % i, value=private_subnet.subnet_id) cdk.CfnOutput(self, 'VPCId', value=vpc.vpc_id) # Create a Bucket data_bucket = s3.Bucket(self, "DataRepository") cdk.CfnOutput(self, 'DataRespository', value=data_bucket.bucket_name) cloudtrail_bucket = s3.Bucket(self, "CloudTrailLogs") quickstart_bucket = s3.Bucket.from_bucket_name(self, 'QuickStartBucket', 'aws-quickstart') # Upload Bootstrap Script to that bucket bootstrap_script = assets.Asset(self, 'BootstrapScript', path='scripts/bootstrap.sh') # Upload parallel cluster post_install_script to that bucket pcluster_post_install_script = assets.Asset( self, 'PclusterPostInstallScript', path='scripts/post_install_script.sh') # Upload parallel cluster post_install_script to that bucket pcluster_config_script = assets.Asset(self, 'PclusterConfigScript', path='scripts/config.ini') # Setup CloudTrail cloudtrail.Trail(self, 'CloudTrail', bucket=cloudtrail_bucket) # Create a Cloud9 instance # Cloud9 doesn't have the ability to provide userdata # Because of this we need to use SSM run command cloud9_instance = cloud9.Ec2Environment( self, 'ResearchWorkspace', vpc=vpc, instance_type=ec2.InstanceType( instance_type_identifier='c5.large')) cdk.CfnOutput(self, 'Research Workspace URL', value=cloud9_instance.ide_url) # Create a keypair in lambda and store the private key in SecretsManager c9_createkeypair_role = iam.Role( self, 'Cloud9CreateKeypairRole', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com')) c9_createkeypair_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaBasicExecutionRole')) # Add IAM permissions to the lambda role c9_createkeypair_role.add_to_policy( iam.PolicyStatement( actions=['ec2:CreateKeyPair', 'ec2:DeleteKeyPair'], resources=['*'], )) # Lambda for Cloud9 keypair c9_createkeypair_lambda = _lambda.Function( self, 'C9CreateKeyPairLambda', runtime=_lambda.Runtime.PYTHON_3_6, handler='lambda_function.handler', timeout=cdk.Duration.seconds(300), role=c9_createkeypair_role, code=_lambda.Code.asset('functions/source/c9keypair'), ) c9_createkeypair_provider = cr.Provider( self, "C9CreateKeyPairProvider", on_event_handler=c9_createkeypair_lambda) c9_createkeypair_cr = cfn.CustomResource( self, "C9CreateKeyPair", provider=c9_createkeypair_provider, properties={'ServiceToken': c9_createkeypair_lambda.function_arn}) #c9_createkeypair_cr.node.add_dependency(instance_id) c9_ssh_private_key_secret = secretsmanager.CfnSecret( self, 'SshPrivateKeySecret', secret_string=c9_createkeypair_cr.get_att_string('PrivateKey')) # The iam policy has a <REGION> parameter that needs to be replaced. # We do it programmatically so future versions of the synth'd stack # template include all regions. with open('iam/ParallelClusterUserPolicy.json') as json_file: data = json.load(json_file) for s in data['Statement']: if s['Sid'] == 'S3ParallelClusterReadOnly': s['Resource'] = [] for r in region_info.RegionInfo.regions: s['Resource'].append( 'arn:aws:s3:::{0}-aws-parallelcluster*'.format( r.name)) parallelcluster_user_policy = iam.CfnManagedPolicy( self, 'ParallelClusterUserPolicy', policy_document=iam.PolicyDocument.from_json(data)) # Cloud9 IAM Role cloud9_role = iam.Role( self, 'Cloud9Role', assumed_by=iam.ServicePrincipal('ec2.amazonaws.com')) cloud9_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonSSMManagedInstanceCore')) cloud9_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name('AWSCloud9User')) cloud9_role.add_managed_policy( iam.ManagedPolicy.from_managed_policy_arn( self, 'AttachParallelClusterUserPolicy', parallelcluster_user_policy.ref)) cloud9_role.add_to_policy( iam.PolicyStatement(resources=['*'], actions=[ 'ec2:DescribeInstances', 'ec2:DescribeVolumes', 'ec2:ModifyVolume' ])) cloud9_role.add_to_policy( iam.PolicyStatement(resources=[c9_ssh_private_key_secret.ref], actions=['secretsmanager:GetSecretValue'])) cloud9_role.add_to_policy( iam.PolicyStatement( actions=["s3:Get*", "s3:List*"], resources=[ "arn:aws:s3:::%s/*" % (data_bucket.bucket_name), "arn:aws:s3:::%s" % (data_bucket.bucket_name) ])) bootstrap_script.grant_read(cloud9_role) pcluster_post_install_script.grant_read(cloud9_role) pcluster_config_script.grant_read(cloud9_role) # Admin Group admin_group = iam.Group(self, 'AdminGroup') admin_group.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'AdministratorAccess')) admin_group.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'AWSCloud9Administrator')) # PowerUser Group poweruser_group = iam.Group(self, 'PowerUserGroup') poweruser_group.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name('PowerUserAccess')) poweruser_group.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'AWSCloud9Administrator')) # HPC User user = iam.CfnUser( self, 'Researcher', groups=[admin_group.node.default_child.ref], login_profile=iam.CfnUser.LoginProfileProperty( password_reset_required=True, password=cdk.SecretValue.cfn_parameter(password).to_string())) create_user = cdk.CfnParameter(self, "CreateUser", default="false", type="String", allowed_values=['true', 'false' ]).value_as_string user_condition = cdk.CfnCondition(self, "UserCondition", expression=cdk.Fn.condition_equals( create_user, "true")) user.cfn_options.condition = user_condition cdk.CfnOutput(self, 'UserLoginUrl', value="".join([ "https://", self.account, ".signin.aws.amazon.com/console" ]), condition=user_condition) cdk.CfnOutput(self, 'UserName', value=user.ref, condition=user_condition) # Cloud9 Setup IAM Role cloud9_setup_role = iam.Role( self, 'Cloud9SetupRole', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com')) cloud9_setup_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaBasicExecutionRole')) # Allow pcluster to be run in bootstrap cloud9_setup_role.add_managed_policy( iam.ManagedPolicy.from_managed_policy_arn( self, 'AttachParallelClusterUserPolicySetup', parallelcluster_user_policy.ref)) # Add IAM permissions to the lambda role cloud9_setup_role.add_to_policy( iam.PolicyStatement( actions=[ 'cloudformation:DescribeStackResources', 'ec2:AssociateIamInstanceProfile', 'ec2:AuthorizeSecurityGroupIngress', 'ec2:DescribeInstances', 'ec2:DescribeInstanceStatus', 'ec2:DescribeInstanceAttribute', 'ec2:DescribeIamInstanceProfileAssociations', 'ec2:DescribeVolumes', 'ec2:DesctibeVolumeAttribute', 'ec2:DescribeVolumesModifications', 'ec2:DescribeVolumeStatus', 'ssm:DescribeInstanceInformation', 'ec2:ModifyVolume', 'ec2:ReplaceIamInstanceProfileAssociation', 'ec2:ReportInstanceStatus', 'ssm:SendCommand', 'ssm:GetCommandInvocation', 's3:GetObject', 'lambda:AddPermission', 'lambda:RemovePermission', 'events:PutRule', 'events:DeleteRule', 'events:PutTargets', 'events:RemoveTargets', 'cloud9:CreateEnvironmentMembership', ], resources=['*'], )) cloud9_setup_role.add_to_policy( iam.PolicyStatement(actions=['iam:PassRole'], resources=[cloud9_role.role_arn])) cloud9_setup_role.add_to_policy( iam.PolicyStatement( actions=['lambda:AddPermission', 'lambda:RemovePermission'], resources=['*'])) # Cloud9 Instance Profile c9_instance_profile = iam.CfnInstanceProfile( self, "Cloud9InstanceProfile", roles=[cloud9_role.role_name]) # Lambda to add Instance Profile to Cloud9 c9_instance_profile_lambda = _lambda.Function( self, 'C9InstanceProfileLambda', runtime=_lambda.Runtime.PYTHON_3_6, handler='lambda_function.handler', timeout=cdk.Duration.seconds(900), role=cloud9_setup_role, code=_lambda.Code.asset('functions/source/c9InstanceProfile'), ) c9_instance_profile_provider = cr.Provider( self, "C9InstanceProfileProvider", on_event_handler=c9_instance_profile_lambda, ) instance_id = cfn.CustomResource(self, "C9InstanceProfile", provider=c9_instance_profile_provider, properties={ 'InstanceProfile': c9_instance_profile.ref, 'Cloud9Environment': cloud9_instance.environment_id, }) instance_id.node.add_dependency(cloud9_instance) # Lambda for Cloud9 Bootstrap c9_bootstrap_lambda = _lambda.Function( self, 'C9BootstrapLambda', runtime=_lambda.Runtime.PYTHON_3_6, handler='lambda_function.handler', timeout=cdk.Duration.seconds(900), role=cloud9_setup_role, code=_lambda.Code.asset('functions/source/c9bootstrap'), ) c9_bootstrap_provider = cr.Provider( self, "C9BootstrapProvider", on_event_handler=c9_bootstrap_lambda) c9_bootstrap_cr = cfn.CustomResource( self, "C9Bootstrap", provider=c9_bootstrap_provider, properties={ 'Cloud9Environment': cloud9_instance.environment_id, 'BootstrapPath': 's3://%s/%s' % (bootstrap_script.s3_bucket_name, bootstrap_script.s3_object_key), 'Config': config, 'VPCID': vpc.vpc_id, 'MasterSubnetID': vpc.public_subnets[0].subnet_id, 'ComputeSubnetID': vpc.private_subnets[0].subnet_id, 'PostInstallScriptS3Url': "".join([ 's3://', pcluster_post_install_script.s3_bucket_name, "/", pcluster_post_install_script.s3_object_key ]), 'PostInstallScriptBucket': pcluster_post_install_script.s3_bucket_name, 'S3ReadWriteResource': data_bucket.bucket_arn, 'S3ReadWriteUrl': 's3://%s' % (data_bucket.bucket_name), 'KeyPairId': c9_createkeypair_cr.ref, 'KeyPairSecretArn': c9_ssh_private_key_secret.ref, 'UserArn': user.attr_arn, 'PclusterVersion': pcluster_version.value_as_string }) c9_bootstrap_cr.node.add_dependency(instance_id) c9_bootstrap_cr.node.add_dependency(c9_createkeypair_cr) c9_bootstrap_cr.node.add_dependency(c9_ssh_private_key_secret) c9_bootstrap_cr.node.add_dependency(data_bucket) enable_budget = cdk.CfnParameter(self, "EnableBudget", default="true", type="String", allowed_values=['true', 'false' ]).value_as_string # Budgets budget_properties = { 'budgetType': "COST", 'timeUnit': "ANNUALLY", 'budgetLimit': { 'amount': cdk.CfnParameter( self, 'BudgetLimit', description= 'The initial budget for this project in USD ($).', default=2000, type='Number').value_as_number, 'unit': "USD", }, 'costFilters': None, 'costTypes': { 'includeCredit': False, 'includeDiscount': True, 'includeOtherSubscription': True, 'includeRecurring': True, 'includeRefund': True, 'includeSubscription': True, 'includeSupport': True, 'includeTax': True, 'includeUpfront': True, 'useAmortized': False, 'useBlended': False, }, 'plannedBudgetLimits': None, 'timePeriod': None, } email = { 'notification': { 'comparisonOperator': "GREATER_THAN", 'notificationType': "ACTUAL", 'threshold': 80, 'thresholdType': "PERCENTAGE", }, 'subscribers': [{ 'address': cdk.CfnParameter( self, 'NotificationEmail', description= 'This email address will receive billing alarm notifications when 80% of the budget limit is reached.', default='*****@*****.**').value_as_string, 'subscriptionType': "EMAIL", }] } overall_budget = budgets.CfnBudget( self, "HPCBudget", budget=budget_properties, notifications_with_subscribers=[email], ) overall_budget.cfn_options.condition = cdk.CfnCondition( self, "BudgetCondition", expression=cdk.Fn.condition_equals(enable_budget, "true"))
def __init__(self, scope: core.Construct, id: str, log_bucket: _s3.Bucket, config_table: _dynamodb.Table, tshirt_size: str, sink_bucket: _s3.Bucket, web_sale_stream: str, web_customer_stream: str, web_customer_address_stream: str, kinesis_key: _kms.Key, vpc: _ec2.Vpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) stack = core.Stack.of(self) stream_source_bucket = AutoEmptyBucket( self, 'StreamSource', bucket_name='ara-stream-source-' + core.Aws.ACCOUNT_ID, uuid='95505f50-0276-11eb-adc1-0242ac120002') service_role = _iam.Role( self, 'StreamEmrServiceRole', assumed_by=_iam.ServicePrincipal('elasticmapreduce.amazonaws.com')) service_role.add_managed_policy( _iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonElasticMapReduceRole')) cluster_role = _iam.Role( self, 'StreamEmrClusterRole', assumed_by=_iam.ServicePrincipal("ec2.amazonaws.com")) _iam.Policy( self, 'StreamEmrClusterPolicy', statements=[ _iam.PolicyStatement(actions=[ "glue:CreateDatabase", "glue:UpdateDatabase", "glue:DeleteDatabase", "glue:GetDatabase", "glue:GetDatabases", "glue:CreateTable", "glue:UpdateTable", "glue:DeleteTable", "glue:GetTable", "glue:GetTables", "glue:GetTableVersions", "glue:CreatePartition", "glue:BatchCreatePartition", "glue:UpdatePartition", "glue:DeletePartition", "glue:BatchDeletePartition", "glue:GetPartition", "glue:GetPartitions", "glue:BatchGetPartition", "glue:CreateUserDefinedFunction", "glue:UpdateUserDefinedFunction", "glue:DeleteUserDefinedFunction", "glue:GetUserDefinedFunction", "glue:GetUserDefinedFunctions", "cloudwatch:PutMetricData", "dynamodb:ListTables", "s3:HeadBucket", "ec2:Describe*", ], resources=['*']), _iam.PolicyStatement( actions=['s3:GetObject'], resources=[ 'arn:aws:s3:::' + ARA_BUCKET_NAME + BINARIES + DataGenConfig.DSDGEN_INSTALL_SCRIPT, 'arn:aws:s3:::' + ARA_BUCKET_NAME + BINARIES + DataGenConfig.JAR_FILE ]), _iam.PolicyStatement( actions=['s3:PutObject'], resources=[log_bucket.bucket_arn + "/data-generator/*"]), _iam.PolicyStatement( actions=[ "s3:AbortMultipartUpload", "s3:CreateBucket", "s3:DeleteObject", "s3:GetBucketVersioning", "s3:GetObject", "s3:GetObjectTagging", "s3:GetObjectVersion", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:ListBucketVersions", "s3:ListMultipartUploadParts", "s3:PutBucketVersioning", "s3:PutObject", "s3:PutObjectTagging" ], resources=[ sink_bucket.bucket_arn + '/*', sink_bucket.bucket_arn, stream_source_bucket.bucket.bucket_arn + '/*', stream_source_bucket.bucket.bucket_arn ]) ], roles=[cluster_role]) cluster_role.add_managed_policy( _iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonSSMManagedInstanceCore')) _iam.CfnInstanceProfile(self, 'StreamEmrClusterInstanceProfile', roles=[cluster_role.role_name], instance_profile_name=cluster_role.role_name) # Security Groups for the EMR cluster (private subnet) # https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-man-sec-groups.html#emr-sg-elasticmapreduce-master-private master_sg = _ec2.SecurityGroup(self, 'ElasticMapReduce-Master-Private', vpc=vpc) slave_sg = _ec2.SecurityGroup(self, 'ElasticMapReduce-Slave-Private', vpc=vpc) service_sg = _ec2.SecurityGroup(self, 'ElasticMapReduce-ServiceAccess', vpc=vpc, allow_all_outbound=False) # Service SG used by the proxy instance service_sg.add_ingress_rule(master_sg, _ec2.Port.tcp(9443)) service_sg.add_egress_rule(master_sg, _ec2.Port.tcp(8443)) service_sg.add_egress_rule(slave_sg, _ec2.Port.tcp(8443)) # EMR Master master_sg.add_ingress_rule(master_sg, _ec2.Port.all_icmp()) master_sg.add_ingress_rule(master_sg, _ec2.Port.all_tcp()) master_sg.add_ingress_rule(master_sg, _ec2.Port.all_udp()) master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_icmp()) master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_tcp()) master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_udp()) master_sg.add_ingress_rule(service_sg, _ec2.Port.tcp(8443)) # EMR Slave slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_icmp()) slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_tcp()) slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_udp()) slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_icmp()) slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_tcp()) slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_udp()) slave_sg.add_ingress_rule(service_sg, _ec2.Port.tcp(8443)) with open('common/common_cdk/lambda/datagen_config.py', 'r') as f: lambda_source = f.read() configure_datagen_function = _lambda.SingletonFunction( self, 'StreamConfigureDatagenLambda', uuid="a9904dec-01cf-11eb-adc1-0242ac120002", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.inline(lambda_source), handler='index.handler', function_name='stream-datagen-config', environment={ 'TABLE_NAME': config_table.table_name, 'JAR_LOCATION': BINARIES_LOCATION + DataGenConfig.JAR_FILE, }, timeout=core.Duration.seconds(10)) configure_datagen_function.role.add_to_policy( _iam.PolicyStatement(actions=[ 'dynamodb:GetItem', 'dynamodb:PutItem', ], resources=[config_table.table_arn])) emr_cluster = _emr.CfnCluster( self, 'StreamEmrCluster', name="StreamDatagenCluster", job_flow_role=cluster_role.role_name, service_role=service_role.role_name, release_label='emr-5.30.1', visible_to_all_users=True, log_uri=log_bucket.s3_url_for_object() + "/data-generator", applications=[ _emr.CfnCluster.ApplicationProperty(name='hadoop'), _emr.CfnCluster.ApplicationProperty(name='spark') ], bootstrap_actions=[ _emr.CfnCluster.BootstrapActionConfigProperty( name="dsdgen-install", script_bootstrap_action=_emr.CfnCluster. ScriptBootstrapActionConfigProperty( path=BINARIES_LOCATION + DataGenConfig.DSDGEN_INSTALL_SCRIPT)) ], instances=_emr.CfnCluster.JobFlowInstancesConfigProperty( emr_managed_master_security_group=master_sg.security_group_id, emr_managed_slave_security_group=slave_sg.security_group_id, service_access_security_group=service_sg.security_group_id, ec2_subnet_id=vpc.private_subnets[0].subnet_id, core_instance_group=_emr.CfnCluster. InstanceGroupConfigProperty(instance_count=DataGenConfig. BATCH_CLUSTER_SIZE[tshirt_size], instance_type='m5.xlarge'), master_instance_group=_emr.CfnCluster. InstanceGroupConfigProperty(instance_count=1, instance_type='m4.large'))) configure_datagen = _sfn_tasks.LambdaInvoke( self, "ConfigureDatagenTask", lambda_function=configure_datagen_function, payload=_sfn.TaskInput.from_text( '{' '"Param": "stream_iterator",' '"Module": "stream",' '"SinkBucket": "' + sink_bucket.s3_url_for_object() + '",' '"Parallelism": "' + str(int(DataGenConfig.STREAM_DATA_SIZE[tshirt_size]) * 2) + '",' '"DataSize": "' + DataGenConfig.STREAM_DATA_SIZE[tshirt_size] + '",' '"TmpBucket": "' + str(stream_source_bucket.bucket.s3_url_for_object()) + '"' '}'), result_path='$.Config') add_datagen_step = _sfn.CustomState( self, 'StreamAddDataGenStep', state_json={ "Type": "Task", "Resource": "arn:aws:states:::elasticmapreduce:addStep.sync", "Parameters": { "ClusterId.$": "$.Emr.Cluster.Id", "Step": { "Name": "DatagenStep", "ActionOnFailure": "CONTINUE", "HadoopJarStep": { "Jar": "command-runner.jar", "Args.$": "$.Config.Payload.StepParam" } } }, "ResultPath": "$.Step", "Next": "StreamUpdateIterator" }) update_iterator = _sfn_tasks.DynamoUpdateItem( self, 'StreamUpdateIterator', table=config_table, key={ 'param': _sfn_tasks.DynamoAttributeValue.from_string('stream_iterator') }, update_expression= 'SET iterator = if_not_exists(iterator, :start) + :inc', expression_attribute_values={ ":inc": _sfn_tasks.DynamoAttributeValue.from_number(1), ":start": _sfn_tasks.DynamoAttributeValue.from_number(0) }, result_path=_sfn.JsonPath.DISCARD) definition = configure_datagen \ .next(add_datagen_step) \ .next(update_iterator) datagen_stepfunctions = _sfn.StateMachine( self, "StreamDataGenStepFunctions", definition=definition, timeout=core.Duration.minutes(30)) datagen_stepfunctions.add_to_role_policy( _iam.PolicyStatement(actions=[ 'elasticmapreduce:AddJobFlowSteps', 'elasticmapreduce:DescribeStep' ], resources=['*'])) step_trigger = _events.Rule(self, 'StreamStepTrigger', schedule=_events.Schedule.cron( minute='0/10', hour='*', month='*', week_day='*', year='*')) step_trigger.add_target( _events_targets.SfnStateMachine( machine=datagen_stepfunctions, input=_events.RuleTargetInput.from_object({ "Emr": { "Cluster": { "Id": core.Fn.ref(emr_cluster.logical_id) } } }))) with open('common/common_cdk/lambda/stepfunctions_trigger.py', 'r') as f: lambda_source = f.read() stepfunctions_trigger_lambda = _lambda.SingletonFunction( self, 'StreamStepFunctionsTriggerLambda', uuid="cf042246-01d0-11eb-adc1-0242ac120002", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.inline(lambda_source), handler='index.handler', function_name='stepfunctions-stream-datagen-trigger') stepfunctions_trigger_lambda.role.add_to_policy( _iam.PolicyStatement(actions=["states:StartExecution"], resources=['*'])) trigger_step_lambda_provider = _custom_resources.Provider( self, 'StreamStepFunctionsTriggerLambdaProvider', on_event_handler=stepfunctions_trigger_lambda) core.CustomResource( self, 'StreamStepFunctionsTrigger', service_token=trigger_step_lambda_provider.service_token, properties={"stepArn": datagen_stepfunctions.state_machine_arn}) with open('common/common_cdk/lambda/stream_generator.py', 'r') as f: lambda_source = f.read() sale_stream_generator_lambda = _lambda.Function( scope=self, id='WebSaleStreamGenerator', runtime=_lambda.Runtime.PYTHON_3_7, memory_size=2048, timeout=core.Duration.minutes(15), code=_lambda.Code.inline(lambda_source), handler='index.lambda_handler', environment={ 'REGION': core.Aws.REGION, 'STREAM_NAME': web_sale_stream }) stream_source_bucket.bucket.add_event_notification( _s3.EventType.OBJECT_CREATED, _s3_notifications.LambdaDestination(sale_stream_generator_lambda), _s3.NotificationKeyFilter(prefix='sale', suffix='csv')) sale_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement( actions=[ "s3:DeleteObject", "s3:GetObject", "s3:ListBucket", ], resources=[ stream_source_bucket.bucket.bucket_arn + '/*', stream_source_bucket.bucket.bucket_arn ])) sale_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement(actions=["kinesis:PutRecords"], resources=[ stack.format_arn( service='kinesis', resource='stream', resource_name=web_sale_stream) ])) sale_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement(actions=['kms:GenerateDataKey'], resources=[ stack.format_arn( service='kms', resource='key', sep='/', resource_name=kinesis_key.key_id) ])) customer_stream_generator_lambda = _lambda.Function( scope=self, id='WebCustomerStreamGenerator', runtime=_lambda.Runtime.PYTHON_3_7, memory_size=2048, timeout=core.Duration.minutes(15), code=_lambda.Code.inline(lambda_source), handler='index.lambda_handler', environment={ 'REGION': core.Aws.REGION, 'STREAM_NAME': web_customer_stream }) stream_source_bucket.bucket.add_event_notification( _s3.EventType.OBJECT_CREATED, _s3_notifications.LambdaDestination( customer_stream_generator_lambda), _s3.NotificationKeyFilter(prefix='customer', suffix='csv')) customer_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement( actions=[ "s3:DeleteObject", "s3:GetObject", "s3:ListBucket", ], resources=[ stream_source_bucket.bucket.bucket_arn + '/*', stream_source_bucket.bucket.bucket_arn ])) customer_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement(actions=["kinesis:PutRecords"], resources=[ stack.format_arn( service='kinesis', resource='stream', resource_name=web_customer_stream) ])) customer_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement(actions=['kms:GenerateDataKey'], resources=[ stack.format_arn( service='kms', resource='key', sep='/', resource_name=kinesis_key.key_id) ])) address_stream_generator_lambda = _lambda.Function( scope=self, id='WebCustomerAddressStreamGenerator', runtime=_lambda.Runtime.PYTHON_3_7, memory_size=2048, timeout=core.Duration.minutes(15), code=_lambda.Code.inline(lambda_source), handler='index.lambda_handler', environment={ 'REGION': core.Aws.REGION, 'STREAM_NAME': web_customer_address_stream }) stream_source_bucket.bucket.add_event_notification( _s3.EventType.OBJECT_CREATED, _s3_notifications.LambdaDestination( address_stream_generator_lambda), _s3.NotificationKeyFilter(prefix='address', suffix='csv')) address_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement( actions=[ "s3:DeleteObject", "s3:GetObject", "s3:ListBucket", ], resources=[ stream_source_bucket.bucket.bucket_arn + '/*', stream_source_bucket.bucket.bucket_arn ])) address_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement( actions=["kinesis:PutRecords"], resources=[ stack.format_arn(service='kinesis', resource='stream', resource_name=web_customer_address_stream) ])) address_stream_generator_lambda.add_to_role_policy( _iam.PolicyStatement(actions=['kms:GenerateDataKey'], resources=[ stack.format_arn( service='kms', resource='key', sep='/', resource_name=kinesis_key.key_id) ]))