def _add_custom_cookbook_policies_to_role(self, role_ref: str, name: str): bucket_info = parse_bucket_url( self._config.dev_settings.cookbook.chef_cookbook) bucket_name = bucket_info.get("bucket_name") object_key = bucket_info.get("object_key") iam.CfnPolicy( Stack.of(self), name, policy_name="CustomCookbookS3Url", policy_document=iam.PolicyDocument(statements=[ iam.PolicyStatement( actions=["s3:GetObject"], effect=iam.Effect.ALLOW, resources=[ self._format_arn(region="", service="s3", account="", resource=bucket_name, resource_name=object_key) ], ), iam.PolicyStatement( actions=["s3:GetBucketLocation"], effect=iam.Effect.ALLOW, resources=[ self._format_arn(service="s3", resource=bucket_name, region="", account="") ], ), ]), roles=[role_ref], )
def _build_policy(self) -> List[iam.PolicyStatement]: return [ iam.PolicyStatement( sid="Ec2", actions=[ "ec2:DescribeInstanceAttribute", ], effect=iam.Effect.ALLOW, resources=["*"], ), iam.PolicyStatement( sid="S3GetObj", actions=["s3:GetObject"], effect=iam.Effect.ALLOW, resources=[ self._format_arn( service="s3", resource="{0}-aws-parallelcluster/*".format( Stack.of(self).region), region="", account="", ) ], ), ]
def _add_node_role(self, node: Union[HeadNode, BaseQueue], name: str): additional_iam_policies = set(node.iam.additional_iam_policy_arns) if self._config.monitoring.logs.cloud_watch.enabled: additional_iam_policies.add( policy_name_to_arn("CloudWatchAgentServerPolicy")) if self._config.scheduling.scheduler == "awsbatch": additional_iam_policies.add( policy_name_to_arn("AWSBatchFullAccess")) return iam.CfnRole( Stack.of(self), name, path=self._cluster_scoped_iam_path(), managed_policy_arns=list(additional_iam_policies), assume_role_policy_document=get_assume_role_policy_document( "ec2.{0}".format(Stack.of(self).url_suffix)), )
def _add_pcluster_policies_to_role(self, role_ref: str, name: str): iam.CfnPolicy( Stack.of(self), name, policy_name="parallelcluster", policy_document=iam.PolicyDocument( statements=self._build_policy()), roles=[role_ref], )
def __init__( self, scope: core.Construct, id_: str, role_alias: str, role_arn: str, credential_duration_seconds: int, log_retention=None, timeout=None, ) -> None: super().__init__(scope, id_) on_create = self.get_on_create( role_alias=role_alias, role_arn=role_arn, credential_duration_seconds=credential_duration_seconds) on_update = self.get_on_update( role_alias=role_alias, role_arn=role_arn, credential_duration_seconds=credential_duration_seconds) on_delete = self.get_on_delete(role_alias=role_alias) account_id = Stack.of(self).account region = Stack.of(self).region policy = AwsCustomResourcePolicy.from_sdk_calls(resources=[ f'arn:aws:iot:{region}:{account_id}:rolealias/{role_alias}' ]) lambda_role_singleton = CustomResourcesLambdaRole(scope) lambda_role_singleton.add_to_policy(actions=["iam:PassRole"], resources=[role_arn]) # lambda_role = self.get_provisioning_lambda_role(role_arn) AwsCustomResource(scope=self, id=f'CustomResource', policy=policy, log_retention=log_retention, on_create=on_create, on_update=on_update, on_delete=on_delete, resource_type='Custom::AWS-IoT-Role-Alias', role=lambda_role_singleton.role, timeout=timeout)
def __init__(self, scope: Construct, stack_id: str, *, props: ComputeStackProps, **kwargs): super().__init__(scope, stack_id, **kwargs) region = Stack.of(self).region version = VersionQuery(self, 'Version', version=props.deadline_version) # Take a Linux image and install Deadline on it to create a new image linux_image = DeadlineMachineImage( self, "LinuxImage", props=ImageBuilderProps( # We use the linux full version string here as there is no Windows equivalent available on the # VersionQuery right now, since it is only exposing Linux installers. deadline_version=version.linux_full_version_string(), parent_ami=MachineImage.latest_amazon_linux(), image_version=props.image_recipe_version)) # Set up a worker fleet that uses the image we just created worker_fleet_linux = WorkerInstanceFleet( self, "WorkerFleetLinux", vpc=props.vpc, render_queue=props.render_queue, worker_machine_image=MachineImage.generic_linux( {region: linux_image.ami_id})) worker_fleet_linux.fleet.node.default_child.node.add_dependency( linux_image.node.default_child) # Take a Windows image and install Deadline on it to create a new image windows_image = DeadlineMachineImage( self, "WindowsImage", props=ImageBuilderProps( deadline_version=version.linux_full_version_string(), parent_ami=MachineImage.latest_windows( WindowsVersion.WINDOWS_SERVER_2019_ENGLISH_CORE_BASE), image_version=props.image_recipe_version)) # Set up a worker fleet that uses the image we just created worker_fleet_windows = WorkerInstanceFleet( self, "WorkerFleetWindows", vpc=props.vpc, render_queue=props.render_queue, worker_machine_image=MachineImage.generic_windows( {region: windows_image.ami_id})) worker_fleet_windows.fleet.node.default_child.node.add_dependency( windows_image.node.default_child)
def _add_s3_access_policies_to_role(self, node: Union[HeadNode, BaseQueue], role_ref: str, name: str): """Attach S3 policies to given role.""" read_only_s3_resources = [] read_write_s3_resources = [] for s3_access in node.iam.s3_access: for resource in s3_access.resource_regex: arn = self._format_arn(service="s3", resource=resource, region="", account="") if s3_access.enable_write_access: read_write_s3_resources.append(arn) else: read_only_s3_resources.append(arn) s3_access_policy = iam.CfnPolicy( Stack.of(self), name, policy_document=iam.PolicyDocument(statements=[]), roles=[role_ref], policy_name="S3Access", ) if read_only_s3_resources: s3_access_policy.policy_document.add_statements( iam.PolicyStatement( sid="S3Read", effect=iam.Effect.ALLOW, actions=["s3:Get*", "s3:List*"], resources=read_only_s3_resources, )) if read_write_s3_resources: s3_access_policy.policy_document.add_statements( iam.PolicyStatement(sid="S3ReadWrite", effect=iam.Effect.ALLOW, actions=["s3:*"], resources=read_write_s3_resources))
def _format_arn(self, **kwargs): return Stack.of(self).format_arn(**kwargs)
def __init__(self, scope: core.Construct, id: str, redshift_cluster_name: str, user_secret: Secret) -> None: super().__init__(scope, id) stack = Stack.of(self) subprocess.call( ['pip', 'install', '-t', 'dwh/dwh_loader_layer/python/lib/python3.8/site-packages', '-r', 'dwh/dwh_loader/requirements.txt', '--platform', 'manylinux1_x86_64', '--only-binary=:all:', '--upgrade']) requirements_layer = _lambda.LayerVersion(scope=self, id='PythonRequirementsTemplate', code=_lambda.Code.from_asset('dwh/dwh_loader_layer'), compatible_runtimes=[_lambda.Runtime.PYTHON_3_8]) dwh_loader_role = _iam.Role( self, 'Role', assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com') ) dwh_loader_role.add_managed_policy(_iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaBasicExecutionRole' )) dwh_loader_role.attach_inline_policy( _iam.Policy( self, 'InlinePolicy', statements=[ _iam.PolicyStatement( actions=[ "redshift-data:ExecuteStatement", "redshift-data:CancelStatement", "redshift-data:ListStatements", "redshift-data:GetStatementResult", "redshift-data:DescribeStatement", "redshift-data:ListDatabases", "redshift-data:ListSchemas", "redshift-data:ListTables", "redshift-data:DescribeTable" ], resources=['*'] ), _iam.PolicyStatement( actions=["secretsmanager:GetSecretValue"], resources=[user_secret.secret_arn] ), _iam.PolicyStatement( actions=["redshift:GetClusterCredentials"], resources=[ "arn:aws:redshift:*:*:dbname:*/*", "arn:aws:redshift:*:*:dbuser:*/"+_config.Redshift.ETL_USER ] ), _iam.PolicyStatement( effect=_iam.Effect('DENY'), actions=["redshift:CreateClusterUser"], resources=["arn:aws:redshift:*:*:dbuser:*/"+_config.Redshift.ETL_USER] ), _iam.PolicyStatement( conditions={ 'StringLike': { "iam:AWSServiceName": "redshift-data.amazonaws.com" } }, actions=["iam:CreateServiceLinkedRole"], resources=["arn:aws:iam::*:role/aws-service-role/redshift-data.amazonaws.com/AWSServiceRoleForRedshift"] ), ] ) ) dwh_loader_function = _lambda.Function( self, 'Lambda', runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.from_asset('dwh/dwh_loader'), handler='dwh_loader.handler', function_name='dwh-loader', environment={ 'CLUSTER_NAME': redshift_cluster_name, 'PROCEDURE': _config.Redshift.ETL_PROCEDURE, 'SECRET_ARN': user_secret.secret_arn, 'DATABASE': _config.Redshift.DATABASE, 'REGION': core.Aws.REGION, 'SCHEMA': _config.Redshift.SCHEMA }, layers=[requirements_layer], timeout=core.Duration.seconds(30), role=dwh_loader_role ) dwh_loader_submit = _sfn_tasks.LambdaInvoke( self, 'Submit', lambda_function=dwh_loader_function, payload_response_only=True ) dwh_loader_wait = _sfn.Wait( self, 'Wait', time=_sfn.WaitTime.duration(core.Duration.seconds(30)) ) dwh_loader_complete = _sfn.Choice( self, 'Complete' ) dwh_loader_failed = _sfn.Fail( self, 'Fail', cause="Redshift Data API statement failed", error="$.Result.Error" ) dwh_loader_status = _sfn_tasks.LambdaInvoke( self, 'Status', lambda_function=dwh_loader_function, result_path='$.Result', payload_response_only=True ) definition = dwh_loader_submit \ .next(dwh_loader_wait) \ .next(dwh_loader_status) \ .next(dwh_loader_complete .when(_sfn.Condition.string_equals('$.Result.Status', 'FAILED'), dwh_loader_failed) .when(_sfn.Condition.string_equals('$.Result.Status', 'FINISHED'), _sfn.Succeed(self, 'DwhLoaderSuccess')) .otherwise(dwh_loader_wait)) dwh_loader_stepfunctions = _sfn.StateMachine( self, 'StepFunctions', definition=definition, timeout=core.Duration.minutes(30) ) step_trigger = _events.Rule( self, 'StepTrigger', schedule=_events.Schedule.cron(minute='0/30', hour='*', month='*', week_day='*', year='*') ) step_trigger.add_target( _events_targets.SfnStateMachine( machine=dwh_loader_stepfunctions, ) )
def __init__( self, scope: core.Construct, id: str, policy_name: str, policy_document: Any, timeout: Duration = None ) -> None: super().__init__(scope, id) if type(policy_document) == dict: policy_document = json.dumps(policy_document) account_id = Stack.of(self).account region = Stack.of(self).region # IMPORTANT! Setting resources to the exact policy name is the most restrictive we can be, but this will cause issues # When trying to update the policy name. # See this issue for more info: https://github.com/aws/aws-cdk/issues/14037 # A possible work around is setting resources to 'arn:aws:iot:{region}:{account_id}:policy/*', which is more permissive. lambda_role = iam.Role( scope=self, id=f'{id}LambdaRole', assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), inline_policies={ "IotPolicyProvisioningPolicy": iam.PolicyDocument(statements=[ iam.PolicyStatement( actions=[ "iot:ListPolicyVersions", "iot:CreatePolicy", "iot:CreatePolicyVersion", "iot:DeletePolicy", "iot:DeletePolicyVersion", "iot:GetPolicy" ], resources=[f'arn:aws:iot:{region}:{account_id}:policy/{policy_name}'], effect=iam.Effect.ALLOW, ) ]) }, managed_policies=[iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole")], ) if not timeout: timeout = Duration.minutes(5) with open(path.join(path.dirname(__file__), 'iot_policy_event_handler.py')) as file: code = file.read() event_handler = aws_lambda.Function( scope=self, id=f'{id}EventHandler', runtime=aws_lambda.Runtime.PYTHON_3_8, code=aws_lambda.Code.from_inline(code), handler='index.on_event', role=lambda_role, timeout=timeout, ) with open(path.join(path.dirname(__file__), 'iot_policy_is_complete_handler.py')) as file: is_complete_code = file.read() is_complete_handler = aws_lambda.Function( scope=self, id=f'{id}IsCompleteHandler', runtime=aws_lambda.Runtime.PYTHON_3_8, code=aws_lambda.Code.from_inline(is_complete_code), handler='index.is_complete', role=lambda_role, ) provider = Provider( scope=self, id=f'{id}Provider', on_event_handler=event_handler, is_complete_handler=is_complete_handler, query_interval=Duration.minutes(2), ) core.CustomResource( scope=self, id=f'{id}IotPolicy', service_token=provider.service_token, removal_policy=RemovalPolicy.DESTROY, resource_type="Custom::IotPolicyAsync", properties={ "policy_name": policy_name, "policy_document": policy_document, }, )
def __init__(self, scope: core.Construct, id: str, es_domain: CfnDomain, kda_role: iam.Role, source_bucket: s3.Bucket, dest_bucket: s3.Bucket, **kwargs): super().__init__(scope, id, **kwargs) stack = Stack.of(self) kda_role.add_to_policy(PolicyStatement(actions=['cloudwatch:PutMetricData'], resources=['*'])) artifacts_bucket_arn = 'arn:aws:s3:::' + _config.ARA_BUCKET.replace("s3://", "") kda_role.add_to_policy(PolicyStatement(actions=['s3:GetObject', 's3:GetObjectVersion'], resources=[artifacts_bucket_arn, artifacts_bucket_arn + '/binaries/*'])) log_group = logs.LogGroup(scope=self, id='KdaLogGroup', retention=logs.RetentionDays.ONE_WEEK, removal_policy=RemovalPolicy.DESTROY) log_stream = logs.LogStream(scope=self, id='KdaLogStream', log_group=log_group, removal_policy=RemovalPolicy.DESTROY) log_stream_arn = stack.format_arn(service='logs', resource='log-group', resource_name=log_group.log_group_name + ':log-stream:' + log_stream.log_stream_name, sep=':') # TODO: restrict kda_role.add_to_policy(PolicyStatement(actions=['logs:*'], resources=[stack.format_arn(service='logs', resource='*')])) kda_role.add_to_policy(PolicyStatement(actions=['logs:DescribeLogStreams', 'logs:DescribeLogGroups'], resources=[log_group.log_group_arn, stack.format_arn(service='logs', resource='log-group', resource_name='*')])) kda_role.add_to_policy(PolicyStatement(actions=['logs:PutLogEvents'], resources=[log_stream_arn])) kda_role.add_to_policy(PolicyStatement(actions=['es:ESHttp*'], resources=[stack.format_arn(service='es', resource='domain', resource_name=es_domain.domain_name + '/*')])) # TODO: restrict kda_role.add_to_policy(PolicyStatement(actions=['s3:*'], resources=['arn:aws:s3::::*'])) # Define delivery stream # delivery_stream_name = 'clean_delivery_stream' # # s3_configuration = { # 'bucketArn': '', # 'compressionFormat': 'Snappy', # 'dataFormatConversionConfiguration': { # 'enabled': True, # 'inputFormatConfiguration': {'deserializer': }, # 'outputFormatConfiguration': {'serializer': {'parquetSerDe': }}, # 'schemaConfiguration': {} # }, # 'prefix': 'streaming' # } # # delivery_stream = CfnDeliveryStream(scope=self, # id='Firehose Delivery Stream', # delivery_stream_name=delivery_stream_name, # delivery_stream_type='DirectPut', # extended_s3_destination_configuration=s3_configuration # ) # Define KDA application application_configuration = { 'environmentProperties': { 'propertyGroups': [ { 'propertyGroupId': 'ConsumerConfigProperties', 'propertyMap': { 'CustomerStream': scope.customer_stream.stream_name, 'AddressStream': scope.address_stream.stream_name, 'SaleStream': scope.sale_stream.stream_name, 'PromoDataPath': source_bucket.s3_url_for_object('promo'), 'ItemDataPath': source_bucket.s3_url_for_object('item'), 'aws.region': scope.region } }, { 'propertyGroupId': 'ProducerConfigProperties', 'propertyMap': { 'ElasticsearchHost': 'https://' + es_domain.attr_domain_endpoint + ':443', 'Region': scope.region, 'DenormalizedSalesS3Path': dest_bucket.s3_url_for_object() + '/', 'IndexName': 'ara-write' } } ] }, 'applicationCodeConfiguration': { 'codeContent': { 's3ContentLocation': { 'bucketArn': artifacts_bucket_arn, 'fileKey': 'binaries/stream-processing-1.1.jar' } }, 'codeContentType': 'ZIPFILE' }, 'flinkApplicationConfiguration': { 'parallelismConfiguration': { 'configurationType': 'DEFAULT' }, 'checkpointConfiguration': { 'configurationType': 'DEFAULT' }, 'monitoringConfiguration': { 'logLevel': 'DEBUG', 'metricsLevel': 'TASK', 'configurationType': 'CUSTOM' } }, 'applicationSnapshotConfiguration': { 'snapshotsEnabled': False } } self.__app = CfnApplicationV2(scope=self, id='KDA application', runtime_environment='FLINK-1_11', application_name='KDA-application', service_execution_role=kda_role.role_arn, application_configuration=application_configuration) logging = CfnApplicationCloudWatchLoggingOptionV2(scope=self, id='KDA application logging', application_name=self.__app.ref, cloud_watch_logging_option={'logStreamArn': log_stream_arn}) logging.apply_removal_policy(policy=RemovalPolicy.RETAIN, apply_to_update_replace_policy=True, default=RemovalPolicy.RETAIN) # Use a custom resource to start the application create_params = { 'ApplicationName': self.__app.ref, 'RunConfiguration': { 'ApplicationRestoreConfiguration': { 'ApplicationRestoreType': 'SKIP_RESTORE_FROM_SNAPSHOT' }, 'FlinkRunConfiguration': { 'AllowNonRestoredState': True } } } # See https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/ for service name, actions and parameters create_action = AwsSdkCall(service='KinesisAnalyticsV2', action='startApplication', parameters=create_params, physical_resource_id=PhysicalResourceId.of(self.__app.ref + '-start')) delete_action = AwsSdkCall(service='KinesisAnalyticsV2', action='stopApplication', parameters={'ApplicationName': self.__app.ref, 'Force': True}) custom_resource = AwsCustomResource(scope=self, id='KdaStartAndStop', on_create=create_action, on_delete=delete_action, policy=AwsCustomResourcePolicy.from_statements([PolicyStatement( actions=['kinesisanalytics:StartApplication', 'kinesisanalytics:StopApplication', 'kinesisanalytics:DescribeApplication', 'kinesisanalytics:UpdateApplication'], resources=[ stack.format_arn(service='kinesisanalytics', resource='application', resource_name=self.app.application_name)])])) custom_resource.node.add_dependency(self.app)
def __init__(self, scope: core.Construct, id: str, vpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) stack = Stack.of(self) self.__key = KeyPair(self, "bastion-keypair", name=_config.Redshift.BASTION_HOST_KEY_PAIR_NAME, description="Key Pair to connect to bastion host", resource_prefix="ara-redshift-bastion") self._bastion_sg = ec2.SecurityGroup(self, id="bastion-sg", vpc=vpc, allow_all_outbound=None, description=None, security_group_name="bastion-sg") # a proper ip address needs to be configured before stack deployment if _config.Redshift.LOCAL_IP is not None: self._bastion_sg.add_ingress_rule( ec2.Peer.ipv4(_config.Redshift.LOCAL_IP), ec2.Port.tcp(22)) # Instance Role and SSM Managed Policy self._role = iam.Role( self, "InstanceSSM", assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), role_name='bastion-host-role') self._role.add_to_policy( iam.PolicyStatement( actions=[ 'secretsmanager:GetResourcePolicy', 'secretsmanager:GetSecretValue', 'secretsmanager:DescribeSecret', 'secretsmanager:ListSecretVersionIds' ], resources=[ stack.format_arn( service='secretsmanager', resource='secret:ec2-private-key/' + _config.Redshift.BASTION_HOST_KEY_PAIR_NAME + '*') ])) self._role.add_managed_policy( _iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonSSMManagedInstanceCore')) bastion_asg = _autoscaling.AutoScalingGroup( self, 'bastion-autoscaling', instance_type=ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE3, ec2.InstanceSize.NANO), machine_image=ec2.AmazonLinuxImage(), vpc=vpc, key_name=_config.Redshift.BASTION_HOST_KEY_PAIR_NAME, role=self._role, security_group=self._bastion_sg, vpc_subnets=ec2.SubnetSelection( availability_zones=None, one_per_az=None, subnet_group_name=None, subnet_name=None, subnets=None, subnet_type=ec2.SubnetType.PRIVATE), cooldown=core.Duration.minutes(1), min_capacity=1, max_capacity=3, spot_price="0.005") self.__bastion_nlb = _elbv2.NetworkLoadBalancer( self, 'bastion_elb', vpc=vpc, internet_facing=True, vpc_subnets=ec2.SubnetSelection(availability_zones=None, one_per_az=None, subnet_group_name=None, subnet_name=None, subnets=None, subnet_type=ec2.SubnetType.PUBLIC)) listener = self.__bastion_nlb.add_listener("Listener", port=22) listener.add_targets("Target", port=22, targets=[bastion_asg])
def _stack_id(self): return Stack.of(self).stack_id
def _add_instance_profile(self, role_ref: str, name: str): return iam.CfnInstanceProfile(Stack.of(self), name, roles=[role_ref], path=self._cluster_scoped_iam_path()).ref
def _stack_unique_id(self): return Fn.select(2, Fn.split("/", Stack.of(self).stack_id))
def _stack_region(self): return Stack.of(self).region
def _build_policy(self) -> List[iam.PolicyStatement]: policy = [ iam.PolicyStatement( sid="Ec2", actions=[ "ec2:DescribeInstanceAttribute", "ec2:DescribeInstances", "ec2:DescribeInstanceStatus", "ec2:CreateTags", "ec2:DescribeVolumes", "ec2:AttachVolume", ], effect=iam.Effect.ALLOW, resources=["*"], ), iam.PolicyStatement( sid="S3GetObj", actions=["s3:GetObject"], effect=iam.Effect.ALLOW, resources=[ self._format_arn( service="s3", resource="{0}-aws-parallelcluster/*".format( Stack.of(self).region), region="", account="", ) ], ), iam.PolicyStatement( sid="ResourcesS3Bucket", effect=iam.Effect.ALLOW, actions=["s3:*"], resources=[ self._format_arn(service="s3", resource=self._cluster_bucket.name, region="", account=""), self._format_arn( service="s3", resource= f"{self._cluster_bucket.name}/{self._cluster_bucket.artifact_directory}/*", region="", account="", ), ], ), iam.PolicyStatement( sid="CloudFormation", actions=[ "cloudformation:DescribeStacks", "cloudformation:DescribeStackResource", "cloudformation:SignalResource", ], effect=iam.Effect.ALLOW, resources=[ self._format_arn( service="cloudformation", resource=f"stack/{Stack.of(self).stack_name}/*"), self._format_arn( service="cloudformation", resource=f"stack/{Stack.of(self).stack_name}-*/*"), ], ), iam.PolicyStatement( sid="DcvLicense", actions=[ "s3:GetObject", ], effect=iam.Effect.ALLOW, resources=[ self._format_arn( service="s3", resource="dcv-license.{0}/*".format( Stack.of(self).region), region="", account="", ) ], ), ] if self._config.scheduling.scheduler != "awsbatch": policy.extend([ iam.PolicyStatement( sid="EC2Terminate", actions=["ec2:TerminateInstances"], effect=iam.Effect.ALLOW, resources=["*"], conditions={ "StringEquals": { f"ec2:ResourceTag/{PCLUSTER_CLUSTER_NAME_TAG}": Stack.of(self).stack_name } }, ), iam.PolicyStatement( sid="EC2RunInstances", actions=["ec2:RunInstances"], effect=iam.Effect.ALLOW, resources=[ self._format_arn(service="ec2", resource=f"subnet/{subnet_id}") for subnet_id in self._config.compute_subnet_ids ] + [ self._format_arn(service="ec2", resource="network-interface/*"), self._format_arn(service="ec2", resource="instance/*"), self._format_arn(service="ec2", resource="volume/*"), self._format_arn( service="ec2", resource= f"key-pair/{self._config.head_node.ssh.key_name}"), self._format_arn(service="ec2", resource="security-group/*"), self._format_arn(service="ec2", resource="launch-template/*"), self._format_arn(service="ec2", resource="placement-group/*"), ] + [ self._format_arn(service="ec2", resource=f"image/{queue_ami}", account="") for _, queue_ami in self._config.image_dict.items() ], ), iam.PolicyStatement( sid="PassRole", actions=["iam:PassRole"], effect=iam.Effect.ALLOW, resources=self._generate_head_node_pass_role_resources(), ), ]) if self._config.scheduling.scheduler == "plugin": cluster_shared_artifacts = get_attr( self._config, "scheduling.settings.scheduler_definition.plugin_resources.cluster_shared_artifacts" ) if cluster_shared_artifacts: for artifacts in cluster_shared_artifacts: if get_url_scheme(artifacts.source) == "s3": bucket_info = parse_bucket_url(artifacts.source) bucket_name = bucket_info.get("bucket_name") object_key = bucket_info.get("object_key") policy.extend([ iam.PolicyStatement( actions=["s3:GetObject"], effect=iam.Effect.ALLOW, resources=[ self._format_arn( region="", service="s3", account="", resource=bucket_name, resource_name=object_key, ) ], ), ]) if self._config.directory_service: policy.append( iam.PolicyStatement( actions=["secretsmanager:GetSecretValue"], effect=iam.Effect.ALLOW, resources=[ self._config.directory_service.password_secret_arn ], )) return policy
def _stack_account(self): return Stack.of(self).account
def _url_suffix(self): return Stack.of(self).url_suffix
def __init__(self, scope: core.Construct, id: str, application_prefix: str, suffix: str, kda_role: Role, **kwargs): super().__init__(scope, id, **kwargs) stack = Stack.of(self) region = stack.region # Create Cognito User Pool self.__user_pool = CfnUserPool( scope=self, id='UserPool', admin_create_user_config={'allowAdminCreateUserOnly': True}, policies={'passwordPolicy': { 'minimumLength': 8 }}, username_attributes=['email'], auto_verified_attributes=['email'], user_pool_name=application_prefix + '_user_pool') # Create a Cognito User Pool Domain using the newly created Cognito User Pool CfnUserPoolDomain(scope=self, id='CognitoDomain', domain=application_prefix + '-' + suffix, user_pool_id=self.user_pool.ref) # Create Cognito Identity Pool self.__id_pool = CfnIdentityPool( scope=self, id='IdentityPool', allow_unauthenticated_identities=False, cognito_identity_providers=[], identity_pool_name=application_prefix + '_identity_pool') trust_relationship = FederatedPrincipal( federated='cognito-identity.amazonaws.com', conditions={ 'StringEquals': { 'cognito-identity.amazonaws.com:aud': self.id_pool.ref }, 'ForAnyValue:StringLike': { 'cognito-identity.amazonaws.com:amr': 'authenticated' } }, assume_role_action='sts:AssumeRoleWithWebIdentity') # IAM role for master user master_auth_role = Role(scope=self, id='MasterAuthRole', assumed_by=trust_relationship) # Role for authenticated user limited_auth_role = Role(scope=self, id='LimitedAuthRole', assumed_by=trust_relationship) # Attach Role to Identity Pool CfnIdentityPoolRoleAttachment( scope=self, id='userPoolRoleAttachment', identity_pool_id=self.id_pool.ref, roles={'authenticated': limited_auth_role.role_arn}) # Create master-user-group CfnUserPoolGroup(scope=self, id='AdminsGroup', user_pool_id=self.user_pool.ref, group_name='master-user-group', role_arn=master_auth_role.role_arn) # Create limited-user-group CfnUserPoolGroup(scope=self, id='UsersGroup', user_pool_id=self.user_pool.ref, group_name='limited-user-group', role_arn=limited_auth_role.role_arn) # Role for the Elasticsearch service to access Cognito es_role = Role(scope=self, id='EsRole', assumed_by=ServicePrincipal(service='es.amazonaws.com'), managed_policies=[ ManagedPolicy.from_aws_managed_policy_name( 'AmazonESCognitoAccess') ]) # Use the following command line to generate the python dependencies layer content # pip3 install -t lambda-layer/python/lib/python3.8/site-packages -r lambda/requirements.txt # Build the lambda layer assets subprocess.call([ 'pip', 'install', '-t', 'streaming/streaming_cdk/lambda-layer/python/lib/python3.8/site-packages', '-r', 'streaming/streaming_cdk/bootstrap-lambda/requirements.txt', '--upgrade' ]) requirements_layer = _lambda.LayerVersion( scope=self, id='PythonRequirementsTemplate', code=_lambda.Code.from_asset( 'streaming/streaming_cdk/lambda-layer'), compatible_runtimes=[_lambda.Runtime.PYTHON_3_8]) # This lambda function will bootstrap the Elasticsearch cluster bootstrap_function_name = 'AESBootstrap' register_template_lambda = _lambda.Function( scope=self, id='RegisterTemplate', runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.from_asset( 'streaming/streaming_cdk/bootstrap-lambda'), handler='es-bootstrap.lambda_handler', environment={ 'REGION': region, 'KDA_ROLE_ARN': kda_role.role_arn, 'MASTER_ROLE_ARN': master_auth_role.role_arn }, layers=[requirements_layer], timeout=Duration.minutes(15), function_name=bootstrap_function_name) lambda_role = register_template_lambda.role lambda_role.add_to_policy( PolicyStatement( actions=['logs:CreateLogGroup'], resources=[stack.format_arn(service='logs', resource='*')])) lambda_role.add_to_policy( PolicyStatement( actions=['logs:CreateLogStream', 'logs:PutLogEvents'], resources=[ stack.format_arn(service='logs', resource='log_group', resource_name='/aws/lambda/' + bootstrap_function_name + ':*') ])) # Let the lambda assume the master role so that actions can be executed on the cluster # https://aws.amazon.com/premiumsupport/knowledge-center/lambda-function-assume-iam-role/ lambda_role.add_to_policy( PolicyStatement(actions=['sts:AssumeRole'], resources=[master_auth_role.role_arn])) master_auth_role.assume_role_policy.add_statements( PolicyStatement(actions=['sts:AssumeRole'], principals=[lambda_role])) # List all the roles that are allowed to access the Elasticsearch cluster. roles = [ ArnPrincipal(limited_auth_role.role_arn), ArnPrincipal(master_auth_role.role_arn), ArnPrincipal(kda_role.role_arn) ] # The users if register_template_lambda and register_template_lambda.role: roles.append(ArnPrincipal( lambda_role.role_arn)) # The lambda used to bootstrap # Create kms key kms_key = Key(scope=self, id='kms-es', alias='custom/es', description='KMS key for Elasticsearch domain', enable_key_rotation=True) # AES Log Groups es_app_log_group = logs.LogGroup(scope=self, id='EsAppLogGroup', retention=logs.RetentionDays.ONE_WEEK, removal_policy=RemovalPolicy.RETAIN) # Create the Elasticsearch domain es_domain_arn = stack.format_arn(service='es', resource='domain', resource_name=application_prefix + '/*') es_access_policy = PolicyDocument(statements=[ PolicyStatement(principals=roles, actions=[ 'es:ESHttpGet', 'es:ESHttpPut', 'es:ESHttpPost', 'es:ESHttpDelete' ], resources=[es_domain_arn]) ]) self.__es_domain = es.CfnDomain( scope=self, id='searchDomain', elasticsearch_cluster_config={ 'instanceType': 'r5.large.elasticsearch', 'instanceCount': 2, 'dedicatedMasterEnabled': True, 'dedicatedMasterCount': 3, 'dedicatedMasterType': 'r5.large.elasticsearch', 'zoneAwarenessEnabled': True, 'zoneAwarenessConfig': { 'AvailabilityZoneCount': '2' }, }, encryption_at_rest_options={ 'enabled': True, 'kmsKeyId': kms_key.key_id }, node_to_node_encryption_options={'enabled': True}, ebs_options={ 'volumeSize': 10, 'ebsEnabled': True }, elasticsearch_version='7.9', domain_name=application_prefix, access_policies=es_access_policy, cognito_options={ 'enabled': True, 'identityPoolId': self.id_pool.ref, 'roleArn': es_role.role_arn, 'userPoolId': self.user_pool.ref }, advanced_security_options={ 'enabled': True, 'internalUserDatabaseEnabled': False, 'masterUserOptions': { 'masterUserArn': master_auth_role.role_arn } }, domain_endpoint_options={ 'enforceHttps': True, 'tlsSecurityPolicy': 'Policy-Min-TLS-1-2-2019-07' }, # log_publishing_options={ # # 'ES_APPLICATION_LOGS': { # # 'enabled': True, # # 'cloud_watch_logs_log_group_arn': es_app_log_group.log_group_arn # # }, # # 'AUDIT_LOGS': { # # 'enabled': True, # # 'cloud_watch_logs_log_group_arn': '' # # }, # # 'SEARCH_SLOW_LOGS': { # # 'enabled': True, # # 'cloud_watch_logs_log_group_arn': '' # # }, # # 'INDEX_SLOW_LOGS': { # # 'enabled': True, # # 'cloud_watch_logs_log_group_arn': '' # # } # } ) # Not yet on the roadmap... # See https://github.com/aws-cloudformation/aws-cloudformation-coverage-roadmap/issues/283 # self.es_domain.add_property_override('ElasticsearchClusterConfig.WarmEnabled', True) # self.es_domain.add_property_override('ElasticsearchClusterConfig.WarmCount', 2) # self.es_domain.add_property_override('ElasticsearchClusterConfig.WarmType', 'ultrawarm1.large.elasticsearch') # Deny all roles from the authentication provider - users must be added to groups # This lambda function will bootstrap the Elasticsearch cluster cognito_function_name = 'CognitoFix' cognito_template_lambda = _lambda.Function( scope=self, id='CognitoFixLambda', runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.from_asset( 'streaming/streaming_cdk/cognito-lambda'), handler='handler.handler', environment={ 'REGION': scope.region, 'USER_POOL_ID': self.__user_pool.ref, 'IDENTITY_POOL_ID': self.__id_pool.ref, 'LIMITED_ROLE_ARN': limited_auth_role.role_arn }, timeout=Duration.minutes(15), function_name=cognito_function_name) lambda_role = cognito_template_lambda.role lambda_role.add_to_policy( PolicyStatement( actions=['logs:CreateLogGroup'], resources=[stack.format_arn(service='logs', resource='*')])) lambda_role.add_to_policy( PolicyStatement( actions=['logs:CreateLogStream', 'logs:PutLogEvents'], resources=[ stack.format_arn(service='logs', resource='log_group', resource_name='/aws/lambda/' + cognito_function_name + ':*') ])) lambda_role.add_to_policy( PolicyStatement(actions=['cognito-idp:ListUserPoolClients'], resources=[self.user_pool.attr_arn])) lambda_role.add_to_policy( PolicyStatement(actions=['iam:PassRole'], resources=[limited_auth_role.role_arn])) cognito_id_res = Fn.join(':', [ 'arn:aws:cognito-identity', scope.region, scope.account, Fn.join('/', ['identitypool', self.__id_pool.ref]) ]) lambda_role.add_to_policy( PolicyStatement(actions=['cognito-identity:SetIdentityPoolRoles'], resources=[cognito_id_res])) # Get the Domain Endpoint and register it with the lambda as environment variable. register_template_lambda.add_environment( 'DOMAIN', self.__es_domain.attr_domain_endpoint) CfnOutput(scope=self, id='createUserUrl', description="Create a new user in the user pool here.", value="https://" + scope.region + ".console.aws.amazon.com/cognito/users?region=" + scope.region + "#/pool/" + self.user_pool.ref + "/users") CfnOutput(scope=self, id='kibanaUrl', description="Access Kibana via this URL.", value="https://" + self.__es_domain.attr_domain_endpoint + "/_plugin/kibana/") bootstrap_lambda_provider = Provider( scope=self, id='BootstrapLambdaProvider', on_event_handler=register_template_lambda) CustomResource(scope=self, id='ExecuteRegisterTemplate', service_token=bootstrap_lambda_provider.service_token, properties={'Timeout': 900}) cognito_lambda_provider = Provider( scope=self, id='CognitoFixLambdaProvider', on_event_handler=cognito_template_lambda) cognito_fix_resource = CustomResource( scope=self, id='ExecuteCognitoFix', service_token=cognito_lambda_provider.service_token) cognito_fix_resource.node.add_dependency(self.__es_domain)
def __init__(self, scope: core.Construct, id: str, vpc: _ec2.Vpc, redshift_secret_arn: str, lambda_sg: _ec2.SecurityGroup, clean_glue_db: _glue.Database, redshift_role_arn: str, redshift_cluster_endpoint: _redshift.Endpoint, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.__vpc = vpc self.__redshift_secret_arn = redshift_secret_arn self.__lambda_sg = lambda_sg self.__clean_glue_db = clean_glue_db self.__redshift_role_arn = redshift_role_arn stack = Stack.of(self) # Generate secrets for Redshift users generator = SecretStringGenerator(exclude_characters="'", exclude_punctuation=True) self.__etl_user_secret = Secret( scope=self, id='ETLUserSecret', description="ETL user Redshift", generate_secret_string=SecretStringGenerator( exclude_characters="'", exclude_punctuation=True, generate_string_key="password", secret_string_template=json.dumps( { 'username': _config.Redshift.ETL_USER, 'dbname': _config.Redshift.DATABASE, 'host': redshift_cluster_endpoint.hostname, 'port': core.Token.as_string(redshift_cluster_endpoint.port) } ) ) ) self.__dataengineer_user_secret = Secret( scope=self, id='DataEngineerUserSecret', description="DataEngineer user Redshift", generate_secret_string=SecretStringGenerator( exclude_characters="'", exclude_punctuation=True, generate_string_key="password", secret_string_template=json.dumps( { 'username': _config.Redshift.DATA_ENGINEER_USER, 'dbname': _config.Redshift.DATABASE, 'host': redshift_cluster_endpoint.hostname, 'port': core.Token.as_string(redshift_cluster_endpoint.port) } ) )) self.__quicksight_user_secret = Secret( scope=self, id='DatavizUserSecret', description="Quicksight user Redshift", generate_secret_string=SecretStringGenerator( exclude_characters="'", exclude_punctuation=True, generate_string_key="password", secret_string_template=json.dumps( { 'username': _config.Redshift.DATAVIZ_USER, 'dbname': _config.Redshift.DATABASE, 'host': redshift_cluster_endpoint.hostname, 'port': core.Token.as_string(redshift_cluster_endpoint.port) } ) )) self.__subnets_selection = _ec2.SubnetSelection(availability_zones=None, one_per_az=None, subnet_group_name=None, subnet_name=None, subnets=None, subnet_type=_ec2.SubnetType.PRIVATE) # Use the following command line to generate the python dependencies layer content # pip3 install -t lambda-layer/python/lib/python3.8/site-packages -r lambda/requirements.txt # Build the lambda layer assets subprocess.call( ['pip', 'install', '-t', 'dwh/dwh_cdk/bootstrap_lambda_layer/python/lib/python3.8/site-packages', '-r', 'dwh/dwh_cdk/bootstrap_lambda/requirements.txt', '--platform', 'manylinux1_x86_64', '--only-binary=:all:', '--upgrade']) requirements_layer = _lambda.LayerVersion(scope=self, id='PythonRequirementsTemplate', code=_lambda.Code.from_asset('dwh/dwh_cdk/bootstrap_lambda_layer'), compatible_runtimes=[_lambda.Runtime.PYTHON_3_8]) # This lambda function will run SQL commands to setup Redshift users and tables bootstrap_function_name = 'RedshiftBootstrap' register_template_lambda = _lambda.Function(scope=self, id='RegisterTemplate', runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.from_asset( 'dwh/dwh_cdk/bootstrap_lambda'), handler='redshift_setup.handler', environment={ 'SQL_SCRIPT_LOCATION': _config.BINARIES_LOCATION + self.SQL_SCRIPT_DIR, 'SECRET_ARN': self.__redshift_secret_arn, 'SQL_SCRIPT_FILES': _config.RedshiftDeploy.SQL_SCRIPT_FILES, 'ETL_SECRET': self.__etl_user_secret.secret_arn, 'DATAENG_SECRET': self.__dataengineer_user_secret.secret_arn, 'DATAVIZ_SECRET': self.__quicksight_user_secret.secret_arn, 'GLUE_DATABASE': self.__clean_glue_db.database_name, 'REDSHIFT_IAM_ROLE': self.__redshift_role_arn }, layers=[requirements_layer], timeout=core.Duration.minutes(3), vpc=self.__vpc, vpc_subnets=self.__subnets_selection, security_group=self.__lambda_sg, function_name=bootstrap_function_name, memory_size=256 ) lambda_role = register_template_lambda.role lambda_role.add_to_policy(PolicyStatement( actions=['secretsmanager:GetResourcePolicy', 'secretsmanager:GetSecretValue', 'secretsmanager:DescribeSecret', 'secretsmanager:ListSecretVersionIds'], resources=[stack.format_arn(service='secretsmanager', resource='*')])) lambda_role.add_to_policy(PolicyStatement(actions=['logs:CreateLogGroup'], resources=[stack.format_arn(service='logs', resource='*')])) lambda_role.add_to_policy(PolicyStatement(actions=['logs:CreateLogStream', 'logs:PutLogEvents'], resources=[stack.format_arn(service='logs', resource='log_group', resource_name='/aws/lambda/' + bootstrap_function_name + ':*')])) artifacts_bucket_arn = 'arn:aws:s3:::' + _config.ARA_BUCKET.replace("s3://", "") lambda_role.add_to_policy(PolicyStatement(actions=['s3:GetObject', 's3:GetObjectVersion'], resources=[artifacts_bucket_arn, artifacts_bucket_arn + '/binaries/*'])) bootstrap_lambda_provider = Provider(scope=self, id='BootstrapLambdaProvider', on_event_handler=register_template_lambda) CustomResource(scope=self, id='ExecuteRegisterTemplate', service_token=bootstrap_lambda_provider.service_token) self.__secrets_manager_vpc_endpoint_sg = _ec2.SecurityGroup(self, id="secrets_manager_vpc_endpoint-sg", vpc=self.__vpc, allow_all_outbound=None, description=None, security_group_name="secrets-manager-vpc_endpoint-sg") self.__secrets_manager_vpc_endpoint_sg.add_ingress_rule(self.__lambda_sg, _ec2.Port.all_tcp() ) self.__security_groups_list = [self.__secrets_manager_vpc_endpoint_sg] self.__endpoint_service_name = 'com.amazonaws.%s.secretsmanager' % stack.region # Create VPC endpoint for SecretsManager secrets_manager_vpc_endpoint = _ec2.InterfaceVpcEndpoint(stack, "Secretsmanager VPC Endpoint", vpc=self.__vpc, service=_ec2.InterfaceVpcEndpointService( self.__endpoint_service_name, 443), subnets=self.__subnets_selection, security_groups=self.__security_groups_list )