def s3_solutions_access(self): return iam.PolicyDocument(statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ "s3:GetObject", "s3:ListBucket", "s3:ListObjects", ], resources=[ Fn.sub( "arn:${AWS::Partition}:s3:::${bucket}-${AWS::Region}/*", variables={ "bucket": Fn.find_in_map("SourceCode", "General", "S3Bucket") }, ), Fn.sub( "arn:${AWS::Partition}:s3:::${bucket}-${AWS::Region}", variables={ "bucket": Fn.find_in_map("SourceCode", "General", "S3Bucket") }, ), ], ) ])
def __init__(self, scope: cdk.Construct, id: str, **kwargs): super().__init__(scope, id, **kwargs) bucketName = CfnParameter(self, "BucketName") self.template_options.metadata = { 'AWS::CloudFormation::Interface': { 'ParameterGroups': [{ 'Label': { 'default': 'Bucket Configuration' }, 'Parameters': [bucketName.logical_id] }], 'ParameterLabels': { bucketName.logical_id: { 'default': 'Which name should the bucket have' } } } } bucket = Bucket(self, 'test-bucket', bucket_name=bucketName.value_as_string) CfnOutput(self, 'S3Id', value=bucket.bucket_arn, export_name=Fn.sub('${AWS::StackName}-S3Id'))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) env = "dev" project = "testproject1" service = "api" component = "buckets" Tag.add(self, "Service", service) Tag.add(self, "Component", component) bucket_a = aws_s3.Bucket( self, "BucketA", bucket_name=generate_resource_name(project, env, service, component, "bucketa"), removal_policy=RemovalPolicy.DESTROY, ) bucket_b = aws_s3.Bucket( self, "BucketB", bucket_name=generate_resource_name(project, env, service, component, "bucketb"), removal_policy=RemovalPolicy.DESTROY, ) value = Fn.sub( "test: ${value_to_import}", { "value_to_import": Fn.import_value( generate_resource_name(project, env, "etl", component, "bucketb")) }, ) aws_ssm.StringParameter( self, "SSMParam", parameter_name=generate_resource_name(project, env, service, component, "ssmparam"), string_value=value, ) core.CfnOutput( self, id="OutputBucketA", value=bucket_a.bucket_name, export_name=generate_resource_name(project, env, service, component, "bucketa"), ) core.CfnOutput( self, id="OutputBucketB", value=bucket_b.bucket_name, export_name=generate_resource_name(project, env, service, component, "bucketb"), )
def get_notebook_prefix(self): if self._is_solution_build(): prefix = Fn.sub( "${prefix}/notebooks", variables={ "prefix": Fn.find_in_map("SourceCode", "General", "KeyPrefix") }, ) else: prefix = "notebooks" return Fn.base64(prefix)
def get_notebook_source(self, data_bucket: IBucket): if self._is_solution_build(): notebook_source_bucket = Fn.sub( "${bucket}-${region}", variables={ "bucket": Fn.find_in_map("SourceCode", "General", "S3Bucket"), "region": Aws.REGION, }, ) else: notebook_source_bucket = data_bucket.bucket_name return Fn.base64(notebook_source_bucket)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) env = "dev" project = "testproject1" service = "etl" component = "workflow" Tag.add(self, "Service", service) Tag.add(self, "Component", component) param_dwh = CfnParameter( self, "ParamDWH", type="String", description="The domain of the DWH to connect to. | team=data,service=dwh", default="fakedwh.host", ) value_raw = "import: ${value_to_import}, param: ${param_dwh}" value = Fn.sub( value_raw, { "value_to_import": Fn.import_value( generate_resource_name(project, env, service, "buckets", "bucketb") ), "param_dwh": Fn.ref(param_dwh.logical_id), }, ) aws_ssm.StringParameter( self, "SSMParam", parameter_name=generate_resource_name( project, env, service, component, "ssmparam" ), string_value=value, )
def _add_compute_resource_launch_template( self, queue, compute_resource, instance_type, queue_pre_install_action, queue_post_install_action, queue_lt_security_groups, queue_placement_group, ): # LT network interfaces compute_lt_nw_interfaces = [ ec2.CfnLaunchTemplate.NetworkInterfaceProperty( device_index=0, associate_public_ip_address=queue.networking.assign_public_ip if compute_resource.max_network_interface_count == 1 else None, # parameter not supported for instance types with multiple network interfaces interface_type="efa" if compute_resource.efa and compute_resource.efa.enabled else None, groups=queue_lt_security_groups, subnet_id=queue.networking.subnet_ids[0], ) ] for device_index in range( 1, compute_resource.max_network_interface_count): compute_lt_nw_interfaces.append( ec2.CfnLaunchTemplate.NetworkInterfaceProperty( device_index=device_index, network_card_index=device_index, interface_type="efa" if compute_resource.efa and compute_resource.efa.enabled else None, groups=queue_lt_security_groups, subnet_id=queue.networking.subnet_ids[0], )) instance_market_options = None if queue.capacity_type == CapacityType.SPOT: instance_market_options = ec2.CfnLaunchTemplate.InstanceMarketOptionsProperty( market_type="spot", spot_options=ec2.CfnLaunchTemplate.SpotOptionsProperty( spot_instance_type="one-time", instance_interruption_behavior="terminate", max_price=None if compute_resource.spot_price is None else str(compute_resource.spot_price), ), ) ec2.CfnLaunchTemplate( self.stack_scope, f"ComputeServerLaunchTemplate{create_hash_suffix(queue.name + instance_type)}", launch_template_name= f"{self.stack_name}-{queue.name}-{instance_type}", launch_template_data=ec2.CfnLaunchTemplate. LaunchTemplateDataProperty( instance_type=instance_type, cpu_options=ec2.CfnLaunchTemplate.CpuOptionsProperty( core_count=compute_resource.vcpus, threads_per_core=1) if compute_resource.pass_cpu_options_in_launch_template else None, block_device_mappings=get_block_device_mappings( queue.compute_settings.local_storage, self.config.image.os), # key_name=, network_interfaces=compute_lt_nw_interfaces, placement=ec2.CfnLaunchTemplate.PlacementProperty( group_name=queue_placement_group), image_id=self.config.image_dict[queue.name], ebs_optimized=compute_resource.is_ebs_optimized, iam_instance_profile=ec2.CfnLaunchTemplate. IamInstanceProfileProperty( name=self.instance_profiles[queue.name]), instance_market_options=instance_market_options, user_data=Fn.base64( Fn.sub( get_user_data_content( "../resources/compute_node/user_data.sh"), { **{ "EnableEfa": "efa" if compute_resource.efa and compute_resource.efa.enabled else "NONE", "RAIDOptions": get_shared_storage_options_by_type( self.shared_storage_options, SharedStorageType.RAID), "DisableHyperThreadingManually": "true" if compute_resource.disable_simultaneous_multithreading_manually else "false", "BaseOS": self.config.image.os, "PreInstallScript": queue_pre_install_action.script if queue_pre_install_action else "NONE", "PreInstallArgs": join_shell_args(queue_pre_install_action.args) if queue_pre_install_action and queue_pre_install_action.args else "NONE", "PostInstallScript": queue_post_install_action.script if queue_post_install_action else "NONE", "PostInstallArgs": join_shell_args(queue_post_install_action.args) if queue_post_install_action and queue_post_install_action.args else "NONE", "EFSId": get_shared_storage_ids_by_type( self.shared_storage_mappings, SharedStorageType.EFS), "EFSOptions": get_shared_storage_options_by_type( self.shared_storage_options, SharedStorageType.EFS), # FIXME "FSXId": get_shared_storage_ids_by_type( self.shared_storage_mappings, SharedStorageType.FSX), "FSXMountName": self.shared_storage_attributes[SharedStorageType.FSX].get( "MountName", ""), "FSXDNSName": self.shared_storage_attributes[SharedStorageType.FSX].get( "DNSName", ""), "FSXOptions": get_shared_storage_options_by_type( self.shared_storage_options, SharedStorageType.FSX), "Scheduler": self.config.scheduling.scheduler, "EphemeralDir": queue.compute_settings.local_storage.ephemeral_volume.mount_dir if queue.compute_settings and queue.compute_settings.local_storage and queue.compute_settings.local_storage.ephemeral_volume else "/scratch", "EbsSharedDirs": get_shared_storage_options_by_type( self.shared_storage_options, SharedStorageType.EBS), "ClusterDNSDomain": str(self.cluster_hosted_zone.name) if self.cluster_hosted_zone else "", "ClusterHostedZone": str(self.cluster_hosted_zone.ref) if self.cluster_hosted_zone else "", "OSUser": OS_MAPPING[self.config.image.os]["user"], "DynamoDBTable": self.dynamodb_table.ref, "LogGroupName": self.log_group.log_group_name if self.config.monitoring.logs.cloud_watch.enabled else "NONE", "IntelHPCPlatform": "true" if self.config.is_intel_hpc_platform_enabled else "false", "CWLoggingEnabled": "true" if self.config.is_cw_logging_enabled else "false", "QueueName": queue.name, "EnableEfaGdr": "compute" if compute_resource.efa and compute_resource.efa.gdr_support else "NONE", "CustomNodePackage": self.config.custom_node_package or "", "CustomAwsBatchCliPackage": self.config.custom_aws_batch_cli_package or "", "ExtraJson": self.config.extra_chef_attributes, }, **get_common_user_data_env(queue, self.config), }, )), monitoring=ec2.CfnLaunchTemplate.MonitoringProperty( enabled=False), tag_specifications=[ ec2.CfnLaunchTemplate.TagSpecificationProperty( resource_type="instance", tags=get_default_instance_tags( self.stack_name, self.config, compute_resource, "Compute", self.shared_storage_mappings) + [ CfnTag(key=PCLUSTER_QUEUE_NAME_TAG, value=queue.name) ] + get_custom_tags(self.config), ), ec2.CfnLaunchTemplate.TagSpecificationProperty( resource_type="volume", tags=get_default_volume_tags( self.stack_name, "Compute") + [ CfnTag(key=PCLUSTER_QUEUE_NAME_TAG, value=queue.name) ] + get_custom_tags(self.config), ), ], ), )
def _add_default_instance_role(self, cleanup_policy_statements, build_tags): """Set default instance role in imagebuilder cfn template.""" managed_policy_arns = [ Fn.sub( "arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" ), Fn.sub( "arn:${AWS::Partition}:iam::aws:policy/EC2InstanceProfileForImageBuilder" ), ] if self.config.build.iam and self.config.build.iam.additional_iam_policies: for policy in self.config.build.iam.additional_iam_policy_arns: managed_policy_arns.append(policy) instancerole_policy_document = iam.PolicyDocument(statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[ self.format_arn( service="ec2", account="", resource="image", resource_name="*", ) ], actions=["ec2:CreateTags", "ec2:ModifyImageAttribute"], ) ]) if self.config.build.components: for custom_component in self.config.build.components: # Check custom component is script, and the url is S3 url if custom_component.type == "script" and utils.get_url_scheme( custom_component.value) == "s3": bucket_info = parse_bucket_url(custom_component.value) bucket_name = bucket_info.get("bucket_name") object_key = bucket_info.get("object_key") instancerole_policy_document.add_statements( iam.PolicyStatement( actions=["s3:GetObject"], effect=iam.Effect.ALLOW, resources=[ self.format_arn( region="", service="s3", account="", resource=bucket_name, resource_name=object_key, ) ], ), ) instancerole_policy = iam.CfnRole.PolicyProperty( policy_name="InstanceRoleInlinePolicy", policy_document=instancerole_policy_document, ) instance_role_resource = iam.CfnRole( self, "InstanceRole", path=IAM_ROLE_PATH, managed_policy_arns=managed_policy_arns, assume_role_policy_document=get_assume_role_policy_document( "ec2.{0}".format(self.url_suffix)), policies=[ instancerole_policy, ], tags=build_tags, role_name=self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX), ) if not self.custom_cleanup_lambda_role: self._add_resource_delete_policy( cleanup_policy_statements, ["iam:DeleteRole"], [ self.format_arn( service="iam", region="", resource="role", resource_name="{0}/{1}".format( IAM_ROLE_PATH.strip("/"), self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX), ), ) ], ) return instance_role_resource
def _add_lambda_cleanup(self, policy_statements, build_tags): lambda_cleanup_execution_role = None if self.custom_cleanup_lambda_role: execution_role = self.custom_cleanup_lambda_role else: # LambdaCleanupPolicies self._add_resource_delete_policy( policy_statements, ["cloudformation:DeleteStack"], [ self.format_arn( service="cloudformation", resource="stack", resource_name="{0}/{1}".format( self.image_id, self._stack_unique_id()), ) ], ) self._add_resource_delete_policy( policy_statements, ["ec2:CreateTags"], [ self.format_arn( service="ec2", account="", resource="image", region=region, resource_name="*", ) for region in self._get_distribution_regions() ], ) self._add_resource_delete_policy( policy_statements, ["tag:TagResources"], ["*"], ) self._add_resource_delete_policy( policy_statements, [ "iam:DetachRolePolicy", "iam:DeleteRole", "iam:DeleteRolePolicy" ], [ self.format_arn( service="iam", resource="role", region="", resource_name="{0}/{1}".format( IAM_ROLE_PATH.strip("/"), self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX + "Cleanup"), ), ) ], ) self._add_resource_delete_policy( policy_statements, ["lambda:DeleteFunction", "lambda:RemovePermission"], [ self.format_arn( service="lambda", resource="function", sep=":", resource_name=self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX), ) ], ) self._add_resource_delete_policy( policy_statements, ["logs:DeleteLogGroup"], [ self.format_arn( service="logs", resource="log-group", sep=":", resource_name="/aws/lambda/{0}:*".format( self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX)), ) ], ) self._add_resource_delete_policy( policy_statements, ["iam:RemoveRoleFromInstanceProfile"], [ self.format_arn( service="iam", resource="instance-profile", region="", resource_name="{0}/{1}".format( IAM_ROLE_PATH.strip("/"), self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX), ), ) ], ) self._add_resource_delete_policy( policy_statements, ["iam:DetachRolePolicy", "iam:DeleteRolePolicy"], [ self.format_arn( service="iam", resource="role", region="", resource_name="{0}/{1}".format( IAM_ROLE_PATH.strip("/"), self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX), ), ) ], ) self._add_resource_delete_policy( policy_statements, [ "SNS:GetTopicAttributes", "SNS:DeleteTopic", "SNS:Unsubscribe" ], [ self.format_arn( service="sns", resource="{0}".format( self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX)), ) ], ) policy_document = iam.PolicyDocument(statements=policy_statements) managed_lambda_policy = [ Fn.sub( "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" ), ] # LambdaCleanupExecutionRole lambda_cleanup_execution_role = iam.CfnRole( self, "DeleteStackFunctionExecutionRole", managed_policy_arns=managed_lambda_policy, assume_role_policy_document=get_assume_role_policy_document( "lambda.amazonaws.com"), path=IAM_ROLE_PATH, policies=[ iam.CfnRole.PolicyProperty( policy_document=policy_document, policy_name="LambdaCleanupPolicy", ), ], tags=build_tags, role_name=self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX + "Cleanup"), ) execution_role = lambda_cleanup_execution_role.attr_arn # LambdaCleanupEnv lambda_env = awslambda.CfnFunction.EnvironmentProperty( variables={"IMAGE_STACK_ARN": self.stack_id}) # LambdaCWLogGroup lambda_log = logs.CfnLogGroup( self, "DeleteStackFunctionLog", log_group_name="/aws/lambda/{0}".format( self._build_resource_name(IMAGEBUILDER_RESOURCE_NAME_PREFIX)), ) # LambdaCleanupFunction lambda_cleanup = awslambda.CfnFunction( self, "DeleteStackFunction", function_name=self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX), code=awslambda.CfnFunction.CodeProperty( s3_bucket=self.config.custom_s3_bucket or S3Bucket.get_bucket_name( AWSApi.instance().sts.get_account_id(), get_region()), s3_key=self.bucket.get_object_key(S3FileType.CUSTOM_RESOURCES, "artifacts.zip"), ), handler="delete_image_stack.handler", memory_size=128, role=execution_role, runtime="python3.8", timeout=900, environment=lambda_env, tags=build_tags, ) permission = awslambda.CfnPermission( self, "DeleteStackFunctionPermission", action="lambda:InvokeFunction", principal="sns.amazonaws.com", function_name=lambda_cleanup.attr_arn, source_arn=Fn.ref("BuildNotificationTopic"), ) lambda_cleanup.add_depends_on(lambda_log) return lambda_cleanup, permission, lambda_cleanup_execution_role, lambda_log
def _add_imagebuilder_components(self, build_tags, lambda_cleanup_policy_statements): imagebuilder_resources_dir = os.path.join( imagebuilder_utils.get_resources_directory(), "imagebuilder") # ImageBuilderComponents components = [] components_resources = [] if self.config.build and self.config.build.update_os_packages and self.config.build.update_os_packages.enabled: update_os_component_resource = imagebuilder.CfnComponent( self, "UpdateOSComponent", name=self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX + "-UpdateOS"), version=utils.get_installed_version(base_version_only=True), tags=build_tags, description="Update OS and Reboot", platform="Linux", data=Fn.sub( _load_yaml(imagebuilder_resources_dir, "update_and_reboot.yaml")), ) components.append( imagebuilder.CfnImageRecipe.ComponentConfigurationProperty( component_arn=Fn.ref("UpdateOSComponent"))) components_resources.append(update_os_component_resource) if not self.custom_cleanup_lambda_role: self._add_resource_delete_policy( lambda_cleanup_policy_statements, ["imagebuilder:DeleteComponent"], [ self.format_arn( service="imagebuilder", resource="component", resource_name="{0}/*".format( self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX + "-UpdateOS", to_lower=True)), ) ], ) disable_pcluster_component = ( self.config.dev_settings.disable_pcluster_component if self.config.dev_settings and self.config.dev_settings.disable_pcluster_component else False) if not disable_pcluster_component: parallelcluster_component_resource = imagebuilder.CfnComponent( self, "ParallelClusterComponent", name=self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX), version=utils.get_installed_version(base_version_only=True), tags=build_tags, description="Install ParallelCluster software stack", platform="Linux", data=Fn.sub( _load_yaml(imagebuilder_resources_dir, "parallelcluster.yaml")), ) components.append( imagebuilder.CfnImageRecipe.ComponentConfigurationProperty( component_arn=Fn.ref("ParallelClusterComponent"))) components_resources.append(parallelcluster_component_resource) if not self.custom_cleanup_lambda_role: self._add_resource_delete_policy( lambda_cleanup_policy_statements, ["imagebuilder:DeleteComponent"], [ self.format_arn( service="imagebuilder", resource="component", resource_name="{0}/*".format( self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX, to_lower=True)), ) ], ) tag_component_resource = imagebuilder.CfnComponent( self, "ParallelClusterTagComponent", name=self._build_resource_name(IMAGEBUILDER_RESOURCE_NAME_PREFIX + "-Tag"), version=utils.get_installed_version(base_version_only=True), tags=build_tags, description="Tag ParallelCluster AMI", platform="Linux", data=_load_yaml(imagebuilder_resources_dir, "parallelcluster_tag.yaml"), ) components.append( imagebuilder.CfnImageRecipe.ComponentConfigurationProperty( component_arn=Fn.ref("ParallelClusterTagComponent"))) components_resources.append(tag_component_resource) if not self.custom_cleanup_lambda_role: self._add_resource_delete_policy( lambda_cleanup_policy_statements, ["imagebuilder:DeleteComponent"], [ self.format_arn( service="imagebuilder", resource="component", resource_name="{0}/*".format( self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX + "-Tag", to_lower=True)), ) ], ) if self.config.build.components: self._add_custom_components(components, lambda_cleanup_policy_statements, components_resources) disable_validate_and_test_component = ( self.config.dev_settings.disable_validate_and_test if self.config.dev_settings and self.config.dev_settings.disable_validate_and_test else False) if not disable_pcluster_component and not disable_validate_and_test_component: validate_component_resource = imagebuilder.CfnComponent( self, id="ParallelClusterValidateComponent", name=self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX + "-Validate"), version=utils.get_installed_version(base_version_only=True), tags=build_tags, description="Validate ParallelCluster AMI", platform="Linux", data=_load_yaml(imagebuilder_resources_dir, "parallelcluster_validate.yaml"), ) components.append( imagebuilder.CfnImageRecipe.ComponentConfigurationProperty( component_arn=Fn.ref("ParallelClusterValidateComponent"))) components_resources.append(validate_component_resource) if not self.custom_cleanup_lambda_role: self._add_resource_delete_policy( lambda_cleanup_policy_statements, ["imagebuilder:DeleteComponent"], [ self.format_arn( service="imagebuilder", resource="component", resource_name="{0}/*".format( self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX + "-Validate", to_lower=True)), ) ], ) test_component_resource = imagebuilder.CfnComponent( self, id="ParallelClusterTestComponent", name=self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX + "-Test"), version=utils.get_installed_version(base_version_only=True), tags=build_tags, description="Test ParallelCluster AMI", platform="Linux", data=_load_yaml(imagebuilder_resources_dir, "parallelcluster_test.yaml"), ) components.append( imagebuilder.CfnImageRecipe.ComponentConfigurationProperty( component_arn=Fn.ref("ParallelClusterTestComponent"))) components_resources.append(test_component_resource) if not self.custom_cleanup_lambda_role: self._add_resource_delete_policy( lambda_cleanup_policy_statements, ["imagebuilder:DeleteComponent"], [ self.format_arn( service="imagebuilder", resource="component", resource_name="{0}/*".format( self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX + "-Test", to_lower=True)), ) ], ) return components, components_resources