def build_file_system_access_point(
scope: core.Construct, team_name: str, shared_fs: efs.FileSystem, path: str, ap_name: str
) -> efs.AccessPoint:
return efs.AccessPoint(
scope=scope,
id=ap_name,
file_system=cast(efs.IFileSystem, shared_fs),
path=f"/{team_name}/{path}",
posix_user=efs.PosixUser(gid="100", uid="1000"),
create_acl=efs.Acl(owner_gid="100", owner_uid="1000", permissions="770"),
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# create vpc
vpc = aws_ec2.Vpc(self, "vpc", max_azs=3, nat_gateways=1)
# create efs share
efs_share = aws_efs.FileSystem(self, "efs-backend", vpc=vpc)
# create efs acl
efs_acl = aws_efs.Acl(owner_gid="1000",
owner_uid="1000",
permissions="0777")
# create efs posix user
efs_user = aws_efs.PosixUser(gid="1000", uid="1000")
# create efs access point
efs_ap = aws_efs.AccessPoint(self,
"efs-accesspoint",
path="/efs",
file_system=efs_share,
posix_user=efs_user,
create_acl=efs_acl)
# create lambda with efs access
efs_lambda = aws_lambda.Function(
self,
"read_efs",
runtime=aws_lambda.Runtime.PYTHON_3_8,
code=aws_lambda.Code.from_asset("./function"),
handler="efsread.handler",
timeout=core.Duration.seconds(20),
memory_size=128,
retry_attempts=0,
filesystem=aws_lambda.FileSystem.from_efs_access_point(
efs_ap, '/mnt/efs'),
tracing=aws_lambda.Tracing.ACTIVE,
vpc=vpc,
environment={"var": "x"})
# create custom iam policy with efs permissions
efs_policy = aws_iam.PolicyStatement(resources=["*"],
actions=["elasticfilesystem:*"])
# add efs iam policy to lambda
efs_lambda.add_to_role_policy(efs_policy)
def __init__(self, scope: core.Construct, config: dict, id: str,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
if 'vpc_id' in config:
vpc = ec2.Vpc.from_lookup(self, "ECS-VPC", vpc_id=config["vpc_id"])
else:
vpc = None
cluster = ecs.Cluster(self,
cluster_name="commvault-cs",
id="commvault",
container_insights=True,
vpc=vpc)
### Create demo bucket
bucket = s3.Bucket(self,
"commvault-bucket",
bucket_name="commvault-demo-bucket-{}-{}".format(
config["region"], config["account"]))
### This will allow the ALB to generate a certificate.
domain_zone = route53.HostedZone.from_lookup(
self, "walkerzone", domain_name="code.awalker.dev")
### Create EFS
# kms_key = kms.Key(self, "comm-vault-key")
commvault_file_system = efs.FileSystem(
self,
"comvault-efs",
vpc=cluster.vpc,
file_system_name="commvault-efs",
encrypted=True,
# kms_key=kms_key ,
)
# kms_key.grant_encrypt_decrypt(commvault_file_system.)
### Define Task Definition and add the container
ecs_task = ecs.FargateTaskDefinition(self, "commvault-task")
ecs_task.add_container(
"commvault-container",
image=ecs.ContainerImage.from_registry(
"store/commvaultrepo/mediaagent:SP7"),
essential=True,
command=[
"-csclientname", "filesys", "-cshost", "-mountpath",
'"/opt/libraryPath"', "-cvdport", "8600", "-clienthost",
"-clientname", "dockermediaagent"
],
logging=ecs.LogDrivers.aws_logs(
stream_prefix="commvault")).add_port_mappings(
ecs.PortMapping(container_port=80,
host_port=80,
protocol=ecs.Protocol.TCP))
ecs_task.add_to_task_role_policy(statement=iam.PolicyStatement(
actions=["efs:*"], resources=['*'], effect=iam.Effect.ALLOW))
### Create the ECS Service using the ApplicationLoadBalancedFargate pattern.
ecs_service = ecs_patterns.ApplicationLoadBalancedFargateService(
self,
"commvault-service",
assign_public_ip=False,
cluster=cluster,
task_definition=ecs_task,
protocol=elbv2.Protocol.HTTPS,
redirect_http=True,
domain_name="commvault.code.awalker.dev",
domain_zone=domain_zone,
platform_version=ecs.FargatePlatformVersion.VERSION1_4,
public_load_balancer=False)
### Grant Read/Write to the s3 Bucket for the task
bucket.grant_read_write(ecs_service.task_definition.task_role)
# -v $TMPDIR/CommvaultLogs:/var/log/commvault/Log_Files
ecs_task.add_volume(
name="CommvaultLogs",
efs_volume_configuration=ecs.EfsVolumeConfiguration(
file_system_id=commvault_file_system.file_system_id,
transit_encryption="ENABLED",
authorization_config=ecs.AuthorizationConfig(
#iam="ENABLED",
access_point_id=efs.AccessPoint(
self,
"CommvaultLog-access-point",
path="/CommvaultLogs",
file_system=commvault_file_system).access_point_id)))
ecs_task.default_container.add_mount_points(
ecs.MountPoint(container_path="/var/log/commvault/Log_Files",
source_volume="CommvaultLogs",
read_only=False))
# -v $TMPDIR/CommvaultRegistry/:/etc/CommVaultRegistry
ecs_task.add_volume(
name="CommVaultRegistry",
efs_volume_configuration=ecs.EfsVolumeConfiguration(
file_system_id=commvault_file_system.file_system_id,
transit_encryption="ENABLED",
authorization_config=ecs.AuthorizationConfig(
#iam="ENABLED",
access_point_id=efs.AccessPoint(
self,
"CommVaultRegistrys-access-point",
path="/CommVaultRegistry",
file_system=commvault_file_system).access_point_id)))
ecs_task.default_container.add_mount_points(
ecs.MountPoint(container_path="/etc/CommVaultRegistry",
source_volume="CommVaultRegistry",
read_only=False))
# -v $TMPDIR/libraryPath/:/opt/libraryPath
ecs_task.add_volume(
name="libraryPath",
efs_volume_configuration=ecs.EfsVolumeConfiguration(
file_system_id=commvault_file_system.file_system_id,
transit_encryption="ENABLED",
authorization_config=ecs.AuthorizationConfig(
#iam="ENABLED",
access_point_id=efs.AccessPoint(
self,
"libraryPath-access-point",
path="/libraryPath",
file_system=commvault_file_system).access_point_id)))
ecs_task.default_container.add_mount_points(
ecs.MountPoint(container_path="/opt/libraryPath",
source_volume="libraryPath",
read_only=False))
# -v $TMPDIR/IndexCache/:/opt/IndexCache
ecs_task.add_volume(
name="IndexCache",
efs_volume_configuration=ecs.EfsVolumeConfiguration(
file_system_id=commvault_file_system.file_system_id,
transit_encryption="ENABLED",
authorization_config=ecs.AuthorizationConfig(
#iam="ENABLED",
access_point_id=efs.AccessPoint(
self,
"IndexCache-access-point",
path="/IndexCache",
file_system=commvault_file_system).access_point_id)))
ecs_task.default_container.add_mount_points(
ecs.MountPoint(container_path="/opt/IndexCache",
source_volume="IndexCache",
read_only=False))
# -v $TMPDIR/jobResults/:/opt/jobResults
ecs_task.add_volume(
name="jobResults",
efs_volume_configuration=ecs.EfsVolumeConfiguration(
file_system_id=commvault_file_system.file_system_id,
transit_encryption="ENABLED",
authorization_config=ecs.AuthorizationConfig(
#iam="ENABLED",
access_point_id=efs.AccessPoint(
self,
"jobResults-access-point",
path="/jobResults",
file_system=commvault_file_system).access_point_id)))
ecs_task.default_container.add_mount_points(
ecs.MountPoint(container_path="/opt/jobResults",
source_volume="jobResults",
read_only=False))
# -v $TMPDIR/certificates:/opt/commvault/Base/certificates
ecs_task.add_volume(
name="certificates",
efs_volume_configuration=ecs.EfsVolumeConfiguration(
file_system_id=commvault_file_system.file_system_id,
transit_encryption="ENABLED",
authorization_config=ecs.AuthorizationConfig(
#iam="ENABLED",
access_point_id=efs.AccessPoint(
self,
"certificates-access-point",
path="/certificates",
file_system=commvault_file_system).access_point_id)))
ecs_task.default_container.add_mount_points(
ecs.MountPoint(container_path="/opt/commvault/Base/certificates",
source_volume="certificates",
read_only=False))
def __init__(self,
scope: core.Construct,
id: str,
vpc,
efs_mnt_path: str = "/efs",
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Create Security Group to connect to EFS
self.efs_sg = _ec2.SecurityGroup(
self,
id="efsSecurityGroup",
vpc=vpc,
security_group_name=f"efs_sg_{id}",
description="Security Group to connect to EFS from the VPC")
self.efs_sg.add_ingress_rule(
peer=_ec2.Peer.ipv4(vpc.vpc_cidr_block),
connection=_ec2.Port.tcp(2049),
description=
"Allow EC2 instances within the same VPC to connect to EFS")
# Let us create the EFS Filesystem
self.efs_share = _efs.FileSystem(
self,
"elasticFileSystem",
file_system_name=f"high-performance-storage",
vpc=vpc,
security_group=self.efs_sg,
encrypted=False,
lifecycle_policy=_efs.LifecyclePolicy.AFTER_7_DAYS,
performance_mode=_efs.PerformanceMode.GENERAL_PURPOSE,
throughput_mode=_efs.ThroughputMode.BURSTING,
removal_policy=core.RemovalPolicy.DESTROY)
# create efs acl
efs_acl = _efs.Acl(owner_gid="1000",
owner_uid="1000",
permissions="0777")
# create efs posix user
efs_user = _efs.PosixUser(gid="1000", uid="1000")
# create efs access point
self.efs_ap = _efs.AccessPoint(self,
"efsAccessPoint",
path=f"{efs_mnt_path}",
file_system=self.efs_share,
posix_user=efs_user,
create_acl=efs_acl)
###########################################
################# OUTPUTS #################
###########################################
output_0 = core.CfnOutput(
self,
"AutomationFrom",
value=f"{GlobalArgs.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output_1 = core.CfnOutput(
self,
"MountEfs",
value=
f"sudo mount -t efs -o tls {self.efs_share.file_system_id}:/ /mnt/efs ",
description=
"Use this command to mount efs using efs helper utility at location /mnt/efs"
)
def __init__(self, scope: core.Construct, construct_id: str,
database: timestream.CfnDatabase, table: timestream.CfnTable,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
vpc = ec2.Vpc(self, "GrafanaVpc", max_azs=2)
vpc.add_interface_endpoint(
'EFSEndpoint',
service=ec2.InterfaceVpcEndpointAwsService.ELASTIC_FILESYSTEM)
vpc.add_interface_endpoint(
'SMEndpoint',
service=ec2.InterfaceVpcEndpointAwsService.SECRETS_MANAGER)
cluster = ecs.Cluster(self, "MyCluster", vpc=vpc)
file_system = efs.FileSystem(
self,
"EfsFileSystem",
vpc=vpc,
encrypted=True,
lifecycle_policy=efs.LifecyclePolicy.AFTER_14_DAYS,
performance_mode=efs.PerformanceMode.GENERAL_PURPOSE,
throughput_mode=efs.ThroughputMode.BURSTING)
access_point = efs.AccessPoint(self,
"EfsAccessPoint",
file_system=file_system,
path="/var/lib/grafana",
posix_user=PosixUser(gid="1000",
uid="1000"),
create_acl=Acl(owner_gid="1000",
owner_uid="1000",
permissions="755"))
log_group = logs.LogGroup(self,
"taskLogGroup",
retention=logs.RetentionDays.ONE_MONTH)
container_log_driver = ecs.LogDrivers.aws_logs(
stream_prefix="fargate-grafana", log_group=log_group)
task_role = iam.Role(
self,
"taskRole",
assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"))
task_role.add_to_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"cloudwatch:DescribeAlarmsForMetric",
"cloudwatch:DescribeAlarmHistory",
"cloudwatch:DescribeAlarms", "cloudwatch:ListMetrics",
"cloudwatch:GetMetricStatistics",
"cloudwatch:GetMetricData", "ec2:DescribeTags",
"ec2:DescribeInstances", "ec2:DescribeRegions",
"tag:GetResources"
],
resources=["*"]))
self.grant_timestream_read(task_role, database, table)
execution_role = iam.Role(
self,
"executionRole",
assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"))
log_group.grant_write(execution_role)
volume_name = "efsGrafanaVolume"
volume_config = ecs.Volume(
name=volume_name,
efs_volume_configuration=EfsVolumeConfiguration(
file_system_id=file_system.file_system_id,
transit_encryption="ENABLED",
authorization_config=AuthorizationConfig(
access_point_id=access_point.access_point_id)))
task_definition = ecs.FargateTaskDefinition(
self,
"TaskDef",
task_role=task_role,
execution_role=execution_role,
volumes=[volume_config])
grafana_admin_password = secretsmanager.Secret(self,
"grafanaAdminPassword")
grafana_admin_password.grant_read(task_role)
container_web = task_definition.add_container(
"grafana",
image=ecs.ContainerImage.from_registry("grafana/grafana"),
logging=container_log_driver,
environment={
"GF_INSTALL_PLUGINS": "grafana-timestream-datasource",
"GF_AWS_default_REGION": core.Aws.REGION
},
secrets={
"GF_SECURITY_ADMIN_PASSWORD":
ecs.Secret.from_secrets_manager(grafana_admin_password)
})
container_web.add_port_mappings(PortMapping(container_port=3000))
container_web.add_mount_points(
MountPoint(container_path="/var/lib/grafana",
read_only=False,
source_volume=volume_config.name))
fargate_service = ecs_patterns.ApplicationLoadBalancedFargateService(
self,
"MyFargateService",
cluster=cluster,
cpu=1024,
desired_count=1,
task_definition=task_definition,
memory_limit_mib=2048,
platform_version=ecs.FargatePlatformVersion.LATEST)
fargate_service.target_group.configure_health_check(path="/api/health")
file_system.connections.allow_default_port_from(
fargate_service.service.connections)
core.CfnOutput(self,
"GrafanaAdminSecret",
value=grafana_admin_password.secret_name,
export_name="GrafanaAdminSecret")
def __init__(
self,
scope: cdk.Construct,
construct_id: str,
vpc: ec2.Vpc,
domain: sagemaker.CfnDomain,
**kwargs,
) -> None:
super().__init__(scope, construct_id, **kwargs)
studio_domain_id = (domain.attr_domain_id
) # cdk.Fn.import_value("StudioDomainId")
# Get the security group associated with the EFS volume managed by SageMaker Studio
get_parameter = cr.AwsCustomResource(
self,
"GetEfsSgId",
on_update={ # will also be called for a CREATE event
"service": "EC2",
"action": "describeSecurityGroups",
"parameters": {
"Filters": [
{"Name": "vpc-id", "Values": [vpc.vpc_id]},
{
"Name": "group-name",
"Values": [
f"security-group-for-inbound-nfs-{studio_domain_id}"
],
},
]
},
"physical_resource_id": cr.PhysicalResourceId.of("GetEfsSgId"),
},
policy=cr.AwsCustomResourcePolicy.from_sdk_calls(
resources=cr.AwsCustomResourcePolicy.ANY_RESOURCE
),
)
sg_name = get_parameter.get_response_field("SecurityGroups.0.GroupId")
sg_efs = ec2.SecurityGroup.from_security_group_id(
self, "SG", security_group_id=sg_name)
# We can now retrive a handler for the EFS volume
StudioDomainEfsId = cdk.Fn.import_value("StudioDomainEfsId")
studio_efs = efs.FileSystem.from_file_system_attributes(
self,
"StudioEFS",
file_system_id=StudioDomainEfsId,
security_group=sg_efs)
# Create EFS access point to enable the lambda fn to mount the EFS volume
efs_ap = efs.AccessPoint(
self,
"EfsAccessPoint",
file_system=studio_efs,
posix_user=efs.PosixUser(gid="0", uid="0"),
)
# Function that takes care of setting up the user environment
self.lambda_fn = lambda_python.PythonFunction(
self,
"UserSetupLambdaFn",
entry="populate_git_fn",
index="populate_from_git.py",
handler="on_event",
vpc=vpc,
layers=[
lambda_.LayerVersion.from_layer_version_arn(
self,
"GitLayer",
layer_version_arn=
f"arn:aws:lambda:{self.region}:553035198032:layer:git-lambda2:8",
),
],
filesystem=lambda_.FileSystem.from_efs_access_point(
efs_ap, "/mnt/efs"),
timeout=cdk.Duration.seconds(300),
initial_policy=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"sagemaker:DescribeUserProfile",
],
resources=["*"],
)
],
)
provider = cr.Provider(
self,
"Provider",
on_event_handler=self.lambda_fn,
)
cdk.CfnOutput(
self,
"StudioUserProviderToken",
value=provider.service_token,
description="StudioUserProviderToken",
export_name="StudioUserProviderToken",
)
self.provider = provider