def __init__(self, app: core.App, id: str, **kwargs) -> None:
super().__init__(app, id, **kwargs)
role = iam.Role(
scope=self,
id="SageMakerDemo1",
role_name="SageMakerDemo1",
assumed_by=iam.ServicePrincipal('sagemaker.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonSageMakerFullAccess'),
iam.ManagedPolicy.from_aws_managed_policy_name(
'IAMReadOnlyAccess')
])
instance = sagemaker.CfnNotebookInstance(
scope=self,
id="My instance",
instance_type="ml.t2.large",
default_code_repository=
"https://github.com/cobusbernard/sagemaker-notebooks.git",
role_arn=role.role_arn)
bucket = s3.Bucket(scope=self,
id="sagemaker-demo-cobus",
bucket_name="sagemaker-demo-cobus")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# enter your code here
instance = sagemaker.CfnNotebookInstance(
scope=self,
id="My instance",
instance_type="ml.t2.medium",
role_arn=self.node.try_get_context("role-arn"),
default_code_repository=
"https://git-codecommit.us-east-1.amazonaws.com/v1/repos/AmazonSageMaker-LinearClassificationRepo",
)
def __init__(
self,
scope: core.Construct,
construct_id: str,
vpc: ec2.Vpc,
role: iam_.Role,
subnet: ec2.PrivateSubnet,
**kwargs
) -> None:
super().__init__(scope, construct_id, **kwargs)
self.security_group = ec2.SecurityGroup(
self,
"WorkshopGroup",
vpc=vpc
)
# Create EFS inside VPC
# self.efs = efs.FileSystem(
# self,
# "commonEFS4Notebooks",
# vpc = vpc,
# encrypted=True,
# enable_automatic_backups=True,
# performance_mode=efs.PerformanceMode('MAX_IO'),
# throughput_mode=efs.ThroughputMode('BURSTING'),
# security_group = self.security_group
# )
# lifecycleconfig = sm.CfnNotebookInstanceLifecycleConfig(
# self,
# "LifeCycleConfig",
# notebook_instance_lifecycle_config_name="LifeCycleConfig",
# on_create=None,
# on_start=code
# )
instance_id = "Workshop"
sm.CfnNotebookInstance(
self,
instance_id,
instance_type='ml.t2.medium',
volume_size_in_gb=20,
security_group_ids=[self.security_group.security_group_id],
subnet_id=subnet.subnet_id,
notebook_instance_name=instance_id,
role_arn=role.role_arn,
direct_internet_access='Enabled',
root_access='Enabled',
default_code_repository="https://github.com/muthukumaranR/workshop_notebooks"
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
bucket1 = s3.Bucket(self,
"BucketCDK",
versioned=True,
bucket_name='cdk-sagemaker-bucket-s843971001',
removal_policy=core.RemovalPolicy.DESTROY)
sm_notebook = sagemaker.CfnNotebookInstance(self,
"SageMakerNotebookInstance",
instance_type='ml.m4.xlarge',
role_arn='arn:aws:iam::304472691870:role/aws-sagemaker-role-s843971',
notebook_instance_name='cdk-sagemaker-notebook-s843971',
default_code_repository='https://github.com/abdul-pfg/sagemaker-iris')
def __init__(self, scope: core.Construct, construct_id: str, vpc: ec2.Vpc,
role: iam_.Role, subnet: ec2.PrivateSubnet, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
self.security_group = ec2.SecurityGroup(self, "WorkshopGroup", vpc=vpc)
instance_id = "Workshop"
sm.CfnNotebookInstance(
self,
instance_id,
instance_type='ml.t2.medium',
volume_size_in_gb=20,
security_group_ids=[self.security_group.security_group_id],
subnet_id=subnet.subnet_id,
notebook_instance_name=instance_id,
role_arn=role.role_arn,
direct_internet_access='Enabled',
root_access='Enabled',
default_code_repository=
"https://github.com/NASA-IMPACT/workshop_notebooks")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Create role for Notebook instance
nrole = iam_.Role(self,
"notebookAccessRole",
assumed_by=iam_.ServicePrincipal("sagemaker"))
nrole.add_managed_policy(
iam_.ManagedPolicy.from_aws_managed_policy_name(
'AmazonSageMakerFullAccess'))
nrole.add_managed_policy(
iam_.ManagedPolicy.from_aws_managed_policy_name(
'AmazonEC2ReadOnlyAccess'))
notebook_uuid = str(uuid.uuid4())
notebook_uuid = str(notebook_uuid[0:notebook_uuid.find('-')])
notebook_instance_id = 'spot-history-notebook-' + notebook_uuid
notebook_instance = sagemaker_.CfnNotebookInstance(
self,
notebook_instance_id,
instance_type='ml.m5.xlarge',
volume_size_in_gb=10,
security_group_ids=default_sg,
subnet_id=default_subnet,
notebook_instance_name=notebook_instance_id,
role_arn=nrole.role_arn,
default_code_repository=github_repo,
)
notebook_url = "https://{}.console.aws.amazon.com/sagemaker/home?region={}#/notebook-instances/openNotebook/{}?view=classic".format(
my_region, my_region, notebook_instance.notebook_instance_name)
core.CfnOutput(
self,
"Notebook Name",
value=notebook_url,
description="Notebook Instance Name",
)
def __init__(self, scope: core.Construct, id: str, users,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
nb_role = iam.Role(
self,
id,
assumed_by=iam.ServicePrincipal('sagemaker.amazonaws.com'))
# XXX: add ro access to some S3 bucket
for user_data in users:
user = user_data['email']
iam_user = iam.User(self,
user,
user_name=user,
password=core.SecretValue.plain_text(
user_data['password']))
nb = sagemaker.CfnNotebookInstance(
self,
'nb_%s' % user,
instance_type='ml.t2.medium',
role_arn=nb_role.role_arn,
notebook_instance_name='nb-%s-%s' %
(id, user.translate(str.maketrans('@.', '--'))))
nb_policy = iam.Policy(
self,
'nb_policy_%s' % user,
statements=[
iam.PolicyStatement(actions=[
'sagemaker:CreatePresignedNotebookInstanceUrl'
],
resources=[nb.ref]),
iam.PolicyStatement(
actions=['sagemaker:ListNotebookInstances'],
resources=["*"])
])
iam_user.attach_inline_policy(nb_policy)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
bucket1 = s3.Bucket(self,
"BucketCDK",
versioned=True,
bucket_name='minotaur-data-bucket',
removal_policy=core.RemovalPolicy.DESTROY)
bucket2 = s3.Bucket(self,
"BucketCDK1",
versioned=True,
bucket_name='minotaur-project-files-bucket',
removal_policy=core.RemovalPolicy.DESTROY)
role_Arn = core.CfnParameter(self, "roleArn", type="String",
description="Sagemaker role for IAM user")
sm_notebook = sagemaker.CfnNotebookInstance(self,
"SageMakerNotebookInstance",
instance_type='ml.t2.medium',
role_arn="role_Arn",
notebook_instance_name='minotaur-notebook')
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Note: typo of role name is copied from original workshop
mysfits_notebook_role = aws_iam.Role(
self,
"MysfitsNotbookRole",
assumed_by=aws_iam.ServicePrincipal("sagemaker.amazonaws.com"),
)
mysfits_notebook_policy = aws_iam.PolicyStatement()
mysfits_notebook_policy.add_actions(
"sagemaker:*",
"ecr:GetAuthorizationToken",
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"ecr:BatchCheckLayerAvailability",
"cloudwatch:PutMetricData",
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:DescribeLogStreams",
"logs:PutLogEvents",
"logs:GetLogEvents",
"s3:CreateBucket",
"s3:ListBucket",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject",
)
mysfits_notebook_policy.add_all_resources()
mysfits_notebook_pass_role_policy = aws_iam.PolicyStatement()
mysfits_notebook_pass_role_policy.add_actions("iam:PassRole")
mysfits_notebook_pass_role_policy.add_all_resources()
mysfits_notebook_pass_role_policy.add_condition(
"StringEquals", {"iam:PassedToService": "sagemaker.amazonaws.com"})
aws_iam.Policy(
self,
"MysfitsNotebookPolicy",
statements=[
mysfits_notebook_pass_role_policy, mysfits_notebook_policy
],
roles=[mysfits_notebook_role],
)
notebook_instance = aws_sagemaker.CfnNotebookInstance(
self,
"MythicalMysfits-SageMaker-Notebook",
instance_type="ml.t2.medium",
role_arn=mysfits_notebook_role.role_arn,
)
lambda_repository = aws_codecommit.Repository(
self,
"RecommendationsLambdaRepository",
repository_name="MythicalMysfits-RecommendationsLambdaRepository",
)
core.CfnOutput(
self,
"recommandationsRepositoryCloneUrlHttp",
value=lambda_repository.repository_clone_url_http,
description="Recommendations Lambda Repository Clone Url HTTP",
)
core.CfnOutput(
self,
"recommandationsRepositoryCloneUrlSsh",
value=lambda_repository.repository_clone_url_ssh,
description="Recommendations Lambda Repository Clone Url SSH",
)
recommendations_lambda_function_policy_statement = aws_iam.PolicyStatement(
)
recommendations_lambda_function_policy_statement.add_actions(
"sagemaker:InvokeEndpoint")
recommendations_lambda_function_policy_statement.add_all_resources()
mysfits_recommendations = aws_lambda.Function(
self,
"Function",
handler="recommendations.recommend",
runtime=aws_lambda.Runtime.PYTHON_3_6,
description="A microservice backend to a SageMaker endpoint",
memory_size=128,
code=aws_lambda.Code.asset(
os.path.join("..", "..", "lambda-recommendations/service")),
timeout=core.Duration.seconds(30),
initial_policy=[recommendations_lambda_function_policy_statement],
)
questions_api_role = aws_iam.Role(
self,
"QuestionsApiRole",
assumed_by=aws_iam.ServicePrincipal("apigateway.amazonaws.com"),
)
api_policy = aws_iam.PolicyStatement()
api_policy.add_actions("lambda:InvokeFunction")
api_policy.add_resources(mysfits_recommendations.function_arn)
aws_iam.Policy(
self,
"QuestionsApiPolicy",
policy_name="questions_api_policy",
statements=[api_policy],
roles=[questions_api_role],
)
questions_integration = aws_apigateway.LambdaIntegration(
mysfits_recommendations,
credentials_role=questions_api_role,
integration_responses=[
aws_apigateway.IntegrationResponse(
status_code="200",
response_templates={
"application/json": '{"status": "OK"}'
},
)
],
)
api = aws_apigateway.LambdaRestApi(
self,
"APIEndpoint",
handler=mysfits_recommendations,
rest_api_name="Recommendation API Service",
proxy=False,
)
recommendations_method = api.root.add_resource("recommendations")
recommendations_method.add_method(
"POST",
questions_integration,
method_responses=[
aws_apigateway.MethodResponse(
status_code="200",
response_parameters={
"method.response.header.Access-Control-Allow-Headers":
True,
"method.response.header.Access-Control-Allow-Methods":
True,
"method.response.header.Access-Control-Allow-Origin":
True,
},
)
],
authorization_type=aws_apigateway.AuthorizationType.NONE,
)
recommendations_method.add_method(
"OPTIONS",
aws_apigateway.MockIntegration(
integration_responses=[
aws_apigateway.IntegrationResponse(
status_code="200",
response_parameters={
"method.response.header.Access-Control-Allow-Headers":
"'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token,X-Amz-User-Agent'",
"method.response.header.Access-Control-Allow-Origin":
"'*'",
"method.response.header.Access-Control-Allow-Credentials":
"'false'",
"method.response.header.Access-Control-Allow-Methods":
"'OPTIONS,GET,PUT,POST,DELETE'",
},
)
],
passthrough_behavior=aws_apigateway.PassthroughBehavior.NEVER,
request_templates={"application/json": '{"statusCode": 200}'},
),
method_responses=[
aws_apigateway.MethodResponse(
status_code="200",
response_parameters={
"method.response.header.Access-Control-Allow-Headers":
True,
"method.response.header.Access-Control-Allow-Methods":
True,
"method.response.header.Access-Control-Allow-Credentials":
True,
"method.response.header.Access-Control-Allow-Origin":
True,
},
)
],
)
def __init__(
self,
scope: core.Construct,
id: str,
vpc: ec2.Vpc,
lambda_sg: ec2.SecurityGroup,
**kwargs
) -> None:
super().__init__(scope, id, **kwargs)
self.PREFIX = id
## **************** Create Knowledge Analyzer Service Role ****************
self.service_role = aws_iam.Role.from_role_arn(
self, f'{self.PREFIX}-IAMROLE-ServiceRole',
f"arn:aws:iam::{self.account}:role/HEALTHLAKE-KNOWLEDGE-ANALYZER-IAMROLE-ServiceRole"
)
## **************** Create a notebook Instance ****************
self.notebook_instance_role = aws_iam.Role(
self,
"AmazonSageMaker-ExecutionRole-20210318",
role_name=f'AmazonSageMaker-ExecutionRole-20210318',
assumed_by=aws_iam.ServicePrincipal("sagemaker.amazonaws.com")
)
roleStmt1=aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
resources=["arn:aws:s3:::*"],
actions=["s3:PutObject", "s3:ListObjects", "s3:GetObject"]
)
roleStmt2=aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
resources=["arn:aws:s3:::*"],
actions=["s3:ListBucket", "s3:GetBucketPublicAccessBlock", "s3:GetEncryptionConfiguration"]
)
self.notebook_instance_role.add_to_policy( roleStmt1 )
self.notebook_instance_role.add_to_policy( roleStmt2 )
self.notebook_instance_role.add_managed_policy(aws_iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSageMakerFullAccess"))
self.nbInstance = sagemaker.CfnNotebookInstance(
self,
f'{self.PREFIX}-HealtLake-Blog-Run',
instance_type = 'ml.t2.medium',
notebook_instance_name=f'{self.PREFIX}-HealtLake-Blog-Run',
role_arn = self.notebook_instance_role.role_arn,
)
## **************** Inherit VPC & Security Group ****************
self.vpc = vpc
self.lambda_sg = lambda_sg
self.comprehend_complete_sqs = sqs.CfnQueue(
self, f'{self.PREFIX}-comprehendCompleteQueue',
visibility_timeout = 900,
queue_name= f'{self.PREFIX}-comprehendCompleteQueue'
)
self.s3export = _s3.Bucket(self, "hl-synthea-export", bucket_name = "hl-synthea-export-%s" % (core.Aws.ACCOUNT_ID), block_public_access=_s3.BlockPublicAccess.BLOCK_ALL, encryption=_s3.BucketEncryption.S3_MANAGED, removal_policy = core.RemovalPolicy.DESTROY, auto_delete_objects= True,)
self.s3_loc = _s3.Bucket(self, "hl-synthea-source", bucket_name = "hl-synthea-source-%s" % (core.Aws.ACCOUNT_ID), block_public_access=_s3.BlockPublicAccess.BLOCK_ALL, removal_policy = core.RemovalPolicy.DESTROY, auto_delete_objects= True,)
self.kendra_instance_role = aws_iam.CfnRole(
self,
f'{self.PREFIX}-Kendra-ServiceRole',
role_name=f'{self.PREFIX}-Kendra-ServiceRole',
assume_role_policy_document=aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[ "sts:AssumeRole" ],
principals=[ aws_iam.ServicePrincipal("kendra.amazonaws.com") ]
)
]
),
policies=[
aws_iam.CfnRole.PolicyProperty(
policy_document=aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"s3:GetObject",
"s3:ListBucket",
"s3:GetBucketPublicAccessBlock",
"s3:GetEncryptionConfiguration"
],
resources=["*"]
)
]
),
policy_name="KendraAllowS3GetListObject"
),
aws_iam.CfnRole.PolicyProperty(
policy_document=aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"cloudwatch:PutMetricData"
],
resources=["*"],
conditions = {
"StringEquals": {
'cloudwatch:namespace': 'AWS/Kendra',
},
}
)
]
),
policy_name="KendraAllowMetricObject"
),
aws_iam.CfnRole.PolicyProperty(
policy_document=aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
'logs:DescribeLogGroups',
'logs:CreateLogGroup',
],
resources=[
"arn:" + "aws" + ":logs:" + core.Aws.REGION + ":" + core.Aws.ACCOUNT_ID + ":log-group:" + "/aws/kendra/*"
],
)
]
),
policy_name="KendraAllowLogObject"
),
aws_iam.CfnRole.PolicyProperty(
policy_document=aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
'logs:DescribeLogStreams',
'logs:CreateLogStream',
'logs:PutLogEvents',
],
resources=[
"arn:" + "aws" + ":logs:" + core.Aws.REGION + ":" + core.Aws.ACCOUNT_ID + ":log-group:" + "/aws/kendra/*:log-stream:*"
],
)
]
),
policy_name="KendraAllowLogStreamsObject"
),
],
)
self.indexKendra = kendra.CfnIndex(
self, f'{self.PREFIX}-KendraIndex',
edition = 'DEVELOPER_EDITION',
name = f'{self.PREFIX}-HealthLakeNotes',
role_arn = self.kendra_instance_role.attr_arn,
)
#--------
# kendra data source role
self.kendra_data_source_instance_role = aws_iam.Role(
self,
f'{self.PREFIX}-KDSrc-ServiceRole',
role_name=f'{self.PREFIX}-KDSrc-ServiceRole',
assumed_by=aws_iam.ServicePrincipal('kendra.amazonaws.com'))
self.kendra_data_source_instance_role.add_to_policy(aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
'kendra:BatchPutDocument',
'kendra:BatchDeleteDocument',
],
resources=[self.indexKendra.attr_arn]
))
self.s3_loc.grant_read(self.kendra_data_source_instance_role)
# print(dir(kendra.CfnDataSource.DataSourceInclusionsExclusionsStringsProperty))
self.datasourceKendra = kendra.CfnDataSource(
self, f'{self.PREFIX}-Data-S3-HealthLake',
name = f'{self.PREFIX}-Data-S3-HealthLake',
index_id = self.indexKendra.attr_id,
type = 'S3',
data_source_configuration=kendra.CfnDataSource.DataSourceConfigurationProperty(
s3_configuration=kendra.CfnDataSource.S3DataSourceConfigurationProperty(
bucket_name=self.s3_loc.bucket_name,
# inclusion_prefixes=kendra.CfnDataSource.DataSourceInclusionsExclusionsStringsProperty(
# data_source_inclusions_exclusions_strings=["source/"]
# ),
)
),
role_arn = self.kendra_data_source_instance_role.role_arn,
)
self.datasourceKendra.add_override("Properties.DataSourceConfiguration.S3Configuration.InclusionPrefixes", ['source/'])
## **************** Create resources ****************
self.createLambdaFunctions()
self.setLambdaTriggers()
# QueuePolicy
self.queue_policy = sqs.CfnQueuePolicy(self, "QueuePolicy",
queues = [self.comprehend_complete_sqs.ref],
policy_document = {
"Version" : "2008-10-17",
"Id" : "__default_policy_ID",
"Statement" : [{
"Sid" : "__owner_statement",
"Effect" : "Allow",
"Principal" : {
"AWS": "*"
},
"Action" : "SQS:SendMessage",
"Resource" : f'arn:aws:sqs:us-east-1:{core.Aws.ACCOUNT_ID}:{self.comprehend_complete_sqs.queue_name}', # self.comprehend_complete_sqs.ref
"Condition": {
"StringEquals": {
"aws:SourceAccount": f'{core.Aws.ACCOUNT_ID}'
},
"ArnLike": {
"aws:SourceArn": f'arn:aws:s3:*:*:{self.s3_loc.bucket_name}'
}
}
}]
}
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# CONTAINER_IMAGE = 'daskdev/dask:0.19.4'
# if use_rapids:
# CONTAINER_IMAGE = 'rapidsai/rapidsai:latest'
# if use_notebook:
# CONTAINER_IMAGE = 'daskdev/dask-notebook:latest'
#TODO : Create ECR repository
#Update: Not required sunce ecs.ContainerImage already creates and pushes using same asset
#ecr = aws_ecr.Repository(self, 'MyECR', repository_name='dask')
# not needed if you use an asset like below:
dockercontainer = ecs.ContainerImage.from_asset(
directory='dockerstuff', build_args=['-t dask .'])
# Create vpc
vpc = ec2.Vpc(self, 'MyVpc', max_azs=3) # default is all AZs in region
subnets = vpc.private_subnets
# Create log groups for the scheduler and workers
s_logs = logs.LogGroup(self, 'SlogGroup', log_group_name='SlogGroup')
w_logs = logs.LogGroup(self, 'WlogGroup', log_group_name='WlogGroup')
#Create private namespace
#nspace = sd.PrivateDnsNamespace(self, 'MyNamespace', vpc=vpc, name='local-dask')
# #Create role for ECS
nRole = iam_.Role(self,
'ECSExecutionRole',
assumed_by=iam_.ServicePrincipal('ecs-tasks'))
nPolicy = iam_.Policy(
self,
"ECSExecutionPolicy",
policy_name="ECSExecutionPolicy",
statements=[
iam_.PolicyStatement(actions=[
'ecr:BatchCheckLayerAvailability',
'ecr:GetDownloadUrlForLayer', 'ecr:BatchGetImage',
'ecr:GetAuthorizationToken', 'logs:CreateLogStream',
'logs:PutLogEvents', 'sagemaker:*', 's3:*'
],
resources=[
'*',
]),
]).attach_to_role(nRole)
# Create ECS cluster
cluster = ecs.Cluster(self,
'DaskCluster',
vpc=vpc,
cluster_name='Fargate-Dask-Cluster')
nspace = cluster.add_default_cloud_map_namespace(
name='local-dask', type=sd.NamespaceType.DNS_PRIVATE, vpc=vpc)
#TO DO: Use default namespace for cluster and use cmap options within fargate service
#Update: done
# schedulerRegistry = sd.Service(self,'serviceRegistryScheduler',
# namespace=nspace,dns_ttl=core.Duration.seconds(60),
# custom_health_check=sd.HealthCheckCustomConfig(failure_threshold=10),
# name='Dask-Scheduler')
# # schedulerRegistry.register_ip_instance(id='serviceRegistryScheduler',ipv4='')
# workerRegistry = sd.Service(self,'workerRegistryScheduler',
# namespace=nspace,dns_ttl=core.Duration.seconds(60),
# custom_health_check=sd.HealthCheckCustomConfig(failure_threshold=10),
# name='Dask-Worker')
# -------------------- Add scheduler task ------------------------
schedulerTask = ecs.TaskDefinition(
self,
'taskDefinitionScheduler',
compatibility=ecs.Compatibility.FARGATE,
cpu='4096',
memory_mib='8192',
network_mode=ecs.NetworkMode.AWS_VPC,
placement_constraints=None,
execution_role=nRole,
family='Dask-Scheduler',
task_role=nRole)
schedulerTask.add_container('MySchedulerImage',
image=dockercontainer,
command=['dask-scheduler'],
cpu=4096,
essential=True,
logging=ecs.LogDriver.aws_logs(
stream_prefix='ecs', log_group=s_logs),
memory_limit_mib=8192,
memory_reservation_mib=8192)
# -------------------- Add worker task -----------------------------
workerTask = ecs.TaskDefinition(
self,
'taskDefinitionWorker',
compatibility=ecs.Compatibility.FARGATE,
cpu='4096',
memory_mib='8192',
network_mode=ecs.NetworkMode.AWS_VPC,
placement_constraints=None,
execution_role=nRole,
family='Dask-Worker',
task_role=nRole)
workerTask.add_container(
'MyWorkerImage',
image=dockercontainer,
command=[
'dask-worker', 'dask-scheduler.local-dask:8786',
'--memory-limit 1800MB', '--worker-port 9000',
'--nanny-port 9001', '--bokeh-port 9002'
],
cpu=4096,
essential=True,
logging=ecs.LogDriver.aws_logs(stream_prefix='ecs',
log_group=s_logs),
memory_limit_mib=8192,
memory_reservation_mib=8192)
# Task security group
sg = ec2.SecurityGroup(self,
'MySG',
vpc=vpc,
description='Enable Scheduler ports access',
security_group_name='DaskSecurityGroup')
# Ingress rule requires IPeer not Peer
# TO DO: fix from any ipv4 to SG
p1 = ec2.Peer().ipv4('0.0.0.0/0')
p2 = ec2.Peer().ipv4('0.0.0.0/0')
sg.add_ingress_rule(peer=p1,
connection=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation='p1',
from_port=8786,
to_port=8789))
sg.add_ingress_rule(peer=p2,
connection=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation='p2',
from_port=9000,
to_port=9002))
# ----------------- Add Scheduler Service -----------------------
# deployconfig = ecs.CfnService.DeploymentConfigurationProperty(maximum_percent=200,minimum_healthy_percent=100)
# vpcconfig = ecs.CfnService.AwsVpcConfigurationProperty(subnets = subnets,assign_public_ip=True,security_groups=[sg])
# networkconfig = ecs.CfnService.NetworkConfigurationProperty(awsvpc_configuration=vpcconfig)
# schedulerService = ecs.CfnService(self, 'DaskSchedulerService',
# task_definition = schedulerTask, deployment_configuration=deployconfig,
# cluster=cluster, desired_count=1, enable_ecs_managed_tags=None,
# launch_type='FARGATE',network_configuration=networkconfig,
# service_registries=schedulerRegistry)
#ecs.CfnService.ServiceRegistryProperty()
# Try fargate service? No service registry option available
#using default cluster namespace
cmap1 = ecs.CloudMapOptions(dns_ttl=core.Duration.seconds(60),
failure_threshold=10,
name='Dask-Scheduler')
schedulerService = ecs.FargateService(
self,
'DaskSchedulerService',
task_definition=schedulerTask,
assign_public_ip=True,
security_group=sg,
#vpc_subnets=subnets,
cluster=cluster,
desired_count=1,
max_healthy_percent=200,
min_healthy_percent=100,
service_name='Dask-Scheduler',
cloud_map_options=cmap1)
# schedulerService.enable_cloud_map(name = 'serviceRegistryScheduler')
# schedulerRegistry.register_non_ip_instance(self,instance_id='DaskSchedulerService')
# ----------------- Add Worker Service -----------------------
#using default cluster namespace
cmap2 = ecs.CloudMapOptions(dns_ttl=core.Duration.seconds(60),
failure_threshold=10,
name='Dask-Worker')
workerService = ecs.FargateService(
self,
'DaskWorkerService',
task_definition=workerTask,
assign_public_ip=True,
security_group=sg,
#vpc_subnets=subnets,
cluster=cluster,
desired_count=1,
max_healthy_percent=200,
min_healthy_percent=100,
service_name='Dask-Worker',
cloud_map_options=cmap2)
# workerService.enable_cloud_map(name = 'workerRegistryScheduler')
#------------------------------------------------------------------------
# Very less control with ECS patterns, did not work
# ecs_patterns.ApplicationLoadBalancedFargateService(self, "DaskFargateStack",
# cluster=cluster, # Required
# cpu=512, # Default is 256
# desired_count=6, # Default is 1
# task_image_options=ecs_patterns.ApplicationLoadBalancedTaskImageOptions(
# image=ecs.ContainerImage.from_registry(CONTAINER_IMAGE)),
# memory_limit_mib=2048, # Default is 512
# public_load_balancer=True) # Default is False
# Start a notebook in the same vpc
# print(type(sg.security_group_id))
# print("------------------------------")
# print(subnets[0].subnet_id)
#Create role for Notebook instance
smRole = iam_.Role(self,
"notebookAccessRole",
assumed_by=iam_.ServicePrincipal('sagemaker'))
smPolicy = iam_.Policy(self,
"notebookAccessPolicy",
policy_name="notebookAccessPolicy",
statements=[
iam_.PolicyStatement(
actions=['s3:*', 'ecs:*'],
resources=[
'*',
]),
]).attach_to_role(smRole)
notebook = sagemaker_.CfnNotebookInstance(
self,
'DaskNotebook',
instance_type='ml.t2.medium',
volume_size_in_gb=50,
security_group_ids=[sg.security_group_id],
subnet_id=subnets[0].subnet_id,
notebook_instance_name='DaskNotebook',
role_arn=smRole.role_arn,
root_access='Enabled',
direct_internet_access='Enabled',
default_code_repository=
'https://github.com/w601sxs/dask-examples.git')
def __init__(self, app: core.App, id: str) -> None:
super().__init__(app, id)
#Create mount instance
efs = efs_.CfnFileSystem(self,
"commonEFS4Notebooks",
encrypted=False,
performance_mode='generalPurpose',
throughput_mode='bursting')
print(efs.ref)
#Create mount target
mount = efs_.CfnMountTarget(
self,
"MountID",
file_system_id=efs.ref,
security_groups=default_sg,
subnet_id=default_subnet,
)
#Create role for Notebook instance
nRole = iam_.Role(self,
"notebookAccessRole",
assumed_by=iam_.ServicePrincipal('sagemaker'))
nPolicy = iam_.Policy(self,
"notebookAccessPolicy",
policy_name="notebookAccessPolicy",
statements=[
iam_.PolicyStatement(actions=[
's3:*',
],
resources=[
'*',
]),
]).attach_to_role(nRole)
#Create notebook instances cluster
instances = []
print(mount.get_att('attr_ip_address').to_string())
encodedScript = LifecycleScriptStr.format(efs.ref)
print("Adding following script to the lifecycle config..\n___\n\n" +
encodedScript)
code = [{"content": core.Fn.base64(encodedScript)}]
lifecycleconfig = sagemaker_.CfnNotebookInstanceLifecycleConfig(
self,
LifeCycleConfigName,
notebook_instance_lifecycle_config_name=LifeCycleConfigName,
on_create=None,
on_start=code)
for i in range(num_instances):
nid = 'CDK-Notebook-Instance-User-' + str(i)
instances.append(
sagemaker_.CfnNotebookInstance(
self,
nid,
instance_type='ml.t2.medium',
volume_size_in_gb=5,
security_group_ids=default_sg,
subnet_id=default_subnet,
notebook_instance_name=nid,
role_arn=nRole.role_arn,
lifecycle_config_name=LifeCycleConfigName,
))
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
SAGEMAKER_NOTEBOOK_INSTANCE_TYPE = cdk.CfnParameter(
self,
'SageMakerNotebookInstanceType',
type='String',
description='Amazon SageMaker Notebook instance type',
default='ml.t2.medium')
#XXX: For createing Amazon MWAA in the existing VPC,
# remove comments from the below codes and
# comments out vpc = aws_ec2.Vpc(..) codes,
# then pass -c vpc_name=your-existing-vpc to cdk command
# for example,
# cdk -c vpc_name=your-existing-vpc syth
#
vpc_name = self.node.try_get_context('vpc_name')
vpc = aws_ec2.Vpc.from_lookup(self,
'ExistingVPC',
is_default=True,
vpc_name=vpc_name)
#XXX: To use more than 2 AZs, be sure to specify the account and region on your stack.
#XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/Vpc.html
# vpc = aws_ec2.Vpc(self, 'SageMakerStudioVPC',
# max_azs=2,
# gateway_endpoints={
# "S3": aws_ec2.GatewayVpcEndpointOptions(
# service=aws_ec2.GatewayVpcEndpointAwsService.S3
# )
# }
# )
sg_sagemaker_notebook_instance = aws_ec2.SecurityGroup(
self,
"SageMakerNotebookSG",
vpc=vpc,
allow_all_outbound=True,
description='Security group with no ingress rule',
security_group_name='sagemaker-nb-{}-sg'.format(''.join(
random.sample((string.ascii_letters), k=5))))
sg_sagemaker_notebook_instance.add_ingress_rule(
peer=sg_sagemaker_notebook_instance,
connection=aws_ec2.Port.all_traffic(),
description='sagemaker notebook security group')
cdk.Tags.of(sg_sagemaker_notebook_instance).add(
'Name', 'sagemaker-nb-sg')
sagemaker_notebook_role_policy_doc = aws_iam.PolicyDocument()
sagemaker_notebook_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"effect":
aws_iam.Effect.ALLOW,
"resources": ["arn:aws:s3:::*"],
"actions": [
"s3:GetObject", "s3:PutObject", "s3:DeleteObject",
"s3:ListBucket"
]
}))
sagemaker_notebook_role = aws_iam.Role(
self,
'SageMakerNotebookRole',
role_name='SageMakerNotebookRole-{suffix}'.format(
suffix=''.join(random.sample((string.ascii_letters), k=5))),
assumed_by=aws_iam.ServicePrincipal('sagemaker.amazonaws.com'),
inline_policies={
'sagemaker-custome-execution-role':
sagemaker_notebook_role_policy_doc
},
managed_policies=[
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonSageMakerFullAccess'),
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'AWSCloudFormationReadOnlyAccess')
])
#XXX: skip downloading rds-combined-ca-bundle.pem if not use SSL with a MySQL DB instance
# https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_MySQL.html#MySQL.Concepts.SSLSupport
sagemaker_nb_lifecycle_content = '''#!/bin/bash
sudo -u ec2-user -i <<'EOF'
echo "export AWS_REGION={AWS_Region}" >> ~/.bashrc
curl -LO https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz
tar zxfv mecab-0.996-ko-0.9.2.tar.gz
pushd mecab-0.996-ko-0.9.2
./configure
make
make check
sudo make install
sudo ldconfig
mecab -v
mecab-config --version
popd
curl -LO https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz
tar -zxvf mecab-ko-dic-2.1.1-20180720.tar.gz
pushd mecab-ko-dic-2.1.1-20180720
./autogen.sh
./configure
make
sudo make install
popd
for each in python3 pytorch_latest_p36
do
source /home/ec2-user/anaconda3/bin/activate ${{each}}
pip install --upgrade pretty_errors
pip install --upgrade pandas-profiling[notebook]
pip install --upgrade ipython-sql
pip install --upgrade PyMySQL
pip install torchvision
pip install torchtext
pip install spacy
pip install nltk
pip install requests
pip install mecab-python
pip install konlpy
pip install jpype1-py3
conda deactivate
done
EOF
'''.format(AWS_Region=cdk.Aws.REGION)
sagemaker_lifecycle_config_prop = aws_sagemaker.CfnNotebookInstanceLifecycleConfig.NotebookInstanceLifecycleHookProperty(
content=cdk.Fn.base64(sagemaker_nb_lifecycle_content))
sagemaker_lifecycle_config = aws_sagemaker.CfnNotebookInstanceLifecycleConfig(
self,
'SageMakerNotebookLifeCycleConfig',
notebook_instance_lifecycle_config_name=
'SageMakerNotebookLifeCycleConfig',
on_start=[sagemaker_lifecycle_config_prop])
sagemaker_notebook_instance = aws_sagemaker.CfnNotebookInstance(
self,
'SageMakerNotebookInstance',
instance_type=SAGEMAKER_NOTEBOOK_INSTANCE_TYPE.value_as_string,
role_arn=sagemaker_notebook_role.role_arn,
lifecycle_config_name=sagemaker_lifecycle_config.
notebook_instance_lifecycle_config_name,
notebook_instance_name='MySageMakerWorkbook',
root_access='Disabled',
security_group_ids=[
sg_sagemaker_notebook_instance.security_group_id
],
subnet_id=vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT).subnet_ids[0])
def __init__(self, scope: core.Construct, id: str, *, prefix: str,
environment: str, configuration, **kwargs):
"""
:param scope: Stack class, used by CDK.
:param id: ID of the construct, used by CDK.
:param prefix: Prefix of the construct, used for naming purposes.
:param environment: Environment of the construct, used for naming purposes.
:param configuration: Configuration of the construct. In this case SAGEMAKER_NOTEBOOK.
:param kwargs: Other parameters that could be used by the construct.
"""
super().__init__(scope, id, **kwargs)
self.prefix = prefix
self.environment_ = environment
self._configuration = configuration
# Validating that the payload passed is correct
validate_configuration(configuration_schema=SAGEMAKER_NOTEBOOK_SCHEMA,
configuration_received=configuration)
base_name = self._configuration["name"]
on_create_list = list()
if validate_file(
file_path=self._configuration["scripts"]["on_create"]):
on_create_file = self._configuration["scripts"]["on_create"]
else:
on_create_file = file_path + "/scripts/iot_analytics_notebook/on_create.sh"
with open(on_create_file) as on_create:
on_create_contents = {"content": core.Fn.base64(on_create.read())}
on_create_list.append(on_create_contents)
on_start_list = list()
if validate_file(
file_path=self._configuration["scripts"]["on_create"]):
on_start_file = self._configuration["scripts"]["on_start"]
else:
on_start_file = file_path + "/scripts/iot_analytics_notebook/on_start.sh"
with open(on_start_file) as on_start:
on_start_contents = {"content": core.Fn.base64(on_start.read())}
on_start_list.append(on_start_contents)
lifecycle_configuration_name = self.prefix + "-" + base_name + "lifecycle-" + self.environment_
self._lifecycle_configuration = sagemaker.CfnNotebookInstanceLifecycleConfig(
self,
id=lifecycle_configuration_name,
notebook_instance_lifecycle_config_name=
lifecycle_configuration_name,
on_create=on_create_list,
on_start=on_start_list,
)
role_name = self.prefix + "_" + base_name + "sagemaker_role_" + self.environment_
self._role = iam.Role(
self,
id=role_name,
role_name=role_name,
assumed_by=iam.ServicePrincipal(service="sagemaker.amazonaws.com"))
managed_policy = iam.ManagedPolicy.from_aws_managed_policy_name(
managed_policy_name="AmazonSageMakerFullAccess")
self._role.add_managed_policy(policy=managed_policy)
policy_name = self.prefix + "_" + base_name + "sagemaker_policy" + self.environment_
ecr_statement = iam.PolicyStatement(
actions=SAGEMAKER_POLICY["ecr_actions"],
resources=SAGEMAKER_POLICY["ecr_resources"])
s3_statement = iam.PolicyStatement(
actions=SAGEMAKER_POLICY["s3_actions"],
resources=SAGEMAKER_POLICY["s3_resources"])
policy = iam.Policy(self,
id=policy_name,
policy_name=policy_name,
statements=[ecr_statement, s3_statement])
self._role.attach_inline_policy(policy=policy)
sagemaker_notebook_name = self.prefix + "_" + base_name + "sagemaker_notebook_" + self.environment_
self._sagemaker_notebook = sagemaker.CfnNotebookInstance(
self,
id=sagemaker_notebook_name,
notebook_instance_name=sagemaker_notebook_name,
lifecycle_config_name=self._lifecycle_configuration.
notebook_instance_lifecycle_config_name,
role_arn=self._role.role_arn,
instance_type=self._configuration["instance_type"],
)
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# The code that defines your stack goes here
vpc_name = self.node.try_get_context("vpc_name")
vpc = aws_ec2.Vpc.from_lookup(self,
"ExistingVPC",
is_default=True,
vpc_name=vpc_name)
sg_use_mysql = aws_ec2.SecurityGroup(
self,
'MySQLClientSG',
vpc=vpc,
allow_all_outbound=True,
description='security group for mysql client',
security_group_name='use-mysql-sg')
core.Tags.of(sg_use_mysql).add('Name', 'mysql-client-sg')
sg_mysql_server = aws_ec2.SecurityGroup(
self,
'MySQLServerSG',
vpc=vpc,
allow_all_outbound=True,
description='security group for mysql',
security_group_name='mysql-server-sg')
sg_mysql_server.add_ingress_rule(peer=sg_use_mysql,
connection=aws_ec2.Port.tcp(3306),
description='use-mysql-sg')
core.Tags.of(sg_mysql_server).add('Name', 'mysql-server-sg')
rds_subnet_group = aws_rds.SubnetGroup(
self,
'RdsSubnetGroup',
description='subnet group for mysql',
subnet_group_name='aurora-mysql',
vpc_subnets=aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.PRIVATE),
vpc=vpc)
rds_engine = aws_rds.DatabaseClusterEngine.aurora_mysql(
version=aws_rds.AuroraMysqlEngineVersion.VER_2_08_1)
rds_cluster_param_group = aws_rds.ParameterGroup(
self,
'AuroraMySQLClusterParamGroup',
engine=rds_engine,
description='Custom cluster parameter group for aurora-mysql5.7',
parameters={
'innodb_flush_log_at_trx_commit': '2',
'slow_query_log': '1',
'tx_isolation': 'READ-COMMITTED',
'wait_timeout': '300',
'character-set-client-handshake': '0',
'character_set_server': 'utf8mb4',
'collation_server': 'utf8mb4_unicode_ci',
'init_connect': 'SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci'
})
rds_db_param_group = aws_rds.ParameterGroup(
self,
'AuroraMySQLDBParamGroup',
engine=rds_engine,
description='Custom parameter group for aurora-mysql5.7',
parameters={
'slow_query_log': '1',
'tx_isolation': 'READ-COMMITTED',
'wait_timeout': '300',
'init_connect': 'SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci'
})
db_cluster_name = self.node.try_get_context('db_cluster_name')
# #XXX: aws_rds.Credentials.from_username(username, ...) can not be given user specific Secret name
# #XXX: therefore, first create Secret and then use it to create database
# db_secret_name = self.node.try_get_context('db_secret_name')
# #XXX: arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name}
# db_secret_arn = 'arn:aws:secretsmanager:{region}:{account}:secret:{resource_name}'.format(
# region=core.Aws.REGION, account=core.Aws.ACCOUNT_ID, resource_name=db_secret_name)
# db_secret = aws_secretsmanager.Secret.from_secret_arn(self, 'DBSecretFromArn', db_secret_arn)
# rds_credentials = aws_rds.Credentials.from_secret(db_secret)
rds_credentials = aws_rds.Credentials.from_generated_secret("admin")
db_cluster = aws_rds.DatabaseCluster(
self,
'Database',
engine=rds_engine,
credentials=rds_credentials,
instance_props={
'instance_type':
aws_ec2.InstanceType.of(aws_ec2.InstanceClass.BURSTABLE3,
aws_ec2.InstanceSize.MEDIUM),
'parameter_group':
rds_db_param_group,
'vpc_subnets': {
'subnet_type': aws_ec2.SubnetType.PRIVATE
},
'vpc':
vpc,
'auto_minor_version_upgrade':
False,
'security_groups': [sg_mysql_server]
},
instances=2,
parameter_group=rds_cluster_param_group,
cloudwatch_logs_retention=aws_logs.RetentionDays.THREE_DAYS,
cluster_identifier=db_cluster_name,
subnet_group=rds_subnet_group,
backup=aws_rds.BackupProps(retention=core.Duration.days(3),
preferred_window="03:00-04:00"))
sagemaker_notebook_role_policy_doc = aws_iam.PolicyDocument()
sagemaker_notebook_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"effect": aws_iam.Effect.ALLOW,
"resources": [db_cluster.secret.secret_full_arn],
"actions": ["secretsmanager:GetSecretValue"]
}))
sagemaker_notebook_role = aws_iam.Role(
self,
'SageMakerNotebookRoleForRDS',
role_name='AWSSageMakerNotebookRoleForRDS',
assumed_by=aws_iam.ServicePrincipal('sagemaker.amazonaws.com'),
inline_policies={
'AuroraMySQLSecretPolicy': sagemaker_notebook_role_policy_doc
})
cf_readonly_access_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'AWSCloudFormationReadOnlyAccess')
sagemaker_notebook_role.add_managed_policy(cf_readonly_access_policy)
#XXX: skip downloading rds-combined-ca-bundle.pem if not use SSL with a MySQL DB instance
# https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_MySQL.html#MySQL.Concepts.SSLSupport
rds_wb_lifecycle_content = '''#!/bin/bash
sudo -u ec2-user -i <<'EOF'
echo "export AWS_REGION={AWS_Region}" >> ~/.bashrc
source /home/ec2-user/anaconda3/bin/activate python3
pip install --upgrade ipython-sql
pip install --upgrade PyMySQL
pip install --upgrade pretty_errors
source /home/ec2-user/anaconda3/bin/deactivate
cd /home/ec2-user/SageMaker
wget -N https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem
wget -N https://raw.githubusercontent.com/ksmin23/my-aws-cdk-examples/main/rds/sagemaker-aurora_mysql/ipython-sql.ipynb
EOF
'''.format(AWS_Region=core.Aws.REGION)
rds_wb_lifecycle_config_prop = aws_sagemaker.CfnNotebookInstanceLifecycleConfig.NotebookInstanceLifecycleHookProperty(
content=core.Fn.base64(rds_wb_lifecycle_content))
rds_wb_lifecycle_config = aws_sagemaker.CfnNotebookInstanceLifecycleConfig(
self,
'MySQLWorkbenchLifeCycleConfig',
notebook_instance_lifecycle_config_name=
'MySQLWorkbenchLifeCycleConfig',
on_start=[rds_wb_lifecycle_config_prop])
rds_workbench = aws_sagemaker.CfnNotebookInstance(
self,
'AuroraMySQLWorkbench',
instance_type='ml.t3.xlarge',
role_arn=sagemaker_notebook_role.role_arn,
lifecycle_config_name=rds_wb_lifecycle_config.
notebook_instance_lifecycle_config_name,
notebook_instance_name='AuroraMySQLWorkbench',
root_access='Disabled',
security_group_ids=[sg_use_mysql.security_group_name],
subnet_id=vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids[0])
core.CfnOutput(self,
'StackName',
value=self.stack_name,
export_name='StackName')
core.CfnOutput(self, 'VpcId', value=vpc.vpc_id, export_name='VpcId')
core.CfnOutput(self,
'DBClusterName',
value=db_cluster.cluster_identifier,
export_name='DBClusterName')
core.CfnOutput(self,
'DBCluster',
value=db_cluster.cluster_endpoint.socket_address,
export_name='DBCluster')
#XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_secretsmanager/README.html
# secret_arn="arn:aws:secretsmanager:<region>:<account-id-number>:secret:<secret-name>-<random-6-characters>",
core.CfnOutput(self,
'DBSecret',
value=db_cluster.secret.secret_name,
export_name='DBSecret')
core.CfnOutput(self,
'SageMakerRole',
value=sagemaker_notebook_role.role_name,
export_name='SageMakerRole')
core.CfnOutput(self,
'SageMakerNotebookInstance',
value=rds_workbench.notebook_instance_name,
export_name='SageMakerNotebookInstance')
core.CfnOutput(self,
'SageMakerNotebookInstanceLifecycleConfig',
value=rds_workbench.lifecycle_config_name,
export_name='SageMakerNotebookInstanceLifecycleConfig')
def __init__(self, scope: Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
vpc = aws_ec2.Vpc(self, "NeptuneHolVPC",
max_azs=2,
gateway_endpoints={
"S3": aws_ec2.GatewayVpcEndpointOptions(
service=aws_ec2.GatewayVpcEndpointAwsService.S3
)
}
)
sg_use_graph_db = aws_ec2.SecurityGroup(self, "NeptuneClientSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for neptune client',
security_group_name='use-neptune-client'
)
cdk.Tags.of(sg_use_graph_db).add('Name', 'use-neptune-client')
sg_graph_db = aws_ec2.SecurityGroup(self, "NeptuneSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for neptune',
security_group_name='neptune-server'
)
cdk.Tags.of(sg_graph_db).add('Name', 'neptune-server')
sg_graph_db.add_ingress_rule(peer=sg_graph_db, connection=aws_ec2.Port.tcp(8182), description='neptune-server')
sg_graph_db.add_ingress_rule(peer=sg_use_graph_db, connection=aws_ec2.Port.tcp(8182), description='use-neptune-client')
graph_db_subnet_group = aws_neptune.CfnDBSubnetGroup(self, 'NeptuneHolSubnetGroup',
db_subnet_group_description='subnet group for neptune hol',
subnet_ids=vpc.select_subnets(subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT).subnet_ids,
db_subnet_group_name='neptune-hol'
)
graph_db = aws_neptune.CfnDBCluster(self, 'NeptuneHol',
availability_zones=vpc.availability_zones,
db_subnet_group_name=graph_db_subnet_group.db_subnet_group_name,
db_cluster_identifier='neptune-hol',
backup_retention_period=1,
preferred_backup_window='08:45-09:15',
preferred_maintenance_window='sun:18:00-sun:18:30',
vpc_security_group_ids=[sg_graph_db.security_group_id]
)
graph_db.add_depends_on(graph_db_subnet_group)
graph_db_instance = aws_neptune.CfnDBInstance(self, 'NeptuneHolInstance',
db_instance_class='db.r5.large',
allow_major_version_upgrade=False,
auto_minor_version_upgrade=False,
availability_zone=vpc.availability_zones[0],
db_cluster_identifier=graph_db.db_cluster_identifier,
db_instance_identifier='neptune-hol',
preferred_maintenance_window='sun:18:00-sun:18:30'
)
graph_db_instance.add_depends_on(graph_db)
graph_db_replica_instance = aws_neptune.CfnDBInstance(self, 'NeptuneHolReplicaInstance',
db_instance_class='db.r5.large',
allow_major_version_upgrade=False,
auto_minor_version_upgrade=False,
availability_zone=vpc.availability_zones[-1],
db_cluster_identifier=graph_db.db_cluster_identifier,
db_instance_identifier='neptune-hol-replica',
preferred_maintenance_window='sun:18:00-sun:18:30'
)
graph_db_replica_instance.add_depends_on(graph_db)
graph_db_replica_instance.add_depends_on(graph_db_instance)
sagemaker_notebook_role_policy_doc = aws_iam.PolicyDocument()
sagemaker_notebook_role_policy_doc.add_statements(aws_iam.PolicyStatement(**{
"effect": aws_iam.Effect.ALLOW,
"resources": ["arn:aws:s3:::aws-neptune-notebook",
"arn:aws:s3:::aws-neptune-notebook/*"],
"actions": ["s3:GetObject",
"s3:ListBucket"]
}))
sagemaker_notebook_role_policy_doc.add_statements(aws_iam.PolicyStatement(**{
"effect": aws_iam.Effect.ALLOW,
"resources": ["arn:aws:neptune-db:{region}:{account}:{cluster_id}/*".format(
region=cdk.Aws.REGION, account=cdk.Aws.ACCOUNT_ID, cluster_id=graph_db.attr_cluster_resource_id)],
"actions": ["neptune-db:connect"]
}))
sagemaker_notebook_role = aws_iam.Role(self, 'SageMakerNotebookForNeptuneWorkbenchRole',
role_name='AWSNeptuneNotebookRole-NeptuneHol',
assumed_by=aws_iam.ServicePrincipal('sagemaker.amazonaws.com'),
#XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221
inline_policies={
'AWSNeptuneNotebook': sagemaker_notebook_role_policy_doc
}
)
neptune_wb_lifecycle_content = '''#!/bin/bash
sudo -u ec2-user -i <<'EOF'
echo "export GRAPH_NOTEBOOK_AUTH_MODE=DEFAULT" >> ~/.bashrc
echo "export GRAPH_NOTEBOOK_HOST={NeptuneClusterEndpoint}" >> ~/.bashrc
echo "export GRAPH_NOTEBOOK_PORT={NeptuneClusterPort}" >> ~/.bashrc
echo "export NEPTUNE_LOAD_FROM_S3_ROLE_ARN=''" >> ~/.bashrc
echo "export AWS_REGION={AWS_Region}" >> ~/.bashrc
aws s3 cp s3://aws-neptune-notebook/graph_notebook.tar.gz /tmp/graph_notebook.tar.gz
rm -rf /tmp/graph_notebook
tar -zxvf /tmp/graph_notebook.tar.gz -C /tmp
/tmp/graph_notebook/install.sh
EOF
'''.format(NeptuneClusterEndpoint=graph_db.attr_endpoint,
NeptuneClusterPort=graph_db.attr_port,
AWS_Region=cdk.Aws.REGION)
neptune_wb_lifecycle_config_prop = aws_sagemaker.CfnNotebookInstanceLifecycleConfig.NotebookInstanceLifecycleHookProperty(
content=cdk.Fn.base64(neptune_wb_lifecycle_content)
)
neptune_wb_lifecycle_config = aws_sagemaker.CfnNotebookInstanceLifecycleConfig(self, 'NpetuneWorkbenchLifeCycleConfig',
notebook_instance_lifecycle_config_name='NeptuneWorkbenchLifeCycleConfig',
on_start=[neptune_wb_lifecycle_config_prop]
)
neptune_workbench = aws_sagemaker.CfnNotebookInstance(self, 'NeptuneWorkbench',
instance_type='ml.t2.medium',
role_arn=sagemaker_notebook_role.role_arn,
lifecycle_config_name=neptune_wb_lifecycle_config.notebook_instance_lifecycle_config_name,
notebook_instance_name='NeptuneHolWorkbench',
root_access='Disabled',
security_group_ids=[sg_use_graph_db.security_group_id],
subnet_id=graph_db_subnet_group.subnet_ids[0]
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
vpc = aws_ec2.Vpc(
self,
"OctemberVPC",
max_azs=2,
# subnet_configuration=[{
# "cidrMask": 24,
# "name": "Public",
# "subnetType": aws_ec2.SubnetType.PUBLIC,
# },
# {
# "cidrMask": 24,
# "name": "Private",
# "subnetType": aws_ec2.SubnetType.PRIVATE
# },
# {
# "cidrMask": 28,
# "name": "Isolated",
# "subnetType": aws_ec2.SubnetType.ISOLATED,
# "reserved": True
# }
# ],
gateway_endpoints={
"S3":
aws_ec2.GatewayVpcEndpointOptions(
service=aws_ec2.GatewayVpcEndpointAwsService.S3)
})
dynamo_db_endpoint = vpc.add_gateway_endpoint(
"DynamoDbEndpoint",
service=aws_ec2.GatewayVpcEndpointAwsService.DYNAMODB)
s3_bucket = s3.Bucket(
self,
"s3bucket",
bucket_name="octember-bizcard-{region}-{account}".format(
region=core.Aws.REGION, account=core.Aws.ACCOUNT_ID))
api = apigw.RestApi(
self,
"BizcardImageUploader",
rest_api_name="BizcardImageUploader",
description="This service serves uploading bizcard images into s3.",
endpoint_types=[apigw.EndpointType.REGIONAL],
binary_media_types=["image/png", "image/jpg"],
deploy=True,
deploy_options=apigw.StageOptions(stage_name="v1"))
rest_api_role = aws_iam.Role(
self,
"ApiGatewayRoleForS3",
role_name="ApiGatewayRoleForS3FullAccess",
assumed_by=aws_iam.ServicePrincipal("apigateway.amazonaws.com"),
managed_policies=[
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3FullAccess")
])
list_objects_responses = [
apigw.IntegrationResponse(
status_code="200",
#XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationResponse.html#aws_cdk.aws_apigateway.IntegrationResponse.response_parameters
# The response parameters from the backend response that API Gateway sends to the method response.
# Use the destination as the key and the source as the value:
# - The destination must be an existing response parameter in the MethodResponse property.
# - The source must be an existing method request parameter or a static value.
response_parameters={
'method.response.header.Timestamp':
'integration.response.header.Date',
'method.response.header.Content-Length':
'integration.response.header.Content-Length',
'method.response.header.Content-Type':
'integration.response.header.Content-Type'
}),
apigw.IntegrationResponse(status_code="400",
selection_pattern="4\d{2}"),
apigw.IntegrationResponse(status_code="500",
selection_pattern="5\d{2}")
]
list_objects_integration_options = apigw.IntegrationOptions(
credentials_role=rest_api_role,
integration_responses=list_objects_responses)
get_s3_integration = apigw.AwsIntegration(
service="s3",
integration_http_method="GET",
path='/',
options=list_objects_integration_options)
api.root.add_method(
"GET",
get_s3_integration,
authorization_type=apigw.AuthorizationType.IAM,
api_key_required=False,
method_responses=[
apigw.MethodResponse(
status_code="200",
response_parameters={
'method.response.header.Timestamp': False,
'method.response.header.Content-Length': False,
'method.response.header.Content-Type': False
},
response_models={'application/json': apigw.EmptyModel()}),
apigw.MethodResponse(status_code="400"),
apigw.MethodResponse(status_code="500")
],
request_parameters={'method.request.header.Content-Type': False})
get_s3_folder_integration_options = apigw.IntegrationOptions(
credentials_role=rest_api_role,
integration_responses=list_objects_responses,
#XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationOptions.html#aws_cdk.aws_apigateway.IntegrationOptions.request_parameters
# Specify request parameters as key-value pairs (string-to-string mappings), with a destination as the key and a source as the value.
# The source must be an existing method request parameter or a static value.
request_parameters={
"integration.request.path.bucket": "method.request.path.folder"
})
get_s3_folder_integration = apigw.AwsIntegration(
service="s3",
integration_http_method="GET",
path="{bucket}",
options=get_s3_folder_integration_options)
s3_folder = api.root.add_resource('{folder}')
s3_folder.add_method(
"GET",
get_s3_folder_integration,
authorization_type=apigw.AuthorizationType.IAM,
api_key_required=False,
method_responses=[
apigw.MethodResponse(
status_code="200",
response_parameters={
'method.response.header.Timestamp': False,
'method.response.header.Content-Length': False,
'method.response.header.Content-Type': False
},
response_models={'application/json': apigw.EmptyModel()}),
apigw.MethodResponse(status_code="400"),
apigw.MethodResponse(status_code="500")
],
request_parameters={
'method.request.header.Content-Type': False,
'method.request.path.folder': True
})
get_s3_item_integration_options = apigw.IntegrationOptions(
credentials_role=rest_api_role,
integration_responses=list_objects_responses,
request_parameters={
"integration.request.path.bucket":
"method.request.path.folder",
"integration.request.path.object": "method.request.path.item"
})
get_s3_item_integration = apigw.AwsIntegration(
service="s3",
integration_http_method="GET",
path="{bucket}/{object}",
options=get_s3_item_integration_options)
s3_item = s3_folder.add_resource('{item}')
s3_item.add_method(
"GET",
get_s3_item_integration,
authorization_type=apigw.AuthorizationType.IAM,
api_key_required=False,
method_responses=[
apigw.MethodResponse(
status_code="200",
response_parameters={
'method.response.header.Timestamp': False,
'method.response.header.Content-Length': False,
'method.response.header.Content-Type': False
},
response_models={'application/json': apigw.EmptyModel()}),
apigw.MethodResponse(status_code="400"),
apigw.MethodResponse(status_code="500")
],
request_parameters={
'method.request.header.Content-Type': False,
'method.request.path.folder': True,
'method.request.path.item': True
})
put_s3_item_integration_options = apigw.IntegrationOptions(
credentials_role=rest_api_role,
integration_responses=[
apigw.IntegrationResponse(status_code="200"),
apigw.IntegrationResponse(status_code="400",
selection_pattern="4\d{2}"),
apigw.IntegrationResponse(status_code="500",
selection_pattern="5\d{2}")
],
request_parameters={
"integration.request.header.Content-Type":
"method.request.header.Content-Type",
"integration.request.path.bucket":
"method.request.path.folder",
"integration.request.path.object": "method.request.path.item"
})
put_s3_item_integration = apigw.AwsIntegration(
service="s3",
integration_http_method="PUT",
path="{bucket}/{object}",
options=put_s3_item_integration_options)
s3_item.add_method(
"PUT",
put_s3_item_integration,
authorization_type=apigw.AuthorizationType.IAM,
api_key_required=False,
method_responses=[
apigw.MethodResponse(
status_code="200",
response_parameters={
'method.response.header.Content-Type': False
},
response_models={'application/json': apigw.EmptyModel()}),
apigw.MethodResponse(status_code="400"),
apigw.MethodResponse(status_code="500")
],
request_parameters={
'method.request.header.Content-Type': False,
'method.request.path.folder': True,
'method.request.path.item': True
})
ddb_table = dynamodb.Table(
self,
"BizcardImageMetaInfoDdbTable",
table_name="OctemberBizcardImgMeta",
partition_key=dynamodb.Attribute(
name="image_id", type=dynamodb.AttributeType.STRING),
billing_mode=dynamodb.BillingMode.PROVISIONED,
read_capacity=15,
write_capacity=5)
img_kinesis_stream = kinesis.Stream(
self, "BizcardImagePath", stream_name="octember-bizcard-image")
# create lambda function
trigger_textract_lambda_fn = _lambda.Function(
self,
"TriggerTextExtractorFromImage",
runtime=_lambda.Runtime.PYTHON_3_7,
function_name="TriggerTextExtractorFromImage",
handler="trigger_text_extract_from_s3_image.lambda_handler",
description="Trigger to extract text from an image in S3",
code=_lambda.Code.asset(
"./src/main/python/TriggerTextExtractFromS3Image"),
environment={
'REGION_NAME': core.Aws.REGION,
'DDB_TABLE_NAME': ddb_table.table_name,
'KINESIS_STREAM_NAME': img_kinesis_stream.stream_name
},
timeout=core.Duration.minutes(5))
ddb_table_rw_policy_statement = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
resources=[ddb_table.table_arn],
actions=[
"dynamodb:BatchGetItem", "dynamodb:Describe*",
"dynamodb:List*", "dynamodb:GetItem", "dynamodb:Query",
"dynamodb:Scan", "dynamodb:BatchWriteItem",
"dynamodb:DeleteItem", "dynamodb:PutItem",
"dynamodb:UpdateItem", "dax:Describe*", "dax:List*",
"dax:GetItem", "dax:BatchGetItem", "dax:Query", "dax:Scan",
"dax:BatchWriteItem", "dax:DeleteItem", "dax:PutItem",
"dax:UpdateItem"
])
trigger_textract_lambda_fn.add_to_role_policy(
ddb_table_rw_policy_statement)
trigger_textract_lambda_fn.add_to_role_policy(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=[img_kinesis_stream.stream_arn],
actions=[
"kinesis:Get*", "kinesis:List*",
"kinesis:Describe*",
"kinesis:PutRecord",
"kinesis:PutRecords"
]))
# assign notification for the s3 event type (ex: OBJECT_CREATED)
s3_event_filter = s3.NotificationKeyFilter(prefix="bizcard-raw-img/",
suffix=".jpg")
s3_event_source = S3EventSource(s3_bucket,
events=[s3.EventType.OBJECT_CREATED],
filters=[s3_event_filter])
trigger_textract_lambda_fn.add_event_source(s3_event_source)
#XXX: https://github.com/aws/aws-cdk/issues/2240
# To avoid to create extra Lambda Functions with names like LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a
# if log_retention=aws_logs.RetentionDays.THREE_DAYS is added to the constructor props
log_group = aws_logs.LogGroup(
self,
"TriggerTextractLogGroup",
log_group_name="/aws/lambda/TriggerTextExtractorFromImage",
retention=aws_logs.RetentionDays.THREE_DAYS)
log_group.grant_write(trigger_textract_lambda_fn)
text_kinesis_stream = kinesis.Stream(
self, "BizcardTextData", stream_name="octember-bizcard-txt")
textract_lambda_fn = _lambda.Function(
self,
"GetTextFromImage",
runtime=_lambda.Runtime.PYTHON_3_7,
function_name="GetTextFromImage",
handler="get_text_from_s3_image.lambda_handler",
description="extract text from an image in S3",
code=_lambda.Code.asset("./src/main/python/GetTextFromS3Image"),
environment={
'REGION_NAME': core.Aws.REGION,
'DDB_TABLE_NAME': ddb_table.table_name,
'KINESIS_STREAM_NAME': text_kinesis_stream.stream_name
},
timeout=core.Duration.minutes(5))
textract_lambda_fn.add_to_role_policy(ddb_table_rw_policy_statement)
textract_lambda_fn.add_to_role_policy(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=[text_kinesis_stream.stream_arn],
actions=[
"kinesis:Get*", "kinesis:List*",
"kinesis:Describe*",
"kinesis:PutRecord",
"kinesis:PutRecords"
]))
textract_lambda_fn.add_to_role_policy(
aws_iam.PolicyStatement(
**{
"effect":
aws_iam.Effect.ALLOW,
"resources": [
s3_bucket.bucket_arn, "{}/*".format(
s3_bucket.bucket_arn)
],
"actions": [
"s3:AbortMultipartUpload", "s3:GetBucketLocation",
"s3:GetObject", "s3:ListBucket",
"s3:ListBucketMultipartUploads", "s3:PutObject"
]
}))
textract_lambda_fn.add_to_role_policy(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=["*"],
actions=["textract:*"]))
img_kinesis_event_source = KinesisEventSource(
img_kinesis_stream,
batch_size=100,
starting_position=_lambda.StartingPosition.LATEST)
textract_lambda_fn.add_event_source(img_kinesis_event_source)
log_group = aws_logs.LogGroup(
self,
"GetTextFromImageLogGroup",
log_group_name="/aws/lambda/GetTextFromImage",
retention=aws_logs.RetentionDays.THREE_DAYS)
log_group.grant_write(textract_lambda_fn)
sg_use_bizcard_es = aws_ec2.SecurityGroup(
self,
"BizcardSearchClientSG",
vpc=vpc,
allow_all_outbound=True,
description=
'security group for octember bizcard elasticsearch client',
security_group_name='use-octember-bizcard-es')
core.Tags.of(sg_use_bizcard_es).add('Name', 'use-octember-bizcard-es')
sg_bizcard_es = aws_ec2.SecurityGroup(
self,
"BizcardSearchSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for octember bizcard elasticsearch',
security_group_name='octember-bizcard-es')
core.Tags.of(sg_bizcard_es).add('Name', 'octember-bizcard-es')
sg_bizcard_es.add_ingress_rule(peer=sg_bizcard_es,
connection=aws_ec2.Port.all_tcp(),
description='octember-bizcard-es')
sg_bizcard_es.add_ingress_rule(peer=sg_use_bizcard_es,
connection=aws_ec2.Port.all_tcp(),
description='use-octember-bizcard-es')
sg_ssh_access = aws_ec2.SecurityGroup(
self,
"BastionHostSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for bastion host',
security_group_name='octember-bastion-host-sg')
core.Tags.of(sg_ssh_access).add('Name', 'octember-bastion-host')
sg_ssh_access.add_ingress_rule(peer=aws_ec2.Peer.any_ipv4(),
connection=aws_ec2.Port.tcp(22),
description='ssh access')
bastion_host = aws_ec2.BastionHostLinux(
self,
"BastionHost",
vpc=vpc,
instance_type=aws_ec2.InstanceType('t3.nano'),
security_group=sg_ssh_access,
subnet_selection=aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.PUBLIC))
bastion_host.instance.add_security_group(sg_use_bizcard_es)
#XXX: aws cdk elastsearch example - https://github.com/aws/aws-cdk/issues/2873
es_cfn_domain = aws_elasticsearch.CfnDomain(
self,
'BizcardSearch',
elasticsearch_cluster_config={
"dedicatedMasterCount": 3,
"dedicatedMasterEnabled": True,
"dedicatedMasterType": "t2.medium.elasticsearch",
"instanceCount": 2,
"instanceType": "t2.medium.elasticsearch",
"zoneAwarenessEnabled": True
},
ebs_options={
"ebsEnabled": True,
"volumeSize": 10,
"volumeType": "gp2"
},
domain_name="octember-bizcard",
elasticsearch_version="7.9",
encryption_at_rest_options={"enabled": False},
access_policies={
"Version":
"2012-10-17",
"Statement": [{
"Effect":
"Allow",
"Principal": {
"AWS": "*"
},
"Action":
["es:Describe*", "es:List*", "es:Get*", "es:ESHttp*"],
"Resource":
self.format_arn(service="es",
resource="domain",
resource_name="octember-bizcard/*")
}]
},
snapshot_options={"automatedSnapshotStartHour": 17},
vpc_options={
"securityGroupIds": [sg_bizcard_es.security_group_id],
"subnetIds":
vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids
})
core.Tags.of(es_cfn_domain).add('Name', 'octember-bizcard-es')
s3_lib_bucket_name = self.node.try_get_context("lib_bucket_name")
#XXX: https://github.com/aws/aws-cdk/issues/1342
s3_lib_bucket = s3.Bucket.from_bucket_name(self, id,
s3_lib_bucket_name)
es_lib_layer = _lambda.LayerVersion(
self,
"ESLib",
layer_version_name="es-lib",
compatible_runtimes=[_lambda.Runtime.PYTHON_3_7],
code=_lambda.Code.from_bucket(s3_lib_bucket,
"var/octember-es-lib.zip"))
redis_lib_layer = _lambda.LayerVersion(
self,
"RedisLib",
layer_version_name="redis-lib",
compatible_runtimes=[_lambda.Runtime.PYTHON_3_7],
code=_lambda.Code.from_bucket(s3_lib_bucket,
"var/octember-redis-lib.zip"))
#XXX: Deploy lambda in VPC - https://github.com/aws/aws-cdk/issues/1342
upsert_to_es_lambda_fn = _lambda.Function(
self,
"UpsertBizcardToES",
runtime=_lambda.Runtime.PYTHON_3_7,
function_name="UpsertBizcardToElasticSearch",
handler="upsert_bizcard_to_es.lambda_handler",
description="Upsert bizcard text into elasticsearch",
code=_lambda.Code.asset("./src/main/python/UpsertBizcardToES"),
environment={
'ES_HOST': es_cfn_domain.attr_domain_endpoint,
'ES_INDEX': 'octember_bizcard',
'ES_TYPE': 'bizcard'
},
timeout=core.Duration.minutes(5),
layers=[es_lib_layer],
security_groups=[sg_use_bizcard_es],
vpc=vpc)
text_kinesis_event_source = KinesisEventSource(
text_kinesis_stream,
batch_size=99,
starting_position=_lambda.StartingPosition.LATEST)
upsert_to_es_lambda_fn.add_event_source(text_kinesis_event_source)
log_group = aws_logs.LogGroup(
self,
"UpsertBizcardToESLogGroup",
log_group_name="/aws/lambda/UpsertBizcardToElasticSearch",
retention=aws_logs.RetentionDays.THREE_DAYS)
log_group.grant_write(upsert_to_es_lambda_fn)
firehose_role_policy_doc = aws_iam.PolicyDocument()
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"effect":
aws_iam.Effect.ALLOW,
"resources": [
s3_bucket.bucket_arn, "{}/*".format(
s3_bucket.bucket_arn)
],
"actions": [
"s3:AbortMultipartUpload", "s3:GetBucketLocation",
"s3:GetObject", "s3:ListBucket",
"s3:ListBucketMultipartUploads", "s3:PutObject"
]
}))
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=["*"],
actions=[
"glue:GetTable",
"glue:GetTableVersion",
"glue:GetTableVersions"
]))
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=[text_kinesis_stream.stream_arn],
actions=[
"kinesis:DescribeStream",
"kinesis:GetShardIterator",
"kinesis:GetRecords"
]))
firehose_log_group_name = "/aws/kinesisfirehose/octember-bizcard-txt-to-s3"
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
#XXX: The ARN will be formatted as follows:
# arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name}
resources=[
self.format_arn(service="logs",
resource="log-group",
resource_name="{}:log-stream:*".format(
firehose_log_group_name),
sep=":")
],
actions=["logs:PutLogEvents"]))
firehose_role = aws_iam.Role(
self,
"FirehoseDeliveryRole",
role_name="FirehoseDeliveryRole",
assumed_by=aws_iam.ServicePrincipal("firehose.amazonaws.com"),
#XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221
inline_policies={"firehose_role_policy": firehose_role_policy_doc})
bizcard_text_to_s3_delivery_stream = aws_kinesisfirehose.CfnDeliveryStream(
self,
"BizcardTextToS3",
delivery_stream_name="octember-bizcard-txt-to-s3",
delivery_stream_type="KinesisStreamAsSource",
kinesis_stream_source_configuration={
"kinesisStreamArn": text_kinesis_stream.stream_arn,
"roleArn": firehose_role.role_arn
},
extended_s3_destination_configuration={
"bucketArn": s3_bucket.bucket_arn,
"bufferingHints": {
"intervalInSeconds": 60,
"sizeInMBs": 1
},
"cloudWatchLoggingOptions": {
"enabled": True,
"logGroupName": firehose_log_group_name,
"logStreamName": "S3Delivery"
},
"compressionFormat": "GZIP",
"prefix": "bizcard-text/",
"roleArn": firehose_role.role_arn
})
sg_use_bizcard_es_cache = aws_ec2.SecurityGroup(
self,
"BizcardSearchCacheClientSG",
vpc=vpc,
allow_all_outbound=True,
description=
'security group for octember bizcard search query cache client',
security_group_name='use-octember-bizcard-es-cache')
core.Tags.of(sg_use_bizcard_es_cache).add(
'Name', 'use-octember-bizcard-es-cache')
sg_bizcard_es_cache = aws_ec2.SecurityGroup(
self,
"BizcardSearchCacheSG",
vpc=vpc,
allow_all_outbound=True,
description=
'security group for octember bizcard search query cache',
security_group_name='octember-bizcard-es-cache')
core.Tags.of(sg_bizcard_es_cache).add('Name',
'octember-bizcard-es-cache')
sg_bizcard_es_cache.add_ingress_rule(
peer=sg_use_bizcard_es_cache,
connection=aws_ec2.Port.tcp(6379),
description='use-octember-bizcard-es-cache')
es_query_cache_subnet_group = aws_elasticache.CfnSubnetGroup(
self,
"QueryCacheSubnetGroup",
description="subnet group for octember-bizcard-es-cache",
subnet_ids=vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids,
cache_subnet_group_name='octember-bizcard-es-cache')
es_query_cache = aws_elasticache.CfnCacheCluster(
self,
"BizcardSearchQueryCache",
cache_node_type="cache.t3.small",
num_cache_nodes=1,
engine="redis",
engine_version="5.0.5",
auto_minor_version_upgrade=False,
cluster_name="octember-bizcard-es-cache",
snapshot_retention_limit=3,
snapshot_window="17:00-19:00",
preferred_maintenance_window="mon:19:00-mon:20:30",
#XXX: Do not use referece for "cache_subnet_group_name" - https://github.com/aws/aws-cdk/issues/3098
#cache_subnet_group_name=es_query_cache_subnet_group.cache_subnet_group_name, # Redis cluster goes to wrong VPC
cache_subnet_group_name='octember-bizcard-es-cache',
vpc_security_group_ids=[sg_bizcard_es_cache.security_group_id])
#XXX: If you're going to launch your cluster in an Amazon VPC, you need to create a subnet group before you start creating a cluster.
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticache-cache-cluster.html#cfn-elasticache-cachecluster-cachesubnetgroupname
es_query_cache.add_depends_on(es_query_cache_subnet_group)
#XXX: add more than 2 security groups
# https://github.com/aws/aws-cdk/blob/ea10f0d141a48819ec0000cd7905feda993870a9/packages/%40aws-cdk/aws-lambda/lib/function.ts#L387
# https://github.com/aws/aws-cdk/issues/1555
# https://github.com/aws/aws-cdk/pull/5049
bizcard_search_lambda_fn = _lambda.Function(
self,
"BizcardSearchServer",
runtime=_lambda.Runtime.PYTHON_3_7,
function_name="BizcardSearchProxy",
handler="es_search_bizcard.lambda_handler",
description="Proxy server to search bizcard text",
code=_lambda.Code.asset("./src/main/python/SearchBizcard"),
environment={
'ES_HOST': es_cfn_domain.attr_domain_endpoint,
'ES_INDEX': 'octember_bizcard',
'ES_TYPE': 'bizcard',
'ELASTICACHE_HOST': es_query_cache.attr_redis_endpoint_address
},
timeout=core.Duration.minutes(1),
layers=[es_lib_layer, redis_lib_layer],
security_groups=[sg_use_bizcard_es, sg_use_bizcard_es_cache],
vpc=vpc)
#XXX: create API Gateway + LambdaProxy
search_api = apigw.LambdaRestApi(
self,
"BizcardSearchAPI",
handler=bizcard_search_lambda_fn,
proxy=False,
rest_api_name="BizcardSearch",
description="This service serves searching bizcard text.",
endpoint_types=[apigw.EndpointType.REGIONAL],
deploy=True,
deploy_options=apigw.StageOptions(stage_name="v1"))
bizcard_search = search_api.root.add_resource('search')
bizcard_search.add_method(
"GET",
method_responses=[
apigw.MethodResponse(
status_code="200",
response_models={'application/json': apigw.EmptyModel()}),
apigw.MethodResponse(status_code="400"),
apigw.MethodResponse(status_code="500")
])
sg_use_bizcard_graph_db = aws_ec2.SecurityGroup(
self,
"BizcardGraphDbClientSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for octember bizcard graph db client',
security_group_name='use-octember-bizcard-neptune')
core.Tags.of(sg_use_bizcard_graph_db).add(
'Name', 'use-octember-bizcard-neptune')
sg_bizcard_graph_db = aws_ec2.SecurityGroup(
self,
"BizcardGraphDbSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for octember bizcard graph db',
security_group_name='octember-bizcard-neptune')
core.Tags.of(sg_bizcard_graph_db).add('Name',
'octember-bizcard-neptune')
sg_bizcard_graph_db.add_ingress_rule(
peer=sg_bizcard_graph_db,
connection=aws_ec2.Port.tcp(8182),
description='octember-bizcard-neptune')
sg_bizcard_graph_db.add_ingress_rule(
peer=sg_use_bizcard_graph_db,
connection=aws_ec2.Port.tcp(8182),
description='use-octember-bizcard-neptune')
bizcard_graph_db_subnet_group = aws_neptune.CfnDBSubnetGroup(
self,
"NeptuneSubnetGroup",
db_subnet_group_description=
"subnet group for octember-bizcard-neptune",
subnet_ids=vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids,
db_subnet_group_name='octember-bizcard-neptune')
bizcard_graph_db = aws_neptune.CfnDBCluster(
self,
"BizcardGraphDB",
availability_zones=vpc.availability_zones,
db_subnet_group_name=bizcard_graph_db_subnet_group.
db_subnet_group_name,
db_cluster_identifier="octember-bizcard",
backup_retention_period=1,
preferred_backup_window="08:45-09:15",
preferred_maintenance_window="sun:18:00-sun:18:30",
vpc_security_group_ids=[sg_bizcard_graph_db.security_group_id])
bizcard_graph_db.add_depends_on(bizcard_graph_db_subnet_group)
bizcard_graph_db_instance = aws_neptune.CfnDBInstance(
self,
"BizcardGraphDBInstance",
db_instance_class="db.r5.large",
allow_major_version_upgrade=False,
auto_minor_version_upgrade=False,
availability_zone=vpc.availability_zones[0],
db_cluster_identifier=bizcard_graph_db.db_cluster_identifier,
db_instance_identifier="octember-bizcard",
preferred_maintenance_window="sun:18:00-sun:18:30")
bizcard_graph_db_instance.add_depends_on(bizcard_graph_db)
bizcard_graph_db_replica_instance = aws_neptune.CfnDBInstance(
self,
"BizcardGraphDBReplicaInstance",
db_instance_class="db.r5.large",
allow_major_version_upgrade=False,
auto_minor_version_upgrade=False,
availability_zone=vpc.availability_zones[-1],
db_cluster_identifier=bizcard_graph_db.db_cluster_identifier,
db_instance_identifier="octember-bizcard-replica",
preferred_maintenance_window="sun:18:00-sun:18:30")
bizcard_graph_db_replica_instance.add_depends_on(bizcard_graph_db)
bizcard_graph_db_replica_instance.add_depends_on(
bizcard_graph_db_instance)
gremlinpython_lib_layer = _lambda.LayerVersion(
self,
"GremlinPythonLib",
layer_version_name="gremlinpython-lib",
compatible_runtimes=[_lambda.Runtime.PYTHON_3_7],
code=_lambda.Code.from_bucket(
s3_lib_bucket, "var/octember-gremlinpython-lib.zip"))
#XXX: https://github.com/aws/aws-cdk/issues/1342
upsert_to_neptune_lambda_fn = _lambda.Function(
self,
"UpsertBizcardToGraphDB",
runtime=_lambda.Runtime.PYTHON_3_7,
function_name="UpsertBizcardToNeptune",
handler="upsert_bizcard_to_graph_db.lambda_handler",
description="Upsert bizcard into neptune",
code=_lambda.Code.asset(
"./src/main/python/UpsertBizcardToGraphDB"),
environment={
'REGION_NAME': core.Aws.REGION,
'NEPTUNE_ENDPOINT': bizcard_graph_db.attr_endpoint,
'NEPTUNE_PORT': bizcard_graph_db.attr_port
},
timeout=core.Duration.minutes(5),
layers=[gremlinpython_lib_layer],
security_groups=[sg_use_bizcard_graph_db],
vpc=vpc)
upsert_to_neptune_lambda_fn.add_event_source(text_kinesis_event_source)
log_group = aws_logs.LogGroup(
self,
"UpsertBizcardToGraphDBLogGroup",
log_group_name="/aws/lambda/UpsertBizcardToNeptune",
retention=aws_logs.RetentionDays.THREE_DAYS)
log_group.grant_write(upsert_to_neptune_lambda_fn)
sg_use_bizcard_neptune_cache = aws_ec2.SecurityGroup(
self,
"BizcardNeptuneCacheClientSG",
vpc=vpc,
allow_all_outbound=True,
description=
'security group for octember bizcard recommendation query cache client',
security_group_name='use-octember-bizcard-neptune-cache')
core.Tags.of(sg_use_bizcard_neptune_cache).add(
'Name', 'use-octember-bizcard-es-cache')
sg_bizcard_neptune_cache = aws_ec2.SecurityGroup(
self,
"BizcardNeptuneCacheSG",
vpc=vpc,
allow_all_outbound=True,
description=
'security group for octember bizcard recommendation query cache',
security_group_name='octember-bizcard-neptune-cache')
core.Tags.of(sg_bizcard_neptune_cache).add(
'Name', 'octember-bizcard-neptune-cache')
sg_bizcard_neptune_cache.add_ingress_rule(
peer=sg_use_bizcard_neptune_cache,
connection=aws_ec2.Port.tcp(6379),
description='use-octember-bizcard-neptune-cache')
recomm_query_cache_subnet_group = aws_elasticache.CfnSubnetGroup(
self,
"RecommQueryCacheSubnetGroup",
description="subnet group for octember-bizcard-neptune-cache",
subnet_ids=vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids,
cache_subnet_group_name='octember-bizcard-neptune-cache')
recomm_query_cache = aws_elasticache.CfnCacheCluster(
self,
"BizcardRecommQueryCache",
cache_node_type="cache.t3.small",
num_cache_nodes=1,
engine="redis",
engine_version="5.0.5",
auto_minor_version_upgrade=False,
cluster_name="octember-bizcard-neptune-cache",
snapshot_retention_limit=3,
snapshot_window="17:00-19:00",
preferred_maintenance_window="mon:19:00-mon:20:30",
#XXX: Do not use referece for "cache_subnet_group_name" - https://github.com/aws/aws-cdk/issues/3098
#cache_subnet_group_name=recomm_query_cache_subnet_group.cache_subnet_group_name, # Redis cluster goes to wrong VPC
cache_subnet_group_name='octember-bizcard-neptune-cache',
vpc_security_group_ids=[
sg_bizcard_neptune_cache.security_group_id
])
recomm_query_cache.add_depends_on(recomm_query_cache_subnet_group)
bizcard_recomm_lambda_fn = _lambda.Function(
self,
"BizcardRecommender",
runtime=_lambda.Runtime.PYTHON_3_7,
function_name="BizcardRecommender",
handler="neptune_recommend_bizcard.lambda_handler",
description="This service serves PYMK(People You May Know).",
code=_lambda.Code.asset("./src/main/python/RecommendBizcard"),
environment={
'REGION_NAME': core.Aws.REGION,
'NEPTUNE_ENDPOINT': bizcard_graph_db.attr_read_endpoint,
'NEPTUNE_PORT': bizcard_graph_db.attr_port,
'ELASTICACHE_HOST':
recomm_query_cache.attr_redis_endpoint_address
},
timeout=core.Duration.minutes(1),
layers=[gremlinpython_lib_layer, redis_lib_layer],
security_groups=[
sg_use_bizcard_graph_db, sg_use_bizcard_neptune_cache
],
vpc=vpc)
#XXX: create API Gateway + LambdaProxy
recomm_api = apigw.LambdaRestApi(
self,
"BizcardRecommendAPI",
handler=bizcard_recomm_lambda_fn,
proxy=False,
rest_api_name="BizcardRecommend",
description="This service serves PYMK(People You May Know).",
endpoint_types=[apigw.EndpointType.REGIONAL],
deploy=True,
deploy_options=apigw.StageOptions(stage_name="v1"))
bizcard_recomm = recomm_api.root.add_resource('pymk')
bizcard_recomm.add_method(
"GET",
method_responses=[
apigw.MethodResponse(
status_code="200",
response_models={'application/json': apigw.EmptyModel()}),
apigw.MethodResponse(status_code="400"),
apigw.MethodResponse(status_code="500")
])
sagemaker_notebook_role_policy_doc = aws_iam.PolicyDocument()
sagemaker_notebook_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"effect":
aws_iam.Effect.ALLOW,
"resources": [
"arn:aws:s3:::aws-neptune-notebook",
"arn:aws:s3:::aws-neptune-notebook/*"
],
"actions": ["s3:GetObject", "s3:ListBucket"]
}))
sagemaker_notebook_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"effect":
aws_iam.Effect.ALLOW,
"resources": [
"arn:aws:neptune-db:{region}:{account}:{cluster_id}/*".
format(region=core.Aws.REGION,
account=core.Aws.ACCOUNT_ID,
cluster_id=bizcard_graph_db.
attr_cluster_resource_id)
],
"actions": ["neptune-db:connect"]
}))
sagemaker_notebook_role = aws_iam.Role(
self,
'SageMakerNotebookForNeptuneWorkbenchRole',
role_name='AWSNeptuneNotebookRole-OctemberBizcard',
assumed_by=aws_iam.ServicePrincipal('sagemaker.amazonaws.com'),
#XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221
inline_policies={
'AWSNeptuneNotebook': sagemaker_notebook_role_policy_doc
})
neptune_wb_lifecycle_content = '''#!/bin/bash
sudo -u ec2-user -i <<'EOF'
echo "export GRAPH_NOTEBOOK_AUTH_MODE=DEFAULT" >> ~/.bashrc
echo "export GRAPH_NOTEBOOK_HOST={NeptuneClusterEndpoint}" >> ~/.bashrc
echo "export GRAPH_NOTEBOOK_PORT={NeptuneClusterPort}" >> ~/.bashrc
echo "export NEPTUNE_LOAD_FROM_S3_ROLE_ARN=''" >> ~/.bashrc
echo "export AWS_REGION={AWS_Region}" >> ~/.bashrc
aws s3 cp s3://aws-neptune-notebook/graph_notebook.tar.gz /tmp/graph_notebook.tar.gz
rm -rf /tmp/graph_notebook
tar -zxvf /tmp/graph_notebook.tar.gz -C /tmp
/tmp/graph_notebook/install.sh
EOF
'''.format(NeptuneClusterEndpoint=bizcard_graph_db.attr_endpoint,
NeptuneClusterPort=bizcard_graph_db.attr_port,
AWS_Region=core.Aws.REGION)
neptune_wb_lifecycle_config_prop = aws_sagemaker.CfnNotebookInstanceLifecycleConfig.NotebookInstanceLifecycleHookProperty(
content=core.Fn.base64(neptune_wb_lifecycle_content))
neptune_wb_lifecycle_config = aws_sagemaker.CfnNotebookInstanceLifecycleConfig(
self,
'NpetuneWorkbenchLifeCycleConfig',
notebook_instance_lifecycle_config_name=
'AWSNeptuneWorkbenchOctemberBizcardLCConfig',
on_start=[neptune_wb_lifecycle_config_prop])
neptune_workbench = aws_sagemaker.CfnNotebookInstance(
self,
'NeptuneWorkbench',
instance_type='ml.t2.medium',
role_arn=sagemaker_notebook_role.role_arn,
lifecycle_config_name=neptune_wb_lifecycle_config.
notebook_instance_lifecycle_config_name,
notebook_instance_name='OctemberBizcard-NeptuneWorkbench',
root_access='Disabled',
security_group_ids=[sg_use_bizcard_graph_db.security_group_name],
subnet_id=bizcard_graph_db_subnet_group.subnet_ids[0])
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# The code that defines your stack goes here
# VPC
self.vpc = ec2.Vpc(
self,
"VPC",
max_azs=2,
cidr="10.10.0.0/16",
subnet_configuration=[
ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.PUBLIC,
name="Public",
cidr_mask=24),
ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.PRIVATE,
name="Private",
cidr_mask=24)
],
nat_gateways=1)
# Security group
# self.sg = ec2.SecurityGroup(self, "securityGroup", self.vpc)
self.sg = ec2.SecurityGroup.from_security_group_id(
self,
"securityGroup",
self.vpc.vpc_default_security_group,
mutable=False)
# Create EFS inside VPC
self.efs = efs.FileSystem(
self,
"commonEFS4Notebooks",
vpc=self.vpc,
encrypted=True,
enable_automatic_backups=True,
performance_mode=efs.PerformanceMode('MAX_IO'),
throughput_mode=efs.ThroughputMode('BURSTING'),
security_group=self.sg)
# Mount target for EFS
# self.mount = efs.CfnMountTarget(
# self,
# "MountID",
# file_system_id=self.efs.file_system_id,security_groups=[self.sg.security_group_id,],
# subnet_id=self.vpc.private_subnets[0].subnet_id,
# )
# IAM Roles
#Create role for Notebook instance
nRole = iam_.Role(self,
"notebookAccessRole",
assumed_by=iam_.ServicePrincipal('sagemaker'))
nPolicy = iam_.Policy(self,
"notebookAccessPolicy",
policy_name="notebookAccessPolicy",
statements=[
iam_.PolicyStatement(actions=[
's3:*',
],
resources=[
'*',
]),
]).attach_to_role(nRole)
#Create notebook instances cluster
# print(self.mount.get_att('attr_ip_address').to_string())
encodedScript = LifecycleScriptStr.format(self.efs.file_system_id)
# print("Adding following script to the lifecycle config..\n___\n\n"+encodedScript)
code = [{"content": core.Fn.base64(encodedScript)}]
lifecycleconfig = sm.CfnNotebookInstanceLifecycleConfig(
self,
"LifeCycleConfig",
notebook_instance_lifecycle_config_name=LifeCycleConfigName,
on_create=None,
on_start=code)
instances = []
for i in range(num_instances):
nid = 'CDK-Notebook-Instance-User-' + str(i)
instances.append(
sm.CfnNotebookInstance(
self,
nid,
instance_type='ml.t2.medium',
volume_size_in_gb=5,
security_group_ids=[self.sg.security_group_id],
subnet_id=self.vpc.private_subnets[0].subnet_id,
notebook_instance_name=nid,
role_arn=nRole.role_arn,
lifecycle_config_name=lifecycleconfig.
notebook_instance_lifecycle_config_name))
core.CfnOutput(self, "VPC_id", value=self.vpc.vpc_id)
core.CfnOutput(self, "EFS_id", value=self.efs.file_system_id)
[
core.CfnOutput(self,
"NotebookInstance_" + str(c),
value=notebook.notebook_instance_name)
for c, notebook in enumerate(instances)
]
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# The code that defines your stack goes here
vpc_name = self.node.try_get_context('vpc_name')
vpc = aws_ec2.Vpc.from_lookup(self,
'ExistingVPC',
is_default=True,
vpc_name=vpc_name)
sg_use_docdb = aws_ec2.SecurityGroup(
self,
'DocDBClientSG',
vpc=vpc,
allow_all_outbound=True,
description='security group for documentdb client',
security_group_name='use-docdb-sg')
core.Tags.of(sg_use_docdb).add('Name', 'docdb-client-sg')
sg_docdb_server = aws_ec2.SecurityGroup(
self,
'DocDBServerSG',
vpc=vpc,
allow_all_outbound=True,
description='security group for documentdb',
security_group_name='{stack_name}-server-sg-docdb'.format(
stack_name=self.stack_name))
sg_docdb_server.add_ingress_rule(peer=sg_use_docdb,
connection=aws_ec2.Port.tcp(27017),
description='docdb-client-sg')
core.Tags.of(sg_docdb_server).add('Name', 'docdb-server-sg')
docdb_cluster_name = self.node.try_get_context('docdb_cluster_name')
docdb_cluster_name = docdb_cluster_name if docdb_cluster_name else self.stack_name
docdb_cluster = aws_docdb.DatabaseCluster(
self,
'DocDB',
db_cluster_name=docdb_cluster_name,
master_user=aws_docdb.Login(username='******'),
instance_props={
#'instance_type': aws_ec2.InstanceType('r5.xlarge'),
'instance_type':
aws_ec2.InstanceType.of(aws_ec2.InstanceClass.MEMORY5,
aws_ec2.InstanceSize.LARGE),
'vpc_subnets': {
'subnet_type': aws_ec2.SubnetType.PRIVATE
},
'vpc':
vpc,
'security_group':
sg_docdb_server
},
instances=3,
preferred_maintenance_window='sun:18:00-sun:18:30',
removal_policy=core.RemovalPolicy.RETAIN)
#[Warning at /docdb-sm/Database/RotationSingleUser/SecurityGroup] Ignoring Egress rule since 'allowAllOutbound' is set to true; To add customize rules, set allowAllOutbound=false on the SecurityGroup
#docdb_cluster.add_rotation_single_user()
sagemaker_notebook_role_policy_doc = aws_iam.PolicyDocument()
sagemaker_notebook_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"effect": aws_iam.Effect.ALLOW,
"resources": [docdb_cluster.secret.secret_full_arn],
"actions": ["secretsmanager:GetSecretValue"]
}))
sagemaker_notebook_role = aws_iam.Role(
self,
'SageMakerNotebookRoleForDocDB',
role_name='AWSSageMakerNotebookRoleForDocDB',
assumed_by=aws_iam.ServicePrincipal('sagemaker.amazonaws.com'),
inline_policies={
'DocumentDBSecretPolicy': sagemaker_notebook_role_policy_doc
})
docdb_wb_lifecycle_content = '''#!/bin/bash
sudo -u ec2-user -i <<'EOF'
echo "export AWS_REGION={AWS_Region}" >> ~/.bashrc
source /home/ec2-user/anaconda3/bin/activate python3
pip install --upgrade pymongo
pip install --upgrade xgboost
source /home/ec2-user/anaconda3/bin/deactivate
cd /home/ec2-user/SageMaker
wget https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem
wget https://raw.githubusercontent.com/aws-samples/documentdb-sagemaker-example/main/script.ipynb
EOF
'''.format(AWS_Region=core.Aws.REGION)
docdb_wb_lifecycle_config_prop = aws_sagemaker.CfnNotebookInstanceLifecycleConfig.NotebookInstanceLifecycleHookProperty(
content=core.Fn.base64(docdb_wb_lifecycle_content))
docdb_wb_lifecycle_config = aws_sagemaker.CfnNotebookInstanceLifecycleConfig(
self,
'DocDBWorkbenchLifeCycleConfig',
notebook_instance_lifecycle_config_name=
'DocDBWorkbenchLifeCycleConfig',
on_start=[docdb_wb_lifecycle_config_prop])
docdb_workbench = aws_sagemaker.CfnNotebookInstance(
self,
'DocDBWorkbench',
instance_type='ml.t3.xlarge',
role_arn=sagemaker_notebook_role.role_arn,
lifecycle_config_name=docdb_wb_lifecycle_config.
notebook_instance_lifecycle_config_name,
notebook_instance_name='DocDBWorkbench',
root_access='Disabled',
security_group_ids=[sg_use_docdb.security_group_name],
subnet_id=vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids[0])
core.CfnOutput(self,
'StackName',
value=self.stack_name,
export_name='StackName')
core.CfnOutput(self, 'VpcId', value=vpc.vpc_id, export_name='VpcId')
core.CfnOutput(self,
'DocumentDBClusterName',
value=docdb_cluster.cluster_identifier,
export_name='DocumentDBClusterName')
core.CfnOutput(self,
'DocumentDBCluster',
value=docdb_cluster.cluster_endpoint.socket_address,
export_name='DocumentDBCluster')
#XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_secretsmanager/README.html
# secret_arn="arn:aws:secretsmanager:<region>:<account-id-number>:secret:<secret-name>-<random-6-characters>",
core.CfnOutput(self,
'DocDBSecret',
value=docdb_cluster.secret.secret_name,
export_name='DocDBSecret')
core.CfnOutput(self,
'SageMakerRole',
value=sagemaker_notebook_role.role_name,
export_name='SageMakerRole')
core.CfnOutput(self,
'SageMakerNotebookInstance',
value=docdb_workbench.notebook_instance_name,
export_name='SageMakerNotebookInstance')
core.CfnOutput(self,
'SageMakerNotebookInstanceLifecycleConfig',
value=docdb_workbench.lifecycle_config_name,
export_name='SageMakerNotebookInstanceLifecycleConfig')
