def __create_cloud_front_lambda_execution_role(self) -> aws_iam.Role:
role = aws_iam.Role(
self,
'CloudFrontLambdaExecutionRole',
assumed_by=ServicePrincipal('lambda.amazonaws.com'),
path='/',
managed_policies=[
ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole')
])
role.assume_role_policy.add_statements(
PolicyStatement(
principals=[ServicePrincipal('edgelambda.amazonaws.com')],
actions=['sts:AssumeRole']))
return role
def __create_s3_trigger_lambda_execution_role(
self, bucket_name: str,
cloudfront_distribution: aws_cloudfront.Distribution
) -> aws_iam.Role:
return aws_iam.Role(
self,
"S3TriggerLambdaExecutionRole",
assumed_by=ServicePrincipal('lambda.amazonaws.com'),
description=
'Execution role that allows the lambda function to get the uploaded zip from S3, upload the '
'unpacked one and invalidate the CDN',
path='/',
managed_policies=[
ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole')
],
inline_policies={
's3_trigger_artifacts-upload-role':
PolicyDocument(statements=[
PolicyStatement(
effect=Effect.ALLOW,
actions=[
's3:PutObject', 's3:GetObject', 's3:ListObject',
's3:DeleteObject', 's3:HeadBucket',
'cloudfront:CreateInvalidation'
],
resources=[
f'arn:aws:cloudfront::{Fn.ref("AWS::AccountId")}:distribution/'
f'{cloudfront_distribution.distribution_id}',
f'arn:aws:s3:::{bucket_name}',
f'arn:aws:s3:::{bucket_name}/*'
])
])
})
def __role(self) -> Role:
"""
A role for custom resource which manages git commits to codecommit.
:return: Custom resource's role.
"""
return Role(
self.__stack,
self.__prefix + 'CiCdLambdaCustomCommitRole',
inline_policies={
self.__prefix + 'CiCdLambdaCustomCommitPolicy':
PolicyDocument(statements=[
PolicyStatement(
actions=[
"codecommit:CreateCommit",
],
resources=[self.__code_repository.repository_arn],
effect=Effect.ALLOW),
PolicyStatement(actions=[
"logs:CreateLogGroup", "logs:CreateLogStream",
"logs:PutLogEvents"
],
resources=['*'],
effect=Effect.ALLOW)
])
},
assumed_by=ServicePrincipal('lambda.amazonaws.com'))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
with open(INPUT_WORKFLOW_FILE) as f:
workflow_dict = json.load(f)
glue_role = Role(
self,
"dummy_glue_job_role",
assumed_by=ServicePrincipal("glue.amazonaws.com"),
managed_policies=[
ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSGlueServiceRole")
],
)
# mock job
job_command = CfnJob.JobCommandProperty(
name='pythonshell', script_location=SCRIPT_LOCATION)
for workflow_item in workflow_dict:
for node in workflow_item['nodes']:
triggered_job_name = node['id']
job = CfnJob(self,
triggered_job_name,
name=triggered_job_name,
command=job_command,
role=glue_role.role_name)
def init_lambda(self):
tmp_dir = install_lambda_code_requirements()
lambda_code = Code.from_asset(str(tmp_dir),
exclude=[
".env",
"__main*",
"*.dist-info",
"bin",
"requirements.txt",
])
lambda_function = Function(self,
"lambda",
code=lambda_code,
handler="main.handler",
runtime=Runtime.PYTHON_3_8)
lambda_function.role.assume_role_policy.add_statements(
PolicyStatement(
actions=["sts:AssumeRole"],
principals=[ServicePrincipal("edgelambda.amazonaws.com")]))
version = Version(self, "version", lambda_=lambda_function)
apply_removal_policy(lambda_function, version, lambda_function.role)
return version
def _create_lambdas(self):
clean_pycache()
for root, dirs, files in os.walk(LAMBDAS_DIR):
for f in files:
if f != "__init__.py":
continue
parent_folder = os.path.basename(os.path.dirname(root))
lambda_folder = os.path.basename(root)
name = f"{parent_folder}-{lambda_folder}"
lambda_config = self.lambdas_config[name]
layers = []
for layer_name in lambda_config["layers"]:
layers.append(self.layers[layer_name])
lambda_role = Role(
self,
f"{name}_role",
assumed_by=ServicePrincipal(service="lambda.amazonaws.com")
)
for policy in lambda_config["policies"]:
lambda_role.add_to_policy(policy)
lambda_role.add_managed_policy(
ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole"))
lambda_args = {
"code": Code.from_asset(root),
"handler": "__init__.handle",
"runtime": Runtime.PYTHON_3_8,
"layers": layers,
"function_name": name,
"environment": lambda_config["variables"],
"role": lambda_role,
"timeout": Duration.seconds(lambda_config["timeout"]),
"memory_size": lambda_config["memory"],
}
if "concurrent_executions" in lambda_config:
lambda_args["reserved_concurrent_executions"] = lambda_config["concurrent_executions"]
self.lambdas[name] = Function(self, name, **lambda_args)
self.lambdas["sqs_handlers-post_anime"].add_event_source(SqsEventSource(self.post_anime_queue))
Rule(
self,
"titles_updater",
schedule=Schedule.cron(hour="2", minute="10"),
targets=[LambdaFunction(self.lambdas["crons-titles_updater"])]
)
Rule(
self,
"episodes_updater",
schedule=Schedule.cron(hour="4", minute="10"),
targets=[LambdaFunction(self.lambdas["crons-episodes_updater"])]
)
def __init__(self, scope: Construct, _id: str, **kwargs) -> None:
super().__init__(scope, _id, **kwargs)
task_definition = FargateTaskDefinition(
self,
'TaskDefinition',
cpu=256,
memory_limit_mib=512,
execution_role=Role(
self,
'ExecutionRole',
assumed_by=cast(IPrincipal, ServicePrincipal('ecs-tasks.amazonaws.com'))
),
task_role=Role(
self,
'TaskRole',
assumed_by=cast(IPrincipal, ServicePrincipal('ecs-tasks.amazonaws.com')),
managed_policies=[
ManagedPolicy.from_aws_managed_policy_name('AmazonSESFullAccess')
]
)
)
task_definition.add_container(
'Container',
image=ContainerImage.from_asset(
getcwd(),
file='Dockerfile',
repository_name='jqbx-bot',
exclude=['cdk.out']
),
command=['pipenv', 'run', 'python', '-u', '-m', 'src.main'],
environment={
'SPOTIFY_USER_ID': environ.get('SPOTIFY_USER_ID'),
'JQBX_ROOM_ID': environ.get('JQBX_ROOM_ID'),
'JQBX_BOT_DISPLAY_NAME': environ.get('JQBX_BOT_DISPLAY_NAME'),
'JQBX_BOT_IMAGE_URL': environ.get('JQBX_BOT_IMAGE_URL'),
'DATA_SERVICE_BASE_URL': environ.get('DATA_SERVICE_BASE_URL')
},
logging=AwsLogDriver(
stream_prefix='jqbx-bot',
log_group=LogGroup(self, 'LogGroup')
)
)
cluster = Cluster(self, '%sCluster' % _id)
FargateService(self, '%sService' % _id, cluster=cluster, task_definition=task_definition, desired_count=1)
def create_role(self):
service_principal = ServicePrincipal('lambda.amazonaws.com')
return Role(
self,
'Role',
assumed_by=service_principal,
role_name='sfn_lambda_role',
managed_policies=[self.managed_policy],
)
def grant_secrets_manager_access_to_lambda(self, rotator_lambda):
"""
Adds a resource based policy to the lambda used for rotation
which gives secrets manager service access to invoke the lambda.
Documentation can be found here: https://docs.aws.amazon.com/secretsmanager/latest/userguide/troubleshoot_rotation.html#tshoot-lambda-initialconfig-perms
Args:
rotator_lambda (Function): The lambda function used for rotating a secret
"""
service_principal = ServicePrincipal(
service='secretsmanager.amazonaws.com')
rotator_lambda.add_permission('invoke_access_to_secrets_manager',
principal=service_principal)
def __init__(self, scope: Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
global g_lambda_edge
Tags.of(self).add("Stack", "Common-Lambda-Edge")
self._role = Role(
self,
"EdgeLambdaRole",
assumed_by=CompositePrincipal(
ServicePrincipal("lambda.amazonaws.com"),
ServicePrincipal("edgelambda.amazonaws.com"),
),
managed_policies=[
ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSLambdaBasicExecutionRole"),
],
)
if g_lambda_edge is not None:
raise Exception("Only a single LambdaEdgeStack instance can exist")
g_lambda_edge = self
def create_default_infrastructure_config(
self, construct_id: str) -> CfnInfrastructureConfiguration:
"""
Create the default infrastructure config, which defines the permissions needed by Image Builder during
image creation.
"""
image_builder_role_name = f"DeadlineMachineImageBuilderRole{construct_id}"
image_builder_role = Role(
self,
image_builder_role_name,
assumed_by=ServicePrincipal("ec2.amazonaws.com"),
role_name=image_builder_role_name)
image_builder_role.add_managed_policy(
ManagedPolicy.from_aws_managed_policy_name(
'EC2InstanceProfileForImageBuilder'))
image_builder_role.add_managed_policy(
ManagedPolicy.from_aws_managed_policy_name(
'AmazonSSMManagedInstanceCore'))
image_builder_role.add_to_policy(
PolicyStatement(actions=[
's3:Get*',
's3:List*',
],
resources=['arn:aws:s3:::thinkbox-installers/*']))
image_builder_profile_name = f"DeadlineMachineImageBuilderPolicy{construct_id}"
image_builder_profile = CfnInstanceProfile(
self,
image_builder_profile_name,
instance_profile_name=image_builder_profile_name,
roles=[image_builder_role_name])
image_builder_profile.add_depends_on(
image_builder_role.node.default_child)
infrastructure_configuration = CfnInfrastructureConfiguration(
self,
f"InfrastructureConfig{construct_id}",
name=f"DeadlineInfrastructureConfig{construct_id}",
instance_profile_name=image_builder_profile_name)
infrastructure_configuration.add_depends_on(image_builder_profile)
return infrastructure_configuration
def _create_lambdas(self):
for root, dirs, files in os.walk(LAMBDAS_DIR):
for f in files:
if f != "__init__.py":
continue
parent_folder = os.path.basename(os.path.dirname(root))
lambda_folder = os.path.basename(root)
name = f"{parent_folder}-{lambda_folder}"
lambda_config = self.lambdas_config[name]
layers = []
for layer_name in lambda_config["layers"]:
layers.append(self.layers[layer_name])
lambda_role = Role(self,
f"{name}_role",
assumed_by=ServicePrincipal(
service="lambda.amazonaws.com"))
for policy in lambda_config["policies"]:
lambda_role.add_to_policy(policy)
lambda_role.add_managed_policy(
ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSLambdaBasicExecutionRole"))
self.lambdas[name] = Function(
self,
name,
code=Code.from_asset(root),
handler="__init__.handle",
runtime=Runtime.PYTHON_3_8,
layers=layers,
function_name=name,
environment=lambda_config["variables"],
role=lambda_role,
timeout=Duration.seconds(lambda_config["timeout"]),
memory_size=lambda_config["memory"],
)
Rule(self,
"update_eps",
schedule=Schedule.cron(hour="2", minute="10"),
targets=[LambdaFunction(self.lambdas["cron-update_eps"])])
def role(self) -> Role:
inline_policies = {
"ElasticsearchIndexPrividerFunctionElasticsearchAccessPolicy":
PolicyDocument(statements=[
PolicyStatement(
# TODO restrict this to appropriate API calls.
actions=["es:*"],
resources=["*"],
)
]),
}
return Role(
scope=self.__scope,
id=f"{self.__name}ElasticsearchIndexResourceProviderRole",
assumed_by=ServicePrincipal("lambda.amazonaws.com"),
description=
f"A role for ElasticsearchIndexResourceProvider lambda function.",
inline_policies=inline_policies,
)
def __init__(self, scope: cdk.Construct, construct_id: str, config, vpc: IVpc, instance: IInstance, neo4j_user_secret: ISecret, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
bucket = Bucket(self, "s3-bucket-altimeter",
bucket_name=config["s3_bucket"],
encryption=BucketEncryption.UNENCRYPTED, #.S3_MANAGED, # Disable encryption since it's not really required and it conflicts with SCP guardrails set by Control Tower on the Audit account.
block_public_access=BlockPublicAccess.BLOCK_ALL
)
cluster = Cluster(self, "ecs-cluster-altimeter",
cluster_name="ecsclstr-altimeter--default",
vpc=vpc
)
task_role = Role(self, "iam-role-altimeter-task-role",
assumed_by=ServicePrincipal("ecs-tasks.amazonaws.com"),
# It appears that within the account where the scanner is running, the task role is (partially) used for scanning resources (rather than the altimeter-scanner-access role).
managed_policies=[
ManagedPolicy.from_aws_managed_policy_name('SecurityAudit'),
ManagedPolicy.from_aws_managed_policy_name('job-function/ViewOnlyAccess')
]
)
task_definition = FargateTaskDefinition(self, "ecs-fgtd-altimeter",
task_role=task_role,
memory_limit_mib=self.MEMORY_LIMIT,
cpu=self.CPU
)
docker_path = os.path.join(os.path.curdir,"..")
image_asset = DockerImageAsset(self, 'ecr-assets-dia-altimeter',
directory=docker_path,
file="scanner.Dockerfile"
)
task_definition.add_container("ecs-container-altimeter",
image= ContainerImage.from_docker_image_asset(image_asset),
# memory_limit_mib=self.MEMORY_LIMIT,
# cpu=self.CPU,
environment= {
"CONFIG_PATH": config["altimeter_config_path"],
"S3_BUCKET": config["s3_bucket"]
},
logging= AwsLogDriver(
stream_prefix= 'altimeter',
log_retention= RetentionDays.TWO_WEEKS
)
)
task_definition.add_to_task_role_policy(PolicyStatement(
resources=["arn:aws:iam::*:role/"+config["account_execution_role"]],
actions=['sts:AssumeRole']
))
task_definition.add_to_task_role_policy(PolicyStatement(
resources=[
"arn:aws:s3:::"+config["s3_bucket"],
"arn:aws:s3:::"+config["s3_bucket"]+"/*"
],
actions=["s3:GetObject*",
"s3:GetBucket*",
"s3:List*",
"s3:DeleteObject*",
"s3:PutObject",
"s3:Abort*",
"s3:PutObjectTagging"]
))
# Grant the ability to record the stdout to CloudWatch Logs
# TODO: Refine
task_definition.add_to_task_role_policy(PolicyStatement(
resources=["*"],
actions=['logs:*']
))
# Trigger task every 24 hours
Rule(self, "events-rule-altimeter-daily-scan",
rule_name="evrule--altimeter-daily-scan",
schedule=Schedule.cron(hour="0", minute="0"),
description="Daily altimeter scan",
targets=[EcsTask(
task_definition=task_definition,
cluster=cluster,
subnet_selection=SubnetSelection(subnet_type=SubnetType.PRIVATE)
)]
)
# Trigger task manually via event
Rule(self, "events-rule-altimeter-manual-scan",
rule_name="evrule--altimeter-manual-scan",
event_pattern=EventPattern(source=['altimeter']),
description="Manual altimeter scan",
targets=[EcsTask(
task_definition=task_definition,
cluster=cluster,
subnet_selection=SubnetSelection(subnet_type=SubnetType.PRIVATE)
)]
)
# Don't put Neo4j Importer lambda in a separate stack since it causes a circular reference with the S3 event source, and using an imported bucket as event source is not possible (you need a Bucket, not an IBucket)
neo4j_importer_function = PythonFunction(self, 'lambda-function-neo4j-importer',
function_name="function-altimeter--neo4j-importer",
entry="../neo4j-importer",
index="app.py",
handler="lambda_handler",
runtime=Runtime.PYTHON_3_8,
memory_size=256,
timeout=cdk.Duration.seconds(60),
vpc=vpc,
vpc_subnets=SubnetSelection(subnets=vpc.select_subnets(subnet_group_name='Private').subnets),
environment={
"neo4j_address": instance.instance_private_ip,
"neo4j_user_secret_name": neo4j_user_secret.secret_name
}
)
neo4j_importer_function.add_event_source(
S3EventSource(bucket,
events= [EventType.OBJECT_CREATED, EventType.OBJECT_REMOVED],
filters= [ { "prefix": "raw/", "suffix": ".rdf"}]
)
)
# Grant lambda read/write access to the S3 bucket for reading raw rdf, writing prepared rdf and generating signed uri
bucket.grant_read_write(neo4j_importer_function.role)
# Grant lambda read access to the neo4j user secret
neo4j_user_secret.grant_read(neo4j_importer_function.role)
cfn role
GetRole
CreateRole
"""
ENDPOINT_SUFFIX = ".amazonaws.com"
API_GATEWAY_ENDPOINT = "apigateway{}".format(ENDPOINT_SUFFIX)
LAMBDA_ENDPOINT = "lambda{}".format(ENDPOINT_SUFFIX)
MINIMAL_FUNCTION_POLICY_STATEMENT = PolicyStatement(
actions=MINIMAL_FUNCTION_ACTIONS, resources=["*"])
MINIMAL_API_POLICY_STATEMENT = PolicyStatement(
actions=MINIMAL_FUNCTION_ACTIONS,
resources=["*"],
principals=[ServicePrincipal(API_GATEWAY_ENDPOINT)],
)
PUBLIC_INVOKE_POLICY_STATEMENT = PolicyStatement(
actions=[API_INVOKE_STATEMENT_ACTION],
resources=["*"],
principals=[Anyone()])
MINIMAL_PUBLIC_API_POLICY_DOCUMENT = PolicyDocument(
statements=[MINIMAL_API_POLICY_STATEMENT, PUBLIC_INVOKE_POLICY_STATEMENT])
def get_ddb_function_statement(table_arns):
return PolicyStatement(actions=MINIMAL_FUNCTION_ACTIONS + DDB_ACTIONS,
resources=table_arns)
def __init__(self,
scope: Construct,
id: str,
elasticsearch_index: ElasticsearchIndexResource,
dynamodb_table: Table,
kms_key: Optional[Key] = None,
*,
sagemaker_endpoint_name: str = None,
sagemaker_endpoint_arn: str = None,
sagemaker_embeddings_key: str = None) -> None:
super().__init__(scope=scope, id=id)
elasticsearch_layer = BElasticsearchLayer(
scope=self, name=f"{id}ElasticsearchLayer")
if bool(sagemaker_endpoint_name) ^ bool(sagemaker_embeddings_key):
raise ValueError(
f'In order to use sentence embedding, all of the following enviroment variables are required: '
f'SAGEMAKER_ENDPOINT_NAME, SAGEMAKER_EMBEDDINGS_KEY. '
f'Else, provide none of above.')
if sagemaker_endpoint_name and not sagemaker_endpoint_arn:
sagemaker_endpoint_arn = self.__resolve_sagemaker_endpoints_arn(
'*')
optional_sagemaker_parameters = {
'SAGEMAKER_ENDPOINT_NAME': sagemaker_endpoint_name or None,
'SAGEMAKER_EMBEDDINGS_KEY': sagemaker_embeddings_key or None
}
initial_cloner_function = SingletonFunction(
scope=self,
id='InitialClonerFunction',
uuid='e01116a4-f939-43f2-8f5b-cc9f862c9e01',
lambda_purpose='InitialClonerSingletonLambda',
code=Code.from_asset(initial_cloner_root),
handler='index.handler',
runtime=Runtime.PYTHON_3_8,
layers=[elasticsearch_layer],
log_retention=RetentionDays.ONE_MONTH,
memory_size=128,
timeout=Duration.minutes(15),
role=Role(
scope=self,
id='InitialClonerFunctionRole',
assumed_by=ServicePrincipal('lambda.amazonaws.com'),
inline_policies={
'LogsPolicy':
PolicyDocument(statements=[
PolicyStatement(
actions=[
'logs:CreateLogGroup',
'logs:CreateLogStream',
'logs:PutLogEvents',
'logs:DescribeLogStreams',
],
resources=['arn:aws:logs:*:*:*'],
effect=Effect.ALLOW,
)
]),
'ElasticsearchPolicy':
PolicyDocument(statements=[
PolicyStatement(
actions=[
'es:ESHttpDelete',
'es:ESHttpGet',
'es:ESHttpHead',
'es:ESHttpPatch',
'es:ESHttpPost',
'es:ESHttpPut',
],
resources=['*'],
effect=Effect.ALLOW,
)
]),
'DynamodbPolicy':
PolicyDocument(statements=[
PolicyStatement(
actions=['dynamodb:*'],
resources=['*'],
effect=Effect.ALLOW,
)
]),
},
description='Role for DynamoDB Initial Cloner Function',
),
)
if kms_key:
initial_cloner_function.add_to_role_policy(
PolicyStatement(
actions=['kms:Decrypt'],
resources=[kms_key.key_arn],
effect=Effect.ALLOW,
), )
initial_cloner = CustomResource(
scope=self,
id='InitialCloner',
service_token=initial_cloner_function.function_arn,
removal_policy=RemovalPolicy.DESTROY,
properties={
'DynamodbTableName':
dynamodb_table.table_name,
'ElasticsearchIndexName':
elasticsearch_index.index_name,
'ElasticsearchEndpoint':
elasticsearch_index.elasticsearch_domain.domain_endpoint,
},
resource_type='Custom::ElasticsearchInitialCloner',
)
primary_key_field = initial_cloner.get_att_string('PrimaryKeyField')
dynamodb_stream_arn = dynamodb_table.table_stream_arn
if not dynamodb_stream_arn:
raise Exception('DynamoDB streams must be enabled for the table')
dynamodb_event_source = DynamoEventSource(
table=dynamodb_table,
starting_position=StartingPosition.LATEST,
enabled=True,
max_batching_window=Duration.seconds(10),
bisect_batch_on_error=True,
parallelization_factor=2,
batch_size=1000,
retry_attempts=10,
)
cloner_inline_policies = {
'LogsPolicy':
PolicyDocument(statements=[
PolicyStatement(
actions=[
'logs:CreateLogGroup',
'logs:CreateLogStream',
'logs:PutLogEvents',
'logs:DescribeLogStreams',
],
resources=['arn:aws:logs:*:*:*'],
effect=Effect.ALLOW,
)
]),
'ElasticsearchPolicy':
PolicyDocument(statements=[
PolicyStatement(
actions=[
'es:ESHttpDelete',
'es:ESHttpGet',
'es:ESHttpHead',
'es:ESHttpPatch',
'es:ESHttpPost',
'es:ESHttpPut',
],
resources=[
f'{elasticsearch_index.elasticsearch_domain.domain_arn}/*'
],
effect=Effect.ALLOW,
)
]),
'DynamodbStreamsPolicy':
PolicyDocument(statements=[
PolicyStatement(
actions=[
'dynamodb:DescribeStream',
'dynamodb:GetRecords',
'dynamodb:GetShardIterator',
'dynamodb:ListStreams',
],
resources=[dynamodb_stream_arn],
effect=Effect.ALLOW,
)
]),
}
if sagemaker_endpoint_arn:
cloner_inline_policies['SagemakerPolicy'] = PolicyDocument(
statements=[
PolicyStatement(actions=['sagemaker:InvokeEndpoint'],
resources=[sagemaker_endpoint_arn],
effect=Effect.ALLOW)
])
cloner_function = Function(
scope=self,
id='ClonerFunction',
code=Code.from_asset(cloner_root),
handler='index.handler',
runtime=Runtime.PYTHON_3_8,
environment={
'ES_INDEX_NAME': elasticsearch_index.index_name,
'ES_DOMAIN_ENDPOINT':
elasticsearch_index.elasticsearch_domain.domain_endpoint,
'PRIMARY_KEY_FIELD': primary_key_field,
**{
k: optional_sagemaker_parameters[k]
for k in optional_sagemaker_parameters if all(optional_sagemaker_parameters.values(
))
}
},
events=[dynamodb_event_source],
layers=[elasticsearch_layer],
log_retention=RetentionDays.ONE_MONTH,
memory_size=128,
role=Role(
scope=self,
id='ClonerFunctionRole',
assumed_by=ServicePrincipal('lambda.amazonaws.com'),
inline_policies=cloner_inline_policies,
description='Role for DynamoDB Cloner Function',
),
timeout=Duration.seconds(30),
)
if kms_key:
cloner_function.add_to_role_policy(
PolicyStatement(
actions=['kms:Decrypt'],
resources=[kms_key.key_arn],
effect=Effect.ALLOW,
))
def __init__(self,
scope: Stack,
id: str,
capacity: Optional[AddCapacityOptions] = None,
cluster_name: Optional[str] = None,
container_insights: Optional[bool] = None,
default_cloud_map_namespace: Optional[
CloudMapNamespaceOptions] = None,
vpc: Optional[IVpc] = None,
**kwargs) -> None:
known_args = dict(
scope=scope,
id=id,
capacity=capacity,
cluster_name=cluster_name,
container_insights=container_insights,
default_cloud_map_namespace=default_cloud_map_namespace,
vpc=vpc)
unknown_args = kwargs
super().__init__(**{**known_args, **unknown_args})
self.__role = Role(
scope=scope,
id=cluster_name + 'CustomResourceRole',
role_name=cluster_name + 'CustomResourceRole',
assumed_by=CompositePrincipal(
ServicePrincipal("lambda.amazonaws.com"),
ServicePrincipal("cloudformation.amazonaws.com")),
inline_policies={
cluster_name + 'CustomResourcePolicy':
PolicyDocument(statements=[
PolicyStatement(actions=[
"ecs:ListClusters",
"ecs:ListContainerInstances",
"ecs:ListServices",
"ecs:ListTaskDefinitions",
"ecs:ListTasks",
"ecs:DescribeClusters",
"ecs:DescribeContainerInstances",
"ecs:DescribeServices",
"ecs:DescribeTaskDefinition",
"ecs:DescribeTasks",
"ecs:CreateCluster",
"ecs:DeleteCluster",
"ecs:DeleteService",
"ecs:DeregisterContainerInstance",
"ecs:DeregisterTaskDefinition",
"ecs:StopTask",
"ecs:UpdateService",
],
effect=Effect.ALLOW,
resources=['*']),
PolicyStatement(actions=[
"logs:CreateLogGroup", "logs:CreateLogStream",
"logs:PutLogEvents"
],
effect=Effect.ALLOW,
resources=['*']),
])
},
managed_policies=[])
self.__custom_backend = Function(
scope=scope,
id=cluster_name + 'Deleter',
code=Code.from_asset(path=package_root),
handler='index.handler',
runtime=Runtime.PYTHON_3_6,
description=
f'A custom resource backend to delete ecs cluster ({cluster_name}) in the right way.',
function_name=cluster_name + 'Deleter',
memory_size=128,
role=self.__role,
timeout=Duration.seconds(900),
)
# noinspection PyTypeChecker
provider: ICustomResourceProvider = CustomResourceProvider.from_lambda(
self.__custom_backend)
self.__custom_resource = CustomResource(
scope=scope,
id=cluster_name + 'CustomResource',
provider=provider,
removal_policy=RemovalPolicy.DESTROY,
properties={'clusterName': cluster_name},
resource_type='Custom::EmptyS3Bucket')
# Make sure that custom resource is deleted before lambda function backend.
self.__custom_resource.node.add_dependency(self.__custom_backend)
# Make sure that custom resource is deleted before the bucket.
self.__custom_resource.node.add_dependency(self)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
table_name = 'items'
items_graphql_api = CfnGraphQLApi(self,
'ItemsApi',
name='items-api',
authentication_type='API_KEY')
CfnApiKey(self, 'ItemsApiKey', api_id=items_graphql_api.attr_api_id)
api_schema = CfnGraphQLSchema(self,
'ItemsSchema',
api_id=items_graphql_api.attr_api_id,
definition=f"""\
type {table_name} {{
{table_name}Id: ID!
name: String
}}
type Paginated{table_name} {{
items: [{table_name}!]!
nextToken: String
}}
type Query {{
all(limit: Int, nextToken: String): Paginated{table_name}!
getOne({table_name}Id: ID!): {table_name}
}}
type Mutation {{
save(name: String!): {table_name}
delete({table_name}Id: ID!): {table_name}
}}
type Schema {{
query: Query
mutation: Mutation
}}""")
items_table = Table(
self,
'ItemsTable',
table_name=table_name,
partition_key=Attribute(name=f'{table_name}Id',
type=AttributeType.STRING),
billing_mode=BillingMode.PAY_PER_REQUEST,
stream=StreamViewType.NEW_IMAGE,
# The default removal policy is RETAIN, which means that cdk
# destroy will not attempt to delete the new table, and it will
# remain in your account until manually deleted. By setting the
# policy to DESTROY, cdk destroy will delete the table (even if it
# has data in it)
removal_policy=core.RemovalPolicy.
DESTROY # NOT recommended for production code
)
items_table_role = Role(
self,
'ItemsDynamoDBRole',
assumed_by=ServicePrincipal('appsync.amazonaws.com'))
items_table_role.add_managed_policy(
ManagedPolicy.from_aws_managed_policy_name(
'AmazonDynamoDBFullAccess'))
data_source = CfnDataSource(
self,
'ItemsDataSource',
api_id=items_graphql_api.attr_api_id,
name='ItemsDynamoDataSource',
type='AMAZON_DYNAMODB',
dynamo_db_config=CfnDataSource.DynamoDBConfigProperty(
table_name=items_table.table_name, aws_region=self.region),
service_role_arn=items_table_role.role_arn)
get_one_resolver = CfnResolver(
self,
'GetOneQueryResolver',
api_id=items_graphql_api.attr_api_id,
type_name='Query',
field_name='getOne',
data_source_name=data_source.name,
request_mapping_template=f"""\
{{
"version": "2017-02-28",
"operation": "GetItem",
"key": {{
"{table_name}Id": $util.dynamodb.toDynamoDBJson($ctx.args.{table_name}Id)
}}
}}""",
response_mapping_template="$util.toJson($ctx.result)")
get_one_resolver.add_depends_on(api_schema)
get_all_resolver = CfnResolver(
self,
'GetAllQueryResolver',
api_id=items_graphql_api.attr_api_id,
type_name='Query',
field_name='all',
data_source_name=data_source.name,
request_mapping_template=f"""\
{{
"version": "2017-02-28",
"operation": "Scan",
"limit": $util.defaultIfNull($ctx.args.limit, 20),
"nextToken": $util.toJson($util.defaultIfNullOrEmpty($ctx.args.nextToken, null))
}}""",
response_mapping_template="$util.toJson($ctx.result)")
get_all_resolver.add_depends_on(api_schema)
save_resolver = CfnResolver(
self,
'SaveMutationResolver',
api_id=items_graphql_api.attr_api_id,
type_name='Mutation',
field_name='save',
data_source_name=data_source.name,
request_mapping_template=f"""\
{{
"version": "2017-02-28",
"operation": "PutItem",
"key": {{
"{table_name}Id": {{ "S": "$util.autoId()" }}
}},
"attributeValues": {{
"name": $util.dynamodb.toDynamoDBJson($ctx.args.name)
}}
}}""",
response_mapping_template="$util.toJson($ctx.result)")
save_resolver.add_depends_on(api_schema)
delete_resolver = CfnResolver(
self,
'DeleteMutationResolver',
api_id=items_graphql_api.attr_api_id,
type_name='Mutation',
field_name='delete',
data_source_name=data_source.name,
request_mapping_template=f"""\
{{
"version": "2017-02-28",
"operation": "DeleteItem",
"key": {{
"{table_name}Id": $util.dynamodb.toDynamoDBJson($ctx.args.{table_name}Id)
}}
}}""",
response_mapping_template="$util.toJson($ctx.result)")
delete_resolver.add_depends_on(api_schema)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# s3
s3_bucket_name = "{}-s3-{}".format(Constant.PROJECT_NAME,
self._get_UUID(4))
_s3.Bucket(
self,
id=s3_bucket_name,
bucket_name=s3_bucket_name,
removal_policy=core.RemovalPolicy.
DESTROY, #TODO: destroy for test
# removal_policy=core.RemovalPolicy.RETAIN
)
# step 1. VPC
# 如果在已有的Vpc 中建立环境， 可以用下面这句， 需要传入 vpc_id
# vpc = ec2.Vpc.from_lookup(self, "VPC", vpc_id='')
vpc = ec2.Vpc(
self,
"VPC",
max_azs=2, # 两个分区， 每个分区建一个子网
cidr="10.10.0.0/16",
# configuration will create 3 groups in 2 AZs = 6 subnets.
subnet_configuration=[
ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.PUBLIC,
name="Public",
cidr_mask=24),
ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.PRIVATE,
name="Private",
cidr_mask=24),
# ec2.SubnetConfiguration(
# subnet_type=ec2.SubnetType.ISOLATED,
# name="DB",
# cidr_mask=24
# )
],
# nat_gateway_provider=ec2.NatProvider.gateway(),
# nat_gateways=2,
)
# ES 需要部署到私有子网中
selection = vpc.select_subnets(subnet_type=ec2.SubnetType.PRIVATE)
# step 2. 访问S3 + ES集群需要的 iam_instance_profile
# action -> statement -> policy -> role -> instance profile -> attach ec2
actions = [
"ec2:CreateNetworkInterface", "ec2:DeleteNetworkInterface",
"ec2:DescribeNetworkInterfaces",
"ec2:ModifyNetworkInterfaceAttribute",
"ec2:DescribeSecurityGroups", "ec2:DescribeSubnets",
"ec2:DescribeVpcs", "s3:*"
]
policyStatement = PolicyStatement(actions=actions, effect=Effect.ALLOW)
policyStatement.add_all_resources()
policyStatement.sid = "Stmt1480452973134"
policy_name = "{}-ec2-es-policy".format(Constant.PROJECT_NAME)
ec2_policy = Policy(self, policy_name, policy_name=policy_name)
ec2_policy.add_statements(policyStatement)
role_name = "{}-ec2-es-role".format(Constant.PROJECT_NAME)
access_es_role = Role(
self,
role_name,
role_name=role_name,
assumed_by=ServicePrincipal('ec2.amazonaws.com.cn'))
ec2_policy.attach_to_role(access_es_role)
profile_name = "{}-ec2-es-profile".format(Constant.PROJECT_NAME)
instance_profile = CfnInstanceProfile(
self,
profile_name,
instance_profile_name=profile_name,
roles=[access_es_role.role_name])
# step 4. ES
# 生产环境建议设置安全组， 只接收VPC内443端口请求
sg_es_cluster_name = "{}-sg-es".format(Constant.PROJECT_NAME)
sg_es_cluster = ec2.SecurityGroup(
self,
id=sg_es_cluster_name,
vpc=vpc,
security_group_name=sg_es_cluster_name)
sg_es_cluster.add_ingress_rule(peer=ec2.Peer.ipv4(vpc.vpc_cidr_block),
connection=ec2.Port.tcp(443))
es_name = Constant.PROJECT_NAME
es_arn = self.format_arn(service="es",
resource="domain",
sep="/",
resource_name=es_name)
es = elasticsearch.CfnDomain(
self,
es_name,
elasticsearch_version='7.1',
domain_name=es_name,
node_to_node_encryption_options={"enabled": False},
vpc_options={
"securityGroupIds": [sg_es_cluster.security_group_id
], # 生产环境建议设置安全组， 只接收VPC内443端口请求
# 如果开启多个节点， 需要配置多个子网， 目前测试只有一个ES 节点， 就只用到一个子网
"subnetIds": selection.subnet_ids[:1]
},
ebs_options={
"ebsEnabled": True,
"volumeSize": 10,
"volumeType": "gp2"
},
elasticsearch_cluster_config={
# 生成环境需要开启三个
# "dedicatedMasterCount": 3,
# "dedicatedMasterEnabled": True,
# "dedicatedMasterType": 'm4.large.elasticsearch',
"instanceCount": 1,
"instanceType": 'm4.large.elasticsearch',
"zoneAwarenessEnabled": False
})
es.access_policies = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": "es:*",
"Resource": "{}/*".format(es_arn)
}]
}
# step 5. SNS
topic = sns.Topic(self, "topic")
topic.add_subscription(subs.EmailSubscription(Constant.EMAIL_ADDRESS))
# 设置SNS endpoint, 让lambda 可以从vpc 内部访问
vpc.add_interface_endpoint(
"SNSEndpoint", service=ec2.InterfaceVpcEndpointAwsService.SNS)
# step 6. Lambda
lambdaFn = lambda_.Function(self,
"Singleton",
code=lambda_.Code.asset('lambda'),
handler='hello.handler',
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE),
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
environment={
'SNS_TOPIC_ARN': topic.topic_arn,
'ES_ENDPOINT': es.attr_domain_endpoint,
'ES_INDEX_NAME': Constant.ES_INDEX_NAME
})
# step 7. Cloud watch event
rule = events.Rule(
self,
"Rule",
schedule=events.Schedule.cron(minute='0/5',
hour='*',
month='*',
week_day='*',
year='*'),
)
rule.add_target(targets.LambdaFunction(lambdaFn))
#给Lambda 添加发布SNS的权限
topic.grant_publish(lambdaFn)
# Create ALB
alb_name = "{}-alb".format(Constant.PROJECT_NAME)
alb = elb.ApplicationLoadBalancer(self,
alb_name,
vpc=vpc,
internet_facing=True,
load_balancer_name=alb_name)
alb.connections.allow_from_any_ipv4(ec2.Port.tcp(80),
"Internet access ALB 80")
listener = alb.add_listener("my80", port=80, open=True)
# Create Autoscaling Group with fixed 2*EC2 hosts
user_data = user_data_content.format(es.attr_domain_endpoint,
Constant.REGION_NAME,
Constant.ES_LOG_PATH,
Constant.ES_INDEX_NAME,
s3_bucket_name)
# step 3. 创建堡垒机
bastion_name = "{}-bastion".format(Constant.PROJECT_NAME)
bastion = ec2.BastionHostLinux(
self,
bastion_name,
vpc=vpc,
subnet_selection=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PUBLIC),
instance_name=bastion_name,
instance_type=ec2.InstanceType(
instance_type_identifier="m4.large"))
bastion.instance.instance.add_property_override(
"KeyName", Constant.EC2_KEY_NAME)
bastion.connections.allow_from_any_ipv4(
ec2.Port.tcp(22), "Internet access SSH") # 生成环境可以限定IP allow_from
bastion.connections.allow_from_any_ipv4(ec2.Port.tcp(8080),
"Internet access HTTP") # 测试需要
# bastion.connections.allow_from_any_ipv4(ec2.Port.tcp(443), "Internet access HTTPS") # 测试需要
bastion.instance.instance.iam_instance_profile = instance_profile.instance_profile_name # 给EC2设置 profile , 相当于Role
bastion.instance.instance.image_id = ami_map.get(
Constant.REGION_NAME) # 指定AMI ID
#堡垒机的user_data 只能执行一次， 如果要执行多次， 请参考 https://amazonaws-china.com/premiumsupport/knowledge-center/execute-user-data-ec2/?nc1=h_ls
bastion_user_data = "/home/ec2-user/start.sh {} {} '{}' {} {}".format(
es.attr_domain_endpoint, Constant.REGION_NAME,
Constant.ES_LOG_PATH, Constant.ES_INDEX_NAME, s3_bucket_name)
bastion.instance.add_user_data(
"date >> /home/ec2-user/root.txt") # 查看启动脚本是否执行
bastion.instance.add_user_data(bastion_user_data)
asg_name = "{}-asg".format(Constant.PROJECT_NAME)
asg = autoscaling.AutoScalingGroup(
self,
asg_name,
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PUBLIC), # PUBLIC for debug
instance_type=ec2.InstanceType(
instance_type_identifier="m4.large"),
machine_image=my_ami,
key_name=Constant.EC2_KEY_NAME,
user_data=ec2.UserData.custom(user_data),
desired_capacity=1,
min_capacity=1,
max_capacity=1,
role=access_es_role)
asg.connections.allow_from(
alb, ec2.Port.tcp(8080),
"ALB access 80 port of EC2 in Autoscaling Group")
# asg.connections.allow_from_any_ipv4(ec2.Port.tcp(8080), "Internet access HTTP for test") # 测试用
asg.connections.allow_from_any_ipv4(ec2.Port.tcp(22),
"Internet access SSH") # for debug
listener.add_targets("addTargetGroup", port=8080, targets=[asg])
core.CfnOutput(self,
"s3_bucket_name",
value=s3_bucket_name,
description='S3 bucket: store web log')
core.CfnOutput(self,
"ElasticSearchEndpointUrl",
value=es.attr_domain_endpoint,
description='Elastic Search Url')
# Elastic Search 统计log 数量， 可以在堡垒机上执行， 快速查看日志数量。
core.CfnOutput(self,
"CmdGetCountIndex",
value='curl https://{}/{}/_count'.format(
es.attr_domain_endpoint, Constant.ES_INDEX_NAME),
description='Count search result. ')
# 堡垒机的登录命令， 可以直接复制使用
core.CfnOutput(self,
"CmdSshToBastion",
value='ssh -i ~/{}.pem ec2-user@{}'.format(
Constant.EC2_KEY_NAME,
bastion.instance_public_dns_name),
description='cmd ssh to bastion')
# 在堡垒机上启动服务的命令， 堡垒机重启以后， 需要执行下面的命令， 可以启动web服务 发送日志到ES
core.CfnOutput(
self,
"CmdSshBastionStartWeb",
value='sudo {}'.format(bastion_user_data),
description="Cmd to start web+logstash+filebeat service")
# ALB 的访问地址
core.CfnOutput(self,
"UrlLoad_Balancer",
value='http://{}'.format(alb.load_balancer_dns_name),
description='ALB url ')
# 堡垒机的web访问地址， 为了调试方便， 在堡垒机上也使用相同的AMI。
core.CfnOutput(self,
"UrlBastion",
value='http://{}:8080'.format(
bastion.instance_public_dns_name),
description="Bastion server web url ")
# 下面这条输出的命令 是通过堡垒机和Elasticsearch 建立隧道， 在本地访问kibana。
core.CfnOutput(
self,
"CmdSshProxyToKibana",
value='ssh -i ~/{}.pem ec2-user@{} -N -L 9200:{}:443'.format(
Constant.EC2_KEY_NAME, bastion.instance_public_dns_name,
es.attr_domain_endpoint),
description="cmd: access kibana from bastion ssh. ")
# 执行完上面的命令后， 在浏览器中打开下面的连接
core.CfnOutput(self,
"UrlKibana",
value='https://localhost:9200/_plugin/kibana/',
description="kibana url ")
def add_low_efs_burst_credit_alarms(self, filesystem: FileSystem,
email_address: str) -> None:
'''
Set up CloudWatch Alarms that will warn when the given filesystem's burst credits are below
four different thresholds. We send an email to the given address when an Alarm breaches.
'''
# Set up the SNS Topic that will send the emails.
# ====================
# 1) KMS key to use to encrypt events within the SNS Topic. The Key is optional
key = Key(
self,
'SNSEncryptionKey',
description=
'Used to encrypt the SNS Topic for sending EFS Burst Credit alerts',
enable_key_rotation=True,
removal_policy=RemovalPolicy.DESTROY,
trust_account_identities=True)
key.grant(ServicePrincipal('cloudwatch.amazonaws.com'), 'kms:Decrypt',
'kms:GenerateDataKey')
# 2) SNS Topic that will be alerted by CloudWatch and will send the email in response.
sns_topic = Topic(self, 'BurstAlertEmailTopic', master_key=key)
sns_topic.grant_publish(ServicePrincipal('cloudwatch.amazonaws.com'))
sns_topic.add_subscription(EmailSubscription(email_address))
alarm_action = SnsAction(sns_topic)
# Set up the CloudWatch Alarm(s) and have them trigger SNS events when breached.
# ======================
# 1) CDK helper to define the CloudWatch Metric that we're interested in.
burst_credits_metric = Metric(
metric_name='BurstCreditBalance',
namespace='AWS/EFS',
dimensions={"FileSystemId": filesystem.file_system_id},
# One 99-th percentile data point sample every hour
period=Duration.hours(1),
statistic='p99')
# 2) Create the alarms
thresholds = [
{
"id": 'CAUTION-EfsBurstCredits',
"name": f"CAUTION Burst Credits - {filesystem.file_system_id}",
"threshold": int(2.00 * 2**40),
"message":
f"CAUTION. 2 TiB Threshold Breached: EFS {filesystem.file_system_id} is depleting burst credits. Add data to the EFS to increase baseline throughput.",
# Alarm after 6 datapoints below threshold. We have 1 datapoint every hour. So, we alarm if below threshold for 6hrs
"datapoints": 6
},
{
"id": 'WARNING-EfsBurstCredits',
"name": f"WARNING Burst Credits - {filesystem.file_system_id}",
"threshold": int(1.25 * 2**40),
"message":
f"WARNING. 1.25 TiB Threshold Breached: EFS {filesystem.file_system_id} is depleting burst credits. Add data to the EFS to increase baseline throughput.",
# Alarm after 6 datapoints below threshold. We have 1 datapoint every hour. So, we alarm if below threshold for 6hrs
"datapoints": 6
},
{
"id": 'ALERT-EfsBurstCredits',
"name": f"ALERT Burst Credits - {filesystem.file_system_id}",
"threshold": int(0.50 * 2**40),
"message":
f"ALERT! 500 GiB Threshold Breached: EFS {filesystem.file_system_id} is running out of burst credits. Add data to the EFS to increase baseline throughput or else the Render Farm may cease operation.",
# Alarm after 6 datapoints below threshold. We have 1 datapoint every hour. So, we alarm if below threshold for 6hrs
"datapoints": 6
},
{
"id": 'EMERGENCY-EfsBurstCredits',
"name":
f"EMERGENCY Burst Credits - {filesystem.file_system_id}",
"threshold": int(0.10 * 2**40),
"message":
f"EMERGENCY! 100 GiB Threshold Breached: EFS {filesystem.file_system_id} is running out of burst credits. Add data to the EFS to increase baseline throughput or else the Render Farm will cease operation.",
# Alarm after 2 datapoints below threshold. We have 1 datapoint every hour. So, we alarm if below threshold for 2hrs
"datapoints": 2
},
]
for config in thresholds:
alarm = burst_credits_metric.create_alarm(
self,
config['id'],
alarm_name=config['name'],
actions_enabled=True,
alarm_description=config['message'],
treat_missing_data=TreatMissingData.NOT_BREACHING,
threshold=config['threshold'],
comparison_operator=ComparisonOperator.LESS_THAN_THRESHOLD,
evaluation_periods=config['datapoints'])
alarm.add_alarm_action(alarm_action)
def __init__(
self,
scope: Stack,
id: str,
on_create_action: Dict[str, Any],
on_update_action: Dict[str, Any],
on_delete_action: Dict[str, Any],
) -> None:
"""
Constructor.
:param scope: CloudFormation stack in which resources should be placed.
:param id: Name (id) or prefix for resources.
:param on_create_action: Create action arguments. Read more on:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ecs.html#ECS.Client.create_service
:param on_update_action: Update action arguments. Read more on:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ecs.html#ECS.Client.update_service
:param on_delete_action: Delete action arguments. Read more on:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ecs.html#ECS.Client.delete_service
"""
self.__role = Role(
scope=scope,
id=id + 'Role',
role_name=id + 'Role',
assumed_by=CompositePrincipal(
ServicePrincipal("lambda.amazonaws.com"),
ServicePrincipal("cloudformation.amazonaws.com")
),
inline_policies={
id + 'Policy': PolicyDocument(
statements=[
PolicyStatement(
actions=[
'ecs:createService',
'ecs:updateService',
'ecs:deleteService',
'ecs:describeServices',
'ecs:listServices',
'ecs:updateServicePrimaryTaskSet'
],
effect=Effect.ALLOW,
resources=['*']
),
PolicyStatement(
actions=[
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
effect=Effect.ALLOW,
resources=['*']
),
]
)
},
managed_policies=[]
)
self.__custom_backend = Function(
scope=scope,
id=id + 'Backend',
code=Code.from_asset(
path=package_root
),
handler='index.handler',
runtime=Runtime.PYTHON_3_6,
description=f'A custom resource backend to manage ecs {id} service.',
function_name=id + 'Backend',
memory_size=128,
role=self.__role,
timeout=Duration.seconds(900),
)
# noinspection PyTypeChecker
provider: ICustomResourceProvider = CustomResourceProvider.from_lambda(self.__custom_backend)
self.__custom_resource = CustomResource(
scope=scope,
id=id + 'CustomResource',
provider=provider,
removal_policy=RemovalPolicy.DESTROY,
properties={
'onCreate': on_create_action,
'onUpdate': on_update_action,
'onDelete': on_delete_action
},
resource_type='Custom::EcsService'
)
# Make sure that custom resource is deleted before lambda function backend.
self.__custom_resource.node.add_dependency(self.__custom_backend)
def __init__(self, scope: core.Construct, id: str, prefix: str,
source_bucket: s3.Bucket, dest_bucket: s3.Bucket,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
suffix = Fn.select(
4, Fn.split('-', Fn.select(2, Fn.split('/', self.stack_id))))
# KMS key for Kinesis Data Streams
self.__kms_key = Key(scope=self,
id='kms-kinesis',
alias='custom/kinesis',
description='KMS key for Kinesis Data Streams',
enable_key_rotation=True)
# Create Kinesis streams
self.__sale_stream = Stream(scope=self,
id="saleStream",
stream_name="ara-web-sale",
encryption_key=self.__kms_key)
self.__address_stream = Stream(scope=self,
id="addressStream",
stream_name="ara-web-customer-address",
encryption_key=self.__kms_key)
self.__customer_stream = Stream(scope=self,
id="customerStream",
stream_name="ara-web-customer",
encryption_key=self.__kms_key)
# Role for the KDA service
kda_role = Role(scope=self,
id='KinesisAnalyticsRole',
assumed_by=ServicePrincipal(
service='kinesisanalytics.amazonaws.com'))
# Grant read on Kinesis streams
self.__customer_stream.grant_read(kda_role)
self.__address_stream.grant_read(kda_role)
self.__sale_stream.grant_read(kda_role)
# Grant read on source bucket (reference data)
source_bucket.grant_read(kda_role)
# Grant write on destination bucket
dest_bucket.grant_write(kda_role)
kda_role.add_to_policy(
PolicyStatement(actions=['kinesis:ListShards'],
resources=[
self.__customer_stream.stream_arn,
self.__address_stream.stream_arn,
self.__sale_stream.stream_arn
]))
# Create Elasticsearch domain
# TODO: use VPC subnets
es_domain = EsDomain(scope=self,
id='EsDomain',
application_prefix=prefix,
suffix=suffix,
kda_role=kda_role)
# Create the KDA application after the Elasticsearch service
kda_app = KdaApplication(scope=self,
id='KdaApplication',
es_domain=es_domain.es_domain,
kda_role=kda_role,
source_bucket=source_bucket,
dest_bucket=dest_bucket)
core.Tags.of(self).add('module-name', 'streaming')
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# The code that defines your stack goes here
table_name = "trainers"
trainers_graphql_api = CfnGraphQLApi(
self,'trainersApi',
name="trainers-api",
authentication_type='API_KEY'
)
CfnApiKey(
self,'TrainersApiKey',
api_id = trainers_graphql_api.attr_api_id
)
api_schema = CfnGraphQLSchema(
self,"TrainersSchema",
api_id = trainers_graphql_api.attr_api_id,
definition=data_schema
)
trainers_table = Table(
self, 'TrainersTable',
table_name=table_name,
partition_key=Attribute(
name='id',
type=AttributeType.STRING,
),
billing_mode=BillingMode.PAY_PER_REQUEST,
stream=StreamViewType.NEW_IMAGE,
# The default removal policy is RETAIN, which means that cdk
# destroy will not attempt to delete the new table, and it will
# remain in your account until manually deleted. By setting the
# policy to DESTROY, cdk destroy will delete the table (even if it
# has data in it)
removal_policy=core.RemovalPolicy.DESTROY # NOT recommended for production code
)
trainers_table_role = Role(
self, 'TrainersDynamoDBRole',
assumed_by=ServicePrincipal('appsync.amazonaws.com')
)
trainers_table_role.add_managed_policy(
ManagedPolicy.from_aws_managed_policy_name(
'AmazonDynamoDBFullAccess'
)
)
data_source = CfnDataSource(
self, 'TrainersDataSource',
api_id=trainers_graphql_api.attr_api_id,
name='TrainersDynamoDataSource',
type='AMAZON_DYNAMODB',
dynamo_db_config=CfnDataSource.DynamoDBConfigProperty(
table_name=trainers_table.table_name,
aws_region=self.region
),
service_role_arn=trainers_table_role.role_arn
)
get_Trainer_resolver = CfnResolver(
self, 'GetOneQueryResolver',
api_id=trainers_graphql_api.attr_api_id,
type_name='Query',
field_name='getTrainer',
data_source_name=data_source.name,
request_mapping_template=get_trainer,
response_mapping_template="$util.toJson($ctx.result)"
)
get_Trainer_resolver.add_depends_on(api_schema)
get_all_trainers_resolver = CfnResolver(
self, 'GetAllQueryResolver',
api_id=trainers_graphql_api.attr_api_id,
type_name='Query',
field_name='allTrainers',
data_source_name=data_source.name,
request_mapping_template=all_trainers,
response_mapping_template="$util.toJson($ctx.result)"
)
get_all_trainers_resolver.add_depends_on(api_schema)
create_trainers_resolver = CfnResolver(
self, 'CreateTrainerMutationResolver',
api_id=trainers_graphql_api.attr_api_id,
type_name='Mutation',
field_name='createTrainer',
data_source_name=data_source.name,
request_mapping_template=create_trainer,
response_mapping_template="$util.toJson($ctx.result)"
)
create_trainers_resolver.add_depends_on(api_schema)
update_trainers_resolver = CfnResolver(
self,'UpdateMutationResolver',
api_id=trainers_graphql_api.attr_api_id,
type_name="Mutation",
field_name="updateTrainers",
data_source_name=data_source.name,
request_mapping_template=update_trainer,
response_mapping_template="$util.toJson($ctx.result)"
)
update_trainers_resolver.add_depends_on(api_schema)
delete_trainer_resolver = CfnResolver(
self, 'DeleteMutationResolver',
api_id=trainers_graphql_api.attr_api_id,
type_name='Mutation',
field_name='deleteTrainer',
data_source_name=data_source.name,
request_mapping_template=delete_trainer,
response_mapping_template="$util.toJson($ctx.result)"
)
#core
delete_trainer_resolver.add_depends_on(api_schema)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# SQS queue
state_change_sqs = Queue(
self,
"state_change_sqs",
visibility_timeout=core.Duration.seconds(60)
)
# Dynamodb Tables
# EC2 state changes
tb_states = Table(
self, "ec2_states", partition_key=Attribute(name="instance-id",
type=AttributeType.STRING),
sort_key=Attribute(
name="time",
type=AttributeType.STRING
),
billing_mode=BillingMode.PAY_PER_REQUEST,
removal_policy=core.RemovalPolicy.DESTROY,
stream=StreamViewType.NEW_IMAGE)
# EC2 inventory
tb_inventory = Table(
self, "ec2_inventory", partition_key=Attribute(name="instance-id",
type=AttributeType.STRING),
sort_key=Attribute(
name="time",
type=AttributeType.STRING
),
billing_mode=BillingMode.PAY_PER_REQUEST,
removal_policy=core.RemovalPolicy.DESTROY,
stream=StreamViewType.KEYS_ONLY)
# IAM policies - AWS managed
basic_exec = ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole")
sqs_access = ManagedPolicy(self, "LambdaSQSExecution",
statements=[
PolicyStatement(
effect=Effect.ALLOW,
actions=[
"sqs:ReceiveMessage",
"sqs:DeleteMessage",
"sqs:GetQueueAttributes"
],
resources=[state_change_sqs.queue_arn]
)])
# IAM Policies
pol_ec2_states_ro = ManagedPolicy(self, "pol_EC2StatesReadOnly",
statements=[
PolicyStatement(
effect=Effect.ALLOW,
actions=[
"dynamodb:DescribeStream",
"dynamodb:GetRecords",
"dynamodb:GetItem",
"dynamodb:GetShardIterator",
"dynamodb:ListStreams"
],
resources=[tb_states.table_arn]
)])
pol_ec2_states_rwd = ManagedPolicy(
self, "pol_EC2StatesWriteDelete",
statements=[
PolicyStatement(
effect=Effect.ALLOW,
actions=[
"dynamodb:DeleteItem",
"dynamodb:DescribeTable",
"dynamodb:PutItem",
"dynamodb:Query",
"dynamodb:UpdateItem"
],
resources=[tb_states.table_arn]
)])
pol_ec2_inventory_full = ManagedPolicy(
self, "pol_EC2InventoryFullAccess",
statements=[
PolicyStatement(
effect=Effect.ALLOW,
actions=[
"dynamodb:DeleteItem",
"dynamodb:DescribeTable",
"dynamodb:GetItem",
"dynamodb:PutItem",
"dynamodb:Query",
"dynamodb:UpdateItem"
],
resources=[tb_inventory.table_arn]
)])
pol_lambda_describe_ec2 = ManagedPolicy(
self, "pol_LambdaDescribeEC2",
statements=[
PolicyStatement(
effect=Effect.ALLOW,
actions=[
"ec2:Describe*"
],
resources=["*"]
)])
# IAM Roles
rl_event_capture = Role(
self,
'rl_state_capture',
assumed_by=ServicePrincipal('lambda.amazonaws.com'),
managed_policies=[basic_exec, sqs_access, pol_ec2_states_rwd]
)
rl_event_processor = Role(
self,
'rl_state_processor',
assumed_by=ServicePrincipal('lambda.amazonaws.com'),
managed_policies=[
basic_exec,
pol_ec2_states_ro,
pol_ec2_states_rwd,
pol_ec2_inventory_full,
pol_lambda_describe_ec2])
# event capture lambda
lambda_event_capture = Function(
self, "lambda_event_capture",
handler="event_capture.handler",
runtime=Runtime.PYTHON_3_7,
code=Code.asset('event_capture'),
role=rl_event_capture,
events=[SqsEventSource(state_change_sqs)],
environment={"state_table": tb_states.table_name}
)
# event processor lambda
lambda_event_processor = Function(
self, "lambda_event_processor",
handler="event_processor.handler",
runtime=Runtime.PYTHON_3_7,
code=Code.asset('event_processor'),
role=rl_event_processor,
events=[
DynamoEventSource(
tb_states,
starting_position=StartingPosition.LATEST)
],
environment={
"inventory_table": tb_inventory.table_name,
}
)
# Cloudwatch Event
event_ec2_change = Rule(
self, "ec2_state_change",
description="trigger on ec2 start, stop and terminate instances",
event_pattern=EventPattern(
source=["aws.ec2"],
detail_type=["EC2 Instance State-change Notification"],
detail={
"state": [
"running",
"stopped",
"terminated"]
}
),
targets=[aws_events_targets.SqsQueue(state_change_sqs)]
)
# Outputs
core.CfnOutput(self, "rl_state_capture_arn", value=rl_event_capture.role_arn)
core.CfnOutput(self, "rl_state_processor_arn", value=rl_event_processor.role_arn)
core.CfnOutput(self, "tb_states_arn", value=tb_states.table_arn)
core.CfnOutput(self, "tb_inventory_arn", value=tb_inventory.table_arn)
core.CfnOutput(self, "sqs_state_change", value=state_change_sqs.queue_arn)
def __init__(
self,
stack: core.Stack,
prefix: str,
code_repository: Repository,
task_definition: str,
app_spec: str,
ecs_application: EcsApplication,
main_listener: CfnListener,
deployments_listener: CfnListener,
production_target_group,
deployment_target_group,
ecs_cluster: Cluster,
) -> None:
"""
Constructor.
:param stack: A CloudFormation stack to which add this resource.
:param prefix: Prefix for resource names.
:param code_repository: A codecommit git repository to push configuration files for ecs deployment.
:param task_definition: A document which describes how ecs deployment should behave.
:param app_spec: A document which describes how ecs deployment should behave.
:param ecs_application: An ecs application for which the deployments are being made.
:param main_listener: A loadbalancer's main listener for main traffic.
:param deployments_listener: A loadbalancer's testing listener for testing traffic.
:param ecs_cluster: An ecs cluster in which our ecs application is located.
"""
self.__stack = stack
self.__prefix = prefix
self.__code_repository = code_repository
self.__task_definition = task_definition
self.__app_spec = app_spec
self.__ecs_application = ecs_application
self.__main_listener = main_listener
self.__deployments_listener = deployments_listener
self.__production_target_group = production_target_group
self.__deployment_target_group = deployment_target_group
self.__ecs_cluster = ecs_cluster
self.__custom_resource_role = Role(
self.__stack,
self.__prefix + 'CustomFargateDeploymentGroupRole',
inline_policies={
self.__prefix + 'CustomFargateDeploymentGroupPolicy': PolicyDocument(
statements=[
PolicyStatement(
actions=[
"codedeploy:GetDeploymentGroup",
"codedeploy:CreateDeploymentGroup",
"codedeploy:DeleteDeploymentGroup",
"codedeploy:UpdateDeploymentGroup",
],
resources=['*'],
effect=Effect.ALLOW
),
PolicyStatement(
actions=[
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
resources=['*'],
effect=Effect.ALLOW
)
]
)},
assumed_by=ServicePrincipal('lambda.amazonaws.com')
)
self.__deployment_group_role = Role(
self.__stack,
self.__prefix + 'FargateDeploymentGroupRole',
path='/',
inline_policies={
self.__prefix + 'FargateDeploymentGroupPolicy': PolicyDocument(
statements=[
PolicyStatement(
actions=[
"ecs:DescribeServices",
"ecs:CreateTaskSet",
"ecs:UpdateServicePrimaryTaskSet",
"ecs:DeleteTaskSet",
"elasticloadbalancing:DescribeTargetGroups",
"elasticloadbalancing:DescribeListeners",
"elasticloadbalancing:ModifyListener",
"elasticloadbalancing:DescribeRules",
"elasticloadbalancing:ModifyRule",
"lambda:InvokeFunction",
"cloudwatch:DescribeAlarms",
"sns:Publish",
"s3:GetObject",
"s3:GetObjectMetadata",
"s3:GetObjectVersion",
"iam:PassRole"
],
resources=['*'],
effect=Effect.ALLOW
)
]
)
},
assumed_by=CompositePrincipal(
ServicePrincipal('ecs-tasks.amazonaws.com'),
ServicePrincipal('codedeploy.amazonaws.com')
)
)
def _create_gateway(self):
cert = Certificate(
self,
"certificate",
domain_name=self.domain_name,
validation_method=ValidationMethod.DNS
)
domain_name = DomainName(
self,
"domain",
domain_name=self.domain_name,
certificate=cert,
security_policy=SecurityPolicy.TLS_1_2
)
http_api = HttpApi(
self,
"movies_gateway",
create_default_stage=False,
api_name="movies",
cors_preflight=CorsPreflightOptions(
allow_methods=[HttpMethod.GET, HttpMethod.POST],
allow_origins=["https://moshan.tv", "https://beta.moshan.tv"],
allow_headers=["authorization", "content-type"]
)
)
authorizer = CfnAuthorizer(
self,
"cognito",
api_id=http_api.http_api_id,
authorizer_type="JWT",
identity_source=["$request.header.Authorization"],
name="cognito",
jwt_configuration=CfnAuthorizer.JWTConfigurationProperty(
audience=["68v5rahd0sdvrmf7fgbq2o1a9u"],
issuer="https://cognito-idp.eu-west-1.amazonaws.com/eu-west-1_sJ3Y4kSv6"
)
)
routes = {
"get_movies": {
"method": "GET",
"route": "/movies",
"target_lambda": self.lambdas["api-movies"]
},
"post_movies": {
"method": "POST",
"route": "/movies",
"target_lambda": self.lambdas["api-movies"]
},
"get_movies_by_id": {
"method": "GET",
"route": "/movies/{id}",
"target_lambda": self.lambdas["api-movies_by_id"]
},
}
for r in routes:
integration = HttpIntegration(
self,
f"{r}_integration",
http_api=http_api,
integration_type=HttpIntegrationType.LAMBDA_PROXY,
integration_uri=routes[r]["target_lambda"].function_arn,
method=getattr(HttpMethod, routes[r]["method"]),
payload_format_version=PayloadFormatVersion.VERSION_2_0,
)
CfnRoute(
self,
r,
api_id=http_api.http_api_id,
route_key=f"{routes[r]['method']} {routes[r]['route']}",
authorization_type="JWT",
authorizer_id=authorizer.ref,
target="integrations/" + integration.integration_id
)
routes[r]["target_lambda"].add_permission(
f"{r}_apigateway_invoke",
principal=ServicePrincipal("apigateway.amazonaws.com"),
source_arn=f"arn:aws:execute-api:{self.region}:{self.account}:{http_api.http_api_id}/*"
)
stage = CfnStage(
self,
"live",
api_id=http_api.http_api_id,
auto_deploy=True,
default_route_settings=CfnStage.RouteSettingsProperty(
throttling_burst_limit=10,
throttling_rate_limit=5
),
stage_name="live"
)
HttpApiMapping(
self,
"mapping",
api=http_api,
domain_name=domain_name,
stage=stage
)
def __init__(self, scope: Construct, stack_id: str, *, props: SEPStackProps, **kwargs):
"""
Initialize a new instance of SEPStack
:param scope: The scope of this construct.
:param stack_id: The ID of this construct.
:param props: The properties for this construct.
:param kwargs: Any kwargs that need to be passed on to the parent class.
"""
super().__init__(scope, stack_id, **kwargs)
# The VPC that all components of the render farm will be created in.
vpc = Vpc(
self,
'Vpc',
max_azs=2,
)
recipes = ThinkboxDockerRecipes(
self,
'Image',
stage=Stage.from_directory(props.docker_recipes_stage_path),
)
repository = Repository(
self,
'Repository',
vpc=vpc,
version=recipes.version,
repository_installation_timeout=Duration.minutes(20),
# TODO - Evaluate deletion protection for your own needs. These properties are set to RemovalPolicy.DESTROY
# to cleanly remove everything when this stack is destroyed. If you would like to ensure
# that these resources are not accidentally deleted, you should set these properties to RemovalPolicy.RETAIN
# or just remove the removal_policy parameter.
removal_policy=RepositoryRemovalPolicies(
database=RemovalPolicy.DESTROY,
filesystem=RemovalPolicy.DESTROY,
),
)
host = 'renderqueue'
zone_name = 'deadline-test.internal'
# Internal DNS zone for the VPC.
dns_zone = PrivateHostedZone(
self,
'DnsZone',
vpc=vpc,
zone_name=zone_name,
)
ca_cert = X509CertificatePem(
self,
'RootCA',
subject=DistinguishedName(
cn='SampleRootCA',
),
)
server_cert = X509CertificatePem(
self,
'RQCert',
subject=DistinguishedName(
cn=f'{host}.{dns_zone.zone_name}',
o='RFDK-Sample',
ou='RenderQueueExternal',
),
signing_certificate=ca_cert,
)
render_queue = RenderQueue(
self,
'RenderQueue',
vpc=vpc,
version=recipes.version,
images=recipes.render_queue_images,
repository=repository,
# TODO - Evaluate deletion protection for your own needs. This is set to false to
# cleanly remove everything when this stack is destroyed. If you would like to ensure
# that this resource is not accidentally deleted, you should set this to true.
deletion_protection=False,
hostname=RenderQueueHostNameProps(
hostname=host,
zone=dns_zone,
),
traffic_encryption=RenderQueueTrafficEncryptionProps(
external_tls=RenderQueueExternalTLSProps(
rfdk_certificate=server_cert,
),
internal_protocol=ApplicationProtocol.HTTPS,
),
)
if props.create_resource_tracker_role:
# Creates the Resource Tracker Access role. This role is required to exist in your account so the resource tracker will work properly
Role(
self,
'ResourceTrackerRole',
assumed_by=ServicePrincipal('lambda.amazonaws.com'),
managed_policies= [ManagedPolicy.from_aws_managed_policy_name('AWSThinkboxDeadlineResourceTrackerAccessPolicy')],
role_name= 'DeadlineResourceTrackerAccessRole',
)
fleet = SpotEventPluginFleet(
self,
'SpotEventPluginFleet',
vpc=vpc,
render_queue=render_queue,
deadline_groups=['group_name'],
instance_types=[InstanceType.of(InstanceClass.BURSTABLE3, InstanceSize.LARGE)],
worker_machine_image=props.worker_machine_image,
max_capacity=1,
)
# Optional: Add additional tags to both spot fleet request and spot instances.
Tags.of(fleet).add('name', 'SEPtest')
ConfigureSpotEventPlugin(
self,
'ConfigureSpotEventPlugin',
vpc=vpc,
render_queue=render_queue,
spot_fleets=[fleet],
configuration=SpotEventPluginSettings(
enable_resource_tracker=True,
),
)
DDB_ACTIONS = ['dynamodb:Scan', 'dynamodb:PutItem']
API_INVOKE_STATEMENT_ACTION = 'execute-api:Invoke'
ROUTE_53_DNS_ENSURER_ACTIONS = [
'route53domains:GetDomainDetail', 'route53domains:UpdateDomainNameservers',
'route53:GetHostedZone'
]
ROUTE_53_DNS_ENSURER_POLICY_STATEMENT = PolicyStatement(
actions=MINIMAL_FUNCTION_ACTIONS + ROUTE_53_DNS_ENSURER_ACTIONS,
resources=['*'])
MINIMAL_FUNCTION_POLICY_STATEMENT = PolicyStatement(
actions=MINIMAL_FUNCTION_ACTIONS, resources=['*'])
MINIMAL_API_POLICY_STATEMENT = PolicyStatement(
actions=MINIMAL_FUNCTION_ACTIONS,
resources=['*'],
principals=[ServicePrincipal('apigateway.amazonaws.com')])
PUBLIC_INVOKE_POLICY_STATEMENT = PolicyStatement(
actions=[API_INVOKE_STATEMENT_ACTION],
resources=['*'],
principals=[Anyone()])
MINIMAL_PUBLIC_API_POLICY_DOCUMENT = PolicyDocument(
statements=[MINIMAL_API_POLICY_STATEMENT, PUBLIC_INVOKE_POLICY_STATEMENT])
DDB_FUNCTION_POLICY_STATEMENT = PolicyStatement(
actions=MINIMAL_FUNCTION_ACTIONS + DDB_ACTIONS, resources=['*'])
def __init__(self, scope: core.Construct, id: str, application_prefix: str,
suffix: str, kda_role: Role, **kwargs):
super().__init__(scope, id, **kwargs)
stack = Stack.of(self)
region = stack.region
# Create Cognito User Pool
self.__user_pool = CfnUserPool(
scope=self,
id='UserPool',
admin_create_user_config={'allowAdminCreateUserOnly': True},
policies={'passwordPolicy': {
'minimumLength': 8
}},
username_attributes=['email'],
auto_verified_attributes=['email'],
user_pool_name=application_prefix + '_user_pool')
# Create a Cognito User Pool Domain using the newly created Cognito User Pool
CfnUserPoolDomain(scope=self,
id='CognitoDomain',
domain=application_prefix + '-' + suffix,
user_pool_id=self.user_pool.ref)
# Create Cognito Identity Pool
self.__id_pool = CfnIdentityPool(
scope=self,
id='IdentityPool',
allow_unauthenticated_identities=False,
cognito_identity_providers=[],
identity_pool_name=application_prefix + '_identity_pool')
trust_relationship = FederatedPrincipal(
federated='cognito-identity.amazonaws.com',
conditions={
'StringEquals': {
'cognito-identity.amazonaws.com:aud': self.id_pool.ref
},
'ForAnyValue:StringLike': {
'cognito-identity.amazonaws.com:amr': 'authenticated'
}
},
assume_role_action='sts:AssumeRoleWithWebIdentity')
# IAM role for master user
master_auth_role = Role(scope=self,
id='MasterAuthRole',
assumed_by=trust_relationship)
# Role for authenticated user
limited_auth_role = Role(scope=self,
id='LimitedAuthRole',
assumed_by=trust_relationship)
# Attach Role to Identity Pool
CfnIdentityPoolRoleAttachment(
scope=self,
id='userPoolRoleAttachment',
identity_pool_id=self.id_pool.ref,
roles={'authenticated': limited_auth_role.role_arn})
# Create master-user-group
CfnUserPoolGroup(scope=self,
id='AdminsGroup',
user_pool_id=self.user_pool.ref,
group_name='master-user-group',
role_arn=master_auth_role.role_arn)
# Create limited-user-group
CfnUserPoolGroup(scope=self,
id='UsersGroup',
user_pool_id=self.user_pool.ref,
group_name='limited-user-group',
role_arn=limited_auth_role.role_arn)
# Role for the Elasticsearch service to access Cognito
es_role = Role(scope=self,
id='EsRole',
assumed_by=ServicePrincipal(service='es.amazonaws.com'),
managed_policies=[
ManagedPolicy.from_aws_managed_policy_name(
'AmazonESCognitoAccess')
])
# Use the following command line to generate the python dependencies layer content
# pip3 install -t lambda-layer/python/lib/python3.8/site-packages -r lambda/requirements.txt
# Build the lambda layer assets
subprocess.call([
'pip', 'install', '-t',
'streaming/streaming_cdk/lambda-layer/python/lib/python3.8/site-packages',
'-r', 'streaming/streaming_cdk/bootstrap-lambda/requirements.txt',
'--upgrade'
])
requirements_layer = _lambda.LayerVersion(
scope=self,
id='PythonRequirementsTemplate',
code=_lambda.Code.from_asset(
'streaming/streaming_cdk/lambda-layer'),
compatible_runtimes=[_lambda.Runtime.PYTHON_3_8])
# This lambda function will bootstrap the Elasticsearch cluster
bootstrap_function_name = 'AESBootstrap'
register_template_lambda = _lambda.Function(
scope=self,
id='RegisterTemplate',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset(
'streaming/streaming_cdk/bootstrap-lambda'),
handler='es-bootstrap.lambda_handler',
environment={
'REGION': region,
'KDA_ROLE_ARN': kda_role.role_arn,
'MASTER_ROLE_ARN': master_auth_role.role_arn
},
layers=[requirements_layer],
timeout=Duration.minutes(15),
function_name=bootstrap_function_name)
lambda_role = register_template_lambda.role
lambda_role.add_to_policy(
PolicyStatement(
actions=['logs:CreateLogGroup'],
resources=[stack.format_arn(service='logs', resource='*')]))
lambda_role.add_to_policy(
PolicyStatement(
actions=['logs:CreateLogStream', 'logs:PutLogEvents'],
resources=[
stack.format_arn(service='logs',
resource='log_group',
resource_name='/aws/lambda/' +
bootstrap_function_name + ':*')
]))
# Let the lambda assume the master role so that actions can be executed on the cluster
# https://aws.amazon.com/premiumsupport/knowledge-center/lambda-function-assume-iam-role/
lambda_role.add_to_policy(
PolicyStatement(actions=['sts:AssumeRole'],
resources=[master_auth_role.role_arn]))
master_auth_role.assume_role_policy.add_statements(
PolicyStatement(actions=['sts:AssumeRole'],
principals=[lambda_role]))
# List all the roles that are allowed to access the Elasticsearch cluster.
roles = [
ArnPrincipal(limited_auth_role.role_arn),
ArnPrincipal(master_auth_role.role_arn),
ArnPrincipal(kda_role.role_arn)
] # The users
if register_template_lambda and register_template_lambda.role:
roles.append(ArnPrincipal(
lambda_role.role_arn)) # The lambda used to bootstrap
# Create kms key
kms_key = Key(scope=self,
id='kms-es',
alias='custom/es',
description='KMS key for Elasticsearch domain',
enable_key_rotation=True)
# AES Log Groups
es_app_log_group = logs.LogGroup(scope=self,
id='EsAppLogGroup',
retention=logs.RetentionDays.ONE_WEEK,
removal_policy=RemovalPolicy.RETAIN)
# Create the Elasticsearch domain
es_domain_arn = stack.format_arn(service='es',
resource='domain',
resource_name=application_prefix +
'/*')
es_access_policy = PolicyDocument(statements=[
PolicyStatement(principals=roles,
actions=[
'es:ESHttpGet', 'es:ESHttpPut',
'es:ESHttpPost', 'es:ESHttpDelete'
],
resources=[es_domain_arn])
])
self.__es_domain = es.CfnDomain(
scope=self,
id='searchDomain',
elasticsearch_cluster_config={
'instanceType': 'r5.large.elasticsearch',
'instanceCount': 2,
'dedicatedMasterEnabled': True,
'dedicatedMasterCount': 3,
'dedicatedMasterType': 'r5.large.elasticsearch',
'zoneAwarenessEnabled': True,
'zoneAwarenessConfig': {
'AvailabilityZoneCount': '2'
},
},
encryption_at_rest_options={
'enabled': True,
'kmsKeyId': kms_key.key_id
},
node_to_node_encryption_options={'enabled': True},
ebs_options={
'volumeSize': 10,
'ebsEnabled': True
},
elasticsearch_version='7.9',
domain_name=application_prefix,
access_policies=es_access_policy,
cognito_options={
'enabled': True,
'identityPoolId': self.id_pool.ref,
'roleArn': es_role.role_arn,
'userPoolId': self.user_pool.ref
},
advanced_security_options={
'enabled': True,
'internalUserDatabaseEnabled': False,
'masterUserOptions': {
'masterUserArn': master_auth_role.role_arn
}
},
domain_endpoint_options={
'enforceHttps': True,
'tlsSecurityPolicy': 'Policy-Min-TLS-1-2-2019-07'
},
# log_publishing_options={
# # 'ES_APPLICATION_LOGS': {
# # 'enabled': True,
# # 'cloud_watch_logs_log_group_arn': es_app_log_group.log_group_arn
# # },
# # 'AUDIT_LOGS': {
# # 'enabled': True,
# # 'cloud_watch_logs_log_group_arn': ''
# # },
# # 'SEARCH_SLOW_LOGS': {
# # 'enabled': True,
# # 'cloud_watch_logs_log_group_arn': ''
# # },
# # 'INDEX_SLOW_LOGS': {
# # 'enabled': True,
# # 'cloud_watch_logs_log_group_arn': ''
# # }
# }
)
# Not yet on the roadmap...
# See https://github.com/aws-cloudformation/aws-cloudformation-coverage-roadmap/issues/283
# self.es_domain.add_property_override('ElasticsearchClusterConfig.WarmEnabled', True)
# self.es_domain.add_property_override('ElasticsearchClusterConfig.WarmCount', 2)
# self.es_domain.add_property_override('ElasticsearchClusterConfig.WarmType', 'ultrawarm1.large.elasticsearch')
# Deny all roles from the authentication provider - users must be added to groups
# This lambda function will bootstrap the Elasticsearch cluster
cognito_function_name = 'CognitoFix'
cognito_template_lambda = _lambda.Function(
scope=self,
id='CognitoFixLambda',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset(
'streaming/streaming_cdk/cognito-lambda'),
handler='handler.handler',
environment={
'REGION': scope.region,
'USER_POOL_ID': self.__user_pool.ref,
'IDENTITY_POOL_ID': self.__id_pool.ref,
'LIMITED_ROLE_ARN': limited_auth_role.role_arn
},
timeout=Duration.minutes(15),
function_name=cognito_function_name)
lambda_role = cognito_template_lambda.role
lambda_role.add_to_policy(
PolicyStatement(
actions=['logs:CreateLogGroup'],
resources=[stack.format_arn(service='logs', resource='*')]))
lambda_role.add_to_policy(
PolicyStatement(
actions=['logs:CreateLogStream', 'logs:PutLogEvents'],
resources=[
stack.format_arn(service='logs',
resource='log_group',
resource_name='/aws/lambda/' +
cognito_function_name + ':*')
]))
lambda_role.add_to_policy(
PolicyStatement(actions=['cognito-idp:ListUserPoolClients'],
resources=[self.user_pool.attr_arn]))
lambda_role.add_to_policy(
PolicyStatement(actions=['iam:PassRole'],
resources=[limited_auth_role.role_arn]))
cognito_id_res = Fn.join(':', [
'arn:aws:cognito-identity', scope.region, scope.account,
Fn.join('/', ['identitypool', self.__id_pool.ref])
])
lambda_role.add_to_policy(
PolicyStatement(actions=['cognito-identity:SetIdentityPoolRoles'],
resources=[cognito_id_res]))
# Get the Domain Endpoint and register it with the lambda as environment variable.
register_template_lambda.add_environment(
'DOMAIN', self.__es_domain.attr_domain_endpoint)
CfnOutput(scope=self,
id='createUserUrl',
description="Create a new user in the user pool here.",
value="https://" + scope.region +
".console.aws.amazon.com/cognito/users?region=" +
scope.region + "#/pool/" + self.user_pool.ref + "/users")
CfnOutput(scope=self,
id='kibanaUrl',
description="Access Kibana via this URL.",
value="https://" + self.__es_domain.attr_domain_endpoint +
"/_plugin/kibana/")
bootstrap_lambda_provider = Provider(
scope=self,
id='BootstrapLambdaProvider',
on_event_handler=register_template_lambda)
CustomResource(scope=self,
id='ExecuteRegisterTemplate',
service_token=bootstrap_lambda_provider.service_token,
properties={'Timeout': 900})
cognito_lambda_provider = Provider(
scope=self,
id='CognitoFixLambdaProvider',
on_event_handler=cognito_template_lambda)
cognito_fix_resource = CustomResource(
scope=self,
id='ExecuteCognitoFix',
service_token=cognito_lambda_provider.service_token)
cognito_fix_resource.node.add_dependency(self.__es_domain)
def _create_gateway(self):
cert = Certificate(self,
"certificate",
domain_name=self.domain_name,
validation_method=ValidationMethod.DNS)
domain_name = DomainName(
self,
"domain_name",
certificate=cert,
domain_name=self.domain_name,
)
http_api = HttpApi(
self,
"shows_gateway",
create_default_stage=False,
api_name="shows",
cors_preflight=CorsPreflightOptions(
allow_methods=[HttpMethod.GET, HttpMethod.POST],
allow_origins=["https://moshan.tv", "https://beta.moshan.tv"],
allow_headers=["authorization", "content-type"]))
routes = {
"get_shows": {
"method": "GET",
"route": "/shows",
"target_lambda": self.lambdas["api-shows"]
},
"post_shows": {
"method": "POST",
"route": "/shows",
"target_lambda": self.lambdas["api-shows"]
},
"get_shows_by_id": {
"method": "GET",
"route": "/shows/{id}",
"target_lambda": self.lambdas["api-shows_by_id"]
},
"get_episodes": {
"method": "GET",
"route": "/episodes",
"target_lambda": self.lambdas["api-episodes"]
},
"post_episodes": {
"method": "POST",
"route": "/shows/{id}/episodes",
"target_lambda": self.lambdas["api-episodes"]
},
"get_episodes_by_id": {
"method": "GET",
"route": "/shows/{id}/episodes/{episode_id}",
"target_lambda": self.lambdas["api-episodes_by_id"]
},
}
for r in routes:
integration = HttpIntegration(
self,
f"{r}_integration",
http_api=http_api,
integration_type=HttpIntegrationType.LAMBDA_PROXY,
integration_uri=routes[r]["target_lambda"].function_arn,
method=getattr(HttpMethod, routes[r]["method"]),
payload_format_version=PayloadFormatVersion.VERSION_2_0,
)
CfnRoute(
self,
r,
api_id=http_api.http_api_id,
route_key=f"{routes[r]['method']} {routes[r]['route']}",
# authorization_type="AWS_IAM", # TODO: add back when: https://github.com/aws/aws-cdk/pull/14853 gets merged (set this manually for now)
target="integrations/" + integration.integration_id)
routes[r]["target_lambda"].add_permission(
f"{r}_apigateway_invoke",
principal=ServicePrincipal("apigateway.amazonaws.com"),
source_arn=
f"arn:aws:execute-api:{self.region}:{self.account}:{http_api.http_api_id}/*"
)
HttpStage(self,
"live",
http_api=http_api,
auto_deploy=True,
stage_name="live",
domain_mapping=DomainMappingOptions(
domain_name=domain_name, ))