def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.platform_resources = ImportedResources(self, self.stack_name)
###
ECS_APP_NAME = "octicketing-microservice"
ECS_DEPLOYMENT_GROUP_NAME = "octicketingECSBlueGreen"
ECS_DEPLOYMENT_CONFIG_NAME = "CodeDeployDefault.ECSLinear10PercentEvery1Minutes"
ECS_DEPLOYMENT_CONFIG_ALL = "CodeDeployDefault.ECSAllAtOnce"
ECS_TASKSET_TERMINATION_WAIT_TIME = 10
ECS_TASK_FAMILY_NAME = "octicketing-service"
ECS_APP_LOG_GROUP_NAME = "/ecs/" + ECS_TASK_FAMILY_NAME
DUMMY_APP_NAME = "hello-world-microservice"
DUMMY_TASK_FAMILY_NAME = "hello-world-service"
DUMMY_APP_LOG_GROUP_NAME = "/ecs/dummy-" + ECS_TASK_FAMILY_NAME
DUMMY_CONTAINER_IMAGE = self.account + ".dkr.ecr." + \
self.region + ".amazonaws.com/hello-world:latest"
Dmmuyvare = ""
# =============================================================================
# ECR and CodeCommit repositories for the Blue/ Green deployment
# =============================================================================
# ECR repository for the docker images
self.octicketing_ecr_repo = aws_ecr.Repository(
self,
"OcticketingECRRepo",
repository_name=ECS_APP_NAME,
removal_policy=core.RemovalPolicy.DESTROY)
self.octicketing_code_repo = aws_codecommit.Repository(
self,
ECS_APP_NAME + "-bg",
repository_name=ECS_APP_NAME + "-bg",
description=ECS_APP_NAME + "blue-green service repository")
core.CfnOutput(self,
'BGRepoName',
value=self.octicketing_code_repo.repository_name,
export_name='OcticketingBGRepoName')
core.CfnOutput(self,
'BGRepoARN',
value=self.octicketing_code_repo.repository_arn,
export_name='OcticketingBGRepoARN')
# =============================================================================
# CODE BUILD and ECS TASK ROLES for the Blue/ Green deployment
# =============================================================================
# IAM role for the Code Build project
codeBuildServiceRole = aws_iam.Role(
self,
"codeBuildServiceRole",
assumed_by=aws_iam.ServicePrincipal('codebuild.amazonaws.com'))
inlinePolicyForCodeBuild = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"ecr:GetAuthorizationToken", "ecr:BatchCheckLayerAvailability",
"ecr:InitiateLayerUpload", "ecr:BatchGetImage",
"ecr:GetDownloadUrlForLayer", "ecr:UploadLayerPart",
"ecr:CompleteLayerUpload", "ecr:PutImage"
],
resources=["*"])
codeBuildServiceRole.add_to_policy(inlinePolicyForCodeBuild)
# ECS task role
ecsTaskRole = aws_iam.Role(
self,
"ecsTaskRoleForWorkshop",
assumed_by=aws_iam.ServicePrincipal('ecs-tasks.amazonaws.com'))
ecsTaskRole.add_managed_policy(
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonECSTaskExecutionRolePolicy"))
ecsTaskRole.add_managed_policy(
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"SecretsManagerReadWrite"))
# =============================================================================
# CODE DEPLOY APPLICATION for the Blue/ Green deployment
# =============================================================================
# Creating the code deploy application
codeDeployApplication = codedeploy.EcsApplication(
self, "OcticketingCodeDeploy")
# Creating the code deploy service role
codeDeployServiceRole = aws_iam.Role(
self,
"codeDeployServiceRole",
assumed_by=aws_iam.ServicePrincipal('codedeploy.amazonaws.com'))
codeDeployServiceRole.add_managed_policy(
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"AWSCodeDeployRoleForECS"))
# IAM role for custom lambda function
customLambdaServiceRole = aws_iam.Role(
self,
"codeDeployCustomLambda",
assumed_by=aws_iam.ServicePrincipal('lambda.amazonaws.com'))
inlinePolicyForLambda = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"iam:PassRole", "sts:AssumeRole", "codedeploy:List*",
"codedeploy:Get*", "codedeploy:UpdateDeploymentGroup",
"codedeploy:CreateDeploymentGroup",
"codedeploy:DeleteDeploymentGroup"
],
resources=["*"])
customLambdaServiceRole.add_managed_policy(
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'))
customLambdaServiceRole.add_to_policy(inlinePolicyForLambda)
# Custom resource to create the deployment group
createDeploymentGroupLambda = aws_lambda.Function(
self,
'createDeploymentGroupLambda',
code=aws_lambda.Code.from_asset("custom_resources"),
runtime=aws_lambda.Runtime.PYTHON_3_8,
handler='create_deployment_group.handler',
role=customLambdaServiceRole,
description="Custom resource to create deployment group",
memory_size=128,
timeout=core.Duration.seconds(60))
# ================================================================================================
# CloudWatch Alarms for 4XX errors
blue4xxMetric = aws_cloudwatch.Metric(
namespace='AWS/ApplicationELB',
metric_name='HTTPCode_Target_4XX_Count',
dimensions={
"TargetGroup": self.platform_resources.blue_target_full_name,
"LoadBalancer": self.platform_resources.alb_full_name
},
statistic="sum",
period=core.Duration.minutes(1))
self.blue_targetAlarm = aws_cloudwatch.Alarm(
self,
"blue4xxErrors",
alarm_name="Blue_4xx_Alarm",
alarm_description=
"CloudWatch Alarm for the 4xx errors of Blue target group",
metric=blue4xxMetric,
threshold=1,
evaluation_periods=1)
green4xxMetric = aws_cloudwatch.Metric(
namespace='AWS/ApplicationELB',
metric_name='HTTPCode_Target_4XX_Count',
dimensions={
"TargetGroup": self.platform_resources.green_target_full_name,
"LoadBalancer": self.platform_resources.alb_full_name
},
statistic="sum",
period=core.Duration.minutes(1))
self.green_targetAlarm = aws_cloudwatch.Alarm(
self,
"green4xxErrors",
alarm_name="Green_4xx_Alarm",
alarm_description=
"CloudWatch Alarm for the 4xx errors of Green target group",
metric=green4xxMetric,
threshold=1,
evaluation_periods=1)
# ================================================================================================
# DUMMY TASK DEFINITION for the initial service creation
# This is required for the service being made available to create the CodeDeploy Deployment Group
# ================================================================================================
sampleTaskDefinition = aws_ecs.FargateTaskDefinition(
self,
"sampleTaskDefn",
family=DUMMY_TASK_FAMILY_NAME,
cpu=256,
memory_limit_mib=1024,
task_role=ecsTaskRole,
execution_role=ecsTaskRole)
sampleContainerDefn = sampleTaskDefinition.add_container(
"sampleAppContainer",
image=aws_ecs.ContainerImage.from_registry(DUMMY_CONTAINER_IMAGE),
logging=aws_ecs.AwsLogDriver(log_group=aws_logs.LogGroup(
self,
"sampleAppLogGroup",
log_group_name=DUMMY_APP_LOG_GROUP_NAME,
removal_policy=core.RemovalPolicy.DESTROY),
stream_prefix=DUMMY_APP_NAME),
docker_labels={"name": DUMMY_APP_NAME})
port_mapping = aws_ecs.PortMapping(container_port=8080,
protocol=aws_ecs.Protocol.TCP)
sampleContainerDefn.add_port_mappings(port_mapping)
# ================================================================================================
# ECS task definition using ECR image
# Will be used by the CODE DEPLOY for Blue/Green deployment
# ================================================================================================
OcticketingTaskDef = aws_ecs.FargateTaskDefinition(
self,
"appTaskDefn",
family=ECS_TASK_FAMILY_NAME,
cpu=256,
memory_limit_mib=1024,
task_role=ecsTaskRole,
execution_role=ecsTaskRole)
# =============================================================================
octicketing_cont_def = OcticketingTaskDef.add_container(
"OcticketingAppContainer",
image=aws_ecs.ContainerImage.from_ecr_repository(
self.octicketing_ecr_repo, "latest"),
logging=aws_ecs.AwsLogDriver(log_group=aws_logs.LogGroup(
self,
"OcticketingAppLogGroup",
log_group_name=ECS_APP_LOG_GROUP_NAME,
removal_policy=core.RemovalPolicy.DESTROY),
stream_prefix=ECS_APP_NAME),
docker_labels={"name": ECS_APP_NAME})
octicketing_cont_def.add_port_mappings(port_mapping)
# =============================================================================
# ECS SERVICE for the Blue/ Green deployment
# =============================================================================
OcticketingAppService = aws_ecs.FargateService(
self,
"OcticketingAppService",
cluster=self.platform_resources.ecs_cluster,
task_definition=sampleTaskDefinition,
health_check_grace_period=core.Duration.seconds(10),
platform_version=aws_ecs.FargatePlatformVersion.VERSION1_4,
desired_count=1,
deployment_controller={
"type": aws_ecs.DeploymentControllerType.CODE_DEPLOY
},
service_name=ECS_APP_NAME)
OcticketingAppService.connections.allow_from(
self.platform_resources.alb, aws_ec2.Port.tcp(80))
OcticketingAppService.connections.allow_from(
self.platform_resources.alb, aws_ec2.Port.tcp(8080))
OcticketingAppService.attach_to_application_target_group(
self.platform_resources.blue_target)
# =============================================================================
# CODE DEPLOY - Deployment Group CUSTOM RESOURCE for the Blue/ Green deployment
# =============================================================================
core.CustomResource(
self,
'customEcsDeploymentGroup',
service_token=createDeploymentGroupLambda.function_arn,
properties={
"ApplicationName": codeDeployApplication.application_name,
"DeploymentGroupName": ECS_DEPLOYMENT_GROUP_NAME,
"DeploymentConfigName": ECS_DEPLOYMENT_CONFIG_NAME,
"ServiceRoleArn": codeDeployServiceRole.role_arn,
"BlueTargetGroup": self.platform_resources.blue_target_name,
"GreenTargetGroup": self.platform_resources.green_target_name,
"ProdListenerArn":
self.platform_resources.prod_listener.listener_arn,
"TestListenerArn":
self.platform_resources.test_listener.listener_arn,
"EcsClusterName":
self.platform_resources.ecs_cluster.cluster_name,
"EcsServiceName": OcticketingAppService.service_name,
"TerminationWaitTime": ECS_TASKSET_TERMINATION_WAIT_TIME,
"BlueGroupAlarm": self.blue_targetAlarm.alarm_name,
"GreenGroupAlarm": self.green_targetAlarm.alarm_name,
})
ecsDeploymentGroup = codedeploy.EcsDeploymentGroup.from_ecs_deployment_group_attributes(
self,
"ecsDeploymentGroup",
application=codeDeployApplication,
deployment_group_name=ECS_DEPLOYMENT_GROUP_NAME,
deployment_config=codedeploy.EcsDeploymentConfig.
from_ecs_deployment_config_name(self, "ecsDeploymentConfig",
ECS_DEPLOYMENT_CONFIG_NAME))
# =============================================================================
# CODE BUILD PROJECT for the Blue/ Green deployment
# =============================================================================
# Creating the code build project
OcticketingAppcodebuild = aws_codebuild.Project(
self,
"OcticketingAppcodebuild",
role=codeBuildServiceRole,
environment=aws_codebuild.BuildEnvironment(
build_image=aws_codebuild.LinuxBuildImage.STANDARD_4_0,
compute_type=aws_codebuild.ComputeType.SMALL,
privileged=True,
environment_variables={
'REPOSITORY_URI': {
'value':
self.octicketing_ecr_repo.repository_uri,
'type':
aws_codebuild.BuildEnvironmentVariableType.PLAINTEXT
},
'TASK_EXECUTION_ARN': {
'value':
ecsTaskRole.role_arn,
'type':
aws_codebuild.BuildEnvironmentVariableType.PLAINTEXT
},
'TASK_FAMILY': {
'value':
ECS_TASK_FAMILY_NAME,
'type':
aws_codebuild.BuildEnvironmentVariableType.PLAINTEXT
}
}),
source=aws_codebuild.Source.code_commit(
repository=self.octicketing_code_repo))
# =============================================================================
# CODE PIPELINE for Blue/Green ECS deployment
# =============================================================================
codePipelineServiceRole = aws_iam.Role(
self,
"codePipelineServiceRole",
assumed_by=aws_iam.ServicePrincipal('codepipeline.amazonaws.com'))
inlinePolicyForCodePipeline = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"iam:PassRole", "sts:AssumeRole", "codecommit:Get*",
"codecommit:List*", "codecommit:GitPull",
"codecommit:UploadArchive", "codecommit:CancelUploadArchive",
"codebuild:BatchGetBuilds", "codebuild:StartBuild",
"codedeploy:CreateDeployment", "codedeploy:Get*",
"codedeploy:RegisterApplicationRevision", "s3:Get*",
"s3:List*", "s3:PutObject"
],
resources=["*"])
codePipelineServiceRole.add_to_policy(inlinePolicyForCodePipeline)
sourceArtifact = codepipeline.Artifact('sourceArtifact')
buildArtifact = codepipeline.Artifact('buildArtifact')
# S3 bucket for storing the code pipeline artifacts
OcticketingAppArtifactsBucket = s3.Bucket(
self,
"OcticketingAppArtifactsBucket",
encryption=s3.BucketEncryption.S3_MANAGED,
block_public_access=s3.BlockPublicAccess.BLOCK_ALL)
# S3 bucket policy for the code pipeline artifacts
denyUnEncryptedObjectUploads = aws_iam.PolicyStatement(
effect=aws_iam.Effect.DENY,
actions=["s3:PutObject"],
principals=[aws_iam.AnyPrincipal()],
resources=[OcticketingAppArtifactsBucket.bucket_arn + "/*"],
conditions={
"StringNotEquals": {
"s3:x-amz-server-side-encryption": "aws:kms"
}
})
denyInsecureConnections = aws_iam.PolicyStatement(
effect=aws_iam.Effect.DENY,
actions=["s3:*"],
principals=[aws_iam.AnyPrincipal()],
resources=[OcticketingAppArtifactsBucket.bucket_arn + "/*"],
conditions={"Bool": {
"aws:SecureTransport": "false"
}})
OcticketingAppArtifactsBucket.add_to_resource_policy(
denyUnEncryptedObjectUploads)
OcticketingAppArtifactsBucket.add_to_resource_policy(
denyInsecureConnections)
# Code Pipeline - CloudWatch trigger event is created by CDK
codepipeline.Pipeline(
self,
"ecsBlueGreen",
role=codePipelineServiceRole,
artifact_bucket=OcticketingAppArtifactsBucket,
stages=[
codepipeline.StageProps(
stage_name='Source',
actions=[
aws_codepipeline_actions.CodeCommitSourceAction(
action_name='Source',
repository=self.octicketing_code_repo,
output=sourceArtifact,
)
]),
codepipeline.StageProps(
stage_name='Build',
actions=[
aws_codepipeline_actions.CodeBuildAction(
action_name='Build',
project=OcticketingAppcodebuild,
input=sourceArtifact,
outputs=[buildArtifact])
]),
codepipeline.StageProps(
stage_name='Deploy',
actions=[
aws_codepipeline_actions.CodeDeployEcsDeployAction(
action_name='Deploy',
deployment_group=ecsDeploymentGroup,
app_spec_template_input=buildArtifact,
task_definition_template_input=buildArtifact,
)
])
])
# =============================================================================
# Export the outputs
# =============================================================================
core.CfnOutput(
self,
"ecsBlueGreenCodeRepo",
description="Demo app code commit repository",
export_name="ecsBlueGreenDemoAppRepo",
value=self.octicketing_code_repo.repository_clone_url_http)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Step Function Starts Here
# The first thing we need to do is see if they are asking for pineapple on a pizza
pineapple_check_lambda = _lambda.Function(
self,
"pineappleCheckLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="orderPizza.handler",
code=_lambda.Code.from_asset("lambda_fns"),
)
# Step functions are built up of steps, we need to define our first step
order_pizza = step_fn_tasks.LambdaInvoke(
self,
'Order Pizza Job',
lambda_function=pineapple_check_lambda,
input_path='$.flavour',
result_path='$.pineappleAnalysis',
payload_response_only=True)
# Pizza Order failure step defined
pineapple_detected = step_fn.Fail(self,
'Sorry, We Dont add Pineapple',
cause='They asked for Pineapple',
error='Failed To Make Pizza')
# If they didnt ask for pineapple let's cook the pizza
cook_pizza = step_fn.Succeed(self,
'Lets make your pizza',
output_path='$.pineappleAnalysis')
# If they ask for a pizza with pineapple, fail. Otherwise cook the pizza
definition = step_fn.Chain \
.start(order_pizza) \
.next(step_fn.Choice(self, 'With Pineapple?')
.when(step_fn.Condition.boolean_equals('$.pineappleAnalysis.containsPineapple', True),
pineapple_detected)
.otherwise(cook_pizza))
state_machine = step_fn.StateMachine(
self,
'StateMachine',
definition=definition,
timeout=core.Duration.minutes(5),
tracing_enabled=True,
state_machine_type=step_fn.StateMachineType.EXPRESS)
# HTTP API Definition
# Give our gateway permissions to interact with SNS
http_api_role = iam.Role(
self,
'HttpApiRole',
assumed_by=iam.ServicePrincipal('apigateway.amazonaws.com'),
inline_policies={
"AllowSFNExec":
iam.PolicyDocument(statements=[
iam.PolicyStatement(
actions=["states:StartSyncExecution"],
effect=iam.Effect.ALLOW,
resources=[state_machine.state_machine_arn])
])
})
api = api_gw.HttpApi(self,
'the_state_machine_api',
create_default_stage=True)
# create an AWS_PROXY integration between the HTTP API and our Step Function
integ = api_gw.CfnIntegration(
self,
'Integ',
api_id=api.http_api_id,
integration_type='AWS_PROXY',
connection_type='INTERNET',
integration_subtype='StepFunctions-StartSyncExecution',
credentials_arn=http_api_role.role_arn,
request_parameters={
"Input": "$request.body",
"StateMachineArn": state_machine.state_machine_arn
},
payload_format_version="1.0",
timeout_in_millis=10000)
api_gw.CfnRoute(self,
'DefaultRoute',
api_id=api.http_api_id,
route_key=api_gw.HttpRouteKey.DEFAULT.key,
target="integrations/" + integ.ref)
core.CfnOutput(self, 'HTTP API URL', value=api.url)
def __init__(self, scope: core.Construct, id: str, eksname: str,
solution_id: str, version: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.template_options.description = "(SO0141) SQL based ETL with Apache Spark on Amazon EKS. This solution provides a SQL based ETL option with a open-source declarative framework powered by Apache Spark."
source_dir = os.path.split(os.environ['VIRTUAL_ENV'])[0] + '/source'
# Cloudformation input params
datalake_bucket = core.CfnParameter(
self,
"datalakebucket",
type="String",
description=
"Your existing S3 bucket to be accessed by Jupyter Notebook and ETL job. Default: blank",
default="")
login_name = core.CfnParameter(
self,
"jhubuser",
type="String",
description="Your username login to jupyter hub",
default="sparkoneks")
# Auto-generate a user login in secrets manager
key = kms.Key(self,
'KMSKey',
removal_policy=core.RemovalPolicy.DESTROY,
enable_key_rotation=True)
key.add_alias("alias/secretsManager")
jhub_secret = secmger.Secret(
self,
'jHubPwd',
generate_secret_string=secmger.SecretStringGenerator(
exclude_punctuation=True,
secret_string_template=json.dumps(
{'username': login_name.value_as_string}),
generate_string_key="password"),
removal_policy=core.RemovalPolicy.DESTROY,
encryption_key=key)
# 1. a new bucket to store app code and logs
self.app_s3 = S3AppCodeConst(self, 'appcode')
# 2. push docker image to ECR via AWS CICD pipeline
ecr_image = DockerPipelineConstruct(self, 'image',
self.app_s3.artifact_bucket)
ecr_image.node.add_dependency(self.app_s3)
core.CfnOutput(self, 'IMAGE_URI', value=ecr_image.image_uri)
# 3. EKS base infrastructure
network_sg = NetworkSgConst(self, 'network-sg', eksname,
self.app_s3.code_bucket)
iam = IamConst(self, 'iam_roles', eksname)
eks_cluster = EksConst(self, 'eks_cluster', eksname, network_sg.vpc,
iam.managed_node_role, iam.admin_role)
EksSAConst(self, 'eks_sa', eks_cluster.my_cluster, jhub_secret)
base_app = EksBaseAppConst(self, 'eks_base_app',
eks_cluster.my_cluster)
# 4. Spark app access control
app_security = SparkOnEksSAConst(self, 'spark_service_account',
eks_cluster.my_cluster,
login_name.value_as_string,
self.app_s3.code_bucket,
datalake_bucket.value_as_string)
app_security.node.add_dependency(base_app.secret_created)
# 5. Install Arc Jupyter notebook in EKS
jhub_install = eks_cluster.my_cluster.add_helm_chart(
'JHubChart',
chart='jupyterhub',
repository='https://jupyterhub.github.io/helm-chart',
release='jhub',
version='0.11.1',
namespace='jupyter',
create_namespace=False,
values=load_yaml_replace_var_local(
source_dir + '/app_resources/jupyter-values.yaml',
fields={
"{{codeBucket}}": self.app_s3.code_bucket,
"{{region}}": core.Aws.REGION
}))
jhub_install.node.add_dependency(app_security)
# EKS get Jupyter login dynamically from secrets manager
name_parts = core.Fn.split('-', jhub_secret.secret_name)
name_no_suffix = core.Fn.join(
'-',
[core.Fn.select(0, name_parts),
core.Fn.select(1, name_parts)])
config_hub = eks.KubernetesManifest(
self,
'JHubConfig',
cluster=eks_cluster.my_cluster,
manifest=load_yaml_replace_var_local(
source_dir + '/app_resources/jupyter-config.yaml',
fields={
"{{MY_SA}}": app_security.jupyter_sa,
"{{REGION}}": core.Aws.REGION,
"{{SECRET_NAME}}": name_no_suffix
},
multi_resource=True))
config_hub.node.add_dependency(jhub_install)
# 6. Install ETL orchestrator - Argo in EKS
# can be replaced by other workflow tool, eg. Airflow
argo_install = eks_cluster.my_cluster.add_helm_chart(
'ARGOChart',
chart='argo-workflows',
repository='https://argoproj.github.io/argo-helm',
release='argo',
version='0.1.4',
namespace='argo',
create_namespace=True,
values=load_yaml_local(source_dir +
'/app_resources/argo-values.yaml'))
argo_install.node.add_dependency(config_hub)
# Create argo workflow template for Spark with T-shirt size
submit_tmpl = eks_cluster.my_cluster.add_manifest(
'SubmitSparkWrktmpl',
load_yaml_local(source_dir + '/app_resources/spark-template.yaml'))
submit_tmpl.node.add_dependency(argo_install)
# 7. (OPTIONAL) retrieve ALB DNS Name to enable CloudFront in the nested stack.
# It is used to serve HTTPS requests with its default domain name.
# Recommend to issue your own TLS certificate, and delete the CF components.
self._jhub_alb = eks.KubernetesObjectValue(
self,
'jhubALB',
cluster=eks_cluster.my_cluster,
json_path='..status.loadBalancer.ingress[0].hostname',
object_type='ingress.networking',
object_name='jupyterhub',
object_namespace='jupyter',
timeout=core.Duration.minutes(10))
self._jhub_alb.node.add_dependency(config_hub)
self._argo_alb = eks.KubernetesObjectValue(
self,
'argoALB',
cluster=eks_cluster.my_cluster,
json_path='..status.loadBalancer.ingress[0].hostname',
object_type='ingress.networking',
object_name='argo-argo-workflows-server',
object_namespace='argo',
timeout=core.Duration.minutes(10))
self._argo_alb.node.add_dependency(argo_install)
# 8. (OPTIONAL) Send solution metrics to AWS
# turn it off from the CloudFormation mapping section if prefer.
send_metrics = solution_metrics.SendAnonymousData(
self,
"SendMetrics",
network_sg.vpc,
self.app_s3.artifact_bucket,
self.app_s3.s3_deploy_contrust,
metrics={
"Solution":
solution_id,
"Region":
core.Aws.REGION,
"SolutionVersion":
version,
"UUID":
"MY_UUID",
"UseDataLakeBucket":
"True" if not datalake_bucket.value_as_string else "False",
"UseAWSCICD":
"True" if ecr_image.image_uri else "False",
"NoAZs":
len(network_sg.vpc.availability_zones)
})
send_metrics.node.add_dependency(self.app_s3.s3_deploy_contrust)
# 9. (OPTIONAL) Override the cfn Nag rules for AWS Solution CICD deployment
# remove the section if your CI/CD pipeline doesn't use the cfn_nag utility to validate the CFN.
k8s_ctl_node = self.node.find_child(
'@aws-cdk--aws-eks.KubectlProvider')
cluster_resrc_node = self.node.find_child(
'@aws-cdk--aws-eks.ClusterResourceProvider')
scan.suppress_cfnnag_rule(
'W12', 'by default the role has * resource',
self.node.find_child('eks_cluster').node.find_child('EKS').node.
default_child.node.find_child('CreationRole').node.find_child(
'DefaultPolicy').node.default_child)
scan.suppress_cfnnag_rule(
'W11', 'by default the role has * resource',
self.node.find_child(
'Custom::AWSCDKOpenIdConnectProviderCustomResourceProvider').
node.find_child('Role'))
scan.suppress_lambda_cfnnag_rule(
k8s_ctl_node.node.find_child('Handler').node.default_child)
scan.suppress_lambda_cfnnag_rule(
k8s_ctl_node.node.find_child('Provider').node.find_child(
'framework-onEvent').node.default_child)
scan.suppress_lambda_cfnnag_rule(
self.node.find_child(
'Custom::CDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756C').
node.default_child)
# scan.suppress_lambda_cfnnag_rule(self.node.find_child('Custom::S3AutoDeleteObjectsCustomResourceProvider').node.find_child('Handler'))
scan.suppress_lambda_cfnnag_rule(
self.node.find_child(
'Custom::AWSCDKOpenIdConnectProviderCustomResourceProvider').
node.find_child('Handler'))
scan.suppress_lambda_cfnnag_rule(
self.node.find_child('AWSCDKCfnUtilsProviderCustomResourceProvider'
).node.find_child('Handler'))
scan.suppress_lambda_cfnnag_rule(
cluster_resrc_node.node.find_child(
'OnEventHandler').node.default_child)
scan.suppress_lambda_cfnnag_rule(
cluster_resrc_node.node.find_child(
'IsCompleteHandler').node.default_child)
scan.suppress_lambda_cfnnag_rule(
cluster_resrc_node.node.find_child('Provider').node.find_child(
'framework-isComplete').node.default_child)
scan.suppress_lambda_cfnnag_rule(
cluster_resrc_node.node.find_child('Provider').node.find_child(
'framework-onTimeout').node.default_child)
scan.suppress_lambda_cfnnag_rule(
cluster_resrc_node.node.find_child('Provider').node.find_child(
'framework-onEvent').node.default_child)
scan.suppress_network_cfnnag_rule(
self.node.find_child('eks_cluster').node.find_child('EKS').node.
find_child('ControlPlaneSecurityGroup').node.default_child)
scan.suppress_lambda_cfnnag_rule(
self.node.find_child('SendMetrics').node.find_child(
'LambdaProvider').node.find_child(
'framework-onEvent').node.default_child)
scan.suppress_network_cfnnag_rule(
self.node.find_child('SendMetrics').node.find_child(
'LambdaProvider').node.find_child('framework-onEvent').node.
find_child('SecurityGroup').node.default_child)
scan.suppress_lambda_cfnnag_rule(
self.node.find_child(
'SingletonLambda75248a819138468c9ba1bca6c7137599').node.
default_child)
scan.suppress_network_cfnnag_rule(
self.node.find_child(
'SingletonLambda75248a819138468c9ba1bca6c7137599').node.
find_child('SecurityGroup').node.default_child)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
# TODO: Create S3 2 buckets:
# RemovalPolicy should be left to the default RETAIN in production to
# avoid any data loss. S3 buckets should be manually deleted.
data_bucket = s3.Bucket(
self, "-data-", removal_policy=core.RemovalPolicy.DESTROY
)
processed_logs_bucket = s3.Bucket(
self, "-processed-logs-", removal_policy=core.RemovalPolicy.DESTROY
)
# TODO: Change destination bucket in the lambda function
process_logs_lambda = _lambda.Function(
self,
"ProcessLogs",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset("lambda"),
handler="process_logs.handler",
environment={"PROCESSED_LOGS_BUCKET": processed_logs_bucket.node.unique_id},
)
data_bucket.add_event_notification(
event=s3.EventType.OBJECT_CREATED, dest=process_logs_lambda
)
# TODO: Create the datalake DB in lake formation
# db_name = id + "-db"
# Check Upgrading AWS Glue Data Permissions to the AWS Lake Formation Model: https://docs.aws.amazon.com/lake-formation/latest/dg/upgrade-glue-lake-formation.html
# TODO: Create Glue crawlers
# One crawler for mysql data:
# Source type: Data stores
# Data store: s3
# Crawl data in specified paths
# Path: s3://data_bucket_name/<table name>
# Add as many paths as tables
# IAM role: LakeFormationWorkflowRole
# Run only once to create the tables
# Database: db_name
# Advanced settings:
# Grouping behavior for S3 data (optional):
# Create a single schema for each S3 path
# When the crawler detects schema changes in the data store, how should AWS Glue handle table updates in the data catalog?
# Ignore the change and don't update the table in the data catalog
# Update all new and existing partitions with metadata from the table
# How should AWS Glue handle deleted objects in the data store?
# Mark the table as deprecated in the data catalog
# TODO: Crate Glue jobs
# TODO: Create Athena link to datalake
# TODO: Create Quicksight basic reports
core.CfnOutput(self, "{}-unique-id".format(id), value=self.node.unique_id)
def __init__(self, scope: core.Construct, id: str, vpc, ecs_cluster, role, target_url: str, number_of_tasks = 1, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
name = id
task_def = ecs.Ec2TaskDefinition(self, name,
network_mode=ecs.NetworkMode.AWS_VPC
)
if role == "slave":
container_env={"TARGET_URL": target_url,
"LOCUST_MODE": role,
#Need to update to pull the name from Cloudmap
"LOCUST_MASTER_HOST": "master.loadgen"
}
else:
container_env={"TARGET_URL": target_url,
"LOCUST_MODE": role
}
locust_container = task_def.add_container(
name + "container",
# Use Locust image from DockerHub
# Or not. we'll use an image we create using the dockerfile in ./locust
image=ecs.ContainerImage.from_asset("locust"),
memory_reservation_mib=512,
essential=True,
logging=ecs.LogDrivers.aws_logs(stream_prefix=name),
environment=container_env
)
web_port_mapping = ecs.PortMapping(container_port=8089)
if role != "standalone":
slave1_port_mapping = ecs.PortMapping(container_port=5557)
slave2_port_mapping = ecs.PortMapping(container_port=5558)
locust_container.add_port_mappings(web_port_mapping,slave1_port_mapping,slave2_port_mapping)
else:
locust_container.add_port_mappings(web_port_mapping)
security_group = ec2.SecurityGroup(
self, "Locust",
vpc=vpc,
allow_all_outbound=True
)
security_group.add_ingress_rule(
ec2.Peer.any_ipv4(),
ec2.Port.tcp(8089)
)
if role != "standalone":
security_group.add_ingress_rule(
ec2.Peer.any_ipv4(),
ec2.Port.tcp(5557)
)
security_group.add_ingress_rule(
ec2.Peer.any_ipv4(),
ec2.Port.tcp(5558)
)
# Create the ecs service
locust_service = ecs.Ec2Service(
self, name +"service",
cluster = ecs_cluster,
task_definition = task_def,
security_group = security_group,
desired_count = number_of_tasks
)
locust_service.enable_cloud_map(name=role)
# Create the ALB to present the Locust UI
if role != "slave":
self.lb = elbv2.ApplicationLoadBalancer(self, "LoustLB", vpc=vpc, internet_facing=True)
listener = self.lb.add_listener("Listener", port=80)
listener.add_targets("ECS1",
port=80,
targets=[locust_service]
)
core.CfnOutput(
self, "lburl",
description = "URL for ALB fronting locust master",
value = self.lb.load_balancer_dns_name
)
def __init__(self, scope: core.Construct, id: str, props,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
alb_subnet = ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.PUBLIC,
name="ALB",
cidr_mask=24)
db_subnet = ec2.SubnetConfiguration(
subnet_type=ec2.SubnetType.ISOLATED, name="DB", cidr_mask=24)
# VPC
vpc = ec2.Vpc(
self,
"VPC",
max_azs=2,
cidr="10.10.0.0/16",
# configuration will create 2 groups in 2 AZs = 4 subnets.
subnet_configuration=[alb_subnet, db_subnet],
nat_gateway_provider=ec2.NatProvider.gateway(),
nat_gateways=1,
)
# Security groups
# Create Security group that allows traffic into the ALB
alb_security_group = ec2.SecurityGroup(
self,
"ALBSecurityGroup",
description="Ghost ALB Security Group",
vpc=vpc)
alb_security_group.add_ingress_rule(ec2.Peer.any_ipv4(),
ec2.Port.tcp(80),
"allow HTTP to ALB")
alb_security_group.add_ingress_rule(ec2.Peer.any_ipv4(),
ec2.Port.tcp(443),
"allow HTTPS to ALB")
# Create Security group for the host/ENI/Fargate that allows 2368
fargate_security_group = ec2.SecurityGroup(
self,
"FargateSecurityGroup",
description="Ghost ECS Fargate Security Group",
vpc=vpc)
fargate_security_group.add_ingress_rule(
alb_security_group, ec2.Port.tcp(2368),
"allow ghost default 2368 to fargate")
# Create the DB's Security group which only allows access to memebers of the Ghost Fargate SG
db_security_group = ec2.SecurityGroup(
self,
"DBSecurityGroup",
description="Security group for RDS DB Instance for ghost cms",
vpc=vpc)
db_security_group.add_ingress_rule(
fargate_security_group, ec2.Port.tcp(3306),
"allow ghost fargate host to connect to db")
ghost_alb = elb.ApplicationLoadBalancer(
self,
"GhostALB",
internet_facing=True,
vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC),
security_group=alb_security_group,
vpc=vpc)
ghost_target_health_check = elb.HealthCheck(
interval=core.Duration.seconds(30),
protocol=elb.Protocol.HTTP,
timeout=core.Duration.seconds(10),
healthy_threshold_count=4,
unhealthy_threshold_count=3,
healthy_http_codes="200,301")
ghost_target_group = elb.ApplicationTargetGroup(
self,
"GhostTargetGroup",
port=2368,
protocol=elb.Protocol.HTTP,
vpc=vpc,
health_check=ghost_target_health_check,
target_type=elb.TargetType.IP)
ghost_alb_listener = elb.ApplicationListener(
self,
"Listener80",
port=80,
protocol=elb.Protocol.HTTP,
load_balancer=ghost_alb,
default_target_groups=[ghost_target_group])
core.CfnOutput(self, "vpcid", value=vpc.vpc_id)
core.CfnOutput(self,
"alb_url",
description="ALB URL",
value=ghost_alb.load_balancer_dns_name)
self.output_props = props.copy()
self.output_props['vpc'] = vpc
self.output_props['subnets'] = vpc.public_subnets
self.output_props['alb_security_group'] = alb_security_group
self.output_props['fargate_security_group'] = fargate_security_group
self.output_props['db_security_group'] = db_security_group
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
prj_name = self.node.try_get_context("project_name")
env_name = self.node.try_get_context("env")
account_id = core.Aws.ACCOUNT_ID
lambda_bucket = s3.Bucket(
self,
'lambda-bucket',
access_control=s3.BucketAccessControl.BUCKET_OWNER_FULL_CONTROL,
encryption=s3.BucketEncryption.S3_MANAGED,
bucket_name=account_id + '-' + env_name +
'-lambda-deploy-packages',
block_public_access=s3.BlockPublicAccess(
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=True),
removal_policy=core.RemovalPolicy.RETAIN)
ssm.StringParameter(self,
'ssm-lambda-bucket',
parameter_name='/' + env_name +
'/lambda-s3-bucket',
string_value=lambda_bucket.bucket_name)
#To Store Build Artifacts
artifacts_bucket = s3.Bucket(
self,
"build-artifacts",
access_control=s3.BucketAccessControl.BUCKET_OWNER_FULL_CONTROL,
encryption=s3.BucketEncryption.S3_MANAGED,
bucket_name=account_id + '-' + env_name + '-build-artifacts',
block_public_access=s3.BlockPublicAccess(
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=True),
removal_policy=core.RemovalPolicy.DESTROY)
core.CfnOutput(self,
's3-build-artifacts-export',
value=artifacts_bucket.bucket_name,
export_name='build-artifacts-bucket')
#To Store Frontend App
frontend_bucket = s3.Bucket(
self,
"frontend",
access_control=s3.BucketAccessControl.BUCKET_OWNER_FULL_CONTROL,
encryption=s3.BucketEncryption.S3_MANAGED,
bucket_name=account_id + '-' + env_name + '-frontend',
block_public_access=s3.BlockPublicAccess(
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=True))
core.CfnOutput(self,
's3-frontend-export',
value=frontend_bucket.bucket_name,
export_name='frontend-bucket')
#CloudTrail Bucket
self.cloudtrail_bucket = s3.Bucket(
self,
"cloudtrail",
access_control=s3.BucketAccessControl.BUCKET_OWNER_FULL_CONTROL,
encryption=s3.BucketEncryption.S3_MANAGED,
bucket_name=account_id + '-' + env_name + '-cloudtrail',
block_public_access=s3.BlockPublicAccess(
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=True))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# store
dynamodb_table = dynamodb.Table(
self,
'dynamodb_table',
table_name=f'{PROJECT}_{STAGE}',
partition_key=dynamodb.Attribute(
name='date', type=dynamodb.AttributeType.STRING),
billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
point_in_time_recovery=False,
removal_policy=core.RemovalPolicy.DESTROY,
server_side_encryption=True,
)
# public api
public_api = appsync.CfnGraphQLApi(
self,
'public_api',
name=f'{PROJECT}_{STAGE}',
authentication_type='API_KEY',
)
now = time.localtime()
epoch = time.mktime(now)
public_api_key = appsync.CfnApiKey(
self,
'public_api_key',
api_id=public_api.attr_api_id,
expires=epoch + core.Duration.days(90).to_seconds(),
)
with open('schema.gql', mode='r') as f:
graphql_schema = f.read()
appsync.CfnGraphQLSchema(self,
'public_api_schema',
api_id=public_api.attr_api_id,
definition=graphql_schema)
public_api_role = iam.Role(
self,
'public_api_role',
assumed_by=iam.ServicePrincipal('appsync.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonDynamoDBFullAccess')
],
)
public_api_datasource = appsync.CfnDataSource(
self,
'public_api_datasource',
api_id=public_api.attr_api_id,
name=f'{PROJECT}_{STAGE}_dynamodb',
type='AMAZON_DYNAMODB',
dynamo_db_config={
'awsRegion': 'us-east-1',
'tableName': dynamodb_table.table_name,
},
service_role_arn=public_api_role.role_arn,
)
with open('mapping_templates/get_holiday.json', mode='r') as f:
get_holiday_json = f.read()
appsync.CfnResolver(
self,
'public_api_resolver_get_holiday',
api_id=public_api.attr_api_id,
type_name='Query',
field_name='getHoliday',
data_source_name=public_api_datasource.attr_name,
kind='UNIT',
request_mapping_template=get_holiday_json,
response_mapping_template='$util.toJson($context.result)',
)
with open('mapping_templates/list_holidays.json', mode='r') as f:
list_holidays_json = f.read()
appsync.CfnResolver(
self,
'public_api_resolver_list_holidays',
api_id=public_api.attr_api_id,
type_name='Query',
field_name='listHolidays',
data_source_name=public_api_datasource.attr_name,
kind='UNIT',
request_mapping_template=list_holidays_json,
response_mapping_template='$util.toJson($context.result)',
)
# lambda source code upload to s3
lambda_assets = s3_assets.Asset(self,
'lambda_assets',
path='./function/.artifact/')
# update function
func_api = lambda_.Function(
self,
f'{PROJECT}-{STAGE}-func',
function_name=f'{PROJECT}-{STAGE}-func',
code=lambda_.Code.from_bucket(bucket=lambda_assets.bucket,
key=lambda_assets.s3_object_key),
handler='app.handler',
runtime=lambda_.Runtime.PYTHON_3_7,
timeout=core.Duration.seconds(120),
log_retention=logs.RetentionDays.SIX_MONTHS,
memory_size=128,
tracing=lambda_.Tracing.ACTIVE,
)
func_api.add_environment('TABLE_NAME', dynamodb_table.table_name)
func_api.add_environment('CSV_URL', CSV_URL)
func_api.add_to_role_policy(
iam.PolicyStatement(
actions=[
'dynamodb:Get*',
'dynamodb:Put*',
'dynamodb:Batch*',
],
resources=[dynamodb_table.table_arn],
))
# schedule execute
events.Rule(
self,
f'{PROJECT}-{STAGE}-schedule',
enabled=True,
schedule=events.Schedule.rate(core.Duration.days(10)),
targets=[events_targets.LambdaFunction(func_api)],
)
# lambda@edge
func_lambdaedge = lambda_.Function(
self,
f'{PROJECT}-{STAGE}-func-lambdaedge',
function_name=f'{PROJECT}-{STAGE}-func-lambdaedge',
code=lambda_.Code.from_inline(
open('./function/src/lambdaedge.py').read().replace(
'__X_API_KEY__', public_api_key.attr_api_key)),
handler='index.handler',
runtime=lambda_.Runtime.PYTHON_3_7,
timeout=core.Duration.seconds(30),
memory_size=128,
role=iam.Role(
self,
f'{PROJECT}-{STAGE}-func-lambdaedge-role',
assumed_by=iam.CompositePrincipal(
iam.ServicePrincipal('edgelambda.amazonaws.com'),
iam.ServicePrincipal('lambda.amazonaws.com'),
),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'),
],
),
)
lambdaedge_version = func_lambdaedge.add_version(
hashlib.sha256(
open('./function/src/lambdaedge.py').read().replace(
'__X_API_KEY__',
public_api_key.attr_api_key).encode()).hexdigest())
# ACM
certificates = acm.Certificate(
self,
'certificates',
domain_name=DOMAIN,
validation_method=acm.ValidationMethod.DNS,
)
# CDN
cdn = cloudfront.CloudFrontWebDistribution(
self,
f'{PROJECT}-{STAGE}-cloudfront',
origin_configs=[
cloudfront.SourceConfiguration(
behaviors=[
# default behavior
cloudfront.Behavior(
allowed_methods=cloudfront.
CloudFrontAllowedMethods.ALL,
default_ttl=core.Duration.seconds(0),
max_ttl=core.Duration.seconds(0),
min_ttl=core.Duration.seconds(0),
is_default_behavior=True,
lambda_function_associations=[
cloudfront.LambdaFunctionAssociation(
event_type=cloudfront.LambdaEdgeEventType.
ORIGIN_REQUEST,
lambda_function=lambdaedge_version,
),
])
],
custom_origin_source=cloudfront.CustomOriginConfig(
domain_name=core.Fn.select(
2, core.Fn.split('/',
public_api.attr_graph_ql_url)), ),
)
],
alias_configuration=cloudfront.AliasConfiguration(
acm_cert_ref=certificates.certificate_arn,
names=[DOMAIN],
security_policy=cloudfront.SecurityPolicyProtocol.
TLS_V1_2_2018,
),
price_class=cloudfront.PriceClass.PRICE_CLASS_ALL,
)
core.CfnOutput(
self,
'cloudfront-domain',
value=cdn.domain_name,
)
def __init__(self, scope: core.Construct, id: str, vpc: ec2.Vpc,
inst_sg: ec2.SecurityGroup, role: iam.IRole, keyname: str,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
cfnhup_restart_handle = ec2.InitServiceRestartHandle()
# https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_autoscaling/AutoScalingGroup.html#aws_cdk.aws_autoscaling.AutoScalingGroup.apply_cloud_formation_init
# Note that configuring init metadata also implies that
# cfn-init and cfn-signal are implicitly added to UserData. Signals property must be configured
# creation policy is updated to wait for cfn-init to finish. (no need to specify in user data)
asg = autoscaling.AutoScalingGroup(
self,
"PublicInstanceASG",
role=role,
vpc=vpc,
instance_type=ec2.InstanceType(
instance_type_identifier=instance_type),
machine_image=ec2.AmazonLinuxImage(
edition=ec2.AmazonLinuxEdition.STANDARD,
generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2,
virtualization=ec2.AmazonLinuxVirt.HVM,
storage=ec2.AmazonLinuxStorage.GENERAL_PURPOSE),
user_data=ec2.UserData.custom(core.Fn.sub(user_data_public)),
key_name=keyname,
vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC),
desired_capacity=2,
max_capacity=2,
min_capacity=2,
signals=autoscaling.Signals.wait_for_all(
timeout=core.Duration.minutes(5)),
security_group=inst_sg)
asg_logical_id = str(asg.node.default_child.logical_id)
#print("logical id:"+str(asg.node.default_child.logical_id))
#print("logical id:"+asg.auto_scaling_group_name)
# The shell environment variable $ASGLOGICALID was set correctly in UserData, and persisted to when config set is executed with ec2.InitCommand.
# However, I had a hard time accessing logical resource ID of AutoScalingGroup because ec2.InitFile does not replace shell environment variable with actual value.
# I have tried all the followings to no avail:
# 1. When referenced as ${ASGLOGICALID} or as "${ASGLOGICALID}", it was interpreted as a literal resource name and cause ValidationError
# 2. When referenced as $ASGLOGICALID, it was interpreted
# As a result, I had to split the initialization of asg into three steps.
# 1. initialize asg without init, or init_options
# 2. store logical ID of asg into python variable asg_logical_id
# 3. add init and init_options to asg, which uses the value of asg_logical_id.
asg.apply_cloud_formation_init(ec2.CloudFormationInit.from_config_sets(
config_sets={
"config_set_1": ["config_step1", "config_step2"],
"config_set_2":
["config_step3", "config_step4", "config_step5"]
},
configs={
"config_step1":
ec2.InitConfig([ec2.InitPackage.yum("git")]),
"config_step2":
ec2.InitConfig([
ec2.InitCommand.shell_command(
"echo configset thinks ASGLOGICALID=$ASGLOGICALID")
]),
"config_step3":
ec2.InitConfig([
ec2.InitFile.from_string(
file_name="/etc/cfn/hooks.d/cfn-auto-reloader.conf",
content=core.Fn.sub('\n'.join((
"[cfn-auto-reloader-hook]", "triggers=post.update",
"path=Resources." + asg_logical_id +
".Metadata.AWS::CloudFormation::Init",
"action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource "
+ asg_logical_id +
" --configsets config_set_1,config_set_2"))),
group='root',
owner='root',
mode='000644',
service_restart_handles=[cfnhup_restart_handle])
]),
"config_step4":
ec2.InitConfig([
ec2.InitFile.from_string(
file_name="/etc/cfn/cfn-hup.conf",
content=core.Fn.sub('\n'.join(
("[main]", "stack=${AWS::StackId}",
"region=${AWS::Region}", "verbose=true",
"interval=5"))),
owner='root',
group='root',
mode='000644',
service_restart_handles=[cfnhup_restart_handle])
]),
"config_step5":
ec2.InitConfig([
ec2.InitService.enable(
service_name='cfn-hup',
enabled=True,
ensure_running=True,
service_restart_handle=cfnhup_restart_handle)
])
}),
config_sets=[
"config_set_1", "config_set_2"
],
print_log=True)
core.CfnOutput(self,
"Output",
value='logical resource id of asg: ' +
str(asg.node.default_child.logical_id))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
### S3 ###
source_csv_bucket = _s3.Bucket(self,
"BYODValidationSourceBucket",
versioned=True)
target_csv_bucket = _s3.Bucket(
self,
"BYODValidationTargetBucket",
removal_policy=core.RemovalPolicy.RETAIN)
### Cognito ###
userpool = _cognito.UserPool(self,
"WebToolUserPool",
user_pool_name="byod-webtool-userpool",
self_sign_up_enabled=True,
auto_verify={
"email": True,
"phone": False
},
user_verification={
"email_subject":
"Your verification code",
"email_body":
"Your verification code is {####}",
"email_style":
_cognito.VerificationEmailStyle.CODE
},
standard_attributes={
"email": {
"required": True,
"mutable": False
}
},
password_policy={})
client = userpool.add_client("webtool-app-client",
auth_flows={
"custom": True,
"user_password": True,
"user_srp": True,
"refresh_token": True
})
identity_pool = _cognito.CfnIdentityPool(
self,
"WebToolCognitoIdentityPool",
allow_unauthenticated_identities=True)
identity_pool.add_property_override(
"CognitoIdentityProviders",
[{
"ClientId": client.user_pool_client_id,
"ProviderName": userpool.user_pool_provider_name
}])
auth_role = _iam.Role(
self,
"CognitoAuthRole",
assumed_by=FederatedPrincipal(
"cognito-identity.amazonaws.com", {
"StringEquals": {
"cognito-identity.amazonaws.com:aud": identity_pool.ref
},
"ForAnyValue:StringLike": {
"cognito-identity.amazonaws.com:amr": "authenticated"
}
}))
auth_role.add_to_policy(
PolicyStatement(effect=Effect.ALLOW,
actions=["s3:GetObject"],
resources=["%s/*" % target_csv_bucket.bucket_arn]))
unauth_role = _iam.Role(
self,
"CognitoUnauthRole",
assumed_by=_iam.FederatedPrincipal(
"cognito-identity.amazonaws.com",
conditions={
"StringEquals": {
"cognito-identity.amazonaws.com:aud": identity_pool.ref
},
"ForAnyValue:StringLike": {
"cognito-identity.amazonaws.com:amr": "unauthenticated"
}
}))
identity_pool_policy = _cognito.CfnIdentityPoolRoleAttachment(
self,
"WebToolCognitoIdentityPoolPolicy",
identity_pool_id=identity_pool.ref,
roles={
'unauthenticated': unauth_role.role_arn,
'authenticated': auth_role.role_arn
})
core.CfnOutput(self, "UserPoolId", value=userpool.user_pool_id)
core.CfnOutput(self, "ClientId", value=client.user_pool_client_id)
core.CfnOutput(self,
"ProviderName",
value=userpool.user_pool_provider_name)
### DynamoDB ###
validation_job_table = _dynamodb.Table(
self,
"ValidationJobTable",
partition_key=_dynamodb.Attribute(
name="id", type=_dynamodb.AttributeType.STRING))
## AppSync ###
api = _appsync.GraphqlApi(
self,
"Api",
name="validation-job-api",
schema=_appsync.Schema.from_asset(
os.path.join(dirname, "api", "schema.graphql")),
authorization_config=AuthorizationConfig(
default_authorization=AuthorizationMode(
authorization_type=AuthorizationType.USER_POOL,
user_pool_config=UserPoolConfig(user_pool=userpool))),
log_config=LogConfig(exclude_verbose_content=False,
field_log_level=FieldLogLevel.ALL))
api_ds = api.add_dynamo_db_data_source("ValidationJobDataSource",
validation_job_table)
core.CfnOutput(self, "GraphQLEndpoint", value=api.graphql_url)
### SQS ###
validation_job_queue = _sqs.Queue(self, "ValidationJobQueue")
profiling_job_queue = _sqs.Queue(self, "ProfilingJobQueue")
### Lambda ###
validation_trigger_function = _lambda.Function(
self,
"ValidationTriggerFunction",
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset(
os.path.join(dirname, "lambda", "validation_trigger")),
handler='lambda_function.lambda_handler')
validation_trigger_function.add_environment(
"TABLE_NAME", validation_job_table.table_name)
validation_trigger_function.add_environment(
"QUEUE_URL", validation_job_queue.queue_url)
validation_trigger_function.add_event_source(
_S3EventSource(source_csv_bucket,
events=[_s3.EventType.OBJECT_CREATED]))
source_csv_bucket.grant_read(validation_trigger_function)
validation_job_table.grant_read_write_data(validation_trigger_function)
validation_job_queue.grant_send_messages(validation_trigger_function)
stager_function = _lambda.Function(self,
"StagerFunction",
runtime=_lambda.Runtime.NODEJS_12_X,
code=_lambda.Code.from_asset(
os.path.join(
dirname, "lambda",
"stager")),
handler='index.handler')
stager_function.add_environment("REGION", self.region)
stager_function.add_environment("SOURCE_BUCKET",
source_csv_bucket.bucket_name)
stager_function.add_environment("STAGE_BUCKET",
target_csv_bucket.bucket_name)
source_csv_bucket.grant_read(stager_function)
target_csv_bucket.grant_put(stager_function)
### ECS Fargate ###
validation_fargate_asset = _ecr_assets.DockerImageAsset(
self,
"ValidationBuildImage",
directory=os.path.join(dirname, "fargate", "validation"))
profiling_fargate_asset = _ecr_assets.DockerImageAsset(
self,
"ProfilingBuildImage",
directory=os.path.join(dirname, "fargate", "profiling"))
vpc = _ec2.Vpc(self, "VPC", max_azs=3)
cluster = _ecs.Cluster(self, "ECSCluster", vpc=vpc)
validation_fargate_service = _ecs_patterns.QueueProcessingFargateService(
self,
"ValidationFargateService",
cluster=cluster,
cpu=4096,
memory_limit_mib=30720,
enable_logging=True,
image=_ecs.ContainerImage.from_docker_image_asset(
validation_fargate_asset),
environment={
"TABLE_NAME": validation_job_table.table_name,
"QUEUE_URL": validation_job_queue.queue_url,
"SOURCE_BUCKET_NAME": source_csv_bucket.bucket_name,
"TARGET_BUCKET_NAME": target_csv_bucket.bucket_name,
"REGION": self.region
},
queue=validation_job_queue,
max_scaling_capacity=2,
max_healthy_percent=200,
min_healthy_percent=66)
validation_fargate_service.task_definition.task_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonDynamoDBFullAccess"))
validation_fargate_service.task_definition.task_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3FullAccess"))
profiling_fargate_service = _ecs_patterns.QueueProcessingFargateService(
self,
"ProfilingFargateService",
cluster=cluster,
cpu=4096,
memory_limit_mib=30720,
enable_logging=True,
image=_ecs.ContainerImage.from_docker_image_asset(
profiling_fargate_asset),
environment={
"TABLE_NAME": validation_job_table.table_name,
"QUEUE_URL": profiling_job_queue.queue_url,
"SOURCE_BUCKET_NAME": source_csv_bucket.bucket_name,
"TARGET_BUCKET_NAME": target_csv_bucket.bucket_name,
"REGION": self.region
},
queue=profiling_job_queue,
max_scaling_capacity=2,
max_healthy_percent=200,
min_healthy_percent=66)
profiling_fargate_service.task_definition.task_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonDynamoDBFullAccess"))
profiling_fargate_service.task_definition.task_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3FullAccess"))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
work_dir = pathlib.Path(__file__).parents[1]
# These below steps allows to reuse ecs cluster which is aleady creatd by shared stack
# Get cluster name from ssm parameter
cluster_name = ssm.StringParameter.from_string_parameter_name(
self,
"GetClusterName",
string_parameter_name="/dev/compute/container/ecs-cluster-name"
).string_value
vpc_az = ssm.StringListParameter.from_string_list_parameter_name(
self,
"GetVpcAz",
string_list_parameter_name="/dev/network/vpc/vpc-az"
).string_list_value
# using string instead of stringlist because of subnets parsing issue
vpc_public_subnets_1 = ssm.StringParameter.from_string_parameter_name(
self,
"GetVpcPublicSubnets1",
string_parameter_name="/dev/network/vpc/vpc-public-subnets-1"
).string_value
vpc_public_subnets_2 = ssm.StringParameter.from_string_parameter_name(
self,
"GetVpcPublicSubnets2",
string_parameter_name="/dev/network/vpc/vpc-public-subnets-2"
).string_value
vpc_id = ssm.StringParameter.from_string_parameter_name(
self, "GetVpcId",
string_parameter_name="/dev/network/vpc/vpc-id").string_value
ec2_vpc = ec2.Vpc.from_vpc_attributes(
self,
"GetVpc",
availability_zones=vpc_az,
vpc_id=vpc_id,
public_subnet_ids=[vpc_public_subnets_1, vpc_public_subnets_2])
# Get security group id from ssm parameter
security_group_id = ssm.StringParameter.from_string_parameter_name(
self,
"GetSgId",
string_parameter_name="/dev/network/vpc/security-group-id"
).string_value
# Get security group from lookup
ec2_sgp = ec2.SecurityGroup.from_security_group_id(
self, "GetSgp", security_group_id=security_group_id)
# Pass vpc, sgp and ecs cluster name to get ecs cluster info
ecs_cluster = ecs.Cluster.from_cluster_attributes(
self,
"GetEcsCluster",
cluster_name=cluster_name,
vpc=ec2_vpc,
security_groups=[ec2_sgp])
# Fargate Service
task_definition = ecs.FargateTaskDefinition(self,
"TaskDef",
memory_limit_mib=512,
cpu=256)
container = task_definition.add_container(
"web",
image=ecs.ContainerImage.from_asset(
os.path.join(work_dir, "container")),
environment=dict(name="latest"))
port_mapping = ecs.PortMapping(container_port=8000,
protocol=ecs.Protocol.TCP)
container.add_port_mappings(port_mapping)
# Create Fargate Service
service = ecs.FargateService(
self,
"Service",
cluster=ecs_cluster,
task_definition=task_definition,
assign_public_ip=True,
deployment_controller=ecs.DeploymentController(
type=ecs.DeploymentControllerType.CODE_DEPLOY))
task_definition_rev = ecs.FargateTaskDefinition(
self,
"TaskDefinitionNew",
cpu=256,
memory_limit_mib=512,
family=task_definition.family)
cfn_task_definition = task_definition_rev.node.default_child
cfn_task_definition.cfn_options.update_replace_policy = core.CfnDeletionPolicy.RETAIN
container_rev = task_definition_rev.add_container(
"docker",
image=ecs.ContainerImage.from_asset(
os.path.join(work_dir, "container_temp")),
environment=dict(name="docker-new"))
container_rev.add_port_mappings(port_mapping)
# Create Application LoadBalancer
lb = elbv2.ApplicationLoadBalancer(self,
"LB",
vpc=ec2_vpc,
internet_facing=True)
# Add test listener to the LB
test_listener = lb.add_listener("TestListener", port=8080, open=True)
# Route to prod container
test_listener.add_targets("TestFargate", port=8080, targets=[service])
# Add prod listener to the LB
prod_listener = lb.add_listener("ProdListener", port=80, open=True)
# Route to prod container
prod_listener.add_targets("ProdFargate", port=80, targets=[service])
# add an output with a well-known name to read it from the integ tests
url_output = core.CfnOutput(
self, "UrlOutput", value=f"http://{lb.load_balancer_dns_name}")
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
"""
First step: Create MediaPackage Channel
"""
channel = mediapackage.CfnChannel(
scope=self,
id_="media-package-channel-{0}".format(
DEFAULT_CONF.get("id_channel")),
id=DEFAULT_CONF.get("id_channel"),
description="Channel {0}".format(DEFAULT_CONF.get("id_channel")))
"""
Second step: Add a HLS endpoint to MediaPackage Channel and output the URL of this endpoint
"""
hsl_endpoint_package = mediapackage.CfnOriginEndpoint.HlsPackageProperty(
segment_duration_seconds=DEFAULT_CONF.get(
"hls_segment_duration_seconds"),
playlist_window_seconds=DEFAULT_CONF.get(
"hls_playlist_window_seconds"),
stream_selection=mediapackage.CfnOriginEndpoint.
StreamSelectionProperty(
max_video_bits_per_second=DEFAULT_CONF.get(
"hls_max_video_bits_per_second"),
min_video_bits_per_second=DEFAULT_CONF.get(
"hls_min_video_bits_per_second"),
stream_order=DEFAULT_CONF.get("hls_stream_order")))
hls_endpoint = mediapackage.CfnOriginEndpoint(
scope=self,
id_="endpoint-{0}".format(DEFAULT_CONF.get("id_channel")),
id=DEFAULT_CONF.get("id_channel"),
channel_id=format(DEFAULT_CONF.get("id_channel")),
description="Endpoint - {0}".format(
DEFAULT_CONF.get("id_channel")),
hls_package=hsl_endpoint_package)
# Output the url stream to player
core.CfnOutput(scope=self,
id="media-package-url-stream",
value=hls_endpoint.attr_url)
"""
Third step: Create MediaLive SG, MediaLive Input and MediaLive Channel
"""
"""
Input Security Group
Allow 0.0.0.0/0 - Modify it if you want """
security_groups_input = medialive.CfnInputSecurityGroup(
scope=self,
id="media-live-sg-input",
whitelist_rules=[{
"cidr": DEFAULT_CONF.get("ip_sg_input")
}])
""" Input destination """
media_live_input_destination = medialive.CfnInput.InputDestinationRequestProperty(
stream_name=DEFAULT_CONF.get("stream_name"))
""" Input with destinations output """
medialive_input = medialive.CfnInput(
scope=self,
id="media-input-channel",
name="input-test1",
type="RTMP_PUSH",
input_security_groups=[security_groups_input.ref],
destinations=[media_live_input_destination])
""" Media Live Channel Block """
media_live_channel_input_spec = medialive.CfnChannel.InputSpecificationProperty(
codec="AVC", maximum_bitrate="MAX_20_MBPS", resolution="HD")
media_live_channel_input_attach = medialive.CfnChannel.InputAttachmentProperty(
input_id=medialive_input.ref,
input_attachment_name="attach-input-test1")
media_live_channel_destination_settings = medialive.CfnChannel.MediaPackageOutputDestinationSettingsProperty(
channel_id=DEFAULT_CONF.get("id_channel"))
media_live_channel_destination = medialive.CfnChannel.OutputDestinationProperty(
id="media-destination",
media_package_settings=[media_live_channel_destination_settings])
"""
We need to control bitrate based on user connection. The manifest will include 2 video quality:
video_720p30 = bitrate 3000000
video_1080p30 = bitrate 5000000
Let's go create them
You can create more resolutions based on bitrate, like:
video_240p30 = bitrate 750000
video_480p30 = bitrate 1500000
"""
# Audio + output + video
""" Audio Section """
audio_aac = medialive.CfnChannel.AacSettingsProperty(
bitrate=192000,
coding_mode="CODING_MODE_2_0",
input_type="NORMAL",
profile="LC",
rate_control_mode="CBR",
raw_format="NONE",
sample_rate=48000,
spec="MPEG4")
audio_codec = medialive.CfnChannel.AudioCodecSettingsProperty(
aac_settings=audio_aac)
audio1 = medialive.CfnChannel.AudioDescriptionProperty(
audio_selector_name="Default",
audio_type_control="FOLLOW_INPUT",
language_code_control="FOLLOW_INPUT",
name="audio_1",
codec_settings=audio_codec)
audio2 = medialive.CfnChannel.AudioDescriptionProperty(
audio_selector_name="Default",
audio_type_control="FOLLOW_INPUT",
language_code_control="FOLLOW_INPUT",
name="audio_2",
codec_settings=audio_codec)
""" Output Section """
output_mediapackage_destination_ref = medialive.CfnChannel.OutputLocationRefProperty(
destination_ref_id="media-destination")
output_mediapackage_destination = medialive.CfnChannel.MediaPackageGroupSettingsProperty(
destination=output_mediapackage_destination_ref)
output_settings = medialive.CfnChannel.OutputGroupSettingsProperty(
media_package_group_settings=output_mediapackage_destination)
output_output1 = medialive.CfnChannel.OutputProperty(
audio_description_names=["audio_1"],
output_name="1080p30",
video_description_name="video_1080p30",
output_settings=medialive.CfnChannel.OutputSettingsProperty(
media_package_output_settings={}))
output_output2 = medialive.CfnChannel.OutputProperty(
audio_description_names=["audio_2"],
output_name="720p30",
video_description_name="video_720p30",
output_settings=medialive.CfnChannel.OutputSettingsProperty(
media_package_output_settings={}))
output = medialive.CfnChannel.OutputGroupProperty(
name="HD",
output_group_settings=output_settings,
outputs=[output_output1, output_output2])
""" Video Section """
video1_h264 = medialive.CfnChannel.H264SettingsProperty(
adaptive_quantization="HIGH",
afd_signaling="NONE",
bitrate=5000000,
color_metadata="INSERT",
entropy_encoding="CABAC",
flicker_aq="ENABLED",
framerate_control="SPECIFIED",
framerate_denominator=1,
framerate_numerator=30,
gop_b_reference="ENABLED",
gop_closed_cadence=1,
gop_num_b_frames=3,
gop_size=60,
gop_size_units="FRAMES",
level="H264_LEVEL_AUTO",
look_ahead_rate_control="HIGH",
num_ref_frames=3,
par_control="SPECIFIED",
profile="HIGH",
rate_control_mode="CBR",
scan_type="PROGRESSIVE",
scene_change_detect="ENABLED",
slices=1,
spatial_aq="ENABLED",
syntax="DEFAULT",
temporal_aq="ENABLED",
timecode_insertion="DISABLED")
video1_codec = medialive.CfnChannel.VideoCodecSettingsProperty(
h264_settings=video1_h264)
video1_description = medialive.CfnChannel.VideoDescriptionProperty(
codec_settings=video1_codec,
height=1080,
name="video_1080p30",
respond_to_afd="NONE",
scaling_behavior="DEFAULT",
sharpness=50,
width=1920)
video2_h264 = medialive.CfnChannel.H264SettingsProperty(
adaptive_quantization="HIGH",
afd_signaling="NONE",
bitrate=3000000,
color_metadata="INSERT",
entropy_encoding="CABAC",
flicker_aq="ENABLED",
framerate_control="SPECIFIED",
framerate_denominator=1,
framerate_numerator=30,
gop_b_reference="ENABLED",
gop_closed_cadence=1,
gop_num_b_frames=3,
gop_size=60,
gop_size_units="FRAMES",
level="H264_LEVEL_AUTO",
look_ahead_rate_control="HIGH",
num_ref_frames=3,
par_control="SPECIFIED",
profile="HIGH",
rate_control_mode="CBR",
scan_type="PROGRESSIVE",
scene_change_detect="ENABLED",
slices=1,
spatial_aq="ENABLED",
syntax="DEFAULT",
temporal_aq="ENABLED",
timecode_insertion="DISABLED")
video2_codec = medialive.CfnChannel.VideoCodecSettingsProperty(
h264_settings=video2_h264)
video2_description = medialive.CfnChannel.VideoDescriptionProperty(
codec_settings=video2_codec,
height=720,
name="video_720p30",
respond_to_afd="NONE",
scaling_behavior="DEFAULT",
sharpness=100,
width=1280)
""" Channel final settings and channel start """
media_live_channel_timecode = medialive.CfnChannel.TimecodeConfigProperty(
source="EMBEDDED")
media_live_channel_encoder = medialive.CfnChannel.EncoderSettingsProperty(
audio_descriptions=[audio1, audio2],
video_descriptions=[video1_description, video2_description],
output_groups=[output],
timecode_config=media_live_channel_timecode)
policy_document = iam.PolicyDocument.from_json(INLINE_POLICY)
medialive_role = iam.Role(
scope=self,
id='medialive_role',
role_name='medialive_role',
assumed_by=iam.ServicePrincipal('medialive.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'AWSElementalMediaLiveFullAccess')
],
inline_policies={"medialivecustom": policy_document})
media_live_channel = medialive.CfnChannel(
scope=self,
id="media-live-channel-{0}".format(DEFAULT_CONF.get("id_channel")),
channel_class="SINGLE_PIPELINE",
name=DEFAULT_CONF.get("id_channel"),
input_specification=media_live_channel_input_spec,
input_attachments=[media_live_channel_input_attach],
destinations=[media_live_channel_destination],
encoder_settings=media_live_channel_encoder,
role_arn=medialive_role.role_arn)
# We need to add dependency because CFN must wait channel creation finish before starting the endpoint creation
mediadep = core.ConcreteDependable()
mediadep.add(channel)
hls_endpoint.node.add_dependency(mediadep)
media_live_channel.node.add_dependency(mediadep)
def __init__(self, scope: cdk.Construct, construct_id: str, vpc, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
""" Read UserData Script: """
try:
with open("app_db_stack/user_data/deploy_app.sh", mode="r") as file:
user_data = file.read()
except OSError:
print("Unable to read UserData script")
""" AMI: """
linux_ami = _ec2.AmazonLinuxImage(generation=_ec2.AmazonLinuxGeneration.AMAZON_LINUX_2,
edition=_ec2.AmazonLinuxEdition.STANDARD,
virtualization=_ec2.AmazonLinuxVirt.HVM,
storage=_ec2.AmazonLinuxStorage.GENERAL_PURPOSE
)
""" Create ALB: """
alb = _elbv2.ApplicationLoadBalancer(
self,
"MyAlbId",
vpc=vpc,
internet_facing=True,
load_balancer_name="WebServerAlb"
)
# Allow alb SG to receive traffic from port 80:
alb.connections.allow_from_any_ipv4(
_ec2.Port.tcp(80),
description="Allow Internet access on ALB port 80"
)
# Add listener to ALB:
listener = alb.add_listener("listenerId",
port=80,
open=True)
""" Web server IAM role: """
web_server_role = _iam.Role(self, "WebServerRoleId",
assumed_by=_iam.ServicePrincipal("ec2.amazonaws.com"),
managed_policies=[
_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonSSMManagedInstanceCore",
),
_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3ReadOnlyAccess"
)
])
""" Create ASG with 2 EC2 instances: """
self.web_server_asg = _autoscaling.AutoScalingGroup(self,
"WebServerAsgId",
vpc=vpc,
key_name="A4L",
vpc_subnets=_ec2.SubnetSelection(
subnet_type=_ec2.SubnetType.PRIVATE
),
instance_type=_ec2.InstanceType(
instance_type_identifier="t2.micro",
),
machine_image=linux_ami,
role=web_server_role,
min_capacity=1,
max_capacity=3,
desired_capacity=3,
user_data=_ec2.UserData.custom(user_data)
)
# Allow ASG SG to receive traffic from ALB SG:
self.web_server_asg.connections.allow_from(alb, _ec2.Port.tcp(80),
description="Allow ASG SG to receive traffic from ALB SG")
# Add ASG Instances to the ALB Target Group:
listener.add_targets("ListenerId", port=80, targets=[self.web_server_asg])
""" Output of the ALB Domain Name: """
output_alb_1 = cdk.CfnOutput(self,
"AlbDomainName",
value=f"http://{alb.load_balancer_dns_name}",
description="Web Server ALB Domain Name")
""" Tags: """
def __init__(self, scope: core.Construct, construct_id: str,
stack_log_level: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# Add your stack resources below):
# Create Kinesis Data Stream
self.data_pipe_stream = _kinesis.Stream(
self,
"dataPipeStream",
retention_period=core.Duration.hours(24),
shard_count=1,
stream_name="data_pipe")
########################################
####### #######
####### Stream Data Producer #######
####### #######
########################################
# Read Lambda Code
try:
with open(
"tumbling_window_stream_analytics/stacks/back_end/lambda_src/stream_data_producer.py",
encoding="utf-8",
mode="r") as f:
data_producer_fn_code = f.read()
except OSError:
print("Unable to read Lambda Function Code")
raise
data_producer_fn = _lambda.Function(
self,
"streamDataProducerFn",
function_name=f"data_producer_fn",
description=
"Produce streaming data events and push to Kinesis stream",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(data_producer_fn_code),
handler="index.lambda_handler",
timeout=core.Duration.seconds(60),
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": "INFO",
"STREAM_NAME": f"{self.data_pipe_stream.stream_name}",
"APP_ENV": "Production",
"STREAM_AWS_REGION": f"{core.Aws.REGION}"
})
# Grant our Lambda Producer privileges to write to Kinesis Data Stream
self.data_pipe_stream.grant_read_write(data_producer_fn)
# Create Custom Loggroup for Producer
data_producer_lg = _logs.LogGroup(
self,
"dataProducerLogGroup",
log_group_name=f"/aws/lambda/{data_producer_fn.function_name}",
removal_policy=core.RemovalPolicy.DESTROY,
retention=_logs.RetentionDays.ONE_DAY)
###########################################
################# OUTPUTS #################
###########################################
output_0 = core.CfnOutput(
self,
"AutomationFrom",
value=f"{GlobalArgs.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output_1 = core.CfnOutput(
self,
"streamDataProducer",
value=
f"https://console.aws.amazon.com/lambda/home?region={core.Aws.REGION}#/functions/{data_producer_fn.function_name}?tab=code",
description=
"Produce streaming data events and push to Kinesis stream.")
def __init__(self, scope: core.Construct, id: str, vpc, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
cluster = ecs.Cluster(
self, 'EKSGraviton2',
vpc=vpc,
container_insights=True
)
task_definition = ecs.Ec2TaskDefinition(
self, "TaskDef")
container_uri = ssm.StringParameter.value_for_string_parameter(self ,"graviton_lab_container_uri")
ecs_ami = ecs.EcsOptimizedAmi(generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2,
hardware_type=ecs.AmiHardwareType.ARM)
asg_ecs = cluster.add_capacity("G2AutoScalingGroup",
instance_type=ec2.InstanceType("m6g.2xlarge"),
machine_image=ecs_ami
)
container = task_definition.add_container(
"web",
image=ecs.ContainerImage.from_registry(container_uri),
memory_limit_mib=512,
logging=ecs.LogDrivers.firelens(
options={
"Name": "cloudwatch",
"log_key": "log",
"region": "us-east-1",
"delivery_stream": "my-stream",
"log_group_name": "firelens-fluent-bit",
"auto_create_group": "true",
"log_stream_prefix": "from-fluent-bit"}
)
)
port_mapping = ecs.PortMapping(
container_port=3000,
host_port=8080,
protocol=ecs.Protocol.TCP
)
container.add_port_mappings(port_mapping)
# Create Service
service = ecs.Ec2Service(
self, "Service",
cluster=cluster,
task_definition=task_definition
)
# Create ALB
lb = elbv2.ApplicationLoadBalancer(
self, "LB",
vpc=vpc,
internet_facing=True
)
listener = lb.add_listener(
"PublicListener",
port=80,
open=True
)
# Attach ALB to ECS Service
listener.add_targets(
"ECS",
port=80,
targets=[service]
)
core.CfnOutput(
self, "LoadBalancerDNS",
value=lb.load_balancer_dns_name
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Parameteres #
assets_bucket_name = create_assets_bucket_name_parameter(self)
blueprint_bucket_name = create_blueprint_bucket_name_parameter(self)
custom_algorithms_ecr_repo_arn = create_custom_algorithms_ecr_repo_arn_parameter(
self)
kms_key_arn = create_kms_key_arn_parameter(self)
algorithm_image_uri = create_algorithm_image_uri_parameter(self)
model_name = create_model_name_parameter(self)
model_artifact_location = create_model_artifact_location_parameter(
self)
data_capture_location = create_data_capture_location_parameter(self)
inference_instance = create_inference_instance_parameter(self)
# Conditions
custom_algorithms_ecr_repo_arn_provided = create_custom_algorithms_ecr_repo_arn_provided_condition(
self, custom_algorithms_ecr_repo_arn)
kms_key_arn_provided = create_kms_key_arn_provided_condition(
self, kms_key_arn)
# Resources #
# getting blueprint bucket object from its name - will be used later in the stack
blueprint_bucket = s3.Bucket.from_bucket_name(
self, "BlueprintBucket", blueprint_bucket_name.value_as_string)
# provision api gateway and lambda for inference using solution constructs
inference_api_gateway = aws_apigateway_lambda.ApiGatewayToLambda(
self,
"BYOMInference",
lambda_function_props={
"runtime":
lambda_.Runtime.PYTHON_3_8,
"handler":
"main.handler",
"code":
lambda_.Code.from_bucket(
blueprint_bucket, "blueprints/byom/lambdas/inference.zip"),
},
api_gateway_props={
"defaultMethodOptions": {
"authorizationType": apigw.AuthorizationType.IAM,
},
"restApiName": f"{core.Aws.STACK_NAME}-inference",
"proxy": False,
},
)
# add supressions
inference_api_gateway.lambda_function.node.default_child.cfn_options.metadata = suppress_lambda_policies(
)
provision_resource = inference_api_gateway.api_gateway.root.add_resource(
"inference")
provision_resource.add_method("POST")
# create Sagemaker role
sagemaker_role = create_sagemaker_role(
self,
"MLOpsRealtimeSagemakerRole",
custom_algorithms_ecr_arn=custom_algorithms_ecr_repo_arn.
value_as_string,
kms_key_arn=kms_key_arn.value_as_string,
assets_bucket_name=assets_bucket_name.value_as_string,
input_bucket_name=assets_bucket_name.value_as_string,
input_s3_location=assets_bucket_name.value_as_string,
output_s3_location=data_capture_location.value_as_string,
ecr_repo_arn_provided_condition=
custom_algorithms_ecr_repo_arn_provided,
kms_key_arn_provided_condition=kms_key_arn_provided,
)
# create sagemaker model
sagemaker_model = create_sagemaker_model(
self,
"MLOpsSagemakerModel",
execution_role=sagemaker_role,
primary_container={
"image":
algorithm_image_uri.value_as_string,
"modelDataUrl":
f"s3://{assets_bucket_name.value_as_string}/{model_artifact_location.value_as_string}",
},
tags=[{
"key": "model_name",
"value": model_name.value_as_string
}],
)
# Create Sagemaker EndpointConfg
sagemaker_endpoint_config = create_sagemaker_endpoint_config(
self,
"MLOpsSagemakerEndpointConfig",
sagemaker_model.attr_model_name,
model_name.value_as_string,
inference_instance.value_as_string,
data_capture_location.value_as_string,
core.Fn.condition_if(kms_key_arn_provided.logical_id,
kms_key_arn.value_as_string,
core.Aws.NO_VALUE).to_string(),
)
# create a dependency on the model
sagemaker_endpoint_config.add_depends_on(sagemaker_model)
# create Sagemaker endpoint
sagemaker_endpoint = create_sagemaker_endpoint(
self,
"MLOpsSagemakerEndpoint",
sagemaker_endpoint_config.attr_endpoint_config_name,
model_name.value_as_string,
)
# add dependency on endpoint config
sagemaker_endpoint.add_depends_on(sagemaker_endpoint_config)
# Create Lambda - sagemakerendpoint
LambdaToSagemakerEndpoint(
self,
"LambdaSagmakerEndpoint",
existing_sagemaker_endpoint_obj=sagemaker_endpoint,
existing_lambda_obj=inference_api_gateway.lambda_function,
)
# Outputs #
core.CfnOutput(
self,
id="SageMakerModelName",
value=sagemaker_model.attr_model_name,
)
core.CfnOutput(
self,
id="SageMakerEndpointConfigName",
value=sagemaker_endpoint_config.attr_endpoint_config_name,
)
core.CfnOutput(
self,
id="SageMakerEndpointName",
value=sagemaker_endpoint.attr_endpoint_name,
)
core.CfnOutput(
self,
id="EndpointDataCaptureLocation",
value=
f"https://s3.console.aws.amazon.com/s3/buckets/{data_capture_location.value_as_string}/",
description=
"Endpoint data capture location (to be used by Model Monitor)",
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
prefix = "test"
cidr = "192.168.0.0/16"
# def name(s): return "{0}/{1}".format(prefix, s)
def name(s):
return "{0} {1}".format(prefix, s)
# VPC
self.vpc = ec2.CfnVPC(self,
"vpc",
cidr_block=cidr,
enable_dns_hostnames=True,
enable_dns_support=True,
tags=[core.CfnTag(key="Name", value=prefix)])
# InternetGateway
igw = ec2.CfnInternetGateway(
self, "igw", tags=[core.CfnTag(key="Name", value=prefix)])
igw_attachment = ec2.CfnVPCGatewayAttachment(
self,
"igw_attachment",
vpc_id=self.vpc.ref,
internet_gateway_id=igw.ref)
dhcpoptions = ec2.CfnDHCPOptions(
self,
"dhcpoptions",
domain_name="ec2.internal " + prefix,
domain_name_servers=["AmazonProvidedDNS"],
tags=[core.CfnTag(key="Name", value=prefix)])
dhcpoptionsassociation = ec2.CfnVPCDHCPOptionsAssociation(
self,
"dhcpoptionsassociation",
dhcp_options_id=dhcpoptions.ref,
vpc_id=self.vpc.ref)
# PrivateSubnetA
# private_subnet_a = ec2.CfnSubnet(
# self, "private_a",
# vpc_id=vpc.ref,
# cidr_block="192.168.0.0/24",
# availability_zone="ap-northeast-1a",
# tags=[
# core.CfnTag(key="Name", value=name("private_a"))
# ]
# )
# PrivateSubnetC
# private_subnet_c = ec2.CfnSubnet(
# self, "private_c",
# vpc_id=vpc.ref,
# cidr_block="192.168.1.0/24",
# availability_zone="ap-northeast-1c",
# tags=[
# core.CfnTag(key="Name", value=name("private_c"))
# ]
# )
# PublicSubnetA
self.public_subnet_a = ec2.CfnSubnet(
self,
"public_a",
vpc_id=self.vpc.ref,
cidr_block="192.168.0.0/20",
# availability_zone="ap-northeast-1a",
availability_zone="us-east-1a",
tags=[core.CfnTag(key="Name", value=prefix + " public_a")])
# PublicSubnetC
self.public_subnet_c = ec2.CfnSubnet(
self,
"public_c",
vpc_id=self.vpc.ref,
cidr_block="192.168.16.0/20",
availability_zone="us-east-1c",
tags=[core.CfnTag(key="Name", value=prefix + " public_c")])
self.public_subnet_d = ec2.CfnSubnet(
self,
"public_d",
vpc_id=self.vpc.ref,
cidr_block="192.168.32.0/20",
availability_zone="us-east-1d",
tags=[core.CfnTag(key="Name", value=prefix + " public_d")])
# EIP1 (for NATGW)
# eip1 = ec2.CfnEIP(
# self, "eip1",
# domain="vpc",
# )
# eip1.add_depends_on(igw_attachment)
# EIP2 (for NATGW)
# eip2 = ec2.CfnEIP(
# self, "eip2",
# domain="vpc",
# )
# eip2.add_depends_on(igw_attachment)
# NatGatewayA
# natgw_a = ec2.CfnNatGateway(
# self, "natgw_a",
# allocation_id=eip1.attr_allocation_id,
# subnet_id=self.public_subnet_a.ref,
# tags=[
# core.CfnTag(key="Name", value=name("natgw_a"))
# ]
# )
# NatGatewayC
# natgw_c = ec2.CfnNatGateway(
# self, "natgw_c",
# allocation_id=eip2.attr_allocation_id,
# subnet_id=public_subnet_c.ref,
# tags=[
# core.CfnTag(key="Name", value=name("natgw_c"))
# ]
# )
# RouteTable of PrivateSubnetA
# rtb_private_a = ec2.CfnRouteTable(
# self, "rtb_private_a",
# vpc_id=vpc.ref,
# tags=[
# core.CfnTag(key="Name", value=name("rtb_private_a"))
# ]
# )
# ec2.CfnSubnetRouteTableAssociation(
# self, "rtb_private_a_association",
# route_table_id=rtb_private_a.ref,
# subnet_id=private_subnet_a.ref
# )
# ec2.CfnRoute(
# self, "route_private_a",
# route_table_id=rtb_private_a.ref,
# destination_cidr_block="0.0.0.0/0",
# nat_gateway_id=natgw_a.ref
# )
# RouteTable of PrivateSubnetC
# rtb_private_c = ec2.CfnRouteTable(
# self, "rtb_private_c",
# vpc_id=vpc.ref,
# tags=[
# core.CfnTag(key="Name", value=name("rtb_private_c"))
# ]
# )
# ec2.CfnSubnetRouteTableAssociation(
# self, "rtb_private_c_association",
# route_table_id=rtb_private_c.ref,
# subnet_id=private_subnet_c.ref
# )
# ec2.CfnRoute(
# self, "route_private_c",
# route_table_id=rtb_private_c.ref,
# destination_cidr_block="0.0.0.0/0",
# nat_gateway_id=natgw_c.ref
# )
# RouteTable of PublicSubnetA
self.rtb_public_a = ec2.CfnRouteTable(
self,
"rtb_public_a",
vpc_id=self.vpc.ref,
tags=[core.CfnTag(key="Name", value=prefix + "rtb_public_a")])
ec2.CfnSubnetRouteTableAssociation(
self,
"rtb_public_a_association",
route_table_id=self.rtb_public_a.ref,
subnet_id=self.public_subnet_a.ref)
ec2.CfnSubnetRouteTableAssociation(
self,
"rtb_public_c_association",
route_table_id=self.rtb_public_a.ref,
subnet_id=self.public_subnet_c.ref)
ec2.CfnSubnetRouteTableAssociation(
self,
"rtb_public_d_association",
route_table_id=self.rtb_public_a.ref,
subnet_id=self.public_subnet_d.ref)
ec2.CfnRoute(self,
"route_public_a",
route_table_id=self.rtb_public_a.ref,
destination_cidr_block="0.0.0.0/0",
gateway_id=igw.ref)
# RouteTable of PublicSubnetC
# rtb_public_c = ec2.CfnRouteTable(
# self, "rtb_public_c",
# vpc_id=vpc.ref,
# tags=[
# core.CfnTag(key="Name", value=name("rtb_public_c"))
# ]
# )
# ec2.CfnSubnetRouteTableAssociation(
# self, "rtb_public_c_association",
# route_table_id=rtb_public_c.ref,
# subnet_id=public_subnet_c.ref
# )
# ec2.CfnRoute(
# self, "route_public_c",
# route_table_id=rtb_public_c.ref,
# destination_cidr_block="0.0.0.0/0",
# gateway_id=igw.ref
# )
ami_id = ec2.AmazonLinuxImage(generation=ec2.AmazonLinuxGeneration.
AMAZON_LINUX_2).get_image(self).image_id
# security_group = ec2.SecurityGroup(
# self,
# id='test',
# vpc=self.vpc,
# security_group_name='test-security-group'
# )
# security_group.add_ingress_rule(
# peer=ec2.Peer.ipv4(cidr),
# connection=ec2.Port.tcp(22),
# )
# red_web_inst = ec2.CfnInstance(self,
# "testInstance01",
# image_id = ami_id,
# instance_type = "t3a.micro",
# monitoring = False,
# key_name = "stg-intrinio-www01",
# security_group_ids=[security_group.security_group_id],
# block_device_mappings = [{
# "deviceName": "/dev/xvda",
# "ebs": {
# "volumeSize": 10,
# "volumeType": "io1",
# "iops": 150,
# "deleteOnTermination": True
# }
# }
# ],
# tags = [
# { "key": "Name", "value": prefix }
# ],
# network_interfaces = [{
# "deviceIndex": "0",
# "associatePublicIpAddress": True,
# "subnetId": self.public_subnet_a.ref,
# # "groupSet": [web_sg.security_group_id]
# }], #https: //github.com/aws/aws-cdk/issues/3419
# )
# RdsSecurityGroup
RdsSecurityGroupStg = ec2.CfnSecurityGroup(
self,
"RdsSecurityGroupStg",
group_name='stg-' + prefix + '-db01',
group_description='stg-' + prefix + '-db01',
vpc_id=self.vpc.ref,
security_group_ingress=[{
"ipProtocol": "tcp",
"fromPort": 3306,
"toPort": 3306,
"cidrIp": "0.0.0.0/0"
}],
security_group_egress=[{
"ipProtocol": "tcp",
"fromPort": 0,
"toPort": 65535,
"cidrIp": "0.0.0.0/0"
}],
)
# MyDBSubnetGroup
rds_subnet_group = rds.CfnDBSubnetGroup(
self,
"DBSubnetGroup",
db_subnet_group_description="DBSubnetGroup",
subnet_ids=[
self.public_subnet_a.ref, self.public_subnet_c.ref,
self.public_subnet_d.ref
])
DBParameterGroupStg = rds.CfnDBParameterGroup(
self,
"DBParameterGroupStg",
description='stg-' + prefix + 'db01',
family="MySQL5.6",
parameters={
'character_set_client': "utf8",
'character_set_connection': "utf8",
'character_set_database': "utf8",
'character_set_results': "utf8",
'character_set_server': "utf8",
'collation_connection': "utf8_general_ci",
'collation_server': "utf8_general_ci",
'long_query_time': "1.2",
'slow_query_log': "1",
'time_zone': "Asia/Tokyo",
},
tags=[core.CfnTag(key="Name", value='stg-' + prefix + 'db01')])
rds_params = {
'db_instance_identifier': "stg-test-db01",
'engine': "mysql",
'engine_version': '5.6.39',
'db_instance_class': 'db.t3.micro',
'allocated_storage': '5',
'storage_type': 'gp2',
'db_name': "test",
'master_username': "******",
'master_user_password': "******",
'db_subnet_group_name': rds_subnet_group.ref,
'publicly_accessible': False,
'multi_az': False,
'preferred_backup_window': "18:00-18:30",
'preferred_maintenance_window': "sat:19:00-sat:19:30",
'auto_minor_version_upgrade': False,
'db_parameter_group_name': DBParameterGroupStg.ref,
'vpc_security_groups': [RdsSecurityGroupStg.ref],
'copy_tags_to_snapshot': True,
'backup_retention_period': 7,
# 'enable_performance_insights': True,
'delete_automated_backups': True,
'deletion_protection': False,
'availability_zone': "us-east-1a",
'enable_cloudwatch_logs_exports': ["error", "slowquery"]
# 'storage_encrypted': False,
}
self.rds = rds.CfnDBInstance(
self,
'staff-rds',
**rds_params,
tags=[core.CfnTag(key="Name", value='stg-' + prefix + 'db01')])
core.CfnOutput(self, "OutputVpc", value=self.vpc.ref)
core.CfnOutput(self, "OutputRds", value=self.rds.ref)
def __init__(self, scope: core.Construct, id: str,vpc: ec2.Vpc, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
prj_name = self.node.try_get_context("project_name")
env_name = self.node.try_get_context("env")
self.lambda_sg = ec2.SecurityGroup(self, 'lambdasg',
security_group_name='lambda-sg',
vpc=vpc,
description="SG for Lambda Functions",
allow_all_outbound=True
)
self.bastion_sg = ec2.SecurityGroup(self, 'bastionsg',
security_group_name='bastion-sg',
vpc=vpc,
description="SG for Bastion Host",
allow_all_outbound=True
)
self.bastion_sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(22), "SSH Access")
redis_sg = ec2.SecurityGroup(self, 'redissg',
security_group_name='redis-sg',
vpc=vpc,
description="SG for Redis Cluster",
allow_all_outbound=True
)
redis_sg.add_ingress_rule(self.lambda_sg, ec2.Port.tcp(6379), 'Access from Lambda functions')
#Kibana
self.kibana_sg = ec2.SecurityGroup(self, 'kibanasg',
security_group_name='kibana-sg',
vpc=vpc,
description='SG for Kibana',
allow_all_outbound=True
)
self.kibana_sg.add_ingress_rule(self.bastion_sg, ec2.Port.tcp(443), "Access from jumpbox")
kibana_role = iam.CfnServiceLinkedRole(self,'kibanarole',
aws_service_name="es.amazonaws.com"
)
lambda_role = iam.Role(self, 'lambdarole',
assumed_by=iam.ServicePrincipal(service='lambda.amazonaws.com'),
role_name='lambda-role',
managed_policies=[iam.ManagedPolicy.from_aws_managed_policy_name(
managed_policy_name='service-role/AWSLambdaVPCAccessExecutionRole'
)]
)
lambda_role.add_to_policy(
statement=iam.PolicyStatement(
actions=['s3:*', 'rds:*'],
resources=['*']
)
)
core.CfnOutput(self, 'redis-export',
export_name='redis-sg-export',
value=redis_sg.security_group_id
)
#SSM Parameters
ssm.StringParameter(self, 'lambdasg-param',
parameter_name='/'+env_name+'/lambda-sg',
string_value=self.lambda_sg.security_group_id
)
ssm.StringParameter(self, 'lambdarole-param-arn',
parameter_name='/'+env_name+'/lambda-role-arn',
string_value=lambda_role.role_arn
)
ssm.StringParameter(self, 'lambdarole-param-name',
parameter_name='/'+env_name+'/lambda-role-name',
string_value=lambda_role.role_name
)
def __init__(self, scope: core.Construct, _id: str, **kwargs) -> None:
super().__init__(scope, _id, **kwargs)
# Setup SSM parameter of credentials, bucket_para, ignore_list
ssm_credential_para = ssm.StringParameter.from_secure_string_parameter_attributes(
self,
"ssm_parameter_credentials",
parameter_name=ssm_parameter_credentials,
version=1)
ssm_bucket_para = ssm.StringParameter(self,
"s3bucket_serverless",
string_value=json.dumps(
bucket_para, indent=4))
ssm_parameter_ignore_list = ssm.StringParameter(
self, "s3_migrate_ignore_list", string_value=ignore_list)
# Setup DynamoDB
ddb_file_list = ddb.Table(self,
"s3migrate_serverless",
partition_key=ddb.Attribute(
name="Key",
type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST)
# Setup SQS
sqs_queue_DLQ = sqs.Queue(self,
"s3migrate_serverless_Q_DLQ",
visibility_timeout=core.Duration.minutes(15),
retention_period=core.Duration.days(14))
sqs_queue = sqs.Queue(self,
"s3migrate_serverless_Q",
visibility_timeout=core.Duration.minutes(15),
retention_period=core.Duration.days(14),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=3, queue=sqs_queue_DLQ))
# Setup API for Lambda to get IP address (for debug networking routing purpose)
checkip = api.RestApi(
self,
"lambda-checkip-api",
cloud_watch_role=True,
deploy=True,
description="For Lambda get IP address",
default_integration=api.MockIntegration(
integration_responses=[
api.IntegrationResponse(status_code="200",
response_templates={
"application/json":
"$context.identity.sourceIp"
})
],
request_templates={"application/json": '{"statusCode": 200}'}),
endpoint_types=[api.EndpointType.REGIONAL])
checkip.root.add_method("GET",
method_responses=[
api.MethodResponse(
status_code="200",
response_models={
"application/json":
api.Model.EMPTY_MODEL
})
])
# Setup Lambda functions
handler = lam.Function(self,
"s3-migrate-worker",
code=lam.Code.asset("./lambda"),
handler="lambda_function_worker.lambda_handler",
runtime=lam.Runtime.PYTHON_3_8,
memory_size=1024,
timeout=core.Duration.minutes(15),
tracing=lam.Tracing.ACTIVE,
environment={
'table_queue_name':
ddb_file_list.table_name,
'Des_bucket_default':
Des_bucket_default,
'Des_prefix_default':
Des_prefix_default,
'StorageClass':
StorageClass,
'checkip_url':
checkip.url,
'ssm_parameter_credentials':
ssm_parameter_credentials
})
handler_jobsender = lam.Function(
self,
"s3-migrate-jobsender",
code=lam.Code.asset("./lambda"),
handler="lambda_function_jobsender.lambda_handler",
runtime=lam.Runtime.PYTHON_3_8,
memory_size=1024,
timeout=core.Duration.minutes(15),
tracing=lam.Tracing.ACTIVE,
environment={
'table_queue_name': ddb_file_list.table_name,
'StorageClass': StorageClass,
'checkip_url': checkip.url,
'sqs_queue': sqs_queue.queue_name,
'ssm_parameter_credentials': ssm_parameter_credentials,
'ssm_parameter_ignore_list':
ssm_parameter_ignore_list.parameter_name,
'ssm_parameter_bucket': ssm_bucket_para.parameter_name
})
# Allow lambda read/write DDB, SQS
ddb_file_list.grant_read_write_data(handler)
ddb_file_list.grant_read_write_data(handler_jobsender)
sqs_queue.grant_send_messages(handler_jobsender)
# SQS trigger Lambda worker
handler.add_event_source(SqsEventSource(sqs_queue, batch_size=1))
# Option1: Create S3 Bucket, all new objects in this bucket will be transmitted by Lambda Worker
s3bucket = s3.Bucket(self, "s3_new_migrate")
s3bucket.grant_read(handler)
s3bucket.add_event_notification(s3.EventType.OBJECT_CREATED,
s3n.SqsDestination(sqs_queue))
# Option2: Allow Exist S3 Buckets to be read by Lambda functions.
# Lambda Jobsender will scan and compare the these buckets and trigger Lambda Workers to transmit
bucket_name = ''
for b in bucket_para:
if bucket_name != b['src_bucket']: # 如果列了多个相同的Bucket，就跳过
bucket_name = b['src_bucket']
s3exist_bucket = s3.Bucket.from_bucket_name(
self,
bucket_name, # 用这个做id
bucket_name=bucket_name)
s3exist_bucket.grant_read(handler_jobsender)
s3exist_bucket.grant_read(handler)
# Allow Lambda read ssm parameters
ssm_bucket_para.grant_read(handler_jobsender)
ssm_credential_para.grant_read(handler)
ssm_credential_para.grant_read(handler_jobsender)
ssm_parameter_ignore_list.grant_read(handler_jobsender)
# Schedule cron event to trigger Lambda Jobsender per hour:
event.Rule(self,
'cron_trigger_jobsender',
schedule=event.Schedule.rate(core.Duration.hours(1)),
targets=[target.LambdaFunction(handler_jobsender)])
# Create Lambda logs filter to create network traffic metric
handler.log_group.add_metric_filter(
"Complete-bytes",
metric_name="Complete-bytes",
metric_namespace="s3_migrate",
metric_value="$bytes",
filter_pattern=logs.FilterPattern.literal(
'[info, date, sn, p="--->Complete", bytes, key]'))
handler.log_group.add_metric_filter(
"Uploading-bytes",
metric_name="Uploading-bytes",
metric_namespace="s3_migrate",
metric_value="$bytes",
filter_pattern=logs.FilterPattern.literal(
'[info, date, sn, p="--->Uploading", bytes, key]'))
handler.log_group.add_metric_filter(
"Downloading-bytes",
metric_name="Downloading-bytes",
metric_namespace="s3_migrate",
metric_value="$bytes",
filter_pattern=logs.FilterPattern.literal(
'[info, date, sn, p="--->Downloading", bytes, key]'))
lambda_metric_Complete = cw.Metric(namespace="s3_migrate",
metric_name="Complete-bytes",
statistic="Sum",
period=core.Duration.minutes(1))
lambda_metric_Upload = cw.Metric(namespace="s3_migrate",
metric_name="Uploading-bytes",
statistic="Sum",
period=core.Duration.minutes(1))
lambda_metric_Download = cw.Metric(namespace="s3_migrate",
metric_name="Downloading-bytes",
statistic="Sum",
period=core.Duration.minutes(1))
handler.log_group.add_metric_filter(
"ERROR",
metric_name="ERROR-Logs",
metric_namespace="s3_migrate",
metric_value="1",
filter_pattern=logs.FilterPattern.literal('"ERROR"'))
handler.log_group.add_metric_filter(
"WARNING",
metric_name="WARNING-Logs",
metric_namespace="s3_migrate",
metric_value="1",
filter_pattern=logs.FilterPattern.literal('"WARNING"'))
log_metric_ERROR = cw.Metric(namespace="s3_migrate",
metric_name="ERROR-Logs",
statistic="Sum",
period=core.Duration.minutes(1))
log_metric_WARNING = cw.Metric(namespace="s3_migrate",
metric_name="WARNING-Logs",
statistic="Sum",
period=core.Duration.minutes(1))
# Dashboard to monitor SQS and Lambda
board = cw.Dashboard(self, "s3_migrate_serverless")
board.add_widgets(
cw.GraphWidget(title="Lambda-NETWORK",
left=[
lambda_metric_Download, lambda_metric_Upload,
lambda_metric_Complete
]),
# TODO: here monitor all lambda concurrency not just the working one. Limitation from CDK
# Lambda now supports monitor single lambda concurrency, will change this after CDK support
cw.GraphWidget(title="Lambda-all-concurrent",
left=[
handler.metric_all_concurrent_executions(
period=core.Duration.minutes(1))
]),
cw.GraphWidget(
title="Lambda-invocations/errors/throttles",
left=[
handler.metric_invocations(
period=core.Duration.minutes(1)),
handler.metric_errors(period=core.Duration.minutes(1)),
handler.metric_throttles(period=core.Duration.minutes(1))
]),
cw.GraphWidget(
title="Lambda-duration",
left=[
handler.metric_duration(period=core.Duration.minutes(1))
]),
)
board.add_widgets(
cw.GraphWidget(
title="SQS-Jobs",
left=[
sqs_queue.metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1)),
sqs_queue.
metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1))
]),
cw.GraphWidget(
title="SQS-DeadLetterQueue",
left=[
sqs_queue_DLQ.
metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1)),
sqs_queue_DLQ.
metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1))
]),
cw.GraphWidget(title="ERROR/WARNING Logs",
left=[log_metric_ERROR],
right=[log_metric_WARNING]),
cw.SingleValueWidget(
title="Running/Waiting and Dead Jobs",
metrics=[
sqs_queue.
metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1)),
sqs_queue.metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1)),
sqs_queue_DLQ.
metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1)),
sqs_queue_DLQ.
metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1))
],
height=6))
# Alarm for queue - DLQ
alarm_DLQ = cw.Alarm(
self,
"SQS_DLQ",
metric=sqs_queue_DLQ.metric_approximate_number_of_messages_visible(
),
threshold=0,
comparison_operator=cw.ComparisonOperator.GREATER_THAN_THRESHOLD,
evaluation_periods=1,
datapoints_to_alarm=1)
alarm_topic = sns.Topic(self, "SQS queue-DLQ has dead letter")
alarm_topic.add_subscription(
subscription=sub.EmailSubscription(alarm_email))
alarm_DLQ.add_alarm_action(action.SnsAction(alarm_topic))
core.CfnOutput(self,
"Dashboard",
value="CloudWatch Dashboard name s3_migrate_serverless")
def __init__(
self,
scope: core.Construct,
id: str,
memory: int = 1024,
timeout: int = 30,
concurrent: Optional[int] = None,
permissions: Optional[List[iam.PolicyStatement]] = None,
layer_arn: Optional[str] = None,
env: dict = {},
code_dir: str = "./",
**kwargs: Any,
) -> None:
"""Define stack."""
super().__init__(scope, id, *kwargs)
permissions = permissions or []
lambda_env = DEFAULT_ENV.copy()
lambda_env.update(env)
lambda_function = aws_lambda.Function(
self,
f"{id}-lambda",
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=self.create_package(code_dir),
handler="handler.handler",
memory_size=memory,
reserved_concurrent_executions=concurrent,
timeout=core.Duration.seconds(timeout),
environment=lambda_env,
)
# # If you use dynamodb mosaic backend you should add IAM roles to read/put Item and maybe create Table
# permissions.append(
# iam.PolicyStatement(
# actions=[
# "dynamodb:GetItem",
# "dynamodb:PutItem",
# "dynamodb:CreateTable",
# "dynamodb:Scan",
# "dynamodb:BatchWriteItem",
# ],
# resources=[f"arn:aws:dynamodb:{self.region}:{self.account}:table/*"],
# )
# )
for perm in permissions:
lambda_function.add_to_role_policy(perm)
if layer_arn:
lambda_function.add_layers(
aws_lambda.LayerVersion.from_layer_version_arn(
self,
layer_arn.split(":")[-2], layer_arn))
api = apigw.HttpApi(
self,
f"{id}-endpoint",
default_integration=apigw.LambdaProxyIntegration(
handler=lambda_function),
)
core.CfnOutput(self, "Endpoint", value=api.url)
def __init__(self, scope: cdk.Construct, construct_id: str,
ecr_repository: ecr.Repository,
ecs_service: ecs.FargateService,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
backend_repository = codecommit.Repository(
self, 'BackendRepository',
repository_name='MythicalMysfits-BackendRepository'
)
codebuild_project = codebuild.PipelineProject(
self, 'BuildProject',
project_name='MythicalMysfitsServiceCodeBuildProject',
environment=codebuild.BuildEnvironment(
build_image=codebuild.LinuxBuildImage.UBUNTU_14_04_PYTHON_3_5_2,
compute_type=codebuild.ComputeType.SMALL,
environment_variables={
'AWS_ACCOUNT_ID': codebuild.BuildEnvironmentVariable(
type=codebuild.BuildEnvironmentVariableType.PLAINTEXT,
value=self.account),
'AWS_DEFAULT_REGION': codebuild.BuildEnvironmentVariable(
type=codebuild.BuildEnvironmentVariableType.PLAINTEXT,
value=self.region),
},
privileged=True
)
)
codebuild_policy_stm = _iam.PolicyStatement()
codebuild_policy_stm.add_resources(backend_repository.repository_arn)
codebuild_policy_stm.add_actions(
"codecommit:ListBranches",
"codecommit:ListRepositories",
"codecommit:BatchGetRepositories",
"codecommit:GitPull"
)
codebuild_project.add_to_role_policy(codebuild_policy_stm)
ecr_repository.grant_pull_push(codebuild_project.grant_principal)
source_output = codepipeline.Artifact()
source_action = actions.CodeCommitSourceAction(
action_name='CodeCommit-Source',
branch='main',
trigger=actions.CodeCommitTrigger.EVENTS,
repository=backend_repository,
output=source_output
)
build_output = codepipeline.Artifact()
build_action = actions.CodeBuildAction(
action_name='Build',
input=source_output,
outputs=[
build_output
],
project=codebuild_project
)
deploy_action = actions.EcsDeployAction(
action_name='DeployAction',
service=ecs_service,
input=build_output
)
pipeline = codepipeline.Pipeline(
self, 'Pipeline',
pipeline_name='MythicalMysfitsPipeline',
)
pipeline.add_stage(stage_name='Source', actions=[source_action])
pipeline.add_stage(stage_name='Build', actions=[build_action])
# # the following pipeline.add_stage doesn't work
# pipeline.add_stage(stage_name='Deploy', actions=[deploy_action])
cdk.CfnOutput(self, 'BackendRepositoryCloneUrlHttp',
description='Backend Repository CloneUrl HTTP',
value=backend_repository.repository_clone_url_http)
cdk.CfnOutput(self, 'BackendRepositoryCloneUrlSsh',
description='Backend Repository CloneUrl SSH',
value=backend_repository.repository_clone_url_ssh)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
lambda_policies = [iam.PolicyStatement(
actions=[ "logs:CreateLogStream", "logs:PutLogEvents", "logs:CreateLogGroup"],
effect=iam.Effect.ALLOW,
resources=["arn:aws:logs:" + core.Aws.REGION + ":" + core.Aws.ACCOUNT_ID + ":*"]
), iam.PolicyStatement(
actions=[ "dynamodb:*"],
effect=iam.Effect.ALLOW,
resources=["arn:aws:dynamodb:" + core.Aws.REGION + ":" + core.Aws.ACCOUNT_ID + ":*"]
)]
base_api = _apigw.RestApi(self, 'PetclinicApiGatewayWithCors',
rest_api_name='PetclinicApiGatewayWithCors')
api_resource = base_api.root.add_resource('api')
website_bucket = _s3.Bucket(self, 'PetclinicWebsite',
website_index_document='index.html',
public_read_access=True,
removal_policy=core.RemovalPolicy.DESTROY
)
deployment = _s3deploy.BucketDeployment(self, 'PetclinicDeployWebsite',
sources=[_s3deploy.Source.asset('./spring-petclinic-static')],
destination_bucket=website_bucket,
retain_on_delete=False
#destination_key_prefix='web/static'
)
# Modify the config.js with CF custome resource
modify_policy = [iam.PolicyStatement(
actions=[ "s3:PutObject","s3:PutObjectAcl","s3:PutObjectVersionAcl","s3:GetObject"],
effect=iam.Effect.ALLOW,
resources=[website_bucket.bucket_arn + "/*"]
),iam.PolicyStatement(
actions=[ "s3:ListBucket"],
effect=iam.Effect.ALLOW,
resources=[website_bucket.bucket_arn]
),iam.PolicyStatement(
actions=[ "dynamodb:*"],
effect=iam.Effect.ALLOW,
resources=["arn:aws:dynamodb:" + core.Aws.REGION + ":" + core.Aws.ACCOUNT_ID + ":*"]
)]
with open("custom-resource-code/init.py", encoding="utf-8") as fp:
code_body = fp.read()
dynamodb_tables = []
# Warm Lambda function Event rule
event_rule = _event.Rule(self, 'PetclinicLambdaWarmRule',
schedule=_event.Schedule.rate(core.Duration.minutes(3))
)
for service in ['customer', 'vet', 'visit']:
table = _dynamodb.Table(self, service.capitalize() + 'Table',
partition_key={ 'name': 'id', 'type': _dynamodb.AttributeType.STRING },
removal_policy=core.RemovalPolicy.DESTROY,
read_capacity=5,
write_capacity=5,
)
dynamodb_tables.append(table.table_name)
base_lambda = _lambda.Function(self,'ApiPetclinic' + service.capitalize() + 'Lambda',
handler='org.springframework.samples.petclinic.' + service + 's.StreamLambdaHandler::handleRequest',
runtime=_lambda.Runtime.JAVA_8,
code=_lambda.Code.asset('./spring-petclinic-serverless/spring-petclinic-' + service +'s-serverless/target/spring-petclinic-' + service +'s-serverless-2.0.7.jar'),
memory_size=1024,
timeout=core.Duration.seconds(300),
initial_policy=lambda_policies,
environment={"DYNAMODB_TABLE_NAME":table.table_name, "SERVER_SERVLET_CONTEXT_PATH":"/api/" + service}
)
entity = api_resource.add_resource(service)
entity.add_proxy(default_integration=_apigw.LambdaIntegration(base_lambda))
self.add_cors_options(entity)
event_rule.add_target(_target.LambdaFunction(handler=base_lambda))
resource = _cfn.CustomResource(self, "S3ModifyCustomResource",
provider=_cfn.CustomResourceProvider.lambda_(
_lambda.SingletonFunction(
self, "CustomResourceSingleton",
uuid="f7d4f730-4ee1-11e8-9c2d-fa7ae01bbebc",
code=_lambda.InlineCode(code_body),
handler="index.handler",
timeout=core.Duration.seconds(300),
runtime=_lambda.Runtime.PYTHON_3_7,
initial_policy=modify_policy
)
),
properties={"Bucket": website_bucket.bucket_name,
"InvokeUrl":base_api.url,
"DynamoDBTables": dynamodb_tables
}
)
core.CfnOutput(self,"PetclinicWebsiteUrl",export_name="PetclinicWebsiteUrl",value=website_bucket.bucket_website_url)
def __init__(self, scope: core.Construct, id: str, config_dict,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
""" Create the datalake database """
createDatalakeDB = glue.Database(
self,
"createDatalakeDB",
database_name=config_dict['datalake_db_name'])
core.CfnOutput(self,
"createDatalakeDBName",
value=createDatalakeDB.database_name)
""" Create Comp Reg Table """
createDatalakeCompRegTable = glue.Table(
self,
"createDatalakeCompRegTable",
columns=[
glue.Column(name="lot_compound_id",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="version_id",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="parent_id",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="smiles",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="parent_mw",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="salt_multiplicity",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="salt_name",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="formula_weight",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="parent_alias",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="stereochemistry",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="stereocomment",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="geometric_isomerism",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="parent_comment",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="parent_project",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="elnref",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="msmethod",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="msmass",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="provider",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="purity",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="puritymethod",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="nmrshifts",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="lotalias",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="lot_comment",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="lot_project",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="molfile",
type=glue.Type(input_string="string",
is_primitive=True)),
glue.Column(name="checksum",
type=glue.Type(input_string="string",
is_primitive=True))
],
database=createDatalakeDB.from_database_arn(
self, "GetDBArn", database_arn=createDatalakeDB.database_arn),
data_format=glue.DataFormat(
input_format=glue.InputFormat.PARQUET,
output_format=glue.OutputFormat.PARQUET,
serialization_library=glue.SerializationLibrary.PARQUET),
table_name="tbl_compound_data",
bucket=s3.Bucket.from_bucket_name(
self,
"getIBucket",
bucket_name=config_dict['datalake_bucket_name']),
compressed=True,
description=
"This table contains data regarding compound registration coming from RDS",
partition_keys=[
glue.Column(name="dt",
type=glue.Type(input_string="string",
is_primitive=True))
],
s3_prefix="compound_reg/compound_data/")
core.CfnOutput(self,
"createDatalakeCompRegTableName",
value=createDatalakeCompRegTable.table_name)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
vahalla_group = Group(self, "Vahalla Project")
dev01 = User(self, "Vahalla Developer")
dev01.add_to_group(vahalla_group)
# Add AWS managed policy for EC2 Read Only access for the console.
vahalla_group.add_managed_policy(
ManagedPolicy.from_aws_managed_policy_name(
managed_policy_name="AmazonEC2ReadOnlyAccess"))
# Import a json policy and create CloudFormation Managed Policy
CfnManagedPolicy(
self,
"VahallaProjectPolicy",
policy_document=json.loads(project_specific_tags),
groups=[vahalla_group.group_name],
)
vpc = Vpc.from_lookup(self, "Default VPC", is_default=True)
instance_type = InstanceType("t2.micro")
ami = MachineImage.latest_amazon_linux()
blocked_instance = Instance(
self,
"Blocked Instance",
machine_image=ami,
instance_type=instance_type,
vpc=vpc,
)
# Re-use the AMI from t
image_id = blocked_instance.instance.image_id
# Can only add tags to CfnInstance as of 1.31
vahalla = core.CfnTag(key="project", value="vahalla")
valid_instance = CfnInstance(
self,
"Valid Instance",
image_id=image_id,
instance_type="t2.micro",
tags=[vahalla],
)
# Empty group as it's not need to complete our tests.
test_security_group = SecurityGroup(self,
"EmptySecurityGroup",
vpc=vpc)
core.CfnOutput(
self,
"BlockedInstance",
value=blocked_instance.instance_id,
export_name="blocked-instance",
)
core.CfnOutput(
self,
"ValidInstance",
value=valid_instance.ref,
export_name="valid-instance",
)
core.CfnOutput(
self,
"TestSecurityGroup",
value=test_security_group.security_group_id,
export_name="test-sg",
)
core.CfnOutput(self,
"DefaultAMI",
value=image_id,
export_name="default-ami")
def __init__(self, scope: cdk.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Version of ParallelCluster for Cloud9.
pcluster_version = cdk.CfnParameter(
self,
'ParallelClusterVersion',
description=
'Specify a custom parallelcluster version. See https://pypi.org/project/aws-parallelcluster/#history for options.',
default='2.8.0',
type='String',
allowed_values=get_version_list('aws-parallelcluster'))
# S3 URI for Config file
config = cdk.CfnParameter(
self,
'ConfigS3URI',
description='Set a custom parallelcluster config file.',
default=
'https://notearshpc-quickstart.s3.amazonaws.com/{0}/config.ini'.
format(__version__))
# Password
password = cdk.CfnParameter(
self,
'UserPasswordParameter',
description='Set a password for the hpc-quickstart user',
no_echo=True)
# create a VPC
vpc = ec2.Vpc(
self,
'VPC',
cidr='10.0.0.0/16',
gateway_endpoints={
"S3":
ec2.GatewayVpcEndpointOptions(
service=ec2.GatewayVpcEndpointAwsService.S3),
"DynamoDB":
ec2.GatewayVpcEndpointOptions(
service=ec2.GatewayVpcEndpointAwsService.DYNAMODB)
},
max_azs=99)
# create a private and public subnet per vpc
selection = vpc.select_subnets(subnet_type=ec2.SubnetType.PRIVATE)
# Output created subnets
for i, public_subnet in enumerate(vpc.public_subnets):
cdk.CfnOutput(self,
'PublicSubnet%i' % i,
value=public_subnet.subnet_id)
for i, private_subnet in enumerate(vpc.private_subnets):
cdk.CfnOutput(self,
'PrivateSubnet%i' % i,
value=private_subnet.subnet_id)
cdk.CfnOutput(self, 'VPCId', value=vpc.vpc_id)
# Create a Bucket
data_bucket = s3.Bucket(self, "DataRepository")
cdk.CfnOutput(self, 'DataRespository', value=data_bucket.bucket_name)
cloudtrail_bucket = s3.Bucket(self, "CloudTrailLogs")
quickstart_bucket = s3.Bucket.from_bucket_name(self,
'QuickStartBucket',
'aws-quickstart')
# Upload Bootstrap Script to that bucket
bootstrap_script = assets.Asset(self,
'BootstrapScript',
path='scripts/bootstrap.sh')
# Upload parallel cluster post_install_script to that bucket
pcluster_post_install_script = assets.Asset(
self,
'PclusterPostInstallScript',
path='scripts/post_install_script.sh')
# Upload parallel cluster post_install_script to that bucket
pcluster_config_script = assets.Asset(self,
'PclusterConfigScript',
path='scripts/config.ini')
# Setup CloudTrail
cloudtrail.Trail(self, 'CloudTrail', bucket=cloudtrail_bucket)
# Create a Cloud9 instance
# Cloud9 doesn't have the ability to provide userdata
# Because of this we need to use SSM run command
cloud9_instance = cloud9.Ec2Environment(
self,
'ResearchWorkspace',
vpc=vpc,
instance_type=ec2.InstanceType(
instance_type_identifier='c5.large'))
cdk.CfnOutput(self,
'Research Workspace URL',
value=cloud9_instance.ide_url)
# Create a keypair in lambda and store the private key in SecretsManager
c9_createkeypair_role = iam.Role(
self,
'Cloud9CreateKeypairRole',
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'))
c9_createkeypair_role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'))
# Add IAM permissions to the lambda role
c9_createkeypair_role.add_to_policy(
iam.PolicyStatement(
actions=['ec2:CreateKeyPair', 'ec2:DeleteKeyPair'],
resources=['*'],
))
# Lambda for Cloud9 keypair
c9_createkeypair_lambda = _lambda.Function(
self,
'C9CreateKeyPairLambda',
runtime=_lambda.Runtime.PYTHON_3_6,
handler='lambda_function.handler',
timeout=cdk.Duration.seconds(300),
role=c9_createkeypair_role,
code=_lambda.Code.asset('functions/source/c9keypair'),
)
c9_createkeypair_provider = cr.Provider(
self,
"C9CreateKeyPairProvider",
on_event_handler=c9_createkeypair_lambda)
c9_createkeypair_cr = cfn.CustomResource(
self,
"C9CreateKeyPair",
provider=c9_createkeypair_provider,
properties={'ServiceToken': c9_createkeypair_lambda.function_arn})
#c9_createkeypair_cr.node.add_dependency(instance_id)
c9_ssh_private_key_secret = secretsmanager.CfnSecret(
self,
'SshPrivateKeySecret',
secret_string=c9_createkeypair_cr.get_att_string('PrivateKey'))
# The iam policy has a <REGION> parameter that needs to be replaced.
# We do it programmatically so future versions of the synth'd stack
# template include all regions.
with open('iam/ParallelClusterUserPolicy.json') as json_file:
data = json.load(json_file)
for s in data['Statement']:
if s['Sid'] == 'S3ParallelClusterReadOnly':
s['Resource'] = []
for r in region_info.RegionInfo.regions:
s['Resource'].append(
'arn:aws:s3:::{0}-aws-parallelcluster*'.format(
r.name))
parallelcluster_user_policy = iam.CfnManagedPolicy(
self,
'ParallelClusterUserPolicy',
policy_document=iam.PolicyDocument.from_json(data))
# Cloud9 IAM Role
cloud9_role = iam.Role(
self,
'Cloud9Role',
assumed_by=iam.ServicePrincipal('ec2.amazonaws.com'))
cloud9_role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonSSMManagedInstanceCore'))
cloud9_role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name('AWSCloud9User'))
cloud9_role.add_managed_policy(
iam.ManagedPolicy.from_managed_policy_arn(
self, 'AttachParallelClusterUserPolicy',
parallelcluster_user_policy.ref))
cloud9_role.add_to_policy(
iam.PolicyStatement(resources=['*'],
actions=[
'ec2:DescribeInstances',
'ec2:DescribeVolumes', 'ec2:ModifyVolume'
]))
cloud9_role.add_to_policy(
iam.PolicyStatement(resources=[c9_ssh_private_key_secret.ref],
actions=['secretsmanager:GetSecretValue']))
cloud9_role.add_to_policy(
iam.PolicyStatement(
actions=["s3:Get*", "s3:List*"],
resources=[
"arn:aws:s3:::%s/*" % (data_bucket.bucket_name),
"arn:aws:s3:::%s" % (data_bucket.bucket_name)
]))
bootstrap_script.grant_read(cloud9_role)
pcluster_post_install_script.grant_read(cloud9_role)
pcluster_config_script.grant_read(cloud9_role)
# Admin Group
admin_group = iam.Group(self, 'AdminGroup')
admin_group.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
'AdministratorAccess'))
admin_group.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
'AWSCloud9Administrator'))
# PowerUser Group
poweruser_group = iam.Group(self, 'PowerUserGroup')
poweruser_group.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name('PowerUserAccess'))
poweruser_group.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
'AWSCloud9Administrator'))
# HPC User
user = iam.CfnUser(
self,
'Researcher',
groups=[admin_group.node.default_child.ref],
login_profile=iam.CfnUser.LoginProfileProperty(
password_reset_required=True,
password=cdk.SecretValue.cfn_parameter(password).to_string()))
create_user = cdk.CfnParameter(self,
"CreateUser",
default="false",
type="String",
allowed_values=['true', 'false'
]).value_as_string
user_condition = cdk.CfnCondition(self,
"UserCondition",
expression=cdk.Fn.condition_equals(
create_user, "true"))
user.cfn_options.condition = user_condition
cdk.CfnOutput(self,
'UserLoginUrl',
value="".join([
"https://", self.account,
".signin.aws.amazon.com/console"
]),
condition=user_condition)
cdk.CfnOutput(self,
'UserName',
value=user.ref,
condition=user_condition)
# Cloud9 Setup IAM Role
cloud9_setup_role = iam.Role(
self,
'Cloud9SetupRole',
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'))
cloud9_setup_role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'))
# Allow pcluster to be run in bootstrap
cloud9_setup_role.add_managed_policy(
iam.ManagedPolicy.from_managed_policy_arn(
self, 'AttachParallelClusterUserPolicySetup',
parallelcluster_user_policy.ref))
# Add IAM permissions to the lambda role
cloud9_setup_role.add_to_policy(
iam.PolicyStatement(
actions=[
'cloudformation:DescribeStackResources',
'ec2:AssociateIamInstanceProfile',
'ec2:AuthorizeSecurityGroupIngress',
'ec2:DescribeInstances',
'ec2:DescribeInstanceStatus',
'ec2:DescribeInstanceAttribute',
'ec2:DescribeIamInstanceProfileAssociations',
'ec2:DescribeVolumes',
'ec2:DesctibeVolumeAttribute',
'ec2:DescribeVolumesModifications',
'ec2:DescribeVolumeStatus',
'ssm:DescribeInstanceInformation',
'ec2:ModifyVolume',
'ec2:ReplaceIamInstanceProfileAssociation',
'ec2:ReportInstanceStatus',
'ssm:SendCommand',
'ssm:GetCommandInvocation',
's3:GetObject',
'lambda:AddPermission',
'lambda:RemovePermission',
'events:PutRule',
'events:DeleteRule',
'events:PutTargets',
'events:RemoveTargets',
'cloud9:CreateEnvironmentMembership',
],
resources=['*'],
))
cloud9_setup_role.add_to_policy(
iam.PolicyStatement(actions=['iam:PassRole'],
resources=[cloud9_role.role_arn]))
cloud9_setup_role.add_to_policy(
iam.PolicyStatement(
actions=['lambda:AddPermission', 'lambda:RemovePermission'],
resources=['*']))
# Cloud9 Instance Profile
c9_instance_profile = iam.CfnInstanceProfile(
self, "Cloud9InstanceProfile", roles=[cloud9_role.role_name])
# Lambda to add Instance Profile to Cloud9
c9_instance_profile_lambda = _lambda.Function(
self,
'C9InstanceProfileLambda',
runtime=_lambda.Runtime.PYTHON_3_6,
handler='lambda_function.handler',
timeout=cdk.Duration.seconds(900),
role=cloud9_setup_role,
code=_lambda.Code.asset('functions/source/c9InstanceProfile'),
)
c9_instance_profile_provider = cr.Provider(
self,
"C9InstanceProfileProvider",
on_event_handler=c9_instance_profile_lambda,
)
instance_id = cfn.CustomResource(self,
"C9InstanceProfile",
provider=c9_instance_profile_provider,
properties={
'InstanceProfile':
c9_instance_profile.ref,
'Cloud9Environment':
cloud9_instance.environment_id,
})
instance_id.node.add_dependency(cloud9_instance)
# Lambda for Cloud9 Bootstrap
c9_bootstrap_lambda = _lambda.Function(
self,
'C9BootstrapLambda',
runtime=_lambda.Runtime.PYTHON_3_6,
handler='lambda_function.handler',
timeout=cdk.Duration.seconds(900),
role=cloud9_setup_role,
code=_lambda.Code.asset('functions/source/c9bootstrap'),
)
c9_bootstrap_provider = cr.Provider(
self, "C9BootstrapProvider", on_event_handler=c9_bootstrap_lambda)
c9_bootstrap_cr = cfn.CustomResource(
self,
"C9Bootstrap",
provider=c9_bootstrap_provider,
properties={
'Cloud9Environment':
cloud9_instance.environment_id,
'BootstrapPath':
's3://%s/%s' % (bootstrap_script.s3_bucket_name,
bootstrap_script.s3_object_key),
'Config':
config,
'VPCID':
vpc.vpc_id,
'MasterSubnetID':
vpc.public_subnets[0].subnet_id,
'ComputeSubnetID':
vpc.private_subnets[0].subnet_id,
'PostInstallScriptS3Url':
"".join([
's3://', pcluster_post_install_script.s3_bucket_name, "/",
pcluster_post_install_script.s3_object_key
]),
'PostInstallScriptBucket':
pcluster_post_install_script.s3_bucket_name,
'S3ReadWriteResource':
data_bucket.bucket_arn,
'S3ReadWriteUrl':
's3://%s' % (data_bucket.bucket_name),
'KeyPairId':
c9_createkeypair_cr.ref,
'KeyPairSecretArn':
c9_ssh_private_key_secret.ref,
'UserArn':
user.attr_arn,
'PclusterVersion':
pcluster_version.value_as_string
})
c9_bootstrap_cr.node.add_dependency(instance_id)
c9_bootstrap_cr.node.add_dependency(c9_createkeypair_cr)
c9_bootstrap_cr.node.add_dependency(c9_ssh_private_key_secret)
c9_bootstrap_cr.node.add_dependency(data_bucket)
enable_budget = cdk.CfnParameter(self,
"EnableBudget",
default="true",
type="String",
allowed_values=['true', 'false'
]).value_as_string
# Budgets
budget_properties = {
'budgetType': "COST",
'timeUnit': "ANNUALLY",
'budgetLimit': {
'amount':
cdk.CfnParameter(
self,
'BudgetLimit',
description=
'The initial budget for this project in USD ($).',
default=2000,
type='Number').value_as_number,
'unit':
"USD",
},
'costFilters': None,
'costTypes': {
'includeCredit': False,
'includeDiscount': True,
'includeOtherSubscription': True,
'includeRecurring': True,
'includeRefund': True,
'includeSubscription': True,
'includeSupport': True,
'includeTax': True,
'includeUpfront': True,
'useAmortized': False,
'useBlended': False,
},
'plannedBudgetLimits': None,
'timePeriod': None,
}
email = {
'notification': {
'comparisonOperator': "GREATER_THAN",
'notificationType': "ACTUAL",
'threshold': 80,
'thresholdType': "PERCENTAGE",
},
'subscribers': [{
'address':
cdk.CfnParameter(
self,
'NotificationEmail',
description=
'This email address will receive billing alarm notifications when 80% of the budget limit is reached.',
default='*****@*****.**').value_as_string,
'subscriptionType':
"EMAIL",
}]
}
overall_budget = budgets.CfnBudget(
self,
"HPCBudget",
budget=budget_properties,
notifications_with_subscribers=[email],
)
overall_budget.cfn_options.condition = cdk.CfnCondition(
self,
"BudgetCondition",
expression=cdk.Fn.condition_equals(enable_budget, "true"))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
current_directory = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
allowed_values = yaml.load(open(
os.path.join(current_directory, "..", "..",
"allowed_values.yaml")),
Loader=yaml.SafeLoader)
ami_mapping = {"AMI": {"OEJITSI": AMI_NAME}}
for region in generated_ami_ids.keys():
ami_mapping[region] = {"OEJITSI": generated_ami_ids[region]}
aws_ami_region_map = core.CfnMapping(self,
"AWSAMIRegionMap",
mapping=ami_mapping)
# utility function to parse the unique id from the stack id for
# shorter resource names using cloudformation functions
def append_stack_uuid(name):
return core.Fn.join("-", [
name,
core.Fn.select(
0,
core.Fn.split(
"-",
core.Fn.select(2, core.Fn.split(
"/", core.Aws.STACK_ID))))
])
#
# PARAMETERS
#
cidr_block_param = core.CfnParameter(
self,
"IngressCidrBlock",
allowed_pattern="((\d{1,3})\.){3}\d{1,3}/\d{1,2}",
default="0.0.0.0/0",
description=
"Required: A CIDR block to restrict access to the Jitsi application. Leave as 0.0.0.0/0 to allow public access from internet."
)
ec2_instance_type_param = core.CfnParameter(
self,
"InstanceType",
allowed_values=allowed_values["allowed_instance_types"],
default="t3.xlarge",
description=
"Required: The EC2 instance type for the application Auto Scaling Group."
)
jitsi_hostname_param = core.CfnParameter(
self,
"JitsiHostname",
description=
"Required: The hostname to access Jitsi. E.G. 'jitsi.internal.mycompany.com'"
)
jitsi_interface_app_name_param = core.CfnParameter(
self,
"JitsiInterfaceAppName",
default="Jitsi Meet",
description=
"Optional: Customize the app name on the Jitsi interface.")
jitsi_interface_default_remote_display_name_param = core.CfnParameter(
self,
"JitsiInterfaceDefaultRemoteDisplayName",
default="Fellow Jitster",
description=
"Optional: Customize the default display name for Jitsi users.")
jitsi_interface_native_app_name_param = core.CfnParameter(
self,
"JitsiInterfaceNativeAppName",
default="Jitsi Meet",
description=
"Optional: Customize the native app name on the Jitsi interface.")
jitsi_interface_show_brand_watermark_param = core.CfnParameter(
self,
"JitsiInterfaceShowBrandWatermark",
allowed_values=["true", "false"],
default="true",
description=
"Optional: Display the watermark logo image in the upper left corner."
)
jitsi_interface_show_watermark_for_guests_param = core.CfnParameter(
self,
"JitsiInterfaceShowWatermarkForGuests",
allowed_values=["true", "false"],
default="true",
description=
"Optional: Display the watermark logo image in the upper left corner for guest users. This can be set to override the general setting behavior for guest users."
)
jitsi_interface_brand_watermark_param = core.CfnParameter(
self,
"JitsiInterfaceBrandWatermark",
default="",
description=
"Optional: Provide a URL to a PNG image to be used as the brand watermark logo image in the upper right corner. File should be publically available for download."
)
jitsi_interface_brand_watermark_link_param = core.CfnParameter(
self,
"JitsiInterfaceBrandWatermarkLink",
default="http://jitsi.org",
description=
"Optional: Provide a link destination for the brand watermark logo image in the upper right corner."
)
jitsi_interface_watermark_param = core.CfnParameter(
self,
"JitsiInterfaceWatermark",
default="",
description=
"Optional: Provide a URL to a PNG image to be used as the watermark logo image in the upper left corner. File should be publically available for download."
)
jitsi_interface_watermark_link_param = core.CfnParameter(
self,
"JitsiInterfaceWatermarkLink",
default="http://jitsi.org",
description=
"Optional: Provide a link destination for the Jitsi watermark logo image in the upper left corner."
)
route_53_hosted_zone_name_param = core.CfnParameter(
self,
"Route53HostedZoneName",
description=
"Required: Route 53 Hosted Zone name in which a DNS record will be created by this template. Must already exist and be the domain part of the Jitsi Hostname parameter, without trailing dot. E.G. 'internal.mycompany.com'"
)
notification_email_param = core.CfnParameter(
self,
"NotificationEmail",
default="",
description=
"Optional: Specify an email address to get emails about deploys, Let's Encrypt, and other system events."
)
#
# CONDITIONS
#
notification_email_exists_condition = core.CfnCondition(
self,
"NotificationEmailExistsCondition",
expression=core.Fn.condition_not(
core.Fn.condition_equals(notification_email_param.value, "")))
#
# RESOURCES
#
# vpc
vpc = Vpc(self, "Vpc")
# sns
sns_notification_topic = aws_sns.CfnTopic(
self,
"NotificationTopic",
topic_name="{}-notifications".format(core.Aws.STACK_NAME))
sns_notification_subscription = aws_sns.CfnSubscription(
self,
"NotificationSubscription",
protocol="email",
topic_arn=sns_notification_topic.ref,
endpoint=notification_email_param.value_as_string)
sns_notification_subscription.cfn_options.condition = notification_email_exists_condition
iam_notification_publish_policy = aws_iam.PolicyDocument(statements=[
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
actions=["sns:Publish"],
resources=[sns_notification_topic.ref])
])
# cloudwatch
app_log_group = aws_logs.CfnLogGroup(
self, "JitsiAppLogGroup", retention_in_days=TWO_YEARS_IN_DAYS)
app_log_group.cfn_options.update_replace_policy = core.CfnDeletionPolicy.RETAIN
app_log_group.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN
system_log_group = aws_logs.CfnLogGroup(
self, "JitsiSystemLogGroup", retention_in_days=TWO_YEARS_IN_DAYS)
system_log_group.cfn_options.update_replace_policy = core.CfnDeletionPolicy.RETAIN
system_log_group.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN
# iam
iam_jitsi_instance_role = aws_iam.CfnRole(
self,
"JitsiInstanceRole",
assume_role_policy_document=aws_iam.PolicyDocument(statements=[
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=["sts:AssumeRole"],
principals=[aws_iam.ServicePrincipal("ec2.amazonaws.com")])
]),
policies=[
aws_iam.CfnRole.PolicyProperty(
policy_document=aws_iam.PolicyDocument(statements=[
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"logs:CreateLogStream",
"logs:DescribeLogStreams", "logs:PutLogEvents"
],
resources=[
app_log_group.attr_arn,
system_log_group.attr_arn
])
]),
policy_name="AllowStreamLogsToCloudWatch"),
aws_iam.CfnRole.PolicyProperty(
policy_document=aws_iam.PolicyDocument(statements=[
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"ec2:AssociateAddress", "ec2:DescribeVolumes",
"ec2:DescribeTags",
"cloudwatch:GetMetricStatistics",
"cloudwatch:ListMetrics",
"cloudwatch:PutMetricData"
],
resources=["*"])
]),
policy_name="AllowStreamMetricsToCloudWatch"),
aws_iam.CfnRole.PolicyProperty(
policy_document=aws_iam.PolicyDocument(statements=[
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=["autoscaling:Describe*"],
resources=["*"])
]),
policy_name="AllowDescribeAutoScaling"),
],
managed_policy_arns=[
"arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
])
# ec2
jitsi_sg = aws_ec2.CfnSecurityGroup(
self,
"JitsiSg",
group_description="Jitsi security group",
vpc_id=vpc.id())
eip = aws_ec2.CfnEIP(self, "Eip", domain="vpc")
core.Tags.of(eip).add("Name", "{}/Eip".format(core.Aws.STACK_NAME))
ec2_instance_profile = aws_iam.CfnInstanceProfile(
self, "JitsiInstanceProfile", roles=[iam_jitsi_instance_role.ref])
with open("jitsi/jitsi_launch_config_user_data.sh") as f:
jitsi_launch_config_user_data = f.read()
ec2_launch_config = aws_autoscaling.CfnLaunchConfiguration(
self,
"JitsiLaunchConfig",
image_id=core.Fn.find_in_map("AWSAMIRegionMap", core.Aws.REGION,
"OEJITSI"),
instance_type=ec2_instance_type_param.value_as_string,
iam_instance_profile=ec2_instance_profile.ref,
security_groups=[jitsi_sg.ref],
user_data=(core.Fn.base64(
core.Fn.sub(
jitsi_launch_config_user_data, {
"JitsiHostname":
jitsi_hostname_param.value_as_string,
"JitsiPublicIP":
eip.ref,
"LetsEncryptCertificateEmail":
notification_email_param.value_as_string
}))))
# autoscaling
asg = aws_autoscaling.CfnAutoScalingGroup(
self,
"JitsiAsg",
launch_configuration_name=ec2_launch_config.ref,
desired_capacity="1",
max_size="1",
min_size="1",
vpc_zone_identifier=vpc.public_subnet_ids())
asg.cfn_options.creation_policy = core.CfnCreationPolicy(
resource_signal=core.CfnResourceSignal(count=1, timeout="PT15M"))
asg.cfn_options.update_policy = core.CfnUpdatePolicy(
auto_scaling_rolling_update=core.CfnAutoScalingRollingUpdate(
max_batch_size=1,
min_instances_in_service=0,
pause_time="PT15M",
wait_on_resource_signals=True))
core.Tags.of(asg).add("Name",
"{}/JitsiAsg".format(core.Aws.STACK_NAME))
jitsi_http_ingress = aws_ec2.CfnSecurityGroupIngress(
self,
"JitsiHttpSgIngress",
cidr_ip=cidr_block_param.value_as_string,
from_port=80,
group_id=jitsi_sg.ref,
ip_protocol="tcp",
to_port=80)
jitsi_https_ingress = aws_ec2.CfnSecurityGroupIngress(
self,
"JitsiHttpsSgIngress",
cidr_ip=cidr_block_param.value_as_string,
from_port=443,
group_id=jitsi_sg.ref,
ip_protocol="tcp",
to_port=443)
jitsi_fallback_network_audio_video_ingress = aws_ec2.CfnSecurityGroupIngress(
self,
"JitsiFallbackNetworkAudioVideoSgIngress",
cidr_ip=cidr_block_param.value_as_string,
from_port=4443,
group_id=jitsi_sg.ref,
ip_protocol="tcp",
to_port=4443)
jitsi_general_network_audio_video_ingress = aws_ec2.CfnSecurityGroupIngress(
self,
"JitsiGeneralNetworkAudioVideoSgIngress",
cidr_ip=cidr_block_param.value_as_string,
from_port=10000,
group_id=jitsi_sg.ref,
ip_protocol="udp",
to_port=10000)
# route 53
record_set = aws_route53.CfnRecordSet(
self,
"RecordSet",
hosted_zone_name=
f"{route_53_hosted_zone_name_param.value_as_string}.",
name=jitsi_hostname_param.value_as_string,
resource_records=[eip.ref],
type="A")
# https://github.com/aws/aws-cdk/issues/8431
record_set.add_property_override("TTL", 60)
# AWS::CloudFormation::Interface
self.template_options.metadata = {
"OE::Patterns::TemplateVersion": template_version,
"AWS::CloudFormation::Interface": {
"ParameterGroups": [{
"Label": {
"default": "Infrastructure Config"
},
"Parameters": [
jitsi_hostname_param.logical_id,
route_53_hosted_zone_name_param.logical_id,
cidr_block_param.logical_id,
ec2_instance_type_param.logical_id,
notification_email_param.logical_id
]
}, {
"Label": {
"default": "Jitsi Config"
},
"Parameters": [
jitsi_interface_app_name_param.logical_id,
jitsi_interface_default_remote_display_name_param.
logical_id,
jitsi_interface_native_app_name_param.logical_id,
jitsi_interface_show_brand_watermark_param.logical_id,
jitsi_interface_show_watermark_for_guests_param.
logical_id,
jitsi_interface_brand_watermark_param.logical_id,
jitsi_interface_brand_watermark_link_param.logical_id,
jitsi_interface_watermark_param.logical_id,
jitsi_interface_watermark_link_param.logical_id,
]
}, *vpc.metadata_parameter_group()],
"ParameterLabels": {
cidr_block_param.logical_id: {
"default": "Ingress CIDR Block"
},
ec2_instance_type_param.logical_id: {
"default": "EC2 instance type"
},
jitsi_hostname_param.logical_id: {
"default": "Jitsi Hostname"
},
jitsi_interface_app_name_param.logical_id: {
"default": "Jitsi Interface App Name"
},
jitsi_interface_default_remote_display_name_param.logical_id:
{
"default":
"Jitsi Interface Default Remote Display Name"
},
jitsi_interface_native_app_name_param.logical_id: {
"default": "Jitsi Interface Native App Name"
},
jitsi_interface_show_brand_watermark_param.logical_id: {
"default": "Jitsi Interface Show Watermark"
},
jitsi_interface_show_watermark_for_guests_param.logical_id:
{
"default": "Jitsi Interface Show Watermark For Guests"
},
jitsi_interface_brand_watermark_param.logical_id: {
"default": "Jitsi Interface Watermark"
},
jitsi_interface_brand_watermark_link_param.logical_id: {
"default": "Jitsi Interface Watermark Link"
},
jitsi_interface_watermark_param.logical_id: {
"default": "Jitsi Interface Watermark"
},
jitsi_interface_watermark_link_param.logical_id: {
"default": "Jitsi Interface Watermark Link"
},
notification_email_param.logical_id: {
"default": "Notification Email"
},
route_53_hosted_zone_name_param.logical_id: {
"default": "AWS Route 53 Hosted Zone Name"
},
**vpc.metadata_parameter_labels()
}
}
}
#
# OUTPUTS
#
eip_output = core.CfnOutput(
self,
"EipOutput",
description=
"The Elastic IP address dynamically mapped to the autoscaling group instance.",
value=eip.ref)
endpoint_output = core.CfnOutput(
self,
"JitsiUrl",
description="The URL for the Jitsi instance.",
value=core.Fn.join(
"", ["https://", jitsi_hostname_param.value_as_string]))
def __init__(self,
scope: cdk.Construct,
construct_id: str,
stack_log_level: str,
custom_bkt_name: str = None,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
self.data_bkt = _s3.Bucket(
self,
"dataBucket",
versioned=True,
# auto_delete_objects=True,
# removal_policy=cdk.RemovalPolicy.DESTROY,
# bucket_name="new-app-bucket-example",
)
##################################################
######## ACCESS POINTS ###########
##################################################
# Lets set custom bucket name if it is set
if custom_bkt_name:
cfn_data_bkt = self.data_bkt.node.default_child
cfn_data_bkt.add_override("Properties.BucketName", custom_bkt_name)
# Delegate Privilege management to Access Point in bucket policy
# https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-points-policies.html#access-points-delegating-control
self.data_bkt.add_to_resource_policy(
_iam.PolicyStatement(actions=["*"],
principals=[_iam.AnyPrincipal()],
resources=[
f"{self.data_bkt.bucket_arn}",
f"{self.data_bkt.arn_for_objects('*')}"
],
conditions={
"StringEquals": {
"s3:DataAccessPointAccount":
f"{cdk.Aws.ACCOUNT_ID}"
}
}))
###########################################
################# OUTPUTS #################
###########################################
output_0 = cdk.CfnOutput(
self,
"AutomationFrom",
value=f"{GlobalArgs.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output_1 = cdk.CfnOutput(self,
"SalesEventsBucket",
value=f"{self.data_bkt.bucket_name}",
description=f"The datasource bucket name")
output_2 = cdk.CfnOutput(
self,
"dataSourceBucketUrl",
value=
f"https://console.aws.amazon.com/s3/buckets/{self.data_bkt.bucket_name}",
description=f"The datasource bucket url")
def __init__(
self, scope: cdk.Construct, construct_id: str, domain: SMSDomainStack, **kwargs
) -> None:
super().__init__(scope, construct_id, **kwargs)
# Create the Lambda Stack for pre-populating the user home directory
studio_user_lambda = StudioUserLambda(
self, "FnPopulateStudioUser", vpc=domain.vpc, domain=domain.domain
)
# Generate the CF template for the studio user
stage = cdk.Stage(self, "IntermediateStage")
SMSIAMUserStack(
stage,
"StudioUserStack",
synthesizer=cdk.BootstraplessSynthesizer(),
)
assembly = stage.synth(force=True)
# Retrive the local path of the CF template
template_full_path = assembly.stacks[0].template_full_path
# Upload CF template to s3 to create an asset to reference
s3_asset = s3assets.Asset(
self,
"TemplateAsset",
path=template_full_path,
)
# Create the Service Catalog product referencing the CF template
sc_product = servicecatalog.CfnCloudFormationProduct(
self,
"StudioUser",
owner="SageMakerStudio",
provisioning_artifact_parameters=[
servicecatalog.CfnCloudFormationProduct.ProvisioningArtifactPropertiesProperty(
info={"LoadTemplateFromURL": s3_asset.s3_url}
)
],
name="StudioUser",
)
# Create the Porduct Portfolio
sc_portfolio = servicecatalog.CfnPortfolio(
self,
"SageMakerPortfolio",
display_name="SageMakerPortfolio",
provider_name="SageMakerTemplate",
)
# Associate the Studio User Template to the Portfolio
servicecatalog.CfnPortfolioProductAssociation(
self,
"ProductAssociation",
portfolio_id=sc_portfolio.ref,
product_id=sc_product.ref,
)
# creat a role and associate it with the portfolio
sc_role = iam.Role(
self,
"StudioAdminRole",
assumed_by=iam.AnyPrincipal(),
role_name="SageMakerStudioAdminRole",
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AWSServiceCatalogEndUserFullAccess"
),
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonSageMakerFullAccess"
),
],
)
sc_role.add_to_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"sagemaker:CreateUserProfile",
],
resources=["*"],
)
)
sc_role.add_to_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"lambda:InvokeFunction",
],
resources=[studio_user_lambda.provider.service_token],
)
)
sc_role.add_to_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"s3:GetObject",
"s3:ListBucket",
],
resources=["*"],
)
)
cdk.CfnOutput(
self,
"SageMakerStudioAdminRole",
value=sc_role.role_arn,
description="SageMakerStudioAdminRole",
# export_name="SageMakerStudioAdminRole",
)
servicecatalog.CfnPortfolioPrincipalAssociation(
self,
"PortfolioPrincipalAssociacion",
portfolio_id=sc_portfolio.ref,
principal_arn=sc_role.role_arn,
principal_type="IAM",
)
def __init__(self, scope: core.Construct, id: str, vpc: _ec2.IVpc,
url: str, LOAD_PARAMS: dict, **kwargs) -> None:
"""
Defines an instance of the traffic generator.
:param scope: construct scope
:param id: construct id
:param vpc: the VPC in which to host the traffic generator cluster
:param url: the URL to hit
:param LOAD_PARAMS: LOCUST load testing parameters
"""
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here):
locust_cluster = _ecs.Cluster(self, "locustCluster", vpc=vpc)
"""
Load Testing Service in Fargate Cluster
"""
locust_task_def = _ecs.FargateTaskDefinition(
self,
"locustAppTaskDef",
)
locust_container = locust_task_def.add_container(
"locustAppContainer",
environment={
"github_profile": "https://github.com/miztiik",
"LOCUSTFILE_PATH": "/locustfile.py",
"TARGET_URL": url,
"LOCUST_OPTS":
f"--clients={LOAD_PARAMS['NO_OF_CLIENTS']} --hatch-rate={LOAD_PARAMS['HATCH_RATE']} --run-time={LOAD_PARAMS['RUN_TIME']} --no-web --print-stats",
# --clients The number of concurrent Locust users.
# --hatch-rate The rate per second in which clients are spawned.
# --run-time The number of seconds to run locust. ( Ensure enough time to hatch all users )
"ADDTIONAL_CUSTOM_OPTIONS": "--reset-stats --print-stats"
},
image=_ecs.ContainerImage.from_registry(
"mystique/xray-lambda-profiler:latest"),
logging=_ecs.LogDrivers.aws_logs(stream_prefix="Mystique"))
locust_container.add_port_mappings(
_ecs.PortMapping(container_port=80, protocol=_ecs.Protocol.TCP))
locust_container.add_port_mappings(
_ecs.PortMapping(container_port=443, protocol=_ecs.Protocol.TCP))
# Defines an AWS Lambda resource
with open(
"load_generator_stacks/lambda_src/create_fargate_run_task.py",
encoding="utf8") as fp:
create_fargate_run_task_fn_handler_code = fp.read()
pub_subnet_ids = []
for subnet in vpc.public_subnets:
pub_subnet_ids.append(subnet.subnet_id)
create_fargate_run_task_fn = _lambda.Function(
self,
id='triggerFargateRunTask',
function_name="create_fargate_run_task_fn",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(create_fargate_run_task_fn_handler_code),
handler='index.lambda_handler',
timeout=core.Duration.seconds(200),
environment={
"CLUSTER_NAME": locust_cluster.cluster_name,
"TASK_DEFINITION": locust_task_def.task_definition_arn,
"SUBNETS": json.dumps(pub_subnet_ids),
"CONTAINER_NAME": locust_container.container_name,
"NO_OF_TASKS": LOAD_PARAMS["NO_OF_TASKS"]
},
reserved_concurrent_executions=1,
)
roleStmt1 = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[locust_task_def.task_definition_arn],
actions=["ecs:RunTask"])
roleStmt1.sid = "AllowLambdaToCreateFargateRunTask"
create_fargate_run_task_fn.add_to_role_policy(roleStmt1)
# task_role = locust_task_def.execution_role.role_arn
roleStmt2 = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
locust_task_def.execution_role.role_arn,
locust_task_def.task_role.role_arn
],
actions=["iam:PassRole"])
roleStmt2.sid = "AllowLambdaToPassRoleToRunTask"
create_fargate_run_task_fn.add_to_role_policy(roleStmt2)
# Create a trigger for our 'create_fargate_run_task' lambda Function
gen_load = trigger_run_task(self,
"triggerLoadGeneratorTask",
config_params={
"RUN_TASK_FN_ARN":
create_fargate_run_task_fn.function_arn
},
message=[{
"RUN_TASK_FN_ARN":
create_fargate_run_task_fn.function_arn
}])
"""
locust_service = _ecs.FargateService(self, 'locustAppService',
cluster=locust_cluster,
task_definition=locust_task_def,
desired_count=2,
assign_public_ip=True,
service_name=f"{global_args.OWNER}-LocustLoadGenerator"
)
"""
output_0 = core.CfnOutput(
self,
"AutomationFrom",
value=f"{global_args.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output_1 = core.CfnOutput(
self,
"LocustClusterName",
value=f"{locust_cluster.cluster_name}",
export_name="locustClusterName",
description=
"The fargate cluster to generate load on APIs using Locust")
def __init__(self, scope: core.Construct, id: str, vpc,
ec2_instance_type: str, es_endpoint_param_name: str,
es_region_param_name: str, stack_log_level: str,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Read BootStrap Script):
try:
with open(
"elastic_fluent_bit_kibana/stacks/back_end/bootstrap_scripts/deploy_app.sh",
encoding="utf-8",
mode="r") as f:
user_data = f.read()
except OSError as e:
print("Unable to read UserData script")
raise e
# Get the latest AMI from AWS SSM
linux_ami = _ec2.AmazonLinuxImage(
generation=_ec2.AmazonLinuxGeneration.AMAZON_LINUX_2)
# Get the latest ami
amzn_linux_ami = _ec2.MachineImage.latest_amazon_linux(
generation=_ec2.AmazonLinuxGeneration.AMAZON_LINUX_2)
# ec2 Instance Role
_instance_role = _iam.Role(
self,
"webAppClientRole",
assumed_by=_iam.ServicePrincipal("ec2.amazonaws.com"),
managed_policies=[
_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonSSMManagedInstanceCore")
])
# Allow CW Agent to create Logs
_instance_role.add_to_policy(
_iam.PolicyStatement(actions=["logs:Create*", "logs:PutLogEvents"],
resources=["arn:aws:logs:*:*:*"]))
# Allow Access to ElasticSearch Domain
# https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-ac.html#es-ac-types-resource
_instance_role.add_to_policy(
_iam.PolicyStatement(actions=[
"es:Describe*",
"es:List*",
"es:ESHttpPost",
"es:ESHttpPut",
],
resources=["*"]))
# fluent_bit_server Instance
self.fluent_bit_server = _ec2.Instance(
self,
"fluentBitLogRouter",
instance_type=_ec2.InstanceType(
instance_type_identifier=f"{ec2_instance_type}"),
instance_name="fluent_bit_log_router_01",
machine_image=amzn_linux_ami,
vpc=vpc,
vpc_subnets=_ec2.SubnetSelection(
subnet_type=_ec2.SubnetType.PUBLIC),
role=_instance_role,
user_data=_ec2.UserData.custom(user_data))
# Allow Web Traffic to WebServer
self.fluent_bit_server.connections.allow_from_any_ipv4(
_ec2.Port.tcp(80), description="Allow Incoming HTTP Traffic")
self.fluent_bit_server.connections.allow_from(
other=_ec2.Peer.ipv4(vpc.vpc_cidr_block),
port_range=_ec2.Port.tcp(443),
description="Allow Incoming FluentBit Traffic")
# Allow CW Agent to create Logs
_instance_role.add_to_policy(
_iam.PolicyStatement(actions=["logs:Create*", "logs:PutLogEvents"],
resources=["arn:aws:logs:*:*:*"]))
# Let us prepare our FluentBit Configuration Script
# Use the script below, if you have a pre-written script, if not use the parts below to ASSEMBLE the script
# Read BootStrap Script):
try:
with open(
"elastic_fluent_bit_kibana/stacks/back_end/bootstrap_scripts/configure_fluent_bit.sh",
encoding="utf-8",
mode="r") as f:
bash_commands_to_run = f.read()
except OSError as e:
print("Unable to read bash commands file")
raise e
es_endpoint = _ssm.StringParameter.value_for_string_parameter(
self, es_endpoint_param_name)
es_region = _ssm.StringParameter.value_for_string_parameter(
self, es_region_param_name)
bash_commands_to_run_01 = """
#!/bin/bash
set -ex
set -o pipefail
# version: 22Nov2020
##################################################
############# SET GLOBALS ################
##################################################
REPO_NAME="elastic-fluent-bit-kibana"
GIT_REPO_URL="https://github.com/miztiik/$REPO_NAME.git"
APP_DIR="/var/$REPO_NAME"
LOG_FILE="/var/log/miztiik-automation-configure-fluent-bit.log"
function install_fluent_bit(){
# https://docs.fluentbit.io/manual/installation/linux/amazon-linux
cat > '/etc/yum.repos.d/td-agent-bit.repo' << "EOF"
[td-agent-bit]
name = TD Agent Bit
baseurl = https://packages.fluentbit.io/amazonlinux/2/$basearch/
gpgcheck=1
gpgkey=https://packages.fluentbit.io/fluentbit.key
enabled=1
EOF
# Install the agent
sudo yum -y install td-agent-bit
sudo service td-agent-bit start
service td-agent-bit status
}
function create_config_files(){
mkdir -p ${APP_DIR}
cd ${APP_DIR}
echo "
[INPUT]
name tail
path /var/log/httpd/*log
tag automate_log_parse
Path_Key filename
[FILTER]
Name record_modifier
Match *
Record hostname ${HOSTNAME}
Record project elastic-fluent-bit-kibana-demo
Add user Mystique
" > ${APP_DIR}/es.conf
"""
bash_commands_to_run_02 = f"""
echo "
[OUTPUT]
Name es
Match automate_log*
Host {es_endpoint}
Port 443
tls On
AWS_Auth On
AWS_Region {es_region}
Index miztiik_automation
Type app_logs
Include_Tag_Key On
" >> ${{APP_DIR}}/es.conf
}}
"""
bash_commands_to_run_03 = """
function configure_fluent_bit(){
# Stop the agent
sudo service td-agent-bit stop
# DO NOT DO THIS IN ANY SERIOUS CONFIG FILE
# Null the defaults and start fresh
> /etc/td-agent-bit/td-agent-bit.conf
echo "
[SERVICE]
Flush 3
@INCLUDE ${APP_DIR}/es.conf
" > /etc/td-agent-bit/td-agent-bit.conf
sudo service td-agent-bit start
sudo service td-agent-bit status
}
install_fluent_bit >> "${LOG_FILE}"
create_config_files >> "${LOG_FILE}"
configure_fluent_bit >> "${LOG_FILE}"
"""
bash_commands_to_run = bash_commands_to_run_01 + \
bash_commands_to_run_02 + bash_commands_to_run_03
# Configure Fluent Bit using SSM Run Commands
config_fluenbit_doc = CreateSsmRunCommandDocument(
self,
"configureFluentBitToEs",
run_document_name="configureFluentBitToEs",
_doc_desc="Bash script to configure FluentBit to send logs to ES",
bash_commands_to_run=bash_commands_to_run,
enable_log=False)
# Create SSM Association to trigger SSM doucment to target (EC2)
_run_commands_on_ec2 = _ssm.CfnAssociation(
self,
"runCommandsOnEc2",
name=config_fluenbit_doc.get_ssm_linux_document_name,
targets=[{
"key": "InstanceIds",
"values": [self.fluent_bit_server.instance_id]
}])
###########################################
################# OUTPUTS #################
###########################################
output_0 = core.CfnOutput(
self,
"AutomationFrom",
value=f"{GlobalArgs.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output_1 = core.CfnOutput(
self,
"FluentBitPrivateIp",
value=f"http://{self.fluent_bit_server.instance_private_ip}",
description=f"Private IP of Fluent Bit Server on EC2")
output_2 = core.CfnOutput(
self,
"FluentBitInstance",
value=(f"https://console.aws.amazon.com/ec2/v2/home?region="
f"{core.Aws.REGION}"
f"#Instances:search="
f"{self.fluent_bit_server.instance_id}"
f";sort=instanceId"),
description=
f"Login to the instance using Systems Manager and use curl to access the Instance"
)
output_3 = core.CfnOutput(
self,
"AwsForFluentBit",
value=
(f"https://github.com/aws/aws-for-fluent-bit/tree/master/examples/fluent-bit/systems-manager-ec2"
),
description=f"Amazon docs on fluent bit")
output_4 = core.CfnOutput(
self,
"WebServerUrl",
value=f"{self.fluent_bit_server.instance_public_dns_name}",
description=f"Public IP of Web Server on EC2")
output_5 = core.CfnOutput(
self,
"GenerateAccessTraffic",
value=
f"ab -n 10 -c 1 http://{self.fluent_bit_server.instance_public_dns_name}/",
description=f"Public IP of Web Server on EC2")
output_6 = core.CfnOutput(
self,
"GenerateFailedTraffic",
value=
f"ab -n 10 -c 1 http://{self.fluent_bit_server.instance_public_dns_name}/${{RANDOM}}",
description=f"Public IP of Web Server on EC2")