def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# s3
s3_bucket_name = "{}-s3-{}".format(Constant.PROJECT_NAME,
self._get_UUID(4))
_s3.Bucket(
self,
id=s3_bucket_name,
bucket_name=s3_bucket_name,
removal_policy=core.RemovalPolicy.
DESTROY, #TODO: destroy for test
# removal_policy=core.RemovalPolicy.RETAIN
)
# step 1. VPC
# 如果在已有的Vpc 中建立环境， 可以用下面这句， 需要传入 vpc_id
# vpc = ec2.Vpc.from_lookup(self, "VPC", vpc_id='')
vpc = ec2.Vpc(
self,
"VPC",
max_azs=2, # 两个分区， 每个分区建一个子网
cidr="10.10.0.0/16",
# configuration will create 3 groups in 2 AZs = 6 subnets.
subnet_configuration=[
ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.PUBLIC,
name="Public",
cidr_mask=24),
ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.PRIVATE,
name="Private",
cidr_mask=24),
# ec2.SubnetConfiguration(
# subnet_type=ec2.SubnetType.ISOLATED,
# name="DB",
# cidr_mask=24
# )
],
# nat_gateway_provider=ec2.NatProvider.gateway(),
# nat_gateways=2,
)
# ES 需要部署到私有子网中
selection = vpc.select_subnets(subnet_type=ec2.SubnetType.PRIVATE)
# step 2. 访问S3 + ES集群需要的 iam_instance_profile
# action -> statement -> policy -> role -> instance profile -> attach ec2
actions = [
"ec2:CreateNetworkInterface", "ec2:DeleteNetworkInterface",
"ec2:DescribeNetworkInterfaces",
"ec2:ModifyNetworkInterfaceAttribute",
"ec2:DescribeSecurityGroups", "ec2:DescribeSubnets",
"ec2:DescribeVpcs", "s3:*"
]
policyStatement = PolicyStatement(actions=actions, effect=Effect.ALLOW)
policyStatement.add_all_resources()
policyStatement.sid = "Stmt1480452973134"
policy_name = "{}-ec2-es-policy".format(Constant.PROJECT_NAME)
ec2_policy = Policy(self, policy_name, policy_name=policy_name)
ec2_policy.add_statements(policyStatement)
role_name = "{}-ec2-es-role".format(Constant.PROJECT_NAME)
access_es_role = Role(
self,
role_name,
role_name=role_name,
assumed_by=ServicePrincipal('ec2.amazonaws.com.cn'))
ec2_policy.attach_to_role(access_es_role)
profile_name = "{}-ec2-es-profile".format(Constant.PROJECT_NAME)
instance_profile = CfnInstanceProfile(
self,
profile_name,
instance_profile_name=profile_name,
roles=[access_es_role.role_name])
# step 4. ES
# 生产环境建议设置安全组， 只接收VPC内443端口请求
sg_es_cluster_name = "{}-sg-es".format(Constant.PROJECT_NAME)
sg_es_cluster = ec2.SecurityGroup(
self,
id=sg_es_cluster_name,
vpc=vpc,
security_group_name=sg_es_cluster_name)
sg_es_cluster.add_ingress_rule(peer=ec2.Peer.ipv4(vpc.vpc_cidr_block),
connection=ec2.Port.tcp(443))
es_name = Constant.PROJECT_NAME
es_arn = self.format_arn(service="es",
resource="domain",
sep="/",
resource_name=es_name)
es = elasticsearch.CfnDomain(
self,
es_name,
elasticsearch_version='7.1',
domain_name=es_name,
node_to_node_encryption_options={"enabled": False},
vpc_options={
"securityGroupIds": [sg_es_cluster.security_group_id
], # 生产环境建议设置安全组， 只接收VPC内443端口请求
# 如果开启多个节点， 需要配置多个子网， 目前测试只有一个ES 节点， 就只用到一个子网
"subnetIds": selection.subnet_ids[:1]
},
ebs_options={
"ebsEnabled": True,
"volumeSize": 10,
"volumeType": "gp2"
},
elasticsearch_cluster_config={
# 生成环境需要开启三个
# "dedicatedMasterCount": 3,
# "dedicatedMasterEnabled": True,
# "dedicatedMasterType": 'm4.large.elasticsearch',
"instanceCount": 1,
"instanceType": 'm4.large.elasticsearch',
"zoneAwarenessEnabled": False
})
es.access_policies = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": "es:*",
"Resource": "{}/*".format(es_arn)
}]
}
# step 5. SNS
topic = sns.Topic(self, "topic")
topic.add_subscription(subs.EmailSubscription(Constant.EMAIL_ADDRESS))
# 设置SNS endpoint, 让lambda 可以从vpc 内部访问
vpc.add_interface_endpoint(
"SNSEndpoint", service=ec2.InterfaceVpcEndpointAwsService.SNS)
# step 6. Lambda
lambdaFn = lambda_.Function(self,
"Singleton",
code=lambda_.Code.asset('lambda'),
handler='hello.handler',
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE),
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
environment={
'SNS_TOPIC_ARN': topic.topic_arn,
'ES_ENDPOINT': es.attr_domain_endpoint,
'ES_INDEX_NAME': Constant.ES_INDEX_NAME
})
# step 7. Cloud watch event
rule = events.Rule(
self,
"Rule",
schedule=events.Schedule.cron(minute='0/5',
hour='*',
month='*',
week_day='*',
year='*'),
)
rule.add_target(targets.LambdaFunction(lambdaFn))
#给Lambda 添加发布SNS的权限
topic.grant_publish(lambdaFn)
# Create ALB
alb_name = "{}-alb".format(Constant.PROJECT_NAME)
alb = elb.ApplicationLoadBalancer(self,
alb_name,
vpc=vpc,
internet_facing=True,
load_balancer_name=alb_name)
alb.connections.allow_from_any_ipv4(ec2.Port.tcp(80),
"Internet access ALB 80")
listener = alb.add_listener("my80", port=80, open=True)
# Create Autoscaling Group with fixed 2*EC2 hosts
user_data = user_data_content.format(es.attr_domain_endpoint,
Constant.REGION_NAME,
Constant.ES_LOG_PATH,
Constant.ES_INDEX_NAME,
s3_bucket_name)
# step 3. 创建堡垒机
bastion_name = "{}-bastion".format(Constant.PROJECT_NAME)
bastion = ec2.BastionHostLinux(
self,
bastion_name,
vpc=vpc,
subnet_selection=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PUBLIC),
instance_name=bastion_name,
instance_type=ec2.InstanceType(
instance_type_identifier="m4.large"))
bastion.instance.instance.add_property_override(
"KeyName", Constant.EC2_KEY_NAME)
bastion.connections.allow_from_any_ipv4(
ec2.Port.tcp(22), "Internet access SSH") # 生成环境可以限定IP allow_from
bastion.connections.allow_from_any_ipv4(ec2.Port.tcp(8080),
"Internet access HTTP") # 测试需要
# bastion.connections.allow_from_any_ipv4(ec2.Port.tcp(443), "Internet access HTTPS") # 测试需要
bastion.instance.instance.iam_instance_profile = instance_profile.instance_profile_name # 给EC2设置 profile , 相当于Role
bastion.instance.instance.image_id = ami_map.get(
Constant.REGION_NAME) # 指定AMI ID
#堡垒机的user_data 只能执行一次， 如果要执行多次， 请参考 https://amazonaws-china.com/premiumsupport/knowledge-center/execute-user-data-ec2/?nc1=h_ls
bastion_user_data = "/home/ec2-user/start.sh {} {} '{}' {} {}".format(
es.attr_domain_endpoint, Constant.REGION_NAME,
Constant.ES_LOG_PATH, Constant.ES_INDEX_NAME, s3_bucket_name)
bastion.instance.add_user_data(
"date >> /home/ec2-user/root.txt") # 查看启动脚本是否执行
bastion.instance.add_user_data(bastion_user_data)
asg_name = "{}-asg".format(Constant.PROJECT_NAME)
asg = autoscaling.AutoScalingGroup(
self,
asg_name,
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PUBLIC), # PUBLIC for debug
instance_type=ec2.InstanceType(
instance_type_identifier="m4.large"),
machine_image=my_ami,
key_name=Constant.EC2_KEY_NAME,
user_data=ec2.UserData.custom(user_data),
desired_capacity=1,
min_capacity=1,
max_capacity=1,
role=access_es_role)
asg.connections.allow_from(
alb, ec2.Port.tcp(8080),
"ALB access 80 port of EC2 in Autoscaling Group")
# asg.connections.allow_from_any_ipv4(ec2.Port.tcp(8080), "Internet access HTTP for test") # 测试用
asg.connections.allow_from_any_ipv4(ec2.Port.tcp(22),
"Internet access SSH") # for debug
listener.add_targets("addTargetGroup", port=8080, targets=[asg])
core.CfnOutput(self,
"s3_bucket_name",
value=s3_bucket_name,
description='S3 bucket: store web log')
core.CfnOutput(self,
"ElasticSearchEndpointUrl",
value=es.attr_domain_endpoint,
description='Elastic Search Url')
# Elastic Search 统计log 数量， 可以在堡垒机上执行， 快速查看日志数量。
core.CfnOutput(self,
"CmdGetCountIndex",
value='curl https://{}/{}/_count'.format(
es.attr_domain_endpoint, Constant.ES_INDEX_NAME),
description='Count search result. ')
# 堡垒机的登录命令， 可以直接复制使用
core.CfnOutput(self,
"CmdSshToBastion",
value='ssh -i ~/{}.pem ec2-user@{}'.format(
Constant.EC2_KEY_NAME,
bastion.instance_public_dns_name),
description='cmd ssh to bastion')
# 在堡垒机上启动服务的命令， 堡垒机重启以后， 需要执行下面的命令， 可以启动web服务 发送日志到ES
core.CfnOutput(
self,
"CmdSshBastionStartWeb",
value='sudo {}'.format(bastion_user_data),
description="Cmd to start web+logstash+filebeat service")
# ALB 的访问地址
core.CfnOutput(self,
"UrlLoad_Balancer",
value='http://{}'.format(alb.load_balancer_dns_name),
description='ALB url ')
# 堡垒机的web访问地址， 为了调试方便， 在堡垒机上也使用相同的AMI。
core.CfnOutput(self,
"UrlBastion",
value='http://{}:8080'.format(
bastion.instance_public_dns_name),
description="Bastion server web url ")
# 下面这条输出的命令 是通过堡垒机和Elasticsearch 建立隧道， 在本地访问kibana。
core.CfnOutput(
self,
"CmdSshProxyToKibana",
value='ssh -i ~/{}.pem ec2-user@{} -N -L 9200:{}:443'.format(
Constant.EC2_KEY_NAME, bastion.instance_public_dns_name,
es.attr_domain_endpoint),
description="cmd: access kibana from bastion ssh. ")
# 执行完上面的命令后， 在浏览器中打开下面的连接
core.CfnOutput(self,
"UrlKibana",
value='https://localhost:9200/_plugin/kibana/',
description="kibana url ")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# 创建sts lambda访问角色
# action -> statement -> policy -> role -> attach lambda
actions = [
"logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents",
"sts:AssumeRole"
]
policyStatement = PolicyStatement(actions=actions, effect=Effect.ALLOW)
policyStatement.add_all_resources()
policy_name = "{}-policy".format(Constant.PROJECT_NAME)
sts_policy = Policy(self, policy_name, policy_name=policy_name)
sts_policy.add_statements(policyStatement)
role_name = "{}-role".format(Constant.PROJECT_NAME)
access_role = Role(self,
role_name,
role_name=role_name,
assumed_by=ServicePrincipal('lambda.amazonaws.com'))
sts_policy.attach_to_role(access_role)
# 创建thum lambda访问角色
# action -> statement -> policy -> role
thum_actions = [
"logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents",
"s3:PutObject"
]
thum_policyStatement = PolicyStatement(actions=thum_actions,
effect=Effect.ALLOW)
thum_policyStatement.add_all_resources()
thum_policy_name = "{}-policy-thum".format(Constant.PROJECT_NAME)
thum_policy = Policy(self,
thum_policy_name,
policy_name=thum_policy_name)
thum_policy.add_statements(thum_policyStatement)
thum_role_name = "{}-role-thum".format(Constant.PROJECT_NAME)
thum_access_role = Role(
self,
thum_role_name,
role_name=thum_role_name,
assumed_by=ServicePrincipal('lambda.amazonaws.com'))
thum_policy.attach_to_role(thum_access_role)
# 创建S3 put的角色
# action -> statement -> policy -> role
s3_actions = [
"s3:PutObject",
"s3:GetObject",
"s3:ListBucket",
"s3:PutObjectTagging",
"s3:PutObjectAcl",
]
s3_policyStatement = PolicyStatement(actions=s3_actions,
effect=Effect.ALLOW)
s3_policyStatement.add_all_resources()
s3_policy_name = "{}-policy-s3".format(Constant.PROJECT_NAME)
s3_policy = Policy(self, s3_policy_name, policy_name=s3_policy_name)
s3_policy.add_statements(s3_policyStatement)
s3_role_name = "{}-role-s3".format(Constant.PROJECT_NAME)
s3_access_role = Role(self,
s3_role_name,
role_name=s3_role_name,
assumed_by=ArnPrincipal(access_role.role_arn))
s3_policy.attach_to_role(s3_access_role)
# 创建STS lambda
sts_lambda = _lambda.Function(
self,
'sts',
function_name='sts',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='auth.handler',
timeout=Duration.minutes(1),
role=access_role,
)
sts_lambda.add_environment("role_to_assume_arn",
s3_access_role.role_arn)
base_api = apigw.RestApi(
self,
'Endpoint',
endpoint_types=[EndpointType.REGIONAL],
)
example_entity = base_api.root.add_resource('auth')
example_entity_lambda_integration = apigw.LambdaIntegration(
sts_lambda,
proxy=False,
integration_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin':
"'*'",
}
}])
example_entity.add_method(
'GET',
example_entity_lambda_integration,
method_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': True,
}
}])
self.add_cors_options(example_entity)
# 创建缩略图lambda
layer_cv2 = _lambda.LayerVersion(
self,
'cv2',
code=_lambda.Code.from_bucket(
s3.Bucket.from_bucket_name(self,
"cdk-data-layer",
bucket_name='nowfox'),
'cdk-data/cv2.zip'),
compatible_runtimes=[_lambda.Runtime.PYTHON_3_7],
)
lambda_thum = _lambda.Function(
self,
'thum',
function_name='thum',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='thum.handler',
timeout=Duration.minutes(1),
role=thum_access_role,
)
lambda_thum.add_environment("frame_second", "3")
lambda_thum.add_layers(layer_cv2)
# 创建存储上传图片的bucket
s3_bucket_name = "{}-{}".format("upload", self._get_UUID(4))
s3_upload = s3.Bucket(
self,
id=s3_bucket_name,
bucket_name=s3_bucket_name,
# access_control=s3.BucketAccessControl.PUBLIC_READ,#不建议使用这个，这个会有list权限，下面这个没有list权限
public_read_access=True,
removal_policy=core.RemovalPolicy.
DESTROY, # TODO: destroy for test
# removal_policy=core.RemovalPolicy.RETAIN
)
notification = aws_s3_notifications.LambdaDestination(lambda_thum)
s3_filter = s3.NotificationKeyFilter(suffix=".mp4")
s3_upload.add_event_notification(s3.EventType.OBJECT_CREATED_PUT,
notification, s3_filter)
s3_upload.add_cors_rule(allowed_methods=[
HttpMethods.POST, HttpMethods.PUT, HttpMethods.GET
],
allowed_origins=["*"],
allowed_headers=["*"])
'''
#创建上传lambda
lambda_upload = _lambda.Function(
self, 'upload',function_name='upload',
runtime=_lambda.Runtime.JAVA_8,
code=_lambda.Code.from_bucket(s3.Bucket.from_bucket_name(self, "cdk-data-upload-jar", bucket_name='nowfox'),
'cdk-data/fileupload.jar'),
handler='fileupload.FileUploadFunctionHandler::handleRequest',
timeout=Duration.minutes(1),
memory_size=512,
role=access_role,
)
lambda_upload.add_environment("bucket", s3_bucket_name)
lambda_upload.add_environment("region", "cn-northwest-1")
'''
core.CfnOutput(self,
"authUrl",
value=base_api.url + "auth",
description="authUrl")
core.CfnOutput(self,
"S3BucketName",
value=s3_bucket_name,
description="S3BucketName")