def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here self.vpc = ec2.Vpc(self, "VPC", cidr='10.10.0.0/16', max_azs=6, subnet_configuration=[ ec2.SubnetConfiguration( subnet_type=ec2.SubnetType.PRIVATE, name='Private', cidr_mask=19 ), ec2.SubnetConfiguration( subnet_type=ec2.SubnetType.PUBLIC, name='Public', cidr_mask=20 ), ec2.SubnetConfiguration( subnet_type=ec2.SubnetType.PRIVATE, name='Spare', cidr_mask=20, reserved=True ) ]) core.CfnOutput(self, "Output", value=self.vpc.vpc_id) self.bastion = ec2.BastionHostLinux(self, id, vpc = self.vpc, instance_name = 'bastione', instance_type = ec2.InstanceType('t3.micro'), machine_image = ec2.AmazonLinuxImage(), subnet_selection = ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), ) core.CfnOutput(self, 'bastion-id', value=self.bastion.instance_id)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here #XXX: To use more than 2 AZs, be sure to specify the account and region on your stack. #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/Vpc.html vpc = aws_ec2.Vpc(self, 'VpcStack', max_azs=6) sg_ssh_access = aws_ec2.SecurityGroup( self, "BastionHostSG", vpc=vpc, allow_all_outbound=True, description='security group for bastion host', security_group_name='bastion-host-sg') core.Tag.of(sg_ssh_access).add('Name', 'bastion-host') sg_ssh_access.add_ingress_rule(peer=aws_ec2.Peer.any_ipv4(), connection=aws_ec2.Port.tcp(22), description='ssh access') bastion_host = aws_ec2.BastionHostLinux( self, "BastionHost", vpc=vpc, instance_type=aws_ec2.InstanceType('t3.nano'), security_group=sg_ssh_access, subnet_selection=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here vpc_name = self.node.try_get_context("vpc_name") vpc = aws_ec2.Vpc.from_lookup( self, "EC2InstanceVPC", is_default= True, # set is_default=False if you want to find your own VPC vpc_name=vpc_name) sg_ssh_access = aws_ec2.SecurityGroup( self, "BastionHostSG", vpc=vpc, allow_all_outbound=True, description='security group for bastion host', security_group_name='bastion-host-sg') core.Tag.of(sg_ssh_access).add('Name', 'bastion-host') sg_ssh_access.add_ingress_rule(peer=aws_ec2.Peer.any_ipv4(), connection=aws_ec2.Port.tcp(22), description='ssh access') bastion_host = aws_ec2.BastionHostLinux( self, "BastionHost", vpc=vpc, instance_type=aws_ec2.InstanceType('t3.nano'), security_group=sg_ssh_access, subnet_selection=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC))
def __init__(self, scope: core.Construct, construct_id: str, vpc, ec2_capacity=False, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) self.cluster = ecs.Cluster(self, "CdkCluster", cluster_name="cdk-ecs", vpc=vpc) if ec2_capacity: key_name = os.getenv("EC2_KEY_PAIR_NAME") asg = self.cluster.add_capacity("CdkClusterCapacity", instance_type=ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE3_AMD, ec2.InstanceSize.SMALL), key_name=key_name, cooldown=cdk_core.Duration.seconds(30), min_capacity=1, max_capacity=5) # note: CDK doesn't support ECS capacity providers yet asg.scale_to_track_metric("CpuReservationScalingPolicy", metric=self.cluster.metric_cpu_reservation(), target_value=50) asg.scale_to_track_metric("MemoryReservationScalingPolicy", metric=self.cluster.metric_memory_reservation(), target_value=50) bastion = ec2.BastionHostLinux(self, "CdkBastion", vpc=vpc, subnet_selection=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC)) bastion.allow_ssh_access_from(ec2.Peer.any_ipv4()) bastion.instance.instance.key_name = key_name asg.connections.allow_from(bastion, ec2.Port.tcp(22))
def build(cls, *, stack): vpc_db_instance = cls() vpc_db_instance.vpc = ec2.Vpc(stack, "vpc", cidr="10.0.0.0/24" ) master_user_name = "exampleadmin" vpc_db_instance.db_security_group = ec2.SecurityGroup(stack, "dbsecuritygroup", security_group_name="DBSG", vpc=vpc_db_instance.vpc, allow_all_outbound=True ) vpc_db_instance.db_instance = rds.DatabaseInstance(stack, 'exampleinstance', master_username=master_user_name, engine=rds.DatabaseInstanceEngine.POSTGRES, instance_type=ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE2, ec2.InstanceSize.SMALL), vpc=vpc_db_instance.vpc, auto_minor_version_upgrade=True, database_name='exampledb', storage_encrypted=True, multi_az=False, security_groups=[vpc_db_instance.db_security_group] ) # create bastion host in public subnet with no KeyPairName (use ec2-instance-connect) vpc_db_instance.bastion_host_security_group = ec2.SecurityGroup(stack, "bastionhostsecuritygroup", security_group_name="bastionhostsecuritygroup", vpc=vpc_db_instance.vpc, allow_all_outbound=True ) vpc_db_instance.bastion_host_linux = ec2.BastionHostLinux(stack, "bastionhostSG", vpc=vpc_db_instance.vpc, instance_name="bastionhost", instance_type=ec2.InstanceType("t2.micro"), subnet_selection={ "subnet_type": ec2.SubnetType.PUBLIC }, security_group=vpc_db_instance.bastion_host_security_group ) vpc_db_instance.db_security_group.add_ingress_rule(vpc_db_instance.bastion_host_security_group, ec2.Port.tcp(5432), 'bastion host') return vpc_db_instance
def __init__(self, scope: core.Construct, id: str, vpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) #Creating Bastion bastion = ec2.BastionHostLinux(self, 'webapp-bastion', vpc=vpc, subnet_selection=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PUBLIC)) bastion.instance.instance.add_property_override("KeyName", key_name) bastion.connections.allow_from_any_ipv4(ec2.Port.tcp(22), "Internet access SSH") #Creating AutoScaling group self.asg = autoscaling.AutoScalingGroup( self, "webapp-autoscaling", vpc=vpc, vpc_subnets=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PRIVATE), instance_type=ec2.InstanceType(instance_type_identifier=ec2_type), machine_image=linux_ami, key_name=key_name, user_data=ec2.UserData.custom(user_data), desired_capacity=2, min_capacity=2, max_capacity=2, ) #Creating Load Balancer alb = elb.ApplicationLoadBalancer(self, "webapp-alb", vpc=vpc, internet_facing=True, load_balancer_name="webapp-alb") alb.connections.allow_from_any_ipv4(ec2.Port.tcp(80), "Internet access ALB 80") listener = alb.add_listener("my80", port=80, open=True) #Final Configuration self.asg.connections.allow_from( alb, ec2.Port.tcp(80), "ALB access 80 port of EC2 in Autoscaling Group") listener.add_targets("addTargetGroup", port=80, targets=[self.asg]) core.CfnOutput(self, "Output", value=alb.load_balancer_dns_name)
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # The code that defines your stack goes here vpc = ec2.Vpc( self, "VPC", cidr="10.0.0.0/21", max_azs=3, nat_gateways=1, subnet_configuration=[ ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.PUBLIC, name="Ingress", cidr_mask=24), ec2.SubnetConfiguration( subnet_type=ec2.SubnetType.PRIVATE, name="Application", cidr_mask=24, ), ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.ISOLATED, name="Database", cidr_mask=28, reserved=True) ]) host = ec2.BastionHostLinux( self, "cdk-vpc-BastionHost", vpc=vpc, subnet_selection=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PUBLIC), # block_devices=[ec2.BlockDevice( # device_name="EBSBastionHost_CDK", # volume=ec2.BlockDeviceVolume.ebs(30, # encrypted=True, # delete_on_termination=True # ) # )] ) core.CfnOutput(self, "Bastion Host Public IP", value=host.instance_public_ip) self.vpc = vpc
def __init__(self, scope: core.Construct, id: str, vpc: ec2.Vpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) # AMI amzn_linux = ec2.MachineImage.latest_amazon_linux( generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2, edition=ec2.AmazonLinuxEdition.STANDARD, virtualization=ec2.AmazonLinuxVirt.HVM, storage=ec2.AmazonLinuxStorage.GENERAL_PURPOSE ) i = 1 for subnet in vpc.stack.vpc.select_subnets(subnet_group_name="BASTION").subnets: bastion_host = ec2.BastionHostLinux(self, f"ec2-BASTION-Instance{i}", vpc=vpc, subnet_selection=ec2.SubnetSelection( availability_zones=[subnet.availability_zone], subnet_group_name="BASTION" ), instance_type=ec2.InstanceType("t1.micro"), machine_image=amzn_linux ) bastion_host.allow_ssh_access_from(ec2.Peer.any_ipv4()) i += 1 host_admin_group = iam.Group(self, "HostAdmins", ) policy = iam.Policy(self, "HostAdminPolicy", groups=[host_admin_group] ) policy.add_statements(iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=["ec2-instance-connect:SendSSHPublicKey"], resources= [f"arn:aws:ec2:{self.region}:{self.account}:instance/*"], conditions={"StringEquals": {"ec2:osuser": "******"}} ))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here private_subnet1 = ec2.SubnetConfiguration( cidr_mask=24, name='PrivateSubnet1', subnet_type=ec2.SubnetType.PRIVATE) private_subnet2 = ec2.SubnetConfiguration( cidr_mask=24, name='PrivateSubnet2', subnet_type=ec2.SubnetType.PRIVATE) public_subnet1 = ec2.SubnetConfiguration( cidr_mask=25, name='PublicSubnet1', subnet_type=ec2.SubnetType.PUBLIC) public_subnet2 = ec2.SubnetConfiguration( cidr_mask=25, name='PublicSubnet2', subnet_type=ec2.SubnetType.PUBLIC) vpc = ec2.Vpc(self, 'TestVPC', cidr='10.100.0.0/16', subnet_configuration=[ private_subnet1, public_subnet1, private_subnet2, public_subnet2 ]) bastion_ssh_sg = ec2.SecurityGroup(self, 'SSHBastion_SecurityGroup', vpc=vpc, security_group_name='Bastion_SSH') bastion_host = ec2.BastionHostLinux(self, 'Bastion-1', vpc=vpc)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here vpc = aws_ec2.Vpc( self, "AnalyticsWorkshopVPC", max_azs=2, gateway_endpoints={ "S3": aws_ec2.GatewayVpcEndpointOptions( service=aws_ec2.GatewayVpcEndpointAwsService.S3) }) sg_bastion_host = aws_ec2.SecurityGroup( self, "BastionHostSG", vpc=vpc, allow_all_outbound=True, description='security group for an bastion host', security_group_name='bastion-host-sg') core.Tag.add(sg_bastion_host, 'Name', 'bastion-host-sg') #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/InstanceClass.html #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/InstanceSize.html#aws_cdk.aws_ec2.InstanceSize ec2_instance_type = aws_ec2.InstanceType.of( aws_ec2.InstanceClass.BURSTABLE3, aws_ec2.InstanceSize.MEDIUM) #XXX: As there are no SSH public keys deployed on this machine, # you need to use EC2 Instance Connect with the command # 'aws ec2-instance-connect send-ssh-public-key' to provide your SSH public key. # https://aws.amazon.com/de/blogs/compute/new-using-amazon-ec2-instance-connect-for-ssh-access-to-your-ec2-instances/ bastion_host = aws_ec2.BastionHostLinux( self, "BastionHost", vpc=vpc, instance_type=ec2_instance_type, subnet_selection=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC), security_group=sg_bastion_host) #TODO: SHOULD restrict IP range allowed to ssh acces bastion_host.allow_ssh_access_from(aws_ec2.Peer.ipv4("0.0.0.0/0")) #XXX: In order to test data pipeline, add {Kinesis, KinesisFirehose}FullAccess Policy to the bastion host. bastion_host.role.add_to_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=["kinesis:*"])) bastion_host.role.add_to_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=["firehose:*"])) sg_use_es = aws_ec2.SecurityGroup( self, "ElasticSearchClientSG", vpc=vpc, allow_all_outbound=True, description='security group for an elasticsearch client', security_group_name='use-es-cluster-sg') core.Tag.add(sg_use_es, 'Name', 'use-es-cluster-sg') sg_es = aws_ec2.SecurityGroup( self, "ElasticSearchSG", vpc=vpc, allow_all_outbound=True, description='security group for an elasticsearch cluster', security_group_name='es-cluster-sg') core.Tag.add(sg_es, 'Name', 'es-cluster-sg') sg_es.add_ingress_rule(peer=sg_es, connection=aws_ec2.Port.all_tcp(), description='es-cluster-sg') sg_es.add_ingress_rule(peer=sg_use_es, connection=aws_ec2.Port.all_tcp(), description='use-es-cluster-sg') sg_es.add_ingress_rule(peer=sg_bastion_host, connection=aws_ec2.Port.all_tcp(), description='bastion-host-sg') s3_bucket = s3.Bucket( self, "s3bucket", bucket_name="aws-analytics-immersion-day-{region}-{account}". format(region=kwargs['env'].region, account=kwargs['env'].account)) trans_kinesis_stream = kinesis.Stream( self, "AnalyticsWorkshopKinesisStreams", stream_name='retail-trans') firehose_role_policy_doc = aws_iam.PolicyDocument() firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ s3_bucket.bucket_arn, "{}/*".format( s3_bucket.bucket_arn) ], "actions": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject" ] })) firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=[ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions" ])) firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, resources=[trans_kinesis_stream.stream_arn], actions=[ "kinesis:DescribeStream", "kinesis:GetShardIterator", "kinesis:GetRecords" ])) firehose_log_group_name = "/aws/kinesisfirehose/retail-trans" firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, #XXX: The ARN will be formatted as follows: # arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name} resources=[ self.format_arn(service="logs", resource="log-group", resource_name="{}:log-stream:*".format( firehose_log_group_name), sep=":") ], actions=["logs:PutLogEvents"])) firehose_role = aws_iam.Role( self, "FirehoseDeliveryRole", role_name="FirehoseDeliveryRole", assumed_by=aws_iam.ServicePrincipal("firehose.amazonaws.com"), #XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221 inline_policies={"firehose_role_policy": firehose_role_policy_doc}) trans_to_s3_delivery_stream = aws_kinesisfirehose.CfnDeliveryStream( self, "KinesisFirehoseToS3", delivery_stream_name="retail-trans", delivery_stream_type="KinesisStreamAsSource", kinesis_stream_source_configuration={ "kinesisStreamArn": trans_kinesis_stream.stream_arn, "roleArn": firehose_role.role_arn }, extended_s3_destination_configuration={ "bucketArn": s3_bucket.bucket_arn, "bufferingHints": { "intervalInSeconds": 60, "sizeInMBs": 1 }, "cloudWatchLoggingOptions": { "enabled": True, "logGroupName": firehose_log_group_name, "logStreamName": "S3Delivery" }, "compressionFormat": "UNCOMPRESSED", # [GZIP | HADOOP_SNAPPY | Snappy | UNCOMPRESSED | ZIP] "prefix": "json-data/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", "errorOutputPrefix": "error-json/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/!{firehose:error-output-type}", "roleArn": firehose_role.role_arn }) #XXX: aws cdk elastsearch example - https://github.com/aws/aws-cdk/issues/2873 es_domain_name = 'retail' es_cfn_domain = aws_elasticsearch.CfnDomain( self, "ElasticSearch", elasticsearch_cluster_config={ "dedicatedMasterCount": 3, "dedicatedMasterEnabled": True, "dedicatedMasterType": "t2.medium.elasticsearch", "instanceCount": 2, "instanceType": "t2.medium.elasticsearch", "zoneAwarenessEnabled": True }, ebs_options={ "ebsEnabled": True, "volumeSize": 10, "volumeType": "gp2" }, domain_name=es_domain_name, elasticsearch_version="7.4", encryption_at_rest_options={"enabled": False}, access_policies={ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "AWS": "*" }, "Action": ["es:Describe*", "es:List*", "es:Get*", "es:ESHttp*"], "Resource": self.format_arn( service="es", resource="domain", resource_name="{}/*".format(es_domain_name)) }] }, snapshot_options={"automatedSnapshotStartHour": 17}, vpc_options={ "securityGroupIds": [sg_es.security_group_id], "subnetIds": vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids }) core.Tag.add(es_cfn_domain, 'Name', 'analytics-workshop-es') #XXX: https://github.com/aws/aws-cdk/issues/1342 s3_lib_bucket = s3.Bucket.from_bucket_name(self, id, S3_BUCKET_LAMBDA_LAYER_LIB) es_lib_layer = _lambda.LayerVersion( self, "ESLib", layer_version_name="es-lib", compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], code=_lambda.Code.from_bucket(s3_lib_bucket, "var/es-lib.zip")) #XXX: add more than 2 security groups # https://github.com/aws/aws-cdk/blob/ea10f0d141a48819ec0000cd7905feda993870a9/packages/%40aws-cdk/aws-lambda/lib/function.ts#L387 # https://github.com/aws/aws-cdk/issues/1555 # https://github.com/aws/aws-cdk/pull/5049 #XXX: Deploy lambda in VPC - https://github.com/aws/aws-cdk/issues/1342 upsert_to_es_lambda_fn = _lambda.Function( self, "UpsertToES", runtime=_lambda.Runtime.PYTHON_3_7, function_name="UpsertToES", handler="upsert_to_es.lambda_handler", description="Upsert records into elasticsearch", code=_lambda.Code.asset("./src/main/python/UpsertToES"), environment={ 'ES_HOST': es_cfn_domain.attr_domain_endpoint, #TODO: MUST set appropriate environment variables for your workloads. 'ES_INDEX': 'retail', 'ES_TYPE': 'trans', 'REQUIRED_FIELDS': 'Invoice,StockCode,Customer_ID', 'REGION_NAME': kwargs['env'].region, 'DATE_TYPE_FIELDS': 'InvoiceDate' }, timeout=core.Duration.minutes(5), layers=[es_lib_layer], security_groups=[sg_use_es], vpc=vpc) trans_kinesis_event_source = KinesisEventSource( trans_kinesis_stream, batch_size=1000, starting_position=_lambda.StartingPosition.LATEST) upsert_to_es_lambda_fn.add_event_source(trans_kinesis_event_source) log_group = aws_logs.LogGroup( self, "UpsertToESLogGroup", log_group_name="/aws/lambda/UpsertToES", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(upsert_to_es_lambda_fn) merge_small_files_lambda_fn = _lambda.Function( self, "MergeSmallFiles", runtime=_lambda.Runtime.PYTHON_3_7, function_name="MergeSmallFiles", handler="athena_ctas.lambda_handler", description="Merge small files in S3", code=_lambda.Code.asset("./src/main/python/MergeSmallFiles"), environment={ #TODO: MUST set appropriate environment variables for your workloads. 'OLD_DATABASE': 'mydatabase', 'OLD_TABLE_NAME': 'retail_trans_json', 'NEW_DATABASE': 'mydatabase', 'NEW_TABLE_NAME': 'ctas_retail_trans_parquet', 'WORK_GROUP': 'primary', 'OUTPUT_PREFIX': 's3://{}'.format( os.path.join(s3_bucket.bucket_name, 'parquet-retail-trans')), 'STAGING_OUTPUT_PREFIX': 's3://{}'.format(os.path.join(s3_bucket.bucket_name, 'tmp')), 'COLUMN_NAMES': 'invoice,stockcode,description,quantity,invoicedate,price,customer_id,country', }, timeout=core.Duration.minutes(5)) merge_small_files_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=["athena:*"])) merge_small_files_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=[ "s3:Get*", "s3:List*", "s3:AbortMultipartUpload", "s3:PutObject", ])) merge_small_files_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, resources=["*"], actions=[ "glue:CreateDatabase", "glue:DeleteDatabase", "glue:GetDatabase", "glue:GetDatabases", "glue:UpdateDatabase", "glue:CreateTable", "glue:DeleteTable", "glue:BatchDeleteTable", "glue:UpdateTable", "glue:GetTable", "glue:GetTables", "glue:BatchCreatePartition", "glue:CreatePartition", "glue:DeletePartition", "glue:BatchDeletePartition", "glue:UpdatePartition", "glue:GetPartition", "glue:GetPartitions", "glue:BatchGetPartition" ])) merge_small_files_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=["lakeformation:GetDataAccess"])) lambda_fn_target = aws_events_targets.LambdaFunction( merge_small_files_lambda_fn) aws_events.Rule(self, "ScheduleRule", schedule=aws_events.Schedule.cron(minute="5"), targets=[lambda_fn_target]) log_group = aws_logs.LogGroup( self, "MergeSmallFilesLogGroup", log_group_name="/aws/lambda/MergeSmallFiles", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(merge_small_files_lambda_fn)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ############################################# ### Setup a basic vpc and security groups ### ############################################# # Create a simple vpc vpc = ec2.Vpc(self, "MyVPC", max_azs=3) # My existing ssh key pair name keypair = 'tom' # Dynamically pull ubuntu ami id - needs environment var set CDK_DEFAULT_REGION dynamic_ubuntu_ami = ec2.MachineImage.lookup( name="*ubuntu-bionic-18.04-amd64-server*", owners=["099720109477"]) # Security group for our test instance my_sg = ec2.SecurityGroup(self, "my_sg", vpc=vpc, description="My sg for testing", allow_all_outbound=True) # Add ssh from anywhere my_sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(22), "Allow ssh access from anywhere") ################################################################# # Single Ubuntu EC2 instance in Private Subnet in an ASG of 1:1 # ################################################################# asg = autoscaling.AutoScalingGroup( self, "Ubuntu-ASG-Instance", vpc=vpc, instance_type=ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE2, ec2.InstanceSize.MICRO), machine_image=dynamic_ubuntu_ami, key_name=keypair, ) asg.add_security_group(my_sg) # add our security group, expects object ######################################################## # Single Ubuntu EC2 instance in Private Subnet, no ASG # ######################################################## ubuntu_ami = dynamic_ubuntu_ami.get_image( self ).image_id # CfnInstances requires image_id to be the ami string instance = ec2.CfnInstance( self, "Ubuntu-Instance", image_id=ubuntu_ami, instance_type='m4.large', monitoring=True, key_name=keypair, network_interfaces=[{ "deviceIndex": "0", "associatePublicIpAddress": False, "subnetId": vpc.private_subnets[0].subnet_id, "groupSet": [my_sg.security_group_id] }], ) ################################################### # Bastion host to access Ubuntu hosts for testing # ################################################### host = ec2.BastionHostLinux( self, "BastionHost", vpc=vpc, subnet_selection=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PUBLIC), ) host.allow_ssh_access_from( ec2.Peer.ipv4("0.0.0.0/0")) # Restrict this to your IP host.instance.instance.add_property_override( "KeyName", keypair) # Add keypair for access unless you use SSM
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.current_dir = os.path.dirname(__file__) self.vpc = ec2.Vpc( self, "VPC", cidr="10.0.0.0/21", max_azs=2, subnet_configuration=[ ec2.SubnetConfiguration( cidr_mask=28, name="Database", subnet_type=ec2.SubnetType.ISOLATED, ), ec2.SubnetConfiguration(cidr_mask=28, name="Private", subnet_type=ec2.SubnetType.PRIVATE), ec2.SubnetConfiguration(cidr_mask=28, name="Public", subnet_type=ec2.SubnetType.PUBLIC) ], nat_gateways=3) self.qs_security_group = ec2.SecurityGroup( self, "quicksight-sg", vpc=self.vpc, allow_all_outbound=True, description="QuickSight security group") self.bastion = ec2.BastionHostLinux( self, "BastionHost", vpc=self.vpc, subnet_selection=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PUBLIC)) self.bastion.connections.allow_from_any_ipv4(ec2.Port.tcp(22), "Internet access SSH") self.vpc.add_interface_endpoint( "redshift_endpoint", service=ec2.InterfaceVpcEndpointAwsService("redshift")) self.vpc.add_interface_endpoint( "rds_endpoint", service=ec2.InterfaceVpcEndpointAwsService("rds")) self.redshift_secret = secrets.Secret( self, 'redshift-admin', secret_name='redshift-admin', description= "This secret has generated admin secret password for Redshift cluster", generate_secret_string=secrets.SecretStringGenerator( secret_string_template='{"username": "******"}', generate_string_key='password', password_length=32, exclude_characters='"@\\\/', exclude_punctuation=True)) self.rs_security_group = ec2.SecurityGroup(self, "redshift-sg", vpc=self.vpc, allow_all_outbound=True, description="Redshift SG") self.rs_security_group.add_ingress_rule(self.rs_security_group, ec2.Port.all_tcp(), 'Redshift-basic') self.rs_security_group.add_ingress_rule( # https://docs.aws.amazon.com/quicksight/latest/user/regions.html ec2.Peer.ipv4('52.23.63.224/27'), ec2.Port.tcp(5439), 'QuickSight-IP') self.rs_security_group.add_ingress_rule(self.qs_security_group, ec2.Port.tcp(5439), 'QuickSight-sg') # self.rs_security_group.add_egress_rule( # self.rs_security_group, # ec2.Port.all_tcp(), # 'Allow outbound for QuickSight' # ) self.redshift_cluster = redshift.Cluster( self, "datasource-redshift", master_user=redshift.Login( master_username="******", master_password=self.redshift_secret.secret_value_from_json( 'password')), vpc=self.vpc, vpc_subnets=ec2.SubnetSelection( subnet_type=ec2.SubnetType.ISOLATED), security_groups=[self.rs_security_group]) self.rds_secret = secrets.Secret( self, 'rds-admin', secret_name='rds-admin', description= "This secret has generated admin secret password for RDS cluster", generate_secret_string=secrets.SecretStringGenerator( secret_string_template='{"username": "******"}', generate_string_key='password', password_length=32, exclude_characters='"@\\\/', exclude_punctuation=True)) self.rds_cluster = rds.DatabaseCluster( self, "datasource-rds", engine=rds.DatabaseClusterEngine.aurora_postgres( version=rds.AuroraPostgresEngineVersion.VER_11_9), instance_props={ "vpc_subnets": { "subnet_type": ec2.SubnetType.ISOLATED }, "vpc": self.vpc }, credentials=rds.Credentials.from_secret(self.rds_secret)) self.rds_cluster.connections.allow_default_port_from( self.bastion, "EC2 Bastion access Aurora") self.rds_cluster.connections.allow_default_port_from( self.qs_security_group, "QuickSight-sg") self.rds_cluster.connections.allow_default_port_from( # https://docs.aws.amazon.com/quicksight/latest/user/regions.html ec2.Peer.ipv4('52.23.63.224/27'), "QuickSight-IP") self.qs_security_group.add_ingress_rule(self.rs_security_group, ec2.Port.all_tcp(), 'AllTCP') for rds_group in self.rds_cluster.connections.security_groups: self.qs_security_group.add_ingress_rule(rds_group, ec2.Port.all_tcp(), 'AllTCP') # self.qs_security_group.add_egress_rule( # self.rs_security_group, # ec2.Port.all_tcp(), # 'AllTCP' # ) core.CfnOutput(self, "vpcId", value=self.vpc.vpc_id) core.CfnOutput(self, "redshiftUsername", value="admin") core.CfnOutput(self, "redshiftPassword", value="redshift-admin") core.CfnOutput(self, "redshiftClusterId", value=self.redshift_cluster.cluster_name) core.CfnOutput(self, "redshiftHost", value=self.redshift_cluster.cluster_endpoint.hostname) core.CfnOutput(self, "redshiftDB", value="dev") core.CfnOutput(self, "rdsUsername", value="administrator") core.CfnOutput(self, "rdsPassword", value="rds-admin") core.CfnOutput(self, "rdsClusterId", value=self.rds_cluster.cluster_identifier) core.CfnOutput(self, "namespace", value="default") core.CfnOutput(self, "version", value="1") core.CfnOutput(self, "quicksightSecurityGroupId", value=self.qs_security_group.security_group_id)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) vpc = aws_ec2.Vpc( self, "OctemberVPC", max_azs=2, # subnet_configuration=[{ # "cidrMask": 24, # "name": "Public", # "subnetType": aws_ec2.SubnetType.PUBLIC, # }, # { # "cidrMask": 24, # "name": "Private", # "subnetType": aws_ec2.SubnetType.PRIVATE # }, # { # "cidrMask": 28, # "name": "Isolated", # "subnetType": aws_ec2.SubnetType.ISOLATED, # "reserved": True # } # ], gateway_endpoints={ "S3": aws_ec2.GatewayVpcEndpointOptions( service=aws_ec2.GatewayVpcEndpointAwsService.S3) }) dynamo_db_endpoint = vpc.add_gateway_endpoint( "DynamoDbEndpoint", service=aws_ec2.GatewayVpcEndpointAwsService.DYNAMODB) s3_bucket = s3.Bucket( self, "s3bucket", bucket_name="octember-bizcard-{region}-{account}".format( region=core.Aws.REGION, account=core.Aws.ACCOUNT_ID)) api = apigw.RestApi( self, "BizcardImageUploader", rest_api_name="BizcardImageUploader", description="This service serves uploading bizcard images into s3.", endpoint_types=[apigw.EndpointType.REGIONAL], binary_media_types=["image/png", "image/jpg"], deploy=True, deploy_options=apigw.StageOptions(stage_name="v1")) rest_api_role = aws_iam.Role( self, "ApiGatewayRoleForS3", role_name="ApiGatewayRoleForS3FullAccess", assumed_by=aws_iam.ServicePrincipal("apigateway.amazonaws.com"), managed_policies=[ aws_iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonS3FullAccess") ]) list_objects_responses = [ apigw.IntegrationResponse( status_code="200", #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationResponse.html#aws_cdk.aws_apigateway.IntegrationResponse.response_parameters # The response parameters from the backend response that API Gateway sends to the method response. # Use the destination as the key and the source as the value: # - The destination must be an existing response parameter in the MethodResponse property. # - The source must be an existing method request parameter or a static value. response_parameters={ 'method.response.header.Timestamp': 'integration.response.header.Date', 'method.response.header.Content-Length': 'integration.response.header.Content-Length', 'method.response.header.Content-Type': 'integration.response.header.Content-Type' }), apigw.IntegrationResponse(status_code="400", selection_pattern="4\d{2}"), apigw.IntegrationResponse(status_code="500", selection_pattern="5\d{2}") ] list_objects_integration_options = apigw.IntegrationOptions( credentials_role=rest_api_role, integration_responses=list_objects_responses) get_s3_integration = apigw.AwsIntegration( service="s3", integration_http_method="GET", path='/', options=list_objects_integration_options) api.root.add_method( "GET", get_s3_integration, authorization_type=apigw.AuthorizationType.IAM, api_key_required=False, method_responses=[ apigw.MethodResponse( status_code="200", response_parameters={ 'method.response.header.Timestamp': False, 'method.response.header.Content-Length': False, 'method.response.header.Content-Type': False }, response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ], request_parameters={'method.request.header.Content-Type': False}) get_s3_folder_integration_options = apigw.IntegrationOptions( credentials_role=rest_api_role, integration_responses=list_objects_responses, #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationOptions.html#aws_cdk.aws_apigateway.IntegrationOptions.request_parameters # Specify request parameters as key-value pairs (string-to-string mappings), with a destination as the key and a source as the value. # The source must be an existing method request parameter or a static value. request_parameters={ "integration.request.path.bucket": "method.request.path.folder" }) get_s3_folder_integration = apigw.AwsIntegration( service="s3", integration_http_method="GET", path="{bucket}", options=get_s3_folder_integration_options) s3_folder = api.root.add_resource('{folder}') s3_folder.add_method( "GET", get_s3_folder_integration, authorization_type=apigw.AuthorizationType.IAM, api_key_required=False, method_responses=[ apigw.MethodResponse( status_code="200", response_parameters={ 'method.response.header.Timestamp': False, 'method.response.header.Content-Length': False, 'method.response.header.Content-Type': False }, response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ], request_parameters={ 'method.request.header.Content-Type': False, 'method.request.path.folder': True }) get_s3_item_integration_options = apigw.IntegrationOptions( credentials_role=rest_api_role, integration_responses=list_objects_responses, request_parameters={ "integration.request.path.bucket": "method.request.path.folder", "integration.request.path.object": "method.request.path.item" }) get_s3_item_integration = apigw.AwsIntegration( service="s3", integration_http_method="GET", path="{bucket}/{object}", options=get_s3_item_integration_options) s3_item = s3_folder.add_resource('{item}') s3_item.add_method( "GET", get_s3_item_integration, authorization_type=apigw.AuthorizationType.IAM, api_key_required=False, method_responses=[ apigw.MethodResponse( status_code="200", response_parameters={ 'method.response.header.Timestamp': False, 'method.response.header.Content-Length': False, 'method.response.header.Content-Type': False }, response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ], request_parameters={ 'method.request.header.Content-Type': False, 'method.request.path.folder': True, 'method.request.path.item': True }) put_s3_item_integration_options = apigw.IntegrationOptions( credentials_role=rest_api_role, integration_responses=[ apigw.IntegrationResponse(status_code="200"), apigw.IntegrationResponse(status_code="400", selection_pattern="4\d{2}"), apigw.IntegrationResponse(status_code="500", selection_pattern="5\d{2}") ], request_parameters={ "integration.request.header.Content-Type": "method.request.header.Content-Type", "integration.request.path.bucket": "method.request.path.folder", "integration.request.path.object": "method.request.path.item" }) put_s3_item_integration = apigw.AwsIntegration( service="s3", integration_http_method="PUT", path="{bucket}/{object}", options=put_s3_item_integration_options) s3_item.add_method( "PUT", put_s3_item_integration, authorization_type=apigw.AuthorizationType.IAM, api_key_required=False, method_responses=[ apigw.MethodResponse( status_code="200", response_parameters={ 'method.response.header.Content-Type': False }, response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ], request_parameters={ 'method.request.header.Content-Type': False, 'method.request.path.folder': True, 'method.request.path.item': True }) ddb_table = dynamodb.Table( self, "BizcardImageMetaInfoDdbTable", table_name="OctemberBizcardImgMeta", partition_key=dynamodb.Attribute( name="image_id", type=dynamodb.AttributeType.STRING), billing_mode=dynamodb.BillingMode.PROVISIONED, read_capacity=15, write_capacity=5) img_kinesis_stream = kinesis.Stream( self, "BizcardImagePath", stream_name="octember-bizcard-image") # create lambda function trigger_textract_lambda_fn = _lambda.Function( self, "TriggerTextExtractorFromImage", runtime=_lambda.Runtime.PYTHON_3_7, function_name="TriggerTextExtractorFromImage", handler="trigger_text_extract_from_s3_image.lambda_handler", description="Trigger to extract text from an image in S3", code=_lambda.Code.asset( "./src/main/python/TriggerTextExtractFromS3Image"), environment={ 'REGION_NAME': core.Aws.REGION, 'DDB_TABLE_NAME': ddb_table.table_name, 'KINESIS_STREAM_NAME': img_kinesis_stream.stream_name }, timeout=core.Duration.minutes(5)) ddb_table_rw_policy_statement = aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, resources=[ddb_table.table_arn], actions=[ "dynamodb:BatchGetItem", "dynamodb:Describe*", "dynamodb:List*", "dynamodb:GetItem", "dynamodb:Query", "dynamodb:Scan", "dynamodb:BatchWriteItem", "dynamodb:DeleteItem", "dynamodb:PutItem", "dynamodb:UpdateItem", "dax:Describe*", "dax:List*", "dax:GetItem", "dax:BatchGetItem", "dax:Query", "dax:Scan", "dax:BatchWriteItem", "dax:DeleteItem", "dax:PutItem", "dax:UpdateItem" ]) trigger_textract_lambda_fn.add_to_role_policy( ddb_table_rw_policy_statement) trigger_textract_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=[img_kinesis_stream.stream_arn], actions=[ "kinesis:Get*", "kinesis:List*", "kinesis:Describe*", "kinesis:PutRecord", "kinesis:PutRecords" ])) # assign notification for the s3 event type (ex: OBJECT_CREATED) s3_event_filter = s3.NotificationKeyFilter(prefix="bizcard-raw-img/", suffix=".jpg") s3_event_source = S3EventSource(s3_bucket, events=[s3.EventType.OBJECT_CREATED], filters=[s3_event_filter]) trigger_textract_lambda_fn.add_event_source(s3_event_source) #XXX: https://github.com/aws/aws-cdk/issues/2240 # To avoid to create extra Lambda Functions with names like LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a # if log_retention=aws_logs.RetentionDays.THREE_DAYS is added to the constructor props log_group = aws_logs.LogGroup( self, "TriggerTextractLogGroup", log_group_name="/aws/lambda/TriggerTextExtractorFromImage", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(trigger_textract_lambda_fn) text_kinesis_stream = kinesis.Stream( self, "BizcardTextData", stream_name="octember-bizcard-txt") textract_lambda_fn = _lambda.Function( self, "GetTextFromImage", runtime=_lambda.Runtime.PYTHON_3_7, function_name="GetTextFromImage", handler="get_text_from_s3_image.lambda_handler", description="extract text from an image in S3", code=_lambda.Code.asset("./src/main/python/GetTextFromS3Image"), environment={ 'REGION_NAME': core.Aws.REGION, 'DDB_TABLE_NAME': ddb_table.table_name, 'KINESIS_STREAM_NAME': text_kinesis_stream.stream_name }, timeout=core.Duration.minutes(5)) textract_lambda_fn.add_to_role_policy(ddb_table_rw_policy_statement) textract_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=[text_kinesis_stream.stream_arn], actions=[ "kinesis:Get*", "kinesis:List*", "kinesis:Describe*", "kinesis:PutRecord", "kinesis:PutRecords" ])) textract_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ s3_bucket.bucket_arn, "{}/*".format( s3_bucket.bucket_arn) ], "actions": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject" ] })) textract_lambda_fn.add_to_role_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=["textract:*"])) img_kinesis_event_source = KinesisEventSource( img_kinesis_stream, batch_size=100, starting_position=_lambda.StartingPosition.LATEST) textract_lambda_fn.add_event_source(img_kinesis_event_source) log_group = aws_logs.LogGroup( self, "GetTextFromImageLogGroup", log_group_name="/aws/lambda/GetTextFromImage", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(textract_lambda_fn) sg_use_bizcard_es = aws_ec2.SecurityGroup( self, "BizcardSearchClientSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard elasticsearch client', security_group_name='use-octember-bizcard-es') core.Tags.of(sg_use_bizcard_es).add('Name', 'use-octember-bizcard-es') sg_bizcard_es = aws_ec2.SecurityGroup( self, "BizcardSearchSG", vpc=vpc, allow_all_outbound=True, description='security group for octember bizcard elasticsearch', security_group_name='octember-bizcard-es') core.Tags.of(sg_bizcard_es).add('Name', 'octember-bizcard-es') sg_bizcard_es.add_ingress_rule(peer=sg_bizcard_es, connection=aws_ec2.Port.all_tcp(), description='octember-bizcard-es') sg_bizcard_es.add_ingress_rule(peer=sg_use_bizcard_es, connection=aws_ec2.Port.all_tcp(), description='use-octember-bizcard-es') sg_ssh_access = aws_ec2.SecurityGroup( self, "BastionHostSG", vpc=vpc, allow_all_outbound=True, description='security group for bastion host', security_group_name='octember-bastion-host-sg') core.Tags.of(sg_ssh_access).add('Name', 'octember-bastion-host') sg_ssh_access.add_ingress_rule(peer=aws_ec2.Peer.any_ipv4(), connection=aws_ec2.Port.tcp(22), description='ssh access') bastion_host = aws_ec2.BastionHostLinux( self, "BastionHost", vpc=vpc, instance_type=aws_ec2.InstanceType('t3.nano'), security_group=sg_ssh_access, subnet_selection=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC)) bastion_host.instance.add_security_group(sg_use_bizcard_es) #XXX: aws cdk elastsearch example - https://github.com/aws/aws-cdk/issues/2873 es_cfn_domain = aws_elasticsearch.CfnDomain( self, 'BizcardSearch', elasticsearch_cluster_config={ "dedicatedMasterCount": 3, "dedicatedMasterEnabled": True, "dedicatedMasterType": "t2.medium.elasticsearch", "instanceCount": 2, "instanceType": "t2.medium.elasticsearch", "zoneAwarenessEnabled": True }, ebs_options={ "ebsEnabled": True, "volumeSize": 10, "volumeType": "gp2" }, domain_name="octember-bizcard", elasticsearch_version="7.9", encryption_at_rest_options={"enabled": False}, access_policies={ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "AWS": "*" }, "Action": ["es:Describe*", "es:List*", "es:Get*", "es:ESHttp*"], "Resource": self.format_arn(service="es", resource="domain", resource_name="octember-bizcard/*") }] }, snapshot_options={"automatedSnapshotStartHour": 17}, vpc_options={ "securityGroupIds": [sg_bizcard_es.security_group_id], "subnetIds": vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids }) core.Tags.of(es_cfn_domain).add('Name', 'octember-bizcard-es') s3_lib_bucket_name = self.node.try_get_context("lib_bucket_name") #XXX: https://github.com/aws/aws-cdk/issues/1342 s3_lib_bucket = s3.Bucket.from_bucket_name(self, id, s3_lib_bucket_name) es_lib_layer = _lambda.LayerVersion( self, "ESLib", layer_version_name="es-lib", compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], code=_lambda.Code.from_bucket(s3_lib_bucket, "var/octember-es-lib.zip")) redis_lib_layer = _lambda.LayerVersion( self, "RedisLib", layer_version_name="redis-lib", compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], code=_lambda.Code.from_bucket(s3_lib_bucket, "var/octember-redis-lib.zip")) #XXX: Deploy lambda in VPC - https://github.com/aws/aws-cdk/issues/1342 upsert_to_es_lambda_fn = _lambda.Function( self, "UpsertBizcardToES", runtime=_lambda.Runtime.PYTHON_3_7, function_name="UpsertBizcardToElasticSearch", handler="upsert_bizcard_to_es.lambda_handler", description="Upsert bizcard text into elasticsearch", code=_lambda.Code.asset("./src/main/python/UpsertBizcardToES"), environment={ 'ES_HOST': es_cfn_domain.attr_domain_endpoint, 'ES_INDEX': 'octember_bizcard', 'ES_TYPE': 'bizcard' }, timeout=core.Duration.minutes(5), layers=[es_lib_layer], security_groups=[sg_use_bizcard_es], vpc=vpc) text_kinesis_event_source = KinesisEventSource( text_kinesis_stream, batch_size=99, starting_position=_lambda.StartingPosition.LATEST) upsert_to_es_lambda_fn.add_event_source(text_kinesis_event_source) log_group = aws_logs.LogGroup( self, "UpsertBizcardToESLogGroup", log_group_name="/aws/lambda/UpsertBizcardToElasticSearch", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(upsert_to_es_lambda_fn) firehose_role_policy_doc = aws_iam.PolicyDocument() firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ s3_bucket.bucket_arn, "{}/*".format( s3_bucket.bucket_arn) ], "actions": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject" ] })) firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=[ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions" ])) firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=[text_kinesis_stream.stream_arn], actions=[ "kinesis:DescribeStream", "kinesis:GetShardIterator", "kinesis:GetRecords" ])) firehose_log_group_name = "/aws/kinesisfirehose/octember-bizcard-txt-to-s3" firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, #XXX: The ARN will be formatted as follows: # arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name} resources=[ self.format_arn(service="logs", resource="log-group", resource_name="{}:log-stream:*".format( firehose_log_group_name), sep=":") ], actions=["logs:PutLogEvents"])) firehose_role = aws_iam.Role( self, "FirehoseDeliveryRole", role_name="FirehoseDeliveryRole", assumed_by=aws_iam.ServicePrincipal("firehose.amazonaws.com"), #XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221 inline_policies={"firehose_role_policy": firehose_role_policy_doc}) bizcard_text_to_s3_delivery_stream = aws_kinesisfirehose.CfnDeliveryStream( self, "BizcardTextToS3", delivery_stream_name="octember-bizcard-txt-to-s3", delivery_stream_type="KinesisStreamAsSource", kinesis_stream_source_configuration={ "kinesisStreamArn": text_kinesis_stream.stream_arn, "roleArn": firehose_role.role_arn }, extended_s3_destination_configuration={ "bucketArn": s3_bucket.bucket_arn, "bufferingHints": { "intervalInSeconds": 60, "sizeInMBs": 1 }, "cloudWatchLoggingOptions": { "enabled": True, "logGroupName": firehose_log_group_name, "logStreamName": "S3Delivery" }, "compressionFormat": "GZIP", "prefix": "bizcard-text/", "roleArn": firehose_role.role_arn }) sg_use_bizcard_es_cache = aws_ec2.SecurityGroup( self, "BizcardSearchCacheClientSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard search query cache client', security_group_name='use-octember-bizcard-es-cache') core.Tags.of(sg_use_bizcard_es_cache).add( 'Name', 'use-octember-bizcard-es-cache') sg_bizcard_es_cache = aws_ec2.SecurityGroup( self, "BizcardSearchCacheSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard search query cache', security_group_name='octember-bizcard-es-cache') core.Tags.of(sg_bizcard_es_cache).add('Name', 'octember-bizcard-es-cache') sg_bizcard_es_cache.add_ingress_rule( peer=sg_use_bizcard_es_cache, connection=aws_ec2.Port.tcp(6379), description='use-octember-bizcard-es-cache') es_query_cache_subnet_group = aws_elasticache.CfnSubnetGroup( self, "QueryCacheSubnetGroup", description="subnet group for octember-bizcard-es-cache", subnet_ids=vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids, cache_subnet_group_name='octember-bizcard-es-cache') es_query_cache = aws_elasticache.CfnCacheCluster( self, "BizcardSearchQueryCache", cache_node_type="cache.t3.small", num_cache_nodes=1, engine="redis", engine_version="5.0.5", auto_minor_version_upgrade=False, cluster_name="octember-bizcard-es-cache", snapshot_retention_limit=3, snapshot_window="17:00-19:00", preferred_maintenance_window="mon:19:00-mon:20:30", #XXX: Do not use referece for "cache_subnet_group_name" - https://github.com/aws/aws-cdk/issues/3098 #cache_subnet_group_name=es_query_cache_subnet_group.cache_subnet_group_name, # Redis cluster goes to wrong VPC cache_subnet_group_name='octember-bizcard-es-cache', vpc_security_group_ids=[sg_bizcard_es_cache.security_group_id]) #XXX: If you're going to launch your cluster in an Amazon VPC, you need to create a subnet group before you start creating a cluster. # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticache-cache-cluster.html#cfn-elasticache-cachecluster-cachesubnetgroupname es_query_cache.add_depends_on(es_query_cache_subnet_group) #XXX: add more than 2 security groups # https://github.com/aws/aws-cdk/blob/ea10f0d141a48819ec0000cd7905feda993870a9/packages/%40aws-cdk/aws-lambda/lib/function.ts#L387 # https://github.com/aws/aws-cdk/issues/1555 # https://github.com/aws/aws-cdk/pull/5049 bizcard_search_lambda_fn = _lambda.Function( self, "BizcardSearchServer", runtime=_lambda.Runtime.PYTHON_3_7, function_name="BizcardSearchProxy", handler="es_search_bizcard.lambda_handler", description="Proxy server to search bizcard text", code=_lambda.Code.asset("./src/main/python/SearchBizcard"), environment={ 'ES_HOST': es_cfn_domain.attr_domain_endpoint, 'ES_INDEX': 'octember_bizcard', 'ES_TYPE': 'bizcard', 'ELASTICACHE_HOST': es_query_cache.attr_redis_endpoint_address }, timeout=core.Duration.minutes(1), layers=[es_lib_layer, redis_lib_layer], security_groups=[sg_use_bizcard_es, sg_use_bizcard_es_cache], vpc=vpc) #XXX: create API Gateway + LambdaProxy search_api = apigw.LambdaRestApi( self, "BizcardSearchAPI", handler=bizcard_search_lambda_fn, proxy=False, rest_api_name="BizcardSearch", description="This service serves searching bizcard text.", endpoint_types=[apigw.EndpointType.REGIONAL], deploy=True, deploy_options=apigw.StageOptions(stage_name="v1")) bizcard_search = search_api.root.add_resource('search') bizcard_search.add_method( "GET", method_responses=[ apigw.MethodResponse( status_code="200", response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ]) sg_use_bizcard_graph_db = aws_ec2.SecurityGroup( self, "BizcardGraphDbClientSG", vpc=vpc, allow_all_outbound=True, description='security group for octember bizcard graph db client', security_group_name='use-octember-bizcard-neptune') core.Tags.of(sg_use_bizcard_graph_db).add( 'Name', 'use-octember-bizcard-neptune') sg_bizcard_graph_db = aws_ec2.SecurityGroup( self, "BizcardGraphDbSG", vpc=vpc, allow_all_outbound=True, description='security group for octember bizcard graph db', security_group_name='octember-bizcard-neptune') core.Tags.of(sg_bizcard_graph_db).add('Name', 'octember-bizcard-neptune') sg_bizcard_graph_db.add_ingress_rule( peer=sg_bizcard_graph_db, connection=aws_ec2.Port.tcp(8182), description='octember-bizcard-neptune') sg_bizcard_graph_db.add_ingress_rule( peer=sg_use_bizcard_graph_db, connection=aws_ec2.Port.tcp(8182), description='use-octember-bizcard-neptune') bizcard_graph_db_subnet_group = aws_neptune.CfnDBSubnetGroup( self, "NeptuneSubnetGroup", db_subnet_group_description= "subnet group for octember-bizcard-neptune", subnet_ids=vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids, db_subnet_group_name='octember-bizcard-neptune') bizcard_graph_db = aws_neptune.CfnDBCluster( self, "BizcardGraphDB", availability_zones=vpc.availability_zones, db_subnet_group_name=bizcard_graph_db_subnet_group. db_subnet_group_name, db_cluster_identifier="octember-bizcard", backup_retention_period=1, preferred_backup_window="08:45-09:15", preferred_maintenance_window="sun:18:00-sun:18:30", vpc_security_group_ids=[sg_bizcard_graph_db.security_group_id]) bizcard_graph_db.add_depends_on(bizcard_graph_db_subnet_group) bizcard_graph_db_instance = aws_neptune.CfnDBInstance( self, "BizcardGraphDBInstance", db_instance_class="db.r5.large", allow_major_version_upgrade=False, auto_minor_version_upgrade=False, availability_zone=vpc.availability_zones[0], db_cluster_identifier=bizcard_graph_db.db_cluster_identifier, db_instance_identifier="octember-bizcard", preferred_maintenance_window="sun:18:00-sun:18:30") bizcard_graph_db_instance.add_depends_on(bizcard_graph_db) bizcard_graph_db_replica_instance = aws_neptune.CfnDBInstance( self, "BizcardGraphDBReplicaInstance", db_instance_class="db.r5.large", allow_major_version_upgrade=False, auto_minor_version_upgrade=False, availability_zone=vpc.availability_zones[-1], db_cluster_identifier=bizcard_graph_db.db_cluster_identifier, db_instance_identifier="octember-bizcard-replica", preferred_maintenance_window="sun:18:00-sun:18:30") bizcard_graph_db_replica_instance.add_depends_on(bizcard_graph_db) bizcard_graph_db_replica_instance.add_depends_on( bizcard_graph_db_instance) gremlinpython_lib_layer = _lambda.LayerVersion( self, "GremlinPythonLib", layer_version_name="gremlinpython-lib", compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], code=_lambda.Code.from_bucket( s3_lib_bucket, "var/octember-gremlinpython-lib.zip")) #XXX: https://github.com/aws/aws-cdk/issues/1342 upsert_to_neptune_lambda_fn = _lambda.Function( self, "UpsertBizcardToGraphDB", runtime=_lambda.Runtime.PYTHON_3_7, function_name="UpsertBizcardToNeptune", handler="upsert_bizcard_to_graph_db.lambda_handler", description="Upsert bizcard into neptune", code=_lambda.Code.asset( "./src/main/python/UpsertBizcardToGraphDB"), environment={ 'REGION_NAME': core.Aws.REGION, 'NEPTUNE_ENDPOINT': bizcard_graph_db.attr_endpoint, 'NEPTUNE_PORT': bizcard_graph_db.attr_port }, timeout=core.Duration.minutes(5), layers=[gremlinpython_lib_layer], security_groups=[sg_use_bizcard_graph_db], vpc=vpc) upsert_to_neptune_lambda_fn.add_event_source(text_kinesis_event_source) log_group = aws_logs.LogGroup( self, "UpsertBizcardToGraphDBLogGroup", log_group_name="/aws/lambda/UpsertBizcardToNeptune", retention=aws_logs.RetentionDays.THREE_DAYS) log_group.grant_write(upsert_to_neptune_lambda_fn) sg_use_bizcard_neptune_cache = aws_ec2.SecurityGroup( self, "BizcardNeptuneCacheClientSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard recommendation query cache client', security_group_name='use-octember-bizcard-neptune-cache') core.Tags.of(sg_use_bizcard_neptune_cache).add( 'Name', 'use-octember-bizcard-es-cache') sg_bizcard_neptune_cache = aws_ec2.SecurityGroup( self, "BizcardNeptuneCacheSG", vpc=vpc, allow_all_outbound=True, description= 'security group for octember bizcard recommendation query cache', security_group_name='octember-bizcard-neptune-cache') core.Tags.of(sg_bizcard_neptune_cache).add( 'Name', 'octember-bizcard-neptune-cache') sg_bizcard_neptune_cache.add_ingress_rule( peer=sg_use_bizcard_neptune_cache, connection=aws_ec2.Port.tcp(6379), description='use-octember-bizcard-neptune-cache') recomm_query_cache_subnet_group = aws_elasticache.CfnSubnetGroup( self, "RecommQueryCacheSubnetGroup", description="subnet group for octember-bizcard-neptune-cache", subnet_ids=vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids, cache_subnet_group_name='octember-bizcard-neptune-cache') recomm_query_cache = aws_elasticache.CfnCacheCluster( self, "BizcardRecommQueryCache", cache_node_type="cache.t3.small", num_cache_nodes=1, engine="redis", engine_version="5.0.5", auto_minor_version_upgrade=False, cluster_name="octember-bizcard-neptune-cache", snapshot_retention_limit=3, snapshot_window="17:00-19:00", preferred_maintenance_window="mon:19:00-mon:20:30", #XXX: Do not use referece for "cache_subnet_group_name" - https://github.com/aws/aws-cdk/issues/3098 #cache_subnet_group_name=recomm_query_cache_subnet_group.cache_subnet_group_name, # Redis cluster goes to wrong VPC cache_subnet_group_name='octember-bizcard-neptune-cache', vpc_security_group_ids=[ sg_bizcard_neptune_cache.security_group_id ]) recomm_query_cache.add_depends_on(recomm_query_cache_subnet_group) bizcard_recomm_lambda_fn = _lambda.Function( self, "BizcardRecommender", runtime=_lambda.Runtime.PYTHON_3_7, function_name="BizcardRecommender", handler="neptune_recommend_bizcard.lambda_handler", description="This service serves PYMK(People You May Know).", code=_lambda.Code.asset("./src/main/python/RecommendBizcard"), environment={ 'REGION_NAME': core.Aws.REGION, 'NEPTUNE_ENDPOINT': bizcard_graph_db.attr_read_endpoint, 'NEPTUNE_PORT': bizcard_graph_db.attr_port, 'ELASTICACHE_HOST': recomm_query_cache.attr_redis_endpoint_address }, timeout=core.Duration.minutes(1), layers=[gremlinpython_lib_layer, redis_lib_layer], security_groups=[ sg_use_bizcard_graph_db, sg_use_bizcard_neptune_cache ], vpc=vpc) #XXX: create API Gateway + LambdaProxy recomm_api = apigw.LambdaRestApi( self, "BizcardRecommendAPI", handler=bizcard_recomm_lambda_fn, proxy=False, rest_api_name="BizcardRecommend", description="This service serves PYMK(People You May Know).", endpoint_types=[apigw.EndpointType.REGIONAL], deploy=True, deploy_options=apigw.StageOptions(stage_name="v1")) bizcard_recomm = recomm_api.root.add_resource('pymk') bizcard_recomm.add_method( "GET", method_responses=[ apigw.MethodResponse( status_code="200", response_models={'application/json': apigw.EmptyModel()}), apigw.MethodResponse(status_code="400"), apigw.MethodResponse(status_code="500") ]) sagemaker_notebook_role_policy_doc = aws_iam.PolicyDocument() sagemaker_notebook_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ "arn:aws:s3:::aws-neptune-notebook", "arn:aws:s3:::aws-neptune-notebook/*" ], "actions": ["s3:GetObject", "s3:ListBucket"] })) sagemaker_notebook_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ "arn:aws:neptune-db:{region}:{account}:{cluster_id}/*". format(region=core.Aws.REGION, account=core.Aws.ACCOUNT_ID, cluster_id=bizcard_graph_db. attr_cluster_resource_id) ], "actions": ["neptune-db:connect"] })) sagemaker_notebook_role = aws_iam.Role( self, 'SageMakerNotebookForNeptuneWorkbenchRole', role_name='AWSNeptuneNotebookRole-OctemberBizcard', assumed_by=aws_iam.ServicePrincipal('sagemaker.amazonaws.com'), #XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221 inline_policies={ 'AWSNeptuneNotebook': sagemaker_notebook_role_policy_doc }) neptune_wb_lifecycle_content = '''#!/bin/bash sudo -u ec2-user -i <<'EOF' echo "export GRAPH_NOTEBOOK_AUTH_MODE=DEFAULT" >> ~/.bashrc echo "export GRAPH_NOTEBOOK_HOST={NeptuneClusterEndpoint}" >> ~/.bashrc echo "export GRAPH_NOTEBOOK_PORT={NeptuneClusterPort}" >> ~/.bashrc echo "export NEPTUNE_LOAD_FROM_S3_ROLE_ARN=''" >> ~/.bashrc echo "export AWS_REGION={AWS_Region}" >> ~/.bashrc aws s3 cp s3://aws-neptune-notebook/graph_notebook.tar.gz /tmp/graph_notebook.tar.gz rm -rf /tmp/graph_notebook tar -zxvf /tmp/graph_notebook.tar.gz -C /tmp /tmp/graph_notebook/install.sh EOF '''.format(NeptuneClusterEndpoint=bizcard_graph_db.attr_endpoint, NeptuneClusterPort=bizcard_graph_db.attr_port, AWS_Region=core.Aws.REGION) neptune_wb_lifecycle_config_prop = aws_sagemaker.CfnNotebookInstanceLifecycleConfig.NotebookInstanceLifecycleHookProperty( content=core.Fn.base64(neptune_wb_lifecycle_content)) neptune_wb_lifecycle_config = aws_sagemaker.CfnNotebookInstanceLifecycleConfig( self, 'NpetuneWorkbenchLifeCycleConfig', notebook_instance_lifecycle_config_name= 'AWSNeptuneWorkbenchOctemberBizcardLCConfig', on_start=[neptune_wb_lifecycle_config_prop]) neptune_workbench = aws_sagemaker.CfnNotebookInstance( self, 'NeptuneWorkbench', instance_type='ml.t2.medium', role_arn=sagemaker_notebook_role.role_arn, lifecycle_config_name=neptune_wb_lifecycle_config. notebook_instance_lifecycle_config_name, notebook_instance_name='OctemberBizcard-NeptuneWorkbench', root_access='Disabled', security_group_ids=[sg_use_bizcard_graph_db.security_group_name], subnet_id=bizcard_graph_db_subnet_group.subnet_ids[0])
def __init__(self, scope: core.Construct, construct_id: str, env, **kwargs) -> None: super().__init__(scope, construct_id, env=env, **kwargs) # The code that defines your stack goes here if self.node.try_get_context("tags"): self.user_defined_tags = self.node.try_get_context("tags").split(' ') else: self.user_defined_tags = None vpc = ec2.Vpc(self, "VPC_EMQ", max_azs=2, cidr="10.10.0.0/16", # configuration will create 3 groups in 2 AZs = 6 subnets. subnet_configuration=[ec2.SubnetConfiguration( subnet_type=ec2.SubnetType.PUBLIC, name="Public", cidr_mask=24 ), ec2.SubnetConfiguration( subnet_type=ec2.SubnetType.PRIVATE, name="Private", cidr_mask=24 ), ec2.SubnetConfiguration( subnet_type=ec2.SubnetType.ISOLATED, name="DB", cidr_mask=24 ) ], nat_gateways=2 ) self.vpc = vpc # Route53 int_zone = r53.PrivateHostedZone(self, r53_zone_name, zone_name = 'int.emqx', vpc = vpc ) self.int_zone = int_zone # Define cfn parameters # ec2_type = CfnParameter(self, "ec2-instance-type", # type="String", default="m5.2xlarge", # description="Specify the instance type you want").value_as_string key_name = CfnParameter(self, "ssh key", type="String", default="key_ireland", description="Specify your SSH key").value_as_string sg = ec2.SecurityGroup(self, id = 'sg_int', vpc = vpc) self.sg = sg sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(22), 'SSH frm anywhere') sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(1883), 'MQTT TCP Port') sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(8883), 'MQTT TCP/TLS Port') sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.udp(14567), 'MQTT Quic Port') sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(18083), 'WEB UI') sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(4369), 'EMQX dist port 1') sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(4370), 'EMQX dist port 2') sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(8081), 'EMQX dashboard') sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(2379), 'etcd client port') sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(2380), 'etcd peer port') # Create Bastion Server bastion = ec2.BastionHostLinux(self, "Bastion", vpc=vpc, subnet_selection=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), instance_name="BastionHostLinux", instance_type=ec2.InstanceType(instance_type_identifier="t3.nano")) bastion.instance.instance.add_property_override("KeyName", key_name) bastion.connections.allow_from_any_ipv4( ec2.Port.tcp(22), "Internet access SSH") # Create NLB nlb = elb.NetworkLoadBalancer(self, "emq-elb", vpc=vpc, internet_facing=False, cross_zone_enabled=True, load_balancer_name="emq-nlb") r53.ARecord(self, "AliasRecord", zone = int_zone, record_name = loadbalancer_dnsname, target = r53.RecordTarget.from_alias(r53_targets.LoadBalancerTarget(nlb)) ) self.nlb = nlb listener = nlb.add_listener("port1883", port=1883) listenerTLS = nlb.add_listener("port8883", port=8883) # TLS, emqx terminataion listenerQuic = nlb.add_listener("port14567", port=14567, protocol=elbv2.Protocol.UDP) listenerUI = nlb.add_listener("port80", port=80) # Create Autoscaling Group with desired 2*EC2 hosts # asg = autoscaling.AutoScalingGroup(self, "emq-asg", # vpc=vpc, # vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE), # instance_type=ec2.InstanceType( # instance_type_identifier=ec2_type), # machine_image=linux_ami, # security_group = sg, # key_name=key_name, # user_data=ec2.UserData.custom(user_data), # health_check=HealthCheck.elb(grace=Duration.seconds(60)), # desired_capacity=3, # min_capacity=2, # max_capacity=4 # ) # if self.user_defined_tags: # core.Tags.of(asg).add(*self.user_defined_tags) # # NLB cannot associate with a security group therefore NLB object has no Connection object # # Must modify manuall inbound rule of the newly created asg security group to allow access # # from NLB IP only # asg.connections.allow_from_any_ipv4( # ec2.Port.tcp(1883), "Allow NLB access 1883 port of EC2 in Autoscaling Group") # asg.connections.allow_from_any_ipv4( # ec2.Port.tcp(18083), "Allow NLB access WEB UI") # asg.connections.allow_from_any_ipv4( # ec2.Port.tcp(4369), "Allow emqx cluster distribution port 1") # asg.connections.allow_from_any_ipv4( # ec2.Port.tcp(4370), "Allow emqx cluster distribution port 2") # asg.connections.allow_from_any_ipv4( # ec2.Port.udp(4369), "Allow emqx cluster discovery port 1") # asg.connections.allow_from_any_ipv4( # ec2.Port.udp(4370), "Allow emqx cluster discovery port 2") # asg.connections.allow_from_any_ipv4( # ec2.Port.tcp(8081), "Allow emqx cluster dashboard access") # asg.connections.allow_from_any_ipv4( # ec2.Port.tcp(2379), "Allow emqx cluster discovery port (etcd)") # asg.connections.allow_from_any_ipv4( # ec2.Port.tcp(2380), "Allow emqx cluster discovery port (etcd)") # asg.connections.allow_from(bastion, # ec2.Port.tcp(22), "Allow SSH from the bastion only") self.setup_emqx(numEmqx, vpc, int_zone, sg, key_name) listener.add_targets('ec2', port=1883, targets= [ target.InstanceTarget(x) for x in self.emqx_vms]) # @todo we need ssl terminataion listenerUI.add_targets('ec2', port=18083, targets=[ target.InstanceTarget(x) for x in self.emqx_vms]) listenerQuic.add_targets('ec2', port=14567, protocol=elbv2.Protocol.UDP, targets=[ target.InstanceTarget(x) for x in self.emqx_vms]) listenerTLS.add_targets('ec2', port=8883, targets=[ target.InstanceTarget(x) for x in self.emqx_vms]) """ db_mysql = rds.DatabaseInstance(self, "EMQ_MySQL_DB", engine=rds.DatabaseInstanceEngine.mysql( version=rds.MysqlEngineVersion.VER_5_7_30), instance_type=ec2.InstanceType.of( ec2.InstanceClass.BURSTABLE2, ec2.InstanceSize.SMALL), vpc=vpc, multi_az=True, allocated_storage=100, storage_type=rds.StorageType.GP2, cloudwatch_logs_exports=["audit", "error", "general", "slowquery"], deletion_protection=False, delete_automated_backups=False, backup_retention=core.Duration.days(7), parameter_group=rds.ParameterGroup.from_parameter_group_name( self, "para-group-mysql", parameter_group_name="default.mysql5.7"), ) asg_security_groups = asg.connections.security_groups for asg_sg in asg_security_groups: db_mysql.connections.allow_default_port_from(asg_sg, "EC2 Autoscaling Group access MySQL") """ #self.setup_monitoring() self.setup_etcd(vpc, int_zone, sg, key_name) self.setup_loadgen(numLg, vpc, int_zone, sg, key_name, nlb.load_balancer_dns_name) self.setup_monitoring() core.CfnOutput(self, "Output", value=nlb.load_balancer_dns_name) core.CfnOutput(self, "SSH Entrypoint", value=bastion.instance_public_ip) core.CfnOutput(self, "SSH cmds", value="ssh -A -l ec2-user %s -L8888:%s:80 -L 9999:%s:80 -L 13000:%s:3000" % (bastion.instance_public_ip, nlb.load_balancer_dns_name, self.mon_lb, self.mon_lb) )
def __init__(self, scope: core.Construct, construct_id: str, env, **kwargs) -> None: super().__init__(scope, construct_id, env=env, **kwargs) # The code that defines your stack goes here vpc = ec2.Vpc( self, "VPC_EMQ", max_azs=2, cidr="10.10.0.0/16", # configuration will create 3 groups in 2 AZs = 6 subnets. subnet_configuration=[ ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.PUBLIC, name="Public", cidr_mask=24), ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.PRIVATE, name="Private", cidr_mask=24), ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.ISOLATED, name="DB", cidr_mask=24) ], nat_gateways=2) # Define cfn parameters ec2_type = CfnParameter( self, "ec2-instance-type", type="String", default="t2.micro", description="Specify the instance type you want").value_as_string key_name = CfnParameter( self, "ssh key", type="String", default="key_ireland", description="Specify your SSH key").value_as_string # Create Bastion Server bastion = ec2.BastionHostLinux( self, "Bastion", vpc=vpc, subnet_selection=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PUBLIC), instance_name="BastionHostLinux", instance_type=ec2.InstanceType( instance_type_identifier="t2.micro")) bastion.instance.instance.add_property_override("KeyName", key_name) bastion.connections.allow_from_any_ipv4(ec2.Port.tcp(22), "Internet access SSH") # Create NLB nlb = elb.NetworkLoadBalancer(self, "emq-elb", vpc=vpc, internet_facing=True, cross_zone_enabled=True, load_balancer_name="emq-nlb") listener = nlb.add_listener("port1883", port=1883) listenerUI = nlb.add_listener("port80", port=80) # Create Autoscaling Group with desired 2*EC2 hosts asg = autoscaling.AutoScalingGroup( self, "emq-asg", vpc=vpc, vpc_subnets=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PRIVATE), instance_type=ec2.InstanceType(instance_type_identifier=ec2_type), machine_image=linux_ami, key_name=key_name, user_data=ec2.UserData.custom(user_data), health_check=HealthCheck.elb(grace=Duration.seconds(60)), desired_capacity=2, min_capacity=2, max_capacity=4) user_defined_tags = self.node.try_get_context("tags") if user_defined_tags: tags = user_defined_tags.split(' ') core.Tags.of(asg).add(*tags) # NLB cannot associate with a security group therefore NLB object has no Connection object # Must modify manuall inbound rule of the newly created asg security group to allow access # from NLB IP only asg.connections.allow_from_any_ipv4( ec2.Port.tcp(1883), "Allow NLB access 1883 port of EC2 in Autoscaling Group") asg.connections.allow_from_any_ipv4(ec2.Port.tcp(18083), "Allow NLB access WEB UI") asg.connections.allow_from_any_ipv4( ec2.Port.tcp(4369), "Allow emqx cluster distribution port 1") asg.connections.allow_from_any_ipv4( ec2.Port.tcp(4370), "Allow emqx cluster distribution port 2") asg.connections.allow_from_any_ipv4( ec2.Port.udp(4369), "Allow emqx cluster discovery port 1") asg.connections.allow_from_any_ipv4( ec2.Port.udp(4370), "Allow emqx cluster discovery port 2") asg.connections.allow_from_any_ipv4( ec2.Port.tcp(2379), "Allow emqx cluster discovery port (etcd)") asg.connections.allow_from_any_ipv4( ec2.Port.tcp(2380), "Allow emqx cluster discovery port (etcd)") asg.connections.allow_from(bastion, ec2.Port.tcp(22), "Allow SSH from the bastion only") listener.add_targets("addTargetGroup", port=1883, targets=[asg]) # @todo we need ssl terminataion # listenerUI.add_targets("addTargetGroup", # port=18083, # targets=[asg]) """ db_mysql = rds.DatabaseInstance(self, "EMQ_MySQL_DB", engine=rds.DatabaseInstanceEngine.mysql( version=rds.MysqlEngineVersion.VER_5_7_30), instance_type=ec2.InstanceType.of( ec2.InstanceClass.BURSTABLE2, ec2.InstanceSize.SMALL), vpc=vpc, multi_az=True, allocated_storage=100, storage_type=rds.StorageType.GP2, cloudwatch_logs_exports=["audit", "error", "general", "slowquery"], deletion_protection=False, delete_automated_backups=False, backup_retention=core.Duration.days(7), parameter_group=rds.ParameterGroup.from_parameter_group_name( self, "para-group-mysql", parameter_group_name="default.mysql5.7"), ) asg_security_groups = asg.connections.security_groups for asg_sg in asg_security_groups: db_mysql.connections.allow_default_port_from(asg_sg, "EC2 Autoscaling Group access MySQL") """ core.CfnOutput(self, "Output", value=nlb.load_balancer_dns_name)
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # vpc_name = self.node.try_get_context("vpc_name") # vpc = aws_ec2.Vpc.from_lookup(self, "ExistingVPC", # is_default=True, # vpc_name=vpc_name) vpc = aws_ec2.Vpc( self, "EKKStackVPC", max_azs=2, gateway_endpoints={ "S3": aws_ec2.GatewayVpcEndpointOptions( service=aws_ec2.GatewayVpcEndpointAwsService.S3) }) #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/InstanceClass.html #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/InstanceSize.html#aws_cdk.aws_ec2.InstanceSize ec2_instance_type = aws_ec2.InstanceType.of( aws_ec2.InstanceClass.BURSTABLE3, aws_ec2.InstanceSize.MEDIUM) sg_bastion_host = aws_ec2.SecurityGroup( self, "BastionHostSG", vpc=vpc, allow_all_outbound=True, description='security group for an bastion host', security_group_name='bastion-host-sg') cdk.Tags.of(sg_bastion_host).add('Name', 'bastion-host-sg') #XXX: As there are no SSH public keys deployed on this machine, # you need to use EC2 Instance Connect with the command # 'aws ec2-instance-connect send-ssh-public-key' to provide your SSH public key. # https://aws.amazon.com/de/blogs/compute/new-using-amazon-ec2-instance-connect-for-ssh-access-to-your-ec2-instances/ bastion_host = aws_ec2.BastionHostLinux( self, "BastionHost", vpc=vpc, instance_type=ec2_instance_type, subnet_selection=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC), security_group=sg_bastion_host) #TODO: SHOULD restrict IP range allowed to ssh acces bastion_host.allow_ssh_access_from(aws_ec2.Peer.ipv4("0.0.0.0/0")) sg_use_es = aws_ec2.SecurityGroup( self, "ElasticSearchClientSG", vpc=vpc, allow_all_outbound=True, description='security group for an elasticsearch client', security_group_name='use-es-cluster-sg') cdk.Tags.of(sg_use_es).add('Name', 'use-es-cluster-sg') sg_es = aws_ec2.SecurityGroup( self, "ElasticSearchSG", vpc=vpc, allow_all_outbound=True, description='security group for an elasticsearch cluster', security_group_name='es-cluster-sg') cdk.Tags.of(sg_es).add('Name', 'es-cluster-sg') sg_es.add_ingress_rule(peer=sg_es, connection=aws_ec2.Port.all_tcp(), description='es-cluster-sg') sg_es.add_ingress_rule(peer=sg_use_es, connection=aws_ec2.Port.all_tcp(), description='use-es-cluster-sg') sg_es.add_ingress_rule(peer=sg_bastion_host, connection=aws_ec2.Port.all_tcp(), description='bastion-host-sg') #XXX: aws cdk elastsearch example - https://github.com/aws/aws-cdk/issues/2873 ES_DOMAIN_NAME = self.node.try_get_context("es_domain_name") es_cfn_domain = aws_elasticsearch.CfnDomain( self, "ElasticSearch", elasticsearch_cluster_config={ "dedicatedMasterCount": 3, "dedicatedMasterEnabled": True, "dedicatedMasterType": "t2.medium.elasticsearch", "instanceCount": 2, "instanceType": "t2.medium.elasticsearch", "zoneAwarenessEnabled": True }, ebs_options={ "ebsEnabled": True, "volumeSize": 10, "volumeType": "gp2" }, domain_name=ES_DOMAIN_NAME, elasticsearch_version="7.10", encryption_at_rest_options={"enabled": False}, access_policies={ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "AWS": "*" }, "Action": ["es:Describe*", "es:List*", "es:Get*", "es:ESHttp*"], "Resource": self.format_arn( service="es", resource="domain", resource_name="{}/*".format(ES_DOMAIN_NAME)) }] }, snapshot_options={"automatedSnapshotStartHour": 17}, vpc_options={ "securityGroupIds": [sg_es.security_group_id], "subnetIds": vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT).subnet_ids }) cdk.Tags.of(es_cfn_domain).add('Name', ES_DOMAIN_NAME) S3_BUCKET_SUFFIX = ''.join( random.sample((string.ascii_lowercase + string.digits), k=7)) s3_bucket = s3.Bucket( self, "s3bucket", removal_policy=cdk.RemovalPolicy. DESTROY, #XXX: Default: cdk.RemovalPolicy.RETAIN - The bucket will be orphaned bucket_name="ekk-stack-{region}-{suffix}".format( region=cdk.Aws.REGION, suffix=S3_BUCKET_SUFFIX)) firehose_role_policy_doc = aws_iam.PolicyDocument() firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( **{ "effect": aws_iam.Effect.ALLOW, "resources": [ s3_bucket.bucket_arn, "{}/*".format( s3_bucket.bucket_arn) ], "actions": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject" ] })) firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, resources=["*"], actions=[ "ec2:DescribeVpcs", "ec2:DescribeVpcAttribute", "ec2:DescribeSubnets", "ec2:DescribeSecurityGroups", "ec2:DescribeNetworkInterfaces", "ec2:CreateNetworkInterface", "ec2:CreateNetworkInterfacePermission", "ec2:DeleteNetworkInterface" ])) firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=[ es_cfn_domain.attr_arn, "{}/*".format(es_cfn_domain.attr_arn) ], actions=[ "es:DescribeElasticsearchDomain", "es:DescribeElasticsearchDomains", "es:DescribeElasticsearchDomainConfig", "es:ESHttpPost", "es:ESHttpPut" ])) ES_INDEX_NAME = self.node.try_get_context("es_index_name") firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, resources=[ es_cfn_domain.attr_arn, "{}/*".format(es_cfn_domain.attr_arn) ], # resources=[ # "{aes_arn}/_all/_settings".format(aes_arn=es_cfn_domain.attr_arn), # "{aes_arn}/_cluster/stats".format(aes_arn=es_cfn_domain.attr_arn), # "{aes_arn}/{es_index_name}*/_mapping".format(aes_arn=es_cfn_domain.attr_arn, es_index_name=ES_INDEX_NAME), # "{aes_arn}/_nodes".format(aes_arn=es_cfn_domain.attr_arn), # "{aes_arn}/_nodes/*/stats".format(aes_arn=es_cfn_domain.attr_arn), # "{aes_arn}/_stats".format(aes_arn=es_cfn_domain.attr_arn), # "{aes_arn}/{es_index_name}*/_stats".format(aes_arn=es_cfn_domain.attr_arn, es_index_name=ES_INDEX_NAME) # ], actions=["es:ESHttpGet"])) firehose_log_group_name = "/aws/kinesisfirehose/{}".format( ES_INDEX_NAME) firehose_role_policy_doc.add_statements( aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, #XXX: The ARN will be formatted as follows: # arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name} resources=[ self.format_arn( service="logs", resource="log-group", resource_name="{}:log-stream:*".format( firehose_log_group_name), arn_format=cdk.ArnFormat.COLON_RESOURCE_NAME) ], actions=["logs:PutLogEvents"])) firehose_role = aws_iam.Role( self, "KinesisFirehoseServiceRole", role_name="KinesisFirehoseServiceRole-{es_index}-{region}".format( es_index=ES_INDEX_NAME, region=cdk.Aws.REGION), assumed_by=aws_iam.ServicePrincipal("firehose.amazonaws.com"), #XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221 inline_policies={"firehose_role_policy": firehose_role_policy_doc}) es_dest_vpc_config = aws_kinesisfirehose.CfnDeliveryStream.VpcConfigurationProperty( role_arn=firehose_role.role_arn, security_group_ids=[sg_use_es.security_group_id], subnet_ids=vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT).subnet_ids) es_dest_config = aws_kinesisfirehose.CfnDeliveryStream.ElasticsearchDestinationConfigurationProperty( index_name=ES_INDEX_NAME, role_arn=firehose_role.role_arn, s3_configuration={ "bucketArn": s3_bucket.bucket_arn, "bufferingHints": { "intervalInSeconds": 60, "sizeInMBs": 1 }, "cloudWatchLoggingOptions": { "enabled": True, "logGroupName": firehose_log_group_name, "logStreamName": "S3Backup" }, "compressionFormat": "UNCOMPRESSED", # [GZIP | HADOOP_SNAPPY | Snappy | UNCOMPRESSED | ZIP] # Kinesis Data Firehose automatically appends the “YYYY/MM/dd/HH/” UTC prefix to delivered S3 files. You can also specify # an extra prefix in front of the time format and add "/" to the end to have it appear as a folder in the S3 console. "prefix": "{}/".format(ES_INDEX_NAME), "roleArn": firehose_role.role_arn }, buffering_hints={ "intervalInSeconds": 60, "sizeInMBs": 1 }, cloud_watch_logging_options={ "enabled": True, "logGroupName": firehose_log_group_name, "logStreamName": "ElasticsearchDelivery" }, domain_arn=es_cfn_domain.attr_arn, index_rotation_period= "NoRotation", # [NoRotation | OneDay | OneHour | OneMonth | OneWeek] retry_options={"durationInSeconds": 60}, s3_backup_mode= "FailedDocumentsOnly", # [AllDocuments | FailedDocumentsOnly] vpc_configuration=es_dest_vpc_config) firehose_to_es_delivery_stream = aws_kinesisfirehose.CfnDeliveryStream( self, "KinesisFirehoseToES", delivery_stream_name=ES_INDEX_NAME, delivery_stream_type="DirectPut", elasticsearch_destination_configuration=es_dest_config, tags=[{ "key": "Name", "value": ES_DOMAIN_NAME }])
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) vpc = aws_ec2.Vpc( self, "ElasticsearchHolVPC", max_azs=2, gateway_endpoints={ "S3": aws_ec2.GatewayVpcEndpointOptions( service=aws_ec2.GatewayVpcEndpointAwsService.S3) }) #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/InstanceClass.html #XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/InstanceSize.html#aws_cdk.aws_ec2.InstanceSize ec2_instance_type = aws_ec2.InstanceType.of( aws_ec2.InstanceClass.BURSTABLE3, aws_ec2.InstanceSize.MEDIUM) sg_bastion_host = aws_ec2.SecurityGroup( self, "BastionHostSG", vpc=vpc, allow_all_outbound=True, description='security group for an bastion host', security_group_name='bastion-host-sg') cdk.Tags.of(sg_bastion_host).add('Name', 'bastion-host-sg') #XXX: As there are no SSH public keys deployed on this machine, # you need to use EC2 Instance Connect with the command # 'aws ec2-instance-connect send-ssh-public-key' to provide your SSH public key. # https://aws.amazon.com/de/blogs/compute/new-using-amazon-ec2-instance-connect-for-ssh-access-to-your-ec2-instances/ bastion_host = aws_ec2.BastionHostLinux( self, "BastionHost", vpc=vpc, instance_type=ec2_instance_type, subnet_selection=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC), security_group=sg_bastion_host) #TODO: SHOULD restrict IP range allowed to ssh acces bastion_host.allow_ssh_access_from(aws_ec2.Peer.ipv4("0.0.0.0/0")) sg_use_es = aws_ec2.SecurityGroup( self, "ElasticSearchClientSG", vpc=vpc, allow_all_outbound=True, description='security group for an elasticsearch client', security_group_name='use-es-cluster-sg') cdk.Tags.of(sg_use_es).add('Name', 'use-es-cluster-sg') sg_es = aws_ec2.SecurityGroup( self, "ElasticSearchSG", vpc=vpc, allow_all_outbound=True, description='security group for an elasticsearch cluster', security_group_name='es-cluster-sg') cdk.Tags.of(sg_es).add('Name', 'es-cluster-sg') sg_es.add_ingress_rule(peer=sg_es, connection=aws_ec2.Port.all_tcp(), description='es-cluster-sg') sg_es.add_ingress_rule(peer=sg_use_es, connection=aws_ec2.Port.all_tcp(), description='use-es-cluster-sg') sg_es.add_ingress_rule(peer=sg_bastion_host, connection=aws_ec2.Port.all_tcp(), description='bastion-host-sg') #XXX: aws cdk elastsearch example - https://github.com/aws/aws-cdk/issues/2873 es_domain_name = 'es-hol' es_cfn_domain = aws_elasticsearch.CfnDomain( self, "ElasticSearch", elasticsearch_cluster_config={ "dedicatedMasterCount": 3, "dedicatedMasterEnabled": True, "dedicatedMasterType": "t2.medium.elasticsearch", "instanceCount": 2, "instanceType": "t2.medium.elasticsearch", "zoneAwarenessEnabled": True }, ebs_options={ "ebsEnabled": True, "volumeSize": 10, "volumeType": "gp2" }, domain_name=es_domain_name, elasticsearch_version="7.7", encryption_at_rest_options={"enabled": False}, access_policies={ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "AWS": "*" }, "Action": ["es:Describe*", "es:List*", "es:Get*", "es:ESHttp*"], "Resource": self.format_arn( service="es", resource="domain", resource_name="{}/*".format(es_domain_name)) }] }, snapshot_options={"automatedSnapshotStartHour": 17}, vpc_options={ "securityGroupIds": [sg_es.security_group_id], "subnetIds": vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT).subnet_ids }) cdk.Tags.of(es_cfn_domain).add('Name', 'es-hol') cdk.CfnOutput(self, 'BastionHostId', value=bastion_host.instance_id, export_name='BastionHostId') cdk.CfnOutput(self, 'BastionHostPublicDNSName', value=bastion_host.instance_public_dns_name, export_name='BastionHostPublicDNSName') cdk.CfnOutput(self, 'ESDomainEndpoint', value=es_cfn_domain.attr_domain_endpoint, export_name='ESDomainEndpoint') cdk.CfnOutput( self, 'ESDashboardsURL', value=f"{es_cfn_domain.attr_domain_endpoint}/_dashboards/", export_name='ESDashboardsURL')
def __init__(self, scope: Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code from zhxinyua to create VPC, s3_endpoint, bastion, EC2, EBS, Cloudwatch event rule stop EC2, Backup for EC2 # create a new VPC vpc_new = aws_ec2.Vpc(self, "VpcFromCDK", cidr="10.0.0.0/16") vpc_new.add_gateway_endpoint( "S3Endpoint", service=aws_ec2.GatewayVpcEndpointAwsService.S3, # Add only to ISOLATED subnets subnets=[ aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType.PUBLIC) ]) # only allow a specific rang of IP to conncet bastion # BastionHostLinux support two way to connect, one is SSM, second is EC2 Instance Connect # EC2 Instance Connect are not supportd in CN host_bastion = aws_ec2.BastionHostLinux( self, "BastionHost", vpc=vpc_new, subnet_selection=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC)) # write your own IP rang to access this bastion instead of 1.2.3.4/32 host_bastion.allow_ssh_access_from(aws_ec2.Peer.ipv4("1.2.3.4/32")) # use amazon linux as OS amzn_linux = aws_ec2.MachineImage.latest_amazon_linux( generation=aws_ec2.AmazonLinuxGeneration.AMAZON_LINUX, edition=aws_ec2.AmazonLinuxEdition.STANDARD, virtualization=aws_ec2.AmazonLinuxVirt.HVM, storage=aws_ec2.AmazonLinuxStorage.GENERAL_PURPOSE) # secure group my_security_group = aws_ec2.SecurityGroup( self, "SecurityGroup", vpc=vpc_new, description="SecurityGroup from CDK", security_group_name="CDK SecurityGroup", allow_all_outbound=True, ) my_security_group.add_ingress_rule(aws_ec2.Peer.ipv4('10.0.0.0/16'), aws_ec2.Port.tcp(22), "allow ssh access from the VPC") # set up an web instance in public subnet work_server = aws_ec2.Instance( self, "WebInstance", instance_type=aws_ec2.InstanceType("Write a EC2 instance type"), machine_image=amzn_linux, vpc=vpc_new, vpc_subnets=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC), security_group=my_security_group, key_name="Your SSH key pair name") # allow web connect work_server.connections.allow_from_any_ipv4(aws_ec2.Port.tcp(80), "allow http from world") work_server.connections.allow_from_any_ipv4(aws_ec2.Port.tcp(443), "allow https from world") # set a second ebs to web instance work_server.instance.add_property_override( "BlockDeviceMappings", [{ "DeviceName": "/dev/sdb", "Ebs": { "VolumeSize": "30", "VolumeType": "gp2", "DeleteOnTermination": "true" } }]) # Cloudwatch event rule to stop instances every day in 15:00 UTC # they only use javascript SDK to call AWS API # https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_events_targets/AwsApi.html stop_EC2 = AwsApi( service="EC2", action="stopInstances", parameters={ "InstanceIds": [work_server.instance_id, host_bastion.instance_id] }) Rule(self, "ScheduleRule", schedule=Schedule.cron(minute="0", hour="15"), targets=[stop_EC2]) # AWS backup part # create a BackupVault vault = backup.BackupVault(self, "BackupVault", backup_vault_name="CDK_Backup_Vault") # create a BackupPlan plan = backup.BackupPlan(self, "AWS-Backup-Plan", backup_plan_name="CDK_Backup") # add buackup resources with two way for two resources plan.add_selection( "Selection", resources=[ backup.BackupResource.from_ec2_instance(work_server), backup.BackupResource.from_tag("Name", "BastionHost") ]) # details with backup rules plan.add_rule( backup.BackupPlanRule( backup_vault=vault, rule_name="CDK_Backup_Rule", schedule_expression=Schedule.cron(minute="0", hour="16", day="1", month="1-12"), delete_after=Duration.days(130), move_to_cold_storage_after=Duration.days(10))) # output information after deploy output = CfnOutput(self, "BastionHost_information", value=host_bastion.instance_public_ip, description="BastionHost's Public IP") output = CfnOutput(self, "WebHost_information", value=work_server.instance_public_ip, description="Web server's Public IP")
def __init__(self, scope: core.Construct, id: str, props, **kwargs) -> None: super().__init__(scope, id, **kwargs) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/Vpc.html vpc = ec2.Vpc(self, "vpc", cidr=props['vpc_CIDR'], max_azs=3, subnet_configuration=[ { 'cidrMask': 28, 'name': 'public', 'subnetType': ec2.SubnetType.PUBLIC }, { 'cidrMask': 28, 'name': 'private', 'subnetType': ec2.SubnetType.PRIVATE }, { 'cidrMask': 28, 'name': 'db', 'subnetType': ec2.SubnetType.ISOLATED } ] ) rds_subnetGroup = rds.SubnetGroup(self, "rds_subnetGroup", description = f"Group for {props['environment']}-{props['application']}-{props['unit']} DB", vpc = vpc, vpc_subnets = ec2.SubnetSelection(subnet_type= ec2.SubnetType.ISOLATED) ) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_rds/DatabaseCluster.html ##TODO:ADD Aurora Serverless Option rds_instance = rds.DatabaseCluster(self,'wordpress-db', engine=rds.DatabaseClusterEngine.aurora_mysql( version=rds.AuroraMysqlEngineVersion.VER_2_07_2 ), instances=1, instance_props=rds.InstanceProps( vpc=vpc, enable_performance_insights=props['rds_enable_performance_insights'], instance_type=ec2.InstanceType(instance_type_identifier=props['rds_instance_type']) ), subnet_group=rds_subnetGroup, storage_encrypted=props['rds_storage_encrypted'], backup=rds.BackupProps( retention=core.Duration.days(props['rds_automated_backup_retention_days']) ) ) EcsToRdsSeurityGroup= ec2.SecurityGroup(self, "EcsToRdsSeurityGroup", vpc = vpc, description = "Allow WordPress containers to talk to RDS" ) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_lambda/Function.html db_cred_generator = _lambda.Function( self, 'db_creds_generator', runtime=_lambda.Runtime.PYTHON_3_8, handler='db_creds_generator.handler', code=_lambda.Code.asset('lambda'), vpc=vpc, vpc_subnets=ec2.SubnetSelection(subnet_type= ec2.SubnetType.ISOLATED), #vpc.select_subnets(subnet_type = ec2.SubnetType("ISOLATED")).subnets , environment={ 'SECRET_NAME': rds_instance.secret.secret_name, } ) #Set Permissions and Sec Groups rds_instance.connections.allow_from(EcsToRdsSeurityGroup, ec2.Port.tcp(3306)) #Open hole to RDS in RDS SG #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_efs/FileSystem.html file_system = efs.FileSystem(self, "MyEfsFileSystem", vpc = vpc, encrypted=True, # file system is not encrypted by default lifecycle_policy = props['efs_lifecycle_policy'], performance_mode = efs.PerformanceMode.GENERAL_PURPOSE, throughput_mode = efs.ThroughputMode.BURSTING, removal_policy = core.RemovalPolicy(props['efs_removal_policy']), enable_automatic_backups = props['efs_automatic_backups'] ) if props['deploy_bastion_host']: #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/BastionHostLinux.html bastion_host = ec2.BastionHostLinux(self, 'bastion_host', vpc = vpc ) rds_instance.connections.allow_from(bastion_host, ec2.Port.tcp(3306)) self.output_props = props.copy() self.output_props["vpc"] = vpc self.output_props["rds_instance"] = rds_instance self.output_props["EcsToRdsSeurityGroup"] = EcsToRdsSeurityGroup self.output_props["file_system"] = file_system
def __init__(self, scope: core.Construct, id: str, props, **kwargs) -> None: super().__init__(scope, id, **kwargs) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/Vpc.html vpc = ec2.Vpc(self, "vpc", cidr=props['vpc_CIDR'], max_azs=3, subnet_configuration=[{ 'cidrMask': 28, 'name': 'public', 'subnetType': ec2.SubnetType.PUBLIC }, { 'cidrMask': 28, 'name': 'private', 'subnetType': ec2.SubnetType.PRIVATE }, { 'cidrMask': 28, 'name': 'db', 'subnetType': ec2.SubnetType.ISOLATED }]) rds_subnetGroup = rds.SubnetGroup( self, "rds_subnetGroup", description= f"Group for {props['environment']}-{props['application']}-{props['unit']} DB", vpc=vpc, vpc_subnets=ec2.SubnetSelection( subnet_type=ec2.SubnetType.ISOLATED)) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_rds/DatabaseCluster.html ##TODO:ADD Aurora Serverless Option rds_instance = rds.DatabaseCluster( self, 'wordpress-db', engine=rds.DatabaseClusterEngine.aurora_mysql( version=rds.AuroraMysqlEngineVersion.VER_2_07_2), instances=1, instance_props=rds.InstanceProps( vpc=vpc, enable_performance_insights=props[ 'rds_enable_performance_insights'], instance_type=ec2.InstanceType( instance_type_identifier=props['rds_instance_type'])), subnet_group=rds_subnetGroup, storage_encrypted=props['rds_storage_encrypted'], backup=rds.BackupProps(retention=core.Duration.days( props['rds_automated_backup_retention_days']))) EcsToRdsSeurityGroup = ec2.SecurityGroup( self, "EcsToRdsSeurityGroup", vpc=vpc, description="Allow WordPress containers to talk to RDS") #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_lambda/Function.html db_cred_generator = _lambda.Function( self, 'db_creds_generator', runtime=_lambda.Runtime.PYTHON_3_8, handler='db_creds_generator.handler', code=_lambda.Code.asset('lambda/db_creds_generator'), vpc=vpc, vpc_subnets=ec2.SubnetSelection( subnet_type=ec2.SubnetType.ISOLATED ), #vpc.select_subnets(subnet_type = ec2.SubnetType("ISOLATED")).subnets , environment={ 'SECRET_NAME': rds_instance.secret.secret_name, }) #Set Permissions and Sec Groups rds_instance.connections.allow_from( EcsToRdsSeurityGroup, ec2.Port.tcp(3306)) #Open hole to RDS in RDS SG #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_efs/FileSystem.html file_system = efs.FileSystem( self, "MyEfsFileSystem", vpc=vpc, encrypted=True, # file system is not encrypted by default lifecycle_policy=props['efs_lifecycle_policy'], performance_mode=efs.PerformanceMode.GENERAL_PURPOSE, throughput_mode=efs.ThroughputMode.BURSTING, removal_policy=core.RemovalPolicy(props['efs_removal_policy']), enable_automatic_backups=props['efs_automatic_backups']) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/Cluster.html?highlight=ecs%20cluster#aws_cdk.aws_ecs.Cluster cluster = ecs.Cluster( self, "Cluster", vpc=vpc, container_insights=props['ecs_enable_container_insights']) if props['deploy_bastion_host']: #ToDo: Deploy bastion host with a key file #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/BastionHostLinux.html bastion_host = ec2.BastionHostLinux(self, 'bastion_host', vpc=vpc) rds_instance.connections.allow_from(bastion_host, ec2.Port.tcp(3306)) ####################### ### Developer Tools ### # SFTP into the EFS Shared File System NetToolsSecret = secretsmanager.Secret( self, "NetToolsSecret", generate_secret_string=secretsmanager.SecretStringGenerator( secret_string_template=json.dumps({ "username": '******', "ip": '' }), generate_string_key="password", exclude_characters='/"')) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_efs/FileSystem.html#aws_cdk.aws_efs.FileSystem.add_access_point AccessPoint = file_system.add_access_point( "access-point", path="/", create_acl=efs.Acl( owner_uid= "100", #https://aws.amazon.com/blogs/containers/developers-guide-to-using-amazon-efs-with-amazon-ecs-and-aws-fargate-part-2/ owner_gid="101", permissions="0755")) EfsVolume = ecs.Volume( name="efs", efs_volume_configuration=ecs.EfsVolumeConfiguration( file_system_id=file_system.file_system_id, transit_encryption="ENABLED", authorization_config=ecs.AuthorizationConfig( access_point_id=AccessPoint.access_point_id))) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/FargateTaskDefinition.html NetToolsTask = ecs.FargateTaskDefinition(self, "TaskDefinition", cpu=256, memory_limit_mib=512, volumes=[EfsVolume]) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/FargateTaskDefinition.html#aws_cdk.aws_ecs.FargateTaskDefinition.add_container NetToolsContainer = NetToolsTask.add_container( "NetTools", image=ecs.ContainerImage.from_registry('netresearch/sftp'), command=['test:test:100:101:efs']) NetToolsContainer.add_port_mappings( ecs.PortMapping(container_port=22, protocol=ecs.Protocol.TCP)) NetToolsContainer.add_mount_points( ecs.MountPoint( container_path= "/home/test/efs", #ToDo build path out with username from secret read_only=False, source_volume=EfsVolume.name, )) #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/FargateService.html?highlight=fargateservice#aws_cdk.aws_ecs.FargateService service = ecs.FargateService( self, "Service", cluster=cluster, task_definition=NetToolsTask, platform_version=ecs.FargatePlatformVersion( "VERSION1_4"), #Required for EFS ) #ToDo somehow store container's IP on deploy #Allow traffic to EFS Volume from Net Tools container service.connections.allow_to(file_system, ec2.Port.tcp(2049)) #ToDo allow bastion host into container on port 22 #https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_lambda/Function.html bastion_ip_locator = _lambda.Function( self, 'bastion_ip_locator', function_name= f"{props['environment']}-{props['application']}-{props['unit']}-SFTP-IP", runtime=_lambda.Runtime.PYTHON_3_8, handler='bastion_ip_locator.handler', code=_lambda.Code.asset('lambda/bastion_ip_locator'), environment={ 'CLUSTER_NAME': cluster.cluster_arn, 'SERVICE_NAME': service.service_name }) #Give needed perms to bastion_ip_locator for reading info from ECS bastion_ip_locator.add_to_role_policy( iam.PolicyStatement( actions=["ecs:DescribeTasks"], resources=[ #f"arn:aws:ecs:us-east-1:348757191778:service/{cluster.cluster_name}/{service.service_name}", f"arn:aws:ecs:us-east-1:348757191778:task/{cluster.cluster_name}/*" ])) bastion_ip_locator.add_to_role_policy( iam.PolicyStatement(actions=[ "ecs:ListTasks", ], resources=["*"], conditions={ 'ArnEquals': { 'ecs:cluster': cluster.cluster_arn } })) self.output_props = props.copy() self.output_props["vpc"] = vpc self.output_props["rds_instance"] = rds_instance self.output_props["EcsToRdsSeurityGroup"] = EcsToRdsSeurityGroup self.output_props["file_system"] = file_system self.output_props["cluster"] = cluster
def __init__(self, scope: core.Construct, id: str, vpc, env, **kwargs) -> None: super().__init__(scope, id, env=env, **kwargs) # Define cfn parameters ec2_type = CfnParameter( self, "ec2-instance-type", type="String", default="t2.micro", description="Specify the instance type you want").value_as_string key_name = CfnParameter( self, "ssh key", type="String", default="key_ireland", description="Specify your SSH key").value_as_string # Create Bastion Server bastion = ec2.BastionHostLinux( self, "Bastion", vpc=vpc, subnet_selection=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PUBLIC), instance_name="BastionHostLinux", instance_type=ec2.InstanceType( instance_type_identifier="t2.micro")) bastion.instance.instance.add_property_override("KeyName", key_name) bastion.connections.allow_from_any_ipv4(ec2.Port.tcp(22), "Internet access SSH") # Create NLB nlb = elb.NetworkLoadBalancer(self, "emq-elb", vpc=vpc, internet_facing=True, cross_zone_enabled=True, load_balancer_name="emq-nlb") listener = nlb.add_listener("port1883", port=1883) # Create Autoscaling Group with desired 2*EC2 hosts asg = autoscaling.AutoScalingGroup( self, "emq-asg", vpc=vpc, vpc_subnets=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PRIVATE), instance_type=ec2.InstanceType(instance_type_identifier=ec2_type), machine_image=linux_ami, key_name=key_name, user_data=ec2.UserData.custom(user_data), health_check=HealthCheck.elb(grace=Duration.seconds(60)), desired_capacity=2, min_capacity=2, max_capacity=4) # NLB cannot associate with a security group therefore NLB object has no Connection object # Must modify manuall inbound rule of the newly created asg security group to allow access # from NLB IP only asg.connections.allow_from_any_ipv4( ec2.Port.tcp(1883), "Allow NLB access 1883 port of EC2 in Autoscaling Group") asg.connections.allow_from(bastion, ec2.Port.tcp(22), "Allow SSH from the bastion only") listener.add_targets("addTargetGroup", port=1883, targets=[asg]) core.CfnOutput(self, "Output", value=nlb.load_balancer_dns_name)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.vpc = ec2.Vpc( self, id="VPC", cidr="10.0.0.0/16", max_azs=2, nat_gateways=1, subnet_configuration=[ ec2.SubnetConfiguration(name="public", cidr_mask=24, reserved=False, subnet_type=ec2.SubnetType.PUBLIC), ec2.SubnetConfiguration(name="private", cidr_mask=24, reserved=False, subnet_type=ec2.SubnetType.PRIVATE), ec2.SubnetConfiguration(name="DB", cidr_mask=24, reserved=False, subnet_type=ec2.SubnetType.ISOLATED), # ec2.SubnetConfiguration( # name="DB2", cidr_mask=24, # reserved=False, subnet_type=ec2.SubnetType.ISOLATED # ) ], enable_dns_hostnames=True, enable_dns_support=True) core.Tag(key="Application", value=self.stack_name) \ .add(self.vpc, key="Application", value=self.stack_name) # core.Tag("Network", "Public").add(vpc) # core.Tag("Name", "VPCName-Environment").add(vpc) # core.Tag("Environment", "Environment").add(vpc) bastion = ec2.BastionHostLinux( self, id="BastionHost", vpc=self.vpc, instance_name="BastionHost", instance_type=ec2.InstanceType(ec2_type), subnet_selection=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PUBLIC)) bastion.allow_ssh_access_from(ec2.Peer.any_ipv4()) # Setup key_name for EC2 instance login if you don't use Session Manager #bastion.instance.instance.add_property_override("KeyName", key_name) ec2.CfnEIP(self, id="BastionHostEIP", domain="vpc", instance_id=bastion.instance_id) core.CfnOutput( self, id="VPCId", value=self.vpc.vpc_id, description="VPC ID", export_name=f"{self.region}:{self.account}:{self.stack_name}:vpc-id" ) core.CfnOutput( self, id="BastionPrivateIP", value=bastion.instance_private_ip, description="BASTION Private IP", export_name= f"{self.region}:{self.account}:{self.stack_name}:bastion-private-ip" ) core.CfnOutput( self, id="BastionPublicIP", value=bastion.instance_public_ip, description="BASTION Public IP", export_name= f"{self.region}:{self.account}:{self.stack_name}:bastion-public-ip" )
def __init__(self, scope: core.Construct, id: str, vpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Create Bastion bastion = ec2.BastionHostLinux( self, "myBastion", vpc=vpc, subnet_selection=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PUBLIC), instance_name="myBastionHostLinux", instance_type=ec2.InstanceType( instance_type_identifier="t2.micro")) # Setup key_name for EC2 instance login if you don't use Session Manager # bastion.instance.instance.add_property_override("KeyName", key_name) bastion.connections.allow_from_any_ipv4(ec2.Port.tcp(22), "Internet access SSH") # Create ALB alb = elb.ApplicationLoadBalancer(self, "myALB", vpc=vpc, internet_facing=True, load_balancer_name="myALB") alb.connections.allow_from_any_ipv4(ec2.Port.tcp(80), "Internet access ALB 80") listener = alb.add_listener("my80", port=80, open=True) # Create Autoscaling Group with fixed 2*EC2 hosts self.asg = autoscaling.AutoScalingGroup( self, "myASG", vpc=vpc, vpc_subnets=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PRIVATE), instance_type=ec2.InstanceType(instance_type_identifier=ec2_type), machine_image=linux_ami, key_name=key_name, user_data=ec2.UserData.custom(user_data), desired_capacity=2, min_capacity=2, max_capacity=2, # block_devices=[ # autoscaling.BlockDevice( # device_name="/dev/xvda", # volume=autoscaling.BlockDeviceVolume.ebs( # volume_type=autoscaling.EbsDeviceVolumeType.GP2, # volume_size=12, # delete_on_termination=True # )), # autoscaling.BlockDevice( # device_name="/dev/sdb", # volume=autoscaling.BlockDeviceVolume.ebs( # volume_size=20) # # 20GB, with default volume_type gp2 # ) # ] ) self.asg.connections.allow_from( alb, ec2.Port.tcp(80), "ALB access 80 port of EC2 in Autoscaling Group") listener.add_targets("addTargetGroup", port=80, targets=[self.asg]) core.CfnOutput(self, "Output", value=alb.load_balancer_dns_name)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # s3 s3_bucket_name = "{}-s3-{}".format(Constant.PROJECT_NAME, self._get_UUID(4)) _s3.Bucket( self, id=s3_bucket_name, bucket_name=s3_bucket_name, removal_policy=core.RemovalPolicy. DESTROY, #TODO: destroy for test # removal_policy=core.RemovalPolicy.RETAIN ) # step 1. VPC # 如果在已有的Vpc 中建立环境, 可以用下面这句, 需要传入 vpc_id # vpc = ec2.Vpc.from_lookup(self, "VPC", vpc_id='') vpc = ec2.Vpc( self, "VPC", max_azs=2, # 两个分区, 每个分区建一个子网 cidr="10.10.0.0/16", # configuration will create 3 groups in 2 AZs = 6 subnets. subnet_configuration=[ ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.PUBLIC, name="Public", cidr_mask=24), ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.PRIVATE, name="Private", cidr_mask=24), # ec2.SubnetConfiguration( # subnet_type=ec2.SubnetType.ISOLATED, # name="DB", # cidr_mask=24 # ) ], # nat_gateway_provider=ec2.NatProvider.gateway(), # nat_gateways=2, ) # ES 需要部署到私有子网中 selection = vpc.select_subnets(subnet_type=ec2.SubnetType.PRIVATE) # step 2. 访问S3 + ES集群需要的 iam_instance_profile # action -> statement -> policy -> role -> instance profile -> attach ec2 actions = [ "ec2:CreateNetworkInterface", "ec2:DeleteNetworkInterface", "ec2:DescribeNetworkInterfaces", "ec2:ModifyNetworkInterfaceAttribute", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", "ec2:DescribeVpcs", "s3:*" ] policyStatement = PolicyStatement(actions=actions, effect=Effect.ALLOW) policyStatement.add_all_resources() policyStatement.sid = "Stmt1480452973134" policy_name = "{}-ec2-es-policy".format(Constant.PROJECT_NAME) ec2_policy = Policy(self, policy_name, policy_name=policy_name) ec2_policy.add_statements(policyStatement) role_name = "{}-ec2-es-role".format(Constant.PROJECT_NAME) access_es_role = Role( self, role_name, role_name=role_name, assumed_by=ServicePrincipal('ec2.amazonaws.com.cn')) ec2_policy.attach_to_role(access_es_role) profile_name = "{}-ec2-es-profile".format(Constant.PROJECT_NAME) instance_profile = CfnInstanceProfile( self, profile_name, instance_profile_name=profile_name, roles=[access_es_role.role_name]) # step 4. ES # 生产环境建议设置安全组, 只接收VPC内443端口请求 sg_es_cluster_name = "{}-sg-es".format(Constant.PROJECT_NAME) sg_es_cluster = ec2.SecurityGroup( self, id=sg_es_cluster_name, vpc=vpc, security_group_name=sg_es_cluster_name) sg_es_cluster.add_ingress_rule(peer=ec2.Peer.ipv4(vpc.vpc_cidr_block), connection=ec2.Port.tcp(443)) es_name = Constant.PROJECT_NAME es_arn = self.format_arn(service="es", resource="domain", sep="/", resource_name=es_name) es = elasticsearch.CfnDomain( self, es_name, elasticsearch_version='7.1', domain_name=es_name, node_to_node_encryption_options={"enabled": False}, vpc_options={ "securityGroupIds": [sg_es_cluster.security_group_id ], # 生产环境建议设置安全组, 只接收VPC内443端口请求 # 如果开启多个节点, 需要配置多个子网, 目前测试只有一个ES 节点, 就只用到一个子网 "subnetIds": selection.subnet_ids[:1] }, ebs_options={ "ebsEnabled": True, "volumeSize": 10, "volumeType": "gp2" }, elasticsearch_cluster_config={ # 生成环境需要开启三个 # "dedicatedMasterCount": 3, # "dedicatedMasterEnabled": True, # "dedicatedMasterType": 'm4.large.elasticsearch', "instanceCount": 1, "instanceType": 'm4.large.elasticsearch', "zoneAwarenessEnabled": False }) es.access_policies = { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "AWS": "*" }, "Action": "es:*", "Resource": "{}/*".format(es_arn) }] } # step 5. SNS topic = sns.Topic(self, "topic") topic.add_subscription(subs.EmailSubscription(Constant.EMAIL_ADDRESS)) # 设置SNS endpoint, 让lambda 可以从vpc 内部访问 vpc.add_interface_endpoint( "SNSEndpoint", service=ec2.InterfaceVpcEndpointAwsService.SNS) # step 6. Lambda lambdaFn = lambda_.Function(self, "Singleton", code=lambda_.Code.asset('lambda'), handler='hello.handler', vpc=vpc, vpc_subnets=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PRIVATE), timeout=core.Duration.seconds(300), runtime=lambda_.Runtime.PYTHON_3_7, environment={ 'SNS_TOPIC_ARN': topic.topic_arn, 'ES_ENDPOINT': es.attr_domain_endpoint, 'ES_INDEX_NAME': Constant.ES_INDEX_NAME }) # step 7. Cloud watch event rule = events.Rule( self, "Rule", schedule=events.Schedule.cron(minute='0/5', hour='*', month='*', week_day='*', year='*'), ) rule.add_target(targets.LambdaFunction(lambdaFn)) #给Lambda 添加发布SNS的权限 topic.grant_publish(lambdaFn) # Create ALB alb_name = "{}-alb".format(Constant.PROJECT_NAME) alb = elb.ApplicationLoadBalancer(self, alb_name, vpc=vpc, internet_facing=True, load_balancer_name=alb_name) alb.connections.allow_from_any_ipv4(ec2.Port.tcp(80), "Internet access ALB 80") listener = alb.add_listener("my80", port=80, open=True) # Create Autoscaling Group with fixed 2*EC2 hosts user_data = user_data_content.format(es.attr_domain_endpoint, Constant.REGION_NAME, Constant.ES_LOG_PATH, Constant.ES_INDEX_NAME, s3_bucket_name) # step 3. 创建堡垒机 bastion_name = "{}-bastion".format(Constant.PROJECT_NAME) bastion = ec2.BastionHostLinux( self, bastion_name, vpc=vpc, subnet_selection=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PUBLIC), instance_name=bastion_name, instance_type=ec2.InstanceType( instance_type_identifier="m4.large")) bastion.instance.instance.add_property_override( "KeyName", Constant.EC2_KEY_NAME) bastion.connections.allow_from_any_ipv4( ec2.Port.tcp(22), "Internet access SSH") # 生成环境可以限定IP allow_from bastion.connections.allow_from_any_ipv4(ec2.Port.tcp(8080), "Internet access HTTP") # 测试需要 # bastion.connections.allow_from_any_ipv4(ec2.Port.tcp(443), "Internet access HTTPS") # 测试需要 bastion.instance.instance.iam_instance_profile = instance_profile.instance_profile_name # 给EC2设置 profile , 相当于Role bastion.instance.instance.image_id = ami_map.get( Constant.REGION_NAME) # 指定AMI ID #堡垒机的user_data 只能执行一次, 如果要执行多次, 请参考 https://amazonaws-china.com/premiumsupport/knowledge-center/execute-user-data-ec2/?nc1=h_ls bastion_user_data = "/home/ec2-user/start.sh {} {} '{}' {} {}".format( es.attr_domain_endpoint, Constant.REGION_NAME, Constant.ES_LOG_PATH, Constant.ES_INDEX_NAME, s3_bucket_name) bastion.instance.add_user_data( "date >> /home/ec2-user/root.txt") # 查看启动脚本是否执行 bastion.instance.add_user_data(bastion_user_data) asg_name = "{}-asg".format(Constant.PROJECT_NAME) asg = autoscaling.AutoScalingGroup( self, asg_name, vpc=vpc, vpc_subnets=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PUBLIC), # PUBLIC for debug instance_type=ec2.InstanceType( instance_type_identifier="m4.large"), machine_image=my_ami, key_name=Constant.EC2_KEY_NAME, user_data=ec2.UserData.custom(user_data), desired_capacity=1, min_capacity=1, max_capacity=1, role=access_es_role) asg.connections.allow_from( alb, ec2.Port.tcp(8080), "ALB access 80 port of EC2 in Autoscaling Group") # asg.connections.allow_from_any_ipv4(ec2.Port.tcp(8080), "Internet access HTTP for test") # 测试用 asg.connections.allow_from_any_ipv4(ec2.Port.tcp(22), "Internet access SSH") # for debug listener.add_targets("addTargetGroup", port=8080, targets=[asg]) core.CfnOutput(self, "s3_bucket_name", value=s3_bucket_name, description='S3 bucket: store web log') core.CfnOutput(self, "ElasticSearchEndpointUrl", value=es.attr_domain_endpoint, description='Elastic Search Url') # Elastic Search 统计log 数量, 可以在堡垒机上执行, 快速查看日志数量。 core.CfnOutput(self, "CmdGetCountIndex", value='curl https://{}/{}/_count'.format( es.attr_domain_endpoint, Constant.ES_INDEX_NAME), description='Count search result. ') # 堡垒机的登录命令, 可以直接复制使用 core.CfnOutput(self, "CmdSshToBastion", value='ssh -i ~/{}.pem ec2-user@{}'.format( Constant.EC2_KEY_NAME, bastion.instance_public_dns_name), description='cmd ssh to bastion') # 在堡垒机上启动服务的命令, 堡垒机重启以后, 需要执行下面的命令, 可以启动web服务 发送日志到ES core.CfnOutput( self, "CmdSshBastionStartWeb", value='sudo {}'.format(bastion_user_data), description="Cmd to start web+logstash+filebeat service") # ALB 的访问地址 core.CfnOutput(self, "UrlLoad_Balancer", value='http://{}'.format(alb.load_balancer_dns_name), description='ALB url ') # 堡垒机的web访问地址, 为了调试方便, 在堡垒机上也使用相同的AMI。 core.CfnOutput(self, "UrlBastion", value='http://{}:8080'.format( bastion.instance_public_dns_name), description="Bastion server web url ") # 下面这条输出的命令 是通过堡垒机和Elasticsearch 建立隧道, 在本地访问kibana。 core.CfnOutput( self, "CmdSshProxyToKibana", value='ssh -i ~/{}.pem ec2-user@{} -N -L 9200:{}:443'.format( Constant.EC2_KEY_NAME, bastion.instance_public_dns_name, es.attr_domain_endpoint), description="cmd: access kibana from bastion ssh. ") # 执行完上面的命令后, 在浏览器中打开下面的连接 core.CfnOutput(self, "UrlKibana", value='https://localhost:9200/_plugin/kibana/', description="kibana url ")