def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) vpc = ec2.Vpc(self, "EKS_Kafka_PocClusterVPC") private_subnets = [snet_id.subnet_id for snet_id in vpc.private_subnets] bngi = msk.CfnCluster.BrokerNodeGroupInfoProperty(instance_type="kafka.m5.large", client_subnets=private_subnets) msk_cluster = msk.CfnCluster(self, "EKS_KafkaPocMSKCluster", broker_node_group_info=bngi, cluster_name="EKSKafkaPOCMKSCluster", kafka_version="2.3.1", number_of_broker_nodes=3) eks_admin_role = iam.Role(self, "EKS_Kafka_PocCluster-AdminRole", assumed_by=iam.AccountPrincipal(account_id=self.account)) eks_cluster = eks.Cluster(self, "EKS_Kafka_PocEKSCluster", cluster_name="EKS_Kafka_PocCluster", masters_role=eks_admin_role, kubectl_enabled=True, version="1.15", vpc=vpc) eks_cluster.add_capacity("worker", instance_type=ec2.InstanceType("t3.large"), min_capacity=1, max_capacity=10)
def __init__(self, scope: core.Construct, id: str, VPC: ec2.Vpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.cluster = msk.CfnCluster( self, "statement_kafka", cluster_name="statement-demo", broker_node_group_info = { "clientSubnets" : [subnet.subnet_id for subnet in VPC.private_subnets], "instanceType": self.node.try_get_context("kafka")["instanceType"], "storageInfo" : { "ebsStorageInfo" : { "volumeSize": self.node.try_get_context("kafka")["volumeSize"] } } }, kafka_version="2.5.1", number_of_broker_nodes=self.node.try_get_context("kafka")["number_of_broker_nodes"], encryption_info={ "encryptionInTransit": { "clientBroker": "TLS" } }, enhanced_monitoring="PER_TOPIC_PER_BROKER" )
def __init__(self, scope: core.Construct, id: str, e2e_security_group: aws_ec2.SecurityGroup, vpc: aws_ec2.Vpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) cluster_sec_group = aws_ec2.SecurityGroup(self, 'msk-cluster-sec-group', vpc=vpc) cluster_sec_group.add_ingress_rule(peer=e2e_security_group, connection=aws_ec2.Port( string_representation='kafka', protocol=aws_ec2.Protocol.TCP, from_port=DEFAULT_KAFKA_PORT, to_port=DEFAULT_KAFKA_PORT)) cluster = aws_msk.CfnCluster( self, 'msk-cluster', cluster_name='cdk-test', number_of_broker_nodes=len( vpc.private_subnets), # this is the minimum number needed kafka_version='2.3.1', broker_node_group_info=aws_msk.CfnCluster. BrokerNodeGroupInfoProperty( instance_type="kafka.m5.large", client_subnets=[ subnet.subnet_id for subnet in vpc.private_subnets ], security_groups=[cluster_sec_group.security_group_id])) core.CfnOutput(self, "arn", value=cluster.ref)
def __init__( self, scope: cdk.Construct, construct_id: str, stack_log_level, vpc, **kwargs ) -> None: super().__init__(scope, construct_id, **kwargs) # Create Security Group for Managed Kafka Instance self.kafka_client_sg = _ec2.SecurityGroup( self, "kafkaClientSG", vpc=vpc, description="kafka client security group", allow_all_outbound=True, ) cdk.Tags.of(self.kafka_client_sg).add("Name", "kafka_client_sg") self.kafka_cluster_sg = _ec2.SecurityGroup( self, "kafkaSG", vpc=vpc, security_group_name=f"kafka_sg_{construct_id}", description="Security Group for Kafka Cluster" ) # https://docs.aws.amazon.com/msk/latest/developerguide/troubleshooting.html#networking-trouble self.kafka_cluster_sg.add_ingress_rule( peer=_ec2.Peer.ipv4(vpc.vpc_cidr_block), connection=_ec2.Port.tcp(443), description="Allow incoming secure traffic from within VPC" ) self.kafka_cluster_sg.add_ingress_rule( peer=_ec2.Peer.ipv4(vpc.vpc_cidr_block), connection=_ec2.Port.tcp(2181), description="Allow incoming secure traffic from within VPC" ) self.kafka_cluster_sg.add_ingress_rule( peer=_ec2.Peer.ipv4(vpc.vpc_cidr_block), connection=_ec2.Port.tcp(9092), description="Allow incoming plaintext traffic REALLY from within VPC" ) self.kafka_cluster_sg.add_ingress_rule( peer=_ec2.Peer.ipv4(vpc.vpc_cidr_block), connection=_ec2.Port.tcp(9094), description="Allow incoming TLS-encrypted traffic from within VPC" ) cdk.Tags.of(self.kafka_cluster_sg).add("Name", "kafka_cluster_sg") # ALLOW kafka clients to connect to kafka cluster sg self.kafka_cluster_sg.add_ingress_rule( peer=self.kafka_client_sg, connection=_ec2.Port.all_tcp(), description="ALLOW kafka clients to connect to kafka cluster sg" ) # ALLOW kafka Lambda Consumer to connect to kafka cluster sg # https://docs.aws.amazon.com/lambda/latest/dg/services-msk-topic-add.html self.kafka_client_sg.add_ingress_rule( peer=_ec2.Peer.ipv4(vpc.vpc_cidr_block), connection=_ec2.Port.all_tcp(), description="ALLOW kafka Lambda consumer to poll kafka cluster" ) # Add your stack resources below): self.msk_cluster = _msk.CfnCluster( self, "managedKafka01", broker_node_group_info=_msk.CfnCluster.BrokerNodeGroupInfoProperty( instance_type="kafka.t3.small", client_subnets=vpc.select_subnets( subnet_type=_ec2.SubnetType.PRIVATE ).subnet_ids, security_groups=[self.kafka_cluster_sg.security_group_id] ), cluster_name="miztiik-msk-cluster-01", kafka_version="2.3.1", number_of_broker_nodes=2 ) ########################################### ################# OUTPUTS ################# ########################################### output_0 = cdk.CfnOutput( self, "AutomationFrom", value=f"{GlobalArgs.SOURCE_INFO}", description="To know more about this automation stack, check out our github page." ) output_1 = cdk.CfnOutput( self, "SalesEventsKafkaRouter", value=f"https://console.aws.amazon.com/msk/home?region={cdk.Aws.REGION}#/clusters/{self.msk_cluster.cluster_name}", description="Sales events Kafka Cluster" )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # VPC vpc = ec2.Vpc( self, f"{prefix}_vpc", nat_gateways=1, enable_dns_hostnames=True, enable_dns_support=True, max_azs=2, subnet_configuration=[ ec2.SubnetConfiguration(name="public", subnet_type=ec2.SubnetType.PUBLIC), ec2.SubnetConfiguration(name="privat", subnet_type=ec2.SubnetType.PRIVATE) ]) # MSK Cluster Security Group sg_msk = ec2.SecurityGroup(self, f"{prefix}_sg", vpc=vpc, allow_all_outbound=True, security_group_name=f"{prefix}_sg_msk") for subnet in vpc.public_subnets: sg_msk.add_ingress_rule(ec2.Peer.ipv4(subnet.ipv4_cidr_block), ec2.Port.tcp(2181), "Zookeeper Plaintext") sg_msk.add_ingress_rule(ec2.Peer.ipv4(subnet.ipv4_cidr_block), ec2.Port.tcp(2182), "Zookeeper TLS") sg_msk.add_ingress_rule(ec2.Peer.ipv4(subnet.ipv4_cidr_block), ec2.Port.tcp(9092), "Broker Plaintext") sg_msk.add_ingress_rule(ec2.Peer.ipv4(subnet.ipv4_cidr_block), ec2.Port.tcp(9094), "Zookeeper Plaintext") for subnet in vpc.private_subnets: sg_msk.add_ingress_rule(ec2.Peer.ipv4(subnet.ipv4_cidr_block), ec2.Port.all_traffic(), "All private traffic") # MSK Cluster msk.CfnCluster( self, f"{prefix}_kafka_cluster", cluster_name="msk-quickstart", number_of_broker_nodes=len(vpc.private_subnets), kafka_version="2.6.0", encryption_info=msk.CfnCluster.EncryptionInfoProperty( encryption_in_transit=msk.CfnCluster. EncryptionInTransitProperty(client_broker="TLS_PLAINTEXT")), broker_node_group_info=msk.CfnCluster.BrokerNodeGroupInfoProperty( instance_type="kafka.m5.large", client_subnets=[ subnet.subnet_id for subnet in vpc.private_subnets ], security_groups=[sg_msk.security_group_id], storage_info=msk.CfnCluster.StorageInfoProperty( ebs_storage_info=msk.CfnCluster.EBSStorageInfoProperty( volume_size=200)))) # EC2 Client AMI amazon_linux = ec2.MachineImage.latest_amazon_linux( generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2, edition=ec2.AmazonLinuxEdition.STANDARD, virtualization=ec2.AmazonLinuxVirt.HVM, storage=ec2.AmazonLinuxStorage.GENERAL_PURPOSE) # Instance Role and SSM Managed Policy role = iam.Role(self, f"{prefix}_ssm_role", assumed_by=iam.ServicePrincipal("ec2.amazonaws.com")) role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AmazonEC2RoleforSSM")) role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonMSKReadOnlyAccess")) # EC2 Client Instance instance = ec2.Instance( self, f"{prefix}_instance", instance_type=ec2.InstanceType("m5.large"), machine_image=amazon_linux, vpc=vpc, role=role, vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC)) # Bootstrap script in S3 as Asset asset_bootstrap = Asset(self, f"{prefix}_bootstrap", path=os.path.join(dirname, "configure.sh")) local_bootstrap_path = instance.user_data.add_s3_download_command( bucket=asset_bootstrap.bucket, bucket_key=asset_bootstrap.s3_object_key) # Loader project in S3 Asset asset_loader = Asset(self, f"{prefix}_loader", path=os.path.join(dirname, "earthquake_loader")) instance.user_data.add_s3_download_command( bucket=asset_loader.bucket, bucket_key=asset_loader.s3_object_key, local_file="earthquake_loader.zip") # Userdata executes bootstrap script from S3 instance.user_data.add_execute_file_command( file_path=local_bootstrap_path) # Grant read permissions to assets asset_bootstrap.grant_read(instance.role) asset_loader.grant_read(instance.role)
def __init__(self, scope: core.Construct, id: str, subnets: List[str], vpc: ec2.IVpc, default_vpc_cidr_block: str, project: str, namespace: cloudmap.HttpNamespace, **kwargs) -> None: super().__init__(scope, id, **kwargs) # firewall for load balancers self.lbFirewall = ec2.SecurityGroup( scope=self, id='LbFirewall', vpc=vpc, description='Load balancer firewall') self.lbFirewall.add_ingress_rule(peer=ec2.Peer.ipv4( vpc.vpc_cidr_block), connection=ec2.Port.all_traffic()) self.lbFirewall.add_ingress_rule( peer=ec2.Peer.ipv4(default_vpc_cidr_block), connection=ec2.Port.all_traffic()) # unified client firewall for both MSK and DocumentDB self.unifiedClientFirewall = ec2.SecurityGroup( scope=self, id='UnifiedClientFirewall', vpc=vpc, description='Client access firewall for DocumentDB and MSK') self.unifiedClientFirewall.add_ingress_rule( peer=self.lbFirewall, connection=ec2.Port.all_traffic()) # DocumentDB cluster projTag = core.CfnTag(key='Project', value=project) subnetGroup = docdb.CfnDBSubnetGroup( scope=self, id='DatabaseSubnetGroup', db_subnet_group_description='Subnet group for database', subnet_ids=subnets, tags=[projTag, core.CfnTag(key='Name', value='DocDbSubnetGroup')]) self.clientFirewall = ec2.SecurityGroup( scope=self, id='DatabaseClientFirewall', vpc=vpc, description='Client access firewall for DocumentDB') self.dbFirewall = ec2.SecurityGroup( scope=self, id='DatabaseInternalFirewall', vpc=vpc, allow_all_outbound=True, description='Firewall for DocumentDB') self.dbFirewall.add_ingress_rule(peer=self.clientFirewall, connection=ec2.Port.all_traffic()) self.dbFirewall.add_ingress_rule(peer=self.unifiedClientFirewall, connection=ec2.Port.all_traffic()) self.dbFirewall.add_ingress_rule( peer=ec2.Peer.ipv4(default_vpc_cidr_block), connection=ec2.Port.all_traffic()) self.docdbCluster = docdb.CfnDBCluster( scope=self, id='DataStore', db_subnet_group_name=subnetGroup.ref, master_username='******', master_user_password='******', vpc_security_group_ids=[self.dbFirewall.security_group_id]) self.docdbInstances = [ docdb.CfnDBInstance(scope=self, id="DataStore-Instance-{0}".format(str(i)), db_cluster_identifier=self.docdbCluster.ref, db_instance_class='db.r5.xlarge') for i in range(3) ] self.docdbCloudMap = namespace.create_service(id='DbSvc') self.docdbCloudMap.register_non_ip_instance( id='dbEndpoint', custom_attributes={ 'endpoint': self.docdbCluster.attr_endpoint, 'user': '******', 'password': '******' }) self.docdbCloudMap.register_non_ip_instance( id='dbReadEndpoint', custom_attributes={ 'endpoint': self.docdbCluster.attr_read_endpoint }) # MSK cluster self.kafkaClientFirewall = ec2.SecurityGroup( scope=self, id='KafkaClientFirewall', vpc=vpc, description='Client access firewall for Kafka') self.kafkaFirewall = ec2.SecurityGroup( scope=self, id='KafkaInternalFirewall', vpc=vpc, allow_all_outbound=True, description='Firewall for Kafka') self.kafkaFirewall.add_ingress_rule(peer=self.kafkaClientFirewall, connection=ec2.Port.all_traffic()) self.kafkaFirewall.add_ingress_rule(peer=self.unifiedClientFirewall, connection=ec2.Port.all_traffic()) self.kafkaFirewall.add_ingress_rule( peer=ec2.Peer.ipv4(default_vpc_cidr_block), connection=ec2.Port.all_traffic()) self.kafkaFirewall.add_ingress_rule(peer=self.kafkaFirewall, connection=ec2.Port.all_traffic()) num_brokers = len(subnets) if num_brokers < 3: num_brokers = 2 * num_brokers self.kafka = msk.CfnCluster( scope=self, id='kafka', cluster_name='kafkafargateworkshop', kafka_version='2.2.1', number_of_broker_nodes=num_brokers, enhanced_monitoring='PER_TOPIC_PER_BROKER', broker_node_group_info=msk.CfnCluster.BrokerNodeGroupInfoProperty( client_subnets=subnets, instance_type='kafka.m5.large', security_groups=[self.kafkaFirewall.security_group_id])) self.kafkaCloudMap = namespace.create_service(id='KafkaSvc') self.kafkaCloudMap.register_non_ip_instance( id='KafkaBrokerArn', custom_attributes={'broker_arn': self.kafka.ref}) # ECR self.docker_repo = ecr.Repository(scope=self, id="FargateImageRepository") ssm.StringParameter(scope=self, id='SSMParamRegion', string_value=self.region, parameter_name='region') ssm.StringParameter(scope=self, id='SSMParamRepoUri', string_value=self.docker_repo.repository_uri, parameter_name='repo_uri') self.docker_repo_s3sink = ecr.Repository( scope=self, id="FargateImageRepositoryS3Sink") self.docker_repo_s3sinkhc = ecr.Repository( scope=self, id="FargateImageRepositoryS3SinkHC") ssm.StringParameter( scope=self, id='SSMParamRepoUriS3Sink', string_value=self.docker_repo_s3sink.repository_uri, parameter_name='repo_uri_s3_sink') ssm.StringParameter( scope=self, id='SSMParamRepoUriS3SinkHC', string_value=self.docker_repo_s3sinkhc.repository_uri, parameter_name='repo_uri_s3_sink_hc') self.docker_repo_sftp = ecr.Repository(scope=self, id="FargateImageRepositorySftp") ssm.StringParameter(scope=self, id='SSMParamRepoUriSftp', string_value=self.docker_repo_sftp.repository_uri, parameter_name='repo_uri_sftp') self.docker_repo_batch = ecr.Repository( scope=self, id="FargateImageRepositoryBatch") ssm.StringParameter(scope=self, id='SSMParamRepoUriBatch', string_value=self.docker_repo_batch.repository_uri, parameter_name='repo_uri_batch') # SFTP server self.sftpBucket = s3.Bucket( scope=self, id="SFTPBucket", block_public_access=s3.BlockPublicAccess.BLOCK_ALL, encryption=s3.BucketEncryption.S3_MANAGED) core.CfnOutput(scope=self, id="SFTPBucketName", value=self.sftpBucket.bucket_name) self.sftp_role = iam.Role( scope=self, id="SFTPRole", assumed_by=iam.ServicePrincipal("transfer.amazonaws.com")) self.sftp_role.add_to_policy( statement=iam.PolicyStatement(resources=[ self.sftpBucket.bucket_arn, self.sftpBucket.bucket_arn + '/*' ], actions=['s3:*'])) self.sftp_vpce = vpc.add_interface_endpoint( id="SftpEndpoint", service=ec2.InterfaceVpcEndpointAwsService.TRANSFER) self.sftp_vpce.connections.allow_default_port_from( other=ec2.Peer.ipv4(vpc.vpc_cidr_block)) self.sftp_vpce.connections.allow_default_port_from( other=ec2.Peer.ipv4(default_vpc_cidr_block)) self.sftp_vpce.connections.allow_from(other=ec2.Peer.ipv4( vpc.vpc_cidr_block), port_range=ec2.Port.tcp(22)) self.sftp_vpce.connections.allow_from( other=ec2.Peer.ipv4(default_vpc_cidr_block), port_range=ec2.Port.tcp(22)) self.sftp = transfer.CfnServer( scope=self, id="SFTP", endpoint_type='VPC_ENDPOINT', endpoint_details=transfer.CfnServer.EndpointDetailsProperty( vpc_endpoint_id=self.sftp_vpce.vpc_endpoint_id), identity_provider_type='SERVICE_MANAGED') self.sftp_user = transfer.CfnUser(scope=self, id="SFTPUser", role=self.sftp_role.role_arn, server_id=self.sftp.attr_server_id, user_name="sftpuser") core.CfnOutput(scope=self, id="SFTPHostVpceOut", value=self.sftp_vpce.vpc_endpoint_id) core.CfnOutput(scope=self, id="SFTPUserOut", value=self.sftp_user.attr_user_name) self.sftpCloudMap = namespace.create_service(id='SftpSvc', name='SFTP') self.sftpCloudMap.register_non_ip_instance( id='sftpEndpoint', custom_attributes={ 'vpce_id': self.sftp_vpce.vpc_endpoint_id, 'user': '******', 'bucket': self.sftpBucket.bucket_name })
def __init__(self, scope: core.Construct, id: str, vpc_stack, client: bool = True, **kwargs) -> None: super().__init__(scope, id, **kwargs) ensure_service_linked_role("kafka.amazonaws.com") client_properties = assets.Asset(self, "client_properties", path=os.path.join( dirname, "client.properties")) self.kafka_client_security_group = ec2.SecurityGroup( self, "kafka_client_security_group", vpc=vpc_stack.get_vpc, description="kafka client security group", allow_all_outbound=True, ) # core.Tag.add(self.kafka_client_security_group, "project", constants["PROJECT_TAG"]) core.Tags.of(self.kafka_client_security_group).add( "project", constants["PROJECT_TAG"]) # core.Tag.add(self.kafka_client_security_group, "Name", "kafka_client_sg") core.Tags.of(self.kafka_client_security_group).add( "Name", "kafka_client_sg") self.kafka_client_security_group.add_ingress_rule( ec2.Peer.ipv4(f"{external_ip}/32"), ec2.Port.tcp(22), "from own public ip", ) self.kafka_security_group = ec2.SecurityGroup( self, "kafka_security_group", vpc=vpc_stack.get_vpc, description="kafka security group", allow_all_outbound=True, ) core.Tags.of(self.kafka_security_group).add("project", constants["PROJECT_TAG"]) core.Tags.of(self.kafka_security_group).add("Name", "kafka_sg") self.kafka_security_group.connections.allow_from( self.kafka_security_group, ec2.Port.all_traffic(), "within kafka", ) self.kafka_security_group.connections.allow_from( self.kafka_client_security_group, ec2.Port.all_traffic(), "from kafka client sg", ) self.kafka_client_security_group.connections.allow_from( self.kafka_security_group, ec2.Port.all_traffic(), "from kafka", ) self.kafka_cluster = msk.CfnCluster( self, "kafka_cluster", broker_node_group_info={ "clientSubnets": vpc_stack.get_vpc_public_subnet_ids, "instanceType": constants["KAFKA_INSTANCE_TYPE"], "numberOfBrokerNodes": constants["KAFKA_BROKER_NODES"], "securityGroups": [self.kafka_security_group.security_group_id], }, encryption_info={ "encryptionInTransit": { "InCluster": "true", "clientBroker": "PLAINTEXT", }, }, cluster_name=constants["PROJECT_TAG"], kafka_version=constants["KAFKA_VERSION"], number_of_broker_nodes=constants["KAFKA_BROKER_NODES"], enhanced_monitoring="DEFAULT", ) core.Tags.of(self.kafka_cluster).add("project", constants["PROJECT_TAG"]) # instance for kafka client if client == True: # userdata for kafka client kafka_client_userdata = user_data_init( log_group_name="filebeat-kafka/kafka/instance") # create the instance kafka_client_instance = ec2.Instance( self, "kafka_client", instance_type=ec2.InstanceType( constants["KAFKA_CLIENT_INSTANCE"]), machine_image=ec2.AmazonLinuxImage( generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2), vpc=vpc_stack.get_vpc, vpc_subnets={"subnet_type": ec2.SubnetType.PUBLIC}, key_name=constants["KEY_PAIR"], security_group=self.kafka_client_security_group, user_data=kafka_client_userdata, ) core.Tags.of(kafka_client_instance).add("project", constants["PROJECT_TAG"]) # needs kafka cluster to be available kafka_client_instance.node.add_dependency(self.kafka_cluster) # create policies for EC2 to connect to Kafka access_kafka_policy = iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ "kafka:ListClusters", "kafka:GetBootstrapBrokers", "kafka:DescribeCluster", ], resources=["*"], ) # add the role permissions kafka_client_instance.add_to_role_policy( statement=access_kafka_policy) # add log permissions instance_add_log_permissions(kafka_client_instance) # add access to the file asset client_properties.grant_read(kafka_client_instance) # update the userdata with commands kafka_client_userdata.add_commands( f"aws s3 cp s3://{client_properties.s3_bucket_name}/{client_properties.s3_object_key} /home/ec2-user/client.properties", "yum install java-1.8.0 -y", f'curl https:/archive.apache.org/dist/kafka/{constants["KAFKA_DOWNLOAD_VERSION"].split("-")[-1]}/{constants["KAFKA_DOWNLOAD_VERSION"]}.tgz -o {constants["KAFKA_DOWNLOAD_VERSION"]}.tgz', f"tar -xvf {constants['KAFKA_DOWNLOAD_VERSION']}.tgz", f"mv {constants['KAFKA_DOWNLOAD_VERSION']} /opt", f"rm {constants['KAFKA_DOWNLOAD_VERSION']}.tgz", f"mv -f /home/ec2-user/client.properties /opt/{constants['KAFKA_DOWNLOAD_VERSION']}/bin/client.properties", # create the topic, if already exists capture error message f"kafka_arn=`aws kafka list-clusters --region {core.Aws.REGION} --output text --query 'ClusterInfoList[*].ClusterArn'`", f"kafka_zookeeper=`aws kafka describe-cluster --cluster-arn $kafka_arn --region {core.Aws.REGION} --output text --query 'ClusterInfo.ZookeeperConnectString'`", f"make_topic=`/opt/{constants['KAFKA_DOWNLOAD_VERSION']}/bin/kafka-topics.sh --create --zookeeper $kafka_zookeeper --replication-factor 3 --partitions 1 --topic topic 2>&1`", f"make_topic=`/opt/{constants['KAFKA_DOWNLOAD_VERSION']}/bin/kafka-topics.sh --create --zookeeper $kafka_zookeeper --replication-factor 3 --partitions 1 --topic apachelog 2>&1`", f"make_topic=`/opt/{constants['KAFKA_DOWNLOAD_VERSION']}/bin/kafka-topics.sh --create --zookeeper $kafka_zookeeper --replication-factor 3 --partitions 1 --topic appevent 2>&1`", ) # add the signal kafka_client_userdata.add_signal_on_exit_command( resource=kafka_client_instance) kafka_client_instance.add_user_data(kafka_client_userdata.render())
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) vpc_name = self.node.try_get_context('vpc_name') vpc = aws_ec2.Vpc.from_lookup(self, 'ExistingVPC', is_default=True, vpc_name=vpc_name) #XXX: create new vpc for msk cluster # vpc = aws_ec2.Vpc(self, 'VpcStack', # max_azs=3, # gateway_endpoints={ # "S3": aws_ec2.GatewayVpcEndpointOptions( # service=aws_ec2.GatewayVpcEndpointAwsService.S3 # ) # } # ) MSK_CLUSTER_NAME = cdk.CfnParameter( self, 'KafkaClusterName', type='String', description='Managed Streaming for Apache Kafka cluster name', default='MSK-{}'.format(''.join( random.sample((string.ascii_letters), k=5))), allowed_pattern='[A-Za-z0-9\-]+') KAFA_VERSION = cdk.CfnParameter( self, 'KafkaVersion', type='String', description='Apache Kafka version', default='2.6.2', # Supported Apache Kafka versions # https://docs.aws.amazon.com/msk/latest/developerguide/supported-kafka-versions.html allowed_values=[ '2.8.1', '2.8.0', '2.7.1', '2.6.2', '2.6.1', '2.6.0', '2.5.1', '2.4.1.1', '2.3.1', '2.2.1' ]) #XXX: change broker instance type KAFA_BROKER_INSTANCE_TYPE = cdk.CfnParameter( self, 'KafkaBrokerInstanceType', type='String', description='Apache Kafka Broker instance type', default='kafka.m5.large') #XXX: change volume size KAFA_BROKER_EBS_VOLUME_SIZE = cdk.CfnParameter( self, 'KafkaBrokerEBSVolumeSize', type='Number', description= 'Apache Kafka Broker EBS Volume size (Minimum: 1 GiB, Maximum: 16384 GiB)', default='100', min_value=1, max_value=16384) MSK_CLIENT_SG_NAME = 'use-msk-sg-{}'.format(''.join( random.sample((string.ascii_lowercase), k=5))) sg_use_msk = aws_ec2.SecurityGroup( self, 'KafkaClientSecurityGroup', vpc=vpc, allow_all_outbound=True, description='security group for Amazon MSK client', security_group_name=MSK_CLIENT_SG_NAME) cdk.Tags.of(sg_use_msk).add('Name', MSK_CLIENT_SG_NAME) MSK_CLUSTER_SG_NAME = 'msk-sg-{}'.format(''.join( random.sample((string.ascii_lowercase), k=5))) sg_msk_cluster = aws_ec2.SecurityGroup( self, 'MSKSecurityGroup', vpc=vpc, allow_all_outbound=True, description='security group for Amazon MSK Cluster', security_group_name=MSK_CLUSTER_SG_NAME) sg_msk_cluster.add_ingress_rule( peer=sg_use_msk, connection=aws_ec2.Port.tcp(2181), description='msk client security group') sg_msk_cluster.add_ingress_rule( peer=sg_use_msk, connection=aws_ec2.Port.tcp(9092), description='msk client security group') sg_msk_cluster.add_ingress_rule( peer=sg_use_msk, connection=aws_ec2.Port.tcp(9094), description='msk client security group') cdk.Tags.of(sg_msk_cluster).add('Name', MSK_CLUSTER_SG_NAME) msk_broker_ebs_storage_info = aws_msk.CfnCluster.EBSStorageInfoProperty( volume_size=KAFA_BROKER_EBS_VOLUME_SIZE.value_as_number) msk_broker_storage_info = aws_msk.CfnCluster.StorageInfoProperty( ebs_storage_info=msk_broker_ebs_storage_info) msk_broker_node_group_info = aws_msk.CfnCluster.BrokerNodeGroupInfoProperty( client_subnets=vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT).subnet_ids, instance_type=KAFA_BROKER_INSTANCE_TYPE.value_as_string, security_groups=[ sg_use_msk.security_group_id, sg_msk_cluster.security_group_id ], storage_info=msk_broker_storage_info) msk_encryption_info = aws_msk.CfnCluster.EncryptionInfoProperty( encryption_in_transit=aws_msk.CfnCluster. EncryptionInTransitProperty(client_broker='TLS_PLAINTEXT', in_cluster=True)) msk_cluster = aws_msk.CfnCluster( self, 'AWSKafkaCluster', broker_node_group_info=msk_broker_node_group_info, cluster_name=MSK_CLUSTER_NAME.value_as_string, #XXX: Supported Apache Kafka versions # https://docs.aws.amazon.com/msk/latest/developerguide/supported-kafka-versions.html kafka_version=KAFA_VERSION.value_as_string, number_of_broker_nodes=3, encryption_info=msk_encryption_info, enhanced_monitoring='PER_TOPIC_PER_BROKER') amzn_linux = aws_ec2.MachineImage.latest_amazon_linux( generation=aws_ec2.AmazonLinuxGeneration.AMAZON_LINUX_2, edition=aws_ec2.AmazonLinuxEdition.STANDARD, virtualization=aws_ec2.AmazonLinuxVirt.HVM, storage=aws_ec2.AmazonLinuxStorage.GENERAL_PURPOSE, cpu_type=aws_ec2.AmazonLinuxCpuType.X86_64) KAFKA_CLIENT_EC2_SG_NAME = 'kafka-client-ec2-sg-{}'.format(''.join( random.sample((string.ascii_lowercase), k=5))) sg_kafka_client_ec2_instance = aws_ec2.SecurityGroup( self, 'KafkaClientEC2InstanceSG', vpc=vpc, allow_all_outbound=True, description='security group for Kafka Client EC2 Instance', security_group_name=KAFKA_CLIENT_EC2_SG_NAME) cdk.Tags.of(sg_kafka_client_ec2_instance).add( 'Name', KAFKA_CLIENT_EC2_SG_NAME) sg_kafka_client_ec2_instance.add_ingress_rule( peer=aws_ec2.Peer.ipv4("0.0.0.0/0"), connection=aws_ec2.Port.tcp(22)) kafka_client_ec2_instance_role = aws_iam.Role( self, 'KafkaClientEC2InstanceRole', role_name='{}-KafkaClientEC2InstanceRole'.format(self.stack_name), assumed_by=aws_iam.ServicePrincipal('ec2.amazonaws.com'), managed_policies=[ aws_iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonSSMManagedInstanceCore'), aws_iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonMSKReadOnlyAccess') ]) msk_client_ec2_instance = aws_ec2.Instance( self, 'KafkaClientEC2Instance', instance_type=aws_ec2.InstanceType.of( instance_class=aws_ec2.InstanceClass.BURSTABLE2, instance_size=aws_ec2.InstanceSize.MICRO), machine_image=amzn_linux, vpc=vpc, availability_zone=vpc.select_subnets( subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT). availability_zones[0], instance_name='KafkaClientInstance', role=kafka_client_ec2_instance_role, security_group=sg_kafka_client_ec2_instance, vpc_subnets=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC)) msk_client_ec2_instance.add_security_group(sg_use_msk) commands = ''' yum update -y yum install python3.7 -y yum install java-1.8.0-openjdk-devel -y cd /home/ec2-user echo "export PATH=.local/bin:$PATH" >> .bash_profile mkdir -p opt cd opt wget https://archive.apache.org/dist/kafka/2.2.1/kafka_2.12-2.2.1.tgz tar -xzf kafka_2.12-2.2.1.tgz ln -nsf kafka_2.12-2.2.1 kafka cd /home/ec2-user wget https://bootstrap.pypa.io/get-pip.py su -c "python3.7 get-pip.py --user" -s /bin/sh ec2-user su -c "/home/ec2-user/.local/bin/pip3 install boto3 --user" -s /bin/sh ec2-user chown -R ec2-user ./opt chgrp -R ec2-user ./opt ''' msk_client_ec2_instance.user_data.add_commands(commands) cdk.CfnOutput(self, 'StackName', value=self.stack_name, export_name='StackName') cdk.CfnOutput(self, 'VpcId', value=vpc.vpc_id, export_name='VpcId') cdk.CfnOutput(self, 'MSKSecurityGroupID', value=sg_msk_cluster.security_group_id, export_name='MSKSecurityGroupID') cdk.CfnOutput(self, 'KafkaClientSecurityGroupID', value=sg_use_msk.security_group_id, export_name='KafkaClientSecurityGroupID') cdk.CfnOutput(self, 'MSKClusterArn', value=msk_cluster.ref, export_name='MSKClusterArn') cdk.CfnOutput(self, 'KafkaClientEC2InstancePublicDNS', value=msk_client_ec2_instance.instance_public_dns_name, export_name='KafkaClientEC2InstancePublicDNS')