def __init__( self, scope: core.Construct, id: str, vpc_stack, logstash_ec2=True, logstash_fargate=True, **kwargs, ) -> None: super().__init__(scope, id, **kwargs) # get s3 bucket name s3client = boto3.client("s3") s3_bucket_list = s3client.list_buckets() s3_bucket_name = "" for bkt in s3_bucket_list["Buckets"]: try: bkt_tags = s3client.get_bucket_tagging( Bucket=bkt["Name"])["TagSet"] for keypairs in bkt_tags: if (keypairs["Key"] == "aws:cloudformation:stack-name" and keypairs["Value"] == "elkk-athena"): s3_bucket_name = bkt["Name"] except ClientError as err: if err.response["Error"]["Code"] in [ "NoSuchTagSet", "NoSuchBucket" ]: pass else: print(f"Unexpected error: {err}") # get elastic endpoint esclient = boto3.client("es") es_domains = esclient.list_domain_names() try: es_domain = [ dom["DomainName"] for dom in es_domains["DomainNames"] if "elkk-" in dom["DomainName"] ][0] es_endpoint = esclient.describe_elasticsearch_domain( DomainName=es_domain) es_endpoint = es_endpoint["DomainStatus"]["Endpoints"]["vpc"] except IndexError: es_endpoint = "" # assets for logstash stack logstash_yml = assets.Asset(self, "logstash_yml", path=os.path.join(dirname, "logstash.yml")) logstash_repo = assets.Asset(self, "logstash_repo", path=os.path.join(dirname, "logstash.repo")) # update conf file to .asset # kafka brokerstring does not need reformatting logstash_conf_asset = file_updated( os.path.join(dirname, "logstash.conf"), { "$s3_bucket": s3_bucket_name, "$es_endpoint": es_endpoint, "$kafka_brokers": kafka_get_brokers(), "$elkk_region": os.environ["CDK_DEFAULT_REGION"], }, ) logstash_conf = assets.Asset( self, "logstash.conf", path=logstash_conf_asset, ) # logstash security group logstash_security_group = ec2.SecurityGroup( self, "logstash_security_group", vpc=vpc_stack.get_vpc, description="logstash security group", allow_all_outbound=True, ) core.Tags.of(logstash_security_group).add("project", constants["PROJECT_TAG"]) core.Tags.of(logstash_security_group).add("Name", "logstash_sg") # Open port 22 for SSH logstash_security_group.add_ingress_rule( ec2.Peer.ipv4(f"{external_ip}/32"), ec2.Port.tcp(22), "from own public ip", ) # get security group for kafka ec2client = boto3.client("ec2") security_groups = ec2client.describe_security_groups(Filters=[{ "Name": "tag-value", "Values": [constants["PROJECT_TAG"]] }], ) # if kafka sg does not exist ... don't add it try: kafka_sg_id = [ sg["GroupId"] for sg in security_groups["SecurityGroups"] if "kafka security group" in sg["Description"] ][0] kafka_security_group = ec2.SecurityGroup.from_security_group_id( self, "kafka_security_group", security_group_id=kafka_sg_id) # let in logstash kafka_security_group.connections.allow_from( logstash_security_group, ec2.Port.all_traffic(), "from logstash", ) except IndexError: # print("kafka_sg_id and kafka_security_group not found") pass # get security group for elastic try: elastic_sg_id = [ sg["GroupId"] for sg in security_groups["SecurityGroups"] if "elastic security group" in sg["Description"] ][0] elastic_security_group = ec2.SecurityGroup.from_security_group_id( self, "elastic_security_group", security_group_id=elastic_sg_id) # let in logstash elastic_security_group.connections.allow_from( logstash_security_group, ec2.Port.all_traffic(), "from logstash", ) except IndexError: pass # elastic policy access_elastic_policy = iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ "es:ListDomainNames", "es:DescribeElasticsearchDomain", "es:ESHttpPut", ], resources=["*"], ) # kafka policy access_kafka_policy = iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=["kafka:ListClusters", "kafka:GetBootstrapBrokers"], resources=["*"], ) # s3 policy access_s3_policy = iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=["s3:ListBucket", "s3:PutObject"], resources=["*"], ) # create the Logstash instance if logstash_ec2: # userdata for Logstash logstash_userdata = user_data_init( log_group_name="elkk/logstash/instance") # create the instance logstash_instance = ec2.Instance( self, "logstash_client", instance_type=ec2.InstanceType(constants["LOGSTASH_INSTANCE"]), machine_image=ec2.AmazonLinuxImage( generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2), vpc=vpc_stack.get_vpc, vpc_subnets=SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), key_name=constants["KEY_PAIR"], security_group=logstash_security_group, user_data=logstash_userdata, ) core.Tag.add(logstash_instance, "project", constants["PROJECT_TAG"]) # add access to the file assets logstash_yml.grant_read(logstash_instance) logstash_repo.grant_read(logstash_instance) logstash_conf.grant_read(logstash_instance) # add permissions to instance logstash_instance.add_to_role_policy( statement=access_elastic_policy) logstash_instance.add_to_role_policy(statement=access_kafka_policy) logstash_instance.add_to_role_policy(statement=access_s3_policy) # add log permissions instance_add_log_permissions(logstash_instance) # add commands to the userdata logstash_userdata.add_commands( # get setup assets files f"aws s3 cp s3://{logstash_yml.s3_bucket_name}/{logstash_yml.s3_object_key} /home/ec2-user/logstash.yml", f"aws s3 cp s3://{logstash_repo.s3_bucket_name}/{logstash_repo.s3_object_key} /home/ec2-user/logstash.repo", f"aws s3 cp s3://{logstash_conf.s3_bucket_name}/{logstash_conf.s3_object_key} /home/ec2-user/logstash.conf", # install java "amazon-linux-extras install java-openjdk11 -y", # install git "yum install git -y", # install pip "yum install python-pip -y", # get elastic output to es "git clone https://github.com/awslabs/logstash-output-amazon_es.git /home/ec2-user/logstash-output-amazon_es", # logstash "rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch", # move logstash repo file "mv -f /home/ec2-user/logstash.repo /etc/yum.repos.d/logstash.repo", # get to the yum "yum install logstash -y", # add user to logstash group "usermod -a -G logstash ec2-user", # move logstash.yml to final location "mv -f /home/ec2-user/logstash.yml /etc/logstash/logstash.yml", # move logstash.conf to final location "mv -f /home/ec2-user/logstash.conf /etc/logstash/conf.d/logstash.conf", # move plugin "mkdir /usr/share/logstash/plugins", "mv -f /home/ec2-user/logstash-output-amazon_es /usr/share/logstash/plugins/logstash-output-amazon_es", # update gemfile """sed -i '5igem "logstash-output-amazon_es", :path => "/usr/share/logstash/plugins/logstash-output-amazon_es"' /usr/share/logstash/Gemfile""", # update ownership "chown -R logstash:logstash /etc/logstash", # start logstash "systemctl start logstash.service", ) # add the signal logstash_userdata.add_signal_on_exit_command( resource=logstash_instance) # add creation policy for instance logstash_instance.instance.cfn_options.creation_policy = core.CfnCreationPolicy( resource_signal=core.CfnResourceSignal(count=1, timeout="PT10M")) # fargate for logstash if logstash_fargate: # cloudwatch log group for containers logstash_logs_containers = logs.LogGroup( self, "logstash_logs_containers", log_group_name="elkk/logstash/container", removal_policy=core.RemovalPolicy.DESTROY, retention=logs.RetentionDays.ONE_WEEK, ) # docker image for logstash logstash_image_asset = ecr_assets.DockerImageAsset( self, "logstash_image_asset", directory=dirname # , file="Dockerfile" ) # create the fargate cluster logstash_cluster = ecs.Cluster(self, "logstash_cluster", vpc=vpc_stack.get_vpc) core.Tag.add(logstash_cluster, "project", constants["PROJECT_TAG"]) # the task logstash_task = ecs.FargateTaskDefinition( self, "logstash_task", cpu=512, memory_limit_mib=1024, ) # add container to the task logstash_task.add_container( logstash_image_asset.source_hash, image=ecs.ContainerImage.from_docker_image_asset( logstash_image_asset), logging=ecs.LogDrivers.aws_logs( stream_prefix="elkk", log_group=logstash_logs_containers), ) # add permissions to the task logstash_task.add_to_task_role_policy(access_s3_policy) logstash_task.add_to_task_role_policy(access_elastic_policy) # the service logstash_service = (ecs.FargateService( self, "logstash_service", cluster=logstash_cluster, task_definition=logstash_task, security_group=logstash_security_group, deployment_controller=ecs.DeploymentController( type=ecs.DeploymentControllerType.ECS), ).auto_scale_task_count( min_capacity=3, max_capacity=10).scale_on_cpu_utilization( "logstash_scaling", target_utilization_percent=75, scale_in_cooldown=core.Duration.seconds(60), scale_out_cooldown=core.Duration.seconds(60), ))
def __init__( self, scope: core.Construct, id: str, vpc_stack, kafka_stack, **kwargs ) -> None: super().__init__(scope, id, **kwargs) # log generator asset log_generator_py = assets.Asset( self, "log_generator", path=os.path.join(dirname, "log_generator.py") ) # log generator requirements.txt asset log_generator_requirements_txt = assets.Asset( self, "log_generator_requirements_txt", path=os.path.join(dirname, "log_generator_requirements.txt"), ) # get kakfa brokers kafka_brokers = f'''"{kafka_get_brokers().replace(",", '", "')}"''' # update filebeat.yml to .asset filebeat_yml_asset = file_updated( os.path.join(dirname, "filebeat.yml"), {"$kafka_brokers": kafka_brokers}, ) filebeat_yml = assets.Asset(self, "filebeat_yml", path=filebeat_yml_asset) elastic_repo = assets.Asset( self, "elastic_repo", path=os.path.join(dirname, "elastic.repo") ) # userdata for Filebeat fb_userdata = user_data_init(log_group_name="elkk/filebeat/instance") # instance for Filebeat fb_instance = ec2.Instance( self, "filebeat_client", instance_type=ec2.InstanceType(constants["FILEBEAT_INSTANCE"]), machine_image=ec2.AmazonLinuxImage( generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2 ), vpc=vpc_stack.get_vpc, vpc_subnets={"subnet_type": ec2.SubnetType.PUBLIC}, key_name=constants["KEY_PAIR"], security_group=kafka_stack.get_kafka_client_security_group, user_data=fb_userdata, ) core.Tag.add(fb_instance, "project", constants["PROJECT_TAG"]) # create policies for EC2 to connect to kafka access_kafka_policy = iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=["kafka:ListClusters", "kafka:GetBootstrapBrokers",], resources=["*"], ) # add the role permissions fb_instance.add_to_role_policy(statement=access_kafka_policy) # add log permissions instance_add_log_permissions(fb_instance) # add access to the file asset filebeat_yml.grant_read(fb_instance) elastic_repo.grant_read(fb_instance) log_generator_py.grant_read(fb_instance) log_generator_requirements_txt.grant_read(fb_instance) # add commands to the userdata fb_userdata.add_commands( # get setup assets files f"aws s3 cp s3://{filebeat_yml.s3_bucket_name}/{filebeat_yml.s3_object_key} /home/ec2-user/filebeat.yml", f"aws s3 cp s3://{elastic_repo.s3_bucket_name}/{elastic_repo.s3_object_key} /home/ec2-user/elastic.repo", f"aws s3 cp s3://{log_generator_py.s3_bucket_name}/{log_generator_py.s3_object_key} /home/ec2-user/log_generator.py", f"aws s3 cp s3://{log_generator_requirements_txt.s3_bucket_name}/{log_generator_requirements_txt.s3_object_key} /home/ec2-user/requirements.txt", # get python3 "yum install python3 -y", # get pip "yum install python-pip -y", # make log generator executable "chmod +x /home/ec2-user/log_generator.py", # get log generator requirements "python3 -m pip install -r /home/ec2-user/requirements.txt", # Filebeat "rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch", # move Filebeat repo file "mv -f /home/ec2-user/elastic.repo /etc/yum.repos.d/elastic.repo", # install Filebeat "yum install filebeat -y", # move filebeat.yml to final location "mv -f /home/ec2-user/filebeat.yml /etc/filebeat/filebeat.yml", # update log generator ownership "chown -R ec2-user:ec2-user /home/ec2-user", # start Filebeat "systemctl start filebeat", ) # add the signal fb_userdata.add_signal_on_exit_command(resource=fb_instance) # attach the userdata fb_instance.add_user_data(fb_userdata.render()) # add creation policy for instance fb_instance.instance.cfn_options.creation_policy = core.CfnCreationPolicy( resource_signal=core.CfnResourceSignal(count=1, timeout="PT10M") )