def __init__( self, scope: core.Construct, id: str, cluster_name: str, **kwargs, ) -> None: super().__init__(scope, id, **kwargs) # EKS admin role self._clusterAdminRole = iam.Role( self, 'clusterAdmin', assumed_by=iam.AccountRootPrincipal()) self._clusterAdminRole.add_to_policy( iam.PolicyStatement( resources=["*"], actions=[ "eks:Describe*", "eks:List*", "eks:AccessKubernetesApi", "ssm:GetParameter", "iam:ListRoles" ], )) core.Tags.of(self._clusterAdminRole).add(key='eks/%s/type' % cluster_name, value='admin-role') # Managed Node Group Instance Role _managed_node_managed_policies = ( iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonEKSWorkerNodePolicy'), iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonEKS_CNI_Policy'), iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonEC2ContainerRegistryReadOnly'), iam.ManagedPolicy.from_aws_managed_policy_name( 'CloudWatchAgentServerPolicy'), ) self._managed_node_role = iam.Role( self, 'NodeInstance-Role', path='/', assumed_by=iam.ServicePrincipal('ec2.amazonaws.com'), managed_policies=list(_managed_node_managed_policies), ) # Override Cfn Nag rule scan.suppress_cfnnag_rule( 'W12', 'by default the role has * resource', self._clusterAdminRole.node.find_child( 'DefaultPolicy').node.default_child)
def add_distribution(scope: core.Construct, id: str, alb_dns_name: str, port: int, logbucket: s3.IBucket) -> cf.IDistribution: load_balancer_arn = core.Fn.get_att(alb_dns_name, "DNSName") security_group_id = core.Fn.get_att(alb_dns_name, "SecurityGroups") alb2 = alb.ApplicationLoadBalancer.from_application_load_balancer_attributes( scope, id, load_balancer_arn=load_balancer_arn.to_string(), security_group_id=security_group_id.to_string(), load_balancer_dns_name=alb_dns_name) _origin = origins.LoadBalancerV2Origin( alb2, http_port=port, protocol_policy=cf.OriginProtocolPolicy.HTTP_ONLY) dist = cf.Distribution( scope, "CF-" + id, default_behavior={ "origin": _origin, "allowed_methods": cf.AllowedMethods.ALLOW_ALL, "cache_policy": cf.CachePolicy.CACHING_DISABLED, "origin_request_policy": cf.OriginRequestPolicy.ALL_VIEWER, "viewer_protocol_policy": cf.ViewerProtocolPolicy.REDIRECT_TO_HTTPS }, minimum_protocol_version=cf.SecurityPolicyProtocol.TLS_V1_2_2019, enable_logging=True, log_bucket=logbucket) # Override Cfn_Nag rule for Cloudfront TLS-1.2 (https://github.com/stelligent/cfn_nag/issues/384) scan.suppress_cfnnag_rule( 'W70', 'the distribution uses CloudFront domain name and automatically sets the policy to TLSv1', dist.node.default_child) return dist.distribution_domain_name
def __init__( self, scope: core.Construct, id: str, codebucket: s3.IBucket, **kwargs, ) -> None: super().__init__(scope, id, **kwargs) # 1. Create ECR repositories self.ecr_repo = ecr.Repository( self, 'ECRRepo', image_scan_on_push=True, removal_policy=core.RemovalPolicy.DESTROY) # 2. Setup deployment CI/CD to deploy docker image to ECR pipeline = codepipeline.Pipeline(self, "Pipeline", pipeline_name='BuildArcDockerImage', artifact_bucket=codebucket) image_builder = codebuild.PipelineProject( self, 'DockerBuild', project_name='BuildArcDockerImage', build_spec=codebuild.BuildSpec.from_source_filename( 'buildspec.yaml'), environment=dict( build_image=codebuild.LinuxBuildImage.AMAZON_LINUX_2_3, privileged=True), environment_variables={ 'REPO_ECR': codebuild.BuildEnvironmentVariable( value=self.ecr_repo.repository_uri), }, description='Pipeline for docker build', timeout=core.Duration.minutes(60)) image_builder.apply_removal_policy(core.RemovalPolicy.DESTROY) # 3. grant permissions for the CI/CD codebucket.grant_read_write(pipeline.role) codebucket.grant_read_write(image_builder) self.ecr_repo.grant_pull_push(image_builder) source_output = codepipeline.Artifact('src') pipeline.add_stage( stage_name='Source', actions=[ codepipeline_actions.S3SourceAction( action_name='S3Trigger', bucket=codebucket, bucket_key='app_code/ecr_build_src.zip', output=source_output, trigger=codepipeline_actions.S3Trigger.POLL), ]) pipeline.add_stage(stage_name='Build', actions=[ codepipeline_actions.CodeBuildAction( action_name='DockerImageBuild', input=source_output, project=image_builder) ]) # Override Cfn Nag warning W12: IAM policy should not allow * resource scan.suppress_cfnnag_rule( 'W12', 'the role for action of ecr:GetAuthorizationToken requires * resource', image_builder.role.node.find_child( 'DefaultPolicy').node.default_child) image_builder.role.node.find_child( 'DefaultPolicy' ).node.default_child.add_metadata( 'cfn_nag', { "rules_to_suppress": [{ "id": "W12", "reason": "the role for action of ecr:GetAuthorizationToken requires * resource" }, { "id": "W76", "reason": "the IAM policy is complex, need to be higher than 25" }] })
def __init__(self, scope: core.Construct, id: str, eks_cluster: ICluster, login_name: str, code_bucket: str, datalake_bucket: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) source_dir = os.path.split(os.environ['VIRTUAL_ENV'])[0] + '/source' # //******************************************************************************************// # //************************ SETUP PERMISSION FOR ARC SPARK JOBS ****************************// # //******* create k8s namespace, service account, and IAM role for service account ********// # //***************************************************************************************// # create k8s namespace etl_ns = eks_cluster.add_manifest( 'SparkNamespace', { "apiVersion": "v1", "kind": "Namespace", "metadata": { "name": "spark", "labels": { "name": "spark" } } }) jupyter_ns = eks_cluster.add_manifest( 'jhubNamespace', { "apiVersion": "v1", "kind": "Namespace", "metadata": { "name": "jupyter", "labels": { "name": "spark" } } }) # create k8s service account self._etl_sa = eks_cluster.add_service_account('ETLSa', name='arcjob', namespace='spark') self._etl_sa.node.add_dependency(etl_ns) _etl_rb = KubernetesManifest( self, 'ETLRoleBinding', cluster=eks_cluster, manifest=load_yaml_replace_var_local( source_dir + '/app_resources/etl-rbac.yaml', fields={"{{MY_SA}}": self._etl_sa.service_account_name}, multi_resource=True)) _etl_rb.node.add_dependency(self._etl_sa) self._jupyter_sa = eks_cluster.add_service_account('jhubServiceAcct', name=login_name, namespace='jupyter') self._jupyter_sa.node.add_dependency(jupyter_ns) # Associate AWS IAM role to K8s Service Account datalake_bucket = code_bucket if not datalake_bucket.strip( ) else datalake_bucket _bucket_setting = { "{{codeBucket}}": code_bucket, "{{datalakeBucket}}": datalake_bucket } _etl_iam = load_yaml_replace_var_local( source_dir + '/app_resources/etl-iam-role.yaml', fields=_bucket_setting) for statmnt in _etl_iam: self._etl_sa.add_to_principal_policy( iam.PolicyStatement.from_json(statmnt)) self._jupyter_sa.add_to_principal_policy( iam.PolicyStatement.from_json(statmnt)) # # //*************************************************************************************// # # //******************** SETUP PERMISSION FOR NATIVE SPARK JOBS **********************// # # //***********************************************************************************// self._spark_sa = eks_cluster.add_service_account('NativeSparkSa', name='nativejob', namespace='spark') self._spark_sa.node.add_dependency(etl_ns) _spark_rb = eks_cluster.add_manifest( 'sparkRoleBinding', load_yaml_replace_var_local( source_dir + '/app_resources/native-spark-rbac.yaml', fields={"{{MY_SA}}": self._spark_sa.service_account_name})) _spark_rb.node.add_dependency(self._spark_sa) _native_spark_iam = load_yaml_replace_var_local( source_dir + '/app_resources/native-spark-iam-role.yaml', fields=_bucket_setting) for statmnt in _native_spark_iam: self._spark_sa.add_to_principal_policy( iam.PolicyStatement.from_json(statmnt)) # Override Cfn Nag warning W12: IAM policy should not allow * resource scan.suppress_cfnnag_rule( 'W12', 'by default the etl_sa role has * resource', self._etl_sa.role.node.find_child( 'DefaultPolicy').node.default_child) scan.suppress_cfnnag_rule( 'W12', 'by default the role spark_sa has * resource', self._spark_sa.role.node.find_child( 'DefaultPolicy').node.default_child) scan.suppress_cfnnag_rule( 'W12', 'by default the role jupyter_sa has * resource', self._jupyter_sa.role.node.find_child( 'DefaultPolicy').node.default_child)
def __init__(self, scope: core.Construct, id: str, eksname: str, solution_id: str, version: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.template_options.description = "(SO0141) SQL based ETL with Apache Spark on Amazon EKS. This solution provides a SQL based ETL option with a open-source declarative framework powered by Apache Spark." source_dir = os.path.split(os.environ['VIRTUAL_ENV'])[0] + '/source' # Cloudformation input params datalake_bucket = core.CfnParameter( self, "datalakebucket", type="String", description= "Your existing S3 bucket to be accessed by Jupyter Notebook and ETL job. Default: blank", default="") login_name = core.CfnParameter( self, "jhubuser", type="String", description="Your username login to jupyter hub", default="sparkoneks") # Auto-generate a user login in secrets manager key = kms.Key(self, 'KMSKey', removal_policy=core.RemovalPolicy.DESTROY, enable_key_rotation=True) key.add_alias("alias/secretsManager") jhub_secret = secmger.Secret( self, 'jHubPwd', generate_secret_string=secmger.SecretStringGenerator( exclude_punctuation=True, secret_string_template=json.dumps( {'username': login_name.value_as_string}), generate_string_key="password"), removal_policy=core.RemovalPolicy.DESTROY, encryption_key=key) # 1. a new bucket to store app code and logs self.app_s3 = S3AppCodeConst(self, 'appcode') # 2. push docker image to ECR via AWS CICD pipeline ecr_image = DockerPipelineConstruct(self, 'image', self.app_s3.artifact_bucket) ecr_image.node.add_dependency(self.app_s3) core.CfnOutput(self, 'IMAGE_URI', value=ecr_image.image_uri) # 3. EKS base infrastructure network_sg = NetworkSgConst(self, 'network-sg', eksname, self.app_s3.code_bucket) iam = IamConst(self, 'iam_roles', eksname) eks_cluster = EksConst(self, 'eks_cluster', eksname, network_sg.vpc, iam.managed_node_role, iam.admin_role) EksSAConst(self, 'eks_sa', eks_cluster.my_cluster, jhub_secret) base_app = EksBaseAppConst(self, 'eks_base_app', eks_cluster.my_cluster) # 4. Spark app access control app_security = SparkOnEksSAConst(self, 'spark_service_account', eks_cluster.my_cluster, login_name.value_as_string, self.app_s3.code_bucket, datalake_bucket.value_as_string) app_security.node.add_dependency(base_app.secret_created) # 5. Install Arc Jupyter notebook in EKS jhub_install = eks_cluster.my_cluster.add_helm_chart( 'JHubChart', chart='jupyterhub', repository='https://jupyterhub.github.io/helm-chart', release='jhub', version='0.11.1', namespace='jupyter', create_namespace=False, values=load_yaml_replace_var_local( source_dir + '/app_resources/jupyter-values.yaml', fields={ "{{codeBucket}}": self.app_s3.code_bucket, "{{region}}": core.Aws.REGION })) jhub_install.node.add_dependency(app_security) # EKS get Jupyter login dynamically from secrets manager name_parts = core.Fn.split('-', jhub_secret.secret_name) name_no_suffix = core.Fn.join( '-', [core.Fn.select(0, name_parts), core.Fn.select(1, name_parts)]) config_hub = eks.KubernetesManifest( self, 'JHubConfig', cluster=eks_cluster.my_cluster, manifest=load_yaml_replace_var_local( source_dir + '/app_resources/jupyter-config.yaml', fields={ "{{MY_SA}}": app_security.jupyter_sa, "{{REGION}}": core.Aws.REGION, "{{SECRET_NAME}}": name_no_suffix }, multi_resource=True)) config_hub.node.add_dependency(jhub_install) # 6. Install ETL orchestrator - Argo in EKS # can be replaced by other workflow tool, eg. Airflow argo_install = eks_cluster.my_cluster.add_helm_chart( 'ARGOChart', chart='argo-workflows', repository='https://argoproj.github.io/argo-helm', release='argo', version='0.1.4', namespace='argo', create_namespace=True, values=load_yaml_local(source_dir + '/app_resources/argo-values.yaml')) argo_install.node.add_dependency(config_hub) # Create argo workflow template for Spark with T-shirt size submit_tmpl = eks_cluster.my_cluster.add_manifest( 'SubmitSparkWrktmpl', load_yaml_local(source_dir + '/app_resources/spark-template.yaml')) submit_tmpl.node.add_dependency(argo_install) # 7. (OPTIONAL) retrieve ALB DNS Name to enable CloudFront in the nested stack. # It is used to serve HTTPS requests with its default domain name. # Recommend to issue your own TLS certificate, and delete the CF components. self._jhub_alb = eks.KubernetesObjectValue( self, 'jhubALB', cluster=eks_cluster.my_cluster, json_path='..status.loadBalancer.ingress[0].hostname', object_type='ingress.networking', object_name='jupyterhub', object_namespace='jupyter', timeout=core.Duration.minutes(10)) self._jhub_alb.node.add_dependency(config_hub) self._argo_alb = eks.KubernetesObjectValue( self, 'argoALB', cluster=eks_cluster.my_cluster, json_path='..status.loadBalancer.ingress[0].hostname', object_type='ingress.networking', object_name='argo-argo-workflows-server', object_namespace='argo', timeout=core.Duration.minutes(10)) self._argo_alb.node.add_dependency(argo_install) # 8. (OPTIONAL) Send solution metrics to AWS # turn it off from the CloudFormation mapping section if prefer. send_metrics = solution_metrics.SendAnonymousData( self, "SendMetrics", network_sg.vpc, self.app_s3.artifact_bucket, self.app_s3.s3_deploy_contrust, metrics={ "Solution": solution_id, "Region": core.Aws.REGION, "SolutionVersion": version, "UUID": "MY_UUID", "UseDataLakeBucket": "True" if not datalake_bucket.value_as_string else "False", "UseAWSCICD": "True" if ecr_image.image_uri else "False", "NoAZs": len(network_sg.vpc.availability_zones) }) send_metrics.node.add_dependency(self.app_s3.s3_deploy_contrust) # 9. (OPTIONAL) Override the cfn Nag rules for AWS Solution CICD deployment # remove the section if your CI/CD pipeline doesn't use the cfn_nag utility to validate the CFN. k8s_ctl_node = self.node.find_child( '@aws-cdk--aws-eks.KubectlProvider') cluster_resrc_node = self.node.find_child( '@aws-cdk--aws-eks.ClusterResourceProvider') scan.suppress_cfnnag_rule( 'W12', 'by default the role has * resource', self.node.find_child('eks_cluster').node.find_child('EKS').node. default_child.node.find_child('CreationRole').node.find_child( 'DefaultPolicy').node.default_child) scan.suppress_cfnnag_rule( 'W11', 'by default the role has * resource', self.node.find_child( 'Custom::AWSCDKOpenIdConnectProviderCustomResourceProvider'). node.find_child('Role')) scan.suppress_lambda_cfnnag_rule( k8s_ctl_node.node.find_child('Handler').node.default_child) scan.suppress_lambda_cfnnag_rule( k8s_ctl_node.node.find_child('Provider').node.find_child( 'framework-onEvent').node.default_child) scan.suppress_lambda_cfnnag_rule( self.node.find_child( 'Custom::CDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756C'). node.default_child) # scan.suppress_lambda_cfnnag_rule(self.node.find_child('Custom::S3AutoDeleteObjectsCustomResourceProvider').node.find_child('Handler')) scan.suppress_lambda_cfnnag_rule( self.node.find_child( 'Custom::AWSCDKOpenIdConnectProviderCustomResourceProvider'). node.find_child('Handler')) scan.suppress_lambda_cfnnag_rule( self.node.find_child('AWSCDKCfnUtilsProviderCustomResourceProvider' ).node.find_child('Handler')) scan.suppress_lambda_cfnnag_rule( cluster_resrc_node.node.find_child( 'OnEventHandler').node.default_child) scan.suppress_lambda_cfnnag_rule( cluster_resrc_node.node.find_child( 'IsCompleteHandler').node.default_child) scan.suppress_lambda_cfnnag_rule( cluster_resrc_node.node.find_child('Provider').node.find_child( 'framework-isComplete').node.default_child) scan.suppress_lambda_cfnnag_rule( cluster_resrc_node.node.find_child('Provider').node.find_child( 'framework-onTimeout').node.default_child) scan.suppress_lambda_cfnnag_rule( cluster_resrc_node.node.find_child('Provider').node.find_child( 'framework-onEvent').node.default_child) scan.suppress_network_cfnnag_rule( self.node.find_child('eks_cluster').node.find_child('EKS').node. find_child('ControlPlaneSecurityGroup').node.default_child) scan.suppress_lambda_cfnnag_rule( self.node.find_child('SendMetrics').node.find_child( 'LambdaProvider').node.find_child( 'framework-onEvent').node.default_child) scan.suppress_network_cfnnag_rule( self.node.find_child('SendMetrics').node.find_child( 'LambdaProvider').node.find_child('framework-onEvent').node. find_child('SecurityGroup').node.default_child) scan.suppress_lambda_cfnnag_rule( self.node.find_child( 'SingletonLambda75248a819138468c9ba1bca6c7137599').node. default_child) scan.suppress_network_cfnnag_rule( self.node.find_child( 'SingletonLambda75248a819138468c9ba1bca6c7137599').node. find_child('SecurityGroup').node.default_child)
def __init__(self, scope: core.Construct, id: str, eksname: str, codebucket: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # //*************************************************// # //******************* NETWORK ********************// # //************************************************// # create VPC self._vpc = ec2.Vpc(self, 'eksVpc', max_azs=2, nat_gateways=1) core.Tags.of(self._vpc).add('Name', eksname + 'EksVpc') self._log_bucket = s3.Bucket.from_bucket_name(self, 'vpc_logbucket', codebucket) self._vpc.add_flow_log("FlowLogCloudWatch", destination=ec2.FlowLogDestination.to_s3( self._log_bucket, 'vpcRejectlog/'), traffic_type=ec2.FlowLogTrafficType.REJECT) # VPC endpoint security group self._vpc_endpoint_sg = ec2.SecurityGroup( self, 'EndpointSg', vpc=self._vpc, description='Security Group for Endpoint', ) self._vpc_endpoint_sg.add_ingress_rule( ec2.Peer.ipv4(self._vpc.vpc_cidr_block), ec2.Port.tcp(port=443)) core.Tags.of(self._vpc_endpoint_sg).add('Name', 'SparkOnEKS-VPCEndpointSg') # Add VPC endpoint self._vpc.add_gateway_endpoint( "S3GatewayEndpoint", service=ec2.GatewayVpcEndpointAwsService.S3, subnets=[ ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE) ]) self._vpc.add_interface_endpoint( "EcrDockerEndpoint", service=ec2.InterfaceVpcEndpointAwsService.ECR_DOCKER, security_groups=[self._vpc_endpoint_sg]) self._vpc.add_interface_endpoint( "CWLogsEndpoint", service=ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS, security_groups=[self._vpc_endpoint_sg]) self._vpc.add_interface_endpoint( "AthenaEndpoint", service=ec2.InterfaceVpcEndpointAwsService.ATHENA, security_groups=[self._vpc_endpoint_sg]) self._vpc.add_interface_endpoint( "KMSEndpoint", service=ec2.InterfaceVpcEndpointAwsService.KMS, security_groups=[self._vpc_endpoint_sg]) # Override Cfn_Nag rule for AWS Solution CICD validation for subnet in self._vpc.public_subnets: scan.suppress_cfnnag_rule( 'W33', 'a public facing ALB is required and ingress from the internet should be permitted.', subnet.node.default_child) self._vpc_endpoint_sg.node.default_child.add_metadata( 'cfn_nag', { "rules_to_suppress": [{ "id": "W40", "reason": "Egress IP Protocol of -1 is default and generally considered OK" }, { "id": "W5", "reason": "Security Groups with cidr open considered OK" }] })