def add_helm_annotation(self, cluster, service_account):
"""
workaround to add helm role to service account
"""
return eks.KubernetesManifest(
self,
"ServiceAccountManifest",
cluster=cluster,
manifest=[{
"apiVersion": "v1",
"kind": "ServiceAccount",
"metadata": {
"name": service_account.service_account_name,
"namespace": service_account.service_account_namespace,
"labels": {
"app.kubernetes.io/name":
service_account.service_account_name,
"app.kubernetes.io/managed-by": "Helm",
},
"annotations": {
"eks.amazonaws.com/role-arn":
service_account.role.role_arn,
"meta.helm.sh/release-name":
service_account.service_account_name,
"meta.helm.sh/release-namespace":
service_account.service_account_namespace,
},
},
}],
)
def add_service_account(self, cluster, name, namespace):
"""
workaround to add helm role to service account
"""
# create role
conditions = core.CfnJson(
self,
'ConditionJson',
value={
"%s:aud" % cluster.cluster_open_id_connect_issuer:
"sts.amazonaws.com",
"%s:sub" % cluster.cluster_open_id_connect_issuer:
"system:serviceaccount:%s:%s" % (namespace, name),
},
)
principal = iam.OpenIdConnectPrincipal(
cluster.open_id_connect_provider).with_conditions({
"StringEquals":
conditions,
})
role = iam.Role(self, 'ServiceAccountRole', assumed_by=principal)
# create policy for the service account
statements = []
with open('backend/iam_policy.json') as f:
data = json.load(f)
for s in data["Statement"]:
statements.append(iam.PolicyStatement.from_json(s))
policy = iam.Policy(self, "LBControllerPolicy", statements=statements)
policy.attach_to_role(role)
return eks.KubernetesManifest(
self,
"ServiceAccount",
cluster=cluster,
manifest=[{
"apiVersion": "v1",
"kind": "ServiceAccount",
"metadata": {
"name": name,
"namespace": namespace,
"labels": {
"app.kubernetes.io/name": name,
"app.kubernetes.io/managed-by": "Helm",
},
"annotations": {
"eks.amazonaws.com/role-arn": role.role_arn,
"meta.helm.sh/release-name": name,
"meta.helm.sh/release-namespace": namespace,
},
},
}],
)
def install_calico(self):
# This produces an obnoxious diff on every subsequent run
# Using a helm chart does not, so we should switch to that
# However, we need to figure out how to get the helm chart
# accessible by the CDK lambda first. Not clear how to give
# s3 perms to it programmatically, and while ECR might be
# an option it also doesn't seem like there's a way to push
# the chart with existing api calls.
# Probably need to do some custom lambda thing.
for manifest in manifests:
filename = f"{manifest[0]}.yaml"
if isfile(filename):
with open(filename) as f:
manifest_text = f.read()
else:
manifest_text = requests_get(manifest[1]).text
loaded_manifests = [
yaml_safe_load(i)
for i in re_split("^---$", manifest_text, flags=MULTILINE) if i
]
crds = eks.KubernetesManifest(
self.scope,
"calico-crds",
cluster=self.eks_cluster,
manifest=[
crd for crd in loaded_manifests
if crd["kind"] == "CustomResourceDefinition"
],
)
non_crds = eks.KubernetesManifest(
self.scope,
"calico",
cluster=self.eks_cluster,
manifest=[
notcrd for notcrd in loaded_manifests
if notcrd["kind"] != "CustomResourceDefinition"
],
)
non_crds.node.add_dependency(crds)
def __init__(self,
scope,
eks_base_cluster,
git_user,
git_password,
git_repository,
git_branch="master"):
main_manifest = self._base_manifest(git_repository)
main_manifest.append(self._secret_manifest(git_user, git_password))
eks_manifest = eks.KubernetesManifest(scope=scope,
id="fluxcd-main-manifest-new",
cluster=eks_base_cluster,
manifest=main_manifest)
def __init__(self, scope: core.Construct, id: str, cluster, manifest,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.cluster = cluster
self.manifest = manifest
def backend_service(self, manifest):
labels = manifest['labels']
deployment = {
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {
"name": manifest['service_name'],
"namespace": "default"
},
"spec": {
"replicas": manifest['replicas'],
"selector": {
"matchLabels": labels
},
"strategy": {
"rollingUpdate": {
"maxSurge": "25%",
"maxUnavailable": "25%"
}
},
"template": {
"metadata": {
"labels": labels
},
"spec": {
"containers": [{
"name":
manifest['service_name'],
"image":
manifest['image'],
"ports": [{
"containerPort": manifest['port'],
"protocol": "TCP"
}]
}]
}
}
}
}
service = {
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": manifest['service_name'],
"namespace": "default"
},
"spec": {
"type": "LoadBalancer",
"ports": [{
"port": 80,
"targetPort": manifest['port']
}],
"selector": manifest['labels']
}
}
return deployment, service
def frontend_service(self, manifest):
labels = manifest['labels']
deployment = {
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {
"name": manifest['service_name'],
"namespace": "default"
},
"spec": {
"replicas": manifest['replicas'],
"selector": {
"matchLabels": labels
},
"strategy": {
"rollingUpdate": {
"maxSurge": "25%",
"maxUnavailable": "25%"
}
},
"template": {
"metadata": {
"labels": labels
},
"spec": {
"containers": [{
"name":
manifest['service_name'],
"image":
manifest['image'],
"ports": [{
"containerPort": manifest['port'],
"protocol": "TCP"
}],
"env":
manifest['env']
}],
}
}
}
}
service = {
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": manifest['service_name'],
"namespace": "default"
},
"spec": {
#"type": "ClusterIP",
"ports": [{
"port": 80,
"targetPort": manifest['port'],
"protocol": "TCP"
}],
"selector":
manifest['labels']
}
}
return deployment, service
fargate_eks_ALB_ingress_manifest = {
"apiVersion": "extensions/v1beta1",
"kind": "Ingress",
"metadata": {
"name": "fargate-eks-alb-ingress",
"namespace": "default",
"annotations": {
"kubernetes.io/ingress.class": "alb",
"alb.ingress.kubernetes.io/scheme": "internet-facing",
"alb.ingress.kubernetes.io/target-type": "ip",
#"alb.ingress.kubernetes.io/listen-ports": "[{"HTTP": 80}, {"HTTP":5000}]"
},
"labels": manifest['labels'],
},
"spec": {
"rules": [{
"http": {
"paths": [{
"path": "/*",
"backend": {
"serviceName": "flask",
"servicePort": 80
},
}],
},
}],
},
}
if self.manifest['service_type'] == 'backend':
deployment_manifest, service_manifest = backend_service(
self, self.manifest)
elif self.manifest['service_type'] == 'frontend':
deployment_manifest, service_manifest = frontend_service(
self, self.manifest)
aws_eks.KubernetesManifest(self,
"DeploymentServiceManifest",
cluster=self.cluster,
manifest=[
deployment_manifest, service_manifest,
fargate_eks_ALB_ingress_manifest
])
def __init__(self, scope: core.Construct, id: str, cluster,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.cluster = cluster
iam_policy = aws_iam.PolicyStatement(
actions=[
"acm:DescribeCertificate", "acm:ListCertificates",
"acm:GetCertificate", "ec2:AuthorizeSecurityGroupIngress",
"ec2:CreateSecurityGroup", "ec2:CreateTags", "ec2:DeleteTags",
"ec2:DeleteSecurityGroup", "ec2:DescribeAccountAttributes",
"ec2:DescribeAddresses", "ec2:DescribeInstances",
"ec2:DescribeInstanceStatus", "ec2:DescribeInternetGateways",
"ec2:DescribeNetworkInterfaces", "ec2:DescribeSecurityGroups",
"ec2:DescribeSubnets", "ec2:DescribeTags", "ec2:DescribeVpcs",
"ec2:ModifyInstanceAttribute",
"ec2:ModifyNetworkInterfaceAttribute",
"ec2:RevokeSecurityGroupIngress",
"elasticloadbalancing:AddListenerCertificates",
"elasticloadbalancing:AddTags",
"elasticloadbalancing:CreateListener",
"elasticloadbalancing:CreateLoadBalancer",
"elasticloadbalancing:CreateRule",
"elasticloadbalancing:CreateTargetGroup",
"elasticloadbalancing:DeleteListener",
"elasticloadbalancing:DeleteLoadBalancer",
"elasticloadbalancing:DeleteRule",
"elasticloadbalancing:DeleteTargetGroup",
"elasticloadbalancing:DeregisterTargets",
"elasticloadbalancing:DescribeListenerCertificates",
"elasticloadbalancing:DescribeListeners",
"elasticloadbalancing:DescribeLoadBalancers",
"elasticloadbalancing:DescribeLoadBalancerAttributes",
"elasticloadbalancing:DescribeRules",
"elasticloadbalancing:DescribeSSLPolicies",
"elasticloadbalancing:DescribeTags",
"elasticloadbalancing:DescribeTargetGroups",
"elasticloadbalancing:DescribeTargetGroupAttributes",
"elasticloadbalancing:DescribeTargetHealth",
"elasticloadbalancing:ModifyListener",
"elasticloadbalancing:ModifyLoadBalancerAttributes",
"elasticloadbalancing:ModifyRule",
"elasticloadbalancing:ModifyTargetGroup",
"elasticloadbalancing:ModifyTargetGroupAttributes",
"elasticloadbalancing:RegisterTargets",
"elasticloadbalancing:RemoveListenerCertificates",
"elasticloadbalancing:RemoveTags",
"elasticloadbalancing:SetIpAddressType",
"elasticloadbalancing:SetSecurityGroups",
"elasticloadbalancing:SetSubnets",
"elasticloadbalancing:SetWebAcl",
"iam:CreateServiceLinkedRole", "iam:GetServerCertificate",
"iam:ListServerCertificates",
"cognito-idp:DescribeUserPoolClient",
"waf-regional:GetWebACLForResource", "waf-regional:GetWebACL",
"waf-regional:AssociateWebACL",
"waf-regional:DisassociateWebACL", "tag:GetResources",
"tag:TagResources", "waf:GetWebACL", "wafv2:GetWebACL",
"wafv2:GetWebACLForResource", "wafv2:AssociateWebACL",
"wafv2:DisassociateWebACL", "shield:DescribeProtection",
"shield:GetSubscriptionState", "shield:DeleteProtection",
"shield:CreateProtection", "shield:DescribeSubscription",
"shield:ListProtections"
],
resources=['*'],
)
eks_role = aws_iam.Role(
self,
"HandsOnClusterServiceAccountRole",
assumed_by=aws_iam.ServicePrincipal('eks.amazonaws.com'),
)
eks_role.add_to_policy(iam_policy)
cluster_role_manifest = {
"apiVersion":
"rbac.authorization.k8s.io/v1",
"kind":
"ClusterRole",
"metadata": {
"name": "alb-ingress-controller",
"labels": {
"app.kubernetes.io/name": "alb-ingress-controller"
}
},
"rules": [{
"apiGroups": ["", "extensions"],
"resources": [
"configmaps", "endpoints", "events", "ingresses",
"ingresses/status", "services", "pods/status", "nodes",
"pods", "secrets", "services", "namespaces"
],
"verbs": ["create", "get", "list", "update", "watch", "patch"]
}]
}
cluster_role_binding_manifest = {
"apiVersion":
"rbac.authorization.k8s.io/v1",
"kind":
"ClusterRoleBinding",
"metadata": {
"name": "alb-ingress-controller",
"labels": {
"app.kubernetes.io/name": "alb-ingress-controller"
}
},
"roleRef": {
"apiGroup": "rbac.authorization.k8s.io",
"kind": "ClusterRole",
"name": "alb-ingress-controller"
},
"subjects": [{
"kind": "ServiceAccount",
"name": "alb-ingress-controller",
"namespace": "kube-system"
}]
}
service_account_manifest = {
"apiVersion": "v1",
"kind": "ServiceAccount",
"metadata": {
"name": "alb-ingress-controller",
"labels": {
"app.kubernetes.io/name": "alb-ingress-controller"
},
"namespace": "kube-system",
"annotations": {
"eks.amazonaws.com/role-arn": eks_role.role_arn
}
}
}
deployment_manifest = {
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {
"name": "alb-ingress-controller",
"labels": {
"app.kubernetes.io/name": "alb-ingress-controller"
},
"namespace": "kube-system"
},
"spec": {
"selector": {
"matchLabels": {
"app.kubernetes.io/name": "alb-ingress-controller"
}
},
"template": {
"metadata": {
"labels": {
"app.kubernetes.io/name": "alb-ingress-controller"
}
},
"spec": {
"containers": [{
"name":
"alb-ingress-controller",
"args": [
"--ingress-class=alb",
"--cluster-name={}".format(
self.cluster.cluster_name),
"--aws-vpc-id={}".format(
self.cluster.vpc.vpc_id),
"--aws-region={}".format(
self.cluster.vpc.env.region),
"--aws-api-debug"
],
"image":
"docker.io/amazon/aws-alb-ingress-controller:v1.1.8",
}],
"serviceAccountName":
"alb-ingress-controller"
}
}
}
}
service_acct = aws_eks.ServiceAccount(self,
"alb-ingress-controller",
cluster=self.cluster,
name="alb-ingress-controller",
namespace="kube-system")
service_acct.add_to_principal_policy(statement=iam_policy)
alb_ingress_access_manifests = aws_eks.KubernetesManifest(
self,
"ClusterRoleALB",
cluster=self.cluster,
manifest=[
cluster_role_manifest, cluster_role_binding_manifest,
service_account_manifest
])
alb_ingress_deployment = aws_eks.KubernetesManifest(
self,
"ALBIngressDeployment",
cluster=self.cluster,
manifest=[deployment_manifest])
def __init__(self, scope: core.Construct, id: str, elastic: Elastic, vpc: ec2.Vpc, roles: list, cluster: eks.Cluster, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
sm_policy = iam.PolicyStatement(
actions=["secretsmanager:GetSecretValue"],
effect=iam.Effect.ALLOW,
resources=[elastic.secret.secret_arn]
)
es_policy = iam.PolicyStatement(
actions=["es:DescribeElasticsearchDomain"],
effect=iam.Effect.ALLOW,
resources=[elastic.domain.domain_arn]
)
function = lbd.SingletonFunction(
self,
"ElasticsearchConfigFunction",
uuid="e579d5f9-1709-43ea-b75f-9d1452ca7690",
code=lbd.Code.from_asset(
"custom_resources/elasticsearch/"
),
handler="config.handler",
runtime=lbd.Runtime.PYTHON_3_7,
function_name="elasticsearchConfig",
initial_policy=[sm_policy,es_policy],
log_retention=logs.RetentionDays.ONE_DAY,
security_group=ec2.SecurityGroup.from_security_group_id(self, "lambdaVPC", vpc.vpc_default_security_group),
timeout=core.Duration.seconds(30),
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
one_per_az=True
)
)
provider = cr.Provider(
self, "ElasticsearchConfigProvider",
on_event_handler=function,
log_retention=logs.RetentionDays.ONE_DAY
)
core.CustomResource(
self, "ElasticSearchConfig",
service_token=provider.service_token,
properties={
"domain": elastic.domain.domain_name,
"secret": elastic.secret.secret_arn,
"roles": [role.role_arn for role in roles],
"shards": self.node.try_get_context("elastic")['shards'],
"user": boto3.client('sts').get_caller_identity().get('Arn'),
"replicas": self.node.try_get_context("elastic")['replicas']
}
)
manifests = []
for namespace in self.node.try_get_context("kubernetes")['namespaces']:
manifests.append({
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": "elasticsearch",
"namespace": namespace
},
"data": {
"url": elastic.domain.domain_endpoint
}
})
eks.KubernetesManifest(
self,
"elastic-search-cm",
cluster=cluster,
manifest=manifests
)
def __init__(self, scope: core.Construct, id: str, VPC: ec2.Vpc,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
cluster_admin = iam.Role(self,
"AdminRole",
assumed_by=iam.AccountRootPrincipal())
self.cluster = eks.Cluster(self,
"cluster",
default_capacity=self.node.try_get_context(
"kubernetes")["default_capacity"],
default_capacity_instance=ec2.InstanceType(
self.node.try_get_context("kubernetes")
["default_capacity_instance"]),
cluster_name="statement-demo",
vpc=VPC,
vpc_subnets=VPC.private_subnets,
masters_role=cluster_admin,
version=eks.KubernetesVersion.V1_17,
endpoint_access=eks.EndpointAccess.PRIVATE)
vpc_security_group = ec2.SecurityGroup.from_security_group_id(
self, "sgVPC", VPC.vpc_default_security_group)
eks_security_group = ec2.SecurityGroup.from_security_group_id(
self, "sgEKS", self.cluster.cluster_security_group_id)
vpc_security_group.add_ingress_rule(eks_security_group,
ec2.Port.all_traffic())
eks_security_group.add_ingress_rule(vpc_security_group,
ec2.Port.all_traffic())
self.cluster.default_nodegroup.role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
"CloudWatchAgentServerPolicy"))
#see https://github.com/kubernetes/kubernetes/issues/61486?#issuecomment-635169272
eks.KubernetesPatch(
self,
"patch",
cluster=self.cluster,
resource_name="daemonset/kube-proxy",
resource_namespace="kube-system",
apply_patch={
"spec": {
"template": {
"spec": {
"containers": [{
"name":
"kube-proxy",
"command": [
"kube-proxy",
"--v=2",
"--hostname-override=$(NODE_NAME)",
"--config=/var/lib/kube-proxy-config/config",
],
"env": [{
"name": "NODE_NAME",
"valueFrom": {
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "spec.nodeName"
}
}
}]
}]
}
}
}
},
restore_patch={
"spec": {
"template": {
"spec": {
"containers": [{
"name":
"kube-proxy",
"command": [
"kube-proxy", "--v=2",
"--config=/var/lib/kube-proxy-config/config"
]
}]
}
}
}
})
# elasticsearch clusters has many nodes, and its DNS records are always truncated by OpenDNS
eks.KubernetesPatch(
self,
"coreDNSTCP",
cluster=self.cluster,
resource_name="configmap/coredns",
resource_namespace="kube-system",
apply_patch={
"data": {
"Corefile":
".:53 {\n errors\n health\n kubernetes cluster.local in-addr.arpa ip6.arpa {\n pods insecure\n upstream\n fallthrough in-addr.arpa ip6.arpa\n }\n prometheus :9153\n forward . /etc/resolv.conf {\n force_tcp\n }\n cache 30\n loop\n reload\n loadbalance\n}\n"
}
},
restore_patch={
"data": {
"Corefile":
".:53 {\n errors\n health\n kubernetes cluster.local in-addr.arpa ip6.arpa {\n pods insecure\n upstream\n fallthrough in-addr.arpa ip6.arpa\n }\n prometheus :9153\n forward . /etc/resolv.conf\n cache 30\n loop\n reload\n loadbalance\n}\n"
}
})
# adding myself as a cluster admin
self.cluster.aws_auth.add_user_mapping(iam.User.from_user_name(
self, "me",
boto3.client('sts').get_caller_identity().get('Arn').partition('/')
[2]),
groups=["system:masters"])
text = requests.get(
"https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluentd-quickstart.yaml"
).text.replace("{{cluster_name}}", self.cluster.cluster_name).replace(
"{{region_name}}",
core.Stack.of(self).region)
eks.KubernetesManifest(
self,
"containerInsights",
cluster=self.cluster,
manifest=[yaml.load(item) for item in text.split("---\n")])
def __init__(self, scope: core.Construct, id: str, cluster, manifest,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.cluster = cluster
self.manifest = manifest
def backend_service(self, manifest):
labels = manifest['labels']
deployment = {
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {
"name": manifest['service_name'],
"namespace": "default"
},
"spec": {
"replicas": manifest['replicas'],
"selector": {
"matchLabels": labels
},
"strategy": {
"rollingUpdate": {
"maxSurge": "25%",
"maxUnavailable": "25%"
}
},
"template": {
"metadata": {
"labels": labels
},
"spec": {
"containers": [{
"name":
manifest['service_name'],
"image":
manifest['image'],
"ports": [{
"containerPort": manifest['port'],
"protocol": "TCP"
}]
}]
}
}
}
}
service = {
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": manifest['service_name'],
"namespace": "default"
},
"spec": {
"type": "LoadBalancer",
"ports": [{
"port": 80,
"targetPort": manifest['port']
}],
"selector": manifest['labels']
}
}
return deployment, service
def frontend_service(self, manifest):
labels = manifest['labels']
deployment = {
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {
"name": manifest['service_name'],
"namespace": "default"
},
"spec": {
"replicas": manifest['replicas'],
"selector": {
"matchLabels": labels
},
"strategy": {
"rollingUpdate": {
"maxSurge": "25%",
"maxUnavailable": "25%"
}
},
"template": {
"metadata": {
"labels": labels
},
"spec": {
"containers": [{
"name":
manifest['service_name'],
"image":
manifest['image'],
"ports": [{
"containerPort": manifest['port'],
"protocol": "TCP"
}],
"env":
manifest['env']
}],
}
}
}
}
service = {
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": manifest['service_name'],
"namespace": "default"
},
"spec": {
"type": "LoadBalancer",
"ports": [{
"port": 80,
"targetPort": manifest['port']
}],
"selector": manifest['labels']
}
}
return deployment, service
if self.manifest['service_type'] == 'backend':
deployment_manifest, service_manifest = backend_service(
self, self.manifest)
elif self.manifest['service_type'] == 'frontend':
deployment_manifest, service_manifest = frontend_service(
self, self.manifest)
aws_eks.KubernetesManifest(
self,
"NodeJSManifest",
cluster=self.cluster,
manifest=[deployment_manifest, service_manifest])
def __init__(self, scope: core.Construct, id: str, eksname: str,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Cloudformation input params
datalake_bucket = core.CfnParameter(
self,
"datalakebucket",
type="String",
description=
"You existing S3 bucket to be accessed by Jupyter Notebook and ETL job. Default: blank",
default="")
login_name = core.CfnParameter(
self,
"jhubuser",
type="String",
description="Your username login to jupyter hub",
default="sparkoneks")
# Auto-generate a user login in secrets manager
jhub_secret = secmger.Secret(
self,
'jHubPwd',
generate_secret_string=secmger.SecretStringGenerator(
exclude_punctuation=True,
secret_string_template=json.dumps(
{'username': login_name.value_as_string}),
generate_string_key="password"))
# A new bucket to store app code and access logs
self.app_s3 = S3AppCodeConst(self, 'appcode')
# 1. Setup EKS base infrastructure
network_sg = NetworkSgConst(self, 'network-sg', eksname,
self.app_s3.code_bucket)
iam = IamConst(self, 'iam_roles', eksname)
eks_cluster = EksConst(self, 'eks_cluster', eksname, network_sg.vpc,
iam.managed_node_role, iam.admin_role,
self.region)
eks_security = EksSAConst(self, 'eks_sa', eks_cluster.my_cluster,
jhub_secret)
eks_base_app = EksBaseAppConst(self, 'eks_base_app',
eks_cluster.my_cluster, self.region)
# 2. Setup Spark application access control
app_security = SparkOnEksSAConst(self, 'spark_service_account',
eks_cluster.my_cluster,
login_name.value_as_string,
self.app_s3.code_bucket,
datalake_bucket.value_as_string)
# 3. Install ETL orchestrator - Argo
# can be replaced by other workflow tool, ie. Airflow
argo_install = eks_cluster.my_cluster.add_helm_chart(
'ARGOChart',
chart='argo',
repository='https://argoproj.github.io/argo-helm',
release='argo',
namespace='argo',
create_namespace=True,
values=loadYamlLocal('../app_resources/argo-values.yaml'))
# Create a Spark workflow template with different T-shirt size
submit_tmpl = eks_cluster.my_cluster.add_manifest(
'SubmitSparkWrktmpl',
loadYamlLocal('../app_resources/spark-template.yaml'))
submit_tmpl.node.add_dependency(argo_install)
# 4. Install Arc Jupyter notebook to as Spark ETL IDE
jhub_install = eks_cluster.my_cluster.add_helm_chart(
'JHubChart',
chart='jupyterhub',
repository='https://jupyterhub.github.io/helm-chart',
release='jhub',
version='0.11.1',
namespace='jupyter',
create_namespace=False,
values=loadYamlReplaceVarLocal(
'../app_resources/jupyter-values.yaml',
fields={
"{{codeBucket}}": self.app_s3.code_bucket,
"{{region}}": self.region
}))
# get Arc Jupyter login from secrets manager
name_parts = core.Fn.split('-', jhub_secret.secret_name)
name_no_suffix = core.Fn.join(
'-',
[core.Fn.select(0, name_parts),
core.Fn.select(1, name_parts)])
config_hub = eks.KubernetesManifest(
self,
'JHubConfig',
cluster=eks_cluster.my_cluster,
manifest=loadYamlReplaceVarLocal(
'../app_resources/jupyter-config.yaml',
fields={
"{{MY_SA}}": app_security.jupyter_sa,
"{{REGION}}": self.region,
"{{SECRET_NAME}}": name_no_suffix
},
multi_resource=True))
config_hub.node.add_dependency(jhub_install)
# 5.(OPTIONAL) retrieve ALB DNS Name to enable Cloudfront in the following nested stack.
# Recommend to remove this section and the rest of CloudFront component.
# Setup your own certificate then add to ALB, to enable the HTTPS.
self._argo_alb = eks.KubernetesObjectValue(
self,
'argoALB',
cluster=eks_cluster.my_cluster,
json_path='.status.loadBalancer.ingress[0].hostname',
object_type='ingress',
object_name='argo-server',
object_namespace='argo')
self._argo_alb.node.add_dependency(argo_install)
self._jhub_alb = eks.KubernetesObjectValue(
self,
'jhubALB',
cluster=eks_cluster.my_cluster,
json_path='.status.loadBalancer.ingress[0].hostname',
object_type='ingress',
object_name='jupyterhub',
object_namespace='jupyter')
self._jhub_alb.node.add_dependency(config_hub)
def __init__(
self,
scope: cdk.Construct,
construct_id: str,
stack_log_level: str,
eks_cluster,
sales_event_bkt,
**kwargs
) -> None:
super().__init__(scope, construct_id, **kwargs)
# Add your stack resources below):
self.reliable_q = _sqs.Queue(
self,
"reliableQueue01",
delivery_delay=cdk.Duration.seconds(2),
queue_name=f"reliable_message_q",
retention_period=cdk.Duration.days(2),
visibility_timeout=cdk.Duration.seconds(30)
)
# Grant our EKS Node Producer privileges to write to SQS
# Due to cyclic dependency should be done before the EKS cluster is created
# self.reliable_q.grant_send_messages(_eks_node_role)
########################################
####### #######
####### Stream Data Producer #######
####### #######
########################################
app_grp_name = "sales-events-producer"
app_grp_label = {"app": f"{app_grp_name}"}
app_grp_ns = eks_cluster.add_manifest(
f"{app_grp_name}-ns-01",
{
"apiVersion": "v1",
"kind": "Namespace",
"metadata": {
"name": f"{app_grp_name}-ns",
"labels": {
"name": f"{app_grp_name}-ns"
}
}
}
)
####### APP 01 #######
app_01_producer_deployment = {
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {
"name": f"{app_grp_name}",
"namespace": f"{app_grp_name}-ns"
},
"spec": {
"replicas": 1,
"selector": {"matchLabels": app_grp_label},
"template": {
"metadata": {"labels": app_grp_label},
"spec": {
"containers": [
{
"name": f"{app_grp_name}",
"image": "python:3.8.10-alpine",
"command": [
"sh",
"-c"
],
"args": [
"wget https://raw.githubusercontent.com/miztiik/event-processor-on-eks/master/stacks/back_end/eks_sqs_producer_stack/lambda_src/stream_data_producer.py;pip3 install --user boto3;python3 stream_data_producer.py;"
],
"env": [
{
"name": "STORE_EVENTS_BKT",
"value": f"{sales_event_bkt.bucket_name}"
},
{
"name": "S3_PREFIX",
"value": "sales_events"
},
{
"name": "RELIABLE_QUEUE_NAME",
"value": f"{self.reliable_q.queue_name}"
},
{
"name": "AWS_REGION",
"value": f"{cdk.Aws.REGION}"
},
{
"name": "TOT_MSGS_TO_PRODUCE",
"value": "10000"
},
{
"name": "WAIT_SECS_BETWEEN_MSGS",
"value": "2"
}
]
}
]
}
}
}
}
# apply a kubernetes manifest to the cluster
app_01_manifest = _eks.KubernetesManifest(
self,
"miztSalesEventproducerSvc",
cluster=eks_cluster,
manifest=[
app_01_producer_deployment,
]
)
app_01_manifest.node.add_dependency(
app_grp_ns)
###########################################
################# OUTPUTS #################
###########################################
output_0 = cdk.CfnOutput(
self,
"AutomationFrom",
value=f"{GlobalArgs.SOURCE_INFO}",
description="To know more about this automation stack, check out our github page.",
)
output_1 = cdk.CfnOutput(
self,
"ReliableMessageQueue",
value=f"https://console.aws.amazon.com/sqs/v2/home?region={cdk.Aws.REGION}#/queues",
description="Reliable Message Queue"
)
def __init__(self, scope: core.Construct, id: str, eks_cluster, service,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.eks_cluster = eks_cluster
self.service = service
def my_demo_service(self, service):
labels = service['labels']
deployment = {
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {
"name": service['service_name'],
"namespace": service['namespace']
},
"spec": {
"replicas": service['replicas'],
"selector": {
"matchLabels": labels
},
"template": {
"metadata": {
"labels": labels
},
"spec": {
"containers": [{
"name":
service['service_name'],
"image":
service['image'],
"ports": [{
"containerPort": service['port'],
"protocol": "TCP"
}]
}],
}
}
}
}
service = {
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": service['service_name'],
"namespace": service['namespace']
},
"spec": {
"type": "LoadBalancer",
"ports": [{
"port": 80,
"targetPort": service['port']
}],
"selector": service['labels']
}
}
namespace = {
"apiVersion": "v1",
"kind": "Namespace",
"metadata": {
"name": "eksdemo",
"labels": {
"name": "eksdemo"
}
}
}
return deployment, service, namespace
deployment_manifest, service_manifest, namespace_manifest = my_demo_service(
self, self.service)
eks.KubernetesManifest(self,
"MySampleService-",
cluster=self.eks_cluster,
manifest=[
namespace_manifest, deployment_manifest,
service_manifest
])
def __init__(
self,
scope: cdk.Construct,
construct_id: str,
stack_log_level: str,
eks_cluster,
reliable_q,
sales_event_bkt,
**kwargs
) -> None:
super().__init__(scope, construct_id, **kwargs)
# Add your stack resources below):
########################################
####### #######
####### Stream Data consumer #######
####### #######
########################################
app_grp_name = "sales-events-consumer"
app_grp_label = {"app": f"{app_grp_name}"}
app_grp_ns = eks_cluster.add_manifest(
f"{app_grp_name}-ns-01",
{
"apiVersion": "v1",
"kind": "Namespace",
"metadata": {
"name": f"{app_grp_name}-ns",
"labels": {
"name": f"{app_grp_name}-ns"
}
}
}
)
app_01_consumer_deployment = {
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {
"name": f"{app_grp_name}",
"namespace": f"{app_grp_name}-ns"
},
"spec": {
"replicas": 1,
"selector": {"matchLabels": app_grp_label},
"template": {
"metadata": {"labels": app_grp_label},
"spec": {
"containers": [
{
"name": f"{app_grp_name}",
"image": "python:3.8.10-alpine",
"command": [
"sh",
"-c"
],
"args": [
"wget https://raw.githubusercontent.com/miztiik/event-processor-on-eks/master/stacks/back_end/eks_sqs_consumer_stack/lambda_src/stream_data_consumer.py;pip3 install --user boto3;python3 stream_data_consumer.py;"
],
"env":
[
{
"name": "STORE_EVENTS_BKT",
"value": f"{sales_event_bkt.bucket_name}"
},
{
"name": "S3_PREFIX",
"value": "sales_events"
},
{
"name": "RELIABLE_QUEUE_NAME",
"value": f"{reliable_q.queue_name}"
},
{
"name": "AWS_REGION",
"value": f"{cdk.Aws.REGION}"
},
{
"name": "MAX_MSGS_PER_BATCH",
"value": "5"
},
{
"name": "MSG_POLL_BACKOFF",
"value": "2"
},
{
"name": "MSG_PROCESS_DELAY",
"value": "10"
},
{
"name": "TOT_MSGS_TO_PROCESS",
"value": "10000"
}
]
}
]
}
}
}
}
# apply a kubernetes manifest to the cluster
app_01_manifest = _eks.KubernetesManifest(
self,
"miztSalesEventConsumerSvc",
cluster=eks_cluster,
manifest=[
app_01_consumer_deployment
]
)
app_01_manifest.node.add_dependency(
app_grp_ns)
###########################################
################# OUTPUTS #################
###########################################
output_0 = cdk.CfnOutput(
self,
"AutomationFrom",
value=f"{GlobalArgs.SOURCE_INFO}",
description="To know more about this automation stack, check out our github page.",
)
def __init__(self, scope: core.Construct, id: str, eksname: str,
solution_id: str, version: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.template_options.description = "(SO0141) SQL based ETL with Apache Spark on Amazon EKS. This solution provides a SQL based ETL option with a open-source declarative framework powered by Apache Spark."
source_dir = os.path.split(os.environ['VIRTUAL_ENV'])[0] + '/source'
# Cloudformation input params
datalake_bucket = core.CfnParameter(
self,
"datalakebucket",
type="String",
description=
"Your existing S3 bucket to be accessed by Jupyter Notebook and ETL job. Default: blank",
default="")
login_name = core.CfnParameter(
self,
"jhubuser",
type="String",
description="Your username login to jupyter hub",
default="sparkoneks")
# Auto-generate a user login in secrets manager
key = kms.Key(self,
'KMSKey',
removal_policy=core.RemovalPolicy.DESTROY,
enable_key_rotation=True)
key.add_alias("alias/secretsManager")
jhub_secret = secmger.Secret(
self,
'jHubPwd',
generate_secret_string=secmger.SecretStringGenerator(
exclude_punctuation=True,
secret_string_template=json.dumps(
{'username': login_name.value_as_string}),
generate_string_key="password"),
removal_policy=core.RemovalPolicy.DESTROY,
encryption_key=key)
# 1. a new bucket to store app code and logs
self.app_s3 = S3AppCodeConst(self, 'appcode')
# 2. push docker image to ECR via AWS CICD pipeline
ecr_image = DockerPipelineConstruct(self, 'image',
self.app_s3.artifact_bucket)
ecr_image.node.add_dependency(self.app_s3)
core.CfnOutput(self, 'IMAGE_URI', value=ecr_image.image_uri)
# 3. EKS base infrastructure
network_sg = NetworkSgConst(self, 'network-sg', eksname,
self.app_s3.code_bucket)
iam = IamConst(self, 'iam_roles', eksname)
eks_cluster = EksConst(self, 'eks_cluster', eksname, network_sg.vpc,
iam.managed_node_role, iam.admin_role)
EksSAConst(self, 'eks_sa', eks_cluster.my_cluster, jhub_secret)
base_app = EksBaseAppConst(self, 'eks_base_app',
eks_cluster.my_cluster)
# 4. Spark app access control
app_security = SparkOnEksSAConst(self, 'spark_service_account',
eks_cluster.my_cluster,
login_name.value_as_string,
self.app_s3.code_bucket,
datalake_bucket.value_as_string)
app_security.node.add_dependency(base_app.secret_created)
# 5. Install Arc Jupyter notebook in EKS
jhub_install = eks_cluster.my_cluster.add_helm_chart(
'JHubChart',
chart='jupyterhub',
repository='https://jupyterhub.github.io/helm-chart',
release='jhub',
version='0.11.1',
namespace='jupyter',
create_namespace=False,
values=load_yaml_replace_var_local(
source_dir + '/app_resources/jupyter-values.yaml',
fields={
"{{codeBucket}}": self.app_s3.code_bucket,
"{{region}}": core.Aws.REGION
}))
jhub_install.node.add_dependency(app_security)
# EKS get Jupyter login dynamically from secrets manager
name_parts = core.Fn.split('-', jhub_secret.secret_name)
name_no_suffix = core.Fn.join(
'-',
[core.Fn.select(0, name_parts),
core.Fn.select(1, name_parts)])
config_hub = eks.KubernetesManifest(
self,
'JHubConfig',
cluster=eks_cluster.my_cluster,
manifest=load_yaml_replace_var_local(
source_dir + '/app_resources/jupyter-config.yaml',
fields={
"{{MY_SA}}": app_security.jupyter_sa,
"{{REGION}}": core.Aws.REGION,
"{{SECRET_NAME}}": name_no_suffix
},
multi_resource=True))
config_hub.node.add_dependency(jhub_install)
# 6. Install ETL orchestrator - Argo in EKS
# can be replaced by other workflow tool, eg. Airflow
argo_install = eks_cluster.my_cluster.add_helm_chart(
'ARGOChart',
chart='argo-workflows',
repository='https://argoproj.github.io/argo-helm',
release='argo',
version='0.1.4',
namespace='argo',
create_namespace=True,
values=load_yaml_local(source_dir +
'/app_resources/argo-values.yaml'))
argo_install.node.add_dependency(config_hub)
# Create argo workflow template for Spark with T-shirt size
submit_tmpl = eks_cluster.my_cluster.add_manifest(
'SubmitSparkWrktmpl',
load_yaml_local(source_dir + '/app_resources/spark-template.yaml'))
submit_tmpl.node.add_dependency(argo_install)
# 7. (OPTIONAL) retrieve ALB DNS Name to enable CloudFront in the nested stack.
# It is used to serve HTTPS requests with its default domain name.
# Recommend to issue your own TLS certificate, and delete the CF components.
self._jhub_alb = eks.KubernetesObjectValue(
self,
'jhubALB',
cluster=eks_cluster.my_cluster,
json_path='..status.loadBalancer.ingress[0].hostname',
object_type='ingress.networking',
object_name='jupyterhub',
object_namespace='jupyter',
timeout=core.Duration.minutes(10))
self._jhub_alb.node.add_dependency(config_hub)
self._argo_alb = eks.KubernetesObjectValue(
self,
'argoALB',
cluster=eks_cluster.my_cluster,
json_path='..status.loadBalancer.ingress[0].hostname',
object_type='ingress.networking',
object_name='argo-argo-workflows-server',
object_namespace='argo',
timeout=core.Duration.minutes(10))
self._argo_alb.node.add_dependency(argo_install)
# 8. (OPTIONAL) Send solution metrics to AWS
# turn it off from the CloudFormation mapping section if prefer.
send_metrics = solution_metrics.SendAnonymousData(
self,
"SendMetrics",
network_sg.vpc,
self.app_s3.artifact_bucket,
self.app_s3.s3_deploy_contrust,
metrics={
"Solution":
solution_id,
"Region":
core.Aws.REGION,
"SolutionVersion":
version,
"UUID":
"MY_UUID",
"UseDataLakeBucket":
"True" if not datalake_bucket.value_as_string else "False",
"UseAWSCICD":
"True" if ecr_image.image_uri else "False",
"NoAZs":
len(network_sg.vpc.availability_zones)
})
send_metrics.node.add_dependency(self.app_s3.s3_deploy_contrust)
# 9. (OPTIONAL) Override the cfn Nag rules for AWS Solution CICD deployment
# remove the section if your CI/CD pipeline doesn't use the cfn_nag utility to validate the CFN.
k8s_ctl_node = self.node.find_child(
'@aws-cdk--aws-eks.KubectlProvider')
cluster_resrc_node = self.node.find_child(
'@aws-cdk--aws-eks.ClusterResourceProvider')
scan.suppress_cfnnag_rule(
'W12', 'by default the role has * resource',
self.node.find_child('eks_cluster').node.find_child('EKS').node.
default_child.node.find_child('CreationRole').node.find_child(
'DefaultPolicy').node.default_child)
scan.suppress_cfnnag_rule(
'W11', 'by default the role has * resource',
self.node.find_child(
'Custom::AWSCDKOpenIdConnectProviderCustomResourceProvider').
node.find_child('Role'))
scan.suppress_lambda_cfnnag_rule(
k8s_ctl_node.node.find_child('Handler').node.default_child)
scan.suppress_lambda_cfnnag_rule(
k8s_ctl_node.node.find_child('Provider').node.find_child(
'framework-onEvent').node.default_child)
scan.suppress_lambda_cfnnag_rule(
self.node.find_child(
'Custom::CDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756C').
node.default_child)
# scan.suppress_lambda_cfnnag_rule(self.node.find_child('Custom::S3AutoDeleteObjectsCustomResourceProvider').node.find_child('Handler'))
scan.suppress_lambda_cfnnag_rule(
self.node.find_child(
'Custom::AWSCDKOpenIdConnectProviderCustomResourceProvider').
node.find_child('Handler'))
scan.suppress_lambda_cfnnag_rule(
self.node.find_child('AWSCDKCfnUtilsProviderCustomResourceProvider'
).node.find_child('Handler'))
scan.suppress_lambda_cfnnag_rule(
cluster_resrc_node.node.find_child(
'OnEventHandler').node.default_child)
scan.suppress_lambda_cfnnag_rule(
cluster_resrc_node.node.find_child(
'IsCompleteHandler').node.default_child)
scan.suppress_lambda_cfnnag_rule(
cluster_resrc_node.node.find_child('Provider').node.find_child(
'framework-isComplete').node.default_child)
scan.suppress_lambda_cfnnag_rule(
cluster_resrc_node.node.find_child('Provider').node.find_child(
'framework-onTimeout').node.default_child)
scan.suppress_lambda_cfnnag_rule(
cluster_resrc_node.node.find_child('Provider').node.find_child(
'framework-onEvent').node.default_child)
scan.suppress_network_cfnnag_rule(
self.node.find_child('eks_cluster').node.find_child('EKS').node.
find_child('ControlPlaneSecurityGroup').node.default_child)
scan.suppress_lambda_cfnnag_rule(
self.node.find_child('SendMetrics').node.find_child(
'LambdaProvider').node.find_child(
'framework-onEvent').node.default_child)
scan.suppress_network_cfnnag_rule(
self.node.find_child('SendMetrics').node.find_child(
'LambdaProvider').node.find_child('framework-onEvent').node.
find_child('SecurityGroup').node.default_child)
scan.suppress_lambda_cfnnag_rule(
self.node.find_child(
'SingletonLambda75248a819138468c9ba1bca6c7137599').node.
default_child)
scan.suppress_network_cfnnag_rule(
self.node.find_child(
'SingletonLambda75248a819138468c9ba1bca6c7137599').node.
find_child('SecurityGroup').node.default_child)
def __init__(self, scope: core.Construct, id: str, cluster: eks.Cluster, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
maps= []
self.roles=[]
ecr_policy = iam.PolicyStatement(
actions=[
"ecr:DescribeImages",
"ecr:ListImages",
"ecr:BatchDeleteImage"
],
effect=iam.Effect.ALLOW,
resources=[
"arn:aws:ecr:%s:%s:repository/%s" % (core.Stack.of(self).region, core.Stack.of(self).account, namespace) for namespace in self.node.try_get_context("kubernetes")['namespaces']
]
)
function = lbd.SingletonFunction(
self,
"ECRDeleteImagesFunction",
uuid="19411b0e-0e80-4ad4-a316-3235940775e4",
code=lbd.Code.from_asset(
"custom_resources/kubernetes/"
),
handler="config.handler",
runtime=lbd.Runtime.PYTHON_3_7,
function_name="kubernetesConfig",
initial_policy=[ecr_policy],
log_retention=logs.RetentionDays.ONE_DAY,
timeout=core.Duration.seconds(30)
)
provider = cr.Provider(
self, "ECRDeleteImagesFunctionProvider",
on_event_handler=function,
log_retention=logs.RetentionDays.ONE_DAY
)
repositores = []
for namespace in self.node.try_get_context("kubernetes")['namespaces']:
manifest = cluster.add_manifest(
"eksConfigNamespace-%s" % namespace,
{
"apiVersion": "v1",
"kind": "Namespace",
"metadata": {
"name": namespace
}
}
)
sa = cluster.add_service_account(
"service-account-%s" % namespace,
name="statement-demo",
namespace=namespace
)
sa.node.add_dependency(manifest)
self.roles.append(sa.role)
repository = ecr.Repository(
self, ("repository-%s" % namespace),
removal_policy=core.RemovalPolicy.DESTROY,
repository_name=namespace,
lifecycle_rules=[ecr.LifecycleRule(max_image_count=1)]
)
repositores.append(repository.repository_arn)
maps.append({
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": "application.properties",
"namespace": namespace
},
"data": {
"application-aws.properties": Path("../%s/src/main/resources/application-aws.properties" % namespace).read_text()
}
})
core.CustomResource(
self, "ECRDeleteImagesFunction-%s" % namespace,
service_token=provider.service_token,
properties={
"repository": namespace
}
).node.add_dependency(repository)
eks.KubernetesManifest(
self,
"eksConfigMaps",
cluster=cluster,
manifest=maps
)
iam.Policy(
self, "saPolicy",
force=True,
policy_name="EKSSAPolicy",
roles=self.roles,
statements=[
iam.PolicyStatement(
actions=["cloudwatch:PutMetricData"],
conditions={
"StringEquals": {
"cloudwatch:namespace": "statement12"
},
},
resources=["*"]
)
]
)
def __init__(self, scope: core.Construct, id: str, cluster: eks.Cluster,
kafka: msk.CfnCluster, vpc: ec2.Vpc, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
pip.main([
"install", "--system", "--target", "custom_resources/kafka/lib",
"kafka-python"
])
arn = cr.AwsCustomResource(
self,
'clusterArn',
policy=cr.AwsCustomResourcePolicy.from_sdk_calls(resources=['*']),
on_create=cr.AwsSdkCall(
action='listClusters',
service='Kafka',
physical_resource_id=cr.PhysicalResourceId.of(
"ClusterNameFilter"),
parameters={
"ClusterNameFilter": kafka.cluster_name,
"MaxResults": 1
},
),
)
bootstraps = cr.AwsCustomResource(
self,
'clusterBootstraps',
policy=cr.AwsCustomResourcePolicy.from_sdk_calls(resources=["*"]),
on_create=cr.AwsSdkCall(
action='getBootstrapBrokers',
service='Kafka',
physical_resource_id=cr.PhysicalResourceId.of("ClusterArn"),
parameters={
"ClusterArn":
arn.get_response_field("ClusterInfoList.0.ClusterArn")
},
),
)
manifests = []
for namespace in self.node.try_get_context("kubernetes")['namespaces']:
manifests.append({
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": "kafka",
"namespace": namespace
},
"data": {
"bootstrap":
bootstraps.get_response_field('BootstrapBrokerStringTls'),
}
})
eks.KubernetesManifest(self,
"kafka-config",
cluster=cluster,
manifest=manifests)
function = lbd.SingletonFunction(
self,
"KafkaConfigFunction",
uuid="b09329a3-5206-46f7-822f-337da714aeac",
code=lbd.Code.from_asset("custom_resources/kafka/"),
handler="config.handler",
runtime=lbd.Runtime.PYTHON_3_7,
function_name="kafkaConfig",
log_retention=logs.RetentionDays.ONE_DAY,
security_group=ec2.SecurityGroup.from_security_group_id(
self, "lambdaKafkaVPC", vpc.vpc_default_security_group),
timeout=core.Duration.seconds(30),
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(one_per_az=True))
provider = cr.Provider(self,
"KafkaConfigProvider",
on_event_handler=function,
log_retention=logs.RetentionDays.ONE_DAY)
core.CustomResource(
self,
"KafkaLoadTopic",
service_token=provider.service_token,
properties={
"bootstrap":
bootstraps.get_response_field('BootstrapBrokerStringTls'),
"topic":
"load",
"partitions":
150,
"replicas":
1
})
core.CustomResource(
self,
"KafkaGenerateTopic",
service_token=provider.service_token,
properties={
"bootstrap":
bootstraps.get_response_field('BootstrapBrokerStringTls'),
"topic":
"generate",
"partitions":
200,
"replicas":
1
})