def __init__(self, scope: core.Construct, id: str, Description: str,
Uuid: str, **kwargs) -> None:
super().__init__(scope, id)
with open(kwargs["HandlerPath"], encoding="utf-8") as fp:
code_body = fp.read()
resource = cfn.CustomResource(
self,
"Resource",
provider=cfn.CustomResourceProvider.lambda_(
lambda_.SingletonFunction(
self,
"Singleton",
description=Description,
uuid=Uuid,
code=lambda_.InlineCode(code_body),
handler="index.main",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
initial_policy=kwargs["ResourcePolicies"],
log_retention=logs.RetentionDays.ONE_DAY,
)),
properties=kwargs,
)
# response
self.response = resource.get_att("Response")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id)
with open("crd_function/crd.py") as fp:
code_body = fp.read()
crd_lambda = lambda_.SingletonFunction(
self, "Singleton",
uuid=str(uuid4()),
code=lambda_.InlineCode(code_body),
handler="index.lambda_handler",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
)
crd_lambda.add_to_role_policy(
statement=iam.PolicyStatement(
actions=["inspector:SubscribeToEvent"],
resources=["*"]
)
)
resource = cfn.CustomResource(
self, "Resource",
provider=cfn.CustomResourceProvider.lambda_(handler=crd_lambda),
properties=kwargs,
)
self.response = resource.get_att("Response").to_string()
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id)
table_name = kwargs.get("table_name")
table_arn = kwargs.get("table_arn")
function = lambda_.SingletonFunction(
self,
"Singleton",
uuid="22fbda4b-ee9f-4317-9489-c118134d8e97",
code=lambda_.Code.asset("./lambda/load-ddb-data.zip"),
handler="lambda_function.handler",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
environment={'TABLE_NAME': table_name})
function.add_to_role_policy(
iam.PolicyStatement(actions=['dynamodb:PutItem'],
resources=["*"],
effect=iam.Effect.ALLOW))
resource = cfn.CustomResource(
self,
"Resource",
provider=cfn.CustomResourceProvider.lambda_(function),
properties=kwargs,
)
self.response = resource.get_att("Response").to_string()
def __init__(self, scope: core.Construct, id: str,
props: CustomResourceProps, **kwargs) -> None:
super().__init__(scope, id)
name = props.name
lambda_directory = props.lambda_directory
handler = props.handler
timeout = props.timeout
runtime = props.runtime
environment = props.environment
resource_properties = props.resource_properties
lambda_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, name + handler))
resource = cfn.CustomResource(
self,
"Resource",
provider=cfn.CustomResourceProvider.lambda_(
lambda_.SingletonFunction(
self,
"Singleton",
environment=environment,
function_name=name,
uuid=lambda_uuid,
code=lambda_.AssetCode(lambda_directory),
handler=handler,
timeout=core.Duration.seconds(timeout),
runtime=runtime,
)),
properties=resource_properties,
)
self.resource = resource
self.response = resource.get_att("Response").to_string()
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id)
es_host = kwargs.get("es_host")
es_region = kwargs.get("es_region")
es_domain_arn = kwargs.get("es_domain_arn")
function = lambda_.SingletonFunction(
self,
"Singleton",
uuid="e43d1f1e-5676-415c-84d5-d376069aa0da",
code=lambda_.Code.asset("./lambda/load-es-index.zip"),
handler="lambda_function.handler",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
environment={
'ES_HOST': es_host,
'ES_REGION': es_region
})
function.add_to_role_policy(
iam.PolicyStatement(actions=['es:ESHttpPost', 'es:ESHttpPut'],
resources=[es_domain_arn],
effect=iam.Effect.ALLOW))
resource = cfn.CustomResource(
self,
"Resource",
provider=cfn.CustomResourceProvider.lambda_(function),
properties=kwargs,
)
self.response = resource.get_att("Response").to_string()
def setup_iot_endpoint_provider(self):
describe_endpoint_policy = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=["iot:DescribeEndpoint"],
resources=["*"],
)
provider_lambda = aws_lambda.SingletonFunction(
self,
"iot_data_ats_endpoint_provider_lambda",
uuid="iot_data_ats_endpoint_provider_lambda_20200507150213",
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=aws_lambda.Code.asset("custom_resources/iot_endpoint"),
handler="iot_endpoint_provider.on_event",
description="Returns iot:Data-ATS endpoint for this account",
current_version_options=aws_lambda.VersionOptions(
removal_policy=core.RemovalPolicy.DESTROY),
initial_policy=[describe_endpoint_policy],
)
provider = custom_resources.Provider(self,
"iot_data_ats_endpoint_provider",
on_event_handler=provider_lambda)
iot_endpoint = core.CustomResource(
self,
"iot_data_ats_endpoint",
resource_type="Custom::IotDataAtsEndpoint",
service_token=provider.service_token,
)
endpoint_address = iot_endpoint.get_att("EndpointAddress").to_string()
self._parameters_to_save["iot_endpoint_address"] = endpoint_address
def __init__(self, scope: cdk.Construct, id: str, **kwargs) -> None:
super().__init__(
scope,
id,
)
with open("custom-resource-handler.py", encoding="utf-8") as fp:
code_body = fp.read()
resource = cfn.CustomResource(
self,
"Resource",
provider=cfn.CustomResourceProvider.lambda_(
lambda_.SingletonFunction(
self,
"Singleton",
uuid="f7d4f730-4ee1-11e8-9c2d-fa7ae01bbebc",
code=lambda_.InlineCode(code_body),
handler="index.main",
timeout=300,
runtime=lambda_.Runtime.PYTHON27,
)),
properties=kwargs,
)
self.response = resource.get_att("Response").to_string()
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id)
with open("custom_resource/random_string_generator_lambda_function.py", encoding="utf-8") as fp:
code_body = fp.read()
# Use `uuidgen` in bash to generate new ones
random_string_generator_fn=lambda_.SingletonFunction(
self, "Singleton",
uuid="RANDOMF2-F7DB-4561-B7AC-4C9730D10E95",
code=lambda_.InlineCode(code_body),
handler="index.lambda_handler",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
)
resource = cfn.CustomResource(
self, "Resource",
provider=cfn.CustomResourceProvider.lambda_(
random_string_generator_fn
),
properties=kwargs,
)
self.response = resource.get_att("Response").to_string()
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id)
# Read Lambda Code:)
try:
with open(
"waf_stacks/custom_resources/waf_rate_rule_creator/lambda_src/index.py",
encoding="utf-8",
mode="r") as f:
waf_rate_rule_creator_fn_code = f.read()
except OSError:
print("Unable to read Lambda Function Code")
raise
# Create IAM Permission Statements that are required by the Lambda
role_stmt1 = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=["*"],
actions=["wafv2:GetWebACL", "wafv2:UpdateWebACL"])
role_stmt1.sid = "AllowLambdaToCreateWafRules"
waf_rate_rule_creator_fn = _lambda.SingletonFunction(
self,
"waFRateRuleCreatorSingleton",
uuid="mystique30-4ee1-11e8-9c2d-fa7ae01bbebc",
code=_lambda.InlineCode(waf_rate_rule_creator_fn_code),
handler="index.lambda_handler",
timeout=core.Duration.seconds(10),
runtime=_lambda.Runtime.PYTHON_3_7,
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": "INFO",
"APP_ENV": "Production"
},
description="Creates a rate based WAF rule")
waf_rate_rule_creator_fn.add_to_role_policy(role_stmt1)
# Create Custom Log group
waf_rate_rule_creator_fn_lg = _logs.LogGroup(
self,
"wafRateRuleCreatorLogGroup",
log_group_name=
f"/aws/lambda/{waf_rate_rule_creator_fn.function_name}",
retention=_logs.RetentionDays.ONE_WEEK,
removal_policy=core.RemovalPolicy.DESTROY)
waf_rate_rule_creator = cfn.CustomResource(
self,
"wafRateRuleCreatorCustomResource",
provider=cfn.CustomResourceProvider.lambda_(
waf_rate_rule_creator_fn),
properties=kwargs,
)
self.response = waf_rate_rule_creator.get_att(
"rule_add_status").to_string()
def __init__(self, scope: core.Construct, id: str, bucket_name: str,
uuid: str, **kwargs):
super().__init__(scope, id, **kwargs)
bucket_storage = _s3.LifecycleRule(transitions=[
_s3.Transition(storage_class=_s3.StorageClass.INTELLIGENT_TIERING,
transition_after=core.Duration.days(1))
])
self.__bucket = _s3.Bucket(self,
'S3Bucket',
bucket_name=bucket_name,
removal_policy=core.RemovalPolicy.DESTROY,
encryption=_s3.BucketEncryption.KMS_MANAGED,
lifecycle_rules=[bucket_storage])
with open('common/common_cdk/lambda/empty_bucket.py', 'r') as f:
lambda_source = f.read()
empty_bucket_lambda = _lambda.SingletonFunction(
self,
'EmptyBucketLambda',
uuid=uuid,
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.inline(lambda_source),
handler='index.handler',
timeout=core.Duration.minutes(15))
empty_bucket_lambda.role.add_to_policy(
_iam.PolicyStatement(actions=[
's3:DeleteObject', 's3:DeleteObjectVersion',
's3:ListBucketVersions', 's3:ListBucket'
],
resources=[
self.__bucket.bucket_arn + '/*',
self.__bucket.bucket_arn
]))
empty_bucket_lambda_provider = _custom_resources.Provider(
self,
'EmptyBucketLambdaProvider',
on_event_handler=empty_bucket_lambda)
custom_resource = core.CustomResource(
self,
'EmptyBucketCustomResource',
service_token=empty_bucket_lambda_provider.service_token,
properties={"bucket_name": self.__bucket.bucket_name})
custom_resource.node.add_dependency(self.__bucket)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id)
with open("custom_resource/iam_user_tagger_lambda_function.py",
encoding="utf-8") as fp:
code_body = fp.read()
statement = iam.PolicyStatement()
# https://docs.aws.amazon.com/IAM/latest/UserGuide/id_tags.html
statement.add_actions("iam:TagUser")
statement.add_actions("iam:UntagUser")
statement.add_all_resources()
statement.effect = iam.Effect.ALLOW
iam_tagger_fn = lambda_.SingletonFunction(
self,
"Singleton",
uuid="tagger30-4ee1-11e8-9c2d-fa7ae01bbebc",
code=lambda_.InlineCode(code_body),
handler="index.lambda_handler",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
)
iam_tagger_fn.add_to_role_policy(statement)
"""
resource = cfn.CustomResource(
self, "Resource",
provider=cfn.CustomResourceProvider.lambda_(
lambda_.SingletonFunction(
self, "Singleton",
uuid="f7d4f730-4ee1-11e8-9c2d-fa7ae01bbebc",
code=lambda_.InlineCode(code_body),
handler="index.main",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
)
),
properties=kwargs,
)
"""
resource = cfn.CustomResource(
self,
"Resource",
provider=cfn.CustomResourceProvider.lambda_(iam_tagger_fn),
properties=kwargs,
)
self.response = resource.get_att("Response").to_string()
def create_custom_authorizer_signing_key_generic(
self, unique_id, description, token_value) -> core.CustomResource:
"""
Uses a Lambda to create an asymmetric key pair, since neither CFn nor CDK support that as of
this writing (2020-05-09)
https://github.com/aws-cloudformation/aws-cloudformation-coverage-roadmap/issues/337
After creating the key, it signs the token value using the private key, and stores all of
`token_value`, `token_value`'s signature, and the public key in the stack's parameter store.
:return: the CustomResource for the signing key
"""
create_authorizer_policy = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"kms:CreateKey", "kms:GetPublicKey", "kms:ScheduleKeyDeletion",
"kms:Sign"
],
resources=["*"],
)
provider_lambda = aws_lambda.SingletonFunction(
self,
f"iot_custom_authorizer_key_provider_lambda_{unique_id}",
uuid=
f"iot_custom_authorizer_key_provider_lambda_20200507150213_{unique_id}",
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=aws_lambda.Code.asset(
"custom_resources/iot_custom_authorizer_key_provider"),
handler="iot_custom_authorizer_key_provider.on_event",
description=description,
current_version_options=aws_lambda.VersionOptions(
removal_policy=core.RemovalPolicy.DESTROY),
initial_policy=[create_authorizer_policy],
)
provider = custom_resources.Provider(
self,
f"iot_custom_authorizer_key_provider_{unique_id}",
on_event_handler=provider_lambda,
)
iot_custom_authorizer_key = core.CustomResource(
self,
f"iot_custom_authorizer_key_{unique_id}",
resource_type="Custom::IoTCustomAuthorizer",
service_token=provider.service_token,
properties={"token_value": token_value},
)
return iot_custom_authorizer_key
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id)
print('reading code source...')
with open("./cdk_blog_vpc/lambda/lambda_function.py",
encoding="utf-8") as fp:
code_body = fp.read()
my_lambda_role = iam.Role(
self,
"Role_lambda",
assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"))
my_lambda_role.add_to_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
resources=["*"],
actions=[
"logs:*", "ec2:DescribeVpcs", "ec2:DescribeInstances",
"ec2:DescribeInstanceAttribute", "dynamodb:PutItem",
"ec2:DescribeSubnets", "ec2:DescribeVpcPeeringConnections",
"ec2:DescribeRouteTables", "ec2:CreateRoute",
"ec2:ReplaceRouteTableAssociation", "ec2:CreateRouteTable",
"ec2:DisassociateRouteTable", "ec2:AssociateRouteTable",
"ec2:DeleteRoute", "ec2:ReplaceRoute",
"ec2:DeleteRouteTable"
]))
_uuid = uuid.uuid1()
resource = cfn.CustomResource(
self,
"Resource",
provider=cfn.CustomResourceProvider.lambda_(
aws_lambda.SingletonFunction(
self,
"Singleton",
uuid=str(_uuid),
code=aws_lambda.Code.from_asset("./cdk_blog_vpc/lambda"),
handler="lambda_function.lambda_handler",
timeout=core.Duration.seconds(300),
runtime=aws_lambda.Runtime.PYTHON_3_7,
role=my_lambda_role)),
properties=kwargs,
)
self.response = resource.get_att("Response").to_string()
def __init__(self, scope: core.Construct, id: str, config_params, ** kwargs) -> None:
super().__init__(scope, id)
# Read LambdaFunction Code
try:
with open("load_generator_stacks/custom_resources/trigger_run_task/lambda_src/trigger_run_task_lambda_function.py", encoding="utf-8") as fp:
code_body = fp.read()
except OSError:
print('Unable to read UserData script')
# Create IAM Permission Statements that are required by the Lambda
trigger_run_task_fn = _lambda.SingletonFunction(
self, "Singleton",
uuid="mystique2010-4ee1-11e8-9c2d-fa7ae01bbebc",
code=_lambda.InlineCode(code_body),
handler="index.lambda_handler",
timeout=core.Duration.seconds(300),
runtime=_lambda.Runtime.PYTHON_3_7,
environment={
"RUN_TASK_FN_ARN": config_params.get("RUN_TASK_FN_ARN")
},
# security_group=config_params.get('RUN_TASK_FN_ARN'),
)
roleStmt1 = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=["*"],
actions=["lambda:InvokeFunction"]
)
roleStmt1.sid = "AllowLambdaToInvokeLambda"
trigger_run_task_fn.add_to_role_policy(roleStmt1)
resource = cfn.CustomResource(
self, "Resource",
provider=cfn.CustomResourceProvider.lambda_(
trigger_run_task_fn
),
properties=kwargs,
)
self.response = resource.get_att("Response").to_string()
def __init__(self, scope: core.Construct, id: str, secgroup_name: str, **kwargs):
super().__init__(scope, id, **kwargs)
with open('common/common_cdk/lambda/empty_security_group.py', 'r') as f:
lambda_source = f.read()
# lambda utils to empty security group before deletion
empty_secgroup_lambda = _lambda.SingletonFunction(self, 'EmptySecurityGroupLambda',
uuid="dfs3k8730-4ee1-11e8-9c2d-fdfs65dfsc",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.inline(lambda_source),
handler='index.handler',
function_name='ara-auto-empty-secgroup'
)
empty_secgroup_lambda_role = _iam.Role(
self, 'AutoEmptyBucketLambdaRole',
assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com')
)
empty_secgroup_lambda_role.add_to_policy(
_iam.PolicyStatement(
actions=[
'ec2:RevokeSecurityGroupIngress',
'ec2:RevokeSecurityGroupEgress'
],
resources=['arn:aws:ec2::'+core.Aws.ACCOUNT_ID+':security-group/'+secgroup_name]
)
)
empty_secgroup_lambda_provider = _custom_resources.Provider(
self, 'EmptyBucketLambdaProvider',
on_event_handler=empty_secgroup_lambda
)
core.CustomResource(
self, 'EmptyBucketCustomResource',
service_token=empty_secgroup_lambda_provider.service_token,
properties={
"secgroup_name": secgroup_name
}
)
def __init__(self, scope: core.Construct, id: str, ** kwargs) -> None:
super().__init__(scope, id)
# Read Lambda Function Code):
# Read Lambda Code
try:
with open("cognito_identity_provider/custom_resources/cognito_app_client_secret_retriever/lambda_src/index.py",
encoding="utf-8",
mode="r") as f:
cognito_app_client_secret_retriever_fn_code = f.read()
except OSError:
print("Unable to read Lambda Function Code")
raise
# Create IAM Permission Statements that are required by the Lambda
roleStmt1 = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=["*"],
actions=["cognito-idp:DescribeUserPoolClient"]
)
roleStmt1.sid = "AllowLambdaToDescribeCognitoUserPool"
roleStmt2 = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=["*"],
actions=["secretsmanager:CreateSecret",
"secretsmanager:TagResource",
"secretsmanager:UpdateSecret",
"secretsmanager:DeleteSecret"]
)
roleStmt2.sid = "AllowLambdaToAddSecrets"
cognito_app_client_secret_retriever_fn = _lambda.SingletonFunction(
self,
"Singleton",
uuid="mystique30-4ee1-11e8-9c2d-fa7ae01bbebc",
code=_lambda.InlineCode(
cognito_app_client_secret_retriever_fn_code),
handler="index.lambda_handler",
timeout=core.Duration.seconds(10),
runtime=_lambda.Runtime.PYTHON_3_7,
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": "INFO",
"APP_ENV": "Production"
}
)
cognito_app_client_secret_retriever_fn.add_to_role_policy(roleStmt1)
cognito_app_client_secret_retriever_fn.add_to_role_policy(roleStmt2)
# Create Custom Loggroup
cognito_app_client_secret_retriever_fn_lg = _logs.LogGroup(
self,
"cognitoAppClientSecretRetriever",
log_group_name=f"/aws/lambda/{cognito_app_client_secret_retriever_fn.function_name}",
retention=_logs.RetentionDays.ONE_WEEK,
removal_policy=core.RemovalPolicy.DESTROY
)
user_pool_secrets_creator = cfn.CustomResource(
self, "Resource",
provider=cfn.CustomResourceProvider.lambda_(
cognito_app_client_secret_retriever_fn
),
properties=kwargs,
)
self.response = user_pool_secrets_creator.get_att(
"user_pool_secrets_arn").to_string()
def setup_custom_authorizer_user_pass(self):
custom_authorizer_name = self.custom_auth_user_pass_default_authorizer_name
self._parameters_to_save[
"custom_authorizer_user_pass_name"] = custom_authorizer_name
token_key_name = "IoTTokenKeyName"
self._parameters_to_save[
"custom_authorizer_user_pass_token_key_name"] = token_key_name
token_value = "allow"
self._parameters_to_save[
"custom_authorizer_user_pass_token_value"] = token_value
self._parameters_to_save[
"custom_authorizer_user_pass_username"] = self.custom_auth_user_pass_username
self._parameters_to_save[
"custom_authorizer_user_pass_password"] = self.custom_auth_user_pass_password
iot_custom_authorizer_key_resource = self.create_custom_authorizer_signing_key_generic(
"2",
"Manages an asymmetric CMK and token signature for iot custom authorizer with username and password.",
token_value,
)
custom_authorizer_token_signature = iot_custom_authorizer_key_resource.get_att(
"custom_authorizer_token_signature").to_string()
self._parameters_to_save[
"custom_authorizer_user_pass_token_signature"] = custom_authorizer_token_signature
# TODO: remove forcing of us-east-1 when enhanced custom authorizers are available in all regions
# Force region to 'us-east-1' due to enhanced custom authorizers only available in this region
authorizer_function_arn = self.setup_custom_authorizer_function(
"2",
"custom_resources/iot_custom_authorizer_user_pass_function",
"iot_custom_authorizer_user_pass.handler",
"Sample custom authorizer that allows or denies based on username and password",
{
"custom_auth_user_pass_username":
self.custom_auth_user_pass_username,
"custom_auth_user_pass_password":
self.custom_auth_user_pass_password
},
"us-east-1",
)
create_authorizer_policy = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"iot:CreateAuthorizer",
"iot:UpdateAuthorizer",
"iot:DeleteAuthorizer",
"iot:UpdateDomainConfiguration",
"iot:CreateDomainConfiguration",
"iot:DescribeDomainConfiguration",
"iot:DeleteDomainConfiguration",
],
resources=["*"],
)
provider_lambda = aws_lambda.SingletonFunction(
self,
"iot_custom_authorizer_user_pass_provider_lambda",
uuid=
"iot_custom_authorizer_user_pass_provider_lambda_20200727123737",
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=aws_lambda.Code.asset(
"custom_resources/iot_custom_authorizer_user_pass_provider"),
handler="iot_custom_authorizer_user_pass_provider.on_event",
description=
"Sets up an IoT custom authorizer for user password & required domain config due to beta status",
environment={
"custom_auth_user_pass_uuid":
self.custom_auth_user_pass_uuid,
"custom_auth_user_pass_default_authorizer_name":
self.custom_auth_user_pass_default_authorizer_name,
"custom_auth_user_pass_domain_configuration_name":
self.custom_auth_user_pass_domain_configuration_name
},
current_version_options=aws_lambda.VersionOptions(
removal_policy=core.RemovalPolicy.DESTROY),
initial_policy=[create_authorizer_policy],
)
provider = custom_resources.Provider(
self,
"iot_custom_authorizer_user_pass_provider",
on_event_handler=provider_lambda)
public_key = iot_custom_authorizer_key_resource.get_att(
"custom_authorizer_public_key").to_string()
iot_endpoint = core.CustomResource(
self,
"iot_custom_authorizer_user_pass",
resource_type="Custom::IoTCustomAuthorizer",
service_token=provider.service_token,
properties={
"authorizer_function_arn": authorizer_function_arn,
"authorizer_name": custom_authorizer_name,
"public_key": public_key,
"token_key_name": token_key_name,
},
)
endpoint_address = iot_endpoint.get_att(
"BetaEndpointAddress").to_string()
self._parameters_to_save[
"iot_beta_endpoint_address"] = endpoint_address
def setup_custom_authorizer(self):
# These values are used in the custom authorizer setup, and exported to Parameter Store
# for use by integration tests
custom_authorizer_name = "iot_custom_authorizer"
self._parameters_to_save[
"custom_authorizer_name"] = custom_authorizer_name
# Note: "key" is a bit overloaded here. In the context of the custom authorizer, "key name"
# refers to the HTTP header field that the custom authorizer looks for a token value in.
#
# In the case of the custom authorizer key provider, the "key" is the KMS asymmetric CMK
# used to sign the token value passed in the `token_key_name` header. In order to keep the
# terminology consistent between client integ tests that are expecting to pass something for
# a "key name" field, we'll let the ambiguity stand.
token_key_name = "iot_custom_authorizer_token"
self._parameters_to_save[
"custom_authorizer_token_key_name"] = token_key_name
token_value = "allow"
self._parameters_to_save["custom_authorizer_token_value"] = token_value
iot_custom_authorizer_key_resource = self.create_custom_authorizer_signing_key_generic(
"1",
"Manages an asymmetric CMK and token signature for iot custom authorizer.",
token_value,
)
custom_authorizer_token_signature = iot_custom_authorizer_key_resource.get_att(
"custom_authorizer_token_signature").to_string()
self._parameters_to_save[
"custom_authorizer_token_signature"] = custom_authorizer_token_signature
authorizer_function_arn = self.setup_custom_authorizer_function(
"1",
"custom_resources/iot_custom_authorizer_function",
"iot_custom_authorizer.handler",
"Sample custom authorizer that allows or denies based on 'token' value",
{},
self.region,
)
create_authorizer_policy = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"iot:CreateAuthorizer", "iot:UpdateAuthorizer",
"iot:DeleteAuthorizer"
],
resources=["*"],
)
provider_lambda = aws_lambda.SingletonFunction(
self,
"iot_custom_authorizer_provider_lambda",
uuid=self.custom_auth_user_pass_uuid,
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=aws_lambda.Code.asset(
"custom_resources/iot_custom_authorizer_provider"),
handler="iot_custom_authorizer_provider.on_event",
description="Sets up an IoT custom authorizer",
current_version_options=aws_lambda.VersionOptions(
removal_policy=core.RemovalPolicy.DESTROY),
initial_policy=[create_authorizer_policy],
)
provider = custom_resources.Provider(self,
"iot_custom_authorizer_provider",
on_event_handler=provider_lambda)
public_key = iot_custom_authorizer_key_resource.get_att(
"custom_authorizer_public_key").to_string()
core.CustomResource(
self,
"iot_custom_authorizer",
resource_type="Custom::IoTCustomAuthorizer",
service_token=provider.service_token,
properties={
"authorizer_function_arn": authorizer_function_arn,
"authorizer_name": custom_authorizer_name,
"public_key": public_key,
"token_key_name": token_key_name,
},
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Create bucket and upload scrips
bucket = s3.Bucket(self, "ScriptBucket")
self.script_bucket = bucket
s3deploy.BucketDeployment(self, "UploadScriptsToBucket",
sources=[s3deploy.Source.asset(os.path.join(dirname, "scripts"))],
destination_bucket=bucket
)
# Greengrass Core Thing policy
greengrass_core_policy = iot.CfnPolicy(self,
'GreenGrassCorePolicy',
policy_name='greengrass-demo-policy',
policy_document={
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"iot:Publish",
"iot:Subscribe",
"iot:Connect",
"iot:Receive"
],
"Resource": [
"*"
]
},
{
"Effect": "Allow",
"Action": [
"iot:GetThingShadow",
"iot:UpdateThingShadow",
"iot:DeleteThingShadow"
],
"Resource": [
"*"
]
},
{
"Effect": "Allow",
"Action": [
"greengrass:*"
],
"Resource": [
"*"
]
}
]
}
)
self.core_policy_name = greengrass_core_policy.policy_name
# Create a Greengrass group role
greengrass_group_role = iam.Role(self, "GroupRole",
assumed_by=iam.ServicePrincipal("greengrass.amazonaws.com")
)
greengrass_group_role.add_to_policy(iam.PolicyStatement(
resources=["arn:aws:logs:*:*:*"],
actions=[
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
]
))
greengrass_group_role.add_to_policy(iam.PolicyStatement(
resources=["*"],
actions=["iot:*"]
))
self.greengrass_group_role_arn = greengrass_group_role.role_arn
# A custom resource to verify that there is a service role for greengrass on the account
greengrass_mgmt_function = awslambda.SingletonFunction(
self,
"MgmttHandler",
uuid="58854ea2-0624-4ca5-b600-fa88d4b9164e",
runtime=awslambda.Runtime.PYTHON_3_7,
code=awslambda.Code.asset("custom_resources"),
handler="greengrassmgmt.handler",
)
greengrass_mgmt_function.add_to_role_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
'greengrass:*',
'iot:*',
'iam:CreateRole',
'iam:AttachRolePolicy',
'iam:PassRole'
],
resources=['*']
)
)
greengrass_mgmt_provider = cust_resource.Provider(self, "MgmtProvider",
on_event_handler=greengrass_mgmt_function
)
core.CustomResource(self, "MgmtCustResource",
service_token=greengrass_mgmt_provider.service_token
)
def __init__(
self,
scope: core.Construct,
id: str,
ssh_key_name="mystique-automation-ssh-key",
** kwargs
) -> None:
super().__init__(scope, id)
# Read Lambda Code:)
try:
with open("custom_resources/ssh_key_generator/lambda_src/index.py",
encoding="utf-8",
mode="r"
) as f:
ssh_key_generator_fn_code = f.read()
except OSError:
print("Unable to read Lambda Function Code")
raise
# Create IAM Permission Statements that are required by the Lambda
role_stmt1 = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=["*"],
actions=[
"ec2:CreateKeyPair",
"ec2:DeleteKeyPair"
]
)
role_stmt1.sid = "AllowLambdaToCreateSshKey"
role_stmt2 = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=["*"],
actions=[
"ssm:PutParameter",
"ssm:DeleteParameter",
"ssm:GetParameter"
]
)
role_stmt2.sid = "AllowLambdaToCreateSSMParameter"
ssh_key_generator_fn = _lambda.SingletonFunction(
self,
"sshKeyGeneratorSingleton",
uuid="mystique30-4ee1-11e8-9c2d-fa7ae01bbebc",
code=_lambda.InlineCode(
ssh_key_generator_fn_code),
handler="index.lambda_handler",
timeout=core.Duration.seconds(10),
runtime=_lambda.Runtime.PYTHON_3_7,
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": "INFO",
"APP_ENV": "Production",
"SSH_KEY_NAME": ssh_key_name
},
description="Creates a SSH Key in the region"
)
ssh_key_generator_fn.add_to_role_policy(role_stmt1)
ssh_key_generator_fn.add_to_role_policy(role_stmt2)
# Create Custom Log group
ssh_key_generator_fn_lg = _logs.LogGroup(
self,
"sshKeyGeneratorLogGroup",
log_group_name=f"/aws/lambda/{ssh_key_generator_fn.function_name}",
retention=_logs.RetentionDays.ONE_WEEK,
removal_policy=core.RemovalPolicy.DESTROY
)
ssh_key_generator = cfn.CustomResource(
self,
"sshKeyGeneratorCustomResource",
provider=cfn.CustomResourceProvider.lambda_(
ssh_key_generator_fn
),
properties=kwargs,
)
self.response = ssh_key_generator.get_att(
"ssh_key_gen_status").to_string()
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Add the VPC connection arn as an input parameter
vpc_conn_arn = core.CfnParameter(
self,
"VpcConnectionArn",
type="String",
description="The Arn of the VPC connection to use for Redshift.")
quicksight_group_arn = core.Fn.import_value('ara-QuickSight-Group-Arn')
secret_arn = core.Fn.import_value('ara-QuickSight-Redshift-Secret-Arn')
# Create the custom resource policy with the necessary permissions
iam_policy = cr.AwsCustomResourcePolicy.from_statements([
iam.PolicyStatement(actions=cfg.CDK_POLICY_ACTIONS,
resources=['*']),
])
redshift_datasource_lambda = lambda_.SingletonFunction(
self,
id='RedshiftDatasourceLambda',
uuid='b438edeb-f5dc-486a-ac2d-bc0918b975b8',
runtime=lambda_.Runtime.PYTHON_3_8,
code=lambda_.Code.from_asset('dataviz_redshift/lambda'),
handler='redshift_datasource.handler',
function_name='ara_redshift_datasource')
redshift_datasource_lambda.role.add_to_policy(
iam.PolicyStatement(actions=['secretsmanager:GetSecretValue'],
resources=[secret_arn]))
redshift_datasource_lambda.role.add_to_policy(
iam.PolicyStatement(actions=[
'quicksight:CreateDataSource', 'quicksight:DeleteDataSource'
],
resources=['*']))
lambda_provider = cr.Provider(
self,
id='LambdaProvider',
on_event_handler=redshift_datasource_lambda)
responseLamb = core.CustomResource(
self,
'RedshiftDatasourceResource',
service_token=lambda_provider.service_token,
properties={
'Secret_arn': secret_arn,
'Datasource_name': cfg.REDSHIFT_DATASOURCE_NAME,
'Aws_account_id': self.account,
'Quicksight_group_arn': quicksight_group_arn,
'Datasource_actions': cfg.DATASOURCE_ACTIONS,
'Vpc_conn_arn': vpc_conn_arn.value_as_string
})
redshift_datasource_arn = responseLamb.get_att_string('datasource_arn')
core.CfnOutput(self,
"RedshiftDataSourceArn",
description="Redshift Data Source Arn",
value=redshift_datasource_arn)
# Create a Redshift dataset with custom SQL
redshift_dataset_arn = QuickSightRedshiftDataset(
self,
'RedshiftDataset',
iam_policy=iam_policy,
quicksight_group_arn=quicksight_group_arn,
redshift_datasource_arn=redshift_datasource_arn,
redshift_dataset_name=cfg.REDSHIFT_DATASET_NAME,
dataset_actions=cfg.DATASET_ACTIONS,
redshift_custom_sql=cfg.REDSHIFT_CUSTOM_SQL,
redshift_columns=cfg.REDSHIFT_COLUMNS,
redshift_data_transformations=cfg.REDSHIFT_DATA_TRANSFORMATIONS
).redshift_dataset_arn
QuickSightRedshiftAnalysis(
self,
'RedshiftAnalysis',
iam_policy=iam_policy,
quicksight_group_arn=quicksight_group_arn,
redshift_dataset_arn=redshift_dataset_arn,
redshift_analysis_name=cfg.REDSHIFT_ANALYSIS_NAME,
redshift_analysis_template_alias=cfg.
REDSHIFT_ANALYSIS_TEMPLATE_ALIAS,
analysis_actions=cfg.ANALYSIS_ACTIONS)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Parameters
parameters = core.CfnParameter(
self, "SourceBucket",
description="Building on AWS Cognito Stack Modified https://github.com/rosberglinhares/CloudFormationCognitoCustomResources",
default="default"
)
LogoutURL = core.CfnParameter(
self, "LogoutURL",
type="String",
default="http://localhost"
)
CallbackURL = core.CfnParameter(
self, "CallbackURL",
type="String",
default="http://localhost/callback"
)
AppDomain = core.CfnParameter(
self, "AppDomain",
type="String",
default="default"
)
# CognitoSNSPolicy
CognitoSNSPolicy = iam.CfnManagedPolicy(
self, 'CognitoSNSPolicy',
description='Managed policy to allow Amazon Cognito to access SNS',
policy_document={
"Version": "2012-10-17",
"Statement": {
"Effect": "Allow",
"Action": ["sns:publish"],
"Resource": "*"
}
})
# SNSRole
SNSRole = iam.CfnRole(
self, "SNSRole",
role_name="SNSRole",
managed_policy_arns=[CognitoSNSPolicy.ref],
assume_role_policy_document={
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["sts:AssumeRole"],
"Principal": {"Service": ["cognito-idp.amazonaws.com"]}
}]
}
)
SNSRole.add_depends_on(CognitoSNSPolicy)
# CognitoUserPool
CognitoUserPool = cognito.CfnUserPool(
self, 'UserPool',
user_pool_name='photos-pool',
alias_attributes=[
"email", "phone_number"],
auto_verified_attributes=[
"email"],
email_verification_message="Hi, Your verification code is <br/>{####}\n",
email_verification_subject="EDX Email Verification",
mfa_configuration="OPTIONAL",
policies={
"passwordPolicy": {
"minimumLength": 8,
"requireLowercase": True,
"requireNumbers": True,
"requireSymbols": True,
"requireUppercase": True
}
},
schema=[{
"attributeDataType": "String",
"mutable": False,
"name": "nickname",
"required": True
},
{
"attributeDataType": "String",
"mutable": False,
"name": "email",
"required": True
},
{
"attributeDataType": "String",
"mutable": False,
"name": "phone_number",
"required": True
}],
sms_configuration={
"externalId": "%s-external" % (core.Aws.STACK_NAME),
"snsCallerArn": SNSRole.attr_arn
}
)
# CognitoUserPoolClient
CognitoUserPoolClient = cognito.CfnUserPoolClient(
self, "UserPoolClient",
client_name="WebsiteClient",
generate_secret=True,
user_pool_id=CognitoUserPool.ref
)
# CognitoCustomResourceRole
CustomResourceRole = iam.CfnRole(
self, "CustomResourceRole",
role_name="cognito-resource-role",
assume_role_policy_document={
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": [
"lambda.amazonaws.com"
]
},
"Action": [
"sts:AssumeRole"
]
}
]
},
policies=[
{
"policyName": "writeCloudWatchLogs",
"policyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "logs:CreateLogGroup",
"Resource": "arn:aws:logs:*:*:*"
},
{
"Effect": "Allow",
"Action": "logs:CreateLogStream",
"Resource": "arn:aws:logs:*:*:*"
},
{
"Effect": "Allow",
"Action": "logs:PutLogEvents",
"Resource": "arn:aws:logs:*:*:*"
}
]
}
},
{
"policyName": "updateUserPoolClient",
"policyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "cognito-idp:UpdateUserPoolClient",
"Resource": "arn:aws:cognito-idp:*:*:userpool/*"
}
]
}
},
{
"policyName": "manageUserPoolDomain",
"policyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "cognito-idp:CreateUserPoolDomain",
"Resource": "arn:aws:cognito-idp:*:*:userpool/*"
},
{
"Effect": "Allow",
"Action": "cognito-idp:DeleteUserPoolDomain",
"Resource": "arn:aws:cognito-idp:*:*:userpool/*"
},
{
"Effect": "Allow",
"Action": "cognito-idp:DescribeUserPoolDomain",
"Resource": "*"
},
{
"Effect": "Allow",
"Action": "cognito-idp:DescribeUserPoolClient",
"Resource": "*"
}
]
}
},
{
"policyName": "invokeLambdaFunction",
"policyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "arn:aws:lambda:*:*:function:*"
}
]
}
},
]
)
# CognitoUserPoolClientClientSettings
with open("./cdk/CognitoUserPoolClientClientSettings/index.js", encoding="utf-8") as fp:
code_body = fp.read()
CognitoUserPoolClientClientSettings = cfn.CustomResource(
self, "CognitoUserPoolClientClientSettings",
provider=cfn.CustomResourceProvider.lambda_(
lambda_.SingletonFunction(
self, "CognitoUserPoolClientClientSettingsLambda",
uuid="f7d4f730-4ee1-11e8-9c2d-fa7ae01bbebc",
code=lambda_.InlineCode(code_body),
handler="index.handler",
runtime=lambda_.Runtime.NODEJS_8_10,
role=iam.Role.from_role_arn(
self, 'CustomResourceRoleiam', role_arn=CustomResourceRole.attr_arn)
)
),
properties={"UserPoolId": CognitoUserPool.ref,
"UserPoolClientId": CognitoUserPoolClient.ref,
"AppDomain": AppDomain.value_as_string,
"SupportedIdentityProviders": ['COGNITO'],
"CallbackURL": CallbackURL.value_as_string,
"LogoutURL": LogoutURL.value_as_string,
"AllowedOAuthFlowsUserPoolClient": True,
"AllowedOAuthFlows": ['code'],
"AllowedOAuthScopes": ['openid']
},
)
# CognitoIdPool
CognitoIdPool = cognito.CfnIdentityPool(
self, 'CognitoIdPool',
identity_pool_name='edxcognitoidpool',
cognito_identity_providers=[{
"clientId": CognitoUserPoolClient.ref,
"providerName": CognitoUserPool.attr_provider_name
}],
allow_unauthenticated_identities=False
)
# Output
core.CfnOutput(self, "CognitoUserPoolIdOutput",
value=CognitoUserPool.ref,
description="The Pool ID of the Cognito User Pool",
export_name="CognitoUserPoolId"
)
core.CfnOutput(self, "CognitoUserPoolProviderURLOutput",
value=CognitoUserPool.attr_provider_url,
description="The Pool ProviderURL of the Cognito User Pool",
export_name="CognitoUserPoolProviderURL"
)
core.CfnOutput(self, "CognitoUserPoolArnOutput",
value=CognitoUserPool.attr_arn,
description="The Pool Arn of the Cognito User Pool",
export_name="CognitoUserPoolArn"
)
core.CfnOutput(self, "CognitoUserPoolClientIdOutput",
value=CognitoUserPoolClient.ref,
description="The App Client ID ",
export_name="CognitoUserPoolClientId"
)
core.CfnOutput(self, "ClientSecretOutput",
value=core.Fn.get_att(
"CognitoUserPoolClientClientSettings", "ClientSecret").to_string(),
description="The Client Secret ",
export_name="ClientSecret"
)
def __init__(self, scope: core.Construct, id: str, cluster: eks.Cluster,
kafka: msk.CfnCluster, vpc: ec2.Vpc, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
pip.main([
"install", "--system", "--target", "custom_resources/kafka/lib",
"kafka-python"
])
arn = cr.AwsCustomResource(
self,
'clusterArn',
policy=cr.AwsCustomResourcePolicy.from_sdk_calls(resources=['*']),
on_create=cr.AwsSdkCall(
action='listClusters',
service='Kafka',
physical_resource_id=cr.PhysicalResourceId.of(
"ClusterNameFilter"),
parameters={
"ClusterNameFilter": kafka.cluster_name,
"MaxResults": 1
},
),
)
bootstraps = cr.AwsCustomResource(
self,
'clusterBootstraps',
policy=cr.AwsCustomResourcePolicy.from_sdk_calls(resources=["*"]),
on_create=cr.AwsSdkCall(
action='getBootstrapBrokers',
service='Kafka',
physical_resource_id=cr.PhysicalResourceId.of("ClusterArn"),
parameters={
"ClusterArn":
arn.get_response_field("ClusterInfoList.0.ClusterArn")
},
),
)
manifests = []
for namespace in self.node.try_get_context("kubernetes")['namespaces']:
manifests.append({
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": "kafka",
"namespace": namespace
},
"data": {
"bootstrap":
bootstraps.get_response_field('BootstrapBrokerStringTls'),
}
})
eks.KubernetesManifest(self,
"kafka-config",
cluster=cluster,
manifest=manifests)
function = lbd.SingletonFunction(
self,
"KafkaConfigFunction",
uuid="b09329a3-5206-46f7-822f-337da714aeac",
code=lbd.Code.from_asset("custom_resources/kafka/"),
handler="config.handler",
runtime=lbd.Runtime.PYTHON_3_7,
function_name="kafkaConfig",
log_retention=logs.RetentionDays.ONE_DAY,
security_group=ec2.SecurityGroup.from_security_group_id(
self, "lambdaKafkaVPC", vpc.vpc_default_security_group),
timeout=core.Duration.seconds(30),
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(one_per_az=True))
provider = cr.Provider(self,
"KafkaConfigProvider",
on_event_handler=function,
log_retention=logs.RetentionDays.ONE_DAY)
core.CustomResource(
self,
"KafkaLoadTopic",
service_token=provider.service_token,
properties={
"bootstrap":
bootstraps.get_response_field('BootstrapBrokerStringTls'),
"topic":
"load",
"partitions":
150,
"replicas":
1
})
core.CustomResource(
self,
"KafkaGenerateTopic",
service_token=provider.service_token,
properties={
"bootstrap":
bootstraps.get_response_field('BootstrapBrokerStringTls'),
"topic":
"generate",
"partitions":
200,
"replicas":
1
})
def __init__(self, scope: core.Construct, id: str, log_bucket: _s3.Bucket,
config_table: _dynamodb.Table, tshirt_size: str,
sink_bucket: _s3.Bucket, web_sale_stream: str,
web_customer_stream: str, web_customer_address_stream: str,
kinesis_key: _kms.Key, vpc: _ec2.Vpc, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
stack = core.Stack.of(self)
stream_source_bucket = AutoEmptyBucket(
self,
'StreamSource',
bucket_name='ara-stream-source-' + core.Aws.ACCOUNT_ID,
uuid='95505f50-0276-11eb-adc1-0242ac120002')
service_role = _iam.Role(
self,
'StreamEmrServiceRole',
assumed_by=_iam.ServicePrincipal('elasticmapreduce.amazonaws.com'))
service_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonElasticMapReduceRole'))
cluster_role = _iam.Role(
self,
'StreamEmrClusterRole',
assumed_by=_iam.ServicePrincipal("ec2.amazonaws.com"))
_iam.Policy(
self,
'StreamEmrClusterPolicy',
statements=[
_iam.PolicyStatement(actions=[
"glue:CreateDatabase",
"glue:UpdateDatabase",
"glue:DeleteDatabase",
"glue:GetDatabase",
"glue:GetDatabases",
"glue:CreateTable",
"glue:UpdateTable",
"glue:DeleteTable",
"glue:GetTable",
"glue:GetTables",
"glue:GetTableVersions",
"glue:CreatePartition",
"glue:BatchCreatePartition",
"glue:UpdatePartition",
"glue:DeletePartition",
"glue:BatchDeletePartition",
"glue:GetPartition",
"glue:GetPartitions",
"glue:BatchGetPartition",
"glue:CreateUserDefinedFunction",
"glue:UpdateUserDefinedFunction",
"glue:DeleteUserDefinedFunction",
"glue:GetUserDefinedFunction",
"glue:GetUserDefinedFunctions",
"cloudwatch:PutMetricData",
"dynamodb:ListTables",
"s3:HeadBucket",
"ec2:Describe*",
],
resources=['*']),
_iam.PolicyStatement(
actions=['s3:GetObject'],
resources=[
'arn:aws:s3:::' + ARA_BUCKET_NAME + BINARIES +
DataGenConfig.DSDGEN_INSTALL_SCRIPT, 'arn:aws:s3:::' +
ARA_BUCKET_NAME + BINARIES + DataGenConfig.JAR_FILE
]),
_iam.PolicyStatement(
actions=['s3:PutObject'],
resources=[log_bucket.bucket_arn + "/data-generator/*"]),
_iam.PolicyStatement(
actions=[
"s3:AbortMultipartUpload", "s3:CreateBucket",
"s3:DeleteObject", "s3:GetBucketVersioning",
"s3:GetObject", "s3:GetObjectTagging",
"s3:GetObjectVersion", "s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:ListBucketVersions", "s3:ListMultipartUploadParts",
"s3:PutBucketVersioning", "s3:PutObject",
"s3:PutObjectTagging"
],
resources=[
sink_bucket.bucket_arn + '/*', sink_bucket.bucket_arn,
stream_source_bucket.bucket.bucket_arn + '/*',
stream_source_bucket.bucket.bucket_arn
])
],
roles=[cluster_role])
cluster_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonSSMManagedInstanceCore'))
_iam.CfnInstanceProfile(self,
'StreamEmrClusterInstanceProfile',
roles=[cluster_role.role_name],
instance_profile_name=cluster_role.role_name)
# Security Groups for the EMR cluster (private subnet)
# https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-man-sec-groups.html#emr-sg-elasticmapreduce-master-private
master_sg = _ec2.SecurityGroup(self,
'ElasticMapReduce-Master-Private',
vpc=vpc)
slave_sg = _ec2.SecurityGroup(self,
'ElasticMapReduce-Slave-Private',
vpc=vpc)
service_sg = _ec2.SecurityGroup(self,
'ElasticMapReduce-ServiceAccess',
vpc=vpc,
allow_all_outbound=False)
# Service SG used by the proxy instance
service_sg.add_ingress_rule(master_sg, _ec2.Port.tcp(9443))
service_sg.add_egress_rule(master_sg, _ec2.Port.tcp(8443))
service_sg.add_egress_rule(slave_sg, _ec2.Port.tcp(8443))
# EMR Master
master_sg.add_ingress_rule(master_sg, _ec2.Port.all_icmp())
master_sg.add_ingress_rule(master_sg, _ec2.Port.all_tcp())
master_sg.add_ingress_rule(master_sg, _ec2.Port.all_udp())
master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_icmp())
master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_tcp())
master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_udp())
master_sg.add_ingress_rule(service_sg, _ec2.Port.tcp(8443))
# EMR Slave
slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_icmp())
slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_tcp())
slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_udp())
slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_icmp())
slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_tcp())
slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_udp())
slave_sg.add_ingress_rule(service_sg, _ec2.Port.tcp(8443))
with open('common/common_cdk/lambda/datagen_config.py', 'r') as f:
lambda_source = f.read()
configure_datagen_function = _lambda.SingletonFunction(
self,
'StreamConfigureDatagenLambda',
uuid="a9904dec-01cf-11eb-adc1-0242ac120002",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.inline(lambda_source),
handler='index.handler',
function_name='stream-datagen-config',
environment={
'TABLE_NAME': config_table.table_name,
'JAR_LOCATION': BINARIES_LOCATION + DataGenConfig.JAR_FILE,
},
timeout=core.Duration.seconds(10))
configure_datagen_function.role.add_to_policy(
_iam.PolicyStatement(actions=[
'dynamodb:GetItem',
'dynamodb:PutItem',
],
resources=[config_table.table_arn]))
emr_cluster = _emr.CfnCluster(
self,
'StreamEmrCluster',
name="StreamDatagenCluster",
job_flow_role=cluster_role.role_name,
service_role=service_role.role_name,
release_label='emr-5.30.1',
visible_to_all_users=True,
log_uri=log_bucket.s3_url_for_object() + "/data-generator",
applications=[
_emr.CfnCluster.ApplicationProperty(name='hadoop'),
_emr.CfnCluster.ApplicationProperty(name='spark')
],
bootstrap_actions=[
_emr.CfnCluster.BootstrapActionConfigProperty(
name="dsdgen-install",
script_bootstrap_action=_emr.CfnCluster.
ScriptBootstrapActionConfigProperty(
path=BINARIES_LOCATION +
DataGenConfig.DSDGEN_INSTALL_SCRIPT))
],
instances=_emr.CfnCluster.JobFlowInstancesConfigProperty(
emr_managed_master_security_group=master_sg.security_group_id,
emr_managed_slave_security_group=slave_sg.security_group_id,
service_access_security_group=service_sg.security_group_id,
ec2_subnet_id=vpc.private_subnets[0].subnet_id,
core_instance_group=_emr.CfnCluster.
InstanceGroupConfigProperty(instance_count=DataGenConfig.
BATCH_CLUSTER_SIZE[tshirt_size],
instance_type='m5.xlarge'),
master_instance_group=_emr.CfnCluster.
InstanceGroupConfigProperty(instance_count=1,
instance_type='m4.large')))
configure_datagen = _sfn_tasks.LambdaInvoke(
self,
"ConfigureDatagenTask",
lambda_function=configure_datagen_function,
payload=_sfn.TaskInput.from_text(
'{'
'"Param": "stream_iterator",'
'"Module": "stream",'
'"SinkBucket": "' + sink_bucket.s3_url_for_object() + '",'
'"Parallelism": "' +
str(int(DataGenConfig.STREAM_DATA_SIZE[tshirt_size]) * 2) +
'",'
'"DataSize": "' + DataGenConfig.STREAM_DATA_SIZE[tshirt_size] +
'",'
'"TmpBucket": "' +
str(stream_source_bucket.bucket.s3_url_for_object()) + '"'
'}'),
result_path='$.Config')
add_datagen_step = _sfn.CustomState(
self,
'StreamAddDataGenStep',
state_json={
"Type": "Task",
"Resource": "arn:aws:states:::elasticmapreduce:addStep.sync",
"Parameters": {
"ClusterId.$": "$.Emr.Cluster.Id",
"Step": {
"Name": "DatagenStep",
"ActionOnFailure": "CONTINUE",
"HadoopJarStep": {
"Jar": "command-runner.jar",
"Args.$": "$.Config.Payload.StepParam"
}
}
},
"ResultPath": "$.Step",
"Next": "StreamUpdateIterator"
})
update_iterator = _sfn_tasks.DynamoUpdateItem(
self,
'StreamUpdateIterator',
table=config_table,
key={
'param':
_sfn_tasks.DynamoAttributeValue.from_string('stream_iterator')
},
update_expression=
'SET iterator = if_not_exists(iterator, :start) + :inc',
expression_attribute_values={
":inc": _sfn_tasks.DynamoAttributeValue.from_number(1),
":start": _sfn_tasks.DynamoAttributeValue.from_number(0)
},
result_path=_sfn.JsonPath.DISCARD)
definition = configure_datagen \
.next(add_datagen_step) \
.next(update_iterator)
datagen_stepfunctions = _sfn.StateMachine(
self,
"StreamDataGenStepFunctions",
definition=definition,
timeout=core.Duration.minutes(30))
datagen_stepfunctions.add_to_role_policy(
_iam.PolicyStatement(actions=[
'elasticmapreduce:AddJobFlowSteps',
'elasticmapreduce:DescribeStep'
],
resources=['*']))
step_trigger = _events.Rule(self,
'StreamStepTrigger',
schedule=_events.Schedule.cron(
minute='0/10',
hour='*',
month='*',
week_day='*',
year='*'))
step_trigger.add_target(
_events_targets.SfnStateMachine(
machine=datagen_stepfunctions,
input=_events.RuleTargetInput.from_object({
"Emr": {
"Cluster": {
"Id": core.Fn.ref(emr_cluster.logical_id)
}
}
})))
with open('common/common_cdk/lambda/stepfunctions_trigger.py',
'r') as f:
lambda_source = f.read()
stepfunctions_trigger_lambda = _lambda.SingletonFunction(
self,
'StreamStepFunctionsTriggerLambda',
uuid="cf042246-01d0-11eb-adc1-0242ac120002",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.inline(lambda_source),
handler='index.handler',
function_name='stepfunctions-stream-datagen-trigger')
stepfunctions_trigger_lambda.role.add_to_policy(
_iam.PolicyStatement(actions=["states:StartExecution"],
resources=['*']))
trigger_step_lambda_provider = _custom_resources.Provider(
self,
'StreamStepFunctionsTriggerLambdaProvider',
on_event_handler=stepfunctions_trigger_lambda)
core.CustomResource(
self,
'StreamStepFunctionsTrigger',
service_token=trigger_step_lambda_provider.service_token,
properties={"stepArn": datagen_stepfunctions.state_machine_arn})
with open('common/common_cdk/lambda/stream_generator.py', 'r') as f:
lambda_source = f.read()
sale_stream_generator_lambda = _lambda.Function(
scope=self,
id='WebSaleStreamGenerator',
runtime=_lambda.Runtime.PYTHON_3_7,
memory_size=2048,
timeout=core.Duration.minutes(15),
code=_lambda.Code.inline(lambda_source),
handler='index.lambda_handler',
environment={
'REGION': core.Aws.REGION,
'STREAM_NAME': web_sale_stream
})
stream_source_bucket.bucket.add_event_notification(
_s3.EventType.OBJECT_CREATED,
_s3_notifications.LambdaDestination(sale_stream_generator_lambda),
_s3.NotificationKeyFilter(prefix='sale', suffix='csv'))
sale_stream_generator_lambda.add_to_role_policy(
_iam.PolicyStatement(
actions=[
"s3:DeleteObject",
"s3:GetObject",
"s3:ListBucket",
],
resources=[
stream_source_bucket.bucket.bucket_arn + '/*',
stream_source_bucket.bucket.bucket_arn
]))
sale_stream_generator_lambda.add_to_role_policy(
_iam.PolicyStatement(actions=["kinesis:PutRecords"],
resources=[
stack.format_arn(
service='kinesis',
resource='stream',
resource_name=web_sale_stream)
]))
sale_stream_generator_lambda.add_to_role_policy(
_iam.PolicyStatement(actions=['kms:GenerateDataKey'],
resources=[
stack.format_arn(
service='kms',
resource='key',
sep='/',
resource_name=kinesis_key.key_id)
]))
customer_stream_generator_lambda = _lambda.Function(
scope=self,
id='WebCustomerStreamGenerator',
runtime=_lambda.Runtime.PYTHON_3_7,
memory_size=2048,
timeout=core.Duration.minutes(15),
code=_lambda.Code.inline(lambda_source),
handler='index.lambda_handler',
environment={
'REGION': core.Aws.REGION,
'STREAM_NAME': web_customer_stream
})
stream_source_bucket.bucket.add_event_notification(
_s3.EventType.OBJECT_CREATED,
_s3_notifications.LambdaDestination(
customer_stream_generator_lambda),
_s3.NotificationKeyFilter(prefix='customer', suffix='csv'))
customer_stream_generator_lambda.add_to_role_policy(
_iam.PolicyStatement(
actions=[
"s3:DeleteObject",
"s3:GetObject",
"s3:ListBucket",
],
resources=[
stream_source_bucket.bucket.bucket_arn + '/*',
stream_source_bucket.bucket.bucket_arn
]))
customer_stream_generator_lambda.add_to_role_policy(
_iam.PolicyStatement(actions=["kinesis:PutRecords"],
resources=[
stack.format_arn(
service='kinesis',
resource='stream',
resource_name=web_customer_stream)
]))
customer_stream_generator_lambda.add_to_role_policy(
_iam.PolicyStatement(actions=['kms:GenerateDataKey'],
resources=[
stack.format_arn(
service='kms',
resource='key',
sep='/',
resource_name=kinesis_key.key_id)
]))
address_stream_generator_lambda = _lambda.Function(
scope=self,
id='WebCustomerAddressStreamGenerator',
runtime=_lambda.Runtime.PYTHON_3_7,
memory_size=2048,
timeout=core.Duration.minutes(15),
code=_lambda.Code.inline(lambda_source),
handler='index.lambda_handler',
environment={
'REGION': core.Aws.REGION,
'STREAM_NAME': web_customer_address_stream
})
stream_source_bucket.bucket.add_event_notification(
_s3.EventType.OBJECT_CREATED,
_s3_notifications.LambdaDestination(
address_stream_generator_lambda),
_s3.NotificationKeyFilter(prefix='address', suffix='csv'))
address_stream_generator_lambda.add_to_role_policy(
_iam.PolicyStatement(
actions=[
"s3:DeleteObject",
"s3:GetObject",
"s3:ListBucket",
],
resources=[
stream_source_bucket.bucket.bucket_arn + '/*',
stream_source_bucket.bucket.bucket_arn
]))
address_stream_generator_lambda.add_to_role_policy(
_iam.PolicyStatement(
actions=["kinesis:PutRecords"],
resources=[
stack.format_arn(service='kinesis',
resource='stream',
resource_name=web_customer_address_stream)
]))
address_stream_generator_lambda.add_to_role_policy(
_iam.PolicyStatement(actions=['kms:GenerateDataKey'],
resources=[
stack.format_arn(
service='kms',
resource='key',
sep='/',
resource_name=kinesis_key.key_id)
]))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
lambda_policies = [
iam.PolicyStatement(actions=[
"logs:CreateLogStream", "logs:PutLogEvents",
"logs:CreateLogGroup"
],
effect=iam.Effect.ALLOW,
resources=[
"arn:aws:logs:" + core.Aws.REGION + ":" +
core.Aws.ACCOUNT_ID + ":*"
]),
iam.PolicyStatement(actions=["dynamodb:*"],
effect=iam.Effect.ALLOW,
resources=[
"arn:aws:dynamodb:" + core.Aws.REGION +
":" + core.Aws.ACCOUNT_ID + ":*"
])
]
base_api = _apigw.RestApi(self,
'PetclinicApiGatewayWithCors',
rest_api_name='PetclinicApiGatewayWithCors')
api_resource = base_api.root.add_resource('api')
website_bucket = _s3.Bucket(self,
'PetclinicWebsite',
website_index_document='index.html',
public_read_access=True,
removal_policy=core.RemovalPolicy.DESTROY)
deployment = _s3deploy.BucketDeployment(
self,
'PetclinicDeployWebsite',
sources=[_s3deploy.Source.asset('./spring-petclinic-static')],
destination_bucket=website_bucket,
retain_on_delete=False
#destination_key_prefix='web/static'
)
# Modify the config.js with CF custome resource
modify_policy = [
iam.PolicyStatement(actions=[
"s3:PutObject", "s3:PutObjectAcl", "s3:PutObjectVersionAcl",
"s3:GetObject"
],
effect=iam.Effect.ALLOW,
resources=[website_bucket.bucket_arn + "/*"]),
iam.PolicyStatement(actions=["s3:ListBucket"],
effect=iam.Effect.ALLOW,
resources=[website_bucket.bucket_arn]),
iam.PolicyStatement(actions=["dynamodb:*"],
effect=iam.Effect.ALLOW,
resources=[
"arn:aws:dynamodb:" + core.Aws.REGION +
":" + core.Aws.ACCOUNT_ID + ":*"
])
]
with open("custom-resource-code/init.py", encoding="utf-8") as fp:
code_body = fp.read()
dynamodb_tables = []
for service in ['customer', 'vet', 'visit']:
table = _dynamodb.Table(
self,
service.capitalize() + 'Table',
partition_key={
'name': 'id',
'type': _dynamodb.AttributeType.STRING
},
removal_policy=core.RemovalPolicy.DESTROY,
read_capacity=5,
write_capacity=5,
)
dynamodb_tables.append(table.table_name)
base_lambda = _lambda.Function(
self,
'ApiPetclinic' + service.capitalize() + 'Lambda',
handler='org.springframework.samples.petclinic.' + service +
's.StreamLambdaHandler::handleRequest',
runtime=_lambda.Runtime.JAVA_8,
code=_lambda.Code.asset(
'./spring-petclinic-serverless/spring-petclinic-' +
service + 's-serverless/target/spring-petclinic-' +
service + 's-serverless-2.0.7.jar'),
memory_size=1024,
timeout=core.Duration.seconds(300),
initial_policy=lambda_policies,
environment={
"DYNAMODB_TABLE_NAME": table.table_name,
"SERVER_SERVLET_CONTEXT_PATH": "/api/" + service
})
base_version = base_lambda.add_version(
name='v1',
provisioned_executions=1) #Added for warm the Java Lambda
entity = api_resource.add_resource(service)
entity.add_proxy(
default_integration=_apigw.LambdaIntegration(base_version))
self.add_cors_options(entity)
resource = _cfn.CustomResource(
self,
"S3ModifyCustomResource",
provider=_cfn.CustomResourceProvider.lambda_(
_lambda.SingletonFunction(
self,
"CustomResourceSingleton",
uuid="f7d4f730-4ee1-11e8-9c2d-fa7ae01bbebc",
code=_lambda.InlineCode(code_body),
handler="index.handler",
timeout=core.Duration.seconds(300),
runtime=_lambda.Runtime.PYTHON_3_7,
initial_policy=modify_policy)),
properties={
"Bucket": website_bucket.bucket_name,
"InvokeUrl": base_api.url,
"DynamoDBTables": dynamodb_tables
})
core.CfnOutput(self,
"PetclinicWebsiteUrl",
export_name="PetclinicWebsiteUrl",
value=website_bucket.bucket_website_url)
def __init__(self, scope: Construct, id: str, vpc: _ec2.IVpc,
codebucket: IBucket, s3_deploy, metrics) -> None:
super().__init__(scope, id)
self._metrics_mapping = CfnMapping(
self,
'AnonymousData',
mapping={'SendAnonymousData': {
'Data': 'Yes'
}})
self._metrics_condition = CfnCondition(
self,
'AnonymousDatatoAWS',
expression=Fn.condition_equals(
self._metrics_mapping.find_in_map('SendAnonymousData', 'Data'),
'Yes'))
self._helper_func = _lambda.SingletonFunction(
self,
'SolutionHelper',
uuid='75248a81-9138-468c-9ba1-bca6c7137599',
runtime=_lambda.Runtime.PYTHON_3_8,
handler='lambda_function.handler',
description=
'This function generates UUID for each deployment and sends anonymous data to the AWS Solutions team',
code=_lambda.Code.from_bucket(bucket=codebucket,
key='app_code/solution_helper.zip'),
vpc=vpc,
timeout=Duration.seconds(30))
self._helper_func.add_dependency(s3_deploy)
self._lambda_provider = _custom_resources.Provider(
self,
'LambdaProvider',
on_event_handler=self._helper_func,
vpc=vpc)
self._uuid = CustomResource(
self,
'UUIDCustomResource',
service_token=self._lambda_provider.service_token,
properties={"Resource": "UUID"},
resource_type="Custom::CreateUUID",
removal_policy=RemovalPolicy.DESTROY)
self._send_data = CustomResource(
self,
'SendDataCustomResource',
service_token=self._lambda_provider.service_token,
properties={
"Resource": "AnonymousMetric",
"UUID": self._uuid.get_att_string("UUID"),
"Solution": metrics["Solution"],
"Data": metrics
},
resource_type='Custom::AnonymousData',
removal_policy=RemovalPolicy.DESTROY)
self._send_data.node.add_dependency(self._uuid)
Aspects.of(self._helper_func).add(Condition(self._metrics_condition))
Aspects.of(self._uuid).add(Condition(self._metrics_condition))
Aspects.of(self._send_data).add(Condition(self._metrics_condition))
def __init__(self, scope: core.Construct, id: str, elastic: Elastic, vpc: ec2.Vpc, roles: list, cluster: eks.Cluster, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
sm_policy = iam.PolicyStatement(
actions=["secretsmanager:GetSecretValue"],
effect=iam.Effect.ALLOW,
resources=[elastic.secret.secret_arn]
)
es_policy = iam.PolicyStatement(
actions=["es:DescribeElasticsearchDomain"],
effect=iam.Effect.ALLOW,
resources=[elastic.domain.domain_arn]
)
function = lbd.SingletonFunction(
self,
"ElasticsearchConfigFunction",
uuid="e579d5f9-1709-43ea-b75f-9d1452ca7690",
code=lbd.Code.from_asset(
"custom_resources/elasticsearch/"
),
handler="config.handler",
runtime=lbd.Runtime.PYTHON_3_7,
function_name="elasticsearchConfig",
initial_policy=[sm_policy,es_policy],
log_retention=logs.RetentionDays.ONE_DAY,
security_group=ec2.SecurityGroup.from_security_group_id(self, "lambdaVPC", vpc.vpc_default_security_group),
timeout=core.Duration.seconds(30),
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
one_per_az=True
)
)
provider = cr.Provider(
self, "ElasticsearchConfigProvider",
on_event_handler=function,
log_retention=logs.RetentionDays.ONE_DAY
)
core.CustomResource(
self, "ElasticSearchConfig",
service_token=provider.service_token,
properties={
"domain": elastic.domain.domain_name,
"secret": elastic.secret.secret_arn,
"roles": [role.role_arn for role in roles],
"shards": self.node.try_get_context("elastic")['shards'],
"user": boto3.client('sts').get_caller_identity().get('Arn'),
"replicas": self.node.try_get_context("elastic")['replicas']
}
)
manifests = []
for namespace in self.node.try_get_context("kubernetes")['namespaces']:
manifests.append({
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": "elasticsearch",
"namespace": namespace
},
"data": {
"url": elastic.domain.domain_endpoint
}
})
eks.KubernetesManifest(
self,
"elastic-search-cm",
cluster=cluster,
manifest=manifests
)
def __init__(self, scope: core.Construct, id: str, cluster: eks.Cluster, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
maps= []
self.roles=[]
ecr_policy = iam.PolicyStatement(
actions=[
"ecr:DescribeImages",
"ecr:ListImages",
"ecr:BatchDeleteImage"
],
effect=iam.Effect.ALLOW,
resources=[
"arn:aws:ecr:%s:%s:repository/%s" % (core.Stack.of(self).region, core.Stack.of(self).account, namespace) for namespace in self.node.try_get_context("kubernetes")['namespaces']
]
)
function = lbd.SingletonFunction(
self,
"ECRDeleteImagesFunction",
uuid="19411b0e-0e80-4ad4-a316-3235940775e4",
code=lbd.Code.from_asset(
"custom_resources/kubernetes/"
),
handler="config.handler",
runtime=lbd.Runtime.PYTHON_3_7,
function_name="kubernetesConfig",
initial_policy=[ecr_policy],
log_retention=logs.RetentionDays.ONE_DAY,
timeout=core.Duration.seconds(30)
)
provider = cr.Provider(
self, "ECRDeleteImagesFunctionProvider",
on_event_handler=function,
log_retention=logs.RetentionDays.ONE_DAY
)
repositores = []
for namespace in self.node.try_get_context("kubernetes")['namespaces']:
manifest = cluster.add_manifest(
"eksConfigNamespace-%s" % namespace,
{
"apiVersion": "v1",
"kind": "Namespace",
"metadata": {
"name": namespace
}
}
)
sa = cluster.add_service_account(
"service-account-%s" % namespace,
name="statement-demo",
namespace=namespace
)
sa.node.add_dependency(manifest)
self.roles.append(sa.role)
repository = ecr.Repository(
self, ("repository-%s" % namespace),
removal_policy=core.RemovalPolicy.DESTROY,
repository_name=namespace,
lifecycle_rules=[ecr.LifecycleRule(max_image_count=1)]
)
repositores.append(repository.repository_arn)
maps.append({
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": "application.properties",
"namespace": namespace
},
"data": {
"application-aws.properties": Path("../%s/src/main/resources/application-aws.properties" % namespace).read_text()
}
})
core.CustomResource(
self, "ECRDeleteImagesFunction-%s" % namespace,
service_token=provider.service_token,
properties={
"repository": namespace
}
).node.add_dependency(repository)
eks.KubernetesManifest(
self,
"eksConfigMaps",
cluster=cluster,
manifest=maps
)
iam.Policy(
self, "saPolicy",
force=True,
policy_name="EKSSAPolicy",
roles=self.roles,
statements=[
iam.PolicyStatement(
actions=["cloudwatch:PutMetricData"],
conditions={
"StringEquals": {
"cloudwatch:namespace": "statement12"
},
},
resources=["*"]
)
]
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
lambda_policies = [
iam.PolicyStatement(actions=[
"logs:CreateLogStream", "logs:PutLogEvents",
"logs:CreateLogGroup"
],
effect=iam.Effect.ALLOW,
resources=[
"arn:aws:logs:" + core.Aws.REGION + ":" +
core.Aws.ACCOUNT_ID + ":*"
]),
iam.PolicyStatement(actions=["dynamodb:*"],
effect=iam.Effect.ALLOW,
resources=[
"arn:aws:dynamodb:" + core.Aws.REGION +
":" + core.Aws.ACCOUNT_ID + ":*"
])
]
table = _dynamodb.Table(
self,
'VisitTable',
partition_key={
'name': 'id',
'type': _dynamodb.AttributeType.STRING
},
removal_policy=core.RemovalPolicy.DESTROY,
read_capacity=5,
write_capacity=5,
)
# Modify the config.js with CF custome resource
modify_policy = [
iam.PolicyStatement(actions=["dynamodb:*"],
effect=iam.Effect.ALLOW,
resources=[
"arn:aws:dynamodb:" + core.Aws.REGION +
":" + core.Aws.ACCOUNT_ID + ":*"
])
]
resource = _cfn.CustomResource(
self,
"VisitDataImportCustomResource",
provider=_cfn.CustomResourceProvider.lambda_(
_lambda.SingletonFunction(
self,
"CustomResourceSingleton",
uuid="f7d4f730-4ee1-11e8-9c2d-fa7ae01bbebc",
code=self.custom_resource,
handler="index.handler",
timeout=core.Duration.seconds(300),
runtime=_lambda.Runtime.PYTHON_3_7,
initial_policy=modify_policy)),
properties={"DynamoDBTable": table.table_name})
base_lambda = _lambda.Function(
self,
'ApiPetclinicVisitLambda',
handler=
'org.springframework.samples.petclinic.visits.StreamLambdaHandler::handleRequest',
runtime=_lambda.Runtime.JAVA_8,
code=self.lambda_code,
memory_size=1024,
timeout=core.Duration.seconds(300),
initial_policy=lambda_policies,
environment={
"DYNAMODB_TABLE_NAME": table.table_name,
"SERVER_SERVLET_CONTEXT_PATH": "/api/visit"
})
version = base_lambda.add_version(str(round(time.time())))
alias = _lambda.Alias(self,
'ApiPetclinicVisitLambdaAlias',
alias_name='Prod',
version=version,
provisioned_concurrent_executions=5)
_deploy.LambdaDeploymentGroup(
self,
'ApiPetclinicVisitDeploymentGroup',
alias=alias,
deployment_config=_deploy.LambdaDeploymentConfig.
LINEAR_10_PERCENT_EVERY_1_MINUTE)
def __init__(self, scope: core.Construct, id: str,
log_bucket: _s3.Bucket,
config_table: _dynamodb.Table,
tshirt_size: str,
sink_bucket: _s3.Bucket,
vpc: _ec2.Vpc,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
service_role = _iam.Role(
self, 'BatchEmrServiceRole',
assumed_by=_iam.ServicePrincipal('elasticmapreduce.amazonaws.com')
)
service_role.add_managed_policy(_iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AmazonElasticMapReduceRole'))
cluster_role = _iam.Role(
self, 'BatchEmrClusterRole',
assumed_by=_iam.ServicePrincipal("ec2.amazonaws.com")
)
_iam.Policy(
self, 'BatchEmrClusterPolicy',
statements=[
_iam.PolicyStatement(
actions=[
"glue:CreateDatabase",
"glue:UpdateDatabase",
"glue:DeleteDatabase",
"glue:GetDatabase",
"glue:GetDatabases",
"glue:CreateTable",
"glue:UpdateTable",
"glue:DeleteTable",
"glue:GetTable",
"glue:GetTables",
"glue:GetTableVersions",
"glue:CreatePartition",
"glue:BatchCreatePartition",
"glue:UpdatePartition",
"glue:DeletePartition",
"glue:BatchDeletePartition",
"glue:GetPartition",
"glue:GetPartitions",
"glue:BatchGetPartition",
"glue:CreateUserDefinedFunction",
"glue:UpdateUserDefinedFunction",
"glue:DeleteUserDefinedFunction",
"glue:GetUserDefinedFunction",
"glue:GetUserDefinedFunctions",
"cloudwatch:PutMetricData",
"dynamodb:ListTables",
"s3:HeadBucket",
"ec2:Describe*",
],
resources=['*']
),
_iam.PolicyStatement(
actions=['s3:GetObject'],
resources=[
'arn:aws:s3:::' + ARA_BUCKET_NAME + BINARIES + DataGenConfig.DSDGEN_INSTALL_SCRIPT,
'arn:aws:s3:::' + ARA_BUCKET_NAME + BINARIES + DataGenConfig.JAR_FILE
]
),
_iam.PolicyStatement(
actions=['s3:PutObject'],
resources=[log_bucket.bucket_arn + "/data-generator/*"]
),
_iam.PolicyStatement(
actions=[
"s3:AbortMultipartUpload",
"s3:CreateBucket",
"s3:DeleteObject",
"s3:GetBucketVersioning",
"s3:GetObject",
"s3:GetObjectTagging",
"s3:GetObjectVersion",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:ListBucketVersions",
"s3:ListMultipartUploadParts",
"s3:PutBucketVersioning",
"s3:PutObject",
"s3:PutObjectTagging"
],
resources=[
sink_bucket.bucket_arn + '/*',
sink_bucket.bucket_arn
]
)
],
roles=[cluster_role]
)
cluster_role.add_managed_policy(_iam.ManagedPolicy.from_aws_managed_policy_name('AmazonSSMManagedInstanceCore'))
_iam.CfnInstanceProfile(
self, 'BatchEmrClusterInstanceProfile',
roles=[cluster_role.role_name],
instance_profile_name=cluster_role.role_name
)
# Security Groups for the EMR cluster (private subnet)
# https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-man-sec-groups.html#emr-sg-elasticmapreduce-master-private
master_sg = _ec2.SecurityGroup(self, 'ElasticMapReduce-Master-Private', vpc=vpc)
slave_sg = _ec2.SecurityGroup(self, 'ElasticMapReduce-Slave-Private', vpc=vpc)
service_sg = _ec2.SecurityGroup(self, 'ElasticMapReduce-ServiceAccess', vpc=vpc, allow_all_outbound=False)
# Service SG used by the proxy instance
service_sg.add_ingress_rule(master_sg, _ec2.Port.tcp(9443))
service_sg.add_egress_rule(master_sg, _ec2.Port.tcp(8443))
service_sg.add_egress_rule(slave_sg, _ec2.Port.tcp(8443))
# EMR Master
master_sg.add_ingress_rule(master_sg, _ec2.Port.all_icmp())
master_sg.add_ingress_rule(master_sg, _ec2.Port.all_tcp())
master_sg.add_ingress_rule(master_sg, _ec2.Port.all_udp())
master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_icmp())
master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_tcp())
master_sg.add_ingress_rule(slave_sg, _ec2.Port.all_udp())
master_sg.add_ingress_rule(service_sg, _ec2.Port.tcp(8443))
# EMR Slave
slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_icmp())
slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_tcp())
slave_sg.add_ingress_rule(master_sg, _ec2.Port.all_udp())
slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_icmp())
slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_tcp())
slave_sg.add_ingress_rule(slave_sg, _ec2.Port.all_udp())
slave_sg.add_ingress_rule(service_sg, _ec2.Port.tcp(8443))
with open('common/common_cdk/lambda/datagen_config.py', 'r') as f:
lambda_source = f.read()
configure_datagen_function = _lambda.SingletonFunction(
self, 'BatchConfigureDatagenLambda',
uuid="58a9a222-ff07-11ea-adc1-0242ac120002",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.inline(lambda_source),
handler='index.handler',
function_name='datagen-config',
environment={
'TABLE_NAME': config_table.table_name,
'JAR_LOCATION': BINARIES_LOCATION + DataGenConfig.JAR_FILE,
},
timeout=core.Duration.seconds(10)
)
configure_datagen_function.role.add_to_policy(
_iam.PolicyStatement(
actions=[
'dynamodb:GetItem',
'dynamodb:PutItem',
],
resources=[config_table.table_arn]
)
)
terminate_cluster = _sfn_tasks.EmrTerminateCluster(
self, 'BatchDeleteCluster',
cluster_id=_sfn.TaskInput.from_data_at("$.Emr.Cluster.Id").value,
integration_pattern=_sfn.IntegrationPattern.RUN_JOB,
)
terminate_cluster_error = _sfn_tasks.EmrTerminateCluster(
self, 'BatchDeleteClusterError',
cluster_id=_sfn.TaskInput.from_data_at("$.Emr.Cluster.Id").value,
integration_pattern=_sfn.IntegrationPattern.RUN_JOB,
).next(_sfn.Fail(self, 'StepFailure'))
create_cluster = _sfn_tasks.EmrCreateCluster(
self, "BatchCreateEMRCluster",
name="BatchDatagenCluster",
result_path="$.Emr",
release_label='emr-5.30.1',
log_uri=log_bucket.s3_url_for_object() + "/data-generator",
cluster_role=cluster_role,
service_role=service_role,
bootstrap_actions=[
_sfn_tasks.EmrCreateCluster.BootstrapActionConfigProperty(
name="dsdgen-install",
script_bootstrap_action=_sfn_tasks.EmrCreateCluster.ScriptBootstrapActionConfigProperty(
path=BINARIES_LOCATION + DataGenConfig.DSDGEN_INSTALL_SCRIPT,
)
)
],
applications=[
_sfn_tasks.EmrCreateCluster.ApplicationConfigProperty(
name="spark"
),
_sfn_tasks.EmrCreateCluster.ApplicationConfigProperty(
name="hadoop"
)
],
instances=_sfn_tasks.EmrCreateCluster.InstancesConfigProperty(
emr_managed_master_security_group=master_sg.security_group_id,
emr_managed_slave_security_group=slave_sg.security_group_id,
service_access_security_group=service_sg.security_group_id,
ec2_subnet_ids=vpc.select_subnets().subnet_ids,
instance_fleets=[
_sfn_tasks.EmrCreateCluster.InstanceFleetConfigProperty(
instance_fleet_type=_sfn_tasks.EmrCreateCluster.InstanceRoleType.MASTER,
instance_type_configs=[
_sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty(
instance_type='m5.xlarge',
weighted_capacity=1
),
_sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty(
instance_type='m5a.xlarge',
weighted_capacity=1
),
_sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty(
instance_type='m4.xlarge',
weighted_capacity=1
),
_sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty(
instance_type='m5d.xlarge',
weighted_capacity=1
),
],
launch_specifications=_sfn_tasks.EmrCreateCluster.InstanceFleetProvisioningSpecificationsProperty(
spot_specification=_sfn_tasks.EmrCreateCluster.SpotProvisioningSpecificationProperty(
timeout_action=_sfn_tasks.EmrCreateCluster.SpotTimeoutAction.SWITCH_TO_ON_DEMAND,
timeout_duration_minutes=5
)
),
target_on_demand_capacity=0,
target_spot_capacity=1
),
_sfn_tasks.EmrCreateCluster.InstanceFleetConfigProperty(
instance_fleet_type=_sfn_tasks.EmrCreateCluster.InstanceRoleType.CORE,
instance_type_configs=[
_sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty(
instance_type='m5.xlarge',
weighted_capacity=1
),
_sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty(
instance_type='m5.2xlarge',
weighted_capacity=2
),
_sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty(
instance_type='m5a.xlarge',
weighted_capacity=1
),
_sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty(
instance_type='m5a.2xlarge',
weighted_capacity=2
),
_sfn_tasks.EmrCreateCluster.InstanceTypeConfigProperty(
instance_type='m4.xlarge',
weighted_capacity=1
)
],
launch_specifications=_sfn_tasks.EmrCreateCluster.InstanceFleetProvisioningSpecificationsProperty(
spot_specification=_sfn_tasks.EmrCreateCluster.SpotProvisioningSpecificationProperty(
timeout_action=_sfn_tasks.EmrCreateCluster.SpotTimeoutAction.SWITCH_TO_ON_DEMAND,
timeout_duration_minutes=5
)
),
target_on_demand_capacity=0,
target_spot_capacity=DataGenConfig.BATCH_CLUSTER_SIZE[tshirt_size]
)
]
)
).add_catch(handler=terminate_cluster_error, result_path="$.error")
configure_datagen = _sfn_tasks.LambdaInvoke(
self, "BatchConfigureDatagenTask",
lambda_function=configure_datagen_function,
payload=_sfn.TaskInput.from_text('{'
'"Param": "batch_iterator",'
'"Module": "batch",'
'"SinkBucket": "'+sink_bucket.s3_url_for_object()+'",'
'"Parallelism": "'+str(int(DataGenConfig.BATCH_DATA_SIZE[tshirt_size])*2)+'",'
'"DataSize": "'+DataGenConfig.BATCH_DATA_SIZE[tshirt_size]+'",'
'"TmpBucket": "fake-bucket"'
'}'),
result_path='$.Config'
).add_catch(handler=terminate_cluster_error, result_path="$.error")
add_datagen_step = _sfn.CustomState(
self, 'BatchAddDataGenStep',
state_json={
"Type": "Task",
"Resource": "arn:aws:states:::elasticmapreduce:addStep.sync",
"Parameters": {
"ClusterId.$": "$.Emr.Cluster.Id",
"Step": {
"Name": "DatagenStep",
"ActionOnFailure": "CONTINUE",
"HadoopJarStep": {
"Jar": "command-runner.jar",
"Args.$": "$.Config.Payload.StepParam"
}
}
},
"ResultPath": "$.Step",
"Next": "BatchUpdateIterator",
"Catch": [
{
"ErrorEquals": ["States.ALL"],
"Next": "BatchDeleteClusterError",
"ResultPath": "$.error"
}
]
}
)
update_iterator = _sfn_tasks.DynamoUpdateItem(
self, 'BatchUpdateIterator',
table=config_table,
key={
'param': _sfn_tasks.DynamoAttributeValue.from_string('batch_iterator')
},
update_expression='SET iterator = if_not_exists(iterator, :start) + :inc',
expression_attribute_values={
":inc": _sfn_tasks.DynamoAttributeValue.from_number(1),
":start": _sfn_tasks.DynamoAttributeValue.from_number(0)
},
result_path=_sfn.JsonPath.DISCARD
)
definition = configure_datagen \
.next(create_cluster) \
.next(add_datagen_step) \
.next(update_iterator) \
.next(terminate_cluster)
datagen_stepfunctions = _sfn.StateMachine(
self, "BatchDataGenStepFunctions",
definition=definition,
timeout=core.Duration.minutes(30)
)
datagen_stepfunctions.add_to_role_policy(
_iam.PolicyStatement(
actions=[
'elasticmapreduce:AddJobFlowSteps',
'elasticmapreduce:DescribeStep'
],
resources=['*']
)
)
datagen_stepfunctions.add_to_role_policy(
_iam.PolicyStatement(
actions= [
"iam:CreateServiceLinkedRole",
"iam:PutRolePolicy"
],
resources=["arn:aws:iam::*:role/aws-service-role/elasticmapreduce.amazonaws.com*/AWSServiceRoleForEMRCleanup*"],
conditions= {
"StringLike": {
"iam:AWSServiceName": [
"elasticmapreduce.amazonaws.com",
"elasticmapreduce.amazonaws.com.cn"
]
}
}
)
)
step_trigger = _events.Rule(
self, 'BatchSteptrigger',
schedule=_events.Schedule.cron(minute='0/30',
hour='*',
month='*',
week_day='*',
year='*')
)
step_trigger.add_target(_events_targets.SfnStateMachine(machine=datagen_stepfunctions))
with open('common/common_cdk/lambda/stepfunctions_trigger.py', 'r') as f:
lambda_source = f.read()
stepfunctions_trigger_lambda = _lambda.SingletonFunction(
self, 'BatchStepFunctionsTriggerLambda',
uuid="9597f6f2-f840-11ea-adc1-0242ac120002",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.inline(lambda_source),
handler='index.handler',
function_name='stepfunctions-batch-datagen-trigger'
)
stepfunctions_trigger_lambda.role.add_to_policy(
_iam.PolicyStatement(
actions=["states:StartExecution"],
resources=['*']
)
)
trigger_step_lambda_provider = _custom_resources.Provider(
self, 'StepFunctionsTriggerLambdaProvider',
on_event_handler=stepfunctions_trigger_lambda
)
core.CustomResource(
self, 'StepFunctionsTrigger',
service_token=trigger_step_lambda_provider.service_token,
properties={
"stepArn": datagen_stepfunctions.state_machine_arn
}
)
# terminate clusters
with open('common/common_cdk/lambda/stepfunctions_terminate_emr.py', 'r') as f:
lambda_source = f.read()
sfn_terminate = _lambda.SingletonFunction(
self, 'StepFuncTerminateBatch',
uuid='58a9a422-ff07-11ea-adc1-0242ac120002',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.inline(lambda_source),
handler='index.handler',
timeout=core.Duration.minutes(5)
)
sfn_terminate.role.add_to_policy(
_iam.PolicyStatement(
actions=[
'elasticmapreduce:ListClusters',
'elasticmapreduce:TerminateJobFlows',
'states:ListStateMachines',
'states:ListExecutions',
'states:StopExecution'
],
resources=['*']
)
)
sfn_terminate_provider = _custom_resources.Provider(
self, 'StepFuncTerminateBatchLambdaProvider',
on_event_handler=sfn_terminate
)
core.CustomResource(
self, 'StepFuncTerminateBatchCustomResource',
service_token=sfn_terminate_provider.service_token,
properties={
"state_machine": 'BatchDatagen'
})
