def __init__(self, scope: Construct, id: str, envs: EnvSettings):
super().__init__(scope, id)
self.key = Key(self, id="Key", alias=f"alias/{envs.project_name}")
self.key.add_to_resource_policy(
PolicyStatement(actions=["kms:Encrypt", "kms:Decrypt"],
principals=[AccountRootPrincipal()],
resources=["*"]))
CfnOutput(
self,
"KmsKeyArnOutput",
export_name=self.get_kms_arn_output_export_name(envs),
value=self.key.key_arn,
)
def _create_ecr_repos(self) -> List[ecr.Repository]:
current_images_names = extract_images_names(env_name=self.context.name)
current_images_names = list(set(current_images_names) - set(self.remove_images))
repos = [self.create_repo(image_name=r) for r in current_images_names]
for image_name in self.add_images:
if image_name not in current_images_names:
repos.append(self.create_repo(image_name=image_name))
current_images_names.append(image_name)
if current_images_names:
current_images_names.sort()
CfnOutput(
scope=self,
id="repos",
export_name=f"orbit-{self.context.name}-repos",
value=",".join([x for x in current_images_names]),
)
return repos
def __init__(
self, scope: Construct, id: str, shared_airflow_env: dict, **kwargs
) -> None:
super().__init__(scope, id, **kwargs)
task_definition = FargateTaskDefinition(
self,
"initdb",
cpu=512,
memory_limit_mib=1024,
)
task_definition.add_container(
"initdb",
image=ContainerImage.from_registry("apache/airflow:1.10.12-python3.8"),
command=["initdb"],
logging=LogDriver.aws_logs(stream_prefix="initdb"),
environment=shared_airflow_env,
)
CfnOutput(self, "init-task-def", value=task_definition.task_definition_arn)
def __init__(self, scope: Construct, construct_id: str, stage: str,
**kwargs) -> None:
super().__init__(scope, construct_id)
_access_logs_bucket = self.get_access_logs_bucket(stage)
_staticsite_bucket = self.create_bucket(_access_logs_bucket, stage)
_cfront_oai = self.create_origin_access_identity()
_policy_statement = self.create_s3_cfront_policy(
_staticsite_bucket, _cfront_oai)
_staticsite_bucket.add_to_resource_policy(_policy_statement)
_cfront_dist = _cfront.CloudFrontWebDistribution(
self,
"staticsitedistribution",
http_version=_cfront.HttpVersion.HTTP2,
origin_configs=[
_cfront.SourceConfiguration(
behaviors=[
_cfront.Behavior(
allowed_methods=_cfront.CloudFrontAllowedMethods.
GET_HEAD_OPTIONS,
is_default_behavior=True)
],
s3_origin_source=_cfront.S3OriginConfig(
s3_bucket_source=_staticsite_bucket),
)
],
default_root_object="index.html",
viewer_protocol_policy=_cfront.ViewerProtocolPolicy.
REDIRECT_TO_HTTPS,
price_class=_cfront.PriceClass.PRICE_CLASS_ALL,
enable_ip_v6=False,
)
self.bucket = _staticsite_bucket
self.access_logs_bucket = _access_logs_bucket
self.cfront_dist = _cfront_dist
self.sourcebucketname = CfnOutput(self,
"sourceBucketName",
value=_staticsite_bucket.bucket_name)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
functions_hello = lambda_function.Function(
self,
"IntroCdkpipelinesStack_HelloHandler",
function_name="IntroCdkpipelinesStack_HelloHandler",
runtime=lambda_function.Runtime.PYTHON_3_7,
code=lambda_function.Code.asset("lambda"),
handler="hello.handler")
api_gw_hello = apigw.LambdaRestApi(
self,
"IntroCdkpipelinesStack_HelloApi",
rest_api_name="IntroCdkpipelinesStack_HelloApi",
description="Endpoint for a simple Lambda-powered web service",
handler=functions_hello)
self.output_url = CfnOutput(self,
"IntroCdkpipelinesStack_HelloApi__url",
value=api_gw_hello.url)
def __init__(
self, scope: Construct, id: str, vpc: Vpc, rds: Rds, batch, **kwargs
) -> None:
super().__init__(scope, id, **kwargs)
cluster = _Cluster(self, "cluster", vpc=vpc.vpc)
cluster.add_capacity(
"ag",
instance_type=InstanceType.of(InstanceClass.STANDARD5, InstanceSize.LARGE),
)
shared_airflow_env = {
"AIRFLOW__CORE__SQL_ALCHEMY_CONN": rds.uri,
"AIRFLOW__CORE__LOAD_EXAMPLES": "True",
"AIRFLOW__CORE__EXECUTOR": "LocalExecutor",
}
init_db = InitTask(self, "init_db", shared_airflow_env=shared_airflow_env)
scheduler_service = SchedulerService(
self,
"scheduler_service",
cluster=cluster,
shared_airflow_env=shared_airflow_env,
)
webserver_service = WebserverService(
self,
"webserver_service",
cluster=cluster,
shared_airflow_env=shared_airflow_env,
vpc=vpc.vpc,
)
CfnOutput(self, "cluster-name", value=cluster.cluster_name)
def __init__(self, scope: Construct, id: str, vpc: Vpc, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.rds = DatabaseInstance(
self,
"rds",
master_username=USER,
master_user_password=SecretValue(PASSWORD),
database_name=DATABASE,
vpc=vpc.vpc,
engine=DatabaseInstanceEngine.POSTGRES,
port=PORT,
instance_type=InstanceType.of(InstanceClass.BURSTABLE3,
InstanceSize.MICRO),
security_groups=[vpc.sg_rds],
deletion_protection=False,
)
CfnOutput(
self,
"rds_address",
value=
f"postgres://{USER}:{PASSWORD}@{self.rds.db_instance_endpoint_address}:{PORT}/{DATABASE}",
)
def __init__(self, scope: core.Construct, id: str, application_prefix: str,
suffix: str, kda_role: Role, **kwargs):
super().__init__(scope, id, **kwargs)
stack = Stack.of(self)
region = stack.region
# Create Cognito User Pool
self.__user_pool = CfnUserPool(
scope=self,
id='UserPool',
admin_create_user_config={'allowAdminCreateUserOnly': True},
policies={'passwordPolicy': {
'minimumLength': 8
}},
username_attributes=['email'],
auto_verified_attributes=['email'],
user_pool_name=application_prefix + '_user_pool')
# Create a Cognito User Pool Domain using the newly created Cognito User Pool
CfnUserPoolDomain(scope=self,
id='CognitoDomain',
domain=application_prefix + '-' + suffix,
user_pool_id=self.user_pool.ref)
# Create Cognito Identity Pool
self.__id_pool = CfnIdentityPool(
scope=self,
id='IdentityPool',
allow_unauthenticated_identities=False,
cognito_identity_providers=[],
identity_pool_name=application_prefix + '_identity_pool')
trust_relationship = FederatedPrincipal(
federated='cognito-identity.amazonaws.com',
conditions={
'StringEquals': {
'cognito-identity.amazonaws.com:aud': self.id_pool.ref
},
'ForAnyValue:StringLike': {
'cognito-identity.amazonaws.com:amr': 'authenticated'
}
},
assume_role_action='sts:AssumeRoleWithWebIdentity')
# IAM role for master user
master_auth_role = Role(scope=self,
id='MasterAuthRole',
assumed_by=trust_relationship)
# Role for authenticated user
limited_auth_role = Role(scope=self,
id='LimitedAuthRole',
assumed_by=trust_relationship)
# Attach Role to Identity Pool
CfnIdentityPoolRoleAttachment(
scope=self,
id='userPoolRoleAttachment',
identity_pool_id=self.id_pool.ref,
roles={'authenticated': limited_auth_role.role_arn})
# Create master-user-group
CfnUserPoolGroup(scope=self,
id='AdminsGroup',
user_pool_id=self.user_pool.ref,
group_name='master-user-group',
role_arn=master_auth_role.role_arn)
# Create limited-user-group
CfnUserPoolGroup(scope=self,
id='UsersGroup',
user_pool_id=self.user_pool.ref,
group_name='limited-user-group',
role_arn=limited_auth_role.role_arn)
# Role for the Elasticsearch service to access Cognito
es_role = Role(scope=self,
id='EsRole',
assumed_by=ServicePrincipal(service='es.amazonaws.com'),
managed_policies=[
ManagedPolicy.from_aws_managed_policy_name(
'AmazonESCognitoAccess')
])
# Use the following command line to generate the python dependencies layer content
# pip3 install -t lambda-layer/python/lib/python3.8/site-packages -r lambda/requirements.txt
# Build the lambda layer assets
subprocess.call([
'pip', 'install', '-t',
'streaming/streaming_cdk/lambda-layer/python/lib/python3.8/site-packages',
'-r', 'streaming/streaming_cdk/bootstrap-lambda/requirements.txt',
'--upgrade'
])
requirements_layer = _lambda.LayerVersion(
scope=self,
id='PythonRequirementsTemplate',
code=_lambda.Code.from_asset(
'streaming/streaming_cdk/lambda-layer'),
compatible_runtimes=[_lambda.Runtime.PYTHON_3_8])
# This lambda function will bootstrap the Elasticsearch cluster
bootstrap_function_name = 'AESBootstrap'
register_template_lambda = _lambda.Function(
scope=self,
id='RegisterTemplate',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset(
'streaming/streaming_cdk/bootstrap-lambda'),
handler='es-bootstrap.lambda_handler',
environment={
'REGION': region,
'KDA_ROLE_ARN': kda_role.role_arn,
'MASTER_ROLE_ARN': master_auth_role.role_arn
},
layers=[requirements_layer],
timeout=Duration.minutes(15),
function_name=bootstrap_function_name)
lambda_role = register_template_lambda.role
lambda_role.add_to_policy(
PolicyStatement(
actions=['logs:CreateLogGroup'],
resources=[stack.format_arn(service='logs', resource='*')]))
lambda_role.add_to_policy(
PolicyStatement(
actions=['logs:CreateLogStream', 'logs:PutLogEvents'],
resources=[
stack.format_arn(service='logs',
resource='log_group',
resource_name='/aws/lambda/' +
bootstrap_function_name + ':*')
]))
# Let the lambda assume the master role so that actions can be executed on the cluster
# https://aws.amazon.com/premiumsupport/knowledge-center/lambda-function-assume-iam-role/
lambda_role.add_to_policy(
PolicyStatement(actions=['sts:AssumeRole'],
resources=[master_auth_role.role_arn]))
master_auth_role.assume_role_policy.add_statements(
PolicyStatement(actions=['sts:AssumeRole'],
principals=[lambda_role]))
# List all the roles that are allowed to access the Elasticsearch cluster.
roles = [
ArnPrincipal(limited_auth_role.role_arn),
ArnPrincipal(master_auth_role.role_arn),
ArnPrincipal(kda_role.role_arn)
] # The users
if register_template_lambda and register_template_lambda.role:
roles.append(ArnPrincipal(
lambda_role.role_arn)) # The lambda used to bootstrap
# Create kms key
kms_key = Key(scope=self,
id='kms-es',
alias='custom/es',
description='KMS key for Elasticsearch domain',
enable_key_rotation=True)
# AES Log Groups
es_app_log_group = logs.LogGroup(scope=self,
id='EsAppLogGroup',
retention=logs.RetentionDays.ONE_WEEK,
removal_policy=RemovalPolicy.RETAIN)
# Create the Elasticsearch domain
es_domain_arn = stack.format_arn(service='es',
resource='domain',
resource_name=application_prefix +
'/*')
es_access_policy = PolicyDocument(statements=[
PolicyStatement(principals=roles,
actions=[
'es:ESHttpGet', 'es:ESHttpPut',
'es:ESHttpPost', 'es:ESHttpDelete'
],
resources=[es_domain_arn])
])
self.__es_domain = es.CfnDomain(
scope=self,
id='searchDomain',
elasticsearch_cluster_config={
'instanceType': 'r5.large.elasticsearch',
'instanceCount': 2,
'dedicatedMasterEnabled': True,
'dedicatedMasterCount': 3,
'dedicatedMasterType': 'r5.large.elasticsearch',
'zoneAwarenessEnabled': True,
'zoneAwarenessConfig': {
'AvailabilityZoneCount': '2'
},
},
encryption_at_rest_options={
'enabled': True,
'kmsKeyId': kms_key.key_id
},
node_to_node_encryption_options={'enabled': True},
ebs_options={
'volumeSize': 10,
'ebsEnabled': True
},
elasticsearch_version='7.9',
domain_name=application_prefix,
access_policies=es_access_policy,
cognito_options={
'enabled': True,
'identityPoolId': self.id_pool.ref,
'roleArn': es_role.role_arn,
'userPoolId': self.user_pool.ref
},
advanced_security_options={
'enabled': True,
'internalUserDatabaseEnabled': False,
'masterUserOptions': {
'masterUserArn': master_auth_role.role_arn
}
},
domain_endpoint_options={
'enforceHttps': True,
'tlsSecurityPolicy': 'Policy-Min-TLS-1-2-2019-07'
},
# log_publishing_options={
# # 'ES_APPLICATION_LOGS': {
# # 'enabled': True,
# # 'cloud_watch_logs_log_group_arn': es_app_log_group.log_group_arn
# # },
# # 'AUDIT_LOGS': {
# # 'enabled': True,
# # 'cloud_watch_logs_log_group_arn': ''
# # },
# # 'SEARCH_SLOW_LOGS': {
# # 'enabled': True,
# # 'cloud_watch_logs_log_group_arn': ''
# # },
# # 'INDEX_SLOW_LOGS': {
# # 'enabled': True,
# # 'cloud_watch_logs_log_group_arn': ''
# # }
# }
)
# Not yet on the roadmap...
# See https://github.com/aws-cloudformation/aws-cloudformation-coverage-roadmap/issues/283
# self.es_domain.add_property_override('ElasticsearchClusterConfig.WarmEnabled', True)
# self.es_domain.add_property_override('ElasticsearchClusterConfig.WarmCount', 2)
# self.es_domain.add_property_override('ElasticsearchClusterConfig.WarmType', 'ultrawarm1.large.elasticsearch')
# Deny all roles from the authentication provider - users must be added to groups
# This lambda function will bootstrap the Elasticsearch cluster
cognito_function_name = 'CognitoFix'
cognito_template_lambda = _lambda.Function(
scope=self,
id='CognitoFixLambda',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset(
'streaming/streaming_cdk/cognito-lambda'),
handler='handler.handler',
environment={
'REGION': scope.region,
'USER_POOL_ID': self.__user_pool.ref,
'IDENTITY_POOL_ID': self.__id_pool.ref,
'LIMITED_ROLE_ARN': limited_auth_role.role_arn
},
timeout=Duration.minutes(15),
function_name=cognito_function_name)
lambda_role = cognito_template_lambda.role
lambda_role.add_to_policy(
PolicyStatement(
actions=['logs:CreateLogGroup'],
resources=[stack.format_arn(service='logs', resource='*')]))
lambda_role.add_to_policy(
PolicyStatement(
actions=['logs:CreateLogStream', 'logs:PutLogEvents'],
resources=[
stack.format_arn(service='logs',
resource='log_group',
resource_name='/aws/lambda/' +
cognito_function_name + ':*')
]))
lambda_role.add_to_policy(
PolicyStatement(actions=['cognito-idp:ListUserPoolClients'],
resources=[self.user_pool.attr_arn]))
lambda_role.add_to_policy(
PolicyStatement(actions=['iam:PassRole'],
resources=[limited_auth_role.role_arn]))
cognito_id_res = Fn.join(':', [
'arn:aws:cognito-identity', scope.region, scope.account,
Fn.join('/', ['identitypool', self.__id_pool.ref])
])
lambda_role.add_to_policy(
PolicyStatement(actions=['cognito-identity:SetIdentityPoolRoles'],
resources=[cognito_id_res]))
# Get the Domain Endpoint and register it with the lambda as environment variable.
register_template_lambda.add_environment(
'DOMAIN', self.__es_domain.attr_domain_endpoint)
CfnOutput(scope=self,
id='createUserUrl',
description="Create a new user in the user pool here.",
value="https://" + scope.region +
".console.aws.amazon.com/cognito/users?region=" +
scope.region + "#/pool/" + self.user_pool.ref + "/users")
CfnOutput(scope=self,
id='kibanaUrl',
description="Access Kibana via this URL.",
value="https://" + self.__es_domain.attr_domain_endpoint +
"/_plugin/kibana/")
bootstrap_lambda_provider = Provider(
scope=self,
id='BootstrapLambdaProvider',
on_event_handler=register_template_lambda)
CustomResource(scope=self,
id='ExecuteRegisterTemplate',
service_token=bootstrap_lambda_provider.service_token,
properties={'Timeout': 900})
cognito_lambda_provider = Provider(
scope=self,
id='CognitoFixLambdaProvider',
on_event_handler=cognito_template_lambda)
cognito_fix_resource = CustomResource(
scope=self,
id='ExecuteCognitoFix',
service_token=cognito_lambda_provider.service_token)
cognito_fix_resource.node.add_dependency(self.__es_domain)
def _add_private_hosted_zone(self):
if self._condition_custom_cluster_dns():
hosted_zone_id = self.config.scheduling.settings.dns.hosted_zone_id
cluster_hosted_zone = CustomDns(
ref=hosted_zone_id,
name=self.cluster_dns_domain.value_as_string)
else:
cluster_hosted_zone = route53.CfnHostedZone(
self.stack_scope,
"Route53HostedZone",
name=self.cluster_dns_domain.value_as_string,
vpcs=[
route53.CfnHostedZone.VPCProperty(
vpc_id=self.config.vpc_id,
vpc_region=self._stack_region)
],
)
# If Headnode InstanceRole is created by ParallelCluster, add Route53 policy for InstanceRole
head_node_role_info = self.instance_roles.get("HeadNode")
if head_node_role_info:
iam.CfnPolicy(
self.stack_scope,
"ParallelClusterSlurmRoute53Policies",
policy_name="parallelcluster-slurm-route53",
policy_document=iam.PolicyDocument(statements=[
iam.PolicyStatement(
sid="Route53Add",
effect=iam.Effect.ALLOW,
actions=["route53:ChangeResourceRecordSets"],
resources=[
self._format_arn(
service="route53",
region="",
account="",
resource=
f"hostedzone/{cluster_hosted_zone.ref}",
),
],
),
]),
roles=[head_node_role_info.get("RoleRef")],
)
cleanup_route53_lambda_execution_role = None
if self.cleanup_lambda_role:
cleanup_route53_lambda_execution_role = add_lambda_cfn_role(
scope=self.stack_scope,
function_id="CleanupRoute53",
statements=[
iam.PolicyStatement(
actions=[
"route53:ListResourceRecordSets",
"route53:ChangeResourceRecordSets"
],
effect=iam.Effect.ALLOW,
resources=[
self._format_arn(
service="route53",
region="",
account="",
resource=
f"hostedzone/{cluster_hosted_zone.ref}",
),
],
sid="Route53DeletePolicy",
),
get_cloud_watch_logs_policy_statement(
resource=self._format_arn(service="logs",
account="*",
region="*",
resource="*")),
],
)
cleanup_route53_lambda = PclusterLambdaConstruct(
scope=self.stack_scope,
id="CleanupRoute53FunctionConstruct",
function_id="CleanupRoute53",
bucket=self.bucket,
config=self.config,
execution_role=cleanup_route53_lambda_execution_role.attr_arn
if cleanup_route53_lambda_execution_role else
self.config.iam.roles.custom_lambda_resources,
handler_func="cleanup_resources",
).lambda_func
self.cleanup_route53_custom_resource = CfnCustomResource(
self.stack_scope,
"CleanupRoute53CustomResource",
service_token=cleanup_route53_lambda.attr_arn,
)
self.cleanup_route53_custom_resource.add_property_override(
"ClusterHostedZone", cluster_hosted_zone.ref)
self.cleanup_route53_custom_resource.add_property_override(
"Action", "DELETE_DNS_RECORDS")
self.cleanup_route53_custom_resource.add_property_override(
"ClusterDNSDomain", cluster_hosted_zone.name)
CfnOutput(
self.stack_scope,
"ClusterHostedZone",
description=
"Id of the private hosted zone created within the cluster",
value=cluster_hosted_zone.ref,
)
return cluster_hosted_zone
def __init__(self, scope: Construct, id: str, context: "FoundationContext",
**kwargs: Any) -> None:
self.env_name = context.name
self.context = context
super().__init__(scope, id, **kwargs)
Tags.of(scope=cast(core.IConstruct, self)).add(
key="Env", value=f"orbit-{self.env_name}")
self.vpc: ec2.Vpc = self._create_vpc()
self.public_subnets = (self.vpc.select_subnets(
subnet_type=ec2.SubnetType.PUBLIC) if self.vpc.public_subnets else
self.vpc.select_subnets(subnet_name=""))
self.private_subnets = (self.vpc.select_subnets(
subnet_type=ec2.SubnetType.PRIVATE) if self.vpc.private_subnets
else self.vpc.select_subnets(subnet_name=""))
self.isolated_subnets = (self.vpc.select_subnets(
subnet_type=ec2.SubnetType.ISOLATED) if self.vpc.isolated_subnets
else self.vpc.select_subnets(subnet_name=""))
self.nodes_subnets = (self.private_subnets
if context.networking.data.internet_accessible
else self.isolated_subnets)
self._create_vpc_endpoints()
if context.toolkit.s3_bucket is None:
raise ValueError("context.toolkit_s3_bucket is not defined")
toolkit_s3_bucket_name: str = context.toolkit.s3_bucket
acct: str = core.Aws.ACCOUNT_ID
self.bucket_names: Dict[str, Any] = {
"scratch-bucket":
f"orbit-foundation-{self.env_name}-scratch-{acct}-{context.toolkit.deploy_id}",
"toolkit-bucket": toolkit_s3_bucket_name,
}
self._build_kms_key_for_env()
self.scratch_bucket: s3.Bucket = S3Builder.build_s3_bucket(
scope=self,
id="scratch_bucket",
name=self.bucket_names["scratch-bucket"],
scratch_retention_days=30,
kms_key=self.env_kms_key,
)
self.efs_fs = EfsBuilder.build_file_system(
scope=self,
name="orbit-fs",
efs_life_cycle="AFTER_7_DAYS",
vpc=self.vpc,
efs_security_group=self._vpc_security_group,
subnets=self.nodes_subnets.subnets,
team_kms_key=self.env_kms_key,
)
self.user_pool: cognito.UserPool = self._create_user_pool()
self._ssm_parameter = ssm.StringParameter(
self,
id="/orbit/DemoParams",
string_value=json.dumps({
"VpcId":
self.vpc.vpc_id,
"PublicSubnets":
self.public_subnets.subnet_ids,
"PrivateSubnets":
self.private_subnets.subnet_ids,
"IsolatedSubnets":
self.isolated_subnets.subnet_ids,
"NodesSubnets":
self.nodes_subnets.subnet_ids,
"LoadBalancersSubnets":
self.public_subnets.subnet_ids,
"KMSKey":
self.env_kms_key.key_arn,
"SharedEfsFsId":
self.efs_fs.file_system_id,
"ScratchBucketArn":
self.scratch_bucket.bucket_arn,
"ScratchBucketName":
self.scratch_bucket.bucket_name,
"UserPoolId":
self.user_pool.user_pool_id,
"SharedEfsSgId":
self._vpc_security_group.security_group_id,
"UserPoolProviderName":
self.user_pool.user_pool_provider_name,
}),
type=ssm.ParameterType.STRING,
description="Orbit Workbench Demo resources.",
parameter_name=context.resources_ssm_parameter_name,
simple_name=False,
tier=ssm.ParameterTier.INTELLIGENT_TIERING,
)
CfnOutput(
scope=self,
id=f"{id}vpcid",
export_name=f"orbit-foundation-{self.env_name}-vpc-id",
value=self.vpc.vpc_id,
)
CfnOutput(
scope=self,
id=f"{id}publicsubnetsids",
export_name=f"orbit-foundation-{self.env_name}-public-subnet-ids",
value=",".join(self.public_subnets.subnet_ids),
)
CfnOutput(
scope=self,
id=f"{id}privatesubnetsids",
export_name=f"orbit-foundation-{self.env_name}-private-subnet-ids",
value=",".join(self.private_subnets.subnet_ids),
)
CfnOutput(
scope=self,
id=f"{id}isolatedsubnetsids",
export_name=f"orbit-foundation-{self.env_name}-isolated-subnet-ids",
value=",".join(self.isolated_subnets.subnet_ids),
)
CfnOutput(
scope=self,
id=f"{id}nodesubnetsids",
export_name=f"orbit-foundation-{self.env_name}-nodes-subnet-ids",
value=",".join(self.nodes_subnets.subnet_ids),
)
def __init__(self,
scope: Construct,
id: str,
*,
sources: str,
hosted_zone: HostedZone = None,
site_domain: str = None,
website_index: str = "index.html",
website_error: str = "error.html"):
"""
This constructs creates:
- S3 bucket
- DNS Validated certificate
- CloudFront web distribution
- Route53 A record
:param scope: The scope in which to define this construct.
:param id: The scoped construct ID. Must be unique amongst siblings. If the ID includes a path separator (``/``), then it will be replaced by double dash ``--``.
:param hosted_zone: A route53 hosted zone
:param site_domain: The domain or subdomain you want to use for the website
:param sources: A path to the location of the code
:param website_index: Name of the index page, defaults to "index.html"
:param website_error: Name of the error page, defaults to "error.html"
"""
super().__init__(scope, id)
# Construct code goes here
if site_domain:
CfnOutput(self, "Site", value=f"https://{site_domain}")
# Content bucket
self.site_bucket = aws_s3.Bucket(self,
"SiteBucket",
bucket_name=site_domain,
website_index_document=website_index,
website_error_document=website_error,
public_read_access=True,
removal_policy=RemovalPolicy.DESTROY)
CfnOutput(self, "BucketArn", value=self.site_bucket.bucket_arn)
CfnOutput(self,
"BucketWebsiteUrl",
value=self.site_bucket.bucket_website_url)
CfnOutput(self,
"BucketWebsiteDomainName",
value=self.site_bucket.bucket_website_domain_name)
# Certificate
alias_config = None
if hosted_zone:
self.cert = DnsValidatedCertificate(self,
f"{id}-bucket",
domain_name=site_domain,
hosted_zone=hosted_zone)
CfnOutput(self, 'CertificateArn', value=self.cert.certificate_arn)
alias_config = AliasConfiguration(
acm_cert_ref=self.cert.certificate_arn,
names=[site_domain],
ssl_method=aws_cloudfront.SSLMethod.SNI,
security_policy=aws_cloudfront.SecurityPolicyProtocol.
TLS_V1_1_2016,
)
self.distr = CloudFrontWebDistribution(
self,
"SiteDistribution",
alias_configuration=alias_config,
origin_configs=[
SourceConfiguration(
s3_origin_source=aws_cloudfront.S3OriginConfig(
s3_bucket_source=self.site_bucket),
behaviors=[
aws_cloudfront.Behavior(is_default_behavior=True)
])
])
CfnOutput(self, "DistributionId", value=self.distr.distribution_id)
# Route 53 alias record for the cloudfront distribution
if hosted_zone:
aws_route53.ARecord(
self,
"SiteAliasRecord",
zone=hosted_zone,
target=aws_route53.AddressRecordTarget.from_alias(
aws_route53_targets.CloudFrontTarget(self.distr)),
record_name=site_domain)
aws_s3_deployment.BucketDeployment(
self,
"DeployWithInvalidation",
sources=[aws_s3_deployment.Source.asset(sources)],
destination_bucket=self.site_bucket,
distribution=self.distr,
distribution_paths=["/*"])
def __init__(
self,
scope: App,
id: str,
envs: EnvSettings,
components: ComponentsStack,
base_resources: BaseResources,
):
super().__init__(scope, id)
self.db_secret_arn = Fn.import_value(
BaseResources.get_database_secret_arn_output_export_name(envs))
self.job_processing_queues = components.data_processing_queues
self.vpc = base_resources.vpc
self.db = base_resources.db
self.app_bucket = Bucket(self, "App", versioned=True)
if self.app_bucket.bucket_arn:
CfnOutput(
self,
id="AppBucketOutput",
export_name=self.get_app_bucket_arn_output_export_name(envs),
value=self.app_bucket.bucket_arn,
)
self.pages_bucket = Bucket(self, "Pages", public_read_access=True)
self.domain_name = StringParameter.from_string_parameter_name(
self,
"DomainNameParameter",
string_parameter_name="/schema-cms-app/DOMAIN_NAME").string_value
self.certificate_arn = StringParameter.from_string_parameter_name(
self,
"CertificateArnParameter",
string_parameter_name="/schema-cms-app/CERTIFICATE_ARN"
).string_value
django_secret = Secret(self,
"DjangoSecretKey",
secret_name="SCHEMA_CMS_DJANGO_SECRET_KEY")
lambda_auth_token_secret = Secret(
self,
"LambdaAuthToken",
secret_name="SCHEMA_CMS_LAMBDA_AUTH_TOKEN")
if lambda_auth_token_secret.secret_arn:
CfnOutput(
self,
id="lambdaAuthTokenArnOutput",
export_name=self.get_lambda_auth_token_arn_output_export_name(
envs),
value=lambda_auth_token_secret.secret_arn,
)
self.django_secret_key = EcsSecret.from_secrets_manager(django_secret)
self.lambda_auth_token = EcsSecret.from_secrets_manager(
lambda_auth_token_secret)
tag_from_context = self.node.try_get_context("app_image_tag")
tag = tag_from_context if tag_from_context != "undefined" else None
api_image = ContainerImage.from_ecr_repository(
repository=Repository.from_repository_name(
self,
id="BackendRepository",
repository_name=BaseECR.get_backend_repository_name(envs)),
tag=tag,
)
nginx_image = ContainerImage.from_ecr_repository(
repository=Repository.from_repository_name(
self,
id="NginxRepository",
repository_name=BaseECR.get_nginx_repository_name(envs)),
tag=tag,
)
self.api = ApplicationLoadBalancedFargateService(
self,
"ApiService",
service_name=f"{envs.project_name}-api-service",
cluster=Cluster.from_cluster_attributes(
self,
id="WorkersCluster",
cluster_name="schema-ecs-cluster",
vpc=self.vpc,
security_groups=[],
),
task_image_options=ApplicationLoadBalancedTaskImageOptions(
image=nginx_image,
container_name="nginx",
container_port=80,
enable_logging=True,
),
desired_count=1,
cpu=512,
memory_limit_mib=1024,
certificate=Certificate.from_certificate_arn(
self, "Cert", certificate_arn=self.certificate_arn),
domain_name=self.domain_name,
domain_zone=PrivateHostedZone(
self,
"zone",
vpc=self.vpc,
zone_name=self.domain_name,
),
)
self.api.task_definition.add_container(
"backend",
image=api_image,
command=[
"sh", "-c",
"/bin/chamber exec $CHAMBER_SERVICE_NAME -- ./scripts/run.sh"
],
logging=AwsLogDriver(stream_prefix="backend-container"),
environment={
"POSTGRES_DB": envs.data_base_name,
"AWS_STORAGE_BUCKET_NAME": self.app_bucket.bucket_name,
"AWS_STORAGE_PAGES_BUCKET_NAME": self.pages_bucket.bucket_name,
"SQS_WORKER_QUEUE_URL":
self.job_processing_queues[0].queue_url,
"SQS_WORKER_EXT_QUEUE_URL":
self.job_processing_queues[1].queue_url,
"SQS_WORKER_MAX_QUEUE_URL":
self.job_processing_queues[2].queue_url,
"CHAMBER_SERVICE_NAME": "schema-cms-app",
"CHAMBER_KMS_KEY_ALIAS": envs.project_name,
},
secrets={
"DB_CONNECTION":
EcsSecret.from_secrets_manager(
Secret.from_secret_arn(self,
id="DbSecret",
secret_arn=self.db_secret_arn)),
"DJANGO_SECRET_KEY":
self.django_secret_key,
"LAMBDA_AUTH_TOKEN":
self.lambda_auth_token,
},
cpu=512,
memory_limit_mib=1024,
)
self.django_secret_key.grant_read(
self.api.service.task_definition.task_role)
self.app_bucket.grant_read_write(
self.api.service.task_definition.task_role)
self.pages_bucket.grant_read_write(
self.api.service.task_definition.task_role)
for queue in self.job_processing_queues:
queue.grant_send_messages(
self.api.service.task_definition.task_role)
self.api.service.connections.allow_to(self.db.connections,
Port.tcp(5432))
self.api.task_definition.add_to_task_role_policy(
PolicyStatement(
actions=["ses:SendRawEmail", "ses:SendBulkTemplatedEmail"],
resources=["*"],
))
self.api.task_definition.add_to_task_role_policy(
PolicyStatement(
actions=[
"kms:Get*", "kms:Describe*", "kms:List*", "kms:Decrypt"
],
resources=[
Fn.import_value(
BaseKMS.get_kms_arn_output_export_name(envs))
],
))
self.api.task_definition.add_to_task_role_policy(
PolicyStatement(actions=["ssm:DescribeParameters"],
resources=["*"]))
self.api.task_definition.add_to_task_role_policy(
PolicyStatement(
actions=["ssm:GetParameters*"],
resources=[
f"arn:aws:ssm:{self.region}:{self.account}:parameter/schema-cms-app/*"
],
))
def __init__(self, scope: Construct, id: str, **kwargs) -> None:
# Set Stack description
deployment_tracking_param = scope.node.try_get_context(
"EnableDeploymentTracking")
stack_description = "Analytics Ref Arch - DataLake stack"
if is_module_enabled(deployment_tracking_param):
stack_description = stack_description + " (uksb-1scq97upu)"
super().__init__(scope, id, description=stack_description, **kwargs)
data_lake = DataLakeFoundations(self, "Foundations")
# Parameters to enable/disable modules
batch_module_param = self.node.try_get_context("EnableBatch")
dwh_module_param = self.node.try_get_context("EnableDWH")
dataviz_module_param = self.node.try_get_context("EnableDataviz")
streaming_module_param = self.node.try_get_context("EnableStreaming")
datagen_config_table = _dynamodb.Table(
self,
'DatagenConfigTable',
partition_key=_dynamodb.Attribute(
name="param", type=_dynamodb.AttributeType.STRING),
billing_mode=_dynamodb.BillingMode.PAY_PER_REQUEST)
# BATCH module
if is_module_enabled(batch_module_param):
BatchModule(self,
"Batch",
raw_bucket=data_lake.raw_s3_bucket,
clean_bucket=data_lake.clean_s3_bucket,
raw_db=data_lake.raw_glue_db,
clean_db=data_lake.clean_glue_db)
BatchDataGenerator(self,
"BatchDatagen",
config_table=datagen_config_table,
tshirt_size='SMALL',
log_bucket=data_lake.logs_s3_bucket,
sink_bucket=data_lake.raw_s3_bucket,
vpc=data_lake.vpc)
# DATA WAREHOUSE module
if is_module_enabled(dwh_module_param):
dwh_stack = DwhModule(self,
"dwh",
vpc=data_lake.vpc,
clean_bucket=data_lake.clean_s3_bucket,
clean_glue_db=data_lake.clean_glue_db)
CfnOutput(self,
'Redshift-QuickSight-Secret-Arn',
value=dwh_stack.quicksight_redshift_secret_arn,
export_name='ara-QuickSight-Redshift-Secret-Arn')
CfnOutput(self,
'bastion_dns',
value=dwh_stack.bastion_dns,
export_name='ara-Redshift-bastion-dns')
CfnOutput(self,
'bastion-host-key-pair',
export_name='ara-Redshift-bastion-keypair-secret',
value=dwh_stack.bastion_keypair_secret)
CfnOutput(self,
'redshift-hostname',
export_name='ara-Redshift-hostname',
value=dwh_stack.redshift_endpoint.hostname)
CfnOutput(self,
'redshift-port',
export_name='ara-Redshift-port',
value=str(dwh_stack.redshift_endpoint.port))
# DATA VISUALIZATION module
if is_module_enabled(dataviz_module_param):
quicksight_username = self.node.try_get_context(
'QuickSightUsername')
quicksight_identity_region = self.node.try_get_context(
'QuickSightIdentityRegion')
if quicksight_username is None or quicksight_identity_region is None:
raise Exception(
'QuickSightUsername and QuickSightIdentityRegion must be specified if data visualization is enabled'
)
dataviz_stack = DataVizModule(
self,
"dataviz",
vpc=data_lake.vpc,
clean_glue_db_name=data_lake.clean_glue_db.database_name,
redshift_sg_id=dwh_stack.redshift_sg_id,
quicksight_username=quicksight_username,
quicksight_identity_region=quicksight_identity_region)
CfnOutput(self,
'QuickSight-Security-Group-Id',
value=dataviz_stack.quicksight_security_group_id)
CfnOutput(self,
'QuickSight-Group-Arn',
value=dataviz_stack.quicksight_group_arn,
export_name='ara-QuickSight-Group-Arn')
# STREAMING module
if is_module_enabled(streaming_module_param):
streaming_stack = StreamingModule(
self,
id="Streaming",
prefix=id,
source_bucket=data_lake.raw_s3_bucket,
dest_bucket=data_lake.curated_s3_bucket)
StreamDataGenerator(
self,
"StreamDatagen",
config_table=datagen_config_table,
tshirt_size='SMALL',
log_bucket=data_lake.logs_s3_bucket,
sink_bucket=data_lake.raw_s3_bucket,
web_sale_stream=streaming_stack.sale_stream.stream_name,
web_customer_stream=streaming_stack.customer_stream.
stream_name,
web_customer_address_stream=streaming_stack.address_stream.
stream_name,
kinesis_key=streaming_stack.kinesis_kms_key,
vpc=data_lake.vpc)
CfnOutput(self,
'Clean-S3-Bucket',
value=data_lake.clean_s3_bucket.bucket_name)
CfnOutput(self,
"VPC-ID",
value=data_lake.vpc.vpc_id,
export_name='ara-Vpc-Id')
for idx, val in enumerate(data_lake.private_subnets_selection.subnets):
CfnOutput(self, 'SUBNET-ID{}'.format(idx), value=val.subnet_id)
Tags.of(self).add('project-name', 'ara')
def __init__(self, scope, id, **kwargs):
super().__init__(scope, id, **kwargs)
# Each publisher instance authenticates as an individual user stored in this pool
user_pool = UserPool(self,
'user-pool',
user_pool_name='amazon-cloudwatch-publisher')
# Set up the client app with simple username/password authentication
user_pool_client = UserPoolClient(
self,
'user-pool-client',
user_pool=user_pool,
enabled_auth_flows=[AuthFlow.USER_PASSWORD],
user_pool_client_name='amazon-cloudwatch-publisher')
# The identity pool exists to associate users to roles they can assume
identity_pool = CfnIdentityPool(
self,
'identity-pool',
identity_pool_name='amazon-cloudwatch-publisher',
allow_unauthenticated_identities=False)
# Setting this property links the identity pool to the user pool client app
identity_pool.add_property_override(
property_path='CognitoIdentityProviders',
value=[{
'ClientId':
user_pool_client.user_pool_client_id,
'ProviderName':
'cognito-idp.{0}.amazonaws.com/{1}'.format(
self.region, user_pool.user_pool_id)
}])
# Only identities that come from Congito users should be able to assume the publisher role
principal = FederatedPrincipal(
federated='cognito-identity.amazonaws.com',
assume_role_action='sts:AssumeRoleWithWebIdentity',
conditions={
'StringEquals': {
'cognito-identity.amazonaws.com:aud': identity_pool.ref
},
'ForAnyValue:StringLike': {
'cognito-identity.amazonaws.com:amr': 'authenticated'
}
})
# Minimum set of permissions required to push metrics and logs
policy = PolicyDocument(statements=[
PolicyStatement(effect=Effect.ALLOW,
actions=[
'cloudwatch:PutMetricData',
'logs:CreateLogGroup',
'logs:CreateLogStream',
'logs:DescribeLogGroups',
'logs:DescribeLogStreams',
'logs:PutLogEvents',
],
resources=[
'*',
])
])
# Create the role itself using the principal and policy defined above
role = Role(self,
'role',
assumed_by=principal,
inline_policies=[policy])
# Associate the above role with the identity pool; we don't want
# any unauthenticated access so explicitly ensure it's set to None
CfnIdentityPoolRoleAttachment(
self,
'identity-pool-role-attachment-authenticated',
identity_pool_id=identity_pool.ref,
roles={
'authenticated': role.role_arn,
'unauthenticated': None
})
# Defining outputs here allows them to be scraped from the `cdk deploy` command
CfnOutput(self, 'Region', value=self.region)
CfnOutput(self, 'AccountId', value=self.account)
CfnOutput(self, 'UserPoolId', value=user_pool.user_pool_id)
CfnOutput(self, 'IdentityPoolId', value=identity_pool.ref)
CfnOutput(self,
'AppClientId',
value=user_pool_client.user_pool_client_id)
def __init__(self,
scope: Stack,
id: str,
ws_api: WsApi,
stage_name: str,
access_log_settings: Optional[
CfnStage.AccessLogSettingsProperty] = None,
create_default_access_log_settings: Optional[bool] = None,
auto_deploy: Optional[bool] = None,
client_certificate_id: Optional[str] = None,
default_route_settings: Optional[
CfnStage.RouteSettingsProperty] = None,
description: Optional[str] = None,
route_settings: Optional[Dict[str, Any]] = None,
stage_variables: Optional[Dict[str, Any]] = None,
tags: Optional[Dict[str, Any]] = None,
*args,
**kwargs) -> None:
"""
Constructor.
:param scope: CloudFormation-equivalent stack.
:param id: AWS-CDK-specific id.
:param ws_api: The web socket API for with this stage should be created.
:param stage_name: The stage name. Stage names can only contain alphanumeric characters, hyphens,
and underscores. Maximum length is 128 characters.
:param access_log_settings: Settings for logging access in this stage.
:param create_default_access_log_settings: Indicate whether to create default access log settings.
:param auto_deploy: Specifies whether updates to an API automatically trigger a new deployment.
The default value is false.
:param client_certificate_id: The identifier of a client certificate for a Stage.
:param default_route_settings: The default route settings for the stage.
:param description: The description for the API stage.
:param route_settings: Route settings for the stage.
:param stage_variables: A map that defines the stage variables for a Stage. Variable names can have
alphanumeric and underscore characters, and the values must match [A-Za-z0-9-._~:/?#&=,]+.
:param tags: The collection of tags. Each tag element is associated with a given resource.
"""
self.__scope = scope
self.__ws_api = ws_api
default_route_settings = default_route_settings or CfnStage.RouteSettingsProperty(
data_trace_enabled=True,
detailed_metrics_enabled=True,
logging_level='INFO',
)
if access_log_settings and create_default_access_log_settings:
raise ValueError(
'Access log settings supplied. Can not request to create default ones.'
)
if create_default_access_log_settings:
log_group = LogGroup(scope=scope,
id=f'{id}LogGroup',
log_group_name=f'{id}LogGroup',
removal_policy=RemovalPolicy.DESTROY,
retention=RetentionDays.ONE_MONTH)
access_log_settings = CfnStage.AccessLogSettingsProperty(
destination_arn=log_group.log_group_arn,
format=("{"
"\"requestId\":\"$context.requestId\", "
"\"ip\": \"$context.identity.sourceIp\", "
"\"caller\":\"$context.identity.caller\", "
"\"user\":\"$context.identity.user\","
"\"requestTime\":\"$context.requestTime\", "
"\"eventType\":\"$context.eventType\","
"\"routeKey\":\"$context.routeKey\", "
"\"status\":\"$context.status\","
"\"connectionId\":\"$context.connectionId\""
"}"))
super().__init__(scope=scope,
id=id,
api_id=ws_api.ref,
stage_name=stage_name,
access_log_settings=access_log_settings,
auto_deploy=auto_deploy,
client_certificate_id=client_certificate_id,
default_route_settings=default_route_settings,
description=description,
route_settings=route_settings,
stage_variables=stage_variables,
tags=tags,
*args,
**kwargs)
CfnOutput(scope=scope,
id=f'{ws_api.name}Url',
value=self.ws_url,
description='A websocket URL.',
export_name=f'{ws_api.name}Url')
CfnOutput(scope=scope,
id=f'{ws_api.name}Id',
value=self.api_id,
description='A websocket ID.',
export_name=f'{ws_api.name}Id')
CfnOutput(scope=scope,
id=f'{ws_api.name}HttpUrl',
value=self.http_url,
description='A websocket http URL.',
export_name=f'{ws_api.name}HttpUrl')
def _export(self, name, value):
CfnOutput(self._stack,
name,
value=value,
export_name=self._config.stack_name + name)
def __init__(self, app: App, id: str, **kwargs) -> None:
super().__init__(app, id, **kwargs)
self.template_options.description = "(SO0123) Improving Forecast Accuracy with Machine Learning %%VERSION%% - This solution provides a mechanism to automate Amazon Forecast predictor and forecast generation and visualize it via an Amazon SageMaker Jupyter Notebook"
# set up the template parameters
email = CfnParameter(
self,
id="Email",
type="String",
description="Email to notify with forecast results",
default="",
max_length=50,
allowed_pattern=
r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$|^$)",
constraint_description="Must be a valid email address or blank",
)
lambda_log_level = CfnParameter(
self,
id="LambdaLogLevel",
type="String",
description="Change the verbosity of the logs output to CloudWatch",
default="WARNING",
allowed_values=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
)
notebook_deploy = CfnParameter(
self,
id="NotebookDeploy",
type="String",
description="Deploy an Amazon SageMaker Jupyter Notebook instance",
default="No",
allowed_values=["Yes", "No"],
)
notebook_volume_size = CfnParameter(
self,
id="NotebookVolumeSize",
type="Number",
description=
"Enter the size of the notebook instance EBS volume in GB",
default=10,
min_value=5,
max_value=16384,
constraint_description=
"Must be an integer between 5 (GB) and 16384 (16 TB)",
)
notebook_instance_type = CfnParameter(
self,
id="NotebookInstanceType",
type="String",
description="Enter the type of the notebook instance",
default="ml.t2.medium",
allowed_values=[
"ml.t2.medium",
"ml.t3.medium",
"ml.r5.large",
"ml.c5.large",
],
)
quicksight_analysis_owner = CfnParameter(
self,
id="QuickSightAnalysisOwner",
description=
"With QuickSight Enterprise enabled, provide a QuickSight ADMIN user ARN to automatically create QuickSight analyses",
default="",
allowed_pattern="(^arn:.*:quicksight:.*:.*:user.*$|^$)",
)
# set up the metadata/ cloudformation interface
template_options = TemplateOptions()
template_options.add_parameter_group(
label=
"Improving Forecast Accuracy with Machine Learning Configuration",
parameters=[email],
)
template_options.add_parameter_group(
label="Visualization Options",
parameters=[
quicksight_analysis_owner,
notebook_deploy,
notebook_instance_type,
notebook_volume_size,
],
)
template_options.add_parameter_group(label="Deployment Configuration",
parameters=[lambda_log_level])
template_options.add_parameter_label(email, "Email")
template_options.add_parameter_label(lambda_log_level,
"CloudWatch Log Level")
template_options.add_parameter_label(notebook_deploy,
"Deploy Jupyter Notebook")
template_options.add_parameter_label(notebook_volume_size,
"Jupyter Notebook volume size")
template_options.add_parameter_label(notebook_instance_type,
"Jupyter Notebook instance type")
template_options.add_parameter_label(quicksight_analysis_owner,
"Deploy QuickSight Dashboards")
self.template_options.metadata = template_options.metadata
solution_mapping = CfnMapping(
self,
"Solution",
mapping={
"Data": {
"ID": "SO0123",
"Version": "%%VERSION%%",
"SendAnonymousUsageData": "Yes",
}
},
)
source_mapping = CfnMapping(
self,
"SourceCode",
mapping={
"General": {
"S3Bucket": "%%BUCKET_NAME%%",
"KeyPrefix": "%%SOLUTION_NAME%%/%%VERSION%%",
"QuickSightSourceTemplateArn": "%%QUICKSIGHT_SOURCE%%",
}
},
)
# conditions
create_notebook = CfnCondition(
self,
"CreateNotebook",
expression=Fn.condition_equals(notebook_deploy, "Yes"),
)
email_provided = CfnCondition(
self,
"EmailProvided",
expression=Fn.condition_not(Fn.condition_equals(email, "")),
)
send_anonymous_usage_data = CfnCondition(
self,
"SendAnonymousUsageData",
expression=Fn.condition_equals(
Fn.find_in_map("Solution", "Data", "SendAnonymousUsageData"),
"Yes"),
)
create_analysis = CfnCondition(
self,
"CreateAnalysis",
expression=Fn.condition_not(
Fn.condition_equals(quicksight_analysis_owner, ""), ),
)
# Step function and state machine
fns = LambdaFunctions(self, "Functions", log_level=lambda_log_level)
# SNS
notifications = Notifications(
self,
"NotificationConfiguration",
lambda_function=fns.functions["SNS"],
email=email,
email_provided=email_provided,
)
# Custom Resources
unique_name = CfnResource(
self,
"UniqueName",
type="Custom::UniqueName",
properties={
"ServiceToken":
fns.functions["CfnResourceUniqueName"].function_arn
},
)
unique_name.override_logical_id("UniqueName")
data_bucket_name_resource = CfnResource(
self,
"DataBucketName",
type="Custom::BucketName",
properties={
"ServiceToken":
fns.functions["CfnResourceBucketName"].function_arn,
"BucketPurpose": "data-bucket",
"StackName": Aws.STACK_NAME,
"Id": unique_name.get_att("Id"),
},
)
data_bucket_name_resource.override_logical_id("DataBucketName")
# Buckets
access_logs_bucket = self.secure_bucket(
"AccessLogsBucket",
suppressions=[
CfnNagSuppression(
"W35",
"This bucket is used as the logging destination for forecast datasets and exports",
)
],
access_control=BucketAccessControl.LOG_DELIVERY_WRITE,
)
athena_bucket = self.secure_bucket(
"AthenaBucket",
server_access_logs_bucket=access_logs_bucket,
server_access_logs_prefix="athena-bucket-access-logs/",
)
data_bucket = self.secure_bucket(
"ForecastBucket",
lifecycle_rules=[
LifecycleRule(
abort_incomplete_multipart_upload_after=Duration.days(3),
enabled=True,
),
LifecycleRule(expiration=Duration.days(1),
prefix="raw/",
enabled=True),
],
bucket_name=data_bucket_name_resource.get_att("Name").to_string(),
server_access_logs_bucket=access_logs_bucket,
server_access_logs_prefix="forecast-bucket-access-logs/",
)
data_bucket.node.default_child.add_property_override(
"NotificationConfiguration",
{
"LambdaConfigurations": [{
"Function":
fns.functions["S3NotificationLambda"].function_arn,
"Event":
"s3:ObjectCreated:*",
"Filter": {
"S3Key": {
"Rules": [
{
"Name": "prefix",
"Value": "train/"
},
{
"Name": "suffix",
"Value": ".csv"
},
]
}
},
}]
},
)
# Glue and Athena
glue = Glue(self, "GlueResources", unique_name)
athena = Athena(self, "AthenaResources", athena_bucket=athena_bucket)
# Configure permissions for functions
fns.set_s3_notification_permissions(data_bucket_name_resource)
fns.set_forecast_s3_access_permissions(
name="DatasetImport",
function=fns.functions["CreateDatasetImportJob"],
data_bucket_name_resource=data_bucket_name_resource,
)
fns.set_forecast_s3_access_permissions(
name="ForecastExport",
function=fns.functions["CreateForecast"],
data_bucket_name_resource=data_bucket_name_resource,
)
fns.set_forecast_etl_permissions(
function=fns.functions["PrepareForecastExport"],
database=glue.database,
workgroup=athena.workgroup,
quicksight_principal=quicksight_analysis_owner,
quicksight_source=source_mapping,
athena_bucket=athena_bucket,
data_bucket_name_resource=data_bucket_name_resource,
)
fns.set_forecast_permissions(
"CreateDatasetGroup",
data_bucket_name_resource=data_bucket_name_resource)
fns.set_forecast_permissions(
"CreateDatasetImportJob",
data_bucket_name_resource=data_bucket_name_resource,
)
fns.set_forecast_permissions(
"CreateForecast",
data_bucket_name_resource=data_bucket_name_resource)
fns.set_forecast_permissions(
"CreatePredictor",
data_bucket_name_resource=data_bucket_name_resource)
fns.set_forecast_permissions(
"PrepareForecastExport",
data_bucket_name_resource=data_bucket_name_resource)
# notebook (conditional on 'create_notebook')
notebook = Notebook(
self,
"Notebook",
buckets=[data_bucket],
instance_type=notebook_instance_type.value_as_string,
instance_volume_size=notebook_volume_size.value_as_number,
notebook_path=Path(__file__).parent.parent.parent.joinpath(
"notebook", "samples", "notebooks"),
notebook_destination_bucket=data_bucket,
notebook_destination_prefix="notebooks",
)
Aspects.of(notebook).add(ConditionalResources(create_notebook))
# solutions metrics (conditional on 'send_anonymous_usage_data')
metrics = Metrics(
self,
"SolutionMetrics",
metrics_function=fns.functions["CfnResourceSolutionMetrics"],
metrics={
"Solution":
solution_mapping.find_in_map("Data", "ID"),
"Version":
solution_mapping.find_in_map("Data", "Version"),
"Region":
Aws.REGION,
"NotebookDeployed":
Fn.condition_if(create_notebook.node.id, "Yes", "No"),
"NotebookType":
Fn.condition_if(
create_notebook.node.id,
notebook_instance_type.value_as_string,
Aws.NO_VALUE,
),
"QuickSightDeployed":
Fn.condition_if(create_analysis.node.id, "Yes", "No"),
},
)
Aspects.of(metrics).add(
ConditionalResources(send_anonymous_usage_data))
# outputs
CfnOutput(self, "ForecastBucketName", value=data_bucket.bucket_name)
CfnOutput(self, "AthenaBucketName", value=athena_bucket.bucket_name)
CfnOutput(self,
"StepFunctionsName",
value=fns.state_machine.state_machine_name)
# and limitations under the License. # #
######################################################################################################################
#!/usr/bin/env python3
from aws_cdk.core import (App, Tags, CfnOutput)
from lib.spark_on_eks_stack import SparkOnEksStack
from lib.cloud_front_stack import NestedStack
app = App()
eks_name = app.node.try_get_context('cluster_name')
solution_id = app.node.try_get_context('solution_id')
solution_version = app.node.try_get_context('version')
# main stack
eks_stack = SparkOnEksStack(app,
'sql-based-etl-with-apache-spark-on-amazon-eks',
eks_name, solution_id, solution_version)
# Recommend to remove the CloudFront nested stack. Setup your own SSL certificate and add it to ALB.
cf_nested_stack = NestedStack(eks_stack, 'CreateCloudFront',
eks_stack.code_bucket, eks_stack.argo_url,
eks_stack.jhub_url)
Tags.of(eks_stack).add('project', 'sqlbasedetl')
Tags.of(cf_nested_stack).add('project', 'sqlbasedetl')
# Deployment Output
CfnOutput(eks_stack, 'CODE_BUCKET', value=eks_stack.code_bucket)
CfnOutput(eks_stack, 'ARGO_URL', value='https://' + cf_nested_stack.argo_cf)
CfnOutput(eks_stack, 'JUPYTER_URL', value='https://' + cf_nested_stack.jhub_cf)
app.synth()
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# set variables for site s3 bucket
bucket_name = DOMAIN_WEBSITE['bucket_name']
website_index = DOMAIN_WEBSITE['website_index']
website_error = DOMAIN_WEBSITE['website_error']
website_code_folder = DOMAIN_WEBSITE['website_code_folder']
site_domain = DOMAIN_WEBSITE['site_domain']
certificate_domain = DOMAIN_WEBSITE['certificate_domain']
api_domain = DOMAIN_WEBSITE['api_domain']
hosted_zone_name = DOMAIN_WEBSITE['hosted_zone_name']
hosted_zone_id = DOMAIN_WEBSITE['hosted_zone_id']
#self.lambda_code = aws_lambda.Code.from_cfn_parameters()
# retrieve hosted zone
hosted_zone = aws_route53.HostedZone.from_hosted_zone_attributes(
self,
'hostedZone',
hosted_zone_id=hosted_zone_id,
zone_name=hosted_zone_name)
# set variables for backend
lambda_code_location = "jukebike/backend/"
# Construct code goes here
CfnOutput(self, "Site", value=f"https://{site_domain}")
# Content bucket
site_bucket = aws_s3.Bucket(self,
"websitebucket",
bucket_name=bucket_name,
website_index_document=website_index,
website_error_document=website_error,
public_read_access=True,
removal_policy=RemovalPolicy.DESTROY)
CfnOutput(self, "BucketArn", value=site_bucket.bucket_arn)
CfnOutput(self, "WebsiteUrl", value=site_bucket.bucket_website_url)
# Certificate
cert = aws_certificatemanager.DnsValidatedCertificate(
self,
"certificate_website",
domain_name=site_domain,
hosted_zone=hosted_zone,
region="us-east-1")
CfnOutput(self, 'CertificateArn', value=cert.certificate_arn)
distr = CloudFrontWebDistribution(
self,
"SiteDistribution",
alias_configuration=AliasConfiguration(
acm_cert_ref=cert.certificate_arn,
names=[site_domain],
ssl_method=aws_cloudfront.SSLMethod.SNI,
security_policy=aws_cloudfront.SecurityPolicyProtocol.
TLS_V1_1_2016,
),
origin_configs=[
SourceConfiguration(
s3_origin_source=aws_cloudfront.S3OriginConfig(
s3_bucket_source=site_bucket),
behaviors=[
aws_cloudfront.Behavior(is_default_behavior=True)
])
])
CfnOutput(self, "DistributionId", value=distr.distribution_id)
#
# Route 53 alias record for the cloudfront distribution
aws_route53.ARecord(self,
"SiteAliasRecord",
zone=hosted_zone,
target=aws_route53.AddressRecordTarget.from_alias(
aws_route53_targets.CloudFrontTarget(distr)),
record_name=site_domain)
aws_s3_deployment.BucketDeployment(
self,
"DeployWithInvalidation",
sources=[aws_s3_deployment.Source.asset(website_code_folder)],
destination_bucket=site_bucket,
distribution=distr,
distribution_paths=["/*"])
########################### Backend #################
certificate = aws_certificatemanager.DnsValidatedCertificate(
self,
"domaincertificate",
hosted_zone=hosted_zone,
region='us-east-1',
domain_name=certificate_domain,
validation_method=aws_certificatemanager.ValidationMethod.DNS)
############# Search API ###################
search_lambda = aws_lambda.Function(
self,
"SearchLambda",
code=aws_lambda.Code.from_asset(lambda_code_location),
handler="search.handler",
runtime=aws_lambda.Runtime.PYTHON_3_7)
CfnOutput(self, "SearchLambda_", value=search_lambda.function_arn)
search_api = aws_apigateway.LambdaRestApi(
self,
'SearchSpotifyEndpoint',
handler=search_lambda,
)
############# Whats-Next API ###################
whats_next_lambda = aws_lambda.Function(
self,
"WhatsNextLambda",
code=aws_lambda.Code.from_asset(lambda_code_location),
handler="whats_next.handler",
runtime=aws_lambda.Runtime.PYTHON_3_7)
CfnOutput(self,
"WhatsNextLambda_",
value=whats_next_lambda.function_arn)
whats_next_api = aws_apigateway.LambdaRestApi(
self,
'WhatsNextEndpoint',
handler=whats_next_lambda,
)
############# Whats-Next API ###################
wish_track_lambda = aws_lambda.Function(
self,
"WishTrackLambda",
code=aws_lambda.Code.from_asset(lambda_code_location),
handler="wish_track.handler",
runtime=aws_lambda.Runtime.PYTHON_3_7)
CfnOutput(self,
"WishTrackLambda_",
value=wish_track_lambda.function_arn)
wish_track_api = aws_apigateway.LambdaRestApi(
self,
'WishTrackEndpoint',
handler=wish_track_lambda,
)
################## Publish APIS with custom domain name ##############
# Pre-Requirements:
# [Manual] 1) Registered Domain with Route 53 (e.g. jacubasch.com)
# 2) Certificate in North Virgina for domain (e.g. api.jacubasch.com) -> AWS Certification Manager
# 3) API Gateway Custom Domain with Edge
# 4) Alias-Record in Route53 forwarding to Cloudfront Target Domain Name (can be found in API Gateway)
# TODO: in separaten Base-Stack auslagern?
# https://medium.com/@maciejtreder/custom-domain-in-aws-api-gateway-a2b7feaf9c74
domain = aws_apigateway.DomainName(
self,
'searchDomain',
certificate=certificate,
endpoint_type=aws_apigateway.EndpointType.EDGE,
domain_name=api_domain)
domain.add_base_path_mapping(target_api=search_api, base_path="search")
domain.add_base_path_mapping(target_api=whats_next_api,
base_path="whats-next")
domain.add_base_path_mapping(target_api=wish_track_api,
base_path="wish-track")
target = aws_route53_targets.ApiGatewayDomain(domain)
record_target = aws_route53.RecordTarget.from_alias(target)
alias_record = aws_route53.ARecord(self,
'aliasRecord',
target=record_target,
record_name=api_domain,
zone=hosted_zone)
CfnOutput(self, "AliasRecord_", value=alias_record.to_string())
################## Dynamo DB ##############
# create dynamo table
track_table = aws_dynamodb.Table(
self,
"track_table",
partition_key=aws_dynamodb.Attribute(
name="track_uri", type=aws_dynamodb.AttributeType.STRING))
# grant permission to lambda & provide environment variable
track_table.grant_write_data(wish_track_lambda)
wish_track_lambda.add_environment("TRACK_TABLE_NAME",
track_table.table_name)
track_table.grant_read_write_data(whats_next_lambda)
whats_next_lambda.add_environment("TRACK_TABLE_NAME",
track_table.table_name)
def __init__(self, app: App, id: str, env: Environment) -> None:
super().__init__(app, id, env=env)
# start by getting the DNS zone we're going to work with
zone = HostedZone.from_lookup(self, "Dominick", domain_name=DOMAIN)
# create a certificate for the web service which matches its hostname
cert = Certificate(self,
"Cletus",
domain_name=HOSTNAME,
validation=CertificateValidation.from_dns(zone))
# the services will live in a vpc, of course
vpc = ec2.Vpc(self, "Virgil")
# we're going to scale this web-service automatically
asg = AutoScalingGroup(
self,
"Alice",
vpc=vpc,
user_data=http_service(),
instance_type=ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE2,
ec2.InstanceSize.MICRO),
machine_image=ec2.AmazonLinuxImage(
generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2))
# explicitly allow internal access from the vpc just to be safe
asg.connections.allow_internally(Port.tcp(WEB_PORT), "web-service")
asg.connections.allow_internally(Port.tcp(NOT_WEB), "not-web")
# expose the scaling group ports and permit egress
asg.connections.allow_from_any_ipv4(Port.tcp(WEB_PORT))
asg.connections.allow_from_any_ipv4(Port.tcp(NOT_WEB))
# create a health check for the not-web service that currently
if NOT_WEB_HEALTH_CHECKS:
# points to the not-web service
checker = HealthCheck(interval=Duration.seconds(10),
port=NOT_WEB,
protocol=Protocol.TCP)
else:
# points to the web port where our demo server listens
checker = HealthCheck(interval=Duration.seconds(10),
port=str(WEB_PORT),
protocol=WEB_PROT)
# put the scaling group behind a network target group for the LB
notwebish = NetworkTargetGroup(self,
"Allison",
vpc=vpc,
health_check=checker,
targets=[asg],
port=NOT_WEB,
protocol=Protocol.TCP)
# for the web-like ports, we can use the default health check
webish = NetworkTargetGroup(
self,
"Alicen",
vpc=vpc,
health_check=HealthCheck(interval=Duration.seconds(10)),
targets=[asg],
port=WEB_PORT,
protocol=WEB_PROT)
if True:
# create the load balancer and put it into dns
lb = NetworkLoadBalancer(self,
"Lisa",
vpc=vpc,
internet_facing=True)
# create a hostname for the service
CnameRecord(self,
"Carl",
domain_name=lb.load_balancer_dns_name,
zone=zone,
record_name=HOSTNAME.split('.')[0],
ttl=Duration.seconds(60))
else:
# a multi-step deployment could allow using an alias in R53
lb = NetworkLoadBalancer.from_network_load_balancer_attributes(
self,
"Larry",
vpc=vpc,
load_balancer_arn=some.load_balancer_arn,
load_balancer_dns_name=HOSTNAME,
load_balancer_canonical_hosted_zone_id=zone.hosted_zone_id)
# create a hostname for the service
AaaaRecord(self,
"Eric",
zone=zone,
record_name=HOSTNAME.split('.')[0],
target=RecordTarget.from_alias(LoadBalancerTarget(lb)))
# point the load balancer to the target group for the ssl service
#
# TODO: determine if we need to use the same cert for pub-facing
# and internal service
listener_cert = ListenerCertificate(cert.certificate_arn)
lb.add_listener("Cecil",
port=443,
certificates=[listener_cert],
default_target_groups=[webish])
# point the load balancer to the target group for the web service
lb.add_listener("Webster", port=80, default_target_groups=[webish])
# point the load balancer to the group for the not-web service
lb.add_listener("NotWeb",
default_target_groups=[notwebish],
port=NOT_WEB,
protocol=Protocol.TCP)
# auto scale the, uh, autoscaling group
asg.scale_on_cpu_utilization("ScaleCPU", target_utilization_percent=80)
# emit some output values, largely for console use
CfnOutput(self,
"LB",
export_name="LB",
value=lb.load_balancer_dns_name)
CfnOutput(self,
"HTTP",
export_name="HTTP",
value="http://{}/".format(HOSTNAME))
CfnOutput(self,
"HTTPS",
export_name="HTTPS",
value="https://{}/".format(HOSTNAME))
CfnOutput(self,
"TCP",
export_name="TCP",
value="tcp://{}:{}/".format(HOSTNAME, NOT_WEB))
CfnOutput(self, "Cert", export_name="Cert", value=cert.certificate_arn)