def __init__(self, scope):
super().__init__(scope, "bug")
bucket = Bucket.from_bucket_name(
self, "artifacts", core.Fn.import_value("CodeArtifactsBucket")
)
pipeline_role = Role.from_role_arn(
self, "pipeline", core.Fn.import_value("CodePipelineRole")
)
pipeline = Pipeline(
self,
"Pipeline",
artifact_bucket=bucket,
role=pipeline_role,
stages=[
StageProps(
stage_name="Source",
actions=[
GitHubSourceAction(
action_name="Source",
run_order=1,
oauth_token=core.SecretValue("something"),
output=Artifact(artifact_name="SourceArtifact"),
owner="me",
repo="repo",
branch="master",
)
],
)
],
)
pipeline.add_stage(
stage_name="Fails",
actions=[
LambdaInvokeAction(
action_name="LambdaInvokeAction",
run_order=1,
lambda_=Function.from_function_arn(
self, "function", core.Fn.import_value("SomeFunction")
),
)
],
)
def __create_cloud_front_www(
self, origin_bucket_name: str, domain_name: str,
alternative_domain_names: Optional[List[str]],
ssl_certificate: aws_certificatemanager.Certificate,
cache_policy: aws_cloudfront.CachePolicy,
origin_access_identity: aws_cloudfront.OriginAccessIdentity,
edge_lambda_viewer_request: aws_lambda.Version
) -> aws_cloudfront.Distribution:
domain_names = alternative_domain_names if alternative_domain_names else []
domain_names.append(domain_name)
domain_names = set(domain_names)
return aws_cloudfront.Distribution(
self,
'CloudFrontWWW',
enabled=True,
certificate=ssl_certificate,
comment='CloudFront Distribution for your WWW static website',
domain_names=list(domain_names),
http_version=HttpVersion.HTTP2,
price_class=PriceClass.PRICE_CLASS_100,
default_behavior=BehaviorOptions(
allowed_methods=AllowedMethods.ALLOW_GET_HEAD,
cached_methods=CachedMethods.CACHE_GET_HEAD,
cache_policy=cache_policy,
viewer_protocol_policy=ViewerProtocolPolicy.REDIRECT_TO_HTTPS,
origin=aws_cloudfront_origins.S3Origin(
bucket=Bucket.from_bucket_name(
self, "OriginProd", bucket_name=origin_bucket_name),
origin_access_identity=origin_access_identity,
origin_path='/src/prod'),
edge_lambdas=[
EdgeLambda(event_type=LambdaEdgeEventType.VIEWER_REQUEST,
include_body=False,
function_version=edge_lambda_viewer_request)
]),
error_responses=[
ErrorResponse(ttl=Duration.seconds(300),
response_page_path='/404.html',
http_status=403,
response_http_status=404)
])
def __create_cloud_front_www_edit_path_for_origin_lambda(
self, webflow_aws_setup_bucket: str,
lambda_execution_role: aws_iam.Role) -> aws_lambda.Function:
return aws_lambda.Function(
self,
'CloudFrontEditPathForOrigin',
description=
'Appends .html extension to universal paths, preserving files with other extensions (ex .css)',
handler='index.handler',
runtime=aws_lambda.Runtime.NODEJS_12_X,
timeout=Duration.seconds(5),
memory_size=128,
role=lambda_execution_role,
code=Code.bucket(
bucket=Bucket.from_bucket_name(
self,
"SourceBucketWWWEditPathForOriginLambda",
bucket_name=webflow_aws_setup_bucket),
key=
'lambda_function/cloudfront_www_edit_path_for_origin/package.zip'
))
def __create_s3_trigger_lambda_function(
self, webflow_aws_setup_bucket: str, execution_role: aws_iam.Role,
cloud_front_distribution: aws_cloudfront.Distribution
) -> aws_lambda.Function:
return aws_lambda.Function(
self,
'S3TriggerLambdaFunction',
description=
'Function responsible of unzipping the zip file uploaded and move the files to the '
'correct folder',
handler='index.handler',
role=execution_role,
runtime=aws_lambda.Runtime.NODEJS_12_X,
timeout=Duration.seconds(300),
environment={
'CDN_DISTRIBUTION_ID': cloud_front_distribution.distribution_id
},
code=Code.bucket(
bucket=Bucket.from_bucket_name(
self,
"WebflowAWSSupport",
bucket_name=webflow_aws_setup_bucket),
key='lambda_function/s3_trigger_artifacts_upload/package.zip'))
def __init__(self, scope: Construct, app_id: str, **kwargs) -> None:
super().__init__(scope, app_id, **kwargs)
# bucket with ui contents can be reached over listener rule on ALB
api_domain_name = "static." + AWS_CONF["private_base_domain"]
host_domain = f"{AWS_CONF['app_name']}.{AWS_CONF['private_base_domain']}"
s3_path = AWS_CONF["app_name"]
if AWS_CONF["deployment_stage"] == "tst":
host_domain = f"{AWS_CONF['branch_id']}." + host_domain
s3_path += "-" + AWS_CONF["branch_path"]
ui_bucket = Bucket.from_bucket_name(
self,
"UiBucket",
bucket_name=AWS_CONF["optionals"]["ui_bucket"],
)
BucketDeployment(
self,
"UiBucketDepl",
destination_bucket=ui_bucket,
destination_key_prefix=s3_path,
sources=[Source.asset(AWS_CONF["optionals"]["node_build_path"])],
)
# ALB rule for http redirect to https
load_balancer_arn = Arn.format(
components=ArnComponents(
service="elasticloadbalancing",
partition="aws",
resource="loadbalancer/app",
resource_name=AWS_CONF["optionals"]["alb"],
),
stack=self,
)
alb = ApplicationLoadBalancer.from_lookup(
self,
"AlbApi",
load_balancer_arn=load_balancer_arn,
)
listener_http = ApplicationListener.from_lookup(
self,
"AlbHttpListenerRule",
load_balancer_arn=alb.load_balancer_arn,
listener_port=80,
)
# listener rule priority is mandatory input and needs to be looked up
# if cdk context not set yet set fixed priority during cdk synth
priority = 1
if AWS_CONF["env"]["account"] in listener_http.listener_arn:
priority = _next_elb_priority(host_domain, listener_http.listener_arn)
# the rule is added to the existing listener
ApplicationListenerRule(
self,
f"ListenerRule{AWS_CONF['branch_id'].capitalize()}",
listener=listener_http,
priority=priority,
action=ListenerAction.redirect(
host=api_domain_name,
path=f"/ui/{s3_path}/index.html",
permanent=True,
port="443",
protocol="HTTPS",
),
conditions=[ListenerCondition.host_headers([host_domain])],
)
# route 53 private zone with listener rule for redirect to alb
ARecord(
self,
f"ARecord{AWS_CONF['branch_id'].capitalize()}",
record_name=host_domain,
target=RecordTarget(alias_target=LoadBalancerTarget(alb)),
zone=PrivateHostedZone.from_lookup(
self,
"PrivZoneCorp",
domain_name=AWS_CONF["private_base_domain"],
private_zone=True,
vpc_id=AWS_CONF["optionals"]["vpc"],
),
)
def __init__(self, scope: Construct, id: str, raw_bucket: Bucket,
clean_bucket: Bucket, raw_db: Database, clean_db: Database,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Create crawler to crawl raw bucket
raw_crawler = Crawler(self,
'Raw',
name='raw',
db=raw_db,
bucket=raw_bucket,
prefix='').crawler
# Retrieve existing shared binary bucket
binary_bucket = Bucket.from_bucket_name(self, 'AraBucketByName',
_config.ARA_BUCKET_NAME)
# Create glue jobs to transform all entities from raw to clean
raw2clean_jobs = [
Raw2CleanJob(self,
'Raw2CleanItem',
source_entity='item',
target_entity='item',
datetime_column='item_datetime',
date_column='item_date',
partition_column='item_date',
script_bucket=binary_bucket,
script_location=_config.Raw2CleanConfig.
PARQUET_GLUE_SCRIPT_LOCATION,
raw_db=raw_db,
clean_db=clean_db,
raw_bucket=raw_bucket,
clean_bucket=clean_bucket),
Raw2CleanJob(self,
'Raw2CleanPromo',
source_entity='promo',
target_entity='promo',
datetime_column='promo_datetime',
date_column='promo_date',
partition_column='promo_date',
script_bucket=binary_bucket,
script_location=_config.Raw2CleanConfig.
PARQUET_GLUE_SCRIPT_LOCATION,
raw_db=raw_db,
clean_db=clean_db,
raw_bucket=raw_bucket,
clean_bucket=clean_bucket),
Raw2CleanJob(self,
'Raw2CleanStore',
source_entity='store',
target_entity='store',
datetime_column='store_datetime',
date_column='store_date',
partition_column='store_date',
script_bucket=binary_bucket,
script_location=_config.Raw2CleanConfig.
PARQUET_GLUE_SCRIPT_LOCATION,
raw_db=raw_db,
clean_db=clean_db,
raw_bucket=raw_bucket,
clean_bucket=clean_bucket),
Raw2CleanJob(self,
'Raw2CleanCustomer',
source_entity='store_customer',
target_entity='store_customer',
datetime_column='customer_datetime',
date_column='customer_date',
script_bucket=binary_bucket,
script_location=_config.Raw2CleanConfig.
HUDI_GLUE_SCRIPT_LOCATION,
raw_db=raw_db,
clean_db=clean_db,
raw_bucket=raw_bucket,
clean_bucket=clean_bucket,
format='hudi',
hudi_primary_key='customer_id'),
Raw2CleanJob(self,
'Raw2CleanCustomerAddress',
source_entity='store_customer_address',
target_entity='store_customer_address',
datetime_column='address_datetime',
date_column='address_date',
script_bucket=binary_bucket,
script_location=_config.Raw2CleanConfig.
HUDI_GLUE_SCRIPT_LOCATION,
raw_db=raw_db,
clean_db=clean_db,
raw_bucket=raw_bucket,
clean_bucket=clean_bucket,
format='hudi',
hudi_primary_key='address_id'),
Raw2CleanJob(self,
'Raw2CleanStoreSale1',
source_entity='store_sale1',
target_entity='store_sale',
datetime_column='sale_datetime',
date_column='sale_date',
partition_column='sale_date',
script_bucket=binary_bucket,
script_location=_config.Raw2CleanConfig.
PARQUET_GLUE_SCRIPT_LOCATION,
raw_db=raw_db,
clean_db=clean_db,
raw_bucket=raw_bucket,
clean_bucket=clean_bucket)
]
# Create a glue workflow that triggers every 30 minutes to execute crawler and all jobs
raw2clean_wf = CfnWorkflow(self, 'Raw2Clean', name='ara-raw2clean')
CfnTrigger(
self,
'Raw2CleanWorkflowStart',
name='ara-raw2clean-workflow-start',
description=
'Trigger that starts the workflow every 30 minutes by triggering the raw crawler',
type='SCHEDULED',
schedule='cron(*/30 * * * ? *)',
start_on_creation=True,
actions=[{
'crawlerName': raw_crawler.ref
}],
workflow_name=raw2clean_wf.ref)
# Add all raw2clean jobs to the predicate condition and actions of the corresponding trigger
pre_job_trigger_actions = []
post_job_trigger_predicate_conditions = []
for r2c_job in raw2clean_jobs:
pre_job_trigger_actions.append({'jobName': r2c_job.job.ref})
post_job_trigger_predicate_conditions.append({
'jobName':
r2c_job.job.ref,
'state':
'SUCCEEDED',
'logicalOperator':
'EQUALS'
})
CfnTrigger(self,
'Raw2CleanJobs',
name='ara-raw2clean-jobs',
description=
'Trigger that starts the transformation from raw to clean',
type='CONDITIONAL',
start_on_creation=True,
predicate={
'conditions': [{
'crawlerName': raw_crawler.ref,
'crawlState': 'SUCCEEDED',
'logicalOperator': 'EQUALS'
}]
},
actions=pre_job_trigger_actions,
workflow_name=raw2clean_wf.ref)
Tags.of(self).add('module-name', 'batch')
