def __init__(self, scope: Construct, id: str, functions: LambdaLib, **kwargs) -> None:
super().__init__(scope, id)
# Step Function
submit_job = tasks.LambdaInvoke(self, "Submit Job",
lambda_function=functions.send_email_approval,
payload=sfn.TaskInput.from_object({'ExecutionContext.$': '$$'}),
result_path=sfn.JsonPath.DISCARD
)
wait_x = sfn.Wait(self, "Wait",
time= sfn.WaitTime.duration(Duration.minutes(2))
)
get_status = tasks.LambdaInvoke(self, "Get Job Status",
lambda_function=functions.check_status_dynamo,
payload=sfn.TaskInput.from_object({'ExecutionContext.$': '$$'}),
result_path="$.status"
)
restrict_es = tasks.LambdaInvoke(self, "Restric ES Policy",
lambda_function=functions.restric_es_policy,
payload=sfn.TaskInput.from_object({'ExecutionContext.$': '$$'}),
)
restrict_rds = tasks.LambdaInvoke(self, "Restric RDS",
lambda_function=functions.restric_rds_policy,
payload=sfn.TaskInput.from_object({'ExecutionContext.$': '$$'}),
)
restrict_es_condition = sfn.Condition.string_equals("$.detail.additionalEventData.configRuleName", constants.CONFIG_RULE_ES_PUBLIC)
restrict_rds_condition = sfn.Condition.string_equals("$.detail.additionalEventData.configRuleName", constants.CONFIG_RULE_RDS_PUBLIC)
definition = (submit_job.next(wait_x)
.next(get_status)
.next(sfn.Choice(self, "Job Complete?")
.when(sfn.Condition.string_equals("$.status.Payload.status", "Rejected!"), wait_x)
# .when(sfn.Condition.string_equals("$.status.Payload.status", "NON_COMPLIANT"), final_task)
# .when(sfn.Condition.string_equals("$.status.Payload.status", "Accepted!"), final_task))
.otherwise(sfn.Choice(self, "Remediation Choice")
.when(restrict_es_condition, restrict_es)
.when(restrict_rds_condition, restrict_rds)))
)
self.state_machine = sfn.StateMachine(self, "StateMachine",
definition=definition,
timeout=Duration.hours(2)
)
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
flip_coin_function = lambda_.Function(
self,
"FlipCoinFunction",
runtime=lambda_.Runtime.PYTHON_3_8,
handler="index.handler",
code=lambda_.Code.from_asset("./sfn/lambda/flip_coin"))
flip_coin_invoke = tasks.LambdaInvoke(
self, "FlipCoin", lambda_function=flip_coin_function)
wait = stepfunctions.Wait(self,
"Wait",
time=stepfunctions.WaitTime.duration(
core.Duration.seconds(5)))
tails_result = stepfunctions.Pass(self, "TailsResult")
tails_result.next(flip_coin_invoke)
choice = stepfunctions.Choice(self,
"HeadsTailsChoice") \
.when(condition=stepfunctions.Condition.string_equals("$.Payload.result", "heads"),
next=stepfunctions.Succeed(self, "HeadsResult")) \
.when(condition=stepfunctions.Condition.string_equals("$.Payload.result", "tails"),
next=tails_result)
stepfunctions.StateMachine(self,
"StateMachine",
definition=flip_coin_invoke.next(
wait.next(choice)))
def __init__(self,
scope: core.Construct,
id: str,
*,
polling_delay: int = 5,
statemachine_timeout: int = 300,
**kwargs):
super().__init__(scope, id, **kwargs)
state_fn = StateHandlerLambda(self, "config-state-handler").function
config_fn = AccountConfigLambda(self,
"account-config-handler").function
config_state = tasks.LambdaInvoke(self,
"Set Configuring State",
lambda_function=state_fn,
output_path="$.Payload")
completed_state = tasks.LambdaInvoke(self,
"Set Completed State",
lambda_function=state_fn,
output_path="$.Payload")
config_task = tasks.LambdaInvoke(self,
"Request Account Configuration",
lambda_function=config_fn,
output_path="$.Payload")
polling_task = tasks.LambdaInvoke(self,
"Poll Account Configuration",
lambda_function=config_fn,
output_path="$.Payload")
delay = sfn.Wait(self,
"Delay Polling",
time=sfn.WaitTime.duration(
core.Duration.seconds(polling_delay)))
is_ready = sfn.Choice(self, "Account Ready?")
acct_ready = sfn.Condition.string_equals('$.state', "READY")
acct_pending = sfn.Condition.string_equals('$.state', "PENDING")
success = sfn.Succeed(self, "Config Succeeded")
failed = sfn.Fail(self,
"Config Failed",
cause="Bad value in Polling loop")
# this is the loop which polls for state change, either looping back to delay or setting completion state and finishing
is_ready.when(acct_pending, delay).when(
acct_ready, completed_state.next(success)).otherwise(failed)
# this is the main chain starting with creation request a delay and then polling loop
config_chain = config_task.next(config_state).next(delay).next(
polling_task).next(is_ready)
self.state_machine = sfn.StateMachine(
self,
"Account-Config-StateMachine",
definition=config_chain,
timeout=core.Duration.seconds(statemachine_timeout))
def __init__(self, scope: core.App, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
pass_through_lambda = _lambda.Function(
self,
'PassThroughLambda',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='pass_through_lambda.handler')
loop_count_lambda = _lambda.Function(
self,
'LoopCountLambda',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='loop_count_lambda.handler')
start_state_machine = sfn.Task(
self,
"Start CodeBuild Lambda",
task=sfn_tasks.InvokeFunction(pass_through_lambda))
wait_x = sfn.Wait(
self,
"Wait X Seconds",
time=sfn.WaitTime.seconds_path('$.wait_time'),
)
get_state_machine_status = sfn.Task(
self,
"Get Build Status",
task=sfn_tasks.InvokeFunction(loop_count_lambda))
is_complete = sfn.Choice(self, "Job Complete?")
state_machine_failed = sfn.Fail(self,
"Build Failed",
cause="AWS Batch Job Failed",
error="DescribeJob returned FAILED")
state_machine_success = sfn.Pass(self, "Build Successs")
definition = start_state_machine\
.next(wait_x)\
.next(get_state_machine_status)\
.next(is_complete
.when(sfn.Condition.string_equals(
"$.status", "FAILED"), state_machine_failed)
.when(sfn.Condition.string_equals(
"$.status", "SUCCEEDED"), state_machine_success)
.otherwise(wait_x))
sfn.StateMachine(
self,
"StateMachine",
definition=definition,
timeout=core.Duration.seconds(60),
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Step Function Starts Here
# The first thing we need to do is see if they are asking for pineapple on a pizza
pineapple_check_lambda = _lambda.Function(self, "pineappleCheckLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="orderPizza.handler",
code=_lambda.Code.from_asset("lambdas"),
)
# Step functions are built up of steps, we need to define our first step
order_pizza = step_fn.Task(self, 'Order Pizza Job',
task=step_fn_tasks.InvokeFunction(pineapple_check_lambda),
input_path='$.flavour',
result_path='$.pineappleAnalysis'
)
# Pizza Order failure step defined
job_failed = step_fn.Fail(self, 'Sorry, We Dont add Pineapple',
cause='Failed To Make Pizza',
error='They asked for Pineapple')
# If they didnt ask for pineapple let's cook the pizza
cook_pizza = step_fn.Pass(self, 'Lets make your pizza')
# If they ask for a pizza with pineapple, fail. Otherwise cook the pizza
definition = step_fn.Chain \
.start(order_pizza) \
.next(step_fn.Choice(self, 'With Pineapple?') \
.when(step_fn.Condition.boolean_equals('$.pineappleAnalysis.containsPineapple', True), job_failed) \
.otherwise(cook_pizza))
state_machine = step_fn.StateMachine(self, 'StateMachine', definition=definition, timeout=core.Duration.minutes(5))
# Dead Letter Queue Setup
dlq = sqs.Queue(self, 'stateMachineLambdaDLQ', visibility_timeout=core.Duration.seconds(300))
# defines an AWS Lambda resource to connect to our API Gateway
state_machine_lambda = _lambda.Function(self, "stateMachineLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="stateMachineLambda.handler",
code=_lambda.Code.from_asset("lambdas"),
environment={
'statemachine_arn': state_machine.state_machine_arn
}
)
state_machine.grant_start_execution(state_machine_lambda)
# defines an API Gateway REST API resource backed by our "sqs_publish_lambda" function.
api_gw.LambdaRestApi(self, 'Endpoint',
handler=state_machine_lambda
)
def __init__(self, app: cdk.App, id: str, **kwargs) -> None:
super().__init__(app, id, **kwargs)
submit_job_activity = sfn.Activity(
self, "SubmitJob"
)
check_job_activity = sfn.Activity(
self, "CheckJob"
)
submit_job = sfn.Task(
self, "Submit Job",
task=sfn_tasks.InvokeActivity(submit_job_activity),
result_path="$.guid",
)
wait_x = sfn.Wait(
self, "Wait X Seconds",
duration=sfn.WaitDuration.seconds_path('$.wait_time'),
)
get_status = sfn.Task(
self, "Get Job Status",
task=sfn_tasks.InvokeActivity(check_job_activity),
input_path="$.guid",
result_path="$.status",
)
is_complete = sfn.Choice(
self, "Job Complete?"
)
job_failed = sfn.Fail(
self, "Job Failed",
cause="AWS Batch Job Failed",
error="DescribeJob returned FAILED"
)
final_status = sfn.Task(
self, "Get Final Job Status",
task=sfn_tasks.InvokeActivity(check_job_activity),
input_path="$.guid",
)
definition = submit_job\
.next(wait_x)\
.next(get_status)\
.next(is_complete
.when(sfn.Condition.string_equals(
"$.status", "FAILED"), job_failed)
.when(sfn.Condition.string_equals(
"$.status", "SUCCEEDED"), final_status)
.otherwise(wait_x))
sfn.StateMachine(
self, "StateMachine",
definition=definition,
timeout_sec=30,
)
def __init__(self, scope: core.Construct, id: builtins.str,
action_name: str, resources: FsiSharedResources,
function: lambda_.Function) -> None:
super().__init__(scope, id)
self.__resources = resources
state_machine_name = id
# Define the state machine definition...
invoke_function = sft.LambdaInvoke(
self,
'InvokeFunction',
lambda_function=function,
invocation_type=sft.LambdaInvocationType.REQUEST_RESPONSE,
input_path='$.Payload',
result_path='$.Result')
choice = sf.Choice(self,
'IsComplete',
comment='Check if theres more to process')
choice.when(
sf.Condition.string_equals('$.Result.Payload.Result.RunState',
'RunStatus.MORE_AVAILABLE'),
invoke_function)
choice.when(
sf.Condition.string_equals('$.Result.Payload.Result.RunState',
'RunStatus.COMPLETE'),
sf.Pass(self, 'Finalize', comment='Workflow Complete'))
choice.otherwise(
sf.Fail(self,
'NotImplemented',
cause='Unknown Choice',
error='NotImplementedException'))
definition = invoke_function.next(choice)
# Register the definition as StateMachine...
zone_name = self.resources.landing_zone.zone_name
self.state_machine = sf.StateMachine(
self,
'StateMachine',
state_machine_name=state_machine_name,
state_machine_type=sf.StateMachineType.STANDARD,
timeout=core.Duration.hours(2),
logs=sf.LogOptions(destination=logs.LogGroup(
self,
'LogGroup',
removal_policy=core.RemovalPolicy.DESTROY,
retention=RetentionDays.TWO_WEEKS,
log_group_name='/homenet/fsi-{}/states/{}/{}'.format(
zone_name, self.component_name, action_name).lower())),
tracing_enabled=True,
definition=definition)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
lambda_role = _iam.Role(
self,
id='lab3-om-role',
assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com'))
cloudwatch_policy_statement = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW)
cloudwatch_policy_statement.add_actions("logs:CreateLogGroup")
cloudwatch_policy_statement.add_actions("logs:CreateLogStream")
cloudwatch_policy_statement.add_actions("logs:PutLogEvents")
cloudwatch_policy_statement.add_actions("logs:DescribeLogStreams")
cloudwatch_policy_statement.add_resources("*")
lambda_role.add_to_policy(cloudwatch_policy_statement)
fn_lambda_approve_reject = aws_lambda.Function(
self,
"lab3-om-approve-reject",
code=aws_lambda.AssetCode(
"../lambda-functions/approve-reject-application/"),
handler="app.handler",
tracing=aws_lambda.Tracing.ACTIVE,
timeout=core.Duration.seconds(30),
role=lambda_role,
runtime=aws_lambda.Runtime.PYTHON_3_8)
fn_lambda_verify_identity = aws_lambda.Function(
self,
"lab3-om-verify-identity",
code=aws_lambda.AssetCode("../lambda-functions/verify-identity/"),
handler="app.handler",
tracing=aws_lambda.Tracing.ACTIVE,
timeout=core.Duration.seconds(30),
role=lambda_role,
runtime=aws_lambda.Runtime.PYTHON_3_8)
fn_lambda_check_address = aws_lambda.Function(
self,
"lab3-om-check-address",
code=aws_lambda.AssetCode("../lambda-functions/check-address/"),
handler="app.handler",
tracing=aws_lambda.Tracing.ACTIVE,
timeout=core.Duration.seconds(30),
role=lambda_role,
runtime=aws_lambda.Runtime.PYTHON_3_8)
'''
[INFO] This is a sample how to define the task and integrate with Lambda Functions. You need to create another 2 tasks for respective Lambda functions
'''
task_verify_identity = _tasks.LambdaInvoke(
self,
"Verify Identity Document",
lambda_function=fn_lambda_verify_identity,
output_path="$.Payload")
task_check_address = _tasks.LambdaInvoke(
self,
"Check Address",
lambda_function=fn_lambda_check_address,
output_path="$.Payload")
task_wait_review = _tasks.LambdaInvoke(
self,
"Wait for Review",
lambda_function=fn_lambda_approve_reject,
output_path="$.Payload")
state_approve = _sfn.Succeed(self, "Approve Application")
state_reject = _sfn.Succeed(self, "Reject Application")
# Let's define the State Machine, step by step
# First, paralell tasks for verification
s_verification = _sfn.Parallel(self, "Verification")
s_verification.branch(task_verify_identity)
s_verification.branch(task_check_address)
# Next, we add a choice state
c_human_review = _sfn.Choice(self, "Human review required?")
c_human_review.when(
_sfn.Condition.and_(
_sfn.Condition.boolean_equals("$[0].humanReviewRequired",
False),
_sfn.Condition.boolean_equals("$[1].humanReviewRequired",
True)), state_approve)
c_human_review.when(
_sfn.Condition.or_(
_sfn.Condition.boolean_equals("$[0].humanReviewRequired",
True),
_sfn.Condition.boolean_equals("$[1].humanReviewRequired",
False)), task_wait_review)
# Another choice state to check if the application passed the review
c_review_approved = _sfn.Choice(self, "Review approved?")
c_review_approved.when(
_sfn.Condition.boolean_equals("$.reviewApproved", True),
state_approve)
c_review_approved.when(
_sfn.Condition.boolean_equals("$.reviewApproved", False),
state_reject)
task_wait_review.next(c_review_approved)
definition = s_verification.next(c_human_review)
_sfn.StateMachine(self,
"lab3-statemachine",
definition=definition,
timeout=core.Duration.minutes(5))
def __init__(
self,
scope: Construct,
stack_id: str,
*,
botocore_lambda_layer: aws_lambda_python.PythonLayerVersion,
env_name: str,
storage_bucket: aws_s3.Bucket,
validation_results_table: Table,
) -> None:
# pylint: disable=too-many-locals, too-many-statements
super().__init__(scope, stack_id)
############################################################################################
# PROCESSING ASSETS TABLE
processing_assets_table = Table(
self,
f"{env_name}-processing-assets",
env_name=env_name,
parameter_name=ParameterName.PROCESSING_ASSETS_TABLE_NAME,
sort_key=aws_dynamodb.Attribute(name="sk", type=aws_dynamodb.AttributeType.STRING),
)
############################################################################################
# BATCH JOB DEPENDENCIES
batch_job_queue = BatchJobQueue(
self,
"batch-job-queue",
env_name=env_name,
processing_assets_table=processing_assets_table,
).job_queue
s3_read_only_access_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3ReadOnlyAccess"
)
############################################################################################
# UPDATE CATALOG UPDATE MESSAGE QUEUE
dead_letter_queue = aws_sqs.Queue(
self,
"dead-letter-queue",
visibility_timeout=LAMBDA_TIMEOUT,
)
self.message_queue = aws_sqs.Queue(
self,
"update-catalog-message-queue",
visibility_timeout=LAMBDA_TIMEOUT,
dead_letter_queue=aws_sqs.DeadLetterQueue(max_receive_count=3, queue=dead_letter_queue),
)
self.message_queue_name_parameter = aws_ssm.StringParameter(
self,
"update-catalog-message-queue-name",
string_value=self.message_queue.queue_name,
description=f"Update Catalog Message Queue Name for {env_name}",
parameter_name=ParameterName.UPDATE_CATALOG_MESSAGE_QUEUE_NAME.value,
)
populate_catalog_lambda = BundledLambdaFunction(
self,
"populate-catalog-bundled-lambda-function",
directory="populate_catalog",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
botocore_lambda_layer=botocore_lambda_layer,
)
self.message_queue.grant_consume_messages(populate_catalog_lambda)
populate_catalog_lambda.add_event_source(
SqsEventSource(self.message_queue, batch_size=1) # type: ignore[arg-type]
)
############################################################################################
# STATE MACHINE TASKS
check_stac_metadata_task = LambdaTask(
self,
"check-stac-metadata-task",
directory="check_stac_metadata",
botocore_lambda_layer=botocore_lambda_layer,
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
assert check_stac_metadata_task.lambda_function.role
check_stac_metadata_task.lambda_function.role.add_managed_policy(
policy=s3_read_only_access_policy
)
for table in [processing_assets_table, validation_results_table]:
table.grant_read_write_data(check_stac_metadata_task.lambda_function)
table.grant(
check_stac_metadata_task.lambda_function,
"dynamodb:DescribeTable",
)
content_iterator_task = LambdaTask(
self,
"content-iterator-task",
directory="content_iterator",
botocore_lambda_layer=botocore_lambda_layer,
result_path=f"$.{CONTENT_KEY}",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
check_files_checksums_directory = "check_files_checksums"
check_files_checksums_default_payload_object = {
f"{DATASET_ID_KEY}.$": f"$.{DATASET_ID_KEY}",
f"{VERSION_ID_KEY}.$": f"$.{VERSION_ID_KEY}",
f"{METADATA_URL_KEY}.$": f"$.{METADATA_URL_KEY}",
f"{FIRST_ITEM_KEY}.$": f"$.{CONTENT_KEY}.{FIRST_ITEM_KEY}",
f"{ASSETS_TABLE_NAME_KEY}.$": f"$.{CONTENT_KEY}.{ASSETS_TABLE_NAME_KEY}",
f"{RESULTS_TABLE_NAME_KEY}.$": f"$.{CONTENT_KEY}.{RESULTS_TABLE_NAME_KEY}",
}
check_files_checksums_single_task = BatchSubmitJobTask(
self,
"check-files-checksums-single-task",
env_name=env_name,
directory=check_files_checksums_directory,
s3_policy=s3_read_only_access_policy,
job_queue=batch_job_queue,
payload_object=check_files_checksums_default_payload_object,
container_overrides_command=[
"--dataset-id",
f"Ref::{DATASET_ID_KEY}",
"--version-id",
f"Ref::{VERSION_ID_KEY}",
"--first-item",
f"Ref::{FIRST_ITEM_KEY}",
"--assets-table-name",
f"Ref::{ASSETS_TABLE_NAME_KEY}",
"--results-table-name",
f"Ref::{RESULTS_TABLE_NAME_KEY}",
],
)
array_size = int(
aws_stepfunctions.JsonPath.number_at(f"$.{CONTENT_KEY}.{ITERATION_SIZE_KEY}")
)
check_files_checksums_array_task = BatchSubmitJobTask(
self,
"check-files-checksums-array-task",
env_name=env_name,
directory=check_files_checksums_directory,
s3_policy=s3_read_only_access_policy,
job_queue=batch_job_queue,
payload_object=check_files_checksums_default_payload_object,
container_overrides_command=[
"--dataset-id",
f"Ref::{DATASET_ID_KEY}",
"--version-id",
f"Ref::{VERSION_ID_KEY}",
"--first-item",
f"Ref::{FIRST_ITEM_KEY}",
"--assets-table-name",
f"Ref::{ASSETS_TABLE_NAME_KEY}",
"--results-table-name",
f"Ref::{RESULTS_TABLE_NAME_KEY}",
],
array_size=array_size,
)
for reader in [
content_iterator_task.lambda_function,
check_files_checksums_single_task.job_role,
check_files_checksums_array_task.job_role,
]:
processing_assets_table.grant_read_data(reader) # type: ignore[arg-type]
processing_assets_table.grant(
reader, "dynamodb:DescribeTable" # type: ignore[arg-type]
)
for writer in [
check_files_checksums_single_task.job_role,
check_files_checksums_array_task.job_role,
]:
validation_results_table.grant_read_write_data(writer) # type: ignore[arg-type]
validation_results_table.grant(
writer, "dynamodb:DescribeTable" # type: ignore[arg-type]
)
validation_summary_task = LambdaTask(
self,
"validation-summary-task",
directory="validation_summary",
botocore_lambda_layer=botocore_lambda_layer,
result_path=f"$.{VALIDATION_KEY}",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
validation_results_table.grant_read_data(validation_summary_task.lambda_function)
validation_results_table.grant(
validation_summary_task.lambda_function, "dynamodb:DescribeTable"
)
import_dataset_role = aws_iam.Role(
self,
"import-dataset",
assumed_by=aws_iam.ServicePrincipal( # type: ignore[arg-type]
"batchoperations.s3.amazonaws.com"
),
)
import_asset_file_function = ImportFileFunction(
self,
directory="import_asset_file",
invoker=import_dataset_role,
env_name=env_name,
botocore_lambda_layer=botocore_lambda_layer,
)
import_metadata_file_function = ImportFileFunction(
self,
directory="import_metadata_file",
invoker=import_dataset_role,
env_name=env_name,
botocore_lambda_layer=botocore_lambda_layer,
)
import_dataset_task = LambdaTask(
self,
"import-dataset-task",
directory="import_dataset",
botocore_lambda_layer=botocore_lambda_layer,
result_path=f"$.{IMPORT_DATASET_KEY}",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
import_dataset_task.lambda_function.add_to_role_policy(
aws_iam.PolicyStatement(
resources=[import_dataset_role.role_arn],
actions=["iam:PassRole"],
),
)
import_dataset_task.lambda_function.add_to_role_policy(
aws_iam.PolicyStatement(resources=["*"], actions=["s3:CreateJob"])
)
for table in [processing_assets_table]:
table.grant_read_data(import_dataset_task.lambda_function)
table.grant(import_dataset_task.lambda_function, "dynamodb:DescribeTable")
# Import status check
wait_before_upload_status_check = Wait(
self,
"wait-before-upload-status-check",
time=WaitTime.duration(Duration.seconds(10)),
)
upload_status_task = LambdaTask(
self,
"upload-status",
directory="upload_status",
botocore_lambda_layer=botocore_lambda_layer,
result_path="$.upload_status",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
validation_results_table.grant_read_data(upload_status_task.lambda_function)
validation_results_table.grant(upload_status_task.lambda_function, "dynamodb:DescribeTable")
upload_status_task.lambda_function.add_to_role_policy(ALLOW_DESCRIBE_ANY_S3_JOB)
# Parameters
import_asset_file_function_arn_parameter = aws_ssm.StringParameter(
self,
"import asset file function arn",
string_value=import_asset_file_function.function_arn,
description=f"Import asset file function ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_IMPORT_ASSET_FILE_FUNCTION_TASK_ARN.value,
)
import_metadata_file_function_arn_parameter = aws_ssm.StringParameter(
self,
"import metadata file function arn",
string_value=import_metadata_file_function.function_arn,
description=f"Import metadata file function ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_IMPORT_METADATA_FILE_FUNCTION_TASK_ARN.value,
)
import_dataset_role_arn_parameter = aws_ssm.StringParameter(
self,
"import dataset role arn",
string_value=import_dataset_role.role_arn,
description=f"Import dataset role ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_IMPORT_DATASET_ROLE_ARN.value,
)
update_dataset_catalog = LambdaTask(
self,
"update-dataset-catalog",
directory="update_dataset_catalog",
botocore_lambda_layer=botocore_lambda_layer,
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
self.message_queue.grant_send_messages(update_dataset_catalog.lambda_function)
for storage_writer in [
import_dataset_role,
import_dataset_task.lambda_function,
import_asset_file_function,
import_metadata_file_function,
populate_catalog_lambda,
update_dataset_catalog.lambda_function,
]:
storage_bucket.grant_read_write(storage_writer) # type: ignore[arg-type]
grant_parameter_read_access(
{
import_asset_file_function_arn_parameter: [import_dataset_task.lambda_function],
import_dataset_role_arn_parameter: [import_dataset_task.lambda_function],
import_metadata_file_function_arn_parameter: [import_dataset_task.lambda_function],
processing_assets_table.name_parameter: [
check_stac_metadata_task.lambda_function,
content_iterator_task.lambda_function,
import_dataset_task.lambda_function,
],
validation_results_table.name_parameter: [
check_stac_metadata_task.lambda_function,
content_iterator_task.lambda_function,
validation_summary_task.lambda_function,
upload_status_task.lambda_function,
],
self.message_queue_name_parameter: [update_dataset_catalog.lambda_function],
}
)
success_task = aws_stepfunctions.Succeed(self, "success")
upload_failure = aws_stepfunctions.Fail(self, "upload failure")
validation_failure = aws_stepfunctions.Succeed(self, "validation failure")
############################################################################################
# STATE MACHINE
dataset_version_creation_definition = (
check_stac_metadata_task.next(content_iterator_task)
.next(
aws_stepfunctions.Choice( # type: ignore[arg-type]
self, "check_files_checksums_maybe_array"
)
.when(
aws_stepfunctions.Condition.number_equals(
f"$.{CONTENT_KEY}.{ITERATION_SIZE_KEY}", 1
),
check_files_checksums_single_task.batch_submit_job,
)
.otherwise(check_files_checksums_array_task.batch_submit_job)
.afterwards()
)
.next(
aws_stepfunctions.Choice(self, "content_iteration_finished")
.when(
aws_stepfunctions.Condition.number_equals(
f"$.{CONTENT_KEY}.{NEXT_ITEM_KEY}", -1
),
validation_summary_task.next(
aws_stepfunctions.Choice( # type: ignore[arg-type]
self, "validation_successful"
)
.when(
aws_stepfunctions.Condition.boolean_equals(
f"$.{VALIDATION_KEY}.{SUCCESS_KEY}", True
),
import_dataset_task.next(
wait_before_upload_status_check # type: ignore[arg-type]
)
.next(upload_status_task)
.next(
aws_stepfunctions.Choice(
self, "import_completed" # type: ignore[arg-type]
)
.when(
aws_stepfunctions.Condition.and_(
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Complete"
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{METADATA_UPLOAD_KEY}.status",
"Complete",
),
),
update_dataset_catalog.next(
success_task # type: ignore[arg-type]
),
)
.when(
aws_stepfunctions.Condition.or_(
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{ASSET_UPLOAD_KEY}.status",
"Cancelled",
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Failed"
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{METADATA_UPLOAD_KEY}.status",
"Cancelled",
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{METADATA_UPLOAD_KEY}.status",
"Failed",
),
),
upload_failure, # type: ignore[arg-type]
)
.otherwise(
wait_before_upload_status_check # type: ignore[arg-type]
)
),
)
.otherwise(validation_failure) # type: ignore[arg-type]
),
)
.otherwise(content_iterator_task)
)
)
self.state_machine = aws_stepfunctions.StateMachine(
self,
f"{env_name}-dataset-version-creation",
definition=dataset_version_creation_definition, # type: ignore[arg-type]
)
self.state_machine_parameter = aws_ssm.StringParameter(
self,
"state machine arn",
description=f"State machine ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_DATASET_VERSION_CREATION_STEP_FUNCTION_ARN.value, # pylint:disable=line-too-long
string_value=self.state_machine.state_machine_arn,
)
Tags.of(self).add("ApplicationLayer", "processing") # type: ignore[arg-type]
def __init__(self, scope: cdk.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
####################################################################################
# IoT Events
# IoT Events: Execution role
iot_events_execution_role = iam.Role(
self,
"IoTEventsExecutionRole",
assumed_by=iam.ServicePrincipal("iotevents.amazonaws.com"))
iot_events_execution_role.add_to_policy(
iam.PolicyStatement(resources=["*"], actions=["iot:Publish"]))
iot_events_execution_role.add_to_policy(
iam.PolicyStatement(resources=["*"], actions=["SNS:Publish"]))
# IoT Events: Input
inputDefinitionProperty = iotevents.CfnInput.InputDefinitionProperty(
attributes=[{
"jsonPath": "gatewayid"
}, {
"jsonPath": "last_uplink_received_timestamp_ms"
}, {
"jsonPath": "last_connection_status"
}, {
"jsonPath": "timestamp_iso8601"
}])
iot_events_input = iotevents.CfnInput(
self,
"LoRaWANGatewayConnectivityStatusInput",
input_definition=inputDefinitionProperty,
input_name="LoRaWANGatewayConnectivityStatusInput",
input_description=
"Input for connectivity status updates for LoRaWAN gateways")
# IoT Events: Detector Model
detector_model_definition = iotevents.CfnDetectorModel.DetectorModelDefinitionProperty(
initial_state_name=lorawan_gateway_monitoring_detectormodel.
initial_state_name,
states=lorawan_gateway_monitoring_detectormodel.get_states(self))
iot_events_model = iotevents.CfnDetectorModel(
self,
"LoRaWANGatewayConnectivityModel",
detector_model_definition=detector_model_definition,
detector_model_name="LoRaWANGatewayConnectivityModel",
detector_model_description=
"Detector model for LoRaWAN gateway connectivity status",
key="gatewayid",
evaluation_method="BATCH",
role_arn=iot_events_execution_role.role_arn)
####################################################################################
# Lambda function GetWirelessGatewayStatisticsLambda
# Lambda function GetWirelessGatewayStatisticsLambda: Execution Role
get_wireless_gateway_statistics_lambda_role = iam.Role(
self,
"GetWirelessGatewayStatisticsLambdaExecutionRole",
assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"))
get_wireless_gateway_statistics_lambda_role.add_to_policy(
iam.PolicyStatement(resources=[
"arn:aws:iotwireless:" + self.region + ":" + self.account +
":WirelessGateway/*"
],
actions=[
"iotwireless:ListWirelessGateways",
"iotwireless:GetWirelessGatewayStatistics"
]))
get_wireless_gateway_statistics_lambda_role.add_to_policy(
iam.PolicyStatement(resources=[
"arn:aws:iotevents:" + self.region + ":" + self.account +
":input/LoRaWANGatewayConnectivityStatusInput"
],
actions=["iotevents:BatchPutMessage"]))
# Lambda function GetWirelessGatewayStatisticsLambda: Lambda function configuration
get_wireless_gateway_statistics_lambda = lambda_.Function(
self,
"GetWirelessGatewayStatisticsLambda",
code=lambda_.Code.asset(
"src_get_wireless_gateway_statistics_lambda"),
runtime=lambda_.Runtime.PYTHON_3_7,
handler="lambda.handler",
role=get_wireless_gateway_statistics_lambda_role,
timeout=cdk.Duration.seconds(25))
get_wireless_gateway_statistics_lambda.add_environment(
"TEST_MODE", "true")
get_wireless_gateway_statistics_lambda.add_environment(
"IOT_EVENTS_INPUT_NAME", "LoRaWANGatewayConnectivityStatusInput")
####################################################################################
# SNS topic
sns_topic = sns.Topic(
self,
"LoRaWANGatewayNotificationTopic",
display_name=
"Topic to use for notifications about LoRaWAN gateway events like connect or disconnect",
topic_name="LoRaWANGatewayNotificationTopic")
email_address = cdk.CfnParameter(self, "emailforalarms")
sns_topic.add_subscription(
subscriptions.EmailSubscription(email_address.value_as_string))
####################################################################################
# Step Function
# State 'Fail'
failure_state = sfn.Fail(self, "Fail")
# State 'Wait'
wait_state = sfn.Wait(self,
"Sleep",
time=sfn.WaitTime.duration(
cdk.Duration.minutes(4)))
# State 'Ingest gateway connectivity status into IoT Events input'
lambda_invoke_state = tasks.LambdaInvoke(
self,
"Ingest gateway connectivity status into IoT Events input",
result_path="$.wireless_gateway_stats",
lambda_function=get_wireless_gateway_statistics_lambda
# payload=task_input_payload
)
# Stat 'Did IoT events ingestion run successfull?'
choice_lambda_state = sfn.Choice(
self, "Did IoT events ingestion run successfull?")
choice_lambda_state.when(
sfn.Condition.number_equals(
"$.wireless_gateway_stats.Payload.status", 200), wait_state)
choice_lambda_state.otherwise(failure_state)
# Define transitions
wait_state.next(lambda_invoke_state)
lambda_invoke_state.next(choice_lambda_state)
# Crreate a state machine
gateway_watchdog_state_machine = sfn.StateMachine(
self,
"LoRaWANGatewayWatchdogStatemachine",
definition=lambda_invoke_state,
state_machine_name="LoRaWANGatewayWatchdogStatemachine")
####################################################################################
# CloudFormation Stack outputs
cdk.CfnOutput(
self,
"StateMachineARN",
value=gateway_watchdog_state_machine.state_machine_arn,
description=
"Please run 'aws stepfunctions start-execution --state-machine-arn <LorawanConnectivityWatchdogStack.StateMachineARN>' to start the monitoring of LoRaWAN gateway connectivity",
)
cdk.CfnOutput(
self,
"StateMachineStartCommand",
value='aws stepfunctions start-execution --state-machine-arn ' +
gateway_watchdog_state_machine.state_machine_arn,
description=
"Please run this command to start the monitoring of LoRaWAN gateway connectivity",
)
cdk.CfnOutput(
self,
"StateMachineStopommand",
value='aws stepfunctions stop-execution --state-machine-arn ' +
gateway_watchdog_state_machine.state_machine_arn,
description=
"Please run this command to stop the monitoring of LoRaWAN gateway connectivity",
)
def __init__(self, scope, id, *args, **kwargs):
super().__init__(scope, id, *args, **kwargs)
# Buckets
source_bucket = s3.Bucket(self, "SourceBucket")
dest_bucket = s3.Bucket(self, "DestinationBucket")
processing_bucket = s3.Bucket(self, "ProcessingBucket")
# Lambda Functions
generate_workflow_input_lambda = aws_lambda.Function(
self, "GenerateWorkflowInputFunction",
code=aws_lambda.Code.from_asset(str(DIST_PATH)),
runtime=aws_lambda.Runtime.PYTHON_3_8,
handler="generate_workflow_input.lambda_handler",
environment={
"InputBucketName": source_bucket.bucket_name,
"ProcessingBucketName": processing_bucket.bucket_name,
"OutputBucketName": dest_bucket.bucket_name
}
)
check_workflow_ready_lambda = aws_lambda.Function(
self, "CheckWorkflowReadyFunction",
code=aws_lambda.Code.from_asset(str(DIST_PATH)),
runtime=aws_lambda.Runtime.PYTHON_3_8,
handler="check_workflow_ready.lambda_handler"
)
string_replace_lambda = aws_lambda.Function(
self, "StringReplaceFunction",
code=aws_lambda.Code.from_asset(str(DIST_PATH)),
runtime=aws_lambda.Runtime.PYTHON_3_8,
handler="string_replace.lambda_handler"
)
calculate_total_earnings_lambda = aws_lambda.Function(
self, "CalculateTotalEarningsFunction",
code=aws_lambda.Code.from_asset(str(DIST_PATH)),
runtime=aws_lambda.Runtime.PYTHON_3_8,
handler="calculate_total_earnings.lambda_handler"
)
convert_csv_to_json_lambda = aws_lambda.Function(
self, "ConvertCsvToJsonFunction",
code=aws_lambda.Code.from_asset(str(DIST_PATH)),
runtime=aws_lambda.Runtime.PYTHON_3_8,
handler="convert_csv_to_json.lambda_handler"
)
# Permissions
source_bucket.grant_read(check_workflow_ready_lambda)
source_bucket.grant_read(string_replace_lambda)
processing_bucket.grant_write(string_replace_lambda)
processing_bucket.grant_read_write(calculate_total_earnings_lambda)
processing_bucket.grant_read(convert_csv_to_json_lambda)
dest_bucket.grant_write(convert_csv_to_json_lambda)
# Outputs
core.CfnOutput(self, "SourceBucketName", value=source_bucket.bucket_name)
core.CfnOutput(self, "DestinationBucketName", value=dest_bucket.bucket_name)
core.CfnOutput(self, "ProcessingBucketName", value=processing_bucket.bucket_name)
core.CfnOutput(self, "GenerateWorkflowInputLambda", value=generate_workflow_input_lambda.function_name)
core.CfnOutput(self, "CheckWorkflowReadyLambda", value=check_workflow_ready_lambda.function_name)
core.CfnOutput(self, "StringReplaceLambda", value=string_replace_lambda.function_name)
core.CfnOutput(self, "CalculateTotalEarningsLambda", value=calculate_total_earnings_lambda.function_name)
core.CfnOutput(self, "ConvertCsvToJsonLambda", value=convert_csv_to_json_lambda.function_name)
# State Machine
generate_workflow_input_task = sf_tasks.LambdaInvoke(
self, "GenerateWorkflowInput",
lambda_function=generate_workflow_input_lambda,
payload_response_only=True
)
check_workflow_ready_task = sf_tasks.LambdaInvoke(
self, "CheckWorkflowReady",
lambda_function=check_workflow_ready_lambda,
input_path="$.CheckWorkflowReady.Input",
result_path="$.CheckWorkflowReady.Output",
payload_response_only=True
)
string_replace_task = sf_tasks.LambdaInvoke(
self, "ReplaceString",
lambda_function=string_replace_lambda,
result_path="$.StringReplace.Output",
payload_response_only=True
)
calculate_total_earnings_task = sf_tasks.LambdaInvoke(
self, "CalculateTotalEarnings",
lambda_function=calculate_total_earnings_lambda,
input_path="$.CalculateTotalEarnings.Input",
result_path="$.CalculateTotalEarnings.Output",
payload_response_only=True
)
convert_csv_to_json_task = sf_tasks.LambdaInvoke(
self, "ConvertCsvToJson",
lambda_function=convert_csv_to_json_lambda,
input_path="$.ConvertCsvToJson.Input",
result_path="$.ConvertCsvToJson.Output",
payload_response_only=True
)
end_task = sf.Succeed(self, "WorkflowEnd")
replace_string_parallel = sf.Map(
self, "ReplaceStringParallel",
items_path="$.StringReplace.Input",
result_path="$.StringReplace.Output"
).iterator(string_replace_task)
workflow_steps = sf.Chain.\
start(replace_string_parallel)\
.next(calculate_total_earnings_task)\
.next(convert_csv_to_json_task)\
.next(end_task)
run_workflow = sf.Choice(self, "RunWorkflowDecision")\
.when(sf.Condition.boolean_equals("$.CheckWorkflowReady.Output", True), workflow_steps)\
.otherwise(end_task)
hello_workflow_state_machine = sf.StateMachine(
self, "HelloWorkflowStateMachine",
definition=sf.Chain\
.start(generate_workflow_input_task)\
.next(check_workflow_ready_task)\
.next(run_workflow)
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
submit_lambda = aws_lambda_python.PythonFunction(
self,
"submit-status",
entry="./lambdas/example",
handler="submit_status",
)
get_status_lambda = aws_lambda_python.PythonFunction(
self,
"get-status",
entry="./lambdas/example",
handler="get_status")
final_status_lambda = aws_lambda_python.PythonFunction(
self,
"final-status",
entry="./lambdas/example",
handler="final_status")
submit_job = tasks.LambdaInvoke(
self,
"Submit Job",
lambda_function=submit_lambda,
payload_response_only=True,
)
wait_x = sfn.Wait(self,
"Wait X Seconds",
time=sfn.WaitTime.seconds_path("$.seconds"))
get_status = tasks.LambdaInvoke(
self,
"Get Job Status",
lambda_function=get_status_lambda,
payload_response_only=True,
)
job_failed = sfn.Fail(
self,
"Job Failed",
cause="AWS Batch Job Failed",
error="DescribeJob returned FAILED",
)
final_status = tasks.LambdaInvoke(
self,
"Get Final Job Status",
lambda_function=final_status_lambda,
payload_response_only=True,
)
definition = (submit_job.next(wait_x).next(get_status).next(
sfn.Choice(self, "Job Complete?").when(
sfn.Condition.string_equals("$.status", "FAILED"),
job_failed).when(
sfn.Condition.string_equals("$.status", "SUCCEEDED"),
final_status).otherwise(wait_x)))
sfn.StateMachine(
self,
"StateMachine",
definition=definition,
timeout=core.Duration.minutes(5),
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Step Function Starts Here
# The first thing we need to do is see if they are asking for pineapple on a pizza
pineapple_check_lambda = _lambda.Function(
self,
"pineappleCheckLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="orderPizza.handler",
code=_lambda.Code.from_asset("lambda_fns"),
)
# Step functions are built up of steps, we need to define our first step
order_pizza = step_fn_tasks.LambdaInvoke(
self,
'Order Pizza Job',
lambda_function=pineapple_check_lambda,
input_path='$.flavour',
result_path='$.pineappleAnalysis',
payload_response_only=True)
# Pizza Order failure step defined
pineapple_detected = step_fn.Fail(self,
'Sorry, We Dont add Pineapple',
cause='They asked for Pineapple',
error='Failed To Make Pizza')
# If they didnt ask for pineapple let's cook the pizza
cook_pizza = step_fn.Succeed(self,
'Lets make your pizza',
output_path='$.pineappleAnalysis')
# If they ask for a pizza with pineapple, fail. Otherwise cook the pizza
definition = step_fn.Chain \
.start(order_pizza) \
.next(step_fn.Choice(self, 'With Pineapple?')
.when(step_fn.Condition.boolean_equals('$.pineappleAnalysis.containsPineapple', True),
pineapple_detected)
.otherwise(cook_pizza))
state_machine = step_fn.StateMachine(
self,
'StateMachine',
definition=definition,
timeout=core.Duration.minutes(5),
tracing_enabled=True,
state_machine_type=step_fn.StateMachineType.EXPRESS)
# HTTP API Definition
# Give our gateway permissions to interact with SNS
http_api_role = iam.Role(
self,
'HttpApiRole',
assumed_by=iam.ServicePrincipal('apigateway.amazonaws.com'),
inline_policies={
"AllowSFNExec":
iam.PolicyDocument(statements=[
iam.PolicyStatement(
actions=["states:StartSyncExecution"],
effect=iam.Effect.ALLOW,
resources=[state_machine.state_machine_arn])
])
})
api = api_gw.HttpApi(self,
'the_state_machine_api',
create_default_stage=True)
# create an AWS_PROXY integration between the HTTP API and our Step Function
integ = api_gw.CfnIntegration(
self,
'Integ',
api_id=api.http_api_id,
integration_type='AWS_PROXY',
connection_type='INTERNET',
integration_subtype='StepFunctions-StartSyncExecution',
credentials_arn=http_api_role.role_arn,
request_parameters={
"Input": "$request.body",
"StateMachineArn": state_machine.state_machine_arn
},
payload_format_version="1.0",
timeout_in_millis=10000)
api_gw.CfnRoute(self,
'DefaultRoute',
api_id=api.http_api_id,
route_key=api_gw.HttpRouteKey.DEFAULT.key,
target="integrations/" + integ.ref)
core.CfnOutput(self, 'HTTP API URL', value=api.url)
def __init__(self, scope: core.Construct, id: str, QueueDefine="default",TaskDefine="default",LambdaDefine="default", SNSDefine="default",**kwargs):
super().__init__(scope, id, **kwargs)
self.Job_String_Split = _sfn.Task(
self,"String_Split",
input_path = "$.TaskInfo",
result_path = "$.JobDetail.String_Split",
output_path = "$",
task = _sfn_tasks.RunBatchJob(
job_name = "String_Split",
job_definition = TaskDefine.getTaskDefine("String_Split"),
job_queue = QueueDefine.getComputeQueue("ComputeQueue"),
container_overrides = _sfn_tasks.ContainerOverrides(
environment = {
"INPUT_BUCKET":_sfn.Data.string_at("$.BasicParameters.INPUT_BUCKET"),
"INPUT_KEY":_sfn.Data.string_at("$.BasicParameters.INPUT_KEY"),
"OUTPUT_BUCKET":_sfn.Data.string_at("$.BasicParameters.OUTPUT_BUCKET"),
"OUTPUT_KEY":_sfn.Data.string_at("$.JobParameter.String_Split.OUTPUT_KEY"),
"SPLIT_NUM":_sfn.Data.string_at("$.JobParameter.String_Split.SPLIT_NUM")
}
)
)
)
self.Job_Map = _sfn.Task(
self,"Job_Map",
input_path = "$.TaskInfo",
result_path = "$.TaskInfo.JobDetail.Job_Map",
output_path = "$",
task = _sfn_tasks.RunLambdaTask(LambdaDefine.getLambdaFunction("Get_Job_List")),
)
self.Job_String_Reverse = _sfn.Task(
self,"String_Reverse",
input_path = "$",
result_path = "$",
output_path = "$",
task = _sfn_tasks.RunBatchJob(
job_name = "String_Reverse",
job_definition = TaskDefine.getTaskDefine("String_Reverse"),
job_queue = QueueDefine.getComputeQueue("ComputeQueue"),
container_overrides = _sfn_tasks.ContainerOverrides(
environment = {
"INDEX":_sfn.Data.string_at("$.INDEX"),
"INPUT_BUCKET":_sfn.Data.string_at("$.INPUT_BUCKET"),
"INPUT_KEY":_sfn.Data.string_at("$.INPUT_KEY"),
"OUTPUT_BUCKET":_sfn.Data.string_at("$.OUTPUT_BUCKET"),
"OUTPUT_KEY":_sfn.Data.string_at("$.String_Reverse.OUTPUT_KEY")
}
)
)
)
self.Job_String_Repeat = _sfn.Task(
self,"String_Repeat",
input_path = "$",
result_path = "$",
output_path = "$",
task = _sfn_tasks.RunBatchJob(
job_name = "String_Repeat",
job_definition = TaskDefine.getTaskDefine("String_Repeat"),
job_queue = QueueDefine.getComputeQueue("ComputeQueue"),
container_overrides = _sfn_tasks.ContainerOverrides(
environment = {
"INDEX":_sfn.Data.string_at("$.INDEX"),
"INPUT_BUCKET":_sfn.Data.string_at("$.INPUT_BUCKET"),
"INPUT_KEY":_sfn.Data.string_at("$.INPUT_KEY"),
"OUTPUT_BUCKET":_sfn.Data.string_at("$.OUTPUT_BUCKET"),
"OUTPUT_KEY":_sfn.Data.string_at("$.String_Repeat.OUTPUT_KEY")
}
)
)
)
self.Job_String_Process_Repeat = _sfn.Map(
self, "String_Process_Repeat",
max_concurrency=50,
input_path = "$.TaskInfo.JobDetail.Job_Map",
result_path = "DISCARD",
items_path = "$.Payload",
output_path = "$",
).iterator(self.Job_String_Repeat)
self.Job_String_Repeat_Merge = _sfn.Task(
self,"String_Repeat_Merge",
input_path = "$.TaskInfo",
result_path = "DISCARD",
output_path = "$",
task = _sfn_tasks.RunBatchJob(
job_name = "String_Repeat_Merge",
job_definition = TaskDefine.getTaskDefine("String_Merge"),
job_queue = QueueDefine.getComputeQueue("ComputeQueue"),
container_overrides = _sfn_tasks.ContainerOverrides(
environment = {
"PERFIX":_sfn.Data.string_at("$.JobParameter.String_Repeat.Prefix"),
"FILE_NAME":_sfn.Data.string_at("$.BasicParameters.INPUT_KEY"),
"INPUT_BUCKET":_sfn.Data.string_at("$.BasicParameters.INPUT_BUCKET"),
"INPUT_KEY":_sfn.Data.string_at("$.JobParameter.String_Repeat.OUTPUT_KEY"),
"OUTPUT_BUCKET":_sfn.Data.string_at("$.BasicParameters.OUTPUT_BUCKET"),
"OUTPUT_KEY":_sfn.Data.string_at("$.JobParameter.String_Repeat.OUTPUT_KEY")
}
)
)
)
self.Job_String_Process_Repeat.next(self.Job_String_Repeat_Merge)
self.Job_String_Process_Reverse = _sfn.Map(
self, "String_Process_Reverse",
max_concurrency=50,
input_path = "$.TaskInfo.JobDetail.Job_Map",
result_path = "DISCARD",
items_path = "$.Payload",
output_path = "$",
).iterator(self.Job_String_Reverse)
self.Job_String_Reverse_Merge = _sfn.Task(
self,"String_Reverse_Merge",
input_path = "$.TaskInfo",
result_path = "DISCARD",
output_path = "$",
task = _sfn_tasks.RunBatchJob(
job_name = "String_Reverse_Merge",
job_definition = TaskDefine.getTaskDefine("String_Merge"),
job_queue = QueueDefine.getComputeQueue("ComputeQueue"),
container_overrides = _sfn_tasks.ContainerOverrides(
environment = {
"PERFIX":_sfn.Data.string_at("$.JobParameter.String_Reverse.Prefix"),
"FILE_NAME":_sfn.Data.string_at("$.BasicParameters.INPUT_KEY"),
"INPUT_BUCKET":_sfn.Data.string_at("$.BasicParameters.INPUT_BUCKET"),
"INPUT_KEY":_sfn.Data.string_at("$.JobParameter.String_Reverse.OUTPUT_KEY"),
"OUTPUT_BUCKET":_sfn.Data.string_at("$.BasicParameters.OUTPUT_BUCKET"),
"OUTPUT_KEY":_sfn.Data.string_at("$.JobParameter.String_Reverse.OUTPUT_KEY")
}
)
)
)
self.Job_String_Process_Reverse.next(self.Job_String_Reverse_Merge)
self.Job_Parallel_Process = _sfn.Parallel(
self,
'Parallel_Process',
input_path = "$",
result_path = "DISCARD"
)
self.Job_Parallel_Process.branch(self.Job_String_Process_Repeat)
self.Job_Parallel_Process.branch(self.Job_String_Process_Reverse)
self.Job_Check_Output = _sfn.Task(
self,"Check_Output",
input_path = "$.TaskInfo",
result_path = "$.JobDetail.Check_Output",
output_path = "$.JobDetail.Check_Output.Payload",
task = _sfn_tasks.RunLambdaTask(LambdaDefine.getLambdaFunction("Get_Output_size")),
)
self.Job_Is_Complete = _sfn.Choice(
self, "Is_Complete",
input_path = "$.TaskInfo",
output_path = "$"
)
self.Job_Finish = _sfn.Wait(
self, "Finish",
time = _sfn.WaitTime.duration(core.Duration.seconds(5))
)
self.Job_Notification = _sfn.Task(self, "Notification",
input_path = "$.TaskInfo",
result_path = "DISCARD",
output_path = "$",
task = _sfn_tasks.PublishToTopic(SNSDefine.getSNSTopic("Topic_Batch_Job_Notification"),
integration_pattern = _sfn.ServiceIntegrationPattern.FIRE_AND_FORGET,
message = _sfn.TaskInput.from_data_at("$.JobStatus.Job_Comment"),
subject = _sfn.Data.string_at("$.JobStatus.SNS_Subject")
)
)
self.Job_Failed = _sfn.Wait(
self, "Failed",
time = _sfn.WaitTime.duration(core.Duration.seconds(5))
)
self.statemachine = _sfn.StateMachine(
self, "StateMachine",
definition = self.Job_String_Split.next(self.Job_Map) \
.next(self.Job_Parallel_Process) \
.next(self.Job_Check_Output) \
.next(self.Job_Notification) \
.next(self.Job_Is_Complete \
.when(_sfn.Condition.string_equals(
"$.JobStatus.OutputStatus", "FAILED"
), self.Job_Failed
.next(self.Job_Map)
)
.when(_sfn.Condition.string_equals(
"$.JobStatus.OutputStatus", "SUCCEEDED"
), self.Job_Finish)
.otherwise(self.Job_Failed)
),
timeout = core.Duration.hours(1),
)
def __init__(self,
scope: core.Construct,
id: str,
id_checker: str,
event_bus: str,
stage: Optional[str] = 'prod',
**kwargs) -> None:
super().__init__(scope, id + '-' + stage, **kwargs)
app_table_name = id + '-applications-table-' + stage
app_table = ddb.Table(self,
id=app_table_name,
table_name=app_table_name,
partition_key=ddb.Attribute(
name='id', type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST)
events_table_name = id + '-events-table-' + stage
events_table = ddb.Table(self,
id=events_table_name,
table_name=events_table_name,
partition_key=ddb.Attribute(
name='id', type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST,
stream=ddb.StreamViewType.NEW_IMAGE)
self._table_stream_arn = events_table.table_stream_arn
# create our Lambda function for the bank account service
func_name = id + '-' + stage + '-' + 'account-application'
lambda_assets = lambda_.Code.from_asset('account_application_service')
handler = lambda_.Function(self,
func_name,
code=lambda_assets,
runtime=lambda_.Runtime.NODEJS_10_X,
handler='main.handler',
environment={
'ACCOUNTS_TABLE_NAME':
app_table.table_name,
'EVENTS_TABLE_NAME':
events_table.table_name,
'REGION': core.Aws.REGION
})
gw.LambdaRestApi(self, id=stage + '-' + id, handler=handler)
# grant main Lambda function access to DynamoDB tables
app_table.grant_read_write_data(handler.role)
events_table.grant_read_write_data(handler.role)
p_statement = iam.PolicyStatement(actions=[
'ssm:Describe*', 'ssm:Get*', 'ssm:List*', 'events:*', 'states:*'
],
effect=iam.Effect.ALLOW,
resources=['*'])
handler.add_to_role_policy(statement=p_statement)
# create the Lambda function for the event publisher
evt_publisher = id + '-' + stage + '-' + 'event-publisher'
evt_handler = lambda_.Function(
self,
evt_publisher,
code=lambda_assets,
runtime=lambda_.Runtime.NODEJS_10_X,
handler='event-publisher.handler',
events=[
lambda_es.DynamoEventSource(
table=events_table,
starting_position=lambda_.StartingPosition.LATEST)
],
environment={
'EVENT_BRIDGE_ARN': event_bus,
'REGION': core.Aws.REGION
})
evt_handler.add_to_role_policy(statement=p_statement)
# set up StepFunctions
approve_application = sf.Task(
self,
'Approve Application',
task=sft.InvokeFunction(handler,
payload={
'body': {
'command':
'APPROVE_ACCOUNT_APPLICATION',
'data': {
'id.$': '$.application.id'
}
}
}),
result_path='$.approveApplication')
reject_application = sf.Task(self,
'Reject Application',
task=sft.InvokeFunction(
handler,
payload={
'body': {
'command':
'REJECT_ACCOUNT_APPLICATION',
'data': {
'id.$': '$.application.id'
}
}
}),
result_path='$.rejectApplication')
id_checker_handler = lambda_.Function.from_function_arn(
self, 'IdentityChecker', function_arn=id_checker)
check_identity = sf.Task(self,
'Check Identity',
task=sft.InvokeFunction(
id_checker_handler,
payload={
'body': {
'command': 'CHECK_IDENTITY',
'data': {
'application.$':
'$.application'
}
}
}))
wait_for_human_review = sf.Task(self, 'Wait for Human Review',
task=sft.RunLambdaTask(handler,
integration_pattern=sf.ServiceIntegrationPattern.WAIT_FOR_TASK_TOKEN,
payload={
'body': {
'command': 'FLAG_ACCOUNT_APPLICATION_FOR_HUMAN_REVIEW',
'data': {
'id.$': '$.application.id',
'taskToken': sf.Context.task_token
}
}
}), result_path='$.humanReview') \
.next(
sf.Choice(self, 'Human Approval Choice')
.when(sf.Condition.string_equals('$.humanReview.decision', 'APPROVE'), next=approve_application)
.when(sf.Condition.string_equals('$.humanReview.decision', 'REJECT'), next=reject_application))
sm_definition = sf.Parallel(self, 'Perform Automated Checks', result_path='$.checks') \
.branch(check_identity) \
.branch(sf.Pass(self, 'Check Fraud Model', result=sf.Result({'flagged': False}))) \
.next(
sf.Choice(self, 'Automated Checks Choice')
.when(sf.Condition.boolean_equals('$.checks[0].flagged', True), next=wait_for_human_review)
.when(sf.Condition.boolean_equals('$.checks[1].flagged', True), next=wait_for_human_review)
.otherwise(approve_application))
state_machine = sf.StateMachine(self,
'OpenAccountStateMachine' + stage,
definition=sm_definition)
ssm.CfnParameter(self,
id='StateMachineArnSSM',
type='String',
value=state_machine.state_machine_arn,
name='StateMachineArnSSM')
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
ProcessPurchase = lambda_.Function(
self,
'process-purchase',
runtime=lambda_.Runtime.PYTHON_3_8,
handler='handler.process_purchase',
code=lambda_.Code.from_asset('./code'),
timeout=(Duration.seconds(5)))
ProcessRefund = lambda_.Function(
self,
'process-refund',
runtime=lambda_.Runtime.PYTHON_3_8,
handler='handler.process_refund',
code=lambda_.Code.from_asset('./code'),
timeout=(Duration.seconds(5)))
GeneratePurchaseReceipt = lambda_.Function(
self,
'generate-purchase-receipt',
runtime=lambda_.Runtime.PYTHON_3_8,
handler='handler.generate_purchase_receipt',
code=lambda_.Code.from_asset('./code'),
timeout=(Duration.seconds(5)))
GenerateRefundReceipt = lambda_.Function(
self,
'generate-refund-receipt',
runtime=lambda_.Runtime.PYTHON_3_8,
handler='handler.generate_refund_receipt',
code=lambda_.Code.from_asset('./code'),
timeout=(Duration.seconds(5)))
ProcessPurchaseState = tasks.LambdaInvoke(
self,
'process purchase task',
lambda_function=ProcessPurchase,
input_path='$')
ProcessRefundState = tasks.LambdaInvoke(self,
'process refund task',
lambda_function=ProcessRefund,
input_path='$')
GeneratePurchaseReceiptState = tasks.LambdaInvoke(
self,
'generate purchase receipt',
lambda_function=GeneratePurchaseReceipt,
input_path='$')
GenerateRefundReceiptState = tasks.LambdaInvoke(
self,
'generate refund receipt',
lambda_function=GenerateRefundReceipt,
input_path='$')
TransactionChoice = sfn.Choice(self,
'process transaction',
output_path='$')
TransactionTypeEqualsPurchase = sfn.Condition.string_equals(
'$.TransactionType', 'PURCHASE')
TransactionTypeEqualsRefund = sfn.Condition.string_equals(
'$.TransactionType', 'REFUND')
# StateMachineDefinition = TransactionChoice.when(TransactionTypeEqualsPurchase, ProcessPurchaseState).when(TransactionTypeEqualsRefund, ProcessRefundState) # works
StateMachineDefinition = TransactionChoice.when(
TransactionTypeEqualsPurchase,
ProcessPurchaseState.next(GeneratePurchaseReceiptState)).when(
TransactionTypeEqualsRefund,
ProcessRefundState.next(GenerateRefundReceiptState))
Workflow = sfn.StateMachine(self,
'transaction workflow',
definition=StateMachineDefinition)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
api = apigateway.RestApi(
scope=self,
id=f'{constants.PREFIX}-approval-api',
rest_api_name='Human approval endpoint',
description='HTTP Endpoint backed by API Gateway and Lambda',
endpoint_types=[apigateway.EndpointType.REGIONAL],
)
v1 = api.root.add_resource("v1")
approve_api = v1.add_resource("approve")
#################################################
email_topic = sns.Topic(
scope=self,
id=f'{constants.PREFIX}-email-topic',
)
email_topic.add_subscription(
subscription=subscriptions.EmailSubscription(
email_address=constants.EMAIL_APPROVER, ))
#################################################
submit_job_lambda = _lambda.Function(
scope=self,
id=f'{constants.PREFIX}-submit-lambda',
runtime=_lambda.Runtime.PYTHON_3_8,
handler='submit.handler',
environment={
"TOPIC_ARN": email_topic.topic_arn,
"END_POINT": approve_api.url,
"TO_ADDRESS": constants.EMAIL_RECIPIENT,
"FROM_ADDRESS": constants.EMAIL_SENDER,
},
code=_lambda.Code.from_asset(
os.path.join('lambdas', 'submit-lambda')),
)
email_topic.grant_publish(submit_job_lambda)
submit_job_lambda.add_to_role_policy(statement=iam.PolicyStatement(
actions=['ses:Send*'],
resources=['*'],
))
submit_job = tasks.LambdaInvoke(
scope=self,
id=f'{constants.PREFIX}-submit-job',
lambda_function=submit_job_lambda,
integration_pattern=sfn.IntegrationPattern.WAIT_FOR_TASK_TOKEN,
heartbeat=core.Duration.minutes(5),
payload=sfn.TaskInput.from_object({
"token": sfn.Context.task_token,
"data": sfn.Data.string_at('$'),
}),
)
success = sfn.Succeed(scope=self,
id=f'{constants.PREFIX}-success',
comment='We did it!')
fail = sfn.Fail(scope=self,
id=f'{constants.PREFIX}-fail',
error='WorkflowFailure',
cause='Something went wrong')
choice = sfn.Choice(scope=self,
id=f'{constants.PREFIX}-choice',
comment='Was it approved?')
choice.when(condition=sfn.Condition.string_equals("$.status", "OK"),
next=success)
choice.otherwise(fail)
definition = submit_job.next(choice)
self._state_machine = sfn.StateMachine(
scope=self,
id=f'{constants.PREFIX}-state-machine',
definition=definition,
# only 10 mins to approve better be quick
timeout=core.Duration.minutes(10))
#################################################
approval_lambda = _lambda.Function(
scope=self,
id=f'{constants.PREFIX}-approval-lambda',
runtime=_lambda.Runtime.PYTHON_3_8,
handler='approve.handler',
code=_lambda.Code.from_asset(
os.path.join('lambdas', 'approve-lambda')),
)
approval_lambda.add_to_role_policy(statement=iam.PolicyStatement(
actions=['states:Send*'], resources=['*']))
approve_integration = apigateway.LambdaIntegration(approval_lambda)
approve_api_get_method = approve_api.add_method(
http_method="GET",
api_key_required=False,
integration=approve_integration,
)
def create_unfurl_statemachine(self):
map_job = sfn.Map(self,
"Unfurl Map",
items_path="$.links",
max_concurrency=10)
get_note_job = tasks.LambdaInvoke(
self,
"Get Note Job for Unfurl",
lambda_function=self.step_lambda,
payload=sfn.TaskInput.from_object({
"action": "get_note_from_url",
"url.$": "$.url"
}),
)
get_tf_job = tasks.LambdaInvoke(
self,
"Get Text Frequency Job for Unfurl",
lambda_function=self.step_lambda,
payload=sfn.TaskInput.from_object({
"action":
"update_tf",
"id.$":
"$.Payload.id",
"url.$":
"$.Payload.url",
"contentUpdatedAt.$":
"$.Payload.contentUpdatedAt",
"isArchived.$":
"$.Payload.isArchived",
}),
)
get_idf_job = tasks.LambdaInvoke(
self,
"Get Inter Document Frequency Job for Unfurl",
lambda_function=self.step_lambda,
payload=sfn.TaskInput.from_object({
"action":
"update_idf",
"id.$":
"$.Payload.id",
"url.$":
"$.Payload.url",
"contentUpdatedAt.$":
"$.Payload.contentUpdatedAt",
"isArchived.$":
"$.Payload.isArchived",
}),
)
get_tfidf_job = tasks.LambdaInvoke(
self,
"Get TF*IDF WordCloud Image Job for Unfurl",
lambda_function=self.step_lambda,
payload=sfn.TaskInput.from_object({
"action":
"update_tfidf_png",
"id.$":
"$.Payload.id",
"url.$":
"$.Payload.url",
"contentUpdatedAt.$":
"$.Payload.contentUpdatedAt",
"isArchived.$":
"$.Payload.isArchived",
}),
)
unfurl_job = tasks.LambdaInvoke(
self,
"Get Attachment Job",
lambda_function=self.step_lambda,
payload=sfn.TaskInput.from_object({
"action": "unfurl",
"id.$": "$.Payload.id",
"url.$": "$.Payload.url",
}),
)
get_tf_job.next(get_idf_job.next(get_tfidf_job.next(unfurl_job)))
choice_job = sfn.Choice(self, "Check for Update")
choice_job.when(
sfn.Condition.and_(
sfn.Condition.is_timestamp("$.Payload.tfidfPngUpdatedAt"),
sfn.Condition.timestamp_less_than_json_path(
"$.Payload.contentUpdatedAt",
"$.Payload.tfidfPngUpdatedAt"),
),
unfurl_job,
).when(
sfn.Condition.and_(
sfn.Condition.is_timestamp("$.Payload.tfTsvUpdatedAt"),
sfn.Condition.timestamp_less_than_json_path(
"$.Payload.contentUpdatedAt", "$.Payload.tfTsvUpdatedAt"),
),
get_tfidf_job,
).otherwise(get_tf_job)
unfurl_definition = map_job.iterator(get_note_job.next(choice_job))
self.unfurl_statemachine = sfn.StateMachine(
self,
"UnfurlStateMachine",
definition=unfurl_definition,
timeout=core.Duration.minutes(20),
state_machine_type=sfn.StateMachineType.EXPRESS,
logs=sfn.LogOptions(
destination=logs.LogGroup(self, "UnfurlStateMachineLogGroup"),
level=sfn.LogLevel.ERROR,
),
)
def __init__(self, scope: cdk.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
submit_lambda = PythonFunction(self,
'Submit',
handler='handler',
index='submit.py',
entry=os.path.join(
os.getcwd(), 'lambdas'),
runtime=lambda_.Runtime.PYTHON_3_8)
get_status_lambda = PythonFunction(self,
'Status',
handler='handler',
index='status.py',
entry=os.path.join(
os.getcwd(), 'lambdas'),
runtime=lambda_.Runtime.PYTHON_3_8)
submit_job = tasks.LambdaInvoke(
self,
"Submit Job",
lambda_function=submit_lambda,
# Lambda's result is in the attribute `Payload`
output_path="$.Payload")
wait_x = sfn.Wait(self,
"Wait X Seconds",
time=sfn.WaitTime.seconds_path("$.waitSeconds"))
get_status = tasks.LambdaInvoke(
self,
"Get Job Status",
lambda_function=get_status_lambda,
# Pass just the field named "guid" into the Lambda, put the
# Lambda's result in a field called "status" in the response
output_path="$.Payload")
job_failed = sfn.Fail(self,
"Job Failed",
cause="AWS Batch Job Failed",
error="DescribeJob returned FAILED")
final_status = tasks.LambdaInvoke(
self,
"Get Final Job Status",
lambda_function=get_status_lambda,
# Use "guid" field as input
output_path="$.Payload")
definition = submit_job.next(wait_x).next(get_status).next(
sfn.Choice(self, "Job Complete?").when(
sfn.Condition.string_equals("$.status", "FAILED"),
job_failed).when(
sfn.Condition.string_equals("$.status", "SUCCEEDED"),
final_status).otherwise(wait_x))
sfn.StateMachine(self,
"StateMachine",
definition=definition,
timeout=cdk.Duration.minutes(5))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
email_subscription_parameter = core.CfnParameter(
self,
"EmailSubscriptionParameter",
description="Email Address for Notification Subscription",
allowed_pattern=
'^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$',
min_length=1,
constraint_description="Must be a valid email.")
email_subscription = email_subscription_parameter.value_as_string
#runtime=aws_lambda.Runtime.PYTHON_3_8
boto3_lambda_layer = None
boto3_lambda_layer = self.create_dependencies_layer(
id="boto3layer",
requirements_path="./layers/boto3/requirements.txt",
output_dir="./layers/boto3")
is_inline = False
context_enrichment = self.create_lambda_function(
boto3_lambda_layer, "./functions/context-enrichment",
"context_enrichment", is_inline)
"""
context_enrichment=aws_lambda.Function(
self,
"context_enrichment",
runtime=runtime,
handler="app.handler",
code=aws_lambda.AssetCode("./functions/context-enrichment"),
layers=[boto3_lambda_layer]
)
"""
handler_statement = iam.PolicyStatement(actions=[
"iam:ListRoleTags", "s3:GetBucketTagging", "lambda:ListTags",
"sqs:ListQueueTags", "kms:ListAliases", "kms:ListResourceTags"
],
effect=iam.Effect.ALLOW,
resources=["*"])
context_enrichment.add_to_role_policy(handler_statement)
cmk_key = kms.Key(
self,
"SNSEncryptionAtRestKey",
description="SNS Encryption at rest key",
alias="sns-encryption-at-rest",
enable_key_rotation=True,
)
email_topic = sns.Topic(
self,
"AccessAnalyzerNotificationTopic",
display_name="Access Analyzer Finding Notification Topic",
master_key=cmk_key)
email_topic.add_subscription(
subscriptions.EmailSubscription(email_subscription))
notification = self.create_lambda_function(
boto3_lambda_layer, "./functions/notification", "notification",
is_inline, {"SNS_TOPIC_ARN": email_topic.topic_arn})
"""
notification=aws_lambda.Function(
self,
"notification",
runtime=runtime,
handler="app.handler",
code=aws_lambda.AssetCode("./functions/notification"),
layers=[boto3_lambda_layer],
environment={"SNS_TOPIC_ARN":email_topic.topic_arn}
)
"""
notification_statement = iam.PolicyStatement(actions=[
"sns:Publish",
],
effect=iam.Effect.ALLOW,
resources=["*"])
notification.add_to_role_policy(notification_statement)
cmk_key.grant_encrypt_decrypt(notification)
archive_access_analyzer_finding = self.create_lambda_function(
boto3_lambda_layer, "./functions/archive-access-analyzer-finding",
"archive-access-analyzer-finding", is_inline)
"""
archive_access_analyzer_finding=aws_lambda.Function(
self,
"archive-access-analyzer-finding",
runtime=runtime,
handler="app.handler",
code=aws_lambda.AssetCode("./functions/archive-access-analyzer-finding"),
layers=[boto3_lambda_layer]
)
"""
archive_statement = iam.PolicyStatement(actions=[
"access-analyzer:UpdateFindings",
],
effect=iam.Effect.ALLOW,
resources=["*"])
archive_access_analyzer_finding.add_to_role_policy(archive_statement)
evaluate_access_analyzer_finding = self.create_lambda_function(
boto3_lambda_layer, "./functions/evaluate-access-analyzer-finding",
"evaluate-access-analyzer-finding", is_inline)
"""
evaluate_access_analyzer_finding=aws_lambda.Function(
self,
"evaluate-access-analyzer-finding",
runtime=runtime,
handler="app.handler",
code=aws_lambda.AssetCode("./functions/evaluate-access-analyzer-finding"),
layers=[boto3_lambda_layer]
)
"""
#https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html
access_analyzer_handler_task = sfn.Task(
self,
"Context Enrichment",
task=sfn_tasks.InvokeFunction(context_enrichment),
result_path="$.guid",
)
notification_task = sfn.Task(
self,
"Send Notification",
task=sfn_tasks.InvokeFunction(notification),
result_path="$.guid",
)
archive_task = sfn.Task(
self,
"Archive Finding",
task=sfn_tasks.InvokeFunction(archive_access_analyzer_finding),
result_path="$.guid",
)
evaluate_task = sfn.Task(
self,
"Evaluate Risk Level",
task=sfn_tasks.InvokeFunction(evaluate_access_analyzer_finding),
result_path="$.guid",
)
definition=access_analyzer_handler_task. \
next(evaluate_task). \
next(sfn.Choice(self, "Archive?"). \
when(sfn.Condition.string_equals("$.guid.status", "ARCHIVE"), archive_task). \
when(sfn.Condition.string_equals("$.guid.status", "NOTIFY"), notification_task) \
)
state_machine = sfn.StateMachine(
self,
"Access-Analyzer-Automatic-Finding-Archive-State-Machine",
definition=definition,
timeout=core.Duration.minutes(5),
)
#https://docs.aws.amazon.com/step-functions/latest/dg/tutorial-cloudwatch-events-s3.html
access_analyzer_finding_rule = aws_events.Rule(
self,
"AccessAnalzyerFindingActiveEventRule",
description="Access Analyzer Finding Event Active",
enabled=True,
event_pattern=aws_events.EventPattern(
source=["aws.access-analyzer"],
detail_type=["Access Analyzer Finding"],
detail={"status": ["ACTIVE"]}),
targets=[
aws_events_targets.SfnStateMachine(state_machine),
aws_events_targets.LambdaFunction(context_enrichment)
])
def __init__(self,
scope: core.Construct,
id: str,
redshift_cluster_name: str,
user_secret: Secret) -> None:
super().__init__(scope, id)
stack = Stack.of(self)
subprocess.call(
['pip', 'install', '-t', 'dwh/dwh_loader_layer/python/lib/python3.8/site-packages', '-r',
'dwh/dwh_loader/requirements.txt', '--platform', 'manylinux1_x86_64', '--only-binary=:all:',
'--upgrade'])
requirements_layer = _lambda.LayerVersion(scope=self,
id='PythonRequirementsTemplate',
code=_lambda.Code.from_asset('dwh/dwh_loader_layer'),
compatible_runtimes=[_lambda.Runtime.PYTHON_3_8])
dwh_loader_role = _iam.Role(
self, 'Role',
assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com')
)
dwh_loader_role.add_managed_policy(_iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'
))
dwh_loader_role.attach_inline_policy(
_iam.Policy(
self, 'InlinePolicy',
statements=[
_iam.PolicyStatement(
actions=[
"redshift-data:ExecuteStatement",
"redshift-data:CancelStatement",
"redshift-data:ListStatements",
"redshift-data:GetStatementResult",
"redshift-data:DescribeStatement",
"redshift-data:ListDatabases",
"redshift-data:ListSchemas",
"redshift-data:ListTables",
"redshift-data:DescribeTable"
],
resources=['*']
),
_iam.PolicyStatement(
actions=["secretsmanager:GetSecretValue"],
resources=[user_secret.secret_arn]
),
_iam.PolicyStatement(
actions=["redshift:GetClusterCredentials"],
resources=[
"arn:aws:redshift:*:*:dbname:*/*",
"arn:aws:redshift:*:*:dbuser:*/"+_config.Redshift.ETL_USER
]
),
_iam.PolicyStatement(
effect=_iam.Effect('DENY'),
actions=["redshift:CreateClusterUser"],
resources=["arn:aws:redshift:*:*:dbuser:*/"+_config.Redshift.ETL_USER]
),
_iam.PolicyStatement(
conditions={
'StringLike': {
"iam:AWSServiceName": "redshift-data.amazonaws.com"
}
},
actions=["iam:CreateServiceLinkedRole"],
resources=["arn:aws:iam::*:role/aws-service-role/redshift-data.amazonaws.com/AWSServiceRoleForRedshift"]
),
]
)
)
dwh_loader_function = _lambda.Function(
self, 'Lambda',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset('dwh/dwh_loader'),
handler='dwh_loader.handler',
function_name='dwh-loader',
environment={
'CLUSTER_NAME': redshift_cluster_name,
'PROCEDURE': _config.Redshift.ETL_PROCEDURE,
'SECRET_ARN': user_secret.secret_arn,
'DATABASE': _config.Redshift.DATABASE,
'REGION': core.Aws.REGION,
'SCHEMA': _config.Redshift.SCHEMA
},
layers=[requirements_layer],
timeout=core.Duration.seconds(30),
role=dwh_loader_role
)
dwh_loader_submit = _sfn_tasks.LambdaInvoke(
self, 'Submit',
lambda_function=dwh_loader_function,
payload_response_only=True
)
dwh_loader_wait = _sfn.Wait(
self, 'Wait',
time=_sfn.WaitTime.duration(core.Duration.seconds(30))
)
dwh_loader_complete = _sfn.Choice(
self, 'Complete'
)
dwh_loader_failed = _sfn.Fail(
self, 'Fail',
cause="Redshift Data API statement failed",
error="$.Result.Error"
)
dwh_loader_status = _sfn_tasks.LambdaInvoke(
self, 'Status',
lambda_function=dwh_loader_function,
result_path='$.Result',
payload_response_only=True
)
definition = dwh_loader_submit \
.next(dwh_loader_wait) \
.next(dwh_loader_status) \
.next(dwh_loader_complete
.when(_sfn.Condition.string_equals('$.Result.Status', 'FAILED'), dwh_loader_failed)
.when(_sfn.Condition.string_equals('$.Result.Status', 'FINISHED'), _sfn.Succeed(self, 'DwhLoaderSuccess'))
.otherwise(dwh_loader_wait))
dwh_loader_stepfunctions = _sfn.StateMachine(
self, 'StepFunctions',
definition=definition,
timeout=core.Duration.minutes(30)
)
step_trigger = _events.Rule(
self, 'StepTrigger',
schedule=_events.Schedule.cron(minute='0/30',
hour='*',
month='*',
week_day='*',
year='*')
)
step_trigger.add_target(
_events_targets.SfnStateMachine(
machine=dwh_loader_stepfunctions,
)
)
def __init__(self, app: core.App, id: str, **kwargs) -> None:
super().__init__(app, id, **kwargs)
submit_job_activity = sfn.Activity(self, "SubmitJob")
check_job_activity = sfn.Activity(self, "CheckJob")
do_mapping_activity1 = sfn.Activity(self, "MapJOb1")
do_mapping_activity2 = sfn.Activity(self, "MapJOb2")
submit_job = sfn.Task(
self,
"Submit Job",
task=sfn_tasks.InvokeActivity(submit_job_activity),
result_path="$.guid",
)
task1 = sfn.Task(
self,
"Task 1 in Mapping",
task=sfn_tasks.InvokeActivity(do_mapping_activity1),
result_path="$.guid",
)
task2 = sfn.Task(
self,
"Task 2 in Mapping",
task=sfn_tasks.InvokeActivity(do_mapping_activity2),
result_path="$.guid",
)
wait_x = sfn.Wait(
self,
"Wait X Seconds",
time=sfn.WaitTime.seconds_path('$.wait_time'),
)
get_status = sfn.Task(
self,
"Get Job Status",
task=sfn_tasks.InvokeActivity(check_job_activity),
input_path="$.guid",
result_path="$.status",
)
is_complete = sfn.Choice(self, "Job Complete?")
job_failed = sfn.Fail(self,
"Job Failed",
cause="AWS Batch Job Failed",
error="DescribeJob returned FAILED")
final_status = sfn.Task(
self,
"Get Final Job Status",
task=sfn_tasks.InvokeActivity(check_job_activity),
input_path="$.guid",
)
definition_map = task1.next(task2)
process_map = sfn.Map(self, "Process_map",
max_concurrency=10).iterator(definition_map)
definition = submit_job \
.next(process_map) \
.next(wait_x) \
.next(get_status) \
.next(is_complete
.when(sfn.Condition.string_equals(
"$.status", "FAILED"), job_failed)
.when(sfn.Condition.string_equals(
"$.status", "SUCCEEDED"), final_status)
.otherwise(wait_x))
sfn.StateMachine(
self,
"StateMachine",
definition=definition,
timeout=core.Duration.seconds(30),
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# ------ Necessary Roles ------
roles = IamRole(
self, 'IamRoles'
)
# ------ S3 Buckets ------
# Create Athena bucket
athena_bucket = _s3.Bucket(self, "AthenaBucket",
removal_policy=core.RemovalPolicy.DESTROY
)
# Create Forecast bucket
forecast_bucket = _s3.Bucket(self, "FoecastBucket",
removal_policy=core.RemovalPolicy.DESTROY
)
# ------ Athena ------
# Config Athena query result output location
workgroup_prop = _athena.CfnWorkGroup.WorkGroupConfigurationProperty(
result_configuration=_athena.CfnWorkGroup.ResultConfigurationProperty(
output_location="s3://"+athena_bucket.bucket_name
)
)
# Create Athena workgroup
athena_workgroup = _athena.CfnWorkGroup(
self, 'ForecastGroup',
name='ForecastGroup',
recursive_delete_option=True,
state='ENABLED',
work_group_configuration=workgroup_prop
)
# ------ SNS Topic ------
topic = sns.Topic(
self, 'NotificationTopic',
display_name='StepsTopic'
)
# SNS email subscription. Get the email address from context value(cdk.json)
topic.add_subscription(subs.EmailSubscription(self.node.try_get_context('my_email')))
# ------ Layers ------
shared_layer = _lambda.LayerVersion(
self, 'LambdaLayer',
layer_version_name='testfolderlayer',
code=_lambda.AssetCode('shared/')
)
# ------ Lambdas for stepfuctions------
create_dataset_lambda = _lambda.Function(
self, 'CreateDataset',
function_name='CreateDataset',
code=_lambda.Code.asset('lambdas/createdataset/'),
handler='dataset.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
timeout=core.Duration.seconds(30),
layers=[shared_layer]
)
create_dataset_group_lambda = _lambda.Function(
self, 'CreateDatasetGroup',
function_name='CreateDatasetGroup',
code = _lambda.Code.asset('lambdas/createdatasetgroup/'),
handler = 'datasetgroup.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
layers=[shared_layer]
)
import_data_lambda = _lambda.Function(
self, 'CreateDatasetImportJob',
function_name='CreateDatasetImportJob',
code = _lambda.Code.asset('lambdas/createdatasetimportjob/'),
handler = 'datasetimport.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
environment= {
'FORECAST_ROLE': roles.forecast_role.role_arn
},
layers=[shared_layer]
)
create_predictor_lambda = _lambda.Function(
self, 'CreatePredictor',
function_name='CreatePredictor',
code = _lambda.Code.asset('lambdas/createpredictor/'),
handler = 'predictor.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
layers=[shared_layer]
)
create_forecast_lambda = _lambda.Function(
self, 'CreateForecast',
function_name='CreateForecast',
code = _lambda.Code.asset('lambdas/createforecast/'),
handler = 'forecast.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
environment= {
'EXPORT_ROLE': roles.forecast_role.role_arn
},
layers=[shared_layer],
timeout=core.Duration.seconds(30)
)
# Deploy lambda with python dependencies from requirements.txt
update_resources_lambda = _lambda_python.PythonFunction(
self, 'UpdateResources',
function_name='UpdateResources',
entry='lambdas/updateresources/',
index='update.py',
handler='lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.update_role,
environment= {
'ATHENA_WORKGROUP': athena_workgroup.name,
'ATHENA_BUCKET' : athena_bucket.bucket_name
},
layers=[shared_layer],
timeout=core.Duration.seconds(900)
)
notify_lambda = _lambda.Function(
self, 'NotifyTopic',
function_name='NotifyTopic',
code = _lambda.Code.asset('lambdas/notify/'),
handler = 'notify.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
environment= {
'SNS_TOPIC_ARN': topic.topic_arn
},
layers=[shared_layer]
)
delete_forecast_lambda = _lambda.Function(
self, 'DeleteForecast',
function_name='DeleteForecast',
code = _lambda.Code.asset('lambdas/deleteforecast/'),
handler = 'deleteforecast.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
layers=[shared_layer]
)
delete_predctor_lambda = _lambda.Function(
self, 'DeletePredictor',
function_name='DeletePredictor',
code = _lambda.Code.asset('lambdas/deletepredictor/'),
handler = 'deletepredictor.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
layers=[shared_layer]
)
delete_importjob_lambda = _lambda.Function(
self, 'DeleteImportJob',
function_name='DeleteImportJob',
code = _lambda.Code.asset('lambdas/deletedatasetimport/'),
handler = 'deletedataset.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
layers=[shared_layer]
)
# ------ StepFunctions ------
strategy_choice = sfn.Choice(
self, 'Strategy-Choice'
)
success_state = sfn.Succeed(
self, 'SuccessState'
)
failed = sfn_tasks.LambdaInvoke(
self, 'Failed',
lambda_function = notify_lambda,
result_path=None
).next(strategy_choice)
create_dataset_job = sfn_tasks.LambdaInvoke(
self, 'Create-Dataset',
lambda_function = create_dataset_lambda,
retry_on_service_exceptions=True,
payload_response_only=True
)
self.add_retry_n_catch(create_dataset_job, failed)
create_dataset_group_job = sfn_tasks.LambdaInvoke(
self, 'Create-DatasetGroup',
lambda_function = create_dataset_group_lambda,
payload_response_only=True
)
self.add_retry_n_catch(create_dataset_group_job, failed)
import_data_job = sfn_tasks.LambdaInvoke(
self, 'Import-Data',
lambda_function = import_data_lambda,
payload_response_only=True
)
self.add_retry_n_catch(import_data_job, failed)
create_predictor_job = sfn_tasks.LambdaInvoke(
self, 'Create-Predictor',
lambda_function = create_predictor_lambda,
payload_response_only=True
)
self.add_retry_n_catch(create_predictor_job, failed)
create_forecast_job = sfn_tasks.LambdaInvoke(
self, 'Create-Forecast',
lambda_function = create_forecast_lambda,
payload_response_only=True
)
self.add_retry_n_catch(create_forecast_job, failed)
update_resources_job = sfn_tasks.LambdaInvoke(
self, 'Update-Resources',
lambda_function = update_resources_lambda,
payload_response_only=True
)
self.add_retry_n_catch(update_resources_job, failed)
notify_success = sfn_tasks.LambdaInvoke(
self, 'Notify-Success',
lambda_function = notify_lambda,
payload_response_only=True
)
delete_forecast_job = sfn_tasks.LambdaInvoke(
self, 'Delete-Forecast',
lambda_function = delete_forecast_lambda,
payload_response_only=True
)
self.delete_retry(delete_forecast_job)
delete_predictor_job = sfn_tasks.LambdaInvoke(
self, 'Delete-Predictor',
lambda_function = delete_predctor_lambda,
payload_response_only=True
)
self.delete_retry(delete_predictor_job)
delete_import_job = sfn_tasks.LambdaInvoke(
self, 'Delete-ImportJob',
lambda_function = delete_importjob_lambda,
payload_response_only=True
)
self.delete_retry(delete_import_job)
definition = create_dataset_job\
.next(create_dataset_group_job)\
.next(import_data_job)\
.next(create_predictor_job)\
.next(create_forecast_job)\
.next(update_resources_job)\
.next(notify_success)\
.next(strategy_choice.when(sfn.Condition.boolean_equals('$.params.PerformDelete', False), success_state)\
.otherwise(delete_forecast_job).afterwards())\
.next(delete_predictor_job)\
.next(delete_import_job)
deployt_state_machine = sfn.StateMachine(
self, 'StateMachine',
definition = definition
# role=roles.states_execution_role
)
# S3 event trigger lambda
s3_lambda = _lambda.Function(
self, 'S3Lambda',
function_name='S3Lambda',
code=_lambda.Code.asset('lambdas/s3lambda/'),
handler='parse.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_7,
role=roles.trigger_role,
environment= {
'STEP_FUNCTIONS_ARN': deployt_state_machine.state_machine_arn,
'PARAMS_FILE': self.node.try_get_context('parameter_file')
}
)
s3_lambda.add_event_source(
event_src.S3EventSource(
bucket=forecast_bucket,
events=[_s3.EventType.OBJECT_CREATED],
filters=[_s3.NotificationKeyFilter(
prefix='train/',
suffix='.csv'
)]
)
)
# CloudFormation output
core.CfnOutput(
self, 'StepFunctionsName',
description='Step Functions Name',
value=deployt_state_machine.state_machine_name
)
core.CfnOutput(
self, 'ForecastBucketName',
description='Forecast bucket name to drop you files',
value=forecast_bucket.bucket_name
)
core.CfnOutput(
self, 'AthenaBucketName',
description='Athena bucket name to drop your files',
value=athena_bucket.bucket_name
)
def __init__(self, scope: core.App, id_: str, stack_env: str, **kwargs) -> None:
super().__init__(scope, id_, **kwargs)
# create dynamo table
demo_table = aws_dynamodb.Table(
scope=self,
id="demo_table",
partition_key=aws_dynamodb.Attribute(
name="id",
type=aws_dynamodb.AttributeType.STRING
),
write_capacity=3,
read_capacity=3,
removal_policy=core.RemovalPolicy.DESTROY
)
queue = aws_sqs.Queue(self, f"{id_}-SQSQueue")
# create producer lambda function
producer_lambda = self._create_lambda_function(
function_name="producer",
environment={
"TABLE_NAME": demo_table.table_name,
"QUEUE_URL": queue.queue_url
}
)
queue.grant_send_messages(producer_lambda)
# grant permission to lambda to write to demo table
demo_table.grant_write_data(producer_lambda)
# create consumer lambda function
consumer_lambda = self._create_lambda_function(
function_name="consumer",
environment={"TABLE_NAME": demo_table.table_name}
)
# grant permission to lambda to read from demo table
demo_table.grant_read_data(consumer_lambda)
# api_gateway for root
base_api = apigw_.RestApi(
scope=self,
id=f"{id_}-{stack_env}-apigw",
rest_api_name=f"{id_}-{stack_env}-apigw",
deploy_options=apigw_.StageOptions(stage_name=stack_env)
)
# /example entity
api_entity = base_api.root.add_resource("example")
# GET /example
api_entity.add_method(
http_method="GET",
integration=apigw_.LambdaIntegration(
handler=consumer_lambda,
integration_responses=[
apigw_.IntegrationResponse(
status_code="200"
)
]
)
)
# POST /example
api_entity.add_method(
http_method="POST",
integration=apigw_.LambdaIntegration(
handler=producer_lambda,
integration_responses=[
apigw_.IntegrationResponse(
status_code="200"
)
]
)
)
# ============= #
# StepFunctions #
# ============= #
dynamodb_update_running_task = self._dynamodb_update_in_sfn(table=demo_table, status="running")
wait_1_min = aws_sfn.Wait(
scope=self,
id="Wait one minutes as heavy task",
time=aws_sfn.WaitTime.duration(core.Duration.minutes(1)),
)
dynamodb_update_complete_task = self._dynamodb_update_in_sfn(table=demo_table, status="complete")
dynamodb_update_failure_task = self._dynamodb_update_in_sfn(table=demo_table, status="failure")
check_task_status = aws_sfn.Choice(scope=self, id="Job Complete?")\
.when(aws_sfn.Condition.string_equals("$.job_status", "success"), dynamodb_update_complete_task) \
.otherwise(dynamodb_update_failure_task)
# StepFunctions
definition = dynamodb_update_running_task \
.next(wait_1_min) \
.next(check_task_status)
sfn_process = aws_sfn.StateMachine(
scope=self,
id=f"{id_}-{stack_env}",
definition=definition
)
# Lambda to invoke StepFunction
sfn_invoke_lambda = self._create_lambda_function(
function_name="invoke_step_function",
environment={
"STEP_FUNCTION_ARN": sfn_process.state_machine_arn,
"QUEUE_URL": queue.queue_url
}
)
# grant
queue.grant_consume_messages(sfn_invoke_lambda)
sfn_process.grant_start_execution(sfn_invoke_lambda)
# ================ #
# CloudWatch Event #
# ================ #
# Runs every 2 hour
invoke_automatically = aws_events.Rule(
scope=self,
id=f"InvokeSFnViaLambda-{stack_env}",
schedule=aws_events.Schedule.rate(core.Duration.hours(2))
)
invoke_automatically.add_target(aws_events_targets.LambdaFunction(sfn_invoke_lambda))
def __init__(self, scope: Construct, construct_id: str, env,
**kwargs) -> None:
super().__init__(scope, construct_id, env=env, **kwargs)
rg_property = network_fw.CfnRuleGroup.RuleGroupProperty(
rule_variables=None,
rules_source=network_fw.CfnRuleGroup.RulesSourceProperty(
stateless_rules_and_custom_actions=network_fw.CfnRuleGroup.
StatelessRulesAndCustomActionsProperty(stateless_rules=[
network_fw.CfnRuleGroup.StatelessRuleProperty(
priority=10,
rule_definition=network_fw.CfnRuleGroup.
RuleDefinitionProperty(
actions=["aws:drop"],
match_attributes=network_fw.CfnRuleGroup.
MatchAttributesProperty(destinations=[
network_fw.CfnRuleGroup.AddressProperty(
address_definition="127.0.0.1/32")
])))
])))
nf_rule_group = network_fw.CfnRuleGroup(
scope=self,
id='GuardDutyNetworkFireWallRuleGroup',
capacity=100,
rule_group_name='guardduty-network-firewall',
type='STATELESS',
description='Guard Duty network firewall rule group',
tags=[CfnTag(key='Name', value='cfn.rule-group.stack')],
rule_group=rg_property)
""" https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-rule-dlq.html#dlq-considerations """
dlq_statemachine = sqs.Queue(self,
'DLQStateMachine',
queue_name='dlq_state_machine')
guardduty_firewall_ddb = ddb.Table(
scope=self,
id=f'GuarddutyFirewallDDB',
table_name='GuardDutyFirewallDDBTable',
removal_policy=RemovalPolicy.DESTROY,
partition_key=ddb.Attribute(name='HostIp',
type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST)
""" IAM role for ddb permission """
nf_iam_role = iam.Role(
self,
'DDBRole',
role_name=f'ddb-nf-role-{env.region}',
assumed_by=iam.ServicePrincipal(service='lambda.amazonaws.com'))
nf_iam_role.add_to_policy(
iam.PolicyStatement(effect=iam.Effect.ALLOW,
resources=["arn:aws:logs:*:*:*"],
actions=[
"logs:CreateLogGroup",
"logs:CreateLogStream", "logs:PutLogEvents"
]))
nf_iam_role.add_to_policy(
iam.PolicyStatement(effect=iam.Effect.ALLOW,
resources=[
guardduty_firewall_ddb.table_arn,
f"{guardduty_firewall_ddb.table_arn}/*"
],
actions=[
"dynamodb:PutItem", "dynamodb:GetItem",
"dynamodb:Scan"
]))
nf_iam_role.add_to_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
resources=[nf_rule_group.ref, f"{nf_rule_group.ref}/*"],
actions=[
"network-firewall:DescribeRuleGroup",
"network-firewall:UpdateRuleGroup"
]))
record_ip_in_db = _lambda.Function(
self,
'RecordIpInDB',
function_name='record-ip-in-ddb',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset('lambda_fns'),
handler='addIPToDDB.handler',
environment=dict(ACLMETATABLE=guardduty_firewall_ddb.table_name),
role=nf_iam_role)
"""
https://docs.amazonaws.cn/en_us/eventbridge/latest/userguide/eb-event-patterns-content-based-filtering.html
"""
record_ip_task = step_fn_task.LambdaInvoke(
self,
'RecordIpDDBTask',
lambda_function=record_ip_in_db,
payload=step_fn.TaskInput.from_object({
"comment":
"Relevant fields from the GuardDuty / Security Hub finding",
"HostIp.$":
"$.detail.findings[0].ProductFields.aws/guardduty/service/action/networkConnectionAction/remoteIpDetails/ipAddressV4",
"Timestamp.$":
"$.detail.findings[0].ProductFields.aws/guardduty/service/eventLastSeen",
"FindingId.$": "$.id",
"AccountId.$": "$.account",
"Region.$": "$.region"
}),
result_path='$',
payload_response_only=True)
firewall_update_rule = _lambda.Function(
scope=self,
id='GuardDutyUpdateNetworkFirewallRule',
function_name='gurdduty-update-networkfirewal-rule-group',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset('lambda_fns'),
handler='updateNetworkFireWall.handler',
environment=dict(
FIREWALLRULEGROUP=nf_rule_group.ref,
RULEGROUPPRI='30000',
CUSTOMACTIONNAME='GuardDutytoFirewall',
CUSTOMACTIONVALUE='gurdduty-update-networkfirewal-rule-group'),
role=nf_iam_role)
firewall_update_rule_task = step_fn_task.LambdaInvoke(
self,
'FirewallUpdateRuleTask',
lambda_function=firewall_update_rule,
input_path='$',
result_path='$',
payload_response_only=True)
firewall_no_update_job = step_fn.Pass(self, 'No Firewall change')
notify_failure_job = step_fn.Fail(self,
'NotifyFailureJob',
cause='Any Failure',
error='Unknown')
send_to_slack = _lambda.Function(
scope=self,
id='SendAlertToSlack',
function_name='gurdduty-networkfirewal-to-slack',
runtime=_lambda.Runtime.PYTHON_3_8,
handler="sendSMSToSlack.handler",
code=_lambda.Code.from_asset('lambda_fns'))
send_slack_task = step_fn_task.LambdaInvoke(
scope=self,
id='LambdaToSlackDemo',
lambda_function=send_to_slack,
input_path='$',
result_path='$')
is_new_ip = step_fn.Choice(self, "New IP?")
is_block_succeed = step_fn.Choice(self, "Block sucessfully?")
definition = step_fn.Chain \
.start(record_ip_task
.add_retry(errors=["States.TaskFailed"],
interval=Duration.seconds(2),
max_attempts=2)
.add_catch(errors=["States.ALL"], handler=notify_failure_job)) \
.next(is_new_ip
.when(step_fn.Condition.boolean_equals('$.NewIP', True),
firewall_update_rule_task
.add_retry(errors=["States.TaskFailed"],
interval=Duration.seconds(2),
max_attempts=2
)
.add_catch(errors=["States.ALL"], handler=notify_failure_job)
.next(
is_block_succeed
.when(step_fn.Condition.boolean_equals('$.Result', False), notify_failure_job)
.otherwise(send_slack_task)
)
)
.otherwise(firewall_no_update_job)
)
guardduty_state_machine = step_fn.StateMachine(
self,
'GuarddutyStateMachine',
definition=definition,
timeout=Duration.minutes(5),
state_machine_name='guardduty-state-machine')
event.Rule(
scope=self,
id='EventBridgeCatchIPv4',
description="Security Hub - GuardDuty findings with remote IP",
rule_name='guardduty-catch-ipv4',
event_pattern=event.EventPattern(
account=['123456789012'],
detail_type=["GuardDuty Finding"],
source=['aws.securityhub'],
detail={
"findings": {
"ProductFields": {
"aws/guardduty/service/action/networkConnectionAction/remoteIpDetails/ipAddressV4":
[{
"exists": True
}]
}
}
}),
targets=[
event_target.SfnStateMachine(
machine=guardduty_state_machine,
dead_letter_queue=dlq_statemachine)
])
""" Send other findings to slack """
send_finding_to_slack = _lambda.Function(
self,
'SendFindingToSlack',
function_name='send-finding-to-slack',
runtime=_lambda.Runtime.PYTHON_3_8,
handler="sendFindingToSlack.handler",
code=_lambda.Code.from_asset('lambda_fns'))
send_findings_task = step_fn_task.LambdaInvoke(
self,
'SendFindingToSlackTask',
lambda_function=send_finding_to_slack,
payload=step_fn.TaskInput.from_object({
"comment":
"Others fields from the GuardDuty / Security Hub finding",
"severity.$":
"$.detail.findings[0].Severity.Label",
"Account_ID.$":
"$.account",
"Finding_ID.$":
"$.id",
"Finding_Type.$":
"$.detail.findings[0].Types",
"Region.$":
"$.region",
"Finding_description.$":
"$.detail.findings[0].Description"
}),
result_path='$')
slack_failure_job = step_fn.Fail(self,
'SlackNotifyFailureJob',
cause='Any Failure',
error='Unknown')
finding_definition = step_fn.Chain \
.start(send_findings_task
.add_retry(errors=["States.TaskFailed"],
interval=Duration.seconds(2),
max_attempts=2)
.add_catch(errors=["States.ALL"], handler=slack_failure_job))
sechub_findings_state_machine = step_fn.StateMachine(
self,
'SecHubFindingsStateMachine',
definition=finding_definition,
timeout=Duration.minutes(5),
state_machine_name='sechub-finding-state-machine')
event.Rule(scope=self,
id='EventBridgeFindings',
description="Security Hub - GuardDuty findings others",
rule_name='others-findings',
event_pattern=event.EventPattern(
account=['123456789012'],
source=['aws.securityhub'],
detail_type=['Security Hub Findings - Imported'],
detail={"severity": [5, 8]}),
targets=[
event_target.SfnStateMachine(
machine=sechub_findings_state_machine,
dead_letter_queue=dlq_statemachine)
])
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Lets create couple of instances to test):
vpc = _ec2.Vpc(self,
"abacVPC",
cidr="10.13.0.0/21",
max_azs=2,
nat_gateways=0,
subnet_configuration=[
_ec2.SubnetConfiguration(
name="pubSubnet",
cidr_mask=24,
subnet_type=_ec2.SubnetType.PUBLIC)
])
core.Tag.add(vpc,
key="ServiceProvider",
value="KonStone",
include_resource_types=[])
weak_sg = _ec2.SecurityGroup(
self,
"web_sec_grp",
vpc=vpc,
description="Allow internet access from the world",
allow_all_outbound=True)
# vpc_cidr_block
# weak_sg.add_ingress_rule(_ec2.Peer.any_ipv4(),
weak_sg.add_ingress_rule(_ec2.Peer.ipv4(vpc.vpc_cidr_block),
_ec2.Port.tcp(22),
"Allow SSH access from the VPC Only.")
# We are using the latest AMAZON LINUX AMI
# Benefit of having SSM Agent pre-installed
ami_id = _ec2.AmazonLinuxImage(generation=_ec2.AmazonLinuxGeneration.
AMAZON_LINUX_2).get_image(self).image_id
# https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_iam/Role.html
instace_profile_role = _iam.Role(
self,
'ec2ssmroleid',
assumed_by=_iam.ServicePrincipal('ec2.amazonaws.com'),
role_name="instace_profile_role")
instace_profile_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonSSMManagedInstanceCore'))
instance_profile_role_additional_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
"arn:aws:logs:*:*:*",
],
actions=["logs:Create*", "logs:PutLogEvents"])
instance_profile_role_additional_perms.sid = "PutBucketPolicy"
instace_profile_role.add_to_policy(
instance_profile_role_additional_perms)
inst_profile_01 = _iam.CfnInstanceProfile(
self,
"instProfile01Id",
roles=[instace_profile_role.role_name],
)
# Let us bootstrap the server with the required agents
try:
with open("./bootstrap_scripts/install_agents.sh",
mode='rb') as file:
bootstrap_data = file.read()
except OSError:
print('Failed to get UserData script')
install_agents = _ec2.UserData.for_linux()
install_agents.add_commands(str(bootstrap_data, 'utf-8'))
# The EC2 Instance to monitor for failed SSH Logins
ssh_monitored_inst_01 = _ec2.CfnInstance(
self,
"sshMonitoredInstance01",
image_id=ami_id,
instance_type="t2.micro",
monitoring=False,
tags=[{
"key": "ServiceProvider",
"value": "KonStone"
}],
iam_instance_profile=inst_profile_01.ref,
network_interfaces=[{
"deviceIndex": "0",
"associatePublicIpAddress": True,
"subnetId": vpc.public_subnets[0].subnet_id,
"groupSet": [weak_sg.security_group_id]
}], #https: //github.com/aws/aws-cdk/issues/3419
user_data=core.Fn.base64(install_agents.render()),
)
"""
linux_ami = _ec2.GenericLinuxImage({ "cn-northwest-1": "ami-0f62e91915e16cfc2","eu-west-1": "ami-12345678"})
ssh_monitored_inst_01_02 = _ec2.Instance(self,
"monitoredInstance02",
instance_type=_ec2.InstanceType(instance_type_identifier="t2.micro"),
instance_name="monitoredInstance02",
machine_image=linux_ami,
vpc=vpc,
security_group=[weak_sg.security_group_id],
# vpc_subnets=_ec2.SubnetSelection(subnet_type=_ec2.SubnetType.PUBLIC)
vpc_subnets=vpc.public_subnets[0].subnet_id,
# user_data=_ec2.UserData.custom(t_user_data)
)
"""
# The log group name to store logs
info_sec_ops_log_group = _logs.LogGroup(
self,
"infoSecOpsLogGroupId",
log_group_name=(f"/Mystique/InfoSec/Automation/"
f"{ssh_monitored_inst_01.ref}"),
retention=_logs.RetentionDays.ONE_WEEK)
# Defines an AWS Lambda resource
with open("lambda_src/quarantine_ec2_instance.py",
encoding="utf8") as fp:
quarantine_ec2_instance_fn_handler_code = fp.read()
quarantine_ec2_instance_fn = _lambda.Function(
self,
id='quarantineEc2InstanceFnId',
function_name="quarantine_ec2_instance",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(quarantine_ec2_instance_fn_handler_code),
handler='index.lambda_handler',
timeout=core.Duration.seconds(5))
quarantine_ec2_instance_fn_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
"*",
],
actions=[
"ec2:RevokeSecurityGroupIngress",
"ec2:DescribeSecurityGroupReferences",
"ec2:RevokeSecurityGroupEgress",
"ec2:ApplySecurityGroupsToClientVpnTargetNetwork",
"ec2:DescribeSecurityGroups", "ec2:CreateSecurityGroup",
"ec2:DescribeInstances", "ec2:CreateTags", "ec2:StopInstances",
"ec2:CreateVolume", "ec2:CreateSnapshots",
"ec2:CreateSnapshot", "ec2:DescribeSnapshots",
"ec2:ModifyInstanceAttribute"
])
quarantine_ec2_instance_fn_perms.sid = "AllowLambdaToQuarantineEC2"
quarantine_ec2_instance_fn.add_to_role_policy(
quarantine_ec2_instance_fn_perms)
info_sec_ops_topic = _sns.Topic(self,
"infoSecOpsTopicId",
display_name="InfoSecTopic",
topic_name="InfoSecOpsTopic")
# Ref: https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html
###############################################################################
################# STEP FUNCTIONS EXPERIMENTAL CODE - UNSTABLE #################
###############################################################################
quarantine_ec2_instance_task = _sfn.Task(
self,
"Quarantine EC2 Instance",
task=_tasks.InvokeFunction(quarantine_ec2_instance_fn),
result_path="$")
notify_secops_task = _sfn.Task(
self,
"Notify InfoSecOps",
task=_tasks.PublishToTopic(
info_sec_ops_topic,
integration_pattern=_sfn.ServiceIntegrationPattern.
FIRE_AND_FORGET,
message=_sfn.TaskInput.from_data_at("$.message"),
subject="SSH Error Response Notification"))
ssh_error_response_failure = _sfn.Fail(
self,
"SSH Error Response Actions Failed",
cause="All Response Actions were NOT completed",
error="Check Logs")
ssh_error_response_success = _sfn.Succeed(
self,
"SSH Error Response Actions Succeeded",
comment="All Response Action Completed Successfully",
)
ssh_error_response_sfn_definition = quarantine_ec2_instance_task\
.next(notify_secops_task\
.next(_sfn.Choice(self, "SSH Errors Response Complete?")\
.when(_sfn.Condition.number_equals("$.SdkHttpMetadata.HttpStatusCode", 200),ssh_error_response_success)\
.when(_sfn.Condition.not_(
_sfn.Condition.number_equals("$.SdkHttpMetadata.HttpStatusCode", 200)), ssh_error_response_failure)\
.otherwise(ssh_error_response_failure)
)
)
ssh_error_response_statemachine = _sfn.StateMachine(
self,
"stateMachineId",
definition=ssh_error_response_sfn_definition,
timeout=core.Duration.minutes(5))
###############################################################################
################# STEP FUNCTIONS EXPERIMENTAL CODE - UNSTABLE #################
###############################################################################
# LAMBDA TO TRIGGER STATE MACHINE - since state cannot be invoked by SNS
with open("lambda_src/trigger_state_machine.py",
encoding="utf8") as fp:
trigger_state_machine_fn_handler_code = fp.read()
trigger_state_machine_fn = _lambda.Function(
self,
id='sshErrorResponseFnId',
function_name="trigger_ssh_error_response_state_machine_fn",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(trigger_state_machine_fn_handler_code),
# code=_lambda.Code.asset("lambda_src/is_policy_permissive.py"),
# code=_lambda.Code.asset('lambda_src'),
# code=_lambda.InlineCode(code_body),
handler='index.lambda_handler',
timeout=core.Duration.seconds(5),
environment={
"STATE_MACHINE_ARN":
f"{ssh_error_response_statemachine.state_machine_arn}",
})
trigger_state_machine_fn_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
f"{ssh_error_response_statemachine.state_machine_arn}",
],
actions=["states:StartExecution"])
trigger_state_machine_fn_perms.sid = "PutBucketPolicy"
trigger_state_machine_fn.add_to_role_policy(
trigger_state_machine_fn_perms)
"""
version = trigger_state_machine_fn.add_version(name=datetime.now().isoformat())
trigger_state_machine_fn_alias = _lambda.Alias(self,
'lmdaAliasId',
alias_name='MystiqueTestAlias',
version=version
)
"""
# Lets add permission to SNS to trigger our lambda function
trigger_lambda_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
trigger_state_machine_fn.function_arn,
],
actions=[
"lambda:InvokeFunction",
])
trigger_lambda_perms.sid = "TriggerLambaFunction"
# info_sec_ops_topic.add_to_resource_policy( trigger_lambda_perms )
# Subscribe InfoSecOps Email to topic
info_sec_ops_topic.add_subscription(
_subs.EmailSubscription(global_args.INFO_SEC_OPS_EMAIL))
# info_sec_ops_topic.add_subscription(_subs.LambdaSubscription(trigger_state_machine_fn))
trigger_state_machine_fn_alarm = trigger_state_machine_fn.metric_all_errors(
).create_alarm(
self,
"fn-error-alarm",
threshold=5,
alarm_name="trigger_state_machine_fn_error_alarm",
evaluation_periods=5,
period=core.Duration.minutes(1),
)
subscribe_trigger_state_machine_fn_to_logs = _logs.SubscriptionFilter(
self,
"sshErrorLogSubscriptionId",
log_group=info_sec_ops_log_group,
destination=_logs_destination.LambdaDestination(
trigger_state_machine_fn),
filter_pattern=_logs.FilterPattern.space_delimited(
"Mon", "day", "timestamp", "ip", "id", "status",
"...").where_string("status", "=", "Invalid"),
)
# https://pypi.org/project/aws-cdk.aws-logs/
# We are creating three filter
# tooManySshDisconnects, invalidSshUser and invalidSshKey:
# When a user tries to SSH with invalid username the next line is logged in the SSH log file:
# Apr 20 02:39:35 ip-172-31-63-56 sshd[17136]: Received disconnect from xxx.xxx.xxx.xxx: 11: [preauth]
too_many_ssh_disconnects_metric = _cloudwatch.Metric(
namespace=f"{global_args.OWNER}",
metric_name="tooManySshDisconnects")
too_many_ssh_disconnects_filter = _logs.MetricFilter(
self,
"tooManySshDisconnectsFilterId",
log_group=info_sec_ops_log_group,
metric_namespace=too_many_ssh_disconnects_metric.namespace,
metric_name=too_many_ssh_disconnects_metric.metric_name,
filter_pattern=_logs.FilterPattern.space_delimited(
"Mon", "day", "timestamp", "ip", "id", "msg1", "msg2",
"...").where_string("msg2", "=", "disconnect"),
metric_value="1")
invalid_ssh_user_metric = _cloudwatch.Metric(
namespace=f"{global_args.OWNER}",
metric_name="invalidSshUser",
)
invalid_ssh_user_filter = _logs.MetricFilter(
self,
"invalidSshUserFilterId",
log_group=info_sec_ops_log_group,
metric_namespace=invalid_ssh_user_metric.namespace,
metric_name=invalid_ssh_user_metric.metric_name,
filter_pattern=_logs.FilterPattern.space_delimited(
"Mon", "day", "timestamp", "ip", "id", "status",
"...").where_string("status", "=", "Invalid"),
metric_value="1")
invalid_ssh_key_metric = _cloudwatch.Metric(
namespace=f"{global_args.OWNER}", metric_name="invalidSshKey")
invalid_ssh_key_filter = _logs.MetricFilter(
self,
"invalidSshKeyFilterId",
log_group=info_sec_ops_log_group,
metric_namespace=invalid_ssh_key_metric.namespace,
metric_name=invalid_ssh_key_metric.metric_name,
filter_pattern=_logs.FilterPattern.space_delimited(
"Mon", "day", "timestamp", "ip", "id", "msg1", "msg2",
"...").where_string("msg1", "=", "Connection").where_string(
"msg2", "=", "closed"),
metric_value="1")
# Now let us create alarms
# alarm is raised there are more than 5(threshold) of the measured metrics in two(datapoint) of the last three seconds(evaluation):
# Period=60Seconds, Eval=3, Threshold=5
too_many_ssh_disconnects_alarm = _cloudwatch.Alarm(
self,
"tooManySshDisconnectsAlarmId",
alarm_name="too_many_ssh_disconnects_alarm",
alarm_description=
"The number disconnect requests is greater then 5, even 1 time in 3 minutes",
metric=too_many_ssh_disconnects_metric,
actions_enabled=True,
period=core.Duration.minutes(1),
threshold=5,
evaluation_periods=3,
datapoints_to_alarm=1,
statistic="sum",
comparison_operator=_cloudwatch.ComparisonOperator.
GREATER_THAN_OR_EQUAL_TO_THRESHOLD)
invalid_ssh_user_alarm = _cloudwatch.Alarm(
self,
"invalidSshUserAlarmId",
alarm_name="too_many_invalid_ssh_users_alarm",
alarm_description=
"The number of invalid ssh users connecting is greater then 5, even 1 time in 3 minutes",
metric=invalid_ssh_user_metric,
actions_enabled=True,
period=core.Duration.minutes(1),
threshold=5,
evaluation_periods=3,
datapoints_to_alarm=1,
statistic="sum",
comparison_operator=_cloudwatch.ComparisonOperator.
GREATER_THAN_THRESHOLD)
invalid_ssh_user_alarm.add_alarm_action(
_cloudwatch_actions.SnsAction(info_sec_ops_topic))
invalid_ssh_key_alarm = _cloudwatch.Alarm(
self,
"invalidSshKeyAlarmId",
alarm_name="too_many_invalid_ssh_key_alarm",
alarm_description=
"The number of invalid ssh keys connecting is greater then 5, even 1 time in 3 minutes",
metric=invalid_ssh_key_metric,
actions_enabled=True,
period=core.Duration.minutes(1),
threshold=5,
evaluation_periods=3,
datapoints_to_alarm=1,
statistic="sum",
comparison_operator=_cloudwatch.ComparisonOperator.
GREATER_THAN_OR_EQUAL_TO_THRESHOLD)
invalid_ssh_key_alarm.add_alarm_action(
_cloudwatch_actions.SnsAction(info_sec_ops_topic))
###########################################
################# OUTPUTS #################
###########################################
output0 = core.CfnOutput(
self,
"SecuirtyAutomationFrom",
value=f"{global_args.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output1_1 = core.Fn.get_att(
logical_name_of_resource="sshMonitoredInstance01",
attribute_name="PublicIp")
output1 = core.CfnOutput(self,
"MonitoredInstance",
value=output1_1.to_string(),
description="Web Server Public IP to attack")
output2 = core.CfnOutput(
self,
"SSHAlarms",
value=
(f"https://console.aws.amazon.com/cloudwatch/home?region="
f"{core.Aws.REGION}"
f"#/configuration/"
f"#alarmsV2:?search=ssh&alarmStateFilter=ALL&alarmTypeFilter=ALL"
),
description="Check out the cloudwatch Alarms")
output3 = core.CfnOutput(
self,
"SubscribeToNotificationTopic",
value=(f"https://console.aws.amazon.com/sns/v3/home?"
f"{core.Aws.REGION}"
f"#/topic/"
f"{info_sec_ops_topic.topic_arn}"),
description=
"Add your email to subscription and confirm subscription")
output_test_1 = core.CfnOutput(
self,
"ToGenInvalidKeyErrors",
value=
(f"for i in {{1..30}}; do ssh -i $RANDOM ec2-user@{output1_1.to_string()}; sleep 2; done &"
),
description=
"Generates random key names and connects to server 30 times over 60 seconds"
)
output_test_2 = core.CfnOutput(
self,
"ToGenInvalidUserErrors",
value=
(f"for i in {{1..30}}; do ssh ec2-user$RANDOM@{output1_1.to_string()}; sleep 2; done &"
),
description=
"Generates random user names and connects to server 30 times over 60 seconds"
)
"""
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
test_queue = sqs.Queue(self, 'test-queue', queue_name='test1')
test_topic = sns.Topic(self, 'test-topic')
sns.Subscription(self,
'test-subscription',
topic=test_topic,
endpoint=test_queue.queue_arn,
protocol=sns.SubscriptionProtocol.SQS)
kinesis.Stream(self,
'test-stream',
stream_name='donut-sales',
shard_count=2)
create_order = step.Pass(self,
'create-order',
result=step.Result.from_object({
"Order": {
"Customer": "Alice",
"Product": "Coffee",
"Billing": {
"Price": 10.0,
"Quantity": 4.0
}
}
}))
calculate_amount = step.Pass(self,
'calculate-amount',
result=step.Result.from_number(40.0),
result_path='$.Order.Billing.Amount',
output_path='$.Order.Billing')
order_definition = create_order.next(calculate_amount)
step.StateMachine(self,
'test-state-machine',
state_machine_name='order-machine',
definition=order_definition)
make_tea = step.Choice(
self, 'make-tea', comment='Input should look like {"tea":"green"}')
green = step.Pass(self,
'green',
result=step.Result.from_string('Green tea'))
make_tea.when(step.Condition.string_equals('$.tea', 'green'), green)
black = step.Pass(self,
'black',
result=step.Result.from_string('Black tea'))
make_tea.when(step.Condition.string_equals('$.tea', 'black'), black)
orange = step.Pass(self,
'orange',
result=step.Result.from_string('Black tea'))
make_tea.when(step.Condition.string_equals('$.tea', 'orange'), orange)
error = step.Pass(self,
'error',
result=step.Result.from_string('Bad input'))
make_tea.otherwise(error)
step.StateMachine(self,
'test-state-machine-2',
state_machine_name='tea-machine',
definition=make_tea)
def __init__(self, app: core.App, id: str, **kwargs) -> None:
super().__init__(app, id, **kwargs)
# Create both lambdas
with open("lambda-submit.py", encoding="utf8") as fp:
lambda_submit_code = fp.read()
lambdaFn1 = lambda_.Function(
self,
"submitsmbatch",
code=lambda_.InlineCode(lambda_submit_code),
handler="index.lambda_handler",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
environment={
"transform_job_name":transform_job_name,
"model_name":model_name,
"max_concurrent":max_concurrent,
"max_payload_size":max_payload_size,
"s3_uri_in":s3_uri_in,
"s3_uri_out":s3_uri_out,
"instance_type":instance_type,
"instance_count":instance_count,
}
)
# Add perms
lambdaFn1.add_to_role_policy(aws_iam.PolicyStatement(
actions = ['sagemaker:CreateTransformJob',],
resources = ['arn:aws:sagemaker:{}:{}:transform-job/{}*'.format(my_region,my_acc_id,transform_job_name),]
))
with open("lambda-check.py", encoding="utf8") as fp:
lambda_check_code = fp.read()
lambdaFn2 = lambda_.Function(
self,
"checksmbatch",
code=lambda_.InlineCode(lambda_check_code),
handler="index.lambda_handler",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
environment={"model_name":model_name, # CHANGE TO YOUR ENDPOINT NAME!!
"content_type":"text/csv"}
)
# Add perms
lambdaFn2.add_to_role_policy(aws_iam.PolicyStatement(
actions = ['sagemaker:DescribeTransformJob',],
resources = ['arn:aws:sagemaker:{}:{}:transform-job/{}*'.format(my_region,my_acc_id,transform_job_name),]
))
# Define state machine
# submit_job_activity = sfn.Activity(
# self, "SubmitJob"
# )
# check_job_activity = sfn.Activity(
# self, "CheckJob"
# )
submit_job = sfn.Task(
self, "Submit Job",
task=sfn_tasks.InvokeFunction(lambdaFn1),
)
wait_x = sfn.Wait(
self, "Wait 1 minute",
time=sfn.WaitTime.duration(core.Duration.minutes(1)),
)
get_status = sfn.Task(
self, "Get Job Status",
task=sfn_tasks.InvokeFunction(lambdaFn2),
)
is_complete = sfn.Choice(
self, "Job Complete?"
)
job_failed = sfn.Fail(
self, "Job Failed",
cause="AWS Batch Job Failed",
error="DescribeJob returned FAILED"
)
final_status = sfn.Task(
self, "Get Final Job Status",
task=sfn_tasks.InvokeFunction(lambdaFn2),
)
definition = submit_job\
.next(wait_x)\
.next(get_status)\
.next(is_complete
.when(sfn.Condition.string_equals(
"$.status", "Failed"), job_failed)
.when(sfn.Condition.string_equals(
"$.status", "Completed"), final_status)
.otherwise(wait_x))
sfn.StateMachine(
self, "SMbatchInference",
definition=definition,
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The start of the image pipeline
imageBucket = aws_s3.Bucket(self, "imageBucket")
# Capture API activity with a trail
imageBucketTrail = aws_cloudtrail.Trail(self,
"imageBucketTrail",
is_multi_region_trail=False)
# Restrict to S3 data-plane events
imageBucketTrail.add_s3_event_selector(
include_management_events=False,
prefixes=[f"{imageBucket.bucket_arn}/"],
read_write_type=aws_cloudtrail.ReadWriteType.WRITE_ONLY)
# Filter to just PutObject and CopyObject events
imageBucketRule = aws_events.Rule(
self,
"imageBucketRule",
event_pattern={
"source": ["aws.s3"],
"detail": {
"eventSource": ["s3.amazonaws.com"],
"eventName": ["PutObject", "CopyObject"],
"requestParameters": {
"bucketName": [imageBucket.bucket_name]
}
}
})
#--
# Lambda Layers
#--------------------#
opencvLayer = aws_lambda.LayerVersion(
self,
'opencvLayer',
code=aws_lambda.AssetCode('layers/opencvLayer'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6])
boto3Layer = aws_lambda.LayerVersion(
self,
'boto3Layer',
code=aws_lambda.AssetCode('layers/boto3Layer'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6])
#--
# Lambda Functions
#--------------------#
# Gather info about an image, name, extension, etc
getImageInfoFunc = aws_lambda.Function(
self,
"getImageInfoFunc",
code=aws_lambda.AssetCode('functions/getImageInfoFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6)
# The home for the website
webBucket = aws_s3.Bucket(self,
"webBucket",
website_index_document='index.html')
# Copy the image to the web bucket
copyImageFunc = aws_lambda.Function(
self,
"copyImageFunc",
code=aws_lambda.AssetCode('functions/copyImageFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
layers=[boto3Layer],
environment={
'OUTPUTBUCKET': webBucket.bucket_name,
'OUTPUTPREFIX': 'images/'
})
# Grant permissions to read from the source and write to the desination
imageBucket.grant_read(copyImageFunc)
webBucket.grant_write(copyImageFunc)
# Create a thumbnail of the image and place in the web bucket
createThumbnailFunc = aws_lambda.Function(
self,
"createThumbnailFunc",
code=aws_lambda.AssetCode('functions/createThumbnailFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
layers=[boto3Layer, opencvLayer],
timeout=core.Duration.seconds(10),
memory_size=256,
environment={
'OUTPUTBUCKET': webBucket.bucket_name,
'OUTPUTPREFIX': 'images/'
})
# Grant permissions to read from the source and write to the desination
imageBucket.grant_read(createThumbnailFunc)
webBucket.grant_write(createThumbnailFunc)
# Store page information
pageTable = aws_dynamodb.Table(
self,
'pageTable',
partition_key={
'name': 'pageName',
'type': aws_dynamodb.AttributeType.STRING
},
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST,
stream=aws_dynamodb.StreamViewType.NEW_IMAGE)
# Save page and image information
updatePageInfoFunc = aws_lambda.Function(
self,
"updatePageInfoFunc",
code=aws_lambda.AssetCode('functions/updatePageInfoFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
layers=[boto3Layer],
environment={
'PAGETABLE': pageTable.table_name,
'PAGEPREFIX': 'posts/'
})
# Grant permissions to write to the page table
pageTable.grant_write_data(updatePageInfoFunc)
imagePipelineDone = aws_stepfunctions.Succeed(self,
"Done processing image")
updatePageInfoJob = aws_stepfunctions.Task(
self,
'Update page info',
task=aws_stepfunctions_tasks.InvokeFunction(updatePageInfoFunc))
updatePageInfoJob.next(imagePipelineDone)
copyImageJob = aws_stepfunctions.Task(
self,
'Copy image',
task=aws_stepfunctions_tasks.InvokeFunction(copyImageFunc))
createThumbnailJob = aws_stepfunctions.Task(
self,
'Create thumbnail',
task=aws_stepfunctions_tasks.InvokeFunction(createThumbnailFunc))
# These tasks can be done in parallel
processImage = aws_stepfunctions.Parallel(self,
'Process image',
result_path="$.images")
processImage.branch(copyImageJob)
processImage.branch(createThumbnailJob)
processImage.next(updatePageInfoJob)
# Results of file extension check
notPng = aws_stepfunctions.Succeed(self, "Not a PNG")
# Verify the file extension
checkForPng = aws_stepfunctions.Choice(self, 'Is a PNG?')
checkForPng.when(
aws_stepfunctions.Condition.string_equals('$.extension', 'png'),
processImage)
checkForPng.otherwise(notPng)
# A single image pipeline job for testing
getImageInfoJob = aws_stepfunctions.Task(
self,
'Get image info',
task=aws_stepfunctions_tasks.InvokeFunction(getImageInfoFunc))
getImageInfoJob.next(checkForPng)
# Configure the image pipeline and starting state
imagePipeline = aws_stepfunctions.StateMachine(
self, "imagePipeline", definition=getImageInfoJob)
# Matching events start the image pipline
imageBucketRule.add_target(
aws_events_targets.SfnStateMachine(
imagePipeline,
input=aws_events.RuleTargetInput.from_event_path(
"$.detail.requestParameters")))
def __init__(self, app: App, id: str, **kwargs) -> None:
super().__init__(app, id, **kwargs)
# Lambda Handlers Definitions
submit_lambda = _lambda.Function(
self,
'submitLambda',
handler='lambda_function.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_9,
code=_lambda.Code.from_asset('lambdas/submit'))
status_lambda = _lambda.Function(
self,
'statusLambda',
handler='lambda_function.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_9,
code=_lambda.Code.from_asset('lambdas/status'))
# Step functions Definition
submit_job = _aws_stepfunctions_tasks.LambdaInvoke(
self,
"Submit Job",
lambda_function=submit_lambda,
output_path="$.Payload",
)
wait_job = _aws_stepfunctions.Wait(
self,
"Wait 30 Seconds",
time=_aws_stepfunctions.WaitTime.duration(Duration.seconds(30)))
status_job = _aws_stepfunctions_tasks.LambdaInvoke(
self,
"Get Status",
lambda_function=status_lambda,
output_path="$.Payload",
)
fail_job = _aws_stepfunctions.Fail(self,
"Fail",
cause='AWS Batch Job Failed',
error='DescribeJob returned FAILED')
succeed_job = _aws_stepfunctions.Succeed(
self, "Succeeded", comment='AWS Batch Job succeeded')
# Create Chain
definition = submit_job.next(wait_job)\
.next(status_job)\
.next(_aws_stepfunctions.Choice(self, 'Job Complete?')
.when(_aws_stepfunctions.Condition.string_equals('$.status', 'FAILED'), fail_job)
.when(_aws_stepfunctions.Condition.string_equals('$.status', 'SUCCEEDED'), succeed_job)
.otherwise(wait_job))
# Create state machine
sm = _aws_stepfunctions.StateMachine(
self,
"StateMachine",
definition=definition,
timeout=Duration.minutes(5),
)
