def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
innerSfnPassState = sfn.Pass(self, 'PassState');
innerSfn = sfn.StateMachine(self, 'InnerStepFunction',
definition = innerSfnPassState,
timeout=Duration.minutes(60)
)
task1 = tasks.StepFunctionsStartExecution(self, "StepFunction1",
state_machine=innerSfn,
integration_pattern=sfn.IntegrationPattern.RUN_JOB,
input=sfn.TaskInput.from_object({
"input.$": "$.Output.input"
}),
output_path="$",
result_selector = {
"Output.$": "$.Output"
}
)
task2 = tasks.StepFunctionsStartExecution(self, "StepFunction2",
state_machine=innerSfn,
integration_pattern=sfn.IntegrationPattern.RUN_JOB,
input=sfn.TaskInput.from_object({
"input.$": "$.Output.input"
}),
output_path="$",
result_selector = {
"Output.$": "$.Output"
}
)
task3 = tasks.StepFunctionsStartExecution(self, "StepFunction3",
state_machine=innerSfn,
integration_pattern=sfn.IntegrationPattern.RUN_JOB,
input=sfn.TaskInput.from_object({
"input.$": "$.Output.input"
}),
output_path="$",
result_selector = {
"Output.$": "$.Output"
}
)
outer_sfn = sfn.StateMachine(self, "OuterStepFunction",
definition=task1.next(task2).next(task3),
timeout=Duration.minutes(60)
)
CfnOutput(self, "StepFunctionArn",
value = outer_sfn.state_machine_arn,
export_name = 'OuterStepFunctionArn',
description = 'Outer Step Function arn')
def createTipWorkflow(self):
notifyTipper = tasks.LambdaInvoke(self, 'notifyTipper',
lambda_function=self.createLambda('notifyTipperLambda', 'tipNotifier.tipNotifier'),
timeout=cdk.Duration.seconds(300)
).next(sfn.Succeed(self, 'withdrawSuccessState'))
self.getTipperInvoice = tasks.StepFunctionsInvokeActivity(self, 'getTipperInvoice',
activity=sfn.Activity(self, 'getTipperInvoiceActivity'),
heartbeat=cdk.Duration.seconds(60),
timeout=cdk.Duration.seconds(86400),
)
self.getTipperInvoice.add_retry(
backoff_rate=1.5,
errors=['States.Timeout'],
interval=cdk.Duration.seconds(60),
max_attempts=7
)
self.getTipperInvoice.add_catch(
handler=sfn.Fail(self, 'withdrawErrorState'),
errors=['States.ALL'],
result_path='$.errorInfo'
)
self.getTipperInvoice.next(notifyTipper)
return sfn.StateMachine(self, 'tipWorkflow',
definition=self.getTipperInvoice,
role=self.statesRole
)
def createWithdrawWorkflow(self):
payInvoiceFailed = tasks.LambdaInvoke(self, 'payInvoiceFailed',
lambda_function=self.createLambda('payInvoiceFailedLambda', 'payInvoiceFailed.payInvoiceFailed'),
timeout=cdk.Duration.seconds(300)
).next(sfn.Fail(self, 'tipErrorState'))
payInvoiceSucceeded = tasks.LambdaInvoke(self, 'payInvoiceSucceeded',
lambda_function=self.createLambda('payInvoiceSucceededLambda', 'payInvoiceSucceeded.payInvoiceSucceeded'),
timeout=cdk.Duration.seconds(300)
).next(sfn.Succeed(self, 'tipSuccessState'))
self.payInvoice = tasks.StepFunctionsInvokeActivity(self, 'payInvoice',
activity=sfn.Activity(self, 'payInvoiceActivity'),
heartbeat=cdk.Duration.seconds(86400),
timeout=cdk.Duration.seconds(86400),
)
self.payInvoice.add_retry(
backoff_rate=2,
errors=['States.Timeout'],
interval=cdk.Duration.seconds(600),
max_attempts=0
)
self.payInvoice.add_catch(
handler=payInvoiceFailed,
errors=['States.ALL'],
result_path='$.errorInfo'
)
self.payInvoice.next(payInvoiceSucceeded)
return sfn.StateMachine(self, 'withdrawWorkflow',
definition=self.payInvoice,
role=self.statesRole
)
def machine(self, name: str='stateMachine', timeout: int=1):
self.statemachine = sfn.StateMachine(
self, name,
definition=self.start,
timeout=core.Duration.minutes(timeout)
)
return self.statemachine
def test_start_execution_task():
default_task_json = {
'End': True,
'Parameters': {
'StateMachineArn': {
'Ref': 'teststatemachine7F4C511D'
},
'Input.$': '$$.Execution.Input'
},
'Type': 'Task',
'Resource': {
'Fn::Join': [
'',
[
'arn:', {
'Ref': 'AWS::Partition'
}, ':states:::states:startExecution.sync'
]
]
}
}
stack = core.Stack(core.App(), 'test-stack')
state_machine = sfn.StateMachine(stack,
'test-state-machine',
definition=sfn.Chain.start(
sfn.Succeed(stack, 'Succeeded')))
task = sfn.Task(stack,
'test-task',
task=emr_tasks.StartExecutionTask(state_machine, ))
print_and_assert(default_task_json, task)
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
flip_coin_function = lambda_.Function(
self,
"FlipCoinFunction",
runtime=lambda_.Runtime.PYTHON_3_8,
handler="index.handler",
code=lambda_.Code.from_asset("./sfn/lambda/flip_coin"))
flip_coin_invoke = tasks.LambdaInvoke(
self, "FlipCoin", lambda_function=flip_coin_function)
wait = stepfunctions.Wait(self,
"Wait",
time=stepfunctions.WaitTime.duration(
core.Duration.seconds(5)))
tails_result = stepfunctions.Pass(self, "TailsResult")
tails_result.next(flip_coin_invoke)
choice = stepfunctions.Choice(self,
"HeadsTailsChoice") \
.when(condition=stepfunctions.Condition.string_equals("$.Payload.result", "heads"),
next=stepfunctions.Succeed(self, "HeadsResult")) \
.when(condition=stepfunctions.Condition.string_equals("$.Payload.result", "tails"),
next=tails_result)
stepfunctions.StateMachine(self,
"StateMachine",
definition=flip_coin_invoke.next(
wait.next(choice)))
def create_enumerate_statemachine(self):
enumerate_job = tasks.LambdaInvoke(
self,
"Enumerate Notes Job",
lambda_function=self.step_lambda,
payload=sfn.TaskInput.from_object({"action": "enumerate_notes"}),
)
get_tf_job = tasks.LambdaInvoke(self,
"Get Text Frequency Job",
lambda_function=self.step_lambda,
payload=sfn.TaskInput.from_object({
"action":
"update_tf",
"id.$":
"$.id",
"contentUpdatedAt.$":
"$.contentUpdatedAt",
"isArchived.$":
"$.isArchived",
}),
output_path="$.Payload")
map_job = sfn.Map(self,
"Notes Map",
items_path="$.Payload.id_list",
max_concurrency=8)
get_idf_job = tasks.LambdaInvoke(
self,
"Get Inter Document Frequency Job",
lambda_function=self.step_lambda,
payload=sfn.TaskInput.from_object({
"action": "update_idf",
"notes.$": "$"
}),
)
map_tfidf_job = sfn.Map(self,
"TF*IDF Notes Map",
items_path="$.Payload.notes",
max_concurrency=100)
get_tfidf_job = tasks.LambdaInvoke(
self,
"Get TF*IDF WordCloud Image Job",
lambda_function=self.step_lambda,
payload=sfn.TaskInput.from_object({
"action": "update_tfidf_png",
"id.$": "$.id",
"contentUpdatedAt.$": "$.contentUpdatedAt",
"isArchived.$": "$.isArchived",
}),
)
definition = (enumerate_job.next(
map_job.iterator(get_tf_job)).next(get_idf_job).next(
map_tfidf_job.iterator(get_tfidf_job)))
self.enumerate_statemachine = sfn.StateMachine(
self,
"EnumerateStateMachine",
definition=definition,
timeout=core.Duration.hours(5),
)
def __init__(self,
scope: core.Construct,
id: str,
*,
polling_delay: int = 5,
statemachine_timeout: int = 300,
**kwargs):
super().__init__(scope, id, **kwargs)
state_fn = StateHandlerLambda(self, "config-state-handler").function
config_fn = AccountConfigLambda(self,
"account-config-handler").function
config_state = tasks.LambdaInvoke(self,
"Set Configuring State",
lambda_function=state_fn,
output_path="$.Payload")
completed_state = tasks.LambdaInvoke(self,
"Set Completed State",
lambda_function=state_fn,
output_path="$.Payload")
config_task = tasks.LambdaInvoke(self,
"Request Account Configuration",
lambda_function=config_fn,
output_path="$.Payload")
polling_task = tasks.LambdaInvoke(self,
"Poll Account Configuration",
lambda_function=config_fn,
output_path="$.Payload")
delay = sfn.Wait(self,
"Delay Polling",
time=sfn.WaitTime.duration(
core.Duration.seconds(polling_delay)))
is_ready = sfn.Choice(self, "Account Ready?")
acct_ready = sfn.Condition.string_equals('$.state', "READY")
acct_pending = sfn.Condition.string_equals('$.state', "PENDING")
success = sfn.Succeed(self, "Config Succeeded")
failed = sfn.Fail(self,
"Config Failed",
cause="Bad value in Polling loop")
# this is the loop which polls for state change, either looping back to delay or setting completion state and finishing
is_ready.when(acct_pending, delay).when(
acct_ready, completed_state.next(success)).otherwise(failed)
# this is the main chain starting with creation request a delay and then polling loop
config_chain = config_task.next(config_state).next(delay).next(
polling_task).next(is_ready)
self.state_machine = sfn.StateMachine(
self,
"Account-Config-StateMachine",
definition=config_chain,
timeout=core.Duration.seconds(statemachine_timeout))
def __init__(self, scope: core.App, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
pass_through_lambda = _lambda.Function(
self,
'PassThroughLambda',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='pass_through_lambda.handler')
loop_count_lambda = _lambda.Function(
self,
'LoopCountLambda',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='loop_count_lambda.handler')
start_state_machine = sfn.Task(
self,
"Start CodeBuild Lambda",
task=sfn_tasks.InvokeFunction(pass_through_lambda))
wait_x = sfn.Wait(
self,
"Wait X Seconds",
time=sfn.WaitTime.seconds_path('$.wait_time'),
)
get_state_machine_status = sfn.Task(
self,
"Get Build Status",
task=sfn_tasks.InvokeFunction(loop_count_lambda))
is_complete = sfn.Choice(self, "Job Complete?")
state_machine_failed = sfn.Fail(self,
"Build Failed",
cause="AWS Batch Job Failed",
error="DescribeJob returned FAILED")
state_machine_success = sfn.Pass(self, "Build Successs")
definition = start_state_machine\
.next(wait_x)\
.next(get_state_machine_status)\
.next(is_complete
.when(sfn.Condition.string_equals(
"$.status", "FAILED"), state_machine_failed)
.when(sfn.Condition.string_equals(
"$.status", "SUCCEEDED"), state_machine_success)
.otherwise(wait_x))
sfn.StateMachine(
self,
"StateMachine",
definition=definition,
timeout=core.Duration.seconds(60),
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Step Function Starts Here
# The first thing we need to do is see if they are asking for pineapple on a pizza
pineapple_check_lambda = _lambda.Function(self, "pineappleCheckLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="orderPizza.handler",
code=_lambda.Code.from_asset("lambdas"),
)
# Step functions are built up of steps, we need to define our first step
order_pizza = step_fn.Task(self, 'Order Pizza Job',
task=step_fn_tasks.InvokeFunction(pineapple_check_lambda),
input_path='$.flavour',
result_path='$.pineappleAnalysis'
)
# Pizza Order failure step defined
job_failed = step_fn.Fail(self, 'Sorry, We Dont add Pineapple',
cause='Failed To Make Pizza',
error='They asked for Pineapple')
# If they didnt ask for pineapple let's cook the pizza
cook_pizza = step_fn.Pass(self, 'Lets make your pizza')
# If they ask for a pizza with pineapple, fail. Otherwise cook the pizza
definition = step_fn.Chain \
.start(order_pizza) \
.next(step_fn.Choice(self, 'With Pineapple?') \
.when(step_fn.Condition.boolean_equals('$.pineappleAnalysis.containsPineapple', True), job_failed) \
.otherwise(cook_pizza))
state_machine = step_fn.StateMachine(self, 'StateMachine', definition=definition, timeout=core.Duration.minutes(5))
# Dead Letter Queue Setup
dlq = sqs.Queue(self, 'stateMachineLambdaDLQ', visibility_timeout=core.Duration.seconds(300))
# defines an AWS Lambda resource to connect to our API Gateway
state_machine_lambda = _lambda.Function(self, "stateMachineLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="stateMachineLambda.handler",
code=_lambda.Code.from_asset("lambdas"),
environment={
'statemachine_arn': state_machine.state_machine_arn
}
)
state_machine.grant_start_execution(state_machine_lambda)
# defines an API Gateway REST API resource backed by our "sqs_publish_lambda" function.
api_gw.LambdaRestApi(self, 'Endpoint',
handler=state_machine_lambda
)
def __init__(self, app: cdk.App, id: str, **kwargs) -> None:
super().__init__(app, id, **kwargs)
submit_job_activity = sfn.Activity(
self, "SubmitJob"
)
check_job_activity = sfn.Activity(
self, "CheckJob"
)
submit_job = sfn.Task(
self, "Submit Job",
task=sfn_tasks.InvokeActivity(submit_job_activity),
result_path="$.guid",
)
wait_x = sfn.Wait(
self, "Wait X Seconds",
duration=sfn.WaitDuration.seconds_path('$.wait_time'),
)
get_status = sfn.Task(
self, "Get Job Status",
task=sfn_tasks.InvokeActivity(check_job_activity),
input_path="$.guid",
result_path="$.status",
)
is_complete = sfn.Choice(
self, "Job Complete?"
)
job_failed = sfn.Fail(
self, "Job Failed",
cause="AWS Batch Job Failed",
error="DescribeJob returned FAILED"
)
final_status = sfn.Task(
self, "Get Final Job Status",
task=sfn_tasks.InvokeActivity(check_job_activity),
input_path="$.guid",
)
definition = submit_job\
.next(wait_x)\
.next(get_status)\
.next(is_complete
.when(sfn.Condition.string_equals(
"$.status", "FAILED"), job_failed)
.when(sfn.Condition.string_equals(
"$.status", "SUCCEEDED"), final_status)
.otherwise(wait_x))
sfn.StateMachine(
self, "StateMachine",
definition=definition,
timeout_sec=30,
)
def __init__(self, scope: core.Construct, id: builtins.str,
action_name: str, resources: FsiSharedResources,
function: lambda_.Function) -> None:
super().__init__(scope, id)
self.__resources = resources
state_machine_name = id
# Define the state machine definition...
invoke_function = sft.LambdaInvoke(
self,
'InvokeFunction',
lambda_function=function,
invocation_type=sft.LambdaInvocationType.REQUEST_RESPONSE,
input_path='$.Payload',
result_path='$.Result')
choice = sf.Choice(self,
'IsComplete',
comment='Check if theres more to process')
choice.when(
sf.Condition.string_equals('$.Result.Payload.Result.RunState',
'RunStatus.MORE_AVAILABLE'),
invoke_function)
choice.when(
sf.Condition.string_equals('$.Result.Payload.Result.RunState',
'RunStatus.COMPLETE'),
sf.Pass(self, 'Finalize', comment='Workflow Complete'))
choice.otherwise(
sf.Fail(self,
'NotImplemented',
cause='Unknown Choice',
error='NotImplementedException'))
definition = invoke_function.next(choice)
# Register the definition as StateMachine...
zone_name = self.resources.landing_zone.zone_name
self.state_machine = sf.StateMachine(
self,
'StateMachine',
state_machine_name=state_machine_name,
state_machine_type=sf.StateMachineType.STANDARD,
timeout=core.Duration.hours(2),
logs=sf.LogOptions(destination=logs.LogGroup(
self,
'LogGroup',
removal_policy=core.RemovalPolicy.DESTROY,
retention=RetentionDays.TWO_WEEKS,
log_group_name='/homenet/fsi-{}/states/{}/{}'.format(
zone_name, self.component_name, action_name).lower())),
tracing_enabled=True,
definition=definition)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
#lf1= Function(self, id="my_stack_lambda", runtime=Runtime.PYTHON_3_7, handler='handlers/my_lambda_handler', code='', function_name='my_example_lambda')
my_table = _dynamodb.Table(self,
id='dynamoTable',
table_name='testcdktabe',
partition_key=_dynamodb.Attribute(
name='lastname',
type=_dynamodb.AttributeType.STRING))
my_s3_bucket = _s3.Bucket(self,
id='s3bucket',
bucket_name='mynpbsample3bucket')
my_lambda_function = _lambda.Function(
self,
id='lambdafunction',
runtime=_lambda.Runtime.PYTHON_3_7,
handler='hello.handler',
code=_lambda.Code.asset('lambdacode'))
process_purchase_function = _lambda.Function(
self,
id='process_purchase',
runtime=_lambda.Runtime.PYTHON_3_7,
handler='process_purchase.handler',
code=_lambda.Code.asset('lambdacode'))
process_refund_function = _lambda.Function(
self,
id='process_refund',
runtime=_lambda.Runtime.PYTHON_3_7,
handler='process_refund.handler',
code=_lambda.Code.asset('lambdacode'))
#start_state = sfn.Pass(self, "start_state")
definition = sfn.Task(
self,
'Get Process Type',
task=tasks.InvokeFunction(process_purchase_function))
sfn.StateMachine(
self,
"MyStateMachine",
definition=definition,
timeout=core.Duration.seconds(30),
)
my_topic = sns.Topic(self,
"MyTopic",
display_name="Customer Subscription")
def __init__(self, scope: Construct, id: str, functions: LambdaLib, **kwargs) -> None:
super().__init__(scope, id)
# Step Function
submit_job = tasks.LambdaInvoke(self, "Submit Job",
lambda_function=functions.send_email_approval,
payload=sfn.TaskInput.from_object({'ExecutionContext.$': '$$'}),
result_path=sfn.JsonPath.DISCARD
)
wait_x = sfn.Wait(self, "Wait",
time= sfn.WaitTime.duration(Duration.minutes(2))
)
get_status = tasks.LambdaInvoke(self, "Get Job Status",
lambda_function=functions.check_status_dynamo,
payload=sfn.TaskInput.from_object({'ExecutionContext.$': '$$'}),
result_path="$.status"
)
restrict_es = tasks.LambdaInvoke(self, "Restric ES Policy",
lambda_function=functions.restric_es_policy,
payload=sfn.TaskInput.from_object({'ExecutionContext.$': '$$'}),
)
restrict_rds = tasks.LambdaInvoke(self, "Restric RDS",
lambda_function=functions.restric_rds_policy,
payload=sfn.TaskInput.from_object({'ExecutionContext.$': '$$'}),
)
restrict_es_condition = sfn.Condition.string_equals("$.detail.additionalEventData.configRuleName", constants.CONFIG_RULE_ES_PUBLIC)
restrict_rds_condition = sfn.Condition.string_equals("$.detail.additionalEventData.configRuleName", constants.CONFIG_RULE_RDS_PUBLIC)
definition = (submit_job.next(wait_x)
.next(get_status)
.next(sfn.Choice(self, "Job Complete?")
.when(sfn.Condition.string_equals("$.status.Payload.status", "Rejected!"), wait_x)
# .when(sfn.Condition.string_equals("$.status.Payload.status", "NON_COMPLIANT"), final_task)
# .when(sfn.Condition.string_equals("$.status.Payload.status", "Accepted!"), final_task))
.otherwise(sfn.Choice(self, "Remediation Choice")
.when(restrict_es_condition, restrict_es)
.when(restrict_rds_condition, restrict_rds)))
)
self.state_machine = sfn.StateMachine(self, "StateMachine",
definition=definition,
timeout=Duration.hours(2)
)
def __init__(self, app: core.App, cfn_name: str, stack_env):
super().__init__(scope=app, id=f"{cfn_name}-{stack_env}")
# lambda
lambda_task = lambda_.Function(
scope=self,
id=f"{cfn_name}-lambda-task",
code=lambda_.AssetCode.from_asset("lambda_script"),
handler="lambda_handler.lambda_task",
timeout=core.Duration.seconds(10),
runtime=self.LAMBDA_PYTHON_RUNTIME,
memory_size=128
)
# StepFunction Tasks
sns_source = sfn.Pass(
scope=self,
id=f"{cfn_name}-sfn-pass",
comment="pass example",
input_path="$",
result_path="$.source",
result=sfn.Result.from_string("example"),
output_path="$"
)
arguments_generation = sfn.Task(
scope=self,
id=f"{cfn_name}-sfn-lambda-task",
task=sfn_tasks.RunLambdaTask(
lambda_function=lambda_task,
payload=sfn.TaskInput.from_object({
"time.$": "$.time",
"source.$": "$.source"
})),
input_path="$",
result_path="$.arguments",
output_path="$.arguments.Payload"
)
# stepfunctions
definition = sns_source.next(arguments_generation)
_ = sfn.StateMachine(
scope=self,
id=f"{cfn_name}-SFn-{stack_env}",
definition=definition
)
def create_state_machine(self, lambda_functions, page_sqs):
task_wrapup = aws_stepfunctions.Task(
self, "task_wrapup",
task = aws_stepfunctions_tasks.RunLambdaTask(lambda_functions["wrapup"])
)
tast_analyze_with_scale = aws_stepfunctions.Task(
self, "AnalyzeWithScale",
task= aws_stepfunctions_tasks.SendToQueue(
queue = page_sqs,
message_body = aws_stepfunctions.TaskInput.from_object(
{
"token": aws_stepfunctions.Context.task_token,
"id.$": "$.id",
"bucket.$": "$.bucket",
"original_upload_pdf.$": "$.original_upload_pdf",
"SAGEMAKER_WORKFLOW_AUGMENTED_AI_ARN.$": "$.SAGEMAKER_WORKFLOW_AUGMENTED_AI_ARN",
"key.$": "$.key"
}
),
delay=None,
integration_pattern=aws_stepfunctions.ServiceIntegrationPattern.WAIT_FOR_TASK_TOKEN
)
)
process_map = aws_stepfunctions.Map(
self, "Process_Map",
items_path = "$.image_keys",
result_path="DISCARD",
parameters = {
"id.$": "$.id",
"bucket.$": "$.bucket",
"original_upload_pdf.$": "$.original_upload_pdf",
"SAGEMAKER_WORKFLOW_AUGMENTED_AI_ARN.$": "$.SAGEMAKER_WORKFLOW_AUGMENTED_AI_ARN",
"key.$": "$$.Map.Item.Value"
}
).iterator(tast_analyze_with_scale)
definition = process_map.next(task_wrapup)
aws_stepfunctions.StateMachine(
scope = self,
id = "multipagepdfa2i_fancy_stepfunction",
state_machine_name = "multipagepdfa2i_fancy_stepfunction",
definition=definition
)
def __init__(self,
scope: core.Construct,
id: str,
factory: AccountFactoryMonitor,
config: AccountConfigMonitor,
statemachine_timeout: int = 300,
**kwargs):
super().__init__(scope, id, **kwargs)
start_factory_machine = tasks.StepFunctionsStartExecution(
self,
"Account-Factory-StateMachine",
state_machine=factory.state_machine,
integration_pattern=sfn.IntegrationPattern.RUN_JOB,
output_path="$.Output")
start_config_machine = tasks.StepFunctionsStartExecution(
self,
"Account-Config-StateMachine",
state_machine=config.state_machine,
integration_pattern=sfn.IntegrationPattern.RUN_JOB,
output_path="$.Output")
start_task = sfn.Pass(self,
"start creation",
parameters={
"uuid.$": "$.uuid",
"action": "CONFIG"
})
inter_task = sfn.Pass(self,
"start configuration",
parameters={
"uuid.$": "$.uuid",
"create_account_poll.$":
"$.create_account_poll",
"action": "CONFIG"
})
end_task = sfn.Pass(self, "end provisioning")
def_chain = start_task.next(start_factory_machine).next(
inter_task).next(start_config_machine).next(end_task)
self.state_machine = sfn.StateMachine(
self,
"Account-Provsioning-StateMachine",
definition=def_chain,
timeout=core.Duration.seconds(statemachine_timeout))
def __init__(self, scope, id, name=None, lambdas=None) -> None:
super().__init__(scope, id)
# ==================================================
# ================= IAM ROLE =======================
# ==================================================
state_machine_role = iam.Role(
scope=self,
id='state_machine_role',
assumed_by=iam.ServicePrincipal(service='states.amazonaws.com'),
)
state_machine_role.add_to_policy(
iam.PolicyStatement(effect=iam.Effect.ALLOW,
actions=['lambda:InvokeFunction'],
resources=['*']))
# ==================================================
# ================= STATE MACHINE ==================
# ==================================================
invoke_lambda_rf = tasks.LambdaInvoke(
scope=self,
id='Random Forest',
lambda_function=lambdas['lambda_rf'],
payload_response_only=True)
invoke_lambda_svr = tasks.LambdaInvoke(
scope=self,
id='Support Vector',
lambda_function=lambdas['lambda_svr'],
payload_response_only=True)
invoke_lambda_lr = tasks.LambdaInvoke(
scope=self,
id='Linear Regressor',
lambda_function=lambdas['lambda_lr'],
payload_response_only=True)
definition = sfn.Parallel(
scope=self,
id='Invoke Predictions').branch(invoke_lambda_rf).branch(
invoke_lambda_svr).branch(invoke_lambda_lr)
self.state_machine = sfn.StateMachine(
scope=self,
id='state_machine',
state_machine_name=name,
definition=definition,
role=state_machine_role,
state_machine_type=sfn.StateMachineType.EXPRESS)
def __init__(self, app: core.Construct, stack_name: str,
batch_job_definition: aws_batch.JobDefinition,
batch_job_queue: aws_batch.JobQueue):
super().__init__(scope=app, id=f"{stack_name}-invoke")
# ============= #
# StepFunctions #
# ============= #
# Ref::{keyword} can be replaced with StepFunction input
command_overrides = ["python", "__init__.py", "--time", "Ref::time"]
batch_task = aws_sfn_tasks.BatchSubmitJob(
scope=self,
id=f"sfn_batch_job",
job_definition=batch_job_definition,
job_name=f"sfn_batch_job",
job_queue=batch_job_queue,
container_overrides=aws_sfn_tasks.BatchContainerOverrides(
command=command_overrides),
payload=aws_sfn.TaskInput.from_object({"time.$": "$.time"}))
# `one step` for StepFunctions
definition = batch_task
sfn_daily_process = aws_sfn.StateMachine(scope=self,
id=f"step_functions",
definition=definition)
# ================ #
# CloudWatch Event #
# ================ #
# Run every day at 21:30 JST
# See https://docs.aws.amazon.com/lambda/latest/dg/tutorial-scheduled-events-schedule-expressions.html
events_daily_process = aws_events.Rule(
scope=self,
id=f"DailySFnProcess",
schedule=aws_events.Schedule.cron(minute="30",
hour="12",
month='*',
day="*",
year='*'),
)
events_daily_process.add_target(
aws_events_targets.SfnStateMachine(sfn_daily_process))
def _create_sfn_pipeline(self):
pipeline_name = "EMRSparkifyDWH"
create_cluster_task = self._emr_create_cluster_task(pipeline_name)
sample_spark_step_task = self._emr_spark_step_task()
terminate_cluster_task = self._emr_terminate_cluster_task()
pipeline = (create_cluster_task.next(sample_spark_step_task).next(
terminate_cluster_task).next(self.lambda_glue_crawler_task).next(
self.lambda_quality_check_task))
# Create & deploy StateMachine
machine = sfn.StateMachine(
self,
pipeline_name,
definition=pipeline,
role=self.sfn_role,
state_machine_name=f"{self.stack_name}-{pipeline_name}",
)
return machine
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
repo = codecommit.Repository(
self,
"repo",
repository_name="demorepo",
description="Repo to test PR with stepfunctions")
proj1 = self.new_build_project(repo, "pr_specs/buildspec.yaml",
"proj1")
proj2 = _codebuild.Project(
self,
"proj_name",
badge=True,
description="Build project for ",
environment=_codebuild.BuildEnvironment(
build_image=_codebuild.LinuxBuildImage.STANDARD_5_0,
compute_type=_codebuild.ComputeType.LARGE,
privileged=True),
project_name="proj_name",
build_spec=_codebuild.BuildSpec.from_source_filename(
filename="pr_specs/buildspec2.yaml"),
timeout=Duration.minutes(10),
)
input_task = _step_fn.Pass(self, "passstate")
proj1_tasks = self.new_codebuild_task(proj1)
proj2_tasks = self.new_codebuild_task(proj2)
definition = input_task.next(proj1_tasks).next(proj2_tasks)
_fn = _step_fn.StateMachine(
self,
"statemachine",
definition=definition,
state_machine_name="statemachine",
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
hello_world = aws_lambda.Function(
self,
"HelloWorld",
code=aws_lambda.Code.from_asset("./lambdas/hello_world"),
handler="function.handler.handler",
timeout=core.Duration.seconds(5),
runtime=aws_lambda.Runtime.PYTHON_3_8,
)
aws_ssm.StringParameter(
self,
"HelloWorldLambdaArn",
string_value=hello_world.function_arn,
parameter_name=f"/integration_tests/{id}/hello_world_lambda_arn",
)
hello_world_task = aws_stepfunctions_tasks.LambdaInvoke(
self,
"InvokeHelloWorld",
lambda_function=hello_world,
result_path="$.hello_message",
)
step_function = aws_stepfunctions.StateMachine(
self,
"Hello World Step Function",
definition=aws_stepfunctions.Chain.start(hello_world_task),
)
aws_ssm.StringParameter(
self,
"HelloWorldStepFunctionArn",
string_value=step_function.state_machine_arn,
parameter_name=f"/integration_tests/{id}/hello_world_step_function_arn",
)
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
detect_sentiment_task = sfn_tasks.CallAwsService(
self,
"DetectSentiment",
service="comprehend",
action="detectSentiment",
iam_resources=["*"],
parameters={
"Text": "$.text",
"LanguageCode": "en"
})
definition = detect_sentiment_task
state_machine = sfn.StateMachine(self,
"DetectSentimentStateMachine",
definition=definition,
timeout=Duration.minutes(5))
CfnOutput(scope=self,
id='StateMachineArn',
value=state_machine.state_machine_arn)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# space for feeder Lambda function
feeder = aws_lambda.Function(self,
id='_feeder',
code=aws_lambda.Code.asset('./code'),
handler='feeder.handler',
runtime=aws_lambda.Runtime.PYTHON_3_7,
description='Feeder function for the Witness project')
# space for saver Lambda function
saver = aws_lambda.Function(self,
id='_saver',
code=aws_lambda.Code.asset('./code'),
handler='saver.handler',
runtime=aws_lambda.Runtime.PYTHON_3_7,
description='Saver function for the Witness project')
# space for feeder lambda trigger
archive.add_event_notification(aws_s3.EventType.OBJECT_CREATED_PUT, s3n.LambdaDestination(feeder))
# space for stepfunction
feederTask = aws_stepfunctions.Task( self,
id='_feederTask',
task=aws_tasks.InvokeFunction(feeder))
saverTask = aws_stepfunctions.Task( self,
id='_saverTask',
task=aws_tasks.InvokeFunction(saver))
definition = feederTask.next(saverTask)
orchestrator = aws_stepfunctions.StateMachine(self,
id='_orchestrator',
state_machine_name='witness_orchestrator',
definition=definition)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
logging_lambda = lambda_func.Function(
scope=self,
id="logging_lambda",
function_name="logging_lambda",
handler="logging-lambda.main",
runtime=lambda_func.Runtime.PYTHON_3_7,
code=lambda_func.Code.from_asset("./code"))
second_lambda = lambda_func.Function(
scope=self,
id="second_lambda",
function_name="second_lambda",
handler="second-lambda.main",
runtime=lambda_func.Runtime.PYTHON_3_7,
code=lambda_func.Code.from_asset("./code"))
logging_lambda_task = tasks.InvokeFunction(logging_lambda)
logging_step = stepfunctions.Task(scope=self,
id="invoke_logging_function",
task=logging_lambda_task)
second_lambda_task = tasks.InvokeFunction(second_lambda)
second_step = stepfunctions.Task(scope=self,
id="invoke_second_function",
task=second_lambda_task)
definition = logging_step.next(second_step)
stepfunctions.StateMachine(
scope=self,
id="state_machine",
state_machine_name="state_machine",
definition=definition,
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The start of the image pipeline
imageBucket = aws_s3.Bucket(self, "imageBucket")
# Capture API activity with a trail
imageBucketTrail = aws_cloudtrail.Trail(self,
"imageBucketTrail",
is_multi_region_trail=False)
# Restrict to S3 data-plane events
imageBucketTrail.add_s3_event_selector(
include_management_events=False,
prefixes=[f"{imageBucket.bucket_arn}/"],
read_write_type=aws_cloudtrail.ReadWriteType.WRITE_ONLY)
# Filter to just PutObject and CopyObject events
imageBucketRule = aws_events.Rule(
self,
"imageBucketRule",
event_pattern={
"source": ["aws.s3"],
"detail": {
"eventSource": ["s3.amazonaws.com"],
"eventName": ["PutObject", "CopyObject"],
"requestParameters": {
"bucketName": [imageBucket.bucket_name]
}
}
})
#--
# Lambda Layers
#--------------------#
opencvLayer = aws_lambda.LayerVersion(
self,
'opencvLayer',
code=aws_lambda.AssetCode('layers/opencvLayer'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6])
boto3Layer = aws_lambda.LayerVersion(
self,
'boto3Layer',
code=aws_lambda.AssetCode('layers/boto3Layer'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6])
#--
# Lambda Functions
#--------------------#
# Gather info about an image, name, extension, etc
getImageInfoFunc = aws_lambda.Function(
self,
"getImageInfoFunc",
code=aws_lambda.AssetCode('functions/getImageInfoFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6)
# The home for the website
webBucket = aws_s3.Bucket(self,
"webBucket",
website_index_document='index.html')
# Copy the image to the web bucket
copyImageFunc = aws_lambda.Function(
self,
"copyImageFunc",
code=aws_lambda.AssetCode('functions/copyImageFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
layers=[boto3Layer],
environment={
'OUTPUTBUCKET': webBucket.bucket_name,
'OUTPUTPREFIX': 'images/'
})
# Grant permissions to read from the source and write to the desination
imageBucket.grant_read(copyImageFunc)
webBucket.grant_write(copyImageFunc)
# Create a thumbnail of the image and place in the web bucket
createThumbnailFunc = aws_lambda.Function(
self,
"createThumbnailFunc",
code=aws_lambda.AssetCode('functions/createThumbnailFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
layers=[boto3Layer, opencvLayer],
timeout=core.Duration.seconds(10),
memory_size=256,
environment={
'OUTPUTBUCKET': webBucket.bucket_name,
'OUTPUTPREFIX': 'images/'
})
# Grant permissions to read from the source and write to the desination
imageBucket.grant_read(createThumbnailFunc)
webBucket.grant_write(createThumbnailFunc)
# Store page information
pageTable = aws_dynamodb.Table(
self,
'pageTable',
partition_key={
'name': 'pageName',
'type': aws_dynamodb.AttributeType.STRING
},
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST,
stream=aws_dynamodb.StreamViewType.NEW_IMAGE)
# Save page and image information
updatePageInfoFunc = aws_lambda.Function(
self,
"updatePageInfoFunc",
code=aws_lambda.AssetCode('functions/updatePageInfoFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
layers=[boto3Layer],
environment={
'PAGETABLE': pageTable.table_name,
'PAGEPREFIX': 'posts/'
})
# Grant permissions to write to the page table
pageTable.grant_write_data(updatePageInfoFunc)
imagePipelineDone = aws_stepfunctions.Succeed(self,
"Done processing image")
updatePageInfoJob = aws_stepfunctions.Task(
self,
'Update page info',
task=aws_stepfunctions_tasks.InvokeFunction(updatePageInfoFunc))
updatePageInfoJob.next(imagePipelineDone)
copyImageJob = aws_stepfunctions.Task(
self,
'Copy image',
task=aws_stepfunctions_tasks.InvokeFunction(copyImageFunc))
createThumbnailJob = aws_stepfunctions.Task(
self,
'Create thumbnail',
task=aws_stepfunctions_tasks.InvokeFunction(createThumbnailFunc))
# These tasks can be done in parallel
processImage = aws_stepfunctions.Parallel(self,
'Process image',
result_path="$.images")
processImage.branch(copyImageJob)
processImage.branch(createThumbnailJob)
processImage.next(updatePageInfoJob)
# Results of file extension check
notPng = aws_stepfunctions.Succeed(self, "Not a PNG")
# Verify the file extension
checkForPng = aws_stepfunctions.Choice(self, 'Is a PNG?')
checkForPng.when(
aws_stepfunctions.Condition.string_equals('$.extension', 'png'),
processImage)
checkForPng.otherwise(notPng)
# A single image pipeline job for testing
getImageInfoJob = aws_stepfunctions.Task(
self,
'Get image info',
task=aws_stepfunctions_tasks.InvokeFunction(getImageInfoFunc))
getImageInfoJob.next(checkForPng)
# Configure the image pipeline and starting state
imagePipeline = aws_stepfunctions.StateMachine(
self, "imagePipeline", definition=getImageInfoJob)
# Matching events start the image pipline
imageBucketRule.add_target(
aws_events_targets.SfnStateMachine(
imagePipeline,
input=aws_events.RuleTargetInput.from_event_path(
"$.detail.requestParameters")))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
lambda_role = _iam.Role(
self,
id='lab3-om-role',
assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com'))
cloudwatch_policy_statement = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW)
cloudwatch_policy_statement.add_actions("logs:CreateLogGroup")
cloudwatch_policy_statement.add_actions("logs:CreateLogStream")
cloudwatch_policy_statement.add_actions("logs:PutLogEvents")
cloudwatch_policy_statement.add_actions("logs:DescribeLogStreams")
cloudwatch_policy_statement.add_resources("*")
lambda_role.add_to_policy(cloudwatch_policy_statement)
fn_lambda_approve_reject = aws_lambda.Function(
self,
"lab3-om-approve-reject",
code=aws_lambda.AssetCode(
"../lambda-functions/approve-reject-application/"),
handler="app.handler",
tracing=aws_lambda.Tracing.ACTIVE,
timeout=core.Duration.seconds(30),
role=lambda_role,
runtime=aws_lambda.Runtime.PYTHON_3_8)
fn_lambda_verify_identity = aws_lambda.Function(
self,
"lab3-om-verify-identity",
code=aws_lambda.AssetCode("../lambda-functions/verify-identity/"),
handler="app.handler",
tracing=aws_lambda.Tracing.ACTIVE,
timeout=core.Duration.seconds(30),
role=lambda_role,
runtime=aws_lambda.Runtime.PYTHON_3_8)
fn_lambda_check_address = aws_lambda.Function(
self,
"lab3-om-check-address",
code=aws_lambda.AssetCode("../lambda-functions/check-address/"),
handler="app.handler",
tracing=aws_lambda.Tracing.ACTIVE,
timeout=core.Duration.seconds(30),
role=lambda_role,
runtime=aws_lambda.Runtime.PYTHON_3_8)
'''
[INFO] This is a sample how to define the task and integrate with Lambda Functions. You need to create another 2 tasks for respective Lambda functions
'''
task_verify_identity = _tasks.LambdaInvoke(
self,
"Verify Identity Document",
lambda_function=fn_lambda_verify_identity,
output_path="$.Payload")
task_check_address = _tasks.LambdaInvoke(
self,
"Check Address",
lambda_function=fn_lambda_check_address,
output_path="$.Payload")
task_wait_review = _tasks.LambdaInvoke(
self,
"Wait for Review",
lambda_function=fn_lambda_approve_reject,
output_path="$.Payload")
state_approve = _sfn.Succeed(self, "Approve Application")
state_reject = _sfn.Succeed(self, "Reject Application")
# Let's define the State Machine, step by step
# First, paralell tasks for verification
s_verification = _sfn.Parallel(self, "Verification")
s_verification.branch(task_verify_identity)
s_verification.branch(task_check_address)
# Next, we add a choice state
c_human_review = _sfn.Choice(self, "Human review required?")
c_human_review.when(
_sfn.Condition.and_(
_sfn.Condition.boolean_equals("$[0].humanReviewRequired",
False),
_sfn.Condition.boolean_equals("$[1].humanReviewRequired",
True)), state_approve)
c_human_review.when(
_sfn.Condition.or_(
_sfn.Condition.boolean_equals("$[0].humanReviewRequired",
True),
_sfn.Condition.boolean_equals("$[1].humanReviewRequired",
False)), task_wait_review)
# Another choice state to check if the application passed the review
c_review_approved = _sfn.Choice(self, "Review approved?")
c_review_approved.when(
_sfn.Condition.boolean_equals("$.reviewApproved", True),
state_approve)
c_review_approved.when(
_sfn.Condition.boolean_equals("$.reviewApproved", False),
state_reject)
task_wait_review.next(c_review_approved)
definition = s_verification.next(c_human_review)
_sfn.StateMachine(self,
"lab3-statemachine",
definition=definition,
timeout=core.Duration.minutes(5))
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
test_queue = sqs.Queue(self, 'test-queue', queue_name='test1')
test_topic = sns.Topic(self, 'test-topic')
sns.Subscription(self,
'test-subscription',
topic=test_topic,
endpoint=test_queue.queue_arn,
protocol=sns.SubscriptionProtocol.SQS)
kinesis.Stream(self,
'test-stream',
stream_name='donut-sales',
shard_count=2)
create_order = step.Pass(self,
'create-order',
result=step.Result.from_object({
"Order": {
"Customer": "Alice",
"Product": "Coffee",
"Billing": {
"Price": 10.0,
"Quantity": 4.0
}
}
}))
calculate_amount = step.Pass(self,
'calculate-amount',
result=step.Result.from_number(40.0),
result_path='$.Order.Billing.Amount',
output_path='$.Order.Billing')
order_definition = create_order.next(calculate_amount)
step.StateMachine(self,
'test-state-machine',
state_machine_name='order-machine',
definition=order_definition)
make_tea = step.Choice(
self, 'make-tea', comment='Input should look like {"tea":"green"}')
green = step.Pass(self,
'green',
result=step.Result.from_string('Green tea'))
make_tea.when(step.Condition.string_equals('$.tea', 'green'), green)
black = step.Pass(self,
'black',
result=step.Result.from_string('Black tea'))
make_tea.when(step.Condition.string_equals('$.tea', 'black'), black)
orange = step.Pass(self,
'orange',
result=step.Result.from_string('Black tea'))
make_tea.when(step.Condition.string_equals('$.tea', 'orange'), orange)
error = step.Pass(self,
'error',
result=step.Result.from_string('Bad input'))
make_tea.otherwise(error)
step.StateMachine(self,
'test-state-machine-2',
state_machine_name='tea-machine',
definition=make_tea)
def __init__(
self,
scope: Construct,
stack_id: str,
*,
botocore_lambda_layer: aws_lambda_python.PythonLayerVersion,
env_name: str,
storage_bucket: aws_s3.Bucket,
validation_results_table: Table,
) -> None:
# pylint: disable=too-many-locals, too-many-statements
super().__init__(scope, stack_id)
############################################################################################
# PROCESSING ASSETS TABLE
processing_assets_table = Table(
self,
f"{env_name}-processing-assets",
env_name=env_name,
parameter_name=ParameterName.PROCESSING_ASSETS_TABLE_NAME,
sort_key=aws_dynamodb.Attribute(name="sk", type=aws_dynamodb.AttributeType.STRING),
)
############################################################################################
# BATCH JOB DEPENDENCIES
batch_job_queue = BatchJobQueue(
self,
"batch-job-queue",
env_name=env_name,
processing_assets_table=processing_assets_table,
).job_queue
s3_read_only_access_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3ReadOnlyAccess"
)
############################################################################################
# UPDATE CATALOG UPDATE MESSAGE QUEUE
dead_letter_queue = aws_sqs.Queue(
self,
"dead-letter-queue",
visibility_timeout=LAMBDA_TIMEOUT,
)
self.message_queue = aws_sqs.Queue(
self,
"update-catalog-message-queue",
visibility_timeout=LAMBDA_TIMEOUT,
dead_letter_queue=aws_sqs.DeadLetterQueue(max_receive_count=3, queue=dead_letter_queue),
)
self.message_queue_name_parameter = aws_ssm.StringParameter(
self,
"update-catalog-message-queue-name",
string_value=self.message_queue.queue_name,
description=f"Update Catalog Message Queue Name for {env_name}",
parameter_name=ParameterName.UPDATE_CATALOG_MESSAGE_QUEUE_NAME.value,
)
populate_catalog_lambda = BundledLambdaFunction(
self,
"populate-catalog-bundled-lambda-function",
directory="populate_catalog",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
botocore_lambda_layer=botocore_lambda_layer,
)
self.message_queue.grant_consume_messages(populate_catalog_lambda)
populate_catalog_lambda.add_event_source(
SqsEventSource(self.message_queue, batch_size=1) # type: ignore[arg-type]
)
############################################################################################
# STATE MACHINE TASKS
check_stac_metadata_task = LambdaTask(
self,
"check-stac-metadata-task",
directory="check_stac_metadata",
botocore_lambda_layer=botocore_lambda_layer,
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
assert check_stac_metadata_task.lambda_function.role
check_stac_metadata_task.lambda_function.role.add_managed_policy(
policy=s3_read_only_access_policy
)
for table in [processing_assets_table, validation_results_table]:
table.grant_read_write_data(check_stac_metadata_task.lambda_function)
table.grant(
check_stac_metadata_task.lambda_function,
"dynamodb:DescribeTable",
)
content_iterator_task = LambdaTask(
self,
"content-iterator-task",
directory="content_iterator",
botocore_lambda_layer=botocore_lambda_layer,
result_path=f"$.{CONTENT_KEY}",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
check_files_checksums_directory = "check_files_checksums"
check_files_checksums_default_payload_object = {
f"{DATASET_ID_KEY}.$": f"$.{DATASET_ID_KEY}",
f"{VERSION_ID_KEY}.$": f"$.{VERSION_ID_KEY}",
f"{METADATA_URL_KEY}.$": f"$.{METADATA_URL_KEY}",
f"{FIRST_ITEM_KEY}.$": f"$.{CONTENT_KEY}.{FIRST_ITEM_KEY}",
f"{ASSETS_TABLE_NAME_KEY}.$": f"$.{CONTENT_KEY}.{ASSETS_TABLE_NAME_KEY}",
f"{RESULTS_TABLE_NAME_KEY}.$": f"$.{CONTENT_KEY}.{RESULTS_TABLE_NAME_KEY}",
}
check_files_checksums_single_task = BatchSubmitJobTask(
self,
"check-files-checksums-single-task",
env_name=env_name,
directory=check_files_checksums_directory,
s3_policy=s3_read_only_access_policy,
job_queue=batch_job_queue,
payload_object=check_files_checksums_default_payload_object,
container_overrides_command=[
"--dataset-id",
f"Ref::{DATASET_ID_KEY}",
"--version-id",
f"Ref::{VERSION_ID_KEY}",
"--first-item",
f"Ref::{FIRST_ITEM_KEY}",
"--assets-table-name",
f"Ref::{ASSETS_TABLE_NAME_KEY}",
"--results-table-name",
f"Ref::{RESULTS_TABLE_NAME_KEY}",
],
)
array_size = int(
aws_stepfunctions.JsonPath.number_at(f"$.{CONTENT_KEY}.{ITERATION_SIZE_KEY}")
)
check_files_checksums_array_task = BatchSubmitJobTask(
self,
"check-files-checksums-array-task",
env_name=env_name,
directory=check_files_checksums_directory,
s3_policy=s3_read_only_access_policy,
job_queue=batch_job_queue,
payload_object=check_files_checksums_default_payload_object,
container_overrides_command=[
"--dataset-id",
f"Ref::{DATASET_ID_KEY}",
"--version-id",
f"Ref::{VERSION_ID_KEY}",
"--first-item",
f"Ref::{FIRST_ITEM_KEY}",
"--assets-table-name",
f"Ref::{ASSETS_TABLE_NAME_KEY}",
"--results-table-name",
f"Ref::{RESULTS_TABLE_NAME_KEY}",
],
array_size=array_size,
)
for reader in [
content_iterator_task.lambda_function,
check_files_checksums_single_task.job_role,
check_files_checksums_array_task.job_role,
]:
processing_assets_table.grant_read_data(reader) # type: ignore[arg-type]
processing_assets_table.grant(
reader, "dynamodb:DescribeTable" # type: ignore[arg-type]
)
for writer in [
check_files_checksums_single_task.job_role,
check_files_checksums_array_task.job_role,
]:
validation_results_table.grant_read_write_data(writer) # type: ignore[arg-type]
validation_results_table.grant(
writer, "dynamodb:DescribeTable" # type: ignore[arg-type]
)
validation_summary_task = LambdaTask(
self,
"validation-summary-task",
directory="validation_summary",
botocore_lambda_layer=botocore_lambda_layer,
result_path=f"$.{VALIDATION_KEY}",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
validation_results_table.grant_read_data(validation_summary_task.lambda_function)
validation_results_table.grant(
validation_summary_task.lambda_function, "dynamodb:DescribeTable"
)
import_dataset_role = aws_iam.Role(
self,
"import-dataset",
assumed_by=aws_iam.ServicePrincipal( # type: ignore[arg-type]
"batchoperations.s3.amazonaws.com"
),
)
import_asset_file_function = ImportFileFunction(
self,
directory="import_asset_file",
invoker=import_dataset_role,
env_name=env_name,
botocore_lambda_layer=botocore_lambda_layer,
)
import_metadata_file_function = ImportFileFunction(
self,
directory="import_metadata_file",
invoker=import_dataset_role,
env_name=env_name,
botocore_lambda_layer=botocore_lambda_layer,
)
import_dataset_task = LambdaTask(
self,
"import-dataset-task",
directory="import_dataset",
botocore_lambda_layer=botocore_lambda_layer,
result_path=f"$.{IMPORT_DATASET_KEY}",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
import_dataset_task.lambda_function.add_to_role_policy(
aws_iam.PolicyStatement(
resources=[import_dataset_role.role_arn],
actions=["iam:PassRole"],
),
)
import_dataset_task.lambda_function.add_to_role_policy(
aws_iam.PolicyStatement(resources=["*"], actions=["s3:CreateJob"])
)
for table in [processing_assets_table]:
table.grant_read_data(import_dataset_task.lambda_function)
table.grant(import_dataset_task.lambda_function, "dynamodb:DescribeTable")
# Import status check
wait_before_upload_status_check = Wait(
self,
"wait-before-upload-status-check",
time=WaitTime.duration(Duration.seconds(10)),
)
upload_status_task = LambdaTask(
self,
"upload-status",
directory="upload_status",
botocore_lambda_layer=botocore_lambda_layer,
result_path="$.upload_status",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
validation_results_table.grant_read_data(upload_status_task.lambda_function)
validation_results_table.grant(upload_status_task.lambda_function, "dynamodb:DescribeTable")
upload_status_task.lambda_function.add_to_role_policy(ALLOW_DESCRIBE_ANY_S3_JOB)
# Parameters
import_asset_file_function_arn_parameter = aws_ssm.StringParameter(
self,
"import asset file function arn",
string_value=import_asset_file_function.function_arn,
description=f"Import asset file function ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_IMPORT_ASSET_FILE_FUNCTION_TASK_ARN.value,
)
import_metadata_file_function_arn_parameter = aws_ssm.StringParameter(
self,
"import metadata file function arn",
string_value=import_metadata_file_function.function_arn,
description=f"Import metadata file function ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_IMPORT_METADATA_FILE_FUNCTION_TASK_ARN.value,
)
import_dataset_role_arn_parameter = aws_ssm.StringParameter(
self,
"import dataset role arn",
string_value=import_dataset_role.role_arn,
description=f"Import dataset role ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_IMPORT_DATASET_ROLE_ARN.value,
)
update_dataset_catalog = LambdaTask(
self,
"update-dataset-catalog",
directory="update_dataset_catalog",
botocore_lambda_layer=botocore_lambda_layer,
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
self.message_queue.grant_send_messages(update_dataset_catalog.lambda_function)
for storage_writer in [
import_dataset_role,
import_dataset_task.lambda_function,
import_asset_file_function,
import_metadata_file_function,
populate_catalog_lambda,
update_dataset_catalog.lambda_function,
]:
storage_bucket.grant_read_write(storage_writer) # type: ignore[arg-type]
grant_parameter_read_access(
{
import_asset_file_function_arn_parameter: [import_dataset_task.lambda_function],
import_dataset_role_arn_parameter: [import_dataset_task.lambda_function],
import_metadata_file_function_arn_parameter: [import_dataset_task.lambda_function],
processing_assets_table.name_parameter: [
check_stac_metadata_task.lambda_function,
content_iterator_task.lambda_function,
import_dataset_task.lambda_function,
],
validation_results_table.name_parameter: [
check_stac_metadata_task.lambda_function,
content_iterator_task.lambda_function,
validation_summary_task.lambda_function,
upload_status_task.lambda_function,
],
self.message_queue_name_parameter: [update_dataset_catalog.lambda_function],
}
)
success_task = aws_stepfunctions.Succeed(self, "success")
upload_failure = aws_stepfunctions.Fail(self, "upload failure")
validation_failure = aws_stepfunctions.Succeed(self, "validation failure")
############################################################################################
# STATE MACHINE
dataset_version_creation_definition = (
check_stac_metadata_task.next(content_iterator_task)
.next(
aws_stepfunctions.Choice( # type: ignore[arg-type]
self, "check_files_checksums_maybe_array"
)
.when(
aws_stepfunctions.Condition.number_equals(
f"$.{CONTENT_KEY}.{ITERATION_SIZE_KEY}", 1
),
check_files_checksums_single_task.batch_submit_job,
)
.otherwise(check_files_checksums_array_task.batch_submit_job)
.afterwards()
)
.next(
aws_stepfunctions.Choice(self, "content_iteration_finished")
.when(
aws_stepfunctions.Condition.number_equals(
f"$.{CONTENT_KEY}.{NEXT_ITEM_KEY}", -1
),
validation_summary_task.next(
aws_stepfunctions.Choice( # type: ignore[arg-type]
self, "validation_successful"
)
.when(
aws_stepfunctions.Condition.boolean_equals(
f"$.{VALIDATION_KEY}.{SUCCESS_KEY}", True
),
import_dataset_task.next(
wait_before_upload_status_check # type: ignore[arg-type]
)
.next(upload_status_task)
.next(
aws_stepfunctions.Choice(
self, "import_completed" # type: ignore[arg-type]
)
.when(
aws_stepfunctions.Condition.and_(
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Complete"
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{METADATA_UPLOAD_KEY}.status",
"Complete",
),
),
update_dataset_catalog.next(
success_task # type: ignore[arg-type]
),
)
.when(
aws_stepfunctions.Condition.or_(
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{ASSET_UPLOAD_KEY}.status",
"Cancelled",
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Failed"
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{METADATA_UPLOAD_KEY}.status",
"Cancelled",
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{METADATA_UPLOAD_KEY}.status",
"Failed",
),
),
upload_failure, # type: ignore[arg-type]
)
.otherwise(
wait_before_upload_status_check # type: ignore[arg-type]
)
),
)
.otherwise(validation_failure) # type: ignore[arg-type]
),
)
.otherwise(content_iterator_task)
)
)
self.state_machine = aws_stepfunctions.StateMachine(
self,
f"{env_name}-dataset-version-creation",
definition=dataset_version_creation_definition, # type: ignore[arg-type]
)
self.state_machine_parameter = aws_ssm.StringParameter(
self,
"state machine arn",
description=f"State machine ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_DATASET_VERSION_CREATION_STEP_FUNCTION_ARN.value, # pylint:disable=line-too-long
string_value=self.state_machine.state_machine_arn,
)
Tags.of(self).add("ApplicationLayer", "processing") # type: ignore[arg-type]
def __init__(self, scope: Construct, construct_id: str, env,
**kwargs) -> None:
super().__init__(scope, construct_id, env=env, **kwargs)
rg_property = network_fw.CfnRuleGroup.RuleGroupProperty(
rule_variables=None,
rules_source=network_fw.CfnRuleGroup.RulesSourceProperty(
stateless_rules_and_custom_actions=network_fw.CfnRuleGroup.
StatelessRulesAndCustomActionsProperty(stateless_rules=[
network_fw.CfnRuleGroup.StatelessRuleProperty(
priority=10,
rule_definition=network_fw.CfnRuleGroup.
RuleDefinitionProperty(
actions=["aws:drop"],
match_attributes=network_fw.CfnRuleGroup.
MatchAttributesProperty(destinations=[
network_fw.CfnRuleGroup.AddressProperty(
address_definition="127.0.0.1/32")
])))
])))
nf_rule_group = network_fw.CfnRuleGroup(
scope=self,
id='GuardDutyNetworkFireWallRuleGroup',
capacity=100,
rule_group_name='guardduty-network-firewall',
type='STATELESS',
description='Guard Duty network firewall rule group',
tags=[CfnTag(key='Name', value='cfn.rule-group.stack')],
rule_group=rg_property)
""" https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-rule-dlq.html#dlq-considerations """
dlq_statemachine = sqs.Queue(self,
'DLQStateMachine',
queue_name='dlq_state_machine')
guardduty_firewall_ddb = ddb.Table(
scope=self,
id=f'GuarddutyFirewallDDB',
table_name='GuardDutyFirewallDDBTable',
removal_policy=RemovalPolicy.DESTROY,
partition_key=ddb.Attribute(name='HostIp',
type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST)
""" IAM role for ddb permission """
nf_iam_role = iam.Role(
self,
'DDBRole',
role_name=f'ddb-nf-role-{env.region}',
assumed_by=iam.ServicePrincipal(service='lambda.amazonaws.com'))
nf_iam_role.add_to_policy(
iam.PolicyStatement(effect=iam.Effect.ALLOW,
resources=["arn:aws:logs:*:*:*"],
actions=[
"logs:CreateLogGroup",
"logs:CreateLogStream", "logs:PutLogEvents"
]))
nf_iam_role.add_to_policy(
iam.PolicyStatement(effect=iam.Effect.ALLOW,
resources=[
guardduty_firewall_ddb.table_arn,
f"{guardduty_firewall_ddb.table_arn}/*"
],
actions=[
"dynamodb:PutItem", "dynamodb:GetItem",
"dynamodb:Scan"
]))
nf_iam_role.add_to_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
resources=[nf_rule_group.ref, f"{nf_rule_group.ref}/*"],
actions=[
"network-firewall:DescribeRuleGroup",
"network-firewall:UpdateRuleGroup"
]))
record_ip_in_db = _lambda.Function(
self,
'RecordIpInDB',
function_name='record-ip-in-ddb',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset('lambda_fns'),
handler='addIPToDDB.handler',
environment=dict(ACLMETATABLE=guardduty_firewall_ddb.table_name),
role=nf_iam_role)
"""
https://docs.amazonaws.cn/en_us/eventbridge/latest/userguide/eb-event-patterns-content-based-filtering.html
"""
record_ip_task = step_fn_task.LambdaInvoke(
self,
'RecordIpDDBTask',
lambda_function=record_ip_in_db,
payload=step_fn.TaskInput.from_object({
"comment":
"Relevant fields from the GuardDuty / Security Hub finding",
"HostIp.$":
"$.detail.findings[0].ProductFields.aws/guardduty/service/action/networkConnectionAction/remoteIpDetails/ipAddressV4",
"Timestamp.$":
"$.detail.findings[0].ProductFields.aws/guardduty/service/eventLastSeen",
"FindingId.$": "$.id",
"AccountId.$": "$.account",
"Region.$": "$.region"
}),
result_path='$',
payload_response_only=True)
firewall_update_rule = _lambda.Function(
scope=self,
id='GuardDutyUpdateNetworkFirewallRule',
function_name='gurdduty-update-networkfirewal-rule-group',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset('lambda_fns'),
handler='updateNetworkFireWall.handler',
environment=dict(
FIREWALLRULEGROUP=nf_rule_group.ref,
RULEGROUPPRI='30000',
CUSTOMACTIONNAME='GuardDutytoFirewall',
CUSTOMACTIONVALUE='gurdduty-update-networkfirewal-rule-group'),
role=nf_iam_role)
firewall_update_rule_task = step_fn_task.LambdaInvoke(
self,
'FirewallUpdateRuleTask',
lambda_function=firewall_update_rule,
input_path='$',
result_path='$',
payload_response_only=True)
firewall_no_update_job = step_fn.Pass(self, 'No Firewall change')
notify_failure_job = step_fn.Fail(self,
'NotifyFailureJob',
cause='Any Failure',
error='Unknown')
send_to_slack = _lambda.Function(
scope=self,
id='SendAlertToSlack',
function_name='gurdduty-networkfirewal-to-slack',
runtime=_lambda.Runtime.PYTHON_3_8,
handler="sendSMSToSlack.handler",
code=_lambda.Code.from_asset('lambda_fns'))
send_slack_task = step_fn_task.LambdaInvoke(
scope=self,
id='LambdaToSlackDemo',
lambda_function=send_to_slack,
input_path='$',
result_path='$')
is_new_ip = step_fn.Choice(self, "New IP?")
is_block_succeed = step_fn.Choice(self, "Block sucessfully?")
definition = step_fn.Chain \
.start(record_ip_task
.add_retry(errors=["States.TaskFailed"],
interval=Duration.seconds(2),
max_attempts=2)
.add_catch(errors=["States.ALL"], handler=notify_failure_job)) \
.next(is_new_ip
.when(step_fn.Condition.boolean_equals('$.NewIP', True),
firewall_update_rule_task
.add_retry(errors=["States.TaskFailed"],
interval=Duration.seconds(2),
max_attempts=2
)
.add_catch(errors=["States.ALL"], handler=notify_failure_job)
.next(
is_block_succeed
.when(step_fn.Condition.boolean_equals('$.Result', False), notify_failure_job)
.otherwise(send_slack_task)
)
)
.otherwise(firewall_no_update_job)
)
guardduty_state_machine = step_fn.StateMachine(
self,
'GuarddutyStateMachine',
definition=definition,
timeout=Duration.minutes(5),
state_machine_name='guardduty-state-machine')
event.Rule(
scope=self,
id='EventBridgeCatchIPv4',
description="Security Hub - GuardDuty findings with remote IP",
rule_name='guardduty-catch-ipv4',
event_pattern=event.EventPattern(
account=['123456789012'],
detail_type=["GuardDuty Finding"],
source=['aws.securityhub'],
detail={
"findings": {
"ProductFields": {
"aws/guardduty/service/action/networkConnectionAction/remoteIpDetails/ipAddressV4":
[{
"exists": True
}]
}
}
}),
targets=[
event_target.SfnStateMachine(
machine=guardduty_state_machine,
dead_letter_queue=dlq_statemachine)
])
""" Send other findings to slack """
send_finding_to_slack = _lambda.Function(
self,
'SendFindingToSlack',
function_name='send-finding-to-slack',
runtime=_lambda.Runtime.PYTHON_3_8,
handler="sendFindingToSlack.handler",
code=_lambda.Code.from_asset('lambda_fns'))
send_findings_task = step_fn_task.LambdaInvoke(
self,
'SendFindingToSlackTask',
lambda_function=send_finding_to_slack,
payload=step_fn.TaskInput.from_object({
"comment":
"Others fields from the GuardDuty / Security Hub finding",
"severity.$":
"$.detail.findings[0].Severity.Label",
"Account_ID.$":
"$.account",
"Finding_ID.$":
"$.id",
"Finding_Type.$":
"$.detail.findings[0].Types",
"Region.$":
"$.region",
"Finding_description.$":
"$.detail.findings[0].Description"
}),
result_path='$')
slack_failure_job = step_fn.Fail(self,
'SlackNotifyFailureJob',
cause='Any Failure',
error='Unknown')
finding_definition = step_fn.Chain \
.start(send_findings_task
.add_retry(errors=["States.TaskFailed"],
interval=Duration.seconds(2),
max_attempts=2)
.add_catch(errors=["States.ALL"], handler=slack_failure_job))
sechub_findings_state_machine = step_fn.StateMachine(
self,
'SecHubFindingsStateMachine',
definition=finding_definition,
timeout=Duration.minutes(5),
state_machine_name='sechub-finding-state-machine')
event.Rule(scope=self,
id='EventBridgeFindings',
description="Security Hub - GuardDuty findings others",
rule_name='others-findings',
event_pattern=event.EventPattern(
account=['123456789012'],
source=['aws.securityhub'],
detail_type=['Security Hub Findings - Imported'],
detail={"severity": [5, 8]}),
targets=[
event_target.SfnStateMachine(
machine=sechub_findings_state_machine,
dead_letter_queue=dlq_statemachine)
])