def createTipWorkflow(self):
notifyTipper = tasks.LambdaInvoke(self, 'notifyTipper',
lambda_function=self.createLambda('notifyTipperLambda', 'tipNotifier.tipNotifier'),
timeout=cdk.Duration.seconds(300)
).next(sfn.Succeed(self, 'withdrawSuccessState'))
self.getTipperInvoice = tasks.StepFunctionsInvokeActivity(self, 'getTipperInvoice',
activity=sfn.Activity(self, 'getTipperInvoiceActivity'),
heartbeat=cdk.Duration.seconds(60),
timeout=cdk.Duration.seconds(86400),
)
self.getTipperInvoice.add_retry(
backoff_rate=1.5,
errors=['States.Timeout'],
interval=cdk.Duration.seconds(60),
max_attempts=7
)
self.getTipperInvoice.add_catch(
handler=sfn.Fail(self, 'withdrawErrorState'),
errors=['States.ALL'],
result_path='$.errorInfo'
)
self.getTipperInvoice.next(notifyTipper)
return sfn.StateMachine(self, 'tipWorkflow',
definition=self.getTipperInvoice,
role=self.statesRole
)
def __init__(self,
scope: core.Construct,
id: str,
*,
message: sfn.TaskInput,
subject: Optional[str] = None,
topic: Optional[sns.Topic] = None,
result_path: str = '$.PublishResult',
output_path: Optional[str] = None):
super().__init__(scope, id)
self._end = sfn.Succeed(self, 'Succeeded', output_path=output_path)
if topic is not None:
self._start = sfn_tasks.SnsPublish(
self,
'Success Notification',
input_path='$',
output_path='$',
result_path=result_path,
topic=topic,
message=message,
subject=subject,
)
self._start.next(self._end)
else:
self._start = self._end
def createWithdrawWorkflow(self):
payInvoiceFailed = tasks.LambdaInvoke(self, 'payInvoiceFailed',
lambda_function=self.createLambda('payInvoiceFailedLambda', 'payInvoiceFailed.payInvoiceFailed'),
timeout=cdk.Duration.seconds(300)
).next(sfn.Fail(self, 'tipErrorState'))
payInvoiceSucceeded = tasks.LambdaInvoke(self, 'payInvoiceSucceeded',
lambda_function=self.createLambda('payInvoiceSucceededLambda', 'payInvoiceSucceeded.payInvoiceSucceeded'),
timeout=cdk.Duration.seconds(300)
).next(sfn.Succeed(self, 'tipSuccessState'))
self.payInvoice = tasks.StepFunctionsInvokeActivity(self, 'payInvoice',
activity=sfn.Activity(self, 'payInvoiceActivity'),
heartbeat=cdk.Duration.seconds(86400),
timeout=cdk.Duration.seconds(86400),
)
self.payInvoice.add_retry(
backoff_rate=2,
errors=['States.Timeout'],
interval=cdk.Duration.seconds(600),
max_attempts=0
)
self.payInvoice.add_catch(
handler=payInvoiceFailed,
errors=['States.ALL'],
result_path='$.errorInfo'
)
self.payInvoice.next(payInvoiceSucceeded)
return sfn.StateMachine(self, 'withdrawWorkflow',
definition=self.payInvoice,
role=self.statesRole
)
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
flip_coin_function = lambda_.Function(
self,
"FlipCoinFunction",
runtime=lambda_.Runtime.PYTHON_3_8,
handler="index.handler",
code=lambda_.Code.from_asset("./sfn/lambda/flip_coin"))
flip_coin_invoke = tasks.LambdaInvoke(
self, "FlipCoin", lambda_function=flip_coin_function)
wait = stepfunctions.Wait(self,
"Wait",
time=stepfunctions.WaitTime.duration(
core.Duration.seconds(5)))
tails_result = stepfunctions.Pass(self, "TailsResult")
tails_result.next(flip_coin_invoke)
choice = stepfunctions.Choice(self,
"HeadsTailsChoice") \
.when(condition=stepfunctions.Condition.string_equals("$.Payload.result", "heads"),
next=stepfunctions.Succeed(self, "HeadsResult")) \
.when(condition=stepfunctions.Condition.string_equals("$.Payload.result", "tails"),
next=tails_result)
stepfunctions.StateMachine(self,
"StateMachine",
definition=flip_coin_invoke.next(
wait.next(choice)))
def test_start_execution_task():
default_task_json = {
'End': True,
'Parameters': {
'StateMachineArn': {
'Ref': 'teststatemachine7F4C511D'
},
'Input.$': '$$.Execution.Input'
},
'Type': 'Task',
'Resource': {
'Fn::Join': [
'',
[
'arn:', {
'Ref': 'AWS::Partition'
}, ':states:::states:startExecution.sync'
]
]
}
}
stack = core.Stack(core.App(), 'test-stack')
state_machine = sfn.StateMachine(stack,
'test-state-machine',
definition=sfn.Chain.start(
sfn.Succeed(stack, 'Succeeded')))
task = sfn.Task(stack,
'test-task',
task=emr_tasks.StartExecutionTask(state_machine, ))
print_and_assert(default_task_json, task)
def __init__(self,
scope: core.Construct,
id: str,
*,
polling_delay: int = 5,
statemachine_timeout: int = 300,
**kwargs):
super().__init__(scope, id, **kwargs)
state_fn = StateHandlerLambda(self, "config-state-handler").function
config_fn = AccountConfigLambda(self,
"account-config-handler").function
config_state = tasks.LambdaInvoke(self,
"Set Configuring State",
lambda_function=state_fn,
output_path="$.Payload")
completed_state = tasks.LambdaInvoke(self,
"Set Completed State",
lambda_function=state_fn,
output_path="$.Payload")
config_task = tasks.LambdaInvoke(self,
"Request Account Configuration",
lambda_function=config_fn,
output_path="$.Payload")
polling_task = tasks.LambdaInvoke(self,
"Poll Account Configuration",
lambda_function=config_fn,
output_path="$.Payload")
delay = sfn.Wait(self,
"Delay Polling",
time=sfn.WaitTime.duration(
core.Duration.seconds(polling_delay)))
is_ready = sfn.Choice(self, "Account Ready?")
acct_ready = sfn.Condition.string_equals('$.state', "READY")
acct_pending = sfn.Condition.string_equals('$.state', "PENDING")
success = sfn.Succeed(self, "Config Succeeded")
failed = sfn.Fail(self,
"Config Failed",
cause="Bad value in Polling loop")
# this is the loop which polls for state change, either looping back to delay or setting completion state and finishing
is_ready.when(acct_pending, delay).when(
acct_ready, completed_state.next(success)).otherwise(failed)
# this is the main chain starting with creation request a delay and then polling loop
config_chain = config_task.next(config_state).next(delay).next(
polling_task).next(is_ready)
self.state_machine = sfn.StateMachine(
self,
"Account-Config-StateMachine",
definition=config_chain,
timeout=core.Duration.seconds(statemachine_timeout))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
api = apigateway.RestApi(
scope=self,
id=f'{constants.PREFIX}-approval-api',
rest_api_name='Human approval endpoint',
description='HTTP Endpoint backed by API Gateway and Lambda',
endpoint_types=[apigateway.EndpointType.REGIONAL],
)
v1 = api.root.add_resource("v1")
approve_api = v1.add_resource("approve")
#################################################
email_topic = sns.Topic(
scope=self,
id=f'{constants.PREFIX}-email-topic',
)
email_topic.add_subscription(
subscription=subscriptions.EmailSubscription(
email_address=constants.EMAIL_APPROVER, ))
#################################################
submit_job_lambda = _lambda.Function(
scope=self,
id=f'{constants.PREFIX}-submit-lambda',
runtime=_lambda.Runtime.PYTHON_3_8,
handler='submit.handler',
environment={
"TOPIC_ARN": email_topic.topic_arn,
"END_POINT": approve_api.url,
"TO_ADDRESS": constants.EMAIL_RECIPIENT,
"FROM_ADDRESS": constants.EMAIL_SENDER,
},
code=_lambda.Code.from_asset(
os.path.join('lambdas', 'submit-lambda')),
)
email_topic.grant_publish(submit_job_lambda)
submit_job_lambda.add_to_role_policy(statement=iam.PolicyStatement(
actions=['ses:Send*'],
resources=['*'],
))
submit_job = tasks.LambdaInvoke(
scope=self,
id=f'{constants.PREFIX}-submit-job',
lambda_function=submit_job_lambda,
integration_pattern=sfn.IntegrationPattern.WAIT_FOR_TASK_TOKEN,
heartbeat=core.Duration.minutes(5),
payload=sfn.TaskInput.from_object({
"token": sfn.Context.task_token,
"data": sfn.Data.string_at('$'),
}),
)
success = sfn.Succeed(scope=self,
id=f'{constants.PREFIX}-success',
comment='We did it!')
fail = sfn.Fail(scope=self,
id=f'{constants.PREFIX}-fail',
error='WorkflowFailure',
cause='Something went wrong')
choice = sfn.Choice(scope=self,
id=f'{constants.PREFIX}-choice',
comment='Was it approved?')
choice.when(condition=sfn.Condition.string_equals("$.status", "OK"),
next=success)
choice.otherwise(fail)
definition = submit_job.next(choice)
self._state_machine = sfn.StateMachine(
scope=self,
id=f'{constants.PREFIX}-state-machine',
definition=definition,
# only 10 mins to approve better be quick
timeout=core.Duration.minutes(10))
#################################################
approval_lambda = _lambda.Function(
scope=self,
id=f'{constants.PREFIX}-approval-lambda',
runtime=_lambda.Runtime.PYTHON_3_8,
handler='approve.handler',
code=_lambda.Code.from_asset(
os.path.join('lambdas', 'approve-lambda')),
)
approval_lambda.add_to_role_policy(statement=iam.PolicyStatement(
actions=['states:Send*'], resources=['*']))
approve_integration = apigateway.LambdaIntegration(approval_lambda)
approve_api_get_method = approve_api.add_method(
http_method="GET",
api_key_required=False,
integration=approve_integration,
)
def __init__(self, app: App, id: str, **kwargs) -> None:
super().__init__(app, id, **kwargs)
# Lambda Handlers Definitions
submit_lambda = _lambda.Function(
self,
'submitLambda',
handler='lambda_function.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_9,
code=_lambda.Code.from_asset('lambdas/submit'))
status_lambda = _lambda.Function(
self,
'statusLambda',
handler='lambda_function.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_9,
code=_lambda.Code.from_asset('lambdas/status'))
# Step functions Definition
submit_job = _aws_stepfunctions_tasks.LambdaInvoke(
self,
"Submit Job",
lambda_function=submit_lambda,
output_path="$.Payload",
)
wait_job = _aws_stepfunctions.Wait(
self,
"Wait 30 Seconds",
time=_aws_stepfunctions.WaitTime.duration(Duration.seconds(30)))
status_job = _aws_stepfunctions_tasks.LambdaInvoke(
self,
"Get Status",
lambda_function=status_lambda,
output_path="$.Payload",
)
fail_job = _aws_stepfunctions.Fail(self,
"Fail",
cause='AWS Batch Job Failed',
error='DescribeJob returned FAILED')
succeed_job = _aws_stepfunctions.Succeed(
self, "Succeeded", comment='AWS Batch Job succeeded')
# Create Chain
definition = submit_job.next(wait_job)\
.next(status_job)\
.next(_aws_stepfunctions.Choice(self, 'Job Complete?')
.when(_aws_stepfunctions.Condition.string_equals('$.status', 'FAILED'), fail_job)
.when(_aws_stepfunctions.Condition.string_equals('$.status', 'SUCCEEDED'), succeed_job)
.otherwise(wait_job))
# Create state machine
sm = _aws_stepfunctions.StateMachine(
self,
"StateMachine",
definition=definition,
timeout=Duration.minutes(5),
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.vpc = aws_ec2.Vpc(self,
"demo-stepfunctions",
cidr="10.100.0.0/16",
max_azs=2,
nat_gateways=0,
subnet_configuration=[
aws_ec2.SubnetConfiguration(
name='demo-stepfunctions',
subnet_type=aws_ec2.SubnetType.ISOLATED,
cidr_mask=24)
])
lambda_role = iam.Role(
self,
'demo-lambda-role',
assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"))
lambda_role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaENIManagementAccess'))
lambda_role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'))
fn_submit = lambda_.Function(
self,
'demo-sfn-submit',
function_name='demo-sfn-submit',
handler='handler.do',
runtime=lambda_.Runtime.PYTHON_3_8,
code=lambda_.Code.asset('./craftaws/func_submit'),
role=lambda_role,
timeout=core.Duration.seconds(900),
allow_public_subnet=False,
vpc=self.vpc,
vpc_subnets=aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.ISOLATED),
environment={})
fn_job_1 = lambda_.Function(
self,
'demo-sfn-job1',
function_name='demo-sfn-job1',
handler='handler.do',
runtime=lambda_.Runtime.PYTHON_3_8,
code=lambda_.Code.asset('./craftaws/func_job_1'),
role=lambda_role,
timeout=core.Duration.seconds(900),
allow_public_subnet=False,
vpc=self.vpc,
vpc_subnets=aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.ISOLATED),
environment={})
fn_job_2 = lambda_.Function(
self,
'demo-sfn-job2',
function_name='demo-sfn-job2',
handler='handler.do',
runtime=lambda_.Runtime.PYTHON_3_8,
code=lambda_.Code.asset('./craftaws/func_job_2'),
role=lambda_role,
timeout=core.Duration.seconds(900),
allow_public_subnet=False,
vpc=self.vpc,
vpc_subnets=aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.ISOLATED),
environment={})
submit_job = tasks.LambdaInvoke(
self,
"Submit Job",
lambda_function=fn_submit,
# Lambda's result is in the attribute `Payload`
output_path="$.Payload")
step_1_job = tasks.LambdaInvoke(
self,
"Job_1",
lambda_function=fn_job_1,
# Lambda's result is in the attribute `Payload`
output_path="$.Payload")
wait_x = sfn.Wait(self,
"Wait X Seconds",
time=sfn.WaitTime.duration(
core.Duration.seconds(60)))
step_2_job = tasks.LambdaInvoke(
self,
"Job_2",
lambda_function=fn_job_1,
# Lambda's result is in the attribute `Payload`
output_path="$.Payload")
job_succeed = sfn.Succeed(self,
"Job Succeed",
comment="AWS Batch Job Succeed")
definition = submit_job.next(step_1_job).next(wait_x).next(
step_2_job).next(job_succeed)
sfn.StateMachine(self,
"StateMachine",
definition=definition,
timeout=core.Duration.minutes(5))
def test_nested_state_machine_chain():
default_fragment_json = {
'Type':
'Parallel',
'End':
True,
'Branches': [{
'StartAt': 'test-fragment: test-nested-state-machine',
'States': {
'test-fragment: test-nested-state-machine': {
'Next':
'test-fragment: test-nested-state-machine - Parse JSON Output',
'Catch': [{
'ErrorEquals': ['States.ALL'],
'ResultPath': '$.Error',
'Next': 'test-fail'
}],
'Parameters': {
'StateMachineArn': {
'Ref': 'teststatemachine7F4C511D'
},
'Input': {
'Key1': 'Value1'
}
},
'Type':
'Task',
'Resource': {
'Fn::Join': [
'',
[
'arn:', {
'Ref': 'AWS::Partition'
}, ':states:::states:startExecution.sync'
]
]
}
},
'test-fragment: test-nested-state-machine - Parse JSON Output':
{
'End':
True,
'Catch': [{
'ErrorEquals': ['States.ALL'],
'ResultPath': '$.Error',
'Next': 'test-fail'
}],
'Parameters': {
'JsonString.$': '$.Output'
},
'Type':
'Task',
'Resource': {
'Fn::GetAtt': ['ParseJsonString859DB4F0', 'Arn']
},
'ResultPath':
'$'
},
'test-fail': {
'Type': 'Fail'
}
}
}]
}
stack = core.Stack(core.App(), 'test-stack')
state_machine = sfn.StateMachine(stack,
'test-state-machine',
definition=sfn.Chain.start(
sfn.Succeed(stack, 'Succeeded')))
fragment = emr_chains.NestedStateMachine(stack,
'test-fragment',
name='test-nested-state-machine',
state_machine=state_machine,
input={'Key1': 'Value1'},
fail_chain=sfn.Fail(
stack, 'test-fail'))
print_and_assert(default_fragment_json, fragment)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
lambda_role = _iam.Role(
self,
id='lab3-om-role',
assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com'))
cloudwatch_policy_statement = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW)
cloudwatch_policy_statement.add_actions("logs:CreateLogGroup")
cloudwatch_policy_statement.add_actions("logs:CreateLogStream")
cloudwatch_policy_statement.add_actions("logs:PutLogEvents")
cloudwatch_policy_statement.add_actions("logs:DescribeLogStreams")
cloudwatch_policy_statement.add_resources("*")
lambda_role.add_to_policy(cloudwatch_policy_statement)
fn_lambda_approve_reject = aws_lambda.Function(
self,
"lab3-om-approve-reject",
code=aws_lambda.AssetCode(
"../lambda-functions/approve-reject-application/"),
handler="app.handler",
tracing=aws_lambda.Tracing.ACTIVE,
timeout=core.Duration.seconds(30),
role=lambda_role,
runtime=aws_lambda.Runtime.PYTHON_3_8)
fn_lambda_verify_identity = aws_lambda.Function(
self,
"lab3-om-verify-identity",
code=aws_lambda.AssetCode("../lambda-functions/verify-identity/"),
handler="app.handler",
tracing=aws_lambda.Tracing.ACTIVE,
timeout=core.Duration.seconds(30),
role=lambda_role,
runtime=aws_lambda.Runtime.PYTHON_3_8)
fn_lambda_check_address = aws_lambda.Function(
self,
"lab3-om-check-address",
code=aws_lambda.AssetCode("../lambda-functions/check-address/"),
handler="app.handler",
tracing=aws_lambda.Tracing.ACTIVE,
timeout=core.Duration.seconds(30),
role=lambda_role,
runtime=aws_lambda.Runtime.PYTHON_3_8)
'''
[INFO] This is a sample how to define the task and integrate with Lambda Functions. You need to create another 2 tasks for respective Lambda functions
'''
task_verify_identity = _tasks.LambdaInvoke(
self,
"Verify Identity Document",
lambda_function=fn_lambda_verify_identity,
output_path="$.Payload")
task_check_address = _tasks.LambdaInvoke(
self,
"Check Address",
lambda_function=fn_lambda_check_address,
output_path="$.Payload")
task_wait_review = _tasks.LambdaInvoke(
self,
"Wait for Review",
lambda_function=fn_lambda_approve_reject,
output_path="$.Payload")
state_approve = _sfn.Succeed(self, "Approve Application")
state_reject = _sfn.Succeed(self, "Reject Application")
# Let's define the State Machine, step by step
# First, paralell tasks for verification
s_verification = _sfn.Parallel(self, "Verification")
s_verification.branch(task_verify_identity)
s_verification.branch(task_check_address)
# Next, we add a choice state
c_human_review = _sfn.Choice(self, "Human review required?")
c_human_review.when(
_sfn.Condition.and_(
_sfn.Condition.boolean_equals("$[0].humanReviewRequired",
False),
_sfn.Condition.boolean_equals("$[1].humanReviewRequired",
True)), state_approve)
c_human_review.when(
_sfn.Condition.or_(
_sfn.Condition.boolean_equals("$[0].humanReviewRequired",
True),
_sfn.Condition.boolean_equals("$[1].humanReviewRequired",
False)), task_wait_review)
# Another choice state to check if the application passed the review
c_review_approved = _sfn.Choice(self, "Review approved?")
c_review_approved.when(
_sfn.Condition.boolean_equals("$.reviewApproved", True),
state_approve)
c_review_approved.when(
_sfn.Condition.boolean_equals("$.reviewApproved", False),
state_reject)
task_wait_review.next(c_review_approved)
definition = s_verification.next(c_human_review)
_sfn.StateMachine(self,
"lab3-statemachine",
definition=definition,
timeout=core.Duration.minutes(5))
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
#the S3 bucket where CloudFront Access Logs will be stored
cf_access_logs = s3.Bucket(self, "LogBucket")
#S3 bucket where Athena will put the results
athena_results = s3.Bucket(self, "AthenaResultsBucket")
#create an Athena database
glue_database_name = "serverlessland_database"
myDatabase = glue.CfnDatabase(
self,
id=glue_database_name,
catalog_id=account,
database_input=glue.CfnDatabase.DatabaseInputProperty(
description=f"Glue database '{glue_database_name}'",
name=glue_database_name,
)
)
#define a table with the structure of CloudFront Logs https://docs.aws.amazon.com/athena/latest/ug/cloudfront-logs.html
athena_table = glue.CfnTable(self,
id='cfaccesslogs',
catalog_id=account,
database_name=glue_database_name,
table_input=glue.CfnTable.TableInputProperty(
name='cf_access_logs',
description='CloudFront access logs',
table_type='EXTERNAL_TABLE',
parameters = {
'skip.header.line.count': '2',
},
storage_descriptor=glue.CfnTable.StorageDescriptorProperty(
location="s3://"+cf_access_logs.bucket_name+"/",
input_format='org.apache.hadoop.mapred.TextInputFormat',
output_format='org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',
compressed=False,
serde_info=glue.CfnTable.SerdeInfoProperty(
serialization_library='org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe',
parameters={
'field.delim' : ' '
}
),
columns=[
glue.CfnTable.ColumnProperty(name='date', type='date'),
glue.CfnTable.ColumnProperty(name='time', type='string'),
glue.CfnTable.ColumnProperty(name='location', type='string'),
glue.CfnTable.ColumnProperty(name='bytes', type='bigint'),
glue.CfnTable.ColumnProperty(name='request_ip', type='string'),
glue.CfnTable.ColumnProperty(name='method', type='string'),
glue.CfnTable.ColumnProperty(name='host', type='string'),
glue.CfnTable.ColumnProperty(name='uri', type='string'),
glue.CfnTable.ColumnProperty(name='status', type='string'),
glue.CfnTable.ColumnProperty(name='referer', type='string'),
glue.CfnTable.ColumnProperty(name='user_agent', type='string'),
glue.CfnTable.ColumnProperty(name='query_string', type='string'),
glue.CfnTable.ColumnProperty(name='cookie', type='string'),
glue.CfnTable.ColumnProperty(name='result_type', type='string'),
glue.CfnTable.ColumnProperty(name='request_id', type='string'),
glue.CfnTable.ColumnProperty(name='host_header', type='string'),
glue.CfnTable.ColumnProperty(name='request_protocol', type='string'),
glue.CfnTable.ColumnProperty(name='request_bytes', type='bigint'),
glue.CfnTable.ColumnProperty(name='time_taken', type='float'),
glue.CfnTable.ColumnProperty(name='xforwarded_for', type='string'),
glue.CfnTable.ColumnProperty(name='ssl_protocol', type='string'),
glue.CfnTable.ColumnProperty(name='ssl_cipher', type='string'),
glue.CfnTable.ColumnProperty(name='response_result_type', type='string'),
glue.CfnTable.ColumnProperty(name='http_version', type='string'),
glue.CfnTable.ColumnProperty(name='fle_status', type='string'),
glue.CfnTable.ColumnProperty(name='fle_encrypted_fields', type='int'),
glue.CfnTable.ColumnProperty(name='c_port', type='int'),
glue.CfnTable.ColumnProperty(name='time_to_first_byte', type='float'),
glue.CfnTable.ColumnProperty(name='x_edge_detailed_result_type', type='string'),
glue.CfnTable.ColumnProperty(name='sc_content_type', type='string'),
glue.CfnTable.ColumnProperty(name='sc_content_len', type='string'),
glue.CfnTable.ColumnProperty(name='sc_range_start', type='bigint'),
glue.CfnTable.ColumnProperty(name='sc_range_end', type='bigint')
]
),
)
)
#submit the query and wait for the results
start_query_execution_job = tasks.AthenaStartQueryExecution(self, "Start Athena Query",
query_string="SELECT uri FROM cf_access_logs limit 10",
integration_pattern=sf.IntegrationPattern.RUN_JOB, #executes the command in SYNC mode
query_execution_context=tasks.QueryExecutionContext(
database_name=glue_database_name
),
result_configuration=tasks.ResultConfiguration(
output_location=s3.Location(
bucket_name=athena_results.bucket_name,
object_key="results"
)
)
)
#get the results
get_query_results_job = tasks.AthenaGetQueryResults(self, "Get Query Results",
query_execution_id=sf.JsonPath.string_at("$.QueryExecution.QueryExecutionId"),
result_path=sf.JsonPath.string_at("$.GetQueryResults"),
)
#prepare the query to see if more results are available (up to 1000 can be retrieved)
prepare_next_params = sf.Pass(self, "Prepare Next Query Params",
parameters={
"QueryExecutionId.$": "$.StartQueryParams.QueryExecutionId",
"NextToken.$": "$.GetQueryResults.NextToken"
},
result_path=sf.JsonPath.string_at("$.StartQueryParams")
)
#check to see if more results are available
has_more_results = sf.Choice(self, "Has More Results?").when(
sf.Condition.is_present("$.GetQueryResults.NextToken"),
prepare_next_params.next(get_query_results_job)
).otherwise(sf.Succeed(self, "Done"))
#do something with each result
#here add your own logic
map = sf.Map(self, "Map State",
max_concurrency=1,
input_path=sf.JsonPath.string_at("$.GetQueryResults.ResultSet.Rows[1:]"),
result_path = sf.JsonPath.DISCARD
)
map.iterator(sf.Pass(self, "DoSomething"))
# Step function to orchestrate Athena query and retrieving the results
workflow = sf.StateMachine(self, "AthenaQuery",
definition=start_query_execution_job.next(get_query_results_job).next(map).next(has_more_results),
timeout=Duration.minutes(60)
)
CfnOutput(self, "Logs",
value=cf_access_logs.bucket_name, export_name='LogsBucket')
CfnOutput(self, "SFName",
value=workflow.state_machine_name, export_name='SFName')
CfnOutput(self, "SFArn",
value = workflow.state_machine_arn,
export_name = 'StepFunctionArn',
description = 'Step Function arn')
def __init__(self,
scope: core.Construct,
id: str,
redshift_cluster_name: str,
user_secret: Secret) -> None:
super().__init__(scope, id)
stack = Stack.of(self)
subprocess.call(
['pip', 'install', '-t', 'dwh/dwh_loader_layer/python/lib/python3.8/site-packages', '-r',
'dwh/dwh_loader/requirements.txt', '--platform', 'manylinux1_x86_64', '--only-binary=:all:',
'--upgrade'])
requirements_layer = _lambda.LayerVersion(scope=self,
id='PythonRequirementsTemplate',
code=_lambda.Code.from_asset('dwh/dwh_loader_layer'),
compatible_runtimes=[_lambda.Runtime.PYTHON_3_8])
dwh_loader_role = _iam.Role(
self, 'Role',
assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com')
)
dwh_loader_role.add_managed_policy(_iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'
))
dwh_loader_role.attach_inline_policy(
_iam.Policy(
self, 'InlinePolicy',
statements=[
_iam.PolicyStatement(
actions=[
"redshift-data:ExecuteStatement",
"redshift-data:CancelStatement",
"redshift-data:ListStatements",
"redshift-data:GetStatementResult",
"redshift-data:DescribeStatement",
"redshift-data:ListDatabases",
"redshift-data:ListSchemas",
"redshift-data:ListTables",
"redshift-data:DescribeTable"
],
resources=['*']
),
_iam.PolicyStatement(
actions=["secretsmanager:GetSecretValue"],
resources=[user_secret.secret_arn]
),
_iam.PolicyStatement(
actions=["redshift:GetClusterCredentials"],
resources=[
"arn:aws:redshift:*:*:dbname:*/*",
"arn:aws:redshift:*:*:dbuser:*/"+_config.Redshift.ETL_USER
]
),
_iam.PolicyStatement(
effect=_iam.Effect('DENY'),
actions=["redshift:CreateClusterUser"],
resources=["arn:aws:redshift:*:*:dbuser:*/"+_config.Redshift.ETL_USER]
),
_iam.PolicyStatement(
conditions={
'StringLike': {
"iam:AWSServiceName": "redshift-data.amazonaws.com"
}
},
actions=["iam:CreateServiceLinkedRole"],
resources=["arn:aws:iam::*:role/aws-service-role/redshift-data.amazonaws.com/AWSServiceRoleForRedshift"]
),
]
)
)
dwh_loader_function = _lambda.Function(
self, 'Lambda',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset('dwh/dwh_loader'),
handler='dwh_loader.handler',
function_name='dwh-loader',
environment={
'CLUSTER_NAME': redshift_cluster_name,
'PROCEDURE': _config.Redshift.ETL_PROCEDURE,
'SECRET_ARN': user_secret.secret_arn,
'DATABASE': _config.Redshift.DATABASE,
'REGION': core.Aws.REGION,
'SCHEMA': _config.Redshift.SCHEMA
},
layers=[requirements_layer],
timeout=core.Duration.seconds(30),
role=dwh_loader_role
)
dwh_loader_submit = _sfn_tasks.LambdaInvoke(
self, 'Submit',
lambda_function=dwh_loader_function,
payload_response_only=True
)
dwh_loader_wait = _sfn.Wait(
self, 'Wait',
time=_sfn.WaitTime.duration(core.Duration.seconds(30))
)
dwh_loader_complete = _sfn.Choice(
self, 'Complete'
)
dwh_loader_failed = _sfn.Fail(
self, 'Fail',
cause="Redshift Data API statement failed",
error="$.Result.Error"
)
dwh_loader_status = _sfn_tasks.LambdaInvoke(
self, 'Status',
lambda_function=dwh_loader_function,
result_path='$.Result',
payload_response_only=True
)
definition = dwh_loader_submit \
.next(dwh_loader_wait) \
.next(dwh_loader_status) \
.next(dwh_loader_complete
.when(_sfn.Condition.string_equals('$.Result.Status', 'FAILED'), dwh_loader_failed)
.when(_sfn.Condition.string_equals('$.Result.Status', 'FINISHED'), _sfn.Succeed(self, 'DwhLoaderSuccess'))
.otherwise(dwh_loader_wait))
dwh_loader_stepfunctions = _sfn.StateMachine(
self, 'StepFunctions',
definition=definition,
timeout=core.Duration.minutes(30)
)
step_trigger = _events.Rule(
self, 'StepTrigger',
schedule=_events.Schedule.cron(minute='0/30',
hour='*',
month='*',
week_day='*',
year='*')
)
step_trigger.add_target(
_events_targets.SfnStateMachine(
machine=dwh_loader_stepfunctions,
)
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
pvt_bkt = _s3.Bucket(self, "s3bucket")
core.Tag.add(pvt_bkt, key="isMonitoredBucket", value="True")
# Lets create a cloudtrail to track s3 data events
s3_data_event_trail = _cloudtrail.Trail(
self,
"s3DataEventTrailId",
is_multi_region_trail=False,
include_global_service_events=False,
enable_file_validation=True)
# Lets capture S3 Data Events only for our bucket- TO REDUCE COST
s3_data_event_trail.add_s3_event_selector(
prefixes=[f"{pvt_bkt.bucket_arn}/"],
include_management_events=True,
read_write_type=_cloudtrail.ReadWriteType.ALL)
# Defines an AWS Lambda resource
"""
with open("lambda_src/make_object_private.py", encoding="utf8") as fp:
make_object_private_fn_handler_code = fp.read()
remediate_object_acl_fn = _lambda.Function(
self,
id='remediateObjAclFn',
function_name="remediate_object_acl_fn",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(make_object_private_fn_handler_code),
handler='index.lambda_handler',
timeout=core.Duration.seconds(10)
)
# Lets add the necessary permission for the lambda function
remediate_object_acl_fn_perms=_iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
"arn:aws:s3:::*",
],
actions=[
"s3:GetObjectAcl",
"s3:PutObjectAcl"
]
)
remediate_object_acl_fn_perms.sid="PutBucketPolicy"
remediate_object_acl_fn.add_to_role_policy( remediate_object_acl_fn_perms )
"""
with open("lambda_src/is_object_private.py", encoding="utf8") as fp:
is_object_private_fn_handler_code = fp.read()
is_object_private_fn = _lambda.Function(
self,
id='isObjPrivateFn',
function_name="is_object_private_fn",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(is_object_private_fn_handler_code),
handler='index.lambda_handler',
timeout=core.Duration.seconds(3))
# Lets add the necessary permission for the lambda function
is_object_private_fn_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
"arn:aws:s3:::*",
],
actions=["s3:GetObjectAcl"])
is_object_private_fn.sid = "CheckObjectAcl"
is_object_private_fn.add_to_role_policy(is_object_private_fn_perms)
with open("lambda_src/make_object_private.py", encoding="utf8") as fp:
make_object_private_fn_handler_code = fp.read()
remediate_object_acl_fn = _lambda.Function(
self,
id='remediateObjAclFn',
function_name="remediate_object_acl_fn",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(make_object_private_fn_handler_code),
handler='index.lambda_handler',
timeout=core.Duration.seconds(10))
# Lets add the necessary permission for the lambda function
remediate_object_acl_fn_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
"arn:aws:s3:::*",
],
actions=["s3:PutObjectAcl"])
remediate_object_acl_fn_perms.sid = "PutObjectAcl"
remediate_object_acl_fn.add_to_role_policy(
remediate_object_acl_fn_perms)
info_sec_ops_topic = _sns.Topic(self,
"infoSecOpsTopicId",
display_name="InfoSecTopic",
topic_name="InfoSecOpsTopic")
# Subscribe InfoSecOps Email to topic
info_sec_ops_topic.add_subscription(
_subs.EmailSubscription(global_args.INFO_SEC_OPS_EMAIL))
# Grant Lambda permission to publish to topic
# info_sec_ops_topic.grant_publish(lambda_notifier)
# State Machine for notifying failed ACLs
# Ref: https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html
###############################################################################
################# STEP FUNCTIONS EXPERIMENTAL CODE - UNSTABLE #################
###############################################################################
is_object_private_task = _sfn.Task(
self,
"isObjectPrivate?",
task=_tasks.InvokeFunction(is_object_private_fn),
result_path="$",
output_path="$")
remediate_object_acl_task = _sfn.Task(
self,
"RemediateObjectAcl",
task=_tasks.InvokeFunction(remediate_object_acl_fn),
result_path="$",
output_path="$")
notify_secops_task = _sfn.Task(
self,
"Notify InfoSecOps",
task=_tasks.PublishToTopic(
info_sec_ops_topic,
integration_pattern=_sfn.ServiceIntegrationPattern.
FIRE_AND_FORGET,
message=_sfn.TaskInput.from_data_at("$.sns_message"),
subject="Object Acl Remediation"))
acl_remediation_failed_task = _sfn.Fail(self,
"Acl Remediation Failed",
cause="Acl Remediation Failed",
error="Check Logs")
acl_compliant_task = _sfn.Succeed(self,
"Object Acl Compliant",
comment="Object Acl is Compliant")
remediate_object_acl_sfn_definition = is_object_private_task\
.next(_sfn.Choice(self, "Is Object Private?")\
.when(_sfn.Condition.boolean_equals("$.is_private", True), acl_compliant_task)\
.when(_sfn.Condition.boolean_equals("$.is_private", False), remediate_object_acl_task\
.next(_sfn.Choice(self, "Object Remediation Complete?")\
.when(_sfn.Condition.boolean_equals("$.status", True),acl_compliant_task)\
.when(_sfn.Condition.boolean_equals("$.status", False), notify_secops_task.next(acl_remediation_failed_task))\
.otherwise(acl_remediation_failed_task)\
)
)
.otherwise(acl_remediation_failed_task)
)
remediate_object_acl_statemachine = _sfn.StateMachine(
self,
"stateMachineId",
definition=remediate_object_acl_sfn_definition,
timeout=core.Duration.minutes(3))
# Cloudwatch Event Triggers
put_object_acl_event_targets = []
"""
put_object_acl_event_targets.append(
_targets.LambdaFunction(
handler=remediate_object_acl_fn
)
)
"""
put_object_acl_event_targets.append(
_targets.SfnStateMachine(
machine=remediate_object_acl_statemachine))
put_object_acl_event_pattern = _events.EventPattern(
source=["aws.s3"],
detail_type=["AWS API Call via CloudTrail"],
detail={
"eventSource": ["s3.amazonaws.com"],
"eventName": ["PutObjectAcl", "PutObject"],
"requestParameters": {
"bucketName": [f"{pvt_bkt.bucket_name}"]
}
})
put_object_acl_event_pattern_rule = _events.Rule(
self,
"putObjectAclEventId",
event_pattern=put_object_acl_event_pattern,
rule_name=f"put_s3_policy_event_{global_args.OWNER}",
enabled=True,
description="Trigger an event for S3 PutObjectAcl or PutObject",
targets=put_object_acl_event_targets)
###########################################
################# OUTPUTS #################
###########################################
output0 = core.CfnOutput(
self,
"SecuirtyAutomationFrom",
value=f"{global_args.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output1 = core.CfnOutput(
self,
"MonitoredS3Bucket",
value=(f"https://console.aws.amazon.com/s3/buckets/"
f"{pvt_bkt.bucket_name}"),
description=f"S3 Bucket for testing purposes")
output2 = core.CfnOutput(
self,
"Helpercommands",
value=
(f"aws s3api get-object-acl --bucket ${pvt_bkt.bucket_name} --key OBJECT-KEY-NAME"
),
description=
f"Commands to set object to public, Update OBJECT-KEY-NAME to your needs"
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# A cache to temporarily hold session info
session_cache_table = aws_dynamodb.Table(
self,
'session_cache_table',
partition_key={
'name': 'code',
'type': aws_dynamodb.AttributeType.STRING
},
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST,
time_to_live_attribute='expires')
#--
# Secrets
#--------------------#
# Twitter secrets are stored external to this stack
twitter_secret = aws_secretsmanager.Secret.from_secret_attributes(
self,
'twitter_secret',
secret_arn=os.environ['TWITTER_SECRET_ARN'])
#--
# Layers
#--------------------#
# Each of these dependencies is used in 2 or more functions, extracted to layer for ease of use
twitter_layer = aws_lambda.LayerVersion(
self,
'twitter_layer',
code=aws_lambda.AssetCode('layers/twitter_layer'),
compatible_runtimes=[
aws_lambda.Runtime.PYTHON_2_7, aws_lambda.Runtime.PYTHON_3_6
])
boto_layer = aws_lambda.LayerVersion(
self,
'boto_layer',
code=aws_lambda.AssetCode('layers/boto_layer'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6])
#--
# Functions
#--------------------#
# Handles CRC validation requests from Twitter
twitter_crc_func = aws_lambda.Function(
self,
"twitter_crc_func",
code=aws_lambda.AssetCode('functions/twitter_crc_func'),
handler="lambda.handler",
layers=[twitter_layer],
runtime=aws_lambda.Runtime.PYTHON_2_7,
environment={'twitter_secret': twitter_secret.secret_arn})
# Grant this function the ability to read Twitter credentials
twitter_secret.grant_read(twitter_crc_func.role)
# Handle schedule requests from Twitter
twitter_webhook_func = aws_lambda.Function(
self,
"twitter_webhook_func",
code=aws_lambda.AssetCode('functions/twitter_webhook_func'),
handler="lambda.handler",
layers=[boto_layer, twitter_layer],
runtime=aws_lambda.Runtime.PYTHON_3_6,
environment={'twitter_secret': twitter_secret.secret_arn})
# Grant this function permission to read Twitter credentials
twitter_secret.grant_read(twitter_webhook_func.role)
# Grant this function permission to publish tweets to EventBridge
twitter_webhook_func.add_to_role_policy(
aws_iam.PolicyStatement(actions=["events:PutEvents"],
resources=["*"]))
# Use API Gateway as the webhook endpoint
twitter_api = aws_apigateway.LambdaRestApi(
self, 'twitter_api', handler=twitter_webhook_func, proxy=False)
# Tweets are POSTed to the endpoint
twitter_api.root.add_method('POST')
# Handles twitter CRC validation requests via GET to the webhook
twitter_api.root.add_method(
'GET', aws_apigateway.LambdaIntegration(twitter_crc_func))
# Extract relevant info from the tweet, including session codes
parse_tweet_func = aws_lambda.Function(
self,
"parse_tweet_func",
code=aws_lambda.AssetCode('functions/parse_tweet_func'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6)
# Get session information for requested codes
get_sessions_func = aws_lambda.Function(
self,
"get_sessions_func",
code=aws_lambda.AssetCode('functions/get_sessions_func'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
timeout=core.Duration.seconds(60),
layers=[boto_layer],
environment={
'CACHE_TABLE': session_cache_table.table_name,
'LOCAL_CACHE_TTL':
str(1 * 60 * 60), # Cache sessions locally for 1 hour
'REMOTE_CACHE_TTL': str(12 * 60 * 60)
}) # Cache sessions removely for 12 hours
# This functions needs permissions to read and write to the table
session_cache_table.grant_write_data(get_sessions_func)
session_cache_table.grant_read_data(get_sessions_func)
# Create a schedule without conflicts
create_schedule_func = aws_lambda.Function(
self,
"create_schedule_func",
code=aws_lambda.AssetCode('functions/create_schedule_func'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
timeout=core.Duration.seconds(60))
# Tweet the response to the user
tweet_schedule_func = aws_lambda.Function(
self,
"tweet_schedule_func",
code=aws_lambda.AssetCode('functions/tweet_schedule_func'),
handler="lambda.handler",
layers=[boto_layer, twitter_layer],
runtime=aws_lambda.Runtime.PYTHON_3_6,
environment={'twitter_secret': twitter_secret.secret_arn})
twitter_secret.grant_read(tweet_schedule_func.role)
#--
# States
#--------------------#
# Step 4
tweet_schedule_job = aws_stepfunctions.Task(
self,
'tweet_schedule_job',
task=aws_stepfunctions_tasks.InvokeFunction(tweet_schedule_func))
# Step 3
create_schedule_job = aws_stepfunctions.Task(
self,
'create_schedule_job',
task=aws_stepfunctions_tasks.InvokeFunction(create_schedule_func),
input_path="$.sessions",
result_path="$.schedule")
create_schedule_job.next(tweet_schedule_job)
# Step 2 - Get associated sessions (scrape or cache)
get_sessions_job = aws_stepfunctions.Task(
self,
'get_sessions_job',
task=aws_stepfunctions_tasks.InvokeFunction(get_sessions_func))
# Prepare to get session info in parallel using the Map state
get_sessions_map = aws_stepfunctions.Map(self,
'get_sessions_map',
items_path="$.codes",
result_path="$.sessions")
get_sessions_map.iterator(get_sessions_job)
get_sessions_map.next(create_schedule_job)
# Shortcut if no session codes are supplied
check_num_codes = aws_stepfunctions.Choice(self, 'check_num_codes')
check_num_codes.when(
aws_stepfunctions.Condition.number_greater_than('$.num_codes', 0),
get_sessions_map)
check_num_codes.otherwise(aws_stepfunctions.Succeed(self, "no_codes"))
# Step 1 - Parse incoming tweet and prepare for scheduling
parse_tweet_job = aws_stepfunctions.Task(
self,
'parse_tweet_job',
task=aws_stepfunctions_tasks.InvokeFunction(parse_tweet_func))
parse_tweet_job.next(check_num_codes)
#--
# State Machines
#--------------------#
schedule_machine = aws_stepfunctions.StateMachine(
self, "schedule_machine", definition=parse_tweet_job)
# A rule to filter reInventSched tweet events
reinvent_sched_rule = aws_events.Rule(
self,
"reinvent_sched_rule",
event_pattern={"source": ["reInventSched"]})
# Matching events start the image pipline
reinvent_sched_rule.add_target(
aws_events_targets.SfnStateMachine(
schedule_machine,
input=aws_events.RuleTargetInput.from_event_path("$.detail")))
def create_stepfunction(self) -> Resource:
"""コンポーネントをビルドしてデプロイするステートマシンの作成
Returns:
Resource: step function
"""
name = f"{self.stack_name}_{self.component_id}_edgedeploy_pipeline"
role_name = self.get_role_name("edgedeploy_pipeline")
sf_role = aws_iam.Role(
self,
id=role_name,
assumed_by=aws_iam.ServicePrincipal("states.amazonaws.com"),
role_name=role_name,
path="/service-role/",
managed_policies=[
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSLambdaBasicExecutionRole")
])
sf_role.attach_inline_policy(
aws_iam.Policy(
self,
"AllowCloudWatchLogsForSF",
document=aws_iam.PolicyDocument(statements=[
aws_iam.PolicyStatement(actions=[
"logs:CreateLogDelivery", "logs:GetLogDelivery",
"logs:UpdateLogDelivery", "logs:DeleteLogDelivery",
"logs:ListLogDeliveries", "logs:PutResourcePolicy",
"logs:DescribeResourcePolicies",
"logs:DescribeLogGroups"
],
resources=["*"])
])))
sf_role.attach_inline_policy(
aws_iam.Policy(
self,
"AllowXRayForSF",
document=aws_iam.PolicyDocument(statements=[
aws_iam.PolicyStatement(actions=[
"xray:PutTraceSegments", "xray:PutTelemetryRecords",
"xray:GetSamplingRules", "xray:GetSamplingTargets"
],
resources=["*"])
])))
sf_role.attach_inline_policy(
aws_iam.Policy(
self,
"AllowInvokeLambda",
document=aws_iam.PolicyDocument(statements=[
aws_iam.PolicyStatement(
actions=["lambda:InvokeFunction"],
resources=[
self._lambda_build_image.function_arn,
self._lambda_build_image.function_arn +
":*", self._lambda_check_image_status.function_arn,
self._lambda_check_image_status.function_arn +
":*", self._lambda_create_component.function_arn,
self._lambda_create_component.function_arn +
":*", self._lambda_deploy_component.function_arn,
self._lambda_deploy_component.function_arn + ":*",
self._lambda_check_deploy_status.function_arn,
self._lambda_check_deploy_status.function_arn +
":*"
])
])))
# dockerコンテナをビルド
task_build_image = aws_sf_tasks.LambdaInvoke(
self,
"BuildInferenceImage",
lambda_function=self._lambda_build_image,
output_path="$.Payload")
# dockerコンテナのビルド結果を確認
task_check_build_image_status = aws_sf_tasks.LambdaInvoke(
self,
"CheckDockerImageBuildStatus",
lambda_function=self._lambda_check_image_status,
output_path="$.Payload")
# dockerコンテナのビルドを待つ
wait_image_build = aws_sf.Wait(self,
"WaitImageBuildFinish",
time=aws_sf.WaitTime.duration(
Duration.seconds(30)))
# Greengrassのコンポーネントを作成
task_create_greengrass_component = aws_sf_tasks.LambdaInvoke(
self,
"CreateComponent",
lambda_function=self._lambda_create_component,
output_path="$.Payload")
# Greengrassへデプロイ
task_deploy_component = aws_sf_tasks.LambdaInvoke(
self,
"DeployComponent",
lambda_function=self._lambda_deploy_component,
output_path="$.Payload")
# Greengrassへのデプロイ終了を待つ
wait_component_deploy = aws_sf.Wait(self,
"WaitDeploymentFinish",
time=aws_sf.WaitTime.duration(
Duration.seconds(30)))
# Greengrassへデプロイ結果を確認
task_check_deployment_status = aws_sf_tasks.LambdaInvoke(
self,
"CheckDeploymentStatus",
lambda_function=self._lambda_check_deploy_status,
output_path="$.Payload")
# デプロイ失敗
pipeline_failed = aws_sf.Fail(self,
"PipelineFailed",
error="DeployPipelineFailed",
cause="Something went wrong")
# 正常終了
pipeline_success = aws_sf.Succeed(self, "PipelineSuccessed")
# dockerコンテナが存在したかを判定
choice_component_exists_result = aws_sf.Choice(self,
"JudgeComponentExists")
# dockerコンテナのビルド結果を判定
choice_image_build_result = aws_sf.Choice(self,
"JudgeImageBuildStatus")
# dockerコンテナのビルド結果を判定
choice_deployment_result = aws_sf.Choice(self, "JudgeDeploymentStatus")
# 正常終了を通知
publish_success_message = aws_sf_tasks.SnsPublish(
self,
"Publish Success message",
topic=aws_sns.Topic(self, "SendDeploySuccess"),
message=aws_sf.TaskInput.from_json_path_at("$.message")).next(
pipeline_success)
# デプロイ失敗を通知
publish_failed_message = aws_sf_tasks.SnsPublish(
self,
"Publish Failed message",
topic=aws_sns.Topic(self, "SendPipelineFailed"),
message=aws_sf.TaskInput.from_json_path_at("$.message")).next(
pipeline_failed)
definition = \
task_build_image.next(
choice_component_exists_result
.when(
aws_sf.Condition.string_equals("$.status", "component_exists"), task_deploy_component)
.otherwise(
wait_image_build.next(
task_check_build_image_status).next(
choice_image_build_result.when(
aws_sf.Condition.string_equals("$.status", "image_exists"), task_create_greengrass_component
.next(task_deploy_component)
.next(wait_component_deploy)
.next(task_check_deployment_status)
.next(
choice_deployment_result
.when(aws_sf.Condition.string_equals("$.status", "RUNNING"), wait_component_deploy)
.when(aws_sf.Condition.string_equals("$.status", "COMPLETED"), publish_success_message)
.otherwise(publish_failed_message).afterwards()))
.when(
aws_sf.Condition.string_equals("$.status", "image_faild"), publish_failed_message)
.otherwise(
wait_image_build).afterwards())
)
)
#.next(aws_sf.Succeed(self, "GreengrassComponentDeployFinished"))
state_machine = aws_sf.StateMachine(
self,
id=name,
state_machine_name=name,
definition=definition,
state_machine_type=aws_sf.StateMachineType.STANDARD,
role=sf_role)
def __init__(self, scope: core.Construct, id: str, group_name: str,
minute_duration: int, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# TODO: Setup alerting of failure to an SNS
# TODO: Failure is not the same as a student not in a group
# TODO: Streamline input data so that lambda's only get the info they really need
# TODO: Comment
# TODO: Need to separate unexpected errors from regular errors
# Setting up monitoring
schedule_stop = lambda_.Function(
self,
id="ScheduleStopLambda",
runtime=lambda_.Runtime.PYTHON_3_7,
code=lambda_.Code.from_inline(
open("./resources/schedule-termination.py", 'r').read()),
handler="index.handler",
log_retention=logs.RetentionDays.ONE_DAY,
environment=dict(GROUP_NAME=group_name),
timeout=core.Duration.seconds(30))
schedule_stop.add_to_role_policy(
statement=iam.PolicyStatement(actions=[
"ec2:Describe*", "iam:ListGroupsForUser", "iam:ListUsers"
],
effect=iam.Effect.ALLOW,
resources=["*"]))
terminate_ec2 = lambda_.Function(
self,
id="TerminateEC2",
runtime=lambda_.Runtime.PYTHON_3_7,
code=lambda_.Code.from_inline(
open("./resources/terminate-ec2.py", 'r').read()),
handler="index.handler",
log_retention=logs.RetentionDays.ONE_DAY,
timeout=core.Duration.seconds(30))
terminate_ec2.add_to_role_policy(
statement=iam.PolicyStatement(actions=[
"ec2:DescribeInstance*",
"ec2:TerminateInstances",
],
effect=iam.Effect.ALLOW,
resources=["*"]))
# The lambda object that will see if we should schedule.
schedule_stop_task = tasks.LambdaInvoke(
self,
id='schedule stop',
lambda_function=schedule_stop,
input_path="$.detail.userIdentity",
result_path="$.Payload",
)
# TODO: Need to change this based on the configuration info above
# Wait state to try and delete
# wait_x = sfn.Wait(self, 'Wait x minutes', time=sfn.WaitTime.seconds_path("10"))
wait_x = sfn.Wait(self,
id='Wait x minutes',
time=sfn.WaitTime.duration(
core.Duration.minutes(minute_duration)))
job_failed = sfn.Fail(self,
id="Failed Job",
cause="Error in the input",
error="Error")
job_finished = sfn.Succeed(self, id="Job Finished")
choice = sfn.Choice(self, 'Can I delete')
choice.when(sfn.Condition.boolean_equals('$.Payload.Payload', False),
job_finished)
choice.otherwise(wait_x)
terminate_ec2_task = tasks.LambdaInvoke(
self,
'terminate',
lambda_function=terminate_ec2,
input_path="$.detail.responseElements.instancesSet")
wait_x.next(terminate_ec2_task).next(job_finished)
state_definition = schedule_stop_task \
.next(choice)
terminate_machine = sfn.StateMachine(self,
id="State Machine",
definition=state_definition)
cloudwatch.Alarm(self,
"EC2ScheduleAlarm",
metric=terminate_machine.metric_failed(),
threshold=1,
evaluation_periods=1)
# TODO Build Rule that monitors for EC2 creation
# Any new creation, the EC2 will have to be destroyed. Including
# other things?
create_event = events.Rule(
self,
id='detect-ec2-start',
description="Detects if an EC2 is created",
enabled=True,
event_pattern=events.EventPattern(
detail_type=["AWS API Call via CloudTrail"],
source=["aws.ec2"],
detail={
"eventName": ["RunInstances"],
"eventSource": ["ec2.amazonaws.com"]
}),
targets=[targets.SfnStateMachine(terminate_machine)])
def __init__(
self,
scope: Construct,
stack_id: str,
*,
botocore_lambda_layer: aws_lambda_python.PythonLayerVersion,
env_name: str,
storage_bucket: aws_s3.Bucket,
validation_results_table: Table,
) -> None:
# pylint: disable=too-many-locals, too-many-statements
super().__init__(scope, stack_id)
############################################################################################
# PROCESSING ASSETS TABLE
processing_assets_table = Table(
self,
f"{env_name}-processing-assets",
env_name=env_name,
parameter_name=ParameterName.PROCESSING_ASSETS_TABLE_NAME,
sort_key=aws_dynamodb.Attribute(name="sk", type=aws_dynamodb.AttributeType.STRING),
)
############################################################################################
# BATCH JOB DEPENDENCIES
batch_job_queue = BatchJobQueue(
self,
"batch-job-queue",
env_name=env_name,
processing_assets_table=processing_assets_table,
).job_queue
s3_read_only_access_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3ReadOnlyAccess"
)
############################################################################################
# UPDATE CATALOG UPDATE MESSAGE QUEUE
dead_letter_queue = aws_sqs.Queue(
self,
"dead-letter-queue",
visibility_timeout=LAMBDA_TIMEOUT,
)
self.message_queue = aws_sqs.Queue(
self,
"update-catalog-message-queue",
visibility_timeout=LAMBDA_TIMEOUT,
dead_letter_queue=aws_sqs.DeadLetterQueue(max_receive_count=3, queue=dead_letter_queue),
)
self.message_queue_name_parameter = aws_ssm.StringParameter(
self,
"update-catalog-message-queue-name",
string_value=self.message_queue.queue_name,
description=f"Update Catalog Message Queue Name for {env_name}",
parameter_name=ParameterName.UPDATE_CATALOG_MESSAGE_QUEUE_NAME.value,
)
populate_catalog_lambda = BundledLambdaFunction(
self,
"populate-catalog-bundled-lambda-function",
directory="populate_catalog",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
botocore_lambda_layer=botocore_lambda_layer,
)
self.message_queue.grant_consume_messages(populate_catalog_lambda)
populate_catalog_lambda.add_event_source(
SqsEventSource(self.message_queue, batch_size=1) # type: ignore[arg-type]
)
############################################################################################
# STATE MACHINE TASKS
check_stac_metadata_task = LambdaTask(
self,
"check-stac-metadata-task",
directory="check_stac_metadata",
botocore_lambda_layer=botocore_lambda_layer,
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
assert check_stac_metadata_task.lambda_function.role
check_stac_metadata_task.lambda_function.role.add_managed_policy(
policy=s3_read_only_access_policy
)
for table in [processing_assets_table, validation_results_table]:
table.grant_read_write_data(check_stac_metadata_task.lambda_function)
table.grant(
check_stac_metadata_task.lambda_function,
"dynamodb:DescribeTable",
)
content_iterator_task = LambdaTask(
self,
"content-iterator-task",
directory="content_iterator",
botocore_lambda_layer=botocore_lambda_layer,
result_path=f"$.{CONTENT_KEY}",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
check_files_checksums_directory = "check_files_checksums"
check_files_checksums_default_payload_object = {
f"{DATASET_ID_KEY}.$": f"$.{DATASET_ID_KEY}",
f"{VERSION_ID_KEY}.$": f"$.{VERSION_ID_KEY}",
f"{METADATA_URL_KEY}.$": f"$.{METADATA_URL_KEY}",
f"{FIRST_ITEM_KEY}.$": f"$.{CONTENT_KEY}.{FIRST_ITEM_KEY}",
f"{ASSETS_TABLE_NAME_KEY}.$": f"$.{CONTENT_KEY}.{ASSETS_TABLE_NAME_KEY}",
f"{RESULTS_TABLE_NAME_KEY}.$": f"$.{CONTENT_KEY}.{RESULTS_TABLE_NAME_KEY}",
}
check_files_checksums_single_task = BatchSubmitJobTask(
self,
"check-files-checksums-single-task",
env_name=env_name,
directory=check_files_checksums_directory,
s3_policy=s3_read_only_access_policy,
job_queue=batch_job_queue,
payload_object=check_files_checksums_default_payload_object,
container_overrides_command=[
"--dataset-id",
f"Ref::{DATASET_ID_KEY}",
"--version-id",
f"Ref::{VERSION_ID_KEY}",
"--first-item",
f"Ref::{FIRST_ITEM_KEY}",
"--assets-table-name",
f"Ref::{ASSETS_TABLE_NAME_KEY}",
"--results-table-name",
f"Ref::{RESULTS_TABLE_NAME_KEY}",
],
)
array_size = int(
aws_stepfunctions.JsonPath.number_at(f"$.{CONTENT_KEY}.{ITERATION_SIZE_KEY}")
)
check_files_checksums_array_task = BatchSubmitJobTask(
self,
"check-files-checksums-array-task",
env_name=env_name,
directory=check_files_checksums_directory,
s3_policy=s3_read_only_access_policy,
job_queue=batch_job_queue,
payload_object=check_files_checksums_default_payload_object,
container_overrides_command=[
"--dataset-id",
f"Ref::{DATASET_ID_KEY}",
"--version-id",
f"Ref::{VERSION_ID_KEY}",
"--first-item",
f"Ref::{FIRST_ITEM_KEY}",
"--assets-table-name",
f"Ref::{ASSETS_TABLE_NAME_KEY}",
"--results-table-name",
f"Ref::{RESULTS_TABLE_NAME_KEY}",
],
array_size=array_size,
)
for reader in [
content_iterator_task.lambda_function,
check_files_checksums_single_task.job_role,
check_files_checksums_array_task.job_role,
]:
processing_assets_table.grant_read_data(reader) # type: ignore[arg-type]
processing_assets_table.grant(
reader, "dynamodb:DescribeTable" # type: ignore[arg-type]
)
for writer in [
check_files_checksums_single_task.job_role,
check_files_checksums_array_task.job_role,
]:
validation_results_table.grant_read_write_data(writer) # type: ignore[arg-type]
validation_results_table.grant(
writer, "dynamodb:DescribeTable" # type: ignore[arg-type]
)
validation_summary_task = LambdaTask(
self,
"validation-summary-task",
directory="validation_summary",
botocore_lambda_layer=botocore_lambda_layer,
result_path=f"$.{VALIDATION_KEY}",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
validation_results_table.grant_read_data(validation_summary_task.lambda_function)
validation_results_table.grant(
validation_summary_task.lambda_function, "dynamodb:DescribeTable"
)
import_dataset_role = aws_iam.Role(
self,
"import-dataset",
assumed_by=aws_iam.ServicePrincipal( # type: ignore[arg-type]
"batchoperations.s3.amazonaws.com"
),
)
import_asset_file_function = ImportFileFunction(
self,
directory="import_asset_file",
invoker=import_dataset_role,
env_name=env_name,
botocore_lambda_layer=botocore_lambda_layer,
)
import_metadata_file_function = ImportFileFunction(
self,
directory="import_metadata_file",
invoker=import_dataset_role,
env_name=env_name,
botocore_lambda_layer=botocore_lambda_layer,
)
import_dataset_task = LambdaTask(
self,
"import-dataset-task",
directory="import_dataset",
botocore_lambda_layer=botocore_lambda_layer,
result_path=f"$.{IMPORT_DATASET_KEY}",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
import_dataset_task.lambda_function.add_to_role_policy(
aws_iam.PolicyStatement(
resources=[import_dataset_role.role_arn],
actions=["iam:PassRole"],
),
)
import_dataset_task.lambda_function.add_to_role_policy(
aws_iam.PolicyStatement(resources=["*"], actions=["s3:CreateJob"])
)
for table in [processing_assets_table]:
table.grant_read_data(import_dataset_task.lambda_function)
table.grant(import_dataset_task.lambda_function, "dynamodb:DescribeTable")
# Import status check
wait_before_upload_status_check = Wait(
self,
"wait-before-upload-status-check",
time=WaitTime.duration(Duration.seconds(10)),
)
upload_status_task = LambdaTask(
self,
"upload-status",
directory="upload_status",
botocore_lambda_layer=botocore_lambda_layer,
result_path="$.upload_status",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
validation_results_table.grant_read_data(upload_status_task.lambda_function)
validation_results_table.grant(upload_status_task.lambda_function, "dynamodb:DescribeTable")
upload_status_task.lambda_function.add_to_role_policy(ALLOW_DESCRIBE_ANY_S3_JOB)
# Parameters
import_asset_file_function_arn_parameter = aws_ssm.StringParameter(
self,
"import asset file function arn",
string_value=import_asset_file_function.function_arn,
description=f"Import asset file function ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_IMPORT_ASSET_FILE_FUNCTION_TASK_ARN.value,
)
import_metadata_file_function_arn_parameter = aws_ssm.StringParameter(
self,
"import metadata file function arn",
string_value=import_metadata_file_function.function_arn,
description=f"Import metadata file function ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_IMPORT_METADATA_FILE_FUNCTION_TASK_ARN.value,
)
import_dataset_role_arn_parameter = aws_ssm.StringParameter(
self,
"import dataset role arn",
string_value=import_dataset_role.role_arn,
description=f"Import dataset role ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_IMPORT_DATASET_ROLE_ARN.value,
)
update_dataset_catalog = LambdaTask(
self,
"update-dataset-catalog",
directory="update_dataset_catalog",
botocore_lambda_layer=botocore_lambda_layer,
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
self.message_queue.grant_send_messages(update_dataset_catalog.lambda_function)
for storage_writer in [
import_dataset_role,
import_dataset_task.lambda_function,
import_asset_file_function,
import_metadata_file_function,
populate_catalog_lambda,
update_dataset_catalog.lambda_function,
]:
storage_bucket.grant_read_write(storage_writer) # type: ignore[arg-type]
grant_parameter_read_access(
{
import_asset_file_function_arn_parameter: [import_dataset_task.lambda_function],
import_dataset_role_arn_parameter: [import_dataset_task.lambda_function],
import_metadata_file_function_arn_parameter: [import_dataset_task.lambda_function],
processing_assets_table.name_parameter: [
check_stac_metadata_task.lambda_function,
content_iterator_task.lambda_function,
import_dataset_task.lambda_function,
],
validation_results_table.name_parameter: [
check_stac_metadata_task.lambda_function,
content_iterator_task.lambda_function,
validation_summary_task.lambda_function,
upload_status_task.lambda_function,
],
self.message_queue_name_parameter: [update_dataset_catalog.lambda_function],
}
)
success_task = aws_stepfunctions.Succeed(self, "success")
upload_failure = aws_stepfunctions.Fail(self, "upload failure")
validation_failure = aws_stepfunctions.Succeed(self, "validation failure")
############################################################################################
# STATE MACHINE
dataset_version_creation_definition = (
check_stac_metadata_task.next(content_iterator_task)
.next(
aws_stepfunctions.Choice( # type: ignore[arg-type]
self, "check_files_checksums_maybe_array"
)
.when(
aws_stepfunctions.Condition.number_equals(
f"$.{CONTENT_KEY}.{ITERATION_SIZE_KEY}", 1
),
check_files_checksums_single_task.batch_submit_job,
)
.otherwise(check_files_checksums_array_task.batch_submit_job)
.afterwards()
)
.next(
aws_stepfunctions.Choice(self, "content_iteration_finished")
.when(
aws_stepfunctions.Condition.number_equals(
f"$.{CONTENT_KEY}.{NEXT_ITEM_KEY}", -1
),
validation_summary_task.next(
aws_stepfunctions.Choice( # type: ignore[arg-type]
self, "validation_successful"
)
.when(
aws_stepfunctions.Condition.boolean_equals(
f"$.{VALIDATION_KEY}.{SUCCESS_KEY}", True
),
import_dataset_task.next(
wait_before_upload_status_check # type: ignore[arg-type]
)
.next(upload_status_task)
.next(
aws_stepfunctions.Choice(
self, "import_completed" # type: ignore[arg-type]
)
.when(
aws_stepfunctions.Condition.and_(
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Complete"
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{METADATA_UPLOAD_KEY}.status",
"Complete",
),
),
update_dataset_catalog.next(
success_task # type: ignore[arg-type]
),
)
.when(
aws_stepfunctions.Condition.or_(
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{ASSET_UPLOAD_KEY}.status",
"Cancelled",
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Failed"
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{METADATA_UPLOAD_KEY}.status",
"Cancelled",
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{METADATA_UPLOAD_KEY}.status",
"Failed",
),
),
upload_failure, # type: ignore[arg-type]
)
.otherwise(
wait_before_upload_status_check # type: ignore[arg-type]
)
),
)
.otherwise(validation_failure) # type: ignore[arg-type]
),
)
.otherwise(content_iterator_task)
)
)
self.state_machine = aws_stepfunctions.StateMachine(
self,
f"{env_name}-dataset-version-creation",
definition=dataset_version_creation_definition, # type: ignore[arg-type]
)
self.state_machine_parameter = aws_ssm.StringParameter(
self,
"state machine arn",
description=f"State machine ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_DATASET_VERSION_CREATION_STEP_FUNCTION_ARN.value, # pylint:disable=line-too-long
string_value=self.state_machine.state_machine_arn,
)
Tags.of(self).add("ApplicationLayer", "processing") # type: ignore[arg-type]
def __init__(self, scope, id, *args, **kwargs):
super().__init__(scope, id, *args, **kwargs)
# Buckets
source_bucket = s3.Bucket(self, "SourceBucket")
dest_bucket = s3.Bucket(self, "DestinationBucket")
processing_bucket = s3.Bucket(self, "ProcessingBucket")
# Lambda Functions
generate_workflow_input_lambda = aws_lambda.Function(
self, "GenerateWorkflowInputFunction",
code=aws_lambda.Code.from_asset(str(DIST_PATH)),
runtime=aws_lambda.Runtime.PYTHON_3_8,
handler="generate_workflow_input.lambda_handler",
environment={
"InputBucketName": source_bucket.bucket_name,
"ProcessingBucketName": processing_bucket.bucket_name,
"OutputBucketName": dest_bucket.bucket_name
}
)
check_workflow_ready_lambda = aws_lambda.Function(
self, "CheckWorkflowReadyFunction",
code=aws_lambda.Code.from_asset(str(DIST_PATH)),
runtime=aws_lambda.Runtime.PYTHON_3_8,
handler="check_workflow_ready.lambda_handler"
)
string_replace_lambda = aws_lambda.Function(
self, "StringReplaceFunction",
code=aws_lambda.Code.from_asset(str(DIST_PATH)),
runtime=aws_lambda.Runtime.PYTHON_3_8,
handler="string_replace.lambda_handler"
)
calculate_total_earnings_lambda = aws_lambda.Function(
self, "CalculateTotalEarningsFunction",
code=aws_lambda.Code.from_asset(str(DIST_PATH)),
runtime=aws_lambda.Runtime.PYTHON_3_8,
handler="calculate_total_earnings.lambda_handler"
)
convert_csv_to_json_lambda = aws_lambda.Function(
self, "ConvertCsvToJsonFunction",
code=aws_lambda.Code.from_asset(str(DIST_PATH)),
runtime=aws_lambda.Runtime.PYTHON_3_8,
handler="convert_csv_to_json.lambda_handler"
)
# Permissions
source_bucket.grant_read(check_workflow_ready_lambda)
source_bucket.grant_read(string_replace_lambda)
processing_bucket.grant_write(string_replace_lambda)
processing_bucket.grant_read_write(calculate_total_earnings_lambda)
processing_bucket.grant_read(convert_csv_to_json_lambda)
dest_bucket.grant_write(convert_csv_to_json_lambda)
# Outputs
core.CfnOutput(self, "SourceBucketName", value=source_bucket.bucket_name)
core.CfnOutput(self, "DestinationBucketName", value=dest_bucket.bucket_name)
core.CfnOutput(self, "ProcessingBucketName", value=processing_bucket.bucket_name)
core.CfnOutput(self, "GenerateWorkflowInputLambda", value=generate_workflow_input_lambda.function_name)
core.CfnOutput(self, "CheckWorkflowReadyLambda", value=check_workflow_ready_lambda.function_name)
core.CfnOutput(self, "StringReplaceLambda", value=string_replace_lambda.function_name)
core.CfnOutput(self, "CalculateTotalEarningsLambda", value=calculate_total_earnings_lambda.function_name)
core.CfnOutput(self, "ConvertCsvToJsonLambda", value=convert_csv_to_json_lambda.function_name)
# State Machine
generate_workflow_input_task = sf_tasks.LambdaInvoke(
self, "GenerateWorkflowInput",
lambda_function=generate_workflow_input_lambda,
payload_response_only=True
)
check_workflow_ready_task = sf_tasks.LambdaInvoke(
self, "CheckWorkflowReady",
lambda_function=check_workflow_ready_lambda,
input_path="$.CheckWorkflowReady.Input",
result_path="$.CheckWorkflowReady.Output",
payload_response_only=True
)
string_replace_task = sf_tasks.LambdaInvoke(
self, "ReplaceString",
lambda_function=string_replace_lambda,
result_path="$.StringReplace.Output",
payload_response_only=True
)
calculate_total_earnings_task = sf_tasks.LambdaInvoke(
self, "CalculateTotalEarnings",
lambda_function=calculate_total_earnings_lambda,
input_path="$.CalculateTotalEarnings.Input",
result_path="$.CalculateTotalEarnings.Output",
payload_response_only=True
)
convert_csv_to_json_task = sf_tasks.LambdaInvoke(
self, "ConvertCsvToJson",
lambda_function=convert_csv_to_json_lambda,
input_path="$.ConvertCsvToJson.Input",
result_path="$.ConvertCsvToJson.Output",
payload_response_only=True
)
end_task = sf.Succeed(self, "WorkflowEnd")
replace_string_parallel = sf.Map(
self, "ReplaceStringParallel",
items_path="$.StringReplace.Input",
result_path="$.StringReplace.Output"
).iterator(string_replace_task)
workflow_steps = sf.Chain.\
start(replace_string_parallel)\
.next(calculate_total_earnings_task)\
.next(convert_csv_to_json_task)\
.next(end_task)
run_workflow = sf.Choice(self, "RunWorkflowDecision")\
.when(sf.Condition.boolean_equals("$.CheckWorkflowReady.Output", True), workflow_steps)\
.otherwise(end_task)
hello_workflow_state_machine = sf.StateMachine(
self, "HelloWorkflowStateMachine",
definition=sf.Chain\
.start(generate_workflow_input_task)\
.next(check_workflow_ready_task)\
.next(run_workflow)
)
def __init__(self, scope: Construct, id: str, log_level: CfnParameter):
super().__init__(scope, id)
self._bundling = {}
self.log_level = log_level.value_as_string
self.source_path = Path(__file__).parent.parent.parent.parent
self.topic = None
self.subscription = None
self.functions: Dict[Function] = {}
self.policies = Policies(self)
self.create_functions()
# step function steps
check_error = sfn.Choice(self, "Check-Error")
notify_failed = tasks.LambdaInvoke(
self,
"Notify-Failed",
lambda_function=self.functions["SNS"],
payload_response_only=True,
retry_on_service_exceptions=True,
result_path=None,
)
notify_failed.next(sfn.Fail(self, "FailureState"))
create_dataset_group = tasks.LambdaInvoke(
self,
"Create-DatasetGroup",
lambda_function=self.functions["CreateDatasetGroup"],
result_path="$.DatasetGroupNames",
payload_response_only=True,
retry_on_service_exceptions=True,
)
create_dataset_group.add_retry(backoff_rate=1.05,
interval=Duration.seconds(5),
errors=["ResourcePending"])
create_dataset_group.add_catch(notify_failed,
errors=["ResourceFailed"],
result_path="$.serviceError")
create_dataset_group.add_catch(notify_failed,
errors=["States.ALL"],
result_path="$.statesError")
import_data = tasks.LambdaInvoke(
self,
"Import-Data",
lambda_function=self.functions["CreateDatasetImportJob"],
result_path="$.DatasetImportJobArn",
payload_response_only=True,
retry_on_service_exceptions=True,
)
import_data.add_retry(
backoff_rate=1.05,
interval=Duration.seconds(5),
max_attempts=100,
errors=["ResourcePending"],
)
import_data.add_catch(notify_failed,
errors=["ResourceFailed"],
result_path="$.serviceError")
import_data.add_catch(notify_failed,
errors=["States.ALL"],
result_path="$.statesError")
update_not_required = sfn.Succeed(self, "Update-Not-Required")
notify_success = tasks.LambdaInvoke(
self,
"Notify-Success",
lambda_function=self.functions["SNS"],
payload_response_only=True,
retry_on_service_exceptions=True,
result_path=None,
)
notify_prediction_failed = tasks.LambdaInvoke(
self,
"Notify-Prediction-Failed",
lambda_function=self.functions["SNS"],
payload_response_only=True,
retry_on_service_exceptions=True,
result_path=None,
)
notify_prediction_failed.next(sfn.Fail(self, "Prediction-Failed"))
create_predictor = tasks.LambdaInvoke(
self,
"Create-Predictor",
lambda_function=self.functions["CreatePredictor"],
result_path="$.PredictorArn",
payload_response_only=True,
retry_on_service_exceptions=True,
)
create_predictor.add_retry(
backoff_rate=1.05,
interval=Duration.seconds(5),
max_attempts=100,
errors=["ResourcePending", "DatasetsImporting"],
)
create_predictor.add_catch(
notify_prediction_failed,
errors=["ResourceFailed"],
result_path="$.serviceError",
)
create_predictor.add_catch(notify_prediction_failed,
errors=["States.ALL"],
result_path="$.statesError")
create_predictor.add_catch(update_not_required,
errors=["NotMostRecentUpdate"])
create_forecast = tasks.LambdaInvoke(
self,
"Create-Forecast",
lambda_function=self.functions["CreateForecast"],
result_path="$.ForecastArn",
payload_response_only=True,
retry_on_service_exceptions=True,
)
create_forecast.add_retry(
backoff_rate=1.05,
interval=Duration.seconds(5),
max_attempts=100,
errors=["ResourcePending"],
)
create_forecast.add_catch(
notify_prediction_failed,
errors=["ResourceFailed"],
result_path="$.serviceError",
)
create_forecast.add_catch(notify_prediction_failed,
errors=["States.ALL"],
result_path="$.statesError")
export_forecast = tasks.LambdaInvoke(
self,
"Export-Forecast",
lambda_function=self.functions["PrepareForecastExport"],
result_path="$.ExportTableName",
payload_response_only=True,
retry_on_service_exceptions=True,
)
export_forecast.add_catch(
notify_prediction_failed,
errors=["ResourceFailed"],
result_path="$.serviceError",
)
export_forecast.add_catch(notify_prediction_failed,
errors=["States.ALL"],
result_path="$.statesError")
create_forecasts = sfn.Map(
self,
"Create-Forecasts",
items_path="$.DatasetGroupNames",
parameters={
"bucket.$": "$.bucket",
"dataset_file.$": "$.dataset_file",
"dataset_group_name.$": "$$.Map.Item.Value",
"config.$": "$.config",
},
)
# step function definition
definition = (check_error.when(
sfn.Condition.is_present("$.serviceError"),
notify_failed).otherwise(create_dataset_group).afterwards().next(
import_data).next(
create_forecasts.iterator(
create_predictor.next(create_forecast).next(
export_forecast).next(notify_success))))
self.state_machine = sfn.StateMachine(self,
"DeployStateMachine",
definition=definition)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# ------ Necessary Roles ------
roles = IamRole(
self, 'IamRoles'
)
# ------ S3 Buckets ------
# Create Athena bucket
athena_bucket = _s3.Bucket(self, "AthenaBucket",
removal_policy=core.RemovalPolicy.DESTROY
)
# Create Forecast bucket
forecast_bucket = _s3.Bucket(self, "FoecastBucket",
removal_policy=core.RemovalPolicy.DESTROY
)
# ------ Athena ------
# Config Athena query result output location
workgroup_prop = _athena.CfnWorkGroup.WorkGroupConfigurationProperty(
result_configuration=_athena.CfnWorkGroup.ResultConfigurationProperty(
output_location="s3://"+athena_bucket.bucket_name
)
)
# Create Athena workgroup
athena_workgroup = _athena.CfnWorkGroup(
self, 'ForecastGroup',
name='ForecastGroup',
recursive_delete_option=True,
state='ENABLED',
work_group_configuration=workgroup_prop
)
# ------ SNS Topic ------
topic = sns.Topic(
self, 'NotificationTopic',
display_name='StepsTopic'
)
# SNS email subscription. Get the email address from context value(cdk.json)
topic.add_subscription(subs.EmailSubscription(self.node.try_get_context('my_email')))
# ------ Layers ------
shared_layer = _lambda.LayerVersion(
self, 'LambdaLayer',
layer_version_name='testfolderlayer',
code=_lambda.AssetCode('shared/')
)
# ------ Lambdas for stepfuctions------
create_dataset_lambda = _lambda.Function(
self, 'CreateDataset',
function_name='CreateDataset',
code=_lambda.Code.asset('lambdas/createdataset/'),
handler='dataset.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
timeout=core.Duration.seconds(30),
layers=[shared_layer]
)
create_dataset_group_lambda = _lambda.Function(
self, 'CreateDatasetGroup',
function_name='CreateDatasetGroup',
code = _lambda.Code.asset('lambdas/createdatasetgroup/'),
handler = 'datasetgroup.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
layers=[shared_layer]
)
import_data_lambda = _lambda.Function(
self, 'CreateDatasetImportJob',
function_name='CreateDatasetImportJob',
code = _lambda.Code.asset('lambdas/createdatasetimportjob/'),
handler = 'datasetimport.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
environment= {
'FORECAST_ROLE': roles.forecast_role.role_arn
},
layers=[shared_layer]
)
create_predictor_lambda = _lambda.Function(
self, 'CreatePredictor',
function_name='CreatePredictor',
code = _lambda.Code.asset('lambdas/createpredictor/'),
handler = 'predictor.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
layers=[shared_layer]
)
create_forecast_lambda = _lambda.Function(
self, 'CreateForecast',
function_name='CreateForecast',
code = _lambda.Code.asset('lambdas/createforecast/'),
handler = 'forecast.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
environment= {
'EXPORT_ROLE': roles.forecast_role.role_arn
},
layers=[shared_layer],
timeout=core.Duration.seconds(30)
)
# Deploy lambda with python dependencies from requirements.txt
update_resources_lambda = _lambda_python.PythonFunction(
self, 'UpdateResources',
function_name='UpdateResources',
entry='lambdas/updateresources/',
index='update.py',
handler='lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.update_role,
environment= {
'ATHENA_WORKGROUP': athena_workgroup.name,
'ATHENA_BUCKET' : athena_bucket.bucket_name
},
layers=[shared_layer],
timeout=core.Duration.seconds(900)
)
notify_lambda = _lambda.Function(
self, 'NotifyTopic',
function_name='NotifyTopic',
code = _lambda.Code.asset('lambdas/notify/'),
handler = 'notify.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
environment= {
'SNS_TOPIC_ARN': topic.topic_arn
},
layers=[shared_layer]
)
delete_forecast_lambda = _lambda.Function(
self, 'DeleteForecast',
function_name='DeleteForecast',
code = _lambda.Code.asset('lambdas/deleteforecast/'),
handler = 'deleteforecast.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
layers=[shared_layer]
)
delete_predctor_lambda = _lambda.Function(
self, 'DeletePredictor',
function_name='DeletePredictor',
code = _lambda.Code.asset('lambdas/deletepredictor/'),
handler = 'deletepredictor.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
layers=[shared_layer]
)
delete_importjob_lambda = _lambda.Function(
self, 'DeleteImportJob',
function_name='DeleteImportJob',
code = _lambda.Code.asset('lambdas/deletedatasetimport/'),
handler = 'deletedataset.lambda_handler',
runtime = _lambda.Runtime.PYTHON_3_7,
role=roles.lambda_role,
layers=[shared_layer]
)
# ------ StepFunctions ------
strategy_choice = sfn.Choice(
self, 'Strategy-Choice'
)
success_state = sfn.Succeed(
self, 'SuccessState'
)
failed = sfn_tasks.LambdaInvoke(
self, 'Failed',
lambda_function = notify_lambda,
result_path=None
).next(strategy_choice)
create_dataset_job = sfn_tasks.LambdaInvoke(
self, 'Create-Dataset',
lambda_function = create_dataset_lambda,
retry_on_service_exceptions=True,
payload_response_only=True
)
self.add_retry_n_catch(create_dataset_job, failed)
create_dataset_group_job = sfn_tasks.LambdaInvoke(
self, 'Create-DatasetGroup',
lambda_function = create_dataset_group_lambda,
payload_response_only=True
)
self.add_retry_n_catch(create_dataset_group_job, failed)
import_data_job = sfn_tasks.LambdaInvoke(
self, 'Import-Data',
lambda_function = import_data_lambda,
payload_response_only=True
)
self.add_retry_n_catch(import_data_job, failed)
create_predictor_job = sfn_tasks.LambdaInvoke(
self, 'Create-Predictor',
lambda_function = create_predictor_lambda,
payload_response_only=True
)
self.add_retry_n_catch(create_predictor_job, failed)
create_forecast_job = sfn_tasks.LambdaInvoke(
self, 'Create-Forecast',
lambda_function = create_forecast_lambda,
payload_response_only=True
)
self.add_retry_n_catch(create_forecast_job, failed)
update_resources_job = sfn_tasks.LambdaInvoke(
self, 'Update-Resources',
lambda_function = update_resources_lambda,
payload_response_only=True
)
self.add_retry_n_catch(update_resources_job, failed)
notify_success = sfn_tasks.LambdaInvoke(
self, 'Notify-Success',
lambda_function = notify_lambda,
payload_response_only=True
)
delete_forecast_job = sfn_tasks.LambdaInvoke(
self, 'Delete-Forecast',
lambda_function = delete_forecast_lambda,
payload_response_only=True
)
self.delete_retry(delete_forecast_job)
delete_predictor_job = sfn_tasks.LambdaInvoke(
self, 'Delete-Predictor',
lambda_function = delete_predctor_lambda,
payload_response_only=True
)
self.delete_retry(delete_predictor_job)
delete_import_job = sfn_tasks.LambdaInvoke(
self, 'Delete-ImportJob',
lambda_function = delete_importjob_lambda,
payload_response_only=True
)
self.delete_retry(delete_import_job)
definition = create_dataset_job\
.next(create_dataset_group_job)\
.next(import_data_job)\
.next(create_predictor_job)\
.next(create_forecast_job)\
.next(update_resources_job)\
.next(notify_success)\
.next(strategy_choice.when(sfn.Condition.boolean_equals('$.params.PerformDelete', False), success_state)\
.otherwise(delete_forecast_job).afterwards())\
.next(delete_predictor_job)\
.next(delete_import_job)
deployt_state_machine = sfn.StateMachine(
self, 'StateMachine',
definition = definition
# role=roles.states_execution_role
)
# S3 event trigger lambda
s3_lambda = _lambda.Function(
self, 'S3Lambda',
function_name='S3Lambda',
code=_lambda.Code.asset('lambdas/s3lambda/'),
handler='parse.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_7,
role=roles.trigger_role,
environment= {
'STEP_FUNCTIONS_ARN': deployt_state_machine.state_machine_arn,
'PARAMS_FILE': self.node.try_get_context('parameter_file')
}
)
s3_lambda.add_event_source(
event_src.S3EventSource(
bucket=forecast_bucket,
events=[_s3.EventType.OBJECT_CREATED],
filters=[_s3.NotificationKeyFilter(
prefix='train/',
suffix='.csv'
)]
)
)
# CloudFormation output
core.CfnOutput(
self, 'StepFunctionsName',
description='Step Functions Name',
value=deployt_state_machine.state_machine_name
)
core.CfnOutput(
self, 'ForecastBucketName',
description='Forecast bucket name to drop you files',
value=forecast_bucket.bucket_name
)
core.CfnOutput(
self, 'AthenaBucketName',
description='Athena bucket name to drop your files',
value=athena_bucket.bucket_name
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Step Function Starts Here
# The first thing we need to do is see if they are asking for pineapple on a pizza
pineapple_check_lambda = _lambda.Function(
self,
"pineappleCheckLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="orderPizza.handler",
code=_lambda.Code.from_asset("lambda_fns"),
)
# Step functions are built up of steps, we need to define our first step
order_pizza = step_fn_tasks.LambdaInvoke(
self,
'Order Pizza Job',
lambda_function=pineapple_check_lambda,
input_path='$.flavour',
result_path='$.pineappleAnalysis',
payload_response_only=True)
# Pizza Order failure step defined
pineapple_detected = step_fn.Fail(self,
'Sorry, We Dont add Pineapple',
cause='They asked for Pineapple',
error='Failed To Make Pizza')
# If they didnt ask for pineapple let's cook the pizza
cook_pizza = step_fn.Succeed(self,
'Lets make your pizza',
output_path='$.pineappleAnalysis')
# If they ask for a pizza with pineapple, fail. Otherwise cook the pizza
definition = step_fn.Chain \
.start(order_pizza) \
.next(step_fn.Choice(self, 'With Pineapple?')
.when(step_fn.Condition.boolean_equals('$.pineappleAnalysis.containsPineapple', True),
pineapple_detected)
.otherwise(cook_pizza))
state_machine = step_fn.StateMachine(
self,
'StateMachine',
definition=definition,
timeout=core.Duration.minutes(5),
tracing_enabled=True,
state_machine_type=step_fn.StateMachineType.EXPRESS)
# HTTP API Definition
# Give our gateway permissions to interact with SNS
http_api_role = iam.Role(
self,
'HttpApiRole',
assumed_by=iam.ServicePrincipal('apigateway.amazonaws.com'),
inline_policies={
"AllowSFNExec":
iam.PolicyDocument(statements=[
iam.PolicyStatement(
actions=["states:StartSyncExecution"],
effect=iam.Effect.ALLOW,
resources=[state_machine.state_machine_arn])
])
})
api = api_gw.HttpApi(self,
'the_state_machine_api',
create_default_stage=True)
# create an AWS_PROXY integration between the HTTP API and our Step Function
integ = api_gw.CfnIntegration(
self,
'Integ',
api_id=api.http_api_id,
integration_type='AWS_PROXY',
connection_type='INTERNET',
integration_subtype='StepFunctions-StartSyncExecution',
credentials_arn=http_api_role.role_arn,
request_parameters={
"Input": "$request.body",
"StateMachineArn": state_machine.state_machine_arn
},
payload_format_version="1.0",
timeout_in_millis=10000)
api_gw.CfnRoute(self,
'DefaultRoute',
api_id=api.http_api_id,
route_key=api_gw.HttpRouteKey.DEFAULT.key,
target="integrations/" + integ.ref)
core.CfnOutput(self, 'HTTP API URL', value=api.url)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
###
# DynamoDB Table
###
# We store Flight, Hotel and Rental Car bookings in the same table.
#
# For more help with single table DB structures see - https://www.dynamodbbook.com/
# pk - the trip_id e.g. 1234
# sk - bookingtype#booking_id e.g. HOTEL#345634, FLIGHT#574576, PAYMENT#45245
table = dynamo_db.Table(self, "Bookings",
partition_key=dynamo_db.Attribute(name="pk", type=dynamo_db.AttributeType.STRING),
sort_key=dynamo_db.Attribute(name="sk", type=dynamo_db.AttributeType.STRING)
)
###
# Lambda Functions
###
# We need Booking and Cancellation functions for our 3 services
#
# All functions need access to our DynamoDB table above.
# We also need to take payment for this trip
#
# 1) Flights
# 2) Hotel
# 3) Payment
# 1) Flights
reserve_flight_lambda = self.create_lambda(scope=self, lambda_id="reserveFlightLambdaHandler",
handler='flights/reserveFlight.handler', table=table)
confirm_flight_lambda = self.create_lambda(scope=self, lambda_id="confirmFlightLambdaHandler",
handler='flights/confirmFlight.handler', table=table)
cancel_flight_lambda = self.create_lambda(scope=self, lambda_id="cancelFlightLambdaHandler",
handler='flights/cancelFlight.handler', table=table)
# 2) Hotel
reserve_hotel_lambda = self.create_lambda(scope=self, lambda_id="reserveHotelLambdaHandler",
handler='hotel/reserveHotel.handler', table=table)
confirm_hotel_lambda = self.create_lambda(scope=self, lambda_id="confirmHotelLambdaHandler",
handler='hotel/confirmHotel.handler', table=table)
cancel_hotel_lambda = self.create_lambda(scope=self, lambda_id="cancelHotelLambdaHandler",
handler='hotel/cancelHotel.handler', table=table)
# 3) Payment For Holiday
take_payment_lambda = self.create_lambda(scope=self, lambda_id="takePaymentLambdaHandler",
handler='payment/takePayment.handler', table=table)
refund_payment_lambda = self.create_lambda(scope=self, lambda_id="refundPaymentLambdaHandler",
handler='payment/refundPayment.handler', table=table)
###
# Saga Pattern Step Function
###
# Follows a strict order:
# 1) Reserve Flights and Hotel
# 2) Take Payment
# 3) Confirm Flight and Hotel booking
# Our two end states
booking_succeeded = step_fn.Succeed(self, 'We have made your booking!')
booking_failed = step_fn.Fail(self, "Sorry, We Couldn't make the booking")
# 1) Reserve Flights and Hotel
cancel_hotel_reservation = step_fn.Task(self, 'CancelHotelReservation',
task=step_fn_tasks.InvokeFunction(cancel_hotel_lambda),
result_path='$.CancelHotelReservationResult'
).add_retry(max_attempts=3).next(booking_failed)
reserve_hotel = step_fn.Task(self, 'ReserveHotel',
task=step_fn_tasks.InvokeFunction(reserve_hotel_lambda),
result_path='$.ReserveHotelResult'
).add_catch(cancel_hotel_reservation, result_path="$.ReserveHotelError")
cancel_flight_reservation = step_fn.Task(self, 'CancelFlightReservation',
task=step_fn_tasks.InvokeFunction(cancel_flight_lambda),
result_path='$.CancelFlightReservationResult'
).add_retry(max_attempts=3).next(cancel_hotel_reservation)
reserve_flight = step_fn.Task(self, 'ReserveFlight',
task=step_fn_tasks.InvokeFunction(reserve_flight_lambda),
result_path='$.ReserveFlightResult'
).add_catch(cancel_flight_reservation, result_path="$.ReserveFlightError")
# 2) Take Payment
refund_payment = step_fn.Task(self, 'RefundPayment',
task=step_fn_tasks.InvokeFunction(refund_payment_lambda),
result_path='$.RefundPaymentResult'
).add_retry(max_attempts=3).next(cancel_flight_reservation)
take_payment = step_fn.Task(self, 'TakePayment',
task=step_fn_tasks.InvokeFunction(take_payment_lambda),
result_path='$.TakePaymentResult'
).add_catch(refund_payment, result_path="$.TakePaymentError")
# 3) Confirm Flight and Hotel Booking
confirm_hotel = step_fn.Task(self, 'ConfirmHotelBooking',
task=step_fn_tasks.InvokeFunction(confirm_hotel_lambda),
result_path='$.ConfirmHotelBookingResult'
).add_catch(refund_payment, result_path="$.ConfirmHotelBookingError")
confirm_flight = step_fn.Task(self, 'ConfirmFlight',
task=step_fn_tasks.InvokeFunction(confirm_flight_lambda),
result_path='$.ConfirmFlightResult'
).add_catch(refund_payment, result_path="$.ConfirmFlightError")
definition = step_fn.Chain \
.start(reserve_hotel) \
.next(reserve_flight) \
.next(take_payment) \
.next(confirm_hotel) \
.next(confirm_flight) \
.next(booking_succeeded)
saga = step_fn.StateMachine(self, 'BookingSaga', definition=definition, timeout=core.Duration.minutes(5))
# defines an AWS Lambda resource to connect to our API Gateway and kick
# off our step function
saga_lambda = _lambda.Function(self, "sagaLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="sagaLambda.handler",
code=_lambda.Code.from_asset("lambdas"),
environment={
'statemachine_arn': saga.state_machine_arn
}
)
saga.grant_start_execution(saga_lambda)
# defines an API Gateway REST API resource backed by our "stateMachineLambda" function.
api_gw.LambdaRestApi(self, 'SagaPatternSingleTable',
handler=saga_lambda
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Lets create couple of instances to test):
vpc = _ec2.Vpc(self,
"abacVPC",
cidr="10.13.0.0/21",
max_azs=2,
nat_gateways=0,
subnet_configuration=[
_ec2.SubnetConfiguration(
name="pubSubnet",
cidr_mask=24,
subnet_type=_ec2.SubnetType.PUBLIC)
])
core.Tag.add(vpc,
key="ServiceProvider",
value="KonStone",
include_resource_types=[])
weak_sg = _ec2.SecurityGroup(
self,
"web_sec_grp",
vpc=vpc,
description="Allow internet access from the world",
allow_all_outbound=True)
# vpc_cidr_block
# weak_sg.add_ingress_rule(_ec2.Peer.any_ipv4(),
weak_sg.add_ingress_rule(_ec2.Peer.ipv4(vpc.vpc_cidr_block),
_ec2.Port.tcp(22),
"Allow SSH access from the VPC Only.")
# We are using the latest AMAZON LINUX AMI
# Benefit of having SSM Agent pre-installed
ami_id = _ec2.AmazonLinuxImage(generation=_ec2.AmazonLinuxGeneration.
AMAZON_LINUX_2).get_image(self).image_id
# https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_iam/Role.html
instace_profile_role = _iam.Role(
self,
'ec2ssmroleid',
assumed_by=_iam.ServicePrincipal('ec2.amazonaws.com'),
role_name="instace_profile_role")
instace_profile_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonSSMManagedInstanceCore'))
instance_profile_role_additional_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
"arn:aws:logs:*:*:*",
],
actions=["logs:Create*", "logs:PutLogEvents"])
instance_profile_role_additional_perms.sid = "PutBucketPolicy"
instace_profile_role.add_to_policy(
instance_profile_role_additional_perms)
inst_profile_01 = _iam.CfnInstanceProfile(
self,
"instProfile01Id",
roles=[instace_profile_role.role_name],
)
# Let us bootstrap the server with the required agents
try:
with open("./bootstrap_scripts/install_agents.sh",
mode='rb') as file:
bootstrap_data = file.read()
except OSError:
print('Failed to get UserData script')
install_agents = _ec2.UserData.for_linux()
install_agents.add_commands(str(bootstrap_data, 'utf-8'))
# The EC2 Instance to monitor for failed SSH Logins
ssh_monitored_inst_01 = _ec2.CfnInstance(
self,
"sshMonitoredInstance01",
image_id=ami_id,
instance_type="t2.micro",
monitoring=False,
tags=[{
"key": "ServiceProvider",
"value": "KonStone"
}],
iam_instance_profile=inst_profile_01.ref,
network_interfaces=[{
"deviceIndex": "0",
"associatePublicIpAddress": True,
"subnetId": vpc.public_subnets[0].subnet_id,
"groupSet": [weak_sg.security_group_id]
}], #https: //github.com/aws/aws-cdk/issues/3419
user_data=core.Fn.base64(install_agents.render()),
)
"""
linux_ami = _ec2.GenericLinuxImage({ "cn-northwest-1": "ami-0f62e91915e16cfc2","eu-west-1": "ami-12345678"})
ssh_monitored_inst_01_02 = _ec2.Instance(self,
"monitoredInstance02",
instance_type=_ec2.InstanceType(instance_type_identifier="t2.micro"),
instance_name="monitoredInstance02",
machine_image=linux_ami,
vpc=vpc,
security_group=[weak_sg.security_group_id],
# vpc_subnets=_ec2.SubnetSelection(subnet_type=_ec2.SubnetType.PUBLIC)
vpc_subnets=vpc.public_subnets[0].subnet_id,
# user_data=_ec2.UserData.custom(t_user_data)
)
"""
# The log group name to store logs
info_sec_ops_log_group = _logs.LogGroup(
self,
"infoSecOpsLogGroupId",
log_group_name=(f"/Mystique/InfoSec/Automation/"
f"{ssh_monitored_inst_01.ref}"),
retention=_logs.RetentionDays.ONE_WEEK)
# Defines an AWS Lambda resource
with open("lambda_src/quarantine_ec2_instance.py",
encoding="utf8") as fp:
quarantine_ec2_instance_fn_handler_code = fp.read()
quarantine_ec2_instance_fn = _lambda.Function(
self,
id='quarantineEc2InstanceFnId',
function_name="quarantine_ec2_instance",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(quarantine_ec2_instance_fn_handler_code),
handler='index.lambda_handler',
timeout=core.Duration.seconds(5))
quarantine_ec2_instance_fn_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
"*",
],
actions=[
"ec2:RevokeSecurityGroupIngress",
"ec2:DescribeSecurityGroupReferences",
"ec2:RevokeSecurityGroupEgress",
"ec2:ApplySecurityGroupsToClientVpnTargetNetwork",
"ec2:DescribeSecurityGroups", "ec2:CreateSecurityGroup",
"ec2:DescribeInstances", "ec2:CreateTags", "ec2:StopInstances",
"ec2:CreateVolume", "ec2:CreateSnapshots",
"ec2:CreateSnapshot", "ec2:DescribeSnapshots",
"ec2:ModifyInstanceAttribute"
])
quarantine_ec2_instance_fn_perms.sid = "AllowLambdaToQuarantineEC2"
quarantine_ec2_instance_fn.add_to_role_policy(
quarantine_ec2_instance_fn_perms)
info_sec_ops_topic = _sns.Topic(self,
"infoSecOpsTopicId",
display_name="InfoSecTopic",
topic_name="InfoSecOpsTopic")
# Ref: https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html
###############################################################################
################# STEP FUNCTIONS EXPERIMENTAL CODE - UNSTABLE #################
###############################################################################
quarantine_ec2_instance_task = _sfn.Task(
self,
"Quarantine EC2 Instance",
task=_tasks.InvokeFunction(quarantine_ec2_instance_fn),
result_path="$")
notify_secops_task = _sfn.Task(
self,
"Notify InfoSecOps",
task=_tasks.PublishToTopic(
info_sec_ops_topic,
integration_pattern=_sfn.ServiceIntegrationPattern.
FIRE_AND_FORGET,
message=_sfn.TaskInput.from_data_at("$.message"),
subject="SSH Error Response Notification"))
ssh_error_response_failure = _sfn.Fail(
self,
"SSH Error Response Actions Failed",
cause="All Response Actions were NOT completed",
error="Check Logs")
ssh_error_response_success = _sfn.Succeed(
self,
"SSH Error Response Actions Succeeded",
comment="All Response Action Completed Successfully",
)
ssh_error_response_sfn_definition = quarantine_ec2_instance_task\
.next(notify_secops_task\
.next(_sfn.Choice(self, "SSH Errors Response Complete?")\
.when(_sfn.Condition.number_equals("$.SdkHttpMetadata.HttpStatusCode", 200),ssh_error_response_success)\
.when(_sfn.Condition.not_(
_sfn.Condition.number_equals("$.SdkHttpMetadata.HttpStatusCode", 200)), ssh_error_response_failure)\
.otherwise(ssh_error_response_failure)
)
)
ssh_error_response_statemachine = _sfn.StateMachine(
self,
"stateMachineId",
definition=ssh_error_response_sfn_definition,
timeout=core.Duration.minutes(5))
###############################################################################
################# STEP FUNCTIONS EXPERIMENTAL CODE - UNSTABLE #################
###############################################################################
# LAMBDA TO TRIGGER STATE MACHINE - since state cannot be invoked by SNS
with open("lambda_src/trigger_state_machine.py",
encoding="utf8") as fp:
trigger_state_machine_fn_handler_code = fp.read()
trigger_state_machine_fn = _lambda.Function(
self,
id='sshErrorResponseFnId',
function_name="trigger_ssh_error_response_state_machine_fn",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(trigger_state_machine_fn_handler_code),
# code=_lambda.Code.asset("lambda_src/is_policy_permissive.py"),
# code=_lambda.Code.asset('lambda_src'),
# code=_lambda.InlineCode(code_body),
handler='index.lambda_handler',
timeout=core.Duration.seconds(5),
environment={
"STATE_MACHINE_ARN":
f"{ssh_error_response_statemachine.state_machine_arn}",
})
trigger_state_machine_fn_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
f"{ssh_error_response_statemachine.state_machine_arn}",
],
actions=["states:StartExecution"])
trigger_state_machine_fn_perms.sid = "PutBucketPolicy"
trigger_state_machine_fn.add_to_role_policy(
trigger_state_machine_fn_perms)
"""
version = trigger_state_machine_fn.add_version(name=datetime.now().isoformat())
trigger_state_machine_fn_alias = _lambda.Alias(self,
'lmdaAliasId',
alias_name='MystiqueTestAlias',
version=version
)
"""
# Lets add permission to SNS to trigger our lambda function
trigger_lambda_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
trigger_state_machine_fn.function_arn,
],
actions=[
"lambda:InvokeFunction",
])
trigger_lambda_perms.sid = "TriggerLambaFunction"
# info_sec_ops_topic.add_to_resource_policy( trigger_lambda_perms )
# Subscribe InfoSecOps Email to topic
info_sec_ops_topic.add_subscription(
_subs.EmailSubscription(global_args.INFO_SEC_OPS_EMAIL))
# info_sec_ops_topic.add_subscription(_subs.LambdaSubscription(trigger_state_machine_fn))
trigger_state_machine_fn_alarm = trigger_state_machine_fn.metric_all_errors(
).create_alarm(
self,
"fn-error-alarm",
threshold=5,
alarm_name="trigger_state_machine_fn_error_alarm",
evaluation_periods=5,
period=core.Duration.minutes(1),
)
subscribe_trigger_state_machine_fn_to_logs = _logs.SubscriptionFilter(
self,
"sshErrorLogSubscriptionId",
log_group=info_sec_ops_log_group,
destination=_logs_destination.LambdaDestination(
trigger_state_machine_fn),
filter_pattern=_logs.FilterPattern.space_delimited(
"Mon", "day", "timestamp", "ip", "id", "status",
"...").where_string("status", "=", "Invalid"),
)
# https://pypi.org/project/aws-cdk.aws-logs/
# We are creating three filter
# tooManySshDisconnects, invalidSshUser and invalidSshKey:
# When a user tries to SSH with invalid username the next line is logged in the SSH log file:
# Apr 20 02:39:35 ip-172-31-63-56 sshd[17136]: Received disconnect from xxx.xxx.xxx.xxx: 11: [preauth]
too_many_ssh_disconnects_metric = _cloudwatch.Metric(
namespace=f"{global_args.OWNER}",
metric_name="tooManySshDisconnects")
too_many_ssh_disconnects_filter = _logs.MetricFilter(
self,
"tooManySshDisconnectsFilterId",
log_group=info_sec_ops_log_group,
metric_namespace=too_many_ssh_disconnects_metric.namespace,
metric_name=too_many_ssh_disconnects_metric.metric_name,
filter_pattern=_logs.FilterPattern.space_delimited(
"Mon", "day", "timestamp", "ip", "id", "msg1", "msg2",
"...").where_string("msg2", "=", "disconnect"),
metric_value="1")
invalid_ssh_user_metric = _cloudwatch.Metric(
namespace=f"{global_args.OWNER}",
metric_name="invalidSshUser",
)
invalid_ssh_user_filter = _logs.MetricFilter(
self,
"invalidSshUserFilterId",
log_group=info_sec_ops_log_group,
metric_namespace=invalid_ssh_user_metric.namespace,
metric_name=invalid_ssh_user_metric.metric_name,
filter_pattern=_logs.FilterPattern.space_delimited(
"Mon", "day", "timestamp", "ip", "id", "status",
"...").where_string("status", "=", "Invalid"),
metric_value="1")
invalid_ssh_key_metric = _cloudwatch.Metric(
namespace=f"{global_args.OWNER}", metric_name="invalidSshKey")
invalid_ssh_key_filter = _logs.MetricFilter(
self,
"invalidSshKeyFilterId",
log_group=info_sec_ops_log_group,
metric_namespace=invalid_ssh_key_metric.namespace,
metric_name=invalid_ssh_key_metric.metric_name,
filter_pattern=_logs.FilterPattern.space_delimited(
"Mon", "day", "timestamp", "ip", "id", "msg1", "msg2",
"...").where_string("msg1", "=", "Connection").where_string(
"msg2", "=", "closed"),
metric_value="1")
# Now let us create alarms
# alarm is raised there are more than 5(threshold) of the measured metrics in two(datapoint) of the last three seconds(evaluation):
# Period=60Seconds, Eval=3, Threshold=5
too_many_ssh_disconnects_alarm = _cloudwatch.Alarm(
self,
"tooManySshDisconnectsAlarmId",
alarm_name="too_many_ssh_disconnects_alarm",
alarm_description=
"The number disconnect requests is greater then 5, even 1 time in 3 minutes",
metric=too_many_ssh_disconnects_metric,
actions_enabled=True,
period=core.Duration.minutes(1),
threshold=5,
evaluation_periods=3,
datapoints_to_alarm=1,
statistic="sum",
comparison_operator=_cloudwatch.ComparisonOperator.
GREATER_THAN_OR_EQUAL_TO_THRESHOLD)
invalid_ssh_user_alarm = _cloudwatch.Alarm(
self,
"invalidSshUserAlarmId",
alarm_name="too_many_invalid_ssh_users_alarm",
alarm_description=
"The number of invalid ssh users connecting is greater then 5, even 1 time in 3 minutes",
metric=invalid_ssh_user_metric,
actions_enabled=True,
period=core.Duration.minutes(1),
threshold=5,
evaluation_periods=3,
datapoints_to_alarm=1,
statistic="sum",
comparison_operator=_cloudwatch.ComparisonOperator.
GREATER_THAN_THRESHOLD)
invalid_ssh_user_alarm.add_alarm_action(
_cloudwatch_actions.SnsAction(info_sec_ops_topic))
invalid_ssh_key_alarm = _cloudwatch.Alarm(
self,
"invalidSshKeyAlarmId",
alarm_name="too_many_invalid_ssh_key_alarm",
alarm_description=
"The number of invalid ssh keys connecting is greater then 5, even 1 time in 3 minutes",
metric=invalid_ssh_key_metric,
actions_enabled=True,
period=core.Duration.minutes(1),
threshold=5,
evaluation_periods=3,
datapoints_to_alarm=1,
statistic="sum",
comparison_operator=_cloudwatch.ComparisonOperator.
GREATER_THAN_OR_EQUAL_TO_THRESHOLD)
invalid_ssh_key_alarm.add_alarm_action(
_cloudwatch_actions.SnsAction(info_sec_ops_topic))
###########################################
################# OUTPUTS #################
###########################################
output0 = core.CfnOutput(
self,
"SecuirtyAutomationFrom",
value=f"{global_args.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output1_1 = core.Fn.get_att(
logical_name_of_resource="sshMonitoredInstance01",
attribute_name="PublicIp")
output1 = core.CfnOutput(self,
"MonitoredInstance",
value=output1_1.to_string(),
description="Web Server Public IP to attack")
output2 = core.CfnOutput(
self,
"SSHAlarms",
value=
(f"https://console.aws.amazon.com/cloudwatch/home?region="
f"{core.Aws.REGION}"
f"#/configuration/"
f"#alarmsV2:?search=ssh&alarmStateFilter=ALL&alarmTypeFilter=ALL"
),
description="Check out the cloudwatch Alarms")
output3 = core.CfnOutput(
self,
"SubscribeToNotificationTopic",
value=(f"https://console.aws.amazon.com/sns/v3/home?"
f"{core.Aws.REGION}"
f"#/topic/"
f"{info_sec_ops_topic.topic_arn}"),
description=
"Add your email to subscription and confirm subscription")
output_test_1 = core.CfnOutput(
self,
"ToGenInvalidKeyErrors",
value=
(f"for i in {{1..30}}; do ssh -i $RANDOM ec2-user@{output1_1.to_string()}; sleep 2; done &"
),
description=
"Generates random key names and connects to server 30 times over 60 seconds"
)
output_test_2 = core.CfnOutput(
self,
"ToGenInvalidUserErrors",
value=
(f"for i in {{1..30}}; do ssh ec2-user$RANDOM@{output1_1.to_string()}; sleep 2; done &"
),
description=
"Generates random user names and connects to server 30 times over 60 seconds"
)
"""
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The start of the image pipeline
imageBucket = aws_s3.Bucket(self, "imageBucket")
# Capture API activity with a trail
imageBucketTrail = aws_cloudtrail.Trail(self,
"imageBucketTrail",
is_multi_region_trail=False)
# Restrict to S3 data-plane events
imageBucketTrail.add_s3_event_selector(
include_management_events=False,
prefixes=[f"{imageBucket.bucket_arn}/"],
read_write_type=aws_cloudtrail.ReadWriteType.WRITE_ONLY)
# Filter to just PutObject and CopyObject events
imageBucketRule = aws_events.Rule(
self,
"imageBucketRule",
event_pattern={
"source": ["aws.s3"],
"detail": {
"eventSource": ["s3.amazonaws.com"],
"eventName": ["PutObject", "CopyObject"],
"requestParameters": {
"bucketName": [imageBucket.bucket_name]
}
}
})
#--
# Lambda Layers
#--------------------#
opencvLayer = aws_lambda.LayerVersion(
self,
'opencvLayer',
code=aws_lambda.AssetCode('layers/opencvLayer'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6])
boto3Layer = aws_lambda.LayerVersion(
self,
'boto3Layer',
code=aws_lambda.AssetCode('layers/boto3Layer'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6])
#--
# Lambda Functions
#--------------------#
# Gather info about an image, name, extension, etc
getImageInfoFunc = aws_lambda.Function(
self,
"getImageInfoFunc",
code=aws_lambda.AssetCode('functions/getImageInfoFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6)
# The home for the website
webBucket = aws_s3.Bucket(self,
"webBucket",
website_index_document='index.html')
# Copy the image to the web bucket
copyImageFunc = aws_lambda.Function(
self,
"copyImageFunc",
code=aws_lambda.AssetCode('functions/copyImageFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
layers=[boto3Layer],
environment={
'OUTPUTBUCKET': webBucket.bucket_name,
'OUTPUTPREFIX': 'images/'
})
# Grant permissions to read from the source and write to the desination
imageBucket.grant_read(copyImageFunc)
webBucket.grant_write(copyImageFunc)
# Create a thumbnail of the image and place in the web bucket
createThumbnailFunc = aws_lambda.Function(
self,
"createThumbnailFunc",
code=aws_lambda.AssetCode('functions/createThumbnailFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
layers=[boto3Layer, opencvLayer],
timeout=core.Duration.seconds(10),
memory_size=256,
environment={
'OUTPUTBUCKET': webBucket.bucket_name,
'OUTPUTPREFIX': 'images/'
})
# Grant permissions to read from the source and write to the desination
imageBucket.grant_read(createThumbnailFunc)
webBucket.grant_write(createThumbnailFunc)
# Store page information
pageTable = aws_dynamodb.Table(
self,
'pageTable',
partition_key={
'name': 'pageName',
'type': aws_dynamodb.AttributeType.STRING
},
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST,
stream=aws_dynamodb.StreamViewType.NEW_IMAGE)
# Save page and image information
updatePageInfoFunc = aws_lambda.Function(
self,
"updatePageInfoFunc",
code=aws_lambda.AssetCode('functions/updatePageInfoFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
layers=[boto3Layer],
environment={
'PAGETABLE': pageTable.table_name,
'PAGEPREFIX': 'posts/'
})
# Grant permissions to write to the page table
pageTable.grant_write_data(updatePageInfoFunc)
imagePipelineDone = aws_stepfunctions.Succeed(self,
"Done processing image")
updatePageInfoJob = aws_stepfunctions.Task(
self,
'Update page info',
task=aws_stepfunctions_tasks.InvokeFunction(updatePageInfoFunc))
updatePageInfoJob.next(imagePipelineDone)
copyImageJob = aws_stepfunctions.Task(
self,
'Copy image',
task=aws_stepfunctions_tasks.InvokeFunction(copyImageFunc))
createThumbnailJob = aws_stepfunctions.Task(
self,
'Create thumbnail',
task=aws_stepfunctions_tasks.InvokeFunction(createThumbnailFunc))
# These tasks can be done in parallel
processImage = aws_stepfunctions.Parallel(self,
'Process image',
result_path="$.images")
processImage.branch(copyImageJob)
processImage.branch(createThumbnailJob)
processImage.next(updatePageInfoJob)
# Results of file extension check
notPng = aws_stepfunctions.Succeed(self, "Not a PNG")
# Verify the file extension
checkForPng = aws_stepfunctions.Choice(self, 'Is a PNG?')
checkForPng.when(
aws_stepfunctions.Condition.string_equals('$.extension', 'png'),
processImage)
checkForPng.otherwise(notPng)
# A single image pipeline job for testing
getImageInfoJob = aws_stepfunctions.Task(
self,
'Get image info',
task=aws_stepfunctions_tasks.InvokeFunction(getImageInfoFunc))
getImageInfoJob.next(checkForPng)
# Configure the image pipeline and starting state
imagePipeline = aws_stepfunctions.StateMachine(
self, "imagePipeline", definition=getImageInfoJob)
# Matching events start the image pipline
imageBucketRule.add_target(
aws_events_targets.SfnStateMachine(
imagePipeline,
input=aws_events.RuleTargetInput.from_event_path(
"$.detail.requestParameters")))
def __init__( # pylint: disable=too-many-arguments
self,
scope: Construct,
stack_id: str,
*,
botocore_lambda_layer: aws_lambda_python.PythonLayerVersion,
datasets_table: Table,
deploy_env: str,
storage_bucket: aws_s3.Bucket,
storage_bucket_parameter: aws_ssm.StringParameter,
validation_results_table: Table,
**kwargs: Any,
) -> None:
# pylint: disable=too-many-locals
super().__init__(scope, stack_id, **kwargs)
############################################################################################
# PROCESSING ASSETS TABLE
processing_assets_table = Table(
self,
f"{deploy_env}-processing-assets",
deploy_env=deploy_env,
parameter_name=ParameterName.PROCESSING_ASSETS_TABLE_NAME,
sort_key=aws_dynamodb.Attribute(
name="sk", type=aws_dynamodb.AttributeType.STRING),
)
############################################################################################
# BATCH JOB DEPENDENCIES
batch_job_queue = BatchJobQueue(
self,
"batch-job-queue",
deploy_env=deploy_env,
processing_assets_table=processing_assets_table,
).job_queue
s3_read_only_access_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3ReadOnlyAccess")
############################################################################################
# STATE MACHINE TASKS
check_stac_metadata_task = LambdaTask(
self,
"check-stac-metadata-task",
directory="check_stac_metadata",
botocore_lambda_layer=botocore_lambda_layer,
extra_environment={"DEPLOY_ENV": deploy_env},
)
assert check_stac_metadata_task.lambda_function.role
check_stac_metadata_task.lambda_function.role.add_managed_policy(
policy=s3_read_only_access_policy)
for table in [processing_assets_table, validation_results_table]:
table.grant_read_write_data(
check_stac_metadata_task.lambda_function)
table.grant(
check_stac_metadata_task.lambda_function,
"dynamodb:DescribeTable",
)
content_iterator_task = LambdaTask(
self,
"content-iterator-task",
directory="content_iterator",
botocore_lambda_layer=botocore_lambda_layer,
result_path="$.content",
extra_environment={"DEPLOY_ENV": deploy_env},
)
check_files_checksums_directory = "check_files_checksums"
check_files_checksums_default_payload_object = {
"dataset_id.$": "$.dataset_id",
"version_id.$": "$.version_id",
"metadata_url.$": "$.metadata_url",
"first_item.$": "$.content.first_item",
"assets_table_name.$": "$.content.assets_table_name",
"results_table_name.$": "$.content.results_table_name",
}
check_files_checksums_single_task = BatchSubmitJobTask(
self,
"check-files-checksums-single-task",
deploy_env=deploy_env,
directory=check_files_checksums_directory,
s3_policy=s3_read_only_access_policy,
job_queue=batch_job_queue,
payload_object=check_files_checksums_default_payload_object,
container_overrides_command=[
"--dataset-id",
"Ref::dataset_id",
"--version-id",
"Ref::version_id",
"--first-item",
"Ref::first_item",
"--assets-table-name",
"Ref::assets_table_name",
"--results-table-name",
"Ref::results_table_name",
],
)
array_size = int(
aws_stepfunctions.JsonPath.number_at("$.content.iteration_size"))
check_files_checksums_array_task = BatchSubmitJobTask(
self,
"check-files-checksums-array-task",
deploy_env=deploy_env,
directory=check_files_checksums_directory,
s3_policy=s3_read_only_access_policy,
job_queue=batch_job_queue,
payload_object=check_files_checksums_default_payload_object,
container_overrides_command=[
"--dataset-id",
"Ref::dataset_id",
"--version-id",
"Ref::version_id",
"--first-item",
"Ref::first_item",
"--assets-table-name",
"Ref::assets_table_name",
"--results-table-name",
"Ref::results_table_name",
],
array_size=array_size,
)
for reader in [
content_iterator_task.lambda_function,
check_files_checksums_single_task.job_role,
check_files_checksums_array_task.job_role,
]:
processing_assets_table.grant_read_data(
reader) # type: ignore[arg-type]
processing_assets_table.grant(
reader,
"dynamodb:DescribeTable" # type: ignore[arg-type]
)
for writer in [
check_files_checksums_single_task.job_role,
check_files_checksums_array_task.job_role,
]:
validation_results_table.grant_read_write_data(
writer) # type: ignore[arg-type]
validation_results_table.grant(
writer,
"dynamodb:DescribeTable" # type: ignore[arg-type]
)
validation_summary_task = LambdaTask(
self,
"validation-summary-task",
directory="validation_summary",
botocore_lambda_layer=botocore_lambda_layer,
result_path="$.validation",
extra_environment={"DEPLOY_ENV": deploy_env},
)
validation_results_table.grant_read_data(
validation_summary_task.lambda_function)
validation_results_table.grant(validation_summary_task.lambda_function,
"dynamodb:DescribeTable")
validation_failure_lambda_invoke = LambdaTask(
self,
"validation-failure-task",
directory="validation_failure",
botocore_lambda_layer=botocore_lambda_layer,
result_path=aws_stepfunctions.JsonPath.DISCARD,
).lambda_invoke
import_dataset_role = aws_iam.Role(
self,
"import-dataset",
assumed_by=aws_iam.ServicePrincipal( # type: ignore[arg-type]
"batchoperations.s3.amazonaws.com"),
)
import_asset_file_function = ImportFileFunction(
self,
directory="import_asset_file",
invoker=import_dataset_role,
deploy_env=deploy_env,
botocore_lambda_layer=botocore_lambda_layer,
)
import_metadata_file_function = ImportFileFunction(
self,
directory="import_metadata_file",
invoker=import_dataset_role,
deploy_env=deploy_env,
botocore_lambda_layer=botocore_lambda_layer,
)
for storage_writer in [
import_dataset_role,
import_asset_file_function.role,
import_metadata_file_function.role,
]:
storage_bucket.grant_read_write(
storage_writer) # type: ignore[arg-type]
import_dataset_task = LambdaTask(
self,
"import-dataset-task",
directory="import_dataset",
botocore_lambda_layer=botocore_lambda_layer,
result_path="$.import_dataset",
extra_environment={"DEPLOY_ENV": deploy_env},
)
assert import_dataset_task.lambda_function.role is not None
import_dataset_task.lambda_function.role.add_to_policy(
aws_iam.PolicyStatement(
resources=[import_dataset_role.role_arn],
actions=["iam:PassRole"],
), )
import_dataset_task.lambda_function.role.add_to_policy(
aws_iam.PolicyStatement(resources=["*"], actions=["s3:CreateJob"]))
storage_bucket.grant_read_write(import_dataset_task.lambda_function)
for table in [datasets_table, processing_assets_table]:
table.grant_read_data(import_dataset_task.lambda_function)
table.grant(import_dataset_task.lambda_function,
"dynamodb:DescribeTable")
# Parameters
import_asset_file_function_arn_parameter = aws_ssm.StringParameter(
self,
"import asset file function arn",
string_value=import_asset_file_function.function_arn,
description=f"Import asset file function ARN for {deploy_env}",
parameter_name=ParameterName.
PROCESSING_IMPORT_ASSET_FILE_FUNCTION_TASK_ARN.value,
)
import_metadata_file_function_arn_parameter = aws_ssm.StringParameter(
self,
"import metadata file function arn",
string_value=import_metadata_file_function.function_arn,
description=f"Import metadata file function ARN for {deploy_env}",
parameter_name=ParameterName.
PROCESSING_IMPORT_METADATA_FILE_FUNCTION_TASK_ARN.value,
)
import_dataset_role_arn_parameter = aws_ssm.StringParameter(
self,
"import dataset role arn",
string_value=import_dataset_role.role_arn,
description=f"Import dataset role ARN for {deploy_env}",
parameter_name=ParameterName.PROCESSING_IMPORT_DATASET_ROLE_ARN.
value,
)
grant_parameter_read_access({
datasets_table.name_parameter:
[import_dataset_task.lambda_function],
import_asset_file_function_arn_parameter:
[import_dataset_task.lambda_function],
import_dataset_role_arn_parameter:
[import_dataset_task.lambda_function],
import_metadata_file_function_arn_parameter:
[import_dataset_task.lambda_function],
processing_assets_table.name_parameter: [
check_stac_metadata_task.lambda_function.role,
content_iterator_task.lambda_function,
import_dataset_task.lambda_function,
],
storage_bucket_parameter: [
import_dataset_task.lambda_function,
],
validation_results_table.name_parameter: [
check_stac_metadata_task.lambda_function.role,
validation_summary_task.lambda_function,
content_iterator_task.lambda_function,
],
})
success_task = aws_stepfunctions.Succeed(self, "success")
############################################################################################
# STATE MACHINE
dataset_version_creation_definition = (
check_stac_metadata_task.lambda_invoke.next(
content_iterator_task.lambda_invoke).next(
aws_stepfunctions.Choice( # type: ignore[arg-type]
self, "check_files_checksums_maybe_array").when(
aws_stepfunctions.Condition.number_equals(
"$.content.iteration_size", 1),
check_files_checksums_single_task.batch_submit_job,
).otherwise(check_files_checksums_array_task.
batch_submit_job).afterwards()).
next(
aws_stepfunctions.Choice(
self, "content_iteration_finished").when(
aws_stepfunctions.Condition.number_equals(
"$.content.next_item", -1),
validation_summary_task.lambda_invoke.next(
aws_stepfunctions.Choice( # type: ignore[arg-type]
self, "validation_successful").when(
aws_stepfunctions.Condition.boolean_equals(
"$.validation.success", True),
import_dataset_task.lambda_invoke.next(
success_task # type: ignore[arg-type]
),
).otherwise(validation_failure_lambda_invoke)),
).otherwise(content_iterator_task.lambda_invoke)))
self.state_machine = aws_stepfunctions.StateMachine(
self,
f"{deploy_env}-dataset-version-creation",
definition=
dataset_version_creation_definition, # type: ignore[arg-type]
)
self.state_machine_parameter = aws_ssm.StringParameter(
self,
"state machine arn",
description=f"State machine ARN for {deploy_env}",
parameter_name=ParameterName.
PROCESSING_DATASET_VERSION_CREATION_STEP_FUNCTION_ARN.value, # pylint:disable=line-too-long
string_value=self.state_machine.state_machine_arn,
)
Tags.of(self).add("ApplicationLayer",
"processing") # type: ignore[arg-type]
