def createTipWorkflow(self): notifyTipper = tasks.LambdaInvoke(self, 'notifyTipper', lambda_function=self.createLambda('notifyTipperLambda', 'tipNotifier.tipNotifier'), timeout=cdk.Duration.seconds(300) ).next(sfn.Succeed(self, 'withdrawSuccessState')) self.getTipperInvoice = tasks.StepFunctionsInvokeActivity(self, 'getTipperInvoice', activity=sfn.Activity(self, 'getTipperInvoiceActivity'), heartbeat=cdk.Duration.seconds(60), timeout=cdk.Duration.seconds(86400), ) self.getTipperInvoice.add_retry( backoff_rate=1.5, errors=['States.Timeout'], interval=cdk.Duration.seconds(60), max_attempts=7 ) self.getTipperInvoice.add_catch( handler=sfn.Fail(self, 'withdrawErrorState'), errors=['States.ALL'], result_path='$.errorInfo' ) self.getTipperInvoice.next(notifyTipper) return sfn.StateMachine(self, 'tipWorkflow', definition=self.getTipperInvoice, role=self.statesRole )
def __init__(self, scope: core.Construct, id: str, *, message: sfn.TaskInput, subject: Optional[str] = None, topic: Optional[sns.Topic] = None, result_path: str = '$.PublishResult', output_path: Optional[str] = None): super().__init__(scope, id) self._end = sfn.Succeed(self, 'Succeeded', output_path=output_path) if topic is not None: self._start = sfn_tasks.SnsPublish( self, 'Success Notification', input_path='$', output_path='$', result_path=result_path, topic=topic, message=message, subject=subject, ) self._start.next(self._end) else: self._start = self._end
def createWithdrawWorkflow(self): payInvoiceFailed = tasks.LambdaInvoke(self, 'payInvoiceFailed', lambda_function=self.createLambda('payInvoiceFailedLambda', 'payInvoiceFailed.payInvoiceFailed'), timeout=cdk.Duration.seconds(300) ).next(sfn.Fail(self, 'tipErrorState')) payInvoiceSucceeded = tasks.LambdaInvoke(self, 'payInvoiceSucceeded', lambda_function=self.createLambda('payInvoiceSucceededLambda', 'payInvoiceSucceeded.payInvoiceSucceeded'), timeout=cdk.Duration.seconds(300) ).next(sfn.Succeed(self, 'tipSuccessState')) self.payInvoice = tasks.StepFunctionsInvokeActivity(self, 'payInvoice', activity=sfn.Activity(self, 'payInvoiceActivity'), heartbeat=cdk.Duration.seconds(86400), timeout=cdk.Duration.seconds(86400), ) self.payInvoice.add_retry( backoff_rate=2, errors=['States.Timeout'], interval=cdk.Duration.seconds(600), max_attempts=0 ) self.payInvoice.add_catch( handler=payInvoiceFailed, errors=['States.ALL'], result_path='$.errorInfo' ) self.payInvoice.next(payInvoiceSucceeded) return sfn.StateMachine(self, 'withdrawWorkflow', definition=self.payInvoice, role=self.statesRole )
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) flip_coin_function = lambda_.Function( self, "FlipCoinFunction", runtime=lambda_.Runtime.PYTHON_3_8, handler="index.handler", code=lambda_.Code.from_asset("./sfn/lambda/flip_coin")) flip_coin_invoke = tasks.LambdaInvoke( self, "FlipCoin", lambda_function=flip_coin_function) wait = stepfunctions.Wait(self, "Wait", time=stepfunctions.WaitTime.duration( core.Duration.seconds(5))) tails_result = stepfunctions.Pass(self, "TailsResult") tails_result.next(flip_coin_invoke) choice = stepfunctions.Choice(self, "HeadsTailsChoice") \ .when(condition=stepfunctions.Condition.string_equals("$.Payload.result", "heads"), next=stepfunctions.Succeed(self, "HeadsResult")) \ .when(condition=stepfunctions.Condition.string_equals("$.Payload.result", "tails"), next=tails_result) stepfunctions.StateMachine(self, "StateMachine", definition=flip_coin_invoke.next( wait.next(choice)))
def test_start_execution_task(): default_task_json = { 'End': True, 'Parameters': { 'StateMachineArn': { 'Ref': 'teststatemachine7F4C511D' }, 'Input.$': '$$.Execution.Input' }, 'Type': 'Task', 'Resource': { 'Fn::Join': [ '', [ 'arn:', { 'Ref': 'AWS::Partition' }, ':states:::states:startExecution.sync' ] ] } } stack = core.Stack(core.App(), 'test-stack') state_machine = sfn.StateMachine(stack, 'test-state-machine', definition=sfn.Chain.start( sfn.Succeed(stack, 'Succeeded'))) task = sfn.Task(stack, 'test-task', task=emr_tasks.StartExecutionTask(state_machine, )) print_and_assert(default_task_json, task)
def __init__(self, scope: core.Construct, id: str, *, polling_delay: int = 5, statemachine_timeout: int = 300, **kwargs): super().__init__(scope, id, **kwargs) state_fn = StateHandlerLambda(self, "config-state-handler").function config_fn = AccountConfigLambda(self, "account-config-handler").function config_state = tasks.LambdaInvoke(self, "Set Configuring State", lambda_function=state_fn, output_path="$.Payload") completed_state = tasks.LambdaInvoke(self, "Set Completed State", lambda_function=state_fn, output_path="$.Payload") config_task = tasks.LambdaInvoke(self, "Request Account Configuration", lambda_function=config_fn, output_path="$.Payload") polling_task = tasks.LambdaInvoke(self, "Poll Account Configuration", lambda_function=config_fn, output_path="$.Payload") delay = sfn.Wait(self, "Delay Polling", time=sfn.WaitTime.duration( core.Duration.seconds(polling_delay))) is_ready = sfn.Choice(self, "Account Ready?") acct_ready = sfn.Condition.string_equals('$.state', "READY") acct_pending = sfn.Condition.string_equals('$.state', "PENDING") success = sfn.Succeed(self, "Config Succeeded") failed = sfn.Fail(self, "Config Failed", cause="Bad value in Polling loop") # this is the loop which polls for state change, either looping back to delay or setting completion state and finishing is_ready.when(acct_pending, delay).when( acct_ready, completed_state.next(success)).otherwise(failed) # this is the main chain starting with creation request a delay and then polling loop config_chain = config_task.next(config_state).next(delay).next( polling_task).next(is_ready) self.state_machine = sfn.StateMachine( self, "Account-Config-StateMachine", definition=config_chain, timeout=core.Duration.seconds(statemachine_timeout))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) api = apigateway.RestApi( scope=self, id=f'{constants.PREFIX}-approval-api', rest_api_name='Human approval endpoint', description='HTTP Endpoint backed by API Gateway and Lambda', endpoint_types=[apigateway.EndpointType.REGIONAL], ) v1 = api.root.add_resource("v1") approve_api = v1.add_resource("approve") ################################################# email_topic = sns.Topic( scope=self, id=f'{constants.PREFIX}-email-topic', ) email_topic.add_subscription( subscription=subscriptions.EmailSubscription( email_address=constants.EMAIL_APPROVER, )) ################################################# submit_job_lambda = _lambda.Function( scope=self, id=f'{constants.PREFIX}-submit-lambda', runtime=_lambda.Runtime.PYTHON_3_8, handler='submit.handler', environment={ "TOPIC_ARN": email_topic.topic_arn, "END_POINT": approve_api.url, "TO_ADDRESS": constants.EMAIL_RECIPIENT, "FROM_ADDRESS": constants.EMAIL_SENDER, }, code=_lambda.Code.from_asset( os.path.join('lambdas', 'submit-lambda')), ) email_topic.grant_publish(submit_job_lambda) submit_job_lambda.add_to_role_policy(statement=iam.PolicyStatement( actions=['ses:Send*'], resources=['*'], )) submit_job = tasks.LambdaInvoke( scope=self, id=f'{constants.PREFIX}-submit-job', lambda_function=submit_job_lambda, integration_pattern=sfn.IntegrationPattern.WAIT_FOR_TASK_TOKEN, heartbeat=core.Duration.minutes(5), payload=sfn.TaskInput.from_object({ "token": sfn.Context.task_token, "data": sfn.Data.string_at('$'), }), ) success = sfn.Succeed(scope=self, id=f'{constants.PREFIX}-success', comment='We did it!') fail = sfn.Fail(scope=self, id=f'{constants.PREFIX}-fail', error='WorkflowFailure', cause='Something went wrong') choice = sfn.Choice(scope=self, id=f'{constants.PREFIX}-choice', comment='Was it approved?') choice.when(condition=sfn.Condition.string_equals("$.status", "OK"), next=success) choice.otherwise(fail) definition = submit_job.next(choice) self._state_machine = sfn.StateMachine( scope=self, id=f'{constants.PREFIX}-state-machine', definition=definition, # only 10 mins to approve better be quick timeout=core.Duration.minutes(10)) ################################################# approval_lambda = _lambda.Function( scope=self, id=f'{constants.PREFIX}-approval-lambda', runtime=_lambda.Runtime.PYTHON_3_8, handler='approve.handler', code=_lambda.Code.from_asset( os.path.join('lambdas', 'approve-lambda')), ) approval_lambda.add_to_role_policy(statement=iam.PolicyStatement( actions=['states:Send*'], resources=['*'])) approve_integration = apigateway.LambdaIntegration(approval_lambda) approve_api_get_method = approve_api.add_method( http_method="GET", api_key_required=False, integration=approve_integration, )
def __init__(self, app: App, id: str, **kwargs) -> None: super().__init__(app, id, **kwargs) # Lambda Handlers Definitions submit_lambda = _lambda.Function( self, 'submitLambda', handler='lambda_function.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_9, code=_lambda.Code.from_asset('lambdas/submit')) status_lambda = _lambda.Function( self, 'statusLambda', handler='lambda_function.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_9, code=_lambda.Code.from_asset('lambdas/status')) # Step functions Definition submit_job = _aws_stepfunctions_tasks.LambdaInvoke( self, "Submit Job", lambda_function=submit_lambda, output_path="$.Payload", ) wait_job = _aws_stepfunctions.Wait( self, "Wait 30 Seconds", time=_aws_stepfunctions.WaitTime.duration(Duration.seconds(30))) status_job = _aws_stepfunctions_tasks.LambdaInvoke( self, "Get Status", lambda_function=status_lambda, output_path="$.Payload", ) fail_job = _aws_stepfunctions.Fail(self, "Fail", cause='AWS Batch Job Failed', error='DescribeJob returned FAILED') succeed_job = _aws_stepfunctions.Succeed( self, "Succeeded", comment='AWS Batch Job succeeded') # Create Chain definition = submit_job.next(wait_job)\ .next(status_job)\ .next(_aws_stepfunctions.Choice(self, 'Job Complete?') .when(_aws_stepfunctions.Condition.string_equals('$.status', 'FAILED'), fail_job) .when(_aws_stepfunctions.Condition.string_equals('$.status', 'SUCCEEDED'), succeed_job) .otherwise(wait_job)) # Create state machine sm = _aws_stepfunctions.StateMachine( self, "StateMachine", definition=definition, timeout=Duration.minutes(5), )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.vpc = aws_ec2.Vpc(self, "demo-stepfunctions", cidr="10.100.0.0/16", max_azs=2, nat_gateways=0, subnet_configuration=[ aws_ec2.SubnetConfiguration( name='demo-stepfunctions', subnet_type=aws_ec2.SubnetType.ISOLATED, cidr_mask=24) ]) lambda_role = iam.Role( self, 'demo-lambda-role', assumed_by=iam.ServicePrincipal("lambda.amazonaws.com")) lambda_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaENIManagementAccess')) lambda_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaBasicExecutionRole')) fn_submit = lambda_.Function( self, 'demo-sfn-submit', function_name='demo-sfn-submit', handler='handler.do', runtime=lambda_.Runtime.PYTHON_3_8, code=lambda_.Code.asset('./craftaws/func_submit'), role=lambda_role, timeout=core.Duration.seconds(900), allow_public_subnet=False, vpc=self.vpc, vpc_subnets=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.ISOLATED), environment={}) fn_job_1 = lambda_.Function( self, 'demo-sfn-job1', function_name='demo-sfn-job1', handler='handler.do', runtime=lambda_.Runtime.PYTHON_3_8, code=lambda_.Code.asset('./craftaws/func_job_1'), role=lambda_role, timeout=core.Duration.seconds(900), allow_public_subnet=False, vpc=self.vpc, vpc_subnets=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.ISOLATED), environment={}) fn_job_2 = lambda_.Function( self, 'demo-sfn-job2', function_name='demo-sfn-job2', handler='handler.do', runtime=lambda_.Runtime.PYTHON_3_8, code=lambda_.Code.asset('./craftaws/func_job_2'), role=lambda_role, timeout=core.Duration.seconds(900), allow_public_subnet=False, vpc=self.vpc, vpc_subnets=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.ISOLATED), environment={}) submit_job = tasks.LambdaInvoke( self, "Submit Job", lambda_function=fn_submit, # Lambda's result is in the attribute `Payload` output_path="$.Payload") step_1_job = tasks.LambdaInvoke( self, "Job_1", lambda_function=fn_job_1, # Lambda's result is in the attribute `Payload` output_path="$.Payload") wait_x = sfn.Wait(self, "Wait X Seconds", time=sfn.WaitTime.duration( core.Duration.seconds(60))) step_2_job = tasks.LambdaInvoke( self, "Job_2", lambda_function=fn_job_1, # Lambda's result is in the attribute `Payload` output_path="$.Payload") job_succeed = sfn.Succeed(self, "Job Succeed", comment="AWS Batch Job Succeed") definition = submit_job.next(step_1_job).next(wait_x).next( step_2_job).next(job_succeed) sfn.StateMachine(self, "StateMachine", definition=definition, timeout=core.Duration.minutes(5))
def test_nested_state_machine_chain(): default_fragment_json = { 'Type': 'Parallel', 'End': True, 'Branches': [{ 'StartAt': 'test-fragment: test-nested-state-machine', 'States': { 'test-fragment: test-nested-state-machine': { 'Next': 'test-fragment: test-nested-state-machine - Parse JSON Output', 'Catch': [{ 'ErrorEquals': ['States.ALL'], 'ResultPath': '$.Error', 'Next': 'test-fail' }], 'Parameters': { 'StateMachineArn': { 'Ref': 'teststatemachine7F4C511D' }, 'Input': { 'Key1': 'Value1' } }, 'Type': 'Task', 'Resource': { 'Fn::Join': [ '', [ 'arn:', { 'Ref': 'AWS::Partition' }, ':states:::states:startExecution.sync' ] ] } }, 'test-fragment: test-nested-state-machine - Parse JSON Output': { 'End': True, 'Catch': [{ 'ErrorEquals': ['States.ALL'], 'ResultPath': '$.Error', 'Next': 'test-fail' }], 'Parameters': { 'JsonString.$': '$.Output' }, 'Type': 'Task', 'Resource': { 'Fn::GetAtt': ['ParseJsonString859DB4F0', 'Arn'] }, 'ResultPath': '$' }, 'test-fail': { 'Type': 'Fail' } } }] } stack = core.Stack(core.App(), 'test-stack') state_machine = sfn.StateMachine(stack, 'test-state-machine', definition=sfn.Chain.start( sfn.Succeed(stack, 'Succeeded'))) fragment = emr_chains.NestedStateMachine(stack, 'test-fragment', name='test-nested-state-machine', state_machine=state_machine, input={'Key1': 'Value1'}, fail_chain=sfn.Fail( stack, 'test-fail')) print_and_assert(default_fragment_json, fragment)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) lambda_role = _iam.Role( self, id='lab3-om-role', assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com')) cloudwatch_policy_statement = _iam.PolicyStatement( effect=_iam.Effect.ALLOW) cloudwatch_policy_statement.add_actions("logs:CreateLogGroup") cloudwatch_policy_statement.add_actions("logs:CreateLogStream") cloudwatch_policy_statement.add_actions("logs:PutLogEvents") cloudwatch_policy_statement.add_actions("logs:DescribeLogStreams") cloudwatch_policy_statement.add_resources("*") lambda_role.add_to_policy(cloudwatch_policy_statement) fn_lambda_approve_reject = aws_lambda.Function( self, "lab3-om-approve-reject", code=aws_lambda.AssetCode( "../lambda-functions/approve-reject-application/"), handler="app.handler", tracing=aws_lambda.Tracing.ACTIVE, timeout=core.Duration.seconds(30), role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) fn_lambda_verify_identity = aws_lambda.Function( self, "lab3-om-verify-identity", code=aws_lambda.AssetCode("../lambda-functions/verify-identity/"), handler="app.handler", tracing=aws_lambda.Tracing.ACTIVE, timeout=core.Duration.seconds(30), role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) fn_lambda_check_address = aws_lambda.Function( self, "lab3-om-check-address", code=aws_lambda.AssetCode("../lambda-functions/check-address/"), handler="app.handler", tracing=aws_lambda.Tracing.ACTIVE, timeout=core.Duration.seconds(30), role=lambda_role, runtime=aws_lambda.Runtime.PYTHON_3_8) ''' [INFO] This is a sample how to define the task and integrate with Lambda Functions. You need to create another 2 tasks for respective Lambda functions ''' task_verify_identity = _tasks.LambdaInvoke( self, "Verify Identity Document", lambda_function=fn_lambda_verify_identity, output_path="$.Payload") task_check_address = _tasks.LambdaInvoke( self, "Check Address", lambda_function=fn_lambda_check_address, output_path="$.Payload") task_wait_review = _tasks.LambdaInvoke( self, "Wait for Review", lambda_function=fn_lambda_approve_reject, output_path="$.Payload") state_approve = _sfn.Succeed(self, "Approve Application") state_reject = _sfn.Succeed(self, "Reject Application") # Let's define the State Machine, step by step # First, paralell tasks for verification s_verification = _sfn.Parallel(self, "Verification") s_verification.branch(task_verify_identity) s_verification.branch(task_check_address) # Next, we add a choice state c_human_review = _sfn.Choice(self, "Human review required?") c_human_review.when( _sfn.Condition.and_( _sfn.Condition.boolean_equals("$[0].humanReviewRequired", False), _sfn.Condition.boolean_equals("$[1].humanReviewRequired", True)), state_approve) c_human_review.when( _sfn.Condition.or_( _sfn.Condition.boolean_equals("$[0].humanReviewRequired", True), _sfn.Condition.boolean_equals("$[1].humanReviewRequired", False)), task_wait_review) # Another choice state to check if the application passed the review c_review_approved = _sfn.Choice(self, "Review approved?") c_review_approved.when( _sfn.Condition.boolean_equals("$.reviewApproved", True), state_approve) c_review_approved.when( _sfn.Condition.boolean_equals("$.reviewApproved", False), state_reject) task_wait_review.next(c_review_approved) definition = s_verification.next(c_human_review) _sfn.StateMachine(self, "lab3-statemachine", definition=definition, timeout=core.Duration.minutes(5))
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) #the S3 bucket where CloudFront Access Logs will be stored cf_access_logs = s3.Bucket(self, "LogBucket") #S3 bucket where Athena will put the results athena_results = s3.Bucket(self, "AthenaResultsBucket") #create an Athena database glue_database_name = "serverlessland_database" myDatabase = glue.CfnDatabase( self, id=glue_database_name, catalog_id=account, database_input=glue.CfnDatabase.DatabaseInputProperty( description=f"Glue database '{glue_database_name}'", name=glue_database_name, ) ) #define a table with the structure of CloudFront Logs https://docs.aws.amazon.com/athena/latest/ug/cloudfront-logs.html athena_table = glue.CfnTable(self, id='cfaccesslogs', catalog_id=account, database_name=glue_database_name, table_input=glue.CfnTable.TableInputProperty( name='cf_access_logs', description='CloudFront access logs', table_type='EXTERNAL_TABLE', parameters = { 'skip.header.line.count': '2', }, storage_descriptor=glue.CfnTable.StorageDescriptorProperty( location="s3://"+cf_access_logs.bucket_name+"/", input_format='org.apache.hadoop.mapred.TextInputFormat', output_format='org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', compressed=False, serde_info=glue.CfnTable.SerdeInfoProperty( serialization_library='org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', parameters={ 'field.delim' : ' ' } ), columns=[ glue.CfnTable.ColumnProperty(name='date', type='date'), glue.CfnTable.ColumnProperty(name='time', type='string'), glue.CfnTable.ColumnProperty(name='location', type='string'), glue.CfnTable.ColumnProperty(name='bytes', type='bigint'), glue.CfnTable.ColumnProperty(name='request_ip', type='string'), glue.CfnTable.ColumnProperty(name='method', type='string'), glue.CfnTable.ColumnProperty(name='host', type='string'), glue.CfnTable.ColumnProperty(name='uri', type='string'), glue.CfnTable.ColumnProperty(name='status', type='string'), glue.CfnTable.ColumnProperty(name='referer', type='string'), glue.CfnTable.ColumnProperty(name='user_agent', type='string'), glue.CfnTable.ColumnProperty(name='query_string', type='string'), glue.CfnTable.ColumnProperty(name='cookie', type='string'), glue.CfnTable.ColumnProperty(name='result_type', type='string'), glue.CfnTable.ColumnProperty(name='request_id', type='string'), glue.CfnTable.ColumnProperty(name='host_header', type='string'), glue.CfnTable.ColumnProperty(name='request_protocol', type='string'), glue.CfnTable.ColumnProperty(name='request_bytes', type='bigint'), glue.CfnTable.ColumnProperty(name='time_taken', type='float'), glue.CfnTable.ColumnProperty(name='xforwarded_for', type='string'), glue.CfnTable.ColumnProperty(name='ssl_protocol', type='string'), glue.CfnTable.ColumnProperty(name='ssl_cipher', type='string'), glue.CfnTable.ColumnProperty(name='response_result_type', type='string'), glue.CfnTable.ColumnProperty(name='http_version', type='string'), glue.CfnTable.ColumnProperty(name='fle_status', type='string'), glue.CfnTable.ColumnProperty(name='fle_encrypted_fields', type='int'), glue.CfnTable.ColumnProperty(name='c_port', type='int'), glue.CfnTable.ColumnProperty(name='time_to_first_byte', type='float'), glue.CfnTable.ColumnProperty(name='x_edge_detailed_result_type', type='string'), glue.CfnTable.ColumnProperty(name='sc_content_type', type='string'), glue.CfnTable.ColumnProperty(name='sc_content_len', type='string'), glue.CfnTable.ColumnProperty(name='sc_range_start', type='bigint'), glue.CfnTable.ColumnProperty(name='sc_range_end', type='bigint') ] ), ) ) #submit the query and wait for the results start_query_execution_job = tasks.AthenaStartQueryExecution(self, "Start Athena Query", query_string="SELECT uri FROM cf_access_logs limit 10", integration_pattern=sf.IntegrationPattern.RUN_JOB, #executes the command in SYNC mode query_execution_context=tasks.QueryExecutionContext( database_name=glue_database_name ), result_configuration=tasks.ResultConfiguration( output_location=s3.Location( bucket_name=athena_results.bucket_name, object_key="results" ) ) ) #get the results get_query_results_job = tasks.AthenaGetQueryResults(self, "Get Query Results", query_execution_id=sf.JsonPath.string_at("$.QueryExecution.QueryExecutionId"), result_path=sf.JsonPath.string_at("$.GetQueryResults"), ) #prepare the query to see if more results are available (up to 1000 can be retrieved) prepare_next_params = sf.Pass(self, "Prepare Next Query Params", parameters={ "QueryExecutionId.$": "$.StartQueryParams.QueryExecutionId", "NextToken.$": "$.GetQueryResults.NextToken" }, result_path=sf.JsonPath.string_at("$.StartQueryParams") ) #check to see if more results are available has_more_results = sf.Choice(self, "Has More Results?").when( sf.Condition.is_present("$.GetQueryResults.NextToken"), prepare_next_params.next(get_query_results_job) ).otherwise(sf.Succeed(self, "Done")) #do something with each result #here add your own logic map = sf.Map(self, "Map State", max_concurrency=1, input_path=sf.JsonPath.string_at("$.GetQueryResults.ResultSet.Rows[1:]"), result_path = sf.JsonPath.DISCARD ) map.iterator(sf.Pass(self, "DoSomething")) # Step function to orchestrate Athena query and retrieving the results workflow = sf.StateMachine(self, "AthenaQuery", definition=start_query_execution_job.next(get_query_results_job).next(map).next(has_more_results), timeout=Duration.minutes(60) ) CfnOutput(self, "Logs", value=cf_access_logs.bucket_name, export_name='LogsBucket') CfnOutput(self, "SFName", value=workflow.state_machine_name, export_name='SFName') CfnOutput(self, "SFArn", value = workflow.state_machine_arn, export_name = 'StepFunctionArn', description = 'Step Function arn')
def __init__(self, scope: core.Construct, id: str, redshift_cluster_name: str, user_secret: Secret) -> None: super().__init__(scope, id) stack = Stack.of(self) subprocess.call( ['pip', 'install', '-t', 'dwh/dwh_loader_layer/python/lib/python3.8/site-packages', '-r', 'dwh/dwh_loader/requirements.txt', '--platform', 'manylinux1_x86_64', '--only-binary=:all:', '--upgrade']) requirements_layer = _lambda.LayerVersion(scope=self, id='PythonRequirementsTemplate', code=_lambda.Code.from_asset('dwh/dwh_loader_layer'), compatible_runtimes=[_lambda.Runtime.PYTHON_3_8]) dwh_loader_role = _iam.Role( self, 'Role', assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com') ) dwh_loader_role.add_managed_policy(_iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaBasicExecutionRole' )) dwh_loader_role.attach_inline_policy( _iam.Policy( self, 'InlinePolicy', statements=[ _iam.PolicyStatement( actions=[ "redshift-data:ExecuteStatement", "redshift-data:CancelStatement", "redshift-data:ListStatements", "redshift-data:GetStatementResult", "redshift-data:DescribeStatement", "redshift-data:ListDatabases", "redshift-data:ListSchemas", "redshift-data:ListTables", "redshift-data:DescribeTable" ], resources=['*'] ), _iam.PolicyStatement( actions=["secretsmanager:GetSecretValue"], resources=[user_secret.secret_arn] ), _iam.PolicyStatement( actions=["redshift:GetClusterCredentials"], resources=[ "arn:aws:redshift:*:*:dbname:*/*", "arn:aws:redshift:*:*:dbuser:*/"+_config.Redshift.ETL_USER ] ), _iam.PolicyStatement( effect=_iam.Effect('DENY'), actions=["redshift:CreateClusterUser"], resources=["arn:aws:redshift:*:*:dbuser:*/"+_config.Redshift.ETL_USER] ), _iam.PolicyStatement( conditions={ 'StringLike': { "iam:AWSServiceName": "redshift-data.amazonaws.com" } }, actions=["iam:CreateServiceLinkedRole"], resources=["arn:aws:iam::*:role/aws-service-role/redshift-data.amazonaws.com/AWSServiceRoleForRedshift"] ), ] ) ) dwh_loader_function = _lambda.Function( self, 'Lambda', runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.from_asset('dwh/dwh_loader'), handler='dwh_loader.handler', function_name='dwh-loader', environment={ 'CLUSTER_NAME': redshift_cluster_name, 'PROCEDURE': _config.Redshift.ETL_PROCEDURE, 'SECRET_ARN': user_secret.secret_arn, 'DATABASE': _config.Redshift.DATABASE, 'REGION': core.Aws.REGION, 'SCHEMA': _config.Redshift.SCHEMA }, layers=[requirements_layer], timeout=core.Duration.seconds(30), role=dwh_loader_role ) dwh_loader_submit = _sfn_tasks.LambdaInvoke( self, 'Submit', lambda_function=dwh_loader_function, payload_response_only=True ) dwh_loader_wait = _sfn.Wait( self, 'Wait', time=_sfn.WaitTime.duration(core.Duration.seconds(30)) ) dwh_loader_complete = _sfn.Choice( self, 'Complete' ) dwh_loader_failed = _sfn.Fail( self, 'Fail', cause="Redshift Data API statement failed", error="$.Result.Error" ) dwh_loader_status = _sfn_tasks.LambdaInvoke( self, 'Status', lambda_function=dwh_loader_function, result_path='$.Result', payload_response_only=True ) definition = dwh_loader_submit \ .next(dwh_loader_wait) \ .next(dwh_loader_status) \ .next(dwh_loader_complete .when(_sfn.Condition.string_equals('$.Result.Status', 'FAILED'), dwh_loader_failed) .when(_sfn.Condition.string_equals('$.Result.Status', 'FINISHED'), _sfn.Succeed(self, 'DwhLoaderSuccess')) .otherwise(dwh_loader_wait)) dwh_loader_stepfunctions = _sfn.StateMachine( self, 'StepFunctions', definition=definition, timeout=core.Duration.minutes(30) ) step_trigger = _events.Rule( self, 'StepTrigger', schedule=_events.Schedule.cron(minute='0/30', hour='*', month='*', week_day='*', year='*') ) step_trigger.add_target( _events_targets.SfnStateMachine( machine=dwh_loader_stepfunctions, ) )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here pvt_bkt = _s3.Bucket(self, "s3bucket") core.Tag.add(pvt_bkt, key="isMonitoredBucket", value="True") # Lets create a cloudtrail to track s3 data events s3_data_event_trail = _cloudtrail.Trail( self, "s3DataEventTrailId", is_multi_region_trail=False, include_global_service_events=False, enable_file_validation=True) # Lets capture S3 Data Events only for our bucket- TO REDUCE COST s3_data_event_trail.add_s3_event_selector( prefixes=[f"{pvt_bkt.bucket_arn}/"], include_management_events=True, read_write_type=_cloudtrail.ReadWriteType.ALL) # Defines an AWS Lambda resource """ with open("lambda_src/make_object_private.py", encoding="utf8") as fp: make_object_private_fn_handler_code = fp.read() remediate_object_acl_fn = _lambda.Function( self, id='remediateObjAclFn', function_name="remediate_object_acl_fn", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(make_object_private_fn_handler_code), handler='index.lambda_handler', timeout=core.Duration.seconds(10) ) # Lets add the necessary permission for the lambda function remediate_object_acl_fn_perms=_iam.PolicyStatement( effect=_iam.Effect.ALLOW, resources=[ "arn:aws:s3:::*", ], actions=[ "s3:GetObjectAcl", "s3:PutObjectAcl" ] ) remediate_object_acl_fn_perms.sid="PutBucketPolicy" remediate_object_acl_fn.add_to_role_policy( remediate_object_acl_fn_perms ) """ with open("lambda_src/is_object_private.py", encoding="utf8") as fp: is_object_private_fn_handler_code = fp.read() is_object_private_fn = _lambda.Function( self, id='isObjPrivateFn', function_name="is_object_private_fn", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(is_object_private_fn_handler_code), handler='index.lambda_handler', timeout=core.Duration.seconds(3)) # Lets add the necessary permission for the lambda function is_object_private_fn_perms = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, resources=[ "arn:aws:s3:::*", ], actions=["s3:GetObjectAcl"]) is_object_private_fn.sid = "CheckObjectAcl" is_object_private_fn.add_to_role_policy(is_object_private_fn_perms) with open("lambda_src/make_object_private.py", encoding="utf8") as fp: make_object_private_fn_handler_code = fp.read() remediate_object_acl_fn = _lambda.Function( self, id='remediateObjAclFn', function_name="remediate_object_acl_fn", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(make_object_private_fn_handler_code), handler='index.lambda_handler', timeout=core.Duration.seconds(10)) # Lets add the necessary permission for the lambda function remediate_object_acl_fn_perms = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, resources=[ "arn:aws:s3:::*", ], actions=["s3:PutObjectAcl"]) remediate_object_acl_fn_perms.sid = "PutObjectAcl" remediate_object_acl_fn.add_to_role_policy( remediate_object_acl_fn_perms) info_sec_ops_topic = _sns.Topic(self, "infoSecOpsTopicId", display_name="InfoSecTopic", topic_name="InfoSecOpsTopic") # Subscribe InfoSecOps Email to topic info_sec_ops_topic.add_subscription( _subs.EmailSubscription(global_args.INFO_SEC_OPS_EMAIL)) # Grant Lambda permission to publish to topic # info_sec_ops_topic.grant_publish(lambda_notifier) # State Machine for notifying failed ACLs # Ref: https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html ############################################################################### ################# STEP FUNCTIONS EXPERIMENTAL CODE - UNSTABLE ################# ############################################################################### is_object_private_task = _sfn.Task( self, "isObjectPrivate?", task=_tasks.InvokeFunction(is_object_private_fn), result_path="$", output_path="$") remediate_object_acl_task = _sfn.Task( self, "RemediateObjectAcl", task=_tasks.InvokeFunction(remediate_object_acl_fn), result_path="$", output_path="$") notify_secops_task = _sfn.Task( self, "Notify InfoSecOps", task=_tasks.PublishToTopic( info_sec_ops_topic, integration_pattern=_sfn.ServiceIntegrationPattern. FIRE_AND_FORGET, message=_sfn.TaskInput.from_data_at("$.sns_message"), subject="Object Acl Remediation")) acl_remediation_failed_task = _sfn.Fail(self, "Acl Remediation Failed", cause="Acl Remediation Failed", error="Check Logs") acl_compliant_task = _sfn.Succeed(self, "Object Acl Compliant", comment="Object Acl is Compliant") remediate_object_acl_sfn_definition = is_object_private_task\ .next(_sfn.Choice(self, "Is Object Private?")\ .when(_sfn.Condition.boolean_equals("$.is_private", True), acl_compliant_task)\ .when(_sfn.Condition.boolean_equals("$.is_private", False), remediate_object_acl_task\ .next(_sfn.Choice(self, "Object Remediation Complete?")\ .when(_sfn.Condition.boolean_equals("$.status", True),acl_compliant_task)\ .when(_sfn.Condition.boolean_equals("$.status", False), notify_secops_task.next(acl_remediation_failed_task))\ .otherwise(acl_remediation_failed_task)\ ) ) .otherwise(acl_remediation_failed_task) ) remediate_object_acl_statemachine = _sfn.StateMachine( self, "stateMachineId", definition=remediate_object_acl_sfn_definition, timeout=core.Duration.minutes(3)) # Cloudwatch Event Triggers put_object_acl_event_targets = [] """ put_object_acl_event_targets.append( _targets.LambdaFunction( handler=remediate_object_acl_fn ) ) """ put_object_acl_event_targets.append( _targets.SfnStateMachine( machine=remediate_object_acl_statemachine)) put_object_acl_event_pattern = _events.EventPattern( source=["aws.s3"], detail_type=["AWS API Call via CloudTrail"], detail={ "eventSource": ["s3.amazonaws.com"], "eventName": ["PutObjectAcl", "PutObject"], "requestParameters": { "bucketName": [f"{pvt_bkt.bucket_name}"] } }) put_object_acl_event_pattern_rule = _events.Rule( self, "putObjectAclEventId", event_pattern=put_object_acl_event_pattern, rule_name=f"put_s3_policy_event_{global_args.OWNER}", enabled=True, description="Trigger an event for S3 PutObjectAcl or PutObject", targets=put_object_acl_event_targets) ########################################### ################# OUTPUTS ################# ########################################### output0 = core.CfnOutput( self, "SecuirtyAutomationFrom", value=f"{global_args.SOURCE_INFO}", description= "To know more about this automation stack, check out our github page." ) output1 = core.CfnOutput( self, "MonitoredS3Bucket", value=(f"https://console.aws.amazon.com/s3/buckets/" f"{pvt_bkt.bucket_name}"), description=f"S3 Bucket for testing purposes") output2 = core.CfnOutput( self, "Helpercommands", value= (f"aws s3api get-object-acl --bucket ${pvt_bkt.bucket_name} --key OBJECT-KEY-NAME" ), description= f"Commands to set object to public, Update OBJECT-KEY-NAME to your needs" )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # A cache to temporarily hold session info session_cache_table = aws_dynamodb.Table( self, 'session_cache_table', partition_key={ 'name': 'code', 'type': aws_dynamodb.AttributeType.STRING }, billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST, time_to_live_attribute='expires') #-- # Secrets #--------------------# # Twitter secrets are stored external to this stack twitter_secret = aws_secretsmanager.Secret.from_secret_attributes( self, 'twitter_secret', secret_arn=os.environ['TWITTER_SECRET_ARN']) #-- # Layers #--------------------# # Each of these dependencies is used in 2 or more functions, extracted to layer for ease of use twitter_layer = aws_lambda.LayerVersion( self, 'twitter_layer', code=aws_lambda.AssetCode('layers/twitter_layer'), compatible_runtimes=[ aws_lambda.Runtime.PYTHON_2_7, aws_lambda.Runtime.PYTHON_3_6 ]) boto_layer = aws_lambda.LayerVersion( self, 'boto_layer', code=aws_lambda.AssetCode('layers/boto_layer'), compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6]) #-- # Functions #--------------------# # Handles CRC validation requests from Twitter twitter_crc_func = aws_lambda.Function( self, "twitter_crc_func", code=aws_lambda.AssetCode('functions/twitter_crc_func'), handler="lambda.handler", layers=[twitter_layer], runtime=aws_lambda.Runtime.PYTHON_2_7, environment={'twitter_secret': twitter_secret.secret_arn}) # Grant this function the ability to read Twitter credentials twitter_secret.grant_read(twitter_crc_func.role) # Handle schedule requests from Twitter twitter_webhook_func = aws_lambda.Function( self, "twitter_webhook_func", code=aws_lambda.AssetCode('functions/twitter_webhook_func'), handler="lambda.handler", layers=[boto_layer, twitter_layer], runtime=aws_lambda.Runtime.PYTHON_3_6, environment={'twitter_secret': twitter_secret.secret_arn}) # Grant this function permission to read Twitter credentials twitter_secret.grant_read(twitter_webhook_func.role) # Grant this function permission to publish tweets to EventBridge twitter_webhook_func.add_to_role_policy( aws_iam.PolicyStatement(actions=["events:PutEvents"], resources=["*"])) # Use API Gateway as the webhook endpoint twitter_api = aws_apigateway.LambdaRestApi( self, 'twitter_api', handler=twitter_webhook_func, proxy=False) # Tweets are POSTed to the endpoint twitter_api.root.add_method('POST') # Handles twitter CRC validation requests via GET to the webhook twitter_api.root.add_method( 'GET', aws_apigateway.LambdaIntegration(twitter_crc_func)) # Extract relevant info from the tweet, including session codes parse_tweet_func = aws_lambda.Function( self, "parse_tweet_func", code=aws_lambda.AssetCode('functions/parse_tweet_func'), handler="lambda.handler", runtime=aws_lambda.Runtime.PYTHON_3_6) # Get session information for requested codes get_sessions_func = aws_lambda.Function( self, "get_sessions_func", code=aws_lambda.AssetCode('functions/get_sessions_func'), handler="lambda.handler", runtime=aws_lambda.Runtime.PYTHON_3_6, timeout=core.Duration.seconds(60), layers=[boto_layer], environment={ 'CACHE_TABLE': session_cache_table.table_name, 'LOCAL_CACHE_TTL': str(1 * 60 * 60), # Cache sessions locally for 1 hour 'REMOTE_CACHE_TTL': str(12 * 60 * 60) }) # Cache sessions removely for 12 hours # This functions needs permissions to read and write to the table session_cache_table.grant_write_data(get_sessions_func) session_cache_table.grant_read_data(get_sessions_func) # Create a schedule without conflicts create_schedule_func = aws_lambda.Function( self, "create_schedule_func", code=aws_lambda.AssetCode('functions/create_schedule_func'), handler="lambda.handler", runtime=aws_lambda.Runtime.PYTHON_3_6, timeout=core.Duration.seconds(60)) # Tweet the response to the user tweet_schedule_func = aws_lambda.Function( self, "tweet_schedule_func", code=aws_lambda.AssetCode('functions/tweet_schedule_func'), handler="lambda.handler", layers=[boto_layer, twitter_layer], runtime=aws_lambda.Runtime.PYTHON_3_6, environment={'twitter_secret': twitter_secret.secret_arn}) twitter_secret.grant_read(tweet_schedule_func.role) #-- # States #--------------------# # Step 4 tweet_schedule_job = aws_stepfunctions.Task( self, 'tweet_schedule_job', task=aws_stepfunctions_tasks.InvokeFunction(tweet_schedule_func)) # Step 3 create_schedule_job = aws_stepfunctions.Task( self, 'create_schedule_job', task=aws_stepfunctions_tasks.InvokeFunction(create_schedule_func), input_path="$.sessions", result_path="$.schedule") create_schedule_job.next(tweet_schedule_job) # Step 2 - Get associated sessions (scrape or cache) get_sessions_job = aws_stepfunctions.Task( self, 'get_sessions_job', task=aws_stepfunctions_tasks.InvokeFunction(get_sessions_func)) # Prepare to get session info in parallel using the Map state get_sessions_map = aws_stepfunctions.Map(self, 'get_sessions_map', items_path="$.codes", result_path="$.sessions") get_sessions_map.iterator(get_sessions_job) get_sessions_map.next(create_schedule_job) # Shortcut if no session codes are supplied check_num_codes = aws_stepfunctions.Choice(self, 'check_num_codes') check_num_codes.when( aws_stepfunctions.Condition.number_greater_than('$.num_codes', 0), get_sessions_map) check_num_codes.otherwise(aws_stepfunctions.Succeed(self, "no_codes")) # Step 1 - Parse incoming tweet and prepare for scheduling parse_tweet_job = aws_stepfunctions.Task( self, 'parse_tweet_job', task=aws_stepfunctions_tasks.InvokeFunction(parse_tweet_func)) parse_tweet_job.next(check_num_codes) #-- # State Machines #--------------------# schedule_machine = aws_stepfunctions.StateMachine( self, "schedule_machine", definition=parse_tweet_job) # A rule to filter reInventSched tweet events reinvent_sched_rule = aws_events.Rule( self, "reinvent_sched_rule", event_pattern={"source": ["reInventSched"]}) # Matching events start the image pipline reinvent_sched_rule.add_target( aws_events_targets.SfnStateMachine( schedule_machine, input=aws_events.RuleTargetInput.from_event_path("$.detail")))
def create_stepfunction(self) -> Resource: """コンポーネントをビルドしてデプロイするステートマシンの作成 Returns: Resource: step function """ name = f"{self.stack_name}_{self.component_id}_edgedeploy_pipeline" role_name = self.get_role_name("edgedeploy_pipeline") sf_role = aws_iam.Role( self, id=role_name, assumed_by=aws_iam.ServicePrincipal("states.amazonaws.com"), role_name=role_name, path="/service-role/", managed_policies=[ aws_iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaBasicExecutionRole") ]) sf_role.attach_inline_policy( aws_iam.Policy( self, "AllowCloudWatchLogsForSF", document=aws_iam.PolicyDocument(statements=[ aws_iam.PolicyStatement(actions=[ "logs:CreateLogDelivery", "logs:GetLogDelivery", "logs:UpdateLogDelivery", "logs:DeleteLogDelivery", "logs:ListLogDeliveries", "logs:PutResourcePolicy", "logs:DescribeResourcePolicies", "logs:DescribeLogGroups" ], resources=["*"]) ]))) sf_role.attach_inline_policy( aws_iam.Policy( self, "AllowXRayForSF", document=aws_iam.PolicyDocument(statements=[ aws_iam.PolicyStatement(actions=[ "xray:PutTraceSegments", "xray:PutTelemetryRecords", "xray:GetSamplingRules", "xray:GetSamplingTargets" ], resources=["*"]) ]))) sf_role.attach_inline_policy( aws_iam.Policy( self, "AllowInvokeLambda", document=aws_iam.PolicyDocument(statements=[ aws_iam.PolicyStatement( actions=["lambda:InvokeFunction"], resources=[ self._lambda_build_image.function_arn, self._lambda_build_image.function_arn + ":*", self._lambda_check_image_status.function_arn, self._lambda_check_image_status.function_arn + ":*", self._lambda_create_component.function_arn, self._lambda_create_component.function_arn + ":*", self._lambda_deploy_component.function_arn, self._lambda_deploy_component.function_arn + ":*", self._lambda_check_deploy_status.function_arn, self._lambda_check_deploy_status.function_arn + ":*" ]) ]))) # dockerコンテナをビルド task_build_image = aws_sf_tasks.LambdaInvoke( self, "BuildInferenceImage", lambda_function=self._lambda_build_image, output_path="$.Payload") # dockerコンテナのビルド結果を確認 task_check_build_image_status = aws_sf_tasks.LambdaInvoke( self, "CheckDockerImageBuildStatus", lambda_function=self._lambda_check_image_status, output_path="$.Payload") # dockerコンテナのビルドを待つ wait_image_build = aws_sf.Wait(self, "WaitImageBuildFinish", time=aws_sf.WaitTime.duration( Duration.seconds(30))) # Greengrassのコンポーネントを作成 task_create_greengrass_component = aws_sf_tasks.LambdaInvoke( self, "CreateComponent", lambda_function=self._lambda_create_component, output_path="$.Payload") # Greengrassへデプロイ task_deploy_component = aws_sf_tasks.LambdaInvoke( self, "DeployComponent", lambda_function=self._lambda_deploy_component, output_path="$.Payload") # Greengrassへのデプロイ終了を待つ wait_component_deploy = aws_sf.Wait(self, "WaitDeploymentFinish", time=aws_sf.WaitTime.duration( Duration.seconds(30))) # Greengrassへデプロイ結果を確認 task_check_deployment_status = aws_sf_tasks.LambdaInvoke( self, "CheckDeploymentStatus", lambda_function=self._lambda_check_deploy_status, output_path="$.Payload") # デプロイ失敗 pipeline_failed = aws_sf.Fail(self, "PipelineFailed", error="DeployPipelineFailed", cause="Something went wrong") # 正常終了 pipeline_success = aws_sf.Succeed(self, "PipelineSuccessed") # dockerコンテナが存在したかを判定 choice_component_exists_result = aws_sf.Choice(self, "JudgeComponentExists") # dockerコンテナのビルド結果を判定 choice_image_build_result = aws_sf.Choice(self, "JudgeImageBuildStatus") # dockerコンテナのビルド結果を判定 choice_deployment_result = aws_sf.Choice(self, "JudgeDeploymentStatus") # 正常終了を通知 publish_success_message = aws_sf_tasks.SnsPublish( self, "Publish Success message", topic=aws_sns.Topic(self, "SendDeploySuccess"), message=aws_sf.TaskInput.from_json_path_at("$.message")).next( pipeline_success) # デプロイ失敗を通知 publish_failed_message = aws_sf_tasks.SnsPublish( self, "Publish Failed message", topic=aws_sns.Topic(self, "SendPipelineFailed"), message=aws_sf.TaskInput.from_json_path_at("$.message")).next( pipeline_failed) definition = \ task_build_image.next( choice_component_exists_result .when( aws_sf.Condition.string_equals("$.status", "component_exists"), task_deploy_component) .otherwise( wait_image_build.next( task_check_build_image_status).next( choice_image_build_result.when( aws_sf.Condition.string_equals("$.status", "image_exists"), task_create_greengrass_component .next(task_deploy_component) .next(wait_component_deploy) .next(task_check_deployment_status) .next( choice_deployment_result .when(aws_sf.Condition.string_equals("$.status", "RUNNING"), wait_component_deploy) .when(aws_sf.Condition.string_equals("$.status", "COMPLETED"), publish_success_message) .otherwise(publish_failed_message).afterwards())) .when( aws_sf.Condition.string_equals("$.status", "image_faild"), publish_failed_message) .otherwise( wait_image_build).afterwards()) ) ) #.next(aws_sf.Succeed(self, "GreengrassComponentDeployFinished")) state_machine = aws_sf.StateMachine( self, id=name, state_machine_name=name, definition=definition, state_machine_type=aws_sf.StateMachineType.STANDARD, role=sf_role)
def __init__(self, scope: core.Construct, id: str, group_name: str, minute_duration: int, **kwargs) -> None: super().__init__(scope, id, **kwargs) # TODO: Setup alerting of failure to an SNS # TODO: Failure is not the same as a student not in a group # TODO: Streamline input data so that lambda's only get the info they really need # TODO: Comment # TODO: Need to separate unexpected errors from regular errors # Setting up monitoring schedule_stop = lambda_.Function( self, id="ScheduleStopLambda", runtime=lambda_.Runtime.PYTHON_3_7, code=lambda_.Code.from_inline( open("./resources/schedule-termination.py", 'r').read()), handler="index.handler", log_retention=logs.RetentionDays.ONE_DAY, environment=dict(GROUP_NAME=group_name), timeout=core.Duration.seconds(30)) schedule_stop.add_to_role_policy( statement=iam.PolicyStatement(actions=[ "ec2:Describe*", "iam:ListGroupsForUser", "iam:ListUsers" ], effect=iam.Effect.ALLOW, resources=["*"])) terminate_ec2 = lambda_.Function( self, id="TerminateEC2", runtime=lambda_.Runtime.PYTHON_3_7, code=lambda_.Code.from_inline( open("./resources/terminate-ec2.py", 'r').read()), handler="index.handler", log_retention=logs.RetentionDays.ONE_DAY, timeout=core.Duration.seconds(30)) terminate_ec2.add_to_role_policy( statement=iam.PolicyStatement(actions=[ "ec2:DescribeInstance*", "ec2:TerminateInstances", ], effect=iam.Effect.ALLOW, resources=["*"])) # The lambda object that will see if we should schedule. schedule_stop_task = tasks.LambdaInvoke( self, id='schedule stop', lambda_function=schedule_stop, input_path="$.detail.userIdentity", result_path="$.Payload", ) # TODO: Need to change this based on the configuration info above # Wait state to try and delete # wait_x = sfn.Wait(self, 'Wait x minutes', time=sfn.WaitTime.seconds_path("10")) wait_x = sfn.Wait(self, id='Wait x minutes', time=sfn.WaitTime.duration( core.Duration.minutes(minute_duration))) job_failed = sfn.Fail(self, id="Failed Job", cause="Error in the input", error="Error") job_finished = sfn.Succeed(self, id="Job Finished") choice = sfn.Choice(self, 'Can I delete') choice.when(sfn.Condition.boolean_equals('$.Payload.Payload', False), job_finished) choice.otherwise(wait_x) terminate_ec2_task = tasks.LambdaInvoke( self, 'terminate', lambda_function=terminate_ec2, input_path="$.detail.responseElements.instancesSet") wait_x.next(terminate_ec2_task).next(job_finished) state_definition = schedule_stop_task \ .next(choice) terminate_machine = sfn.StateMachine(self, id="State Machine", definition=state_definition) cloudwatch.Alarm(self, "EC2ScheduleAlarm", metric=terminate_machine.metric_failed(), threshold=1, evaluation_periods=1) # TODO Build Rule that monitors for EC2 creation # Any new creation, the EC2 will have to be destroyed. Including # other things? create_event = events.Rule( self, id='detect-ec2-start', description="Detects if an EC2 is created", enabled=True, event_pattern=events.EventPattern( detail_type=["AWS API Call via CloudTrail"], source=["aws.ec2"], detail={ "eventName": ["RunInstances"], "eventSource": ["ec2.amazonaws.com"] }), targets=[targets.SfnStateMachine(terminate_machine)])
def __init__( self, scope: Construct, stack_id: str, *, botocore_lambda_layer: aws_lambda_python.PythonLayerVersion, env_name: str, storage_bucket: aws_s3.Bucket, validation_results_table: Table, ) -> None: # pylint: disable=too-many-locals, too-many-statements super().__init__(scope, stack_id) ############################################################################################ # PROCESSING ASSETS TABLE processing_assets_table = Table( self, f"{env_name}-processing-assets", env_name=env_name, parameter_name=ParameterName.PROCESSING_ASSETS_TABLE_NAME, sort_key=aws_dynamodb.Attribute(name="sk", type=aws_dynamodb.AttributeType.STRING), ) ############################################################################################ # BATCH JOB DEPENDENCIES batch_job_queue = BatchJobQueue( self, "batch-job-queue", env_name=env_name, processing_assets_table=processing_assets_table, ).job_queue s3_read_only_access_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonS3ReadOnlyAccess" ) ############################################################################################ # UPDATE CATALOG UPDATE MESSAGE QUEUE dead_letter_queue = aws_sqs.Queue( self, "dead-letter-queue", visibility_timeout=LAMBDA_TIMEOUT, ) self.message_queue = aws_sqs.Queue( self, "update-catalog-message-queue", visibility_timeout=LAMBDA_TIMEOUT, dead_letter_queue=aws_sqs.DeadLetterQueue(max_receive_count=3, queue=dead_letter_queue), ) self.message_queue_name_parameter = aws_ssm.StringParameter( self, "update-catalog-message-queue-name", string_value=self.message_queue.queue_name, description=f"Update Catalog Message Queue Name for {env_name}", parameter_name=ParameterName.UPDATE_CATALOG_MESSAGE_QUEUE_NAME.value, ) populate_catalog_lambda = BundledLambdaFunction( self, "populate-catalog-bundled-lambda-function", directory="populate_catalog", extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, botocore_lambda_layer=botocore_lambda_layer, ) self.message_queue.grant_consume_messages(populate_catalog_lambda) populate_catalog_lambda.add_event_source( SqsEventSource(self.message_queue, batch_size=1) # type: ignore[arg-type] ) ############################################################################################ # STATE MACHINE TASKS check_stac_metadata_task = LambdaTask( self, "check-stac-metadata-task", directory="check_stac_metadata", botocore_lambda_layer=botocore_lambda_layer, extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, ) assert check_stac_metadata_task.lambda_function.role check_stac_metadata_task.lambda_function.role.add_managed_policy( policy=s3_read_only_access_policy ) for table in [processing_assets_table, validation_results_table]: table.grant_read_write_data(check_stac_metadata_task.lambda_function) table.grant( check_stac_metadata_task.lambda_function, "dynamodb:DescribeTable", ) content_iterator_task = LambdaTask( self, "content-iterator-task", directory="content_iterator", botocore_lambda_layer=botocore_lambda_layer, result_path=f"$.{CONTENT_KEY}", extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, ) check_files_checksums_directory = "check_files_checksums" check_files_checksums_default_payload_object = { f"{DATASET_ID_KEY}.$": f"$.{DATASET_ID_KEY}", f"{VERSION_ID_KEY}.$": f"$.{VERSION_ID_KEY}", f"{METADATA_URL_KEY}.$": f"$.{METADATA_URL_KEY}", f"{FIRST_ITEM_KEY}.$": f"$.{CONTENT_KEY}.{FIRST_ITEM_KEY}", f"{ASSETS_TABLE_NAME_KEY}.$": f"$.{CONTENT_KEY}.{ASSETS_TABLE_NAME_KEY}", f"{RESULTS_TABLE_NAME_KEY}.$": f"$.{CONTENT_KEY}.{RESULTS_TABLE_NAME_KEY}", } check_files_checksums_single_task = BatchSubmitJobTask( self, "check-files-checksums-single-task", env_name=env_name, directory=check_files_checksums_directory, s3_policy=s3_read_only_access_policy, job_queue=batch_job_queue, payload_object=check_files_checksums_default_payload_object, container_overrides_command=[ "--dataset-id", f"Ref::{DATASET_ID_KEY}", "--version-id", f"Ref::{VERSION_ID_KEY}", "--first-item", f"Ref::{FIRST_ITEM_KEY}", "--assets-table-name", f"Ref::{ASSETS_TABLE_NAME_KEY}", "--results-table-name", f"Ref::{RESULTS_TABLE_NAME_KEY}", ], ) array_size = int( aws_stepfunctions.JsonPath.number_at(f"$.{CONTENT_KEY}.{ITERATION_SIZE_KEY}") ) check_files_checksums_array_task = BatchSubmitJobTask( self, "check-files-checksums-array-task", env_name=env_name, directory=check_files_checksums_directory, s3_policy=s3_read_only_access_policy, job_queue=batch_job_queue, payload_object=check_files_checksums_default_payload_object, container_overrides_command=[ "--dataset-id", f"Ref::{DATASET_ID_KEY}", "--version-id", f"Ref::{VERSION_ID_KEY}", "--first-item", f"Ref::{FIRST_ITEM_KEY}", "--assets-table-name", f"Ref::{ASSETS_TABLE_NAME_KEY}", "--results-table-name", f"Ref::{RESULTS_TABLE_NAME_KEY}", ], array_size=array_size, ) for reader in [ content_iterator_task.lambda_function, check_files_checksums_single_task.job_role, check_files_checksums_array_task.job_role, ]: processing_assets_table.grant_read_data(reader) # type: ignore[arg-type] processing_assets_table.grant( reader, "dynamodb:DescribeTable" # type: ignore[arg-type] ) for writer in [ check_files_checksums_single_task.job_role, check_files_checksums_array_task.job_role, ]: validation_results_table.grant_read_write_data(writer) # type: ignore[arg-type] validation_results_table.grant( writer, "dynamodb:DescribeTable" # type: ignore[arg-type] ) validation_summary_task = LambdaTask( self, "validation-summary-task", directory="validation_summary", botocore_lambda_layer=botocore_lambda_layer, result_path=f"$.{VALIDATION_KEY}", extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, ) validation_results_table.grant_read_data(validation_summary_task.lambda_function) validation_results_table.grant( validation_summary_task.lambda_function, "dynamodb:DescribeTable" ) import_dataset_role = aws_iam.Role( self, "import-dataset", assumed_by=aws_iam.ServicePrincipal( # type: ignore[arg-type] "batchoperations.s3.amazonaws.com" ), ) import_asset_file_function = ImportFileFunction( self, directory="import_asset_file", invoker=import_dataset_role, env_name=env_name, botocore_lambda_layer=botocore_lambda_layer, ) import_metadata_file_function = ImportFileFunction( self, directory="import_metadata_file", invoker=import_dataset_role, env_name=env_name, botocore_lambda_layer=botocore_lambda_layer, ) import_dataset_task = LambdaTask( self, "import-dataset-task", directory="import_dataset", botocore_lambda_layer=botocore_lambda_layer, result_path=f"$.{IMPORT_DATASET_KEY}", extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, ) import_dataset_task.lambda_function.add_to_role_policy( aws_iam.PolicyStatement( resources=[import_dataset_role.role_arn], actions=["iam:PassRole"], ), ) import_dataset_task.lambda_function.add_to_role_policy( aws_iam.PolicyStatement(resources=["*"], actions=["s3:CreateJob"]) ) for table in [processing_assets_table]: table.grant_read_data(import_dataset_task.lambda_function) table.grant(import_dataset_task.lambda_function, "dynamodb:DescribeTable") # Import status check wait_before_upload_status_check = Wait( self, "wait-before-upload-status-check", time=WaitTime.duration(Duration.seconds(10)), ) upload_status_task = LambdaTask( self, "upload-status", directory="upload_status", botocore_lambda_layer=botocore_lambda_layer, result_path="$.upload_status", extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, ) validation_results_table.grant_read_data(upload_status_task.lambda_function) validation_results_table.grant(upload_status_task.lambda_function, "dynamodb:DescribeTable") upload_status_task.lambda_function.add_to_role_policy(ALLOW_DESCRIBE_ANY_S3_JOB) # Parameters import_asset_file_function_arn_parameter = aws_ssm.StringParameter( self, "import asset file function arn", string_value=import_asset_file_function.function_arn, description=f"Import asset file function ARN for {env_name}", parameter_name=ParameterName.PROCESSING_IMPORT_ASSET_FILE_FUNCTION_TASK_ARN.value, ) import_metadata_file_function_arn_parameter = aws_ssm.StringParameter( self, "import metadata file function arn", string_value=import_metadata_file_function.function_arn, description=f"Import metadata file function ARN for {env_name}", parameter_name=ParameterName.PROCESSING_IMPORT_METADATA_FILE_FUNCTION_TASK_ARN.value, ) import_dataset_role_arn_parameter = aws_ssm.StringParameter( self, "import dataset role arn", string_value=import_dataset_role.role_arn, description=f"Import dataset role ARN for {env_name}", parameter_name=ParameterName.PROCESSING_IMPORT_DATASET_ROLE_ARN.value, ) update_dataset_catalog = LambdaTask( self, "update-dataset-catalog", directory="update_dataset_catalog", botocore_lambda_layer=botocore_lambda_layer, extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, ) self.message_queue.grant_send_messages(update_dataset_catalog.lambda_function) for storage_writer in [ import_dataset_role, import_dataset_task.lambda_function, import_asset_file_function, import_metadata_file_function, populate_catalog_lambda, update_dataset_catalog.lambda_function, ]: storage_bucket.grant_read_write(storage_writer) # type: ignore[arg-type] grant_parameter_read_access( { import_asset_file_function_arn_parameter: [import_dataset_task.lambda_function], import_dataset_role_arn_parameter: [import_dataset_task.lambda_function], import_metadata_file_function_arn_parameter: [import_dataset_task.lambda_function], processing_assets_table.name_parameter: [ check_stac_metadata_task.lambda_function, content_iterator_task.lambda_function, import_dataset_task.lambda_function, ], validation_results_table.name_parameter: [ check_stac_metadata_task.lambda_function, content_iterator_task.lambda_function, validation_summary_task.lambda_function, upload_status_task.lambda_function, ], self.message_queue_name_parameter: [update_dataset_catalog.lambda_function], } ) success_task = aws_stepfunctions.Succeed(self, "success") upload_failure = aws_stepfunctions.Fail(self, "upload failure") validation_failure = aws_stepfunctions.Succeed(self, "validation failure") ############################################################################################ # STATE MACHINE dataset_version_creation_definition = ( check_stac_metadata_task.next(content_iterator_task) .next( aws_stepfunctions.Choice( # type: ignore[arg-type] self, "check_files_checksums_maybe_array" ) .when( aws_stepfunctions.Condition.number_equals( f"$.{CONTENT_KEY}.{ITERATION_SIZE_KEY}", 1 ), check_files_checksums_single_task.batch_submit_job, ) .otherwise(check_files_checksums_array_task.batch_submit_job) .afterwards() ) .next( aws_stepfunctions.Choice(self, "content_iteration_finished") .when( aws_stepfunctions.Condition.number_equals( f"$.{CONTENT_KEY}.{NEXT_ITEM_KEY}", -1 ), validation_summary_task.next( aws_stepfunctions.Choice( # type: ignore[arg-type] self, "validation_successful" ) .when( aws_stepfunctions.Condition.boolean_equals( f"$.{VALIDATION_KEY}.{SUCCESS_KEY}", True ), import_dataset_task.next( wait_before_upload_status_check # type: ignore[arg-type] ) .next(upload_status_task) .next( aws_stepfunctions.Choice( self, "import_completed" # type: ignore[arg-type] ) .when( aws_stepfunctions.Condition.and_( aws_stepfunctions.Condition.string_equals( f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Complete" ), aws_stepfunctions.Condition.string_equals( f"$.upload_status.{METADATA_UPLOAD_KEY}.status", "Complete", ), ), update_dataset_catalog.next( success_task # type: ignore[arg-type] ), ) .when( aws_stepfunctions.Condition.or_( aws_stepfunctions.Condition.string_equals( f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Cancelled", ), aws_stepfunctions.Condition.string_equals( f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Failed" ), aws_stepfunctions.Condition.string_equals( f"$.upload_status.{METADATA_UPLOAD_KEY}.status", "Cancelled", ), aws_stepfunctions.Condition.string_equals( f"$.upload_status.{METADATA_UPLOAD_KEY}.status", "Failed", ), ), upload_failure, # type: ignore[arg-type] ) .otherwise( wait_before_upload_status_check # type: ignore[arg-type] ) ), ) .otherwise(validation_failure) # type: ignore[arg-type] ), ) .otherwise(content_iterator_task) ) ) self.state_machine = aws_stepfunctions.StateMachine( self, f"{env_name}-dataset-version-creation", definition=dataset_version_creation_definition, # type: ignore[arg-type] ) self.state_machine_parameter = aws_ssm.StringParameter( self, "state machine arn", description=f"State machine ARN for {env_name}", parameter_name=ParameterName.PROCESSING_DATASET_VERSION_CREATION_STEP_FUNCTION_ARN.value, # pylint:disable=line-too-long string_value=self.state_machine.state_machine_arn, ) Tags.of(self).add("ApplicationLayer", "processing") # type: ignore[arg-type]
def __init__(self, scope, id, *args, **kwargs): super().__init__(scope, id, *args, **kwargs) # Buckets source_bucket = s3.Bucket(self, "SourceBucket") dest_bucket = s3.Bucket(self, "DestinationBucket") processing_bucket = s3.Bucket(self, "ProcessingBucket") # Lambda Functions generate_workflow_input_lambda = aws_lambda.Function( self, "GenerateWorkflowInputFunction", code=aws_lambda.Code.from_asset(str(DIST_PATH)), runtime=aws_lambda.Runtime.PYTHON_3_8, handler="generate_workflow_input.lambda_handler", environment={ "InputBucketName": source_bucket.bucket_name, "ProcessingBucketName": processing_bucket.bucket_name, "OutputBucketName": dest_bucket.bucket_name } ) check_workflow_ready_lambda = aws_lambda.Function( self, "CheckWorkflowReadyFunction", code=aws_lambda.Code.from_asset(str(DIST_PATH)), runtime=aws_lambda.Runtime.PYTHON_3_8, handler="check_workflow_ready.lambda_handler" ) string_replace_lambda = aws_lambda.Function( self, "StringReplaceFunction", code=aws_lambda.Code.from_asset(str(DIST_PATH)), runtime=aws_lambda.Runtime.PYTHON_3_8, handler="string_replace.lambda_handler" ) calculate_total_earnings_lambda = aws_lambda.Function( self, "CalculateTotalEarningsFunction", code=aws_lambda.Code.from_asset(str(DIST_PATH)), runtime=aws_lambda.Runtime.PYTHON_3_8, handler="calculate_total_earnings.lambda_handler" ) convert_csv_to_json_lambda = aws_lambda.Function( self, "ConvertCsvToJsonFunction", code=aws_lambda.Code.from_asset(str(DIST_PATH)), runtime=aws_lambda.Runtime.PYTHON_3_8, handler="convert_csv_to_json.lambda_handler" ) # Permissions source_bucket.grant_read(check_workflow_ready_lambda) source_bucket.grant_read(string_replace_lambda) processing_bucket.grant_write(string_replace_lambda) processing_bucket.grant_read_write(calculate_total_earnings_lambda) processing_bucket.grant_read(convert_csv_to_json_lambda) dest_bucket.grant_write(convert_csv_to_json_lambda) # Outputs core.CfnOutput(self, "SourceBucketName", value=source_bucket.bucket_name) core.CfnOutput(self, "DestinationBucketName", value=dest_bucket.bucket_name) core.CfnOutput(self, "ProcessingBucketName", value=processing_bucket.bucket_name) core.CfnOutput(self, "GenerateWorkflowInputLambda", value=generate_workflow_input_lambda.function_name) core.CfnOutput(self, "CheckWorkflowReadyLambda", value=check_workflow_ready_lambda.function_name) core.CfnOutput(self, "StringReplaceLambda", value=string_replace_lambda.function_name) core.CfnOutput(self, "CalculateTotalEarningsLambda", value=calculate_total_earnings_lambda.function_name) core.CfnOutput(self, "ConvertCsvToJsonLambda", value=convert_csv_to_json_lambda.function_name) # State Machine generate_workflow_input_task = sf_tasks.LambdaInvoke( self, "GenerateWorkflowInput", lambda_function=generate_workflow_input_lambda, payload_response_only=True ) check_workflow_ready_task = sf_tasks.LambdaInvoke( self, "CheckWorkflowReady", lambda_function=check_workflow_ready_lambda, input_path="$.CheckWorkflowReady.Input", result_path="$.CheckWorkflowReady.Output", payload_response_only=True ) string_replace_task = sf_tasks.LambdaInvoke( self, "ReplaceString", lambda_function=string_replace_lambda, result_path="$.StringReplace.Output", payload_response_only=True ) calculate_total_earnings_task = sf_tasks.LambdaInvoke( self, "CalculateTotalEarnings", lambda_function=calculate_total_earnings_lambda, input_path="$.CalculateTotalEarnings.Input", result_path="$.CalculateTotalEarnings.Output", payload_response_only=True ) convert_csv_to_json_task = sf_tasks.LambdaInvoke( self, "ConvertCsvToJson", lambda_function=convert_csv_to_json_lambda, input_path="$.ConvertCsvToJson.Input", result_path="$.ConvertCsvToJson.Output", payload_response_only=True ) end_task = sf.Succeed(self, "WorkflowEnd") replace_string_parallel = sf.Map( self, "ReplaceStringParallel", items_path="$.StringReplace.Input", result_path="$.StringReplace.Output" ).iterator(string_replace_task) workflow_steps = sf.Chain.\ start(replace_string_parallel)\ .next(calculate_total_earnings_task)\ .next(convert_csv_to_json_task)\ .next(end_task) run_workflow = sf.Choice(self, "RunWorkflowDecision")\ .when(sf.Condition.boolean_equals("$.CheckWorkflowReady.Output", True), workflow_steps)\ .otherwise(end_task) hello_workflow_state_machine = sf.StateMachine( self, "HelloWorkflowStateMachine", definition=sf.Chain\ .start(generate_workflow_input_task)\ .next(check_workflow_ready_task)\ .next(run_workflow) )
def __init__(self, scope: Construct, id: str, log_level: CfnParameter): super().__init__(scope, id) self._bundling = {} self.log_level = log_level.value_as_string self.source_path = Path(__file__).parent.parent.parent.parent self.topic = None self.subscription = None self.functions: Dict[Function] = {} self.policies = Policies(self) self.create_functions() # step function steps check_error = sfn.Choice(self, "Check-Error") notify_failed = tasks.LambdaInvoke( self, "Notify-Failed", lambda_function=self.functions["SNS"], payload_response_only=True, retry_on_service_exceptions=True, result_path=None, ) notify_failed.next(sfn.Fail(self, "FailureState")) create_dataset_group = tasks.LambdaInvoke( self, "Create-DatasetGroup", lambda_function=self.functions["CreateDatasetGroup"], result_path="$.DatasetGroupNames", payload_response_only=True, retry_on_service_exceptions=True, ) create_dataset_group.add_retry(backoff_rate=1.05, interval=Duration.seconds(5), errors=["ResourcePending"]) create_dataset_group.add_catch(notify_failed, errors=["ResourceFailed"], result_path="$.serviceError") create_dataset_group.add_catch(notify_failed, errors=["States.ALL"], result_path="$.statesError") import_data = tasks.LambdaInvoke( self, "Import-Data", lambda_function=self.functions["CreateDatasetImportJob"], result_path="$.DatasetImportJobArn", payload_response_only=True, retry_on_service_exceptions=True, ) import_data.add_retry( backoff_rate=1.05, interval=Duration.seconds(5), max_attempts=100, errors=["ResourcePending"], ) import_data.add_catch(notify_failed, errors=["ResourceFailed"], result_path="$.serviceError") import_data.add_catch(notify_failed, errors=["States.ALL"], result_path="$.statesError") update_not_required = sfn.Succeed(self, "Update-Not-Required") notify_success = tasks.LambdaInvoke( self, "Notify-Success", lambda_function=self.functions["SNS"], payload_response_only=True, retry_on_service_exceptions=True, result_path=None, ) notify_prediction_failed = tasks.LambdaInvoke( self, "Notify-Prediction-Failed", lambda_function=self.functions["SNS"], payload_response_only=True, retry_on_service_exceptions=True, result_path=None, ) notify_prediction_failed.next(sfn.Fail(self, "Prediction-Failed")) create_predictor = tasks.LambdaInvoke( self, "Create-Predictor", lambda_function=self.functions["CreatePredictor"], result_path="$.PredictorArn", payload_response_only=True, retry_on_service_exceptions=True, ) create_predictor.add_retry( backoff_rate=1.05, interval=Duration.seconds(5), max_attempts=100, errors=["ResourcePending", "DatasetsImporting"], ) create_predictor.add_catch( notify_prediction_failed, errors=["ResourceFailed"], result_path="$.serviceError", ) create_predictor.add_catch(notify_prediction_failed, errors=["States.ALL"], result_path="$.statesError") create_predictor.add_catch(update_not_required, errors=["NotMostRecentUpdate"]) create_forecast = tasks.LambdaInvoke( self, "Create-Forecast", lambda_function=self.functions["CreateForecast"], result_path="$.ForecastArn", payload_response_only=True, retry_on_service_exceptions=True, ) create_forecast.add_retry( backoff_rate=1.05, interval=Duration.seconds(5), max_attempts=100, errors=["ResourcePending"], ) create_forecast.add_catch( notify_prediction_failed, errors=["ResourceFailed"], result_path="$.serviceError", ) create_forecast.add_catch(notify_prediction_failed, errors=["States.ALL"], result_path="$.statesError") export_forecast = tasks.LambdaInvoke( self, "Export-Forecast", lambda_function=self.functions["PrepareForecastExport"], result_path="$.ExportTableName", payload_response_only=True, retry_on_service_exceptions=True, ) export_forecast.add_catch( notify_prediction_failed, errors=["ResourceFailed"], result_path="$.serviceError", ) export_forecast.add_catch(notify_prediction_failed, errors=["States.ALL"], result_path="$.statesError") create_forecasts = sfn.Map( self, "Create-Forecasts", items_path="$.DatasetGroupNames", parameters={ "bucket.$": "$.bucket", "dataset_file.$": "$.dataset_file", "dataset_group_name.$": "$$.Map.Item.Value", "config.$": "$.config", }, ) # step function definition definition = (check_error.when( sfn.Condition.is_present("$.serviceError"), notify_failed).otherwise(create_dataset_group).afterwards().next( import_data).next( create_forecasts.iterator( create_predictor.next(create_forecast).next( export_forecast).next(notify_success)))) self.state_machine = sfn.StateMachine(self, "DeployStateMachine", definition=definition)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # ------ Necessary Roles ------ roles = IamRole( self, 'IamRoles' ) # ------ S3 Buckets ------ # Create Athena bucket athena_bucket = _s3.Bucket(self, "AthenaBucket", removal_policy=core.RemovalPolicy.DESTROY ) # Create Forecast bucket forecast_bucket = _s3.Bucket(self, "FoecastBucket", removal_policy=core.RemovalPolicy.DESTROY ) # ------ Athena ------ # Config Athena query result output location workgroup_prop = _athena.CfnWorkGroup.WorkGroupConfigurationProperty( result_configuration=_athena.CfnWorkGroup.ResultConfigurationProperty( output_location="s3://"+athena_bucket.bucket_name ) ) # Create Athena workgroup athena_workgroup = _athena.CfnWorkGroup( self, 'ForecastGroup', name='ForecastGroup', recursive_delete_option=True, state='ENABLED', work_group_configuration=workgroup_prop ) # ------ SNS Topic ------ topic = sns.Topic( self, 'NotificationTopic', display_name='StepsTopic' ) # SNS email subscription. Get the email address from context value(cdk.json) topic.add_subscription(subs.EmailSubscription(self.node.try_get_context('my_email'))) # ------ Layers ------ shared_layer = _lambda.LayerVersion( self, 'LambdaLayer', layer_version_name='testfolderlayer', code=_lambda.AssetCode('shared/') ) # ------ Lambdas for stepfuctions------ create_dataset_lambda = _lambda.Function( self, 'CreateDataset', function_name='CreateDataset', code=_lambda.Code.asset('lambdas/createdataset/'), handler='dataset.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, timeout=core.Duration.seconds(30), layers=[shared_layer] ) create_dataset_group_lambda = _lambda.Function( self, 'CreateDatasetGroup', function_name='CreateDatasetGroup', code = _lambda.Code.asset('lambdas/createdatasetgroup/'), handler = 'datasetgroup.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) import_data_lambda = _lambda.Function( self, 'CreateDatasetImportJob', function_name='CreateDatasetImportJob', code = _lambda.Code.asset('lambdas/createdatasetimportjob/'), handler = 'datasetimport.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, environment= { 'FORECAST_ROLE': roles.forecast_role.role_arn }, layers=[shared_layer] ) create_predictor_lambda = _lambda.Function( self, 'CreatePredictor', function_name='CreatePredictor', code = _lambda.Code.asset('lambdas/createpredictor/'), handler = 'predictor.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) create_forecast_lambda = _lambda.Function( self, 'CreateForecast', function_name='CreateForecast', code = _lambda.Code.asset('lambdas/createforecast/'), handler = 'forecast.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, environment= { 'EXPORT_ROLE': roles.forecast_role.role_arn }, layers=[shared_layer], timeout=core.Duration.seconds(30) ) # Deploy lambda with python dependencies from requirements.txt update_resources_lambda = _lambda_python.PythonFunction( self, 'UpdateResources', function_name='UpdateResources', entry='lambdas/updateresources/', index='update.py', handler='lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.update_role, environment= { 'ATHENA_WORKGROUP': athena_workgroup.name, 'ATHENA_BUCKET' : athena_bucket.bucket_name }, layers=[shared_layer], timeout=core.Duration.seconds(900) ) notify_lambda = _lambda.Function( self, 'NotifyTopic', function_name='NotifyTopic', code = _lambda.Code.asset('lambdas/notify/'), handler = 'notify.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, environment= { 'SNS_TOPIC_ARN': topic.topic_arn }, layers=[shared_layer] ) delete_forecast_lambda = _lambda.Function( self, 'DeleteForecast', function_name='DeleteForecast', code = _lambda.Code.asset('lambdas/deleteforecast/'), handler = 'deleteforecast.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) delete_predctor_lambda = _lambda.Function( self, 'DeletePredictor', function_name='DeletePredictor', code = _lambda.Code.asset('lambdas/deletepredictor/'), handler = 'deletepredictor.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) delete_importjob_lambda = _lambda.Function( self, 'DeleteImportJob', function_name='DeleteImportJob', code = _lambda.Code.asset('lambdas/deletedatasetimport/'), handler = 'deletedataset.lambda_handler', runtime = _lambda.Runtime.PYTHON_3_7, role=roles.lambda_role, layers=[shared_layer] ) # ------ StepFunctions ------ strategy_choice = sfn.Choice( self, 'Strategy-Choice' ) success_state = sfn.Succeed( self, 'SuccessState' ) failed = sfn_tasks.LambdaInvoke( self, 'Failed', lambda_function = notify_lambda, result_path=None ).next(strategy_choice) create_dataset_job = sfn_tasks.LambdaInvoke( self, 'Create-Dataset', lambda_function = create_dataset_lambda, retry_on_service_exceptions=True, payload_response_only=True ) self.add_retry_n_catch(create_dataset_job, failed) create_dataset_group_job = sfn_tasks.LambdaInvoke( self, 'Create-DatasetGroup', lambda_function = create_dataset_group_lambda, payload_response_only=True ) self.add_retry_n_catch(create_dataset_group_job, failed) import_data_job = sfn_tasks.LambdaInvoke( self, 'Import-Data', lambda_function = import_data_lambda, payload_response_only=True ) self.add_retry_n_catch(import_data_job, failed) create_predictor_job = sfn_tasks.LambdaInvoke( self, 'Create-Predictor', lambda_function = create_predictor_lambda, payload_response_only=True ) self.add_retry_n_catch(create_predictor_job, failed) create_forecast_job = sfn_tasks.LambdaInvoke( self, 'Create-Forecast', lambda_function = create_forecast_lambda, payload_response_only=True ) self.add_retry_n_catch(create_forecast_job, failed) update_resources_job = sfn_tasks.LambdaInvoke( self, 'Update-Resources', lambda_function = update_resources_lambda, payload_response_only=True ) self.add_retry_n_catch(update_resources_job, failed) notify_success = sfn_tasks.LambdaInvoke( self, 'Notify-Success', lambda_function = notify_lambda, payload_response_only=True ) delete_forecast_job = sfn_tasks.LambdaInvoke( self, 'Delete-Forecast', lambda_function = delete_forecast_lambda, payload_response_only=True ) self.delete_retry(delete_forecast_job) delete_predictor_job = sfn_tasks.LambdaInvoke( self, 'Delete-Predictor', lambda_function = delete_predctor_lambda, payload_response_only=True ) self.delete_retry(delete_predictor_job) delete_import_job = sfn_tasks.LambdaInvoke( self, 'Delete-ImportJob', lambda_function = delete_importjob_lambda, payload_response_only=True ) self.delete_retry(delete_import_job) definition = create_dataset_job\ .next(create_dataset_group_job)\ .next(import_data_job)\ .next(create_predictor_job)\ .next(create_forecast_job)\ .next(update_resources_job)\ .next(notify_success)\ .next(strategy_choice.when(sfn.Condition.boolean_equals('$.params.PerformDelete', False), success_state)\ .otherwise(delete_forecast_job).afterwards())\ .next(delete_predictor_job)\ .next(delete_import_job) deployt_state_machine = sfn.StateMachine( self, 'StateMachine', definition = definition # role=roles.states_execution_role ) # S3 event trigger lambda s3_lambda = _lambda.Function( self, 'S3Lambda', function_name='S3Lambda', code=_lambda.Code.asset('lambdas/s3lambda/'), handler='parse.lambda_handler', runtime=_lambda.Runtime.PYTHON_3_7, role=roles.trigger_role, environment= { 'STEP_FUNCTIONS_ARN': deployt_state_machine.state_machine_arn, 'PARAMS_FILE': self.node.try_get_context('parameter_file') } ) s3_lambda.add_event_source( event_src.S3EventSource( bucket=forecast_bucket, events=[_s3.EventType.OBJECT_CREATED], filters=[_s3.NotificationKeyFilter( prefix='train/', suffix='.csv' )] ) ) # CloudFormation output core.CfnOutput( self, 'StepFunctionsName', description='Step Functions Name', value=deployt_state_machine.state_machine_name ) core.CfnOutput( self, 'ForecastBucketName', description='Forecast bucket name to drop you files', value=forecast_bucket.bucket_name ) core.CfnOutput( self, 'AthenaBucketName', description='Athena bucket name to drop your files', value=athena_bucket.bucket_name )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Step Function Starts Here # The first thing we need to do is see if they are asking for pineapple on a pizza pineapple_check_lambda = _lambda.Function( self, "pineappleCheckLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="orderPizza.handler", code=_lambda.Code.from_asset("lambda_fns"), ) # Step functions are built up of steps, we need to define our first step order_pizza = step_fn_tasks.LambdaInvoke( self, 'Order Pizza Job', lambda_function=pineapple_check_lambda, input_path='$.flavour', result_path='$.pineappleAnalysis', payload_response_only=True) # Pizza Order failure step defined pineapple_detected = step_fn.Fail(self, 'Sorry, We Dont add Pineapple', cause='They asked for Pineapple', error='Failed To Make Pizza') # If they didnt ask for pineapple let's cook the pizza cook_pizza = step_fn.Succeed(self, 'Lets make your pizza', output_path='$.pineappleAnalysis') # If they ask for a pizza with pineapple, fail. Otherwise cook the pizza definition = step_fn.Chain \ .start(order_pizza) \ .next(step_fn.Choice(self, 'With Pineapple?') .when(step_fn.Condition.boolean_equals('$.pineappleAnalysis.containsPineapple', True), pineapple_detected) .otherwise(cook_pizza)) state_machine = step_fn.StateMachine( self, 'StateMachine', definition=definition, timeout=core.Duration.minutes(5), tracing_enabled=True, state_machine_type=step_fn.StateMachineType.EXPRESS) # HTTP API Definition # Give our gateway permissions to interact with SNS http_api_role = iam.Role( self, 'HttpApiRole', assumed_by=iam.ServicePrincipal('apigateway.amazonaws.com'), inline_policies={ "AllowSFNExec": iam.PolicyDocument(statements=[ iam.PolicyStatement( actions=["states:StartSyncExecution"], effect=iam.Effect.ALLOW, resources=[state_machine.state_machine_arn]) ]) }) api = api_gw.HttpApi(self, 'the_state_machine_api', create_default_stage=True) # create an AWS_PROXY integration between the HTTP API and our Step Function integ = api_gw.CfnIntegration( self, 'Integ', api_id=api.http_api_id, integration_type='AWS_PROXY', connection_type='INTERNET', integration_subtype='StepFunctions-StartSyncExecution', credentials_arn=http_api_role.role_arn, request_parameters={ "Input": "$request.body", "StateMachineArn": state_machine.state_machine_arn }, payload_format_version="1.0", timeout_in_millis=10000) api_gw.CfnRoute(self, 'DefaultRoute', api_id=api.http_api_id, route_key=api_gw.HttpRouteKey.DEFAULT.key, target="integrations/" + integ.ref) core.CfnOutput(self, 'HTTP API URL', value=api.url)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ### # DynamoDB Table ### # We store Flight, Hotel and Rental Car bookings in the same table. # # For more help with single table DB structures see - https://www.dynamodbbook.com/ # pk - the trip_id e.g. 1234 # sk - bookingtype#booking_id e.g. HOTEL#345634, FLIGHT#574576, PAYMENT#45245 table = dynamo_db.Table(self, "Bookings", partition_key=dynamo_db.Attribute(name="pk", type=dynamo_db.AttributeType.STRING), sort_key=dynamo_db.Attribute(name="sk", type=dynamo_db.AttributeType.STRING) ) ### # Lambda Functions ### # We need Booking and Cancellation functions for our 3 services # # All functions need access to our DynamoDB table above. # We also need to take payment for this trip # # 1) Flights # 2) Hotel # 3) Payment # 1) Flights reserve_flight_lambda = self.create_lambda(scope=self, lambda_id="reserveFlightLambdaHandler", handler='flights/reserveFlight.handler', table=table) confirm_flight_lambda = self.create_lambda(scope=self, lambda_id="confirmFlightLambdaHandler", handler='flights/confirmFlight.handler', table=table) cancel_flight_lambda = self.create_lambda(scope=self, lambda_id="cancelFlightLambdaHandler", handler='flights/cancelFlight.handler', table=table) # 2) Hotel reserve_hotel_lambda = self.create_lambda(scope=self, lambda_id="reserveHotelLambdaHandler", handler='hotel/reserveHotel.handler', table=table) confirm_hotel_lambda = self.create_lambda(scope=self, lambda_id="confirmHotelLambdaHandler", handler='hotel/confirmHotel.handler', table=table) cancel_hotel_lambda = self.create_lambda(scope=self, lambda_id="cancelHotelLambdaHandler", handler='hotel/cancelHotel.handler', table=table) # 3) Payment For Holiday take_payment_lambda = self.create_lambda(scope=self, lambda_id="takePaymentLambdaHandler", handler='payment/takePayment.handler', table=table) refund_payment_lambda = self.create_lambda(scope=self, lambda_id="refundPaymentLambdaHandler", handler='payment/refundPayment.handler', table=table) ### # Saga Pattern Step Function ### # Follows a strict order: # 1) Reserve Flights and Hotel # 2) Take Payment # 3) Confirm Flight and Hotel booking # Our two end states booking_succeeded = step_fn.Succeed(self, 'We have made your booking!') booking_failed = step_fn.Fail(self, "Sorry, We Couldn't make the booking") # 1) Reserve Flights and Hotel cancel_hotel_reservation = step_fn.Task(self, 'CancelHotelReservation', task=step_fn_tasks.InvokeFunction(cancel_hotel_lambda), result_path='$.CancelHotelReservationResult' ).add_retry(max_attempts=3).next(booking_failed) reserve_hotel = step_fn.Task(self, 'ReserveHotel', task=step_fn_tasks.InvokeFunction(reserve_hotel_lambda), result_path='$.ReserveHotelResult' ).add_catch(cancel_hotel_reservation, result_path="$.ReserveHotelError") cancel_flight_reservation = step_fn.Task(self, 'CancelFlightReservation', task=step_fn_tasks.InvokeFunction(cancel_flight_lambda), result_path='$.CancelFlightReservationResult' ).add_retry(max_attempts=3).next(cancel_hotel_reservation) reserve_flight = step_fn.Task(self, 'ReserveFlight', task=step_fn_tasks.InvokeFunction(reserve_flight_lambda), result_path='$.ReserveFlightResult' ).add_catch(cancel_flight_reservation, result_path="$.ReserveFlightError") # 2) Take Payment refund_payment = step_fn.Task(self, 'RefundPayment', task=step_fn_tasks.InvokeFunction(refund_payment_lambda), result_path='$.RefundPaymentResult' ).add_retry(max_attempts=3).next(cancel_flight_reservation) take_payment = step_fn.Task(self, 'TakePayment', task=step_fn_tasks.InvokeFunction(take_payment_lambda), result_path='$.TakePaymentResult' ).add_catch(refund_payment, result_path="$.TakePaymentError") # 3) Confirm Flight and Hotel Booking confirm_hotel = step_fn.Task(self, 'ConfirmHotelBooking', task=step_fn_tasks.InvokeFunction(confirm_hotel_lambda), result_path='$.ConfirmHotelBookingResult' ).add_catch(refund_payment, result_path="$.ConfirmHotelBookingError") confirm_flight = step_fn.Task(self, 'ConfirmFlight', task=step_fn_tasks.InvokeFunction(confirm_flight_lambda), result_path='$.ConfirmFlightResult' ).add_catch(refund_payment, result_path="$.ConfirmFlightError") definition = step_fn.Chain \ .start(reserve_hotel) \ .next(reserve_flight) \ .next(take_payment) \ .next(confirm_hotel) \ .next(confirm_flight) \ .next(booking_succeeded) saga = step_fn.StateMachine(self, 'BookingSaga', definition=definition, timeout=core.Duration.minutes(5)) # defines an AWS Lambda resource to connect to our API Gateway and kick # off our step function saga_lambda = _lambda.Function(self, "sagaLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="sagaLambda.handler", code=_lambda.Code.from_asset("lambdas"), environment={ 'statemachine_arn': saga.state_machine_arn } ) saga.grant_start_execution(saga_lambda) # defines an API Gateway REST API resource backed by our "stateMachineLambda" function. api_gw.LambdaRestApi(self, 'SagaPatternSingleTable', handler=saga_lambda )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Lets create couple of instances to test): vpc = _ec2.Vpc(self, "abacVPC", cidr="10.13.0.0/21", max_azs=2, nat_gateways=0, subnet_configuration=[ _ec2.SubnetConfiguration( name="pubSubnet", cidr_mask=24, subnet_type=_ec2.SubnetType.PUBLIC) ]) core.Tag.add(vpc, key="ServiceProvider", value="KonStone", include_resource_types=[]) weak_sg = _ec2.SecurityGroup( self, "web_sec_grp", vpc=vpc, description="Allow internet access from the world", allow_all_outbound=True) # vpc_cidr_block # weak_sg.add_ingress_rule(_ec2.Peer.any_ipv4(), weak_sg.add_ingress_rule(_ec2.Peer.ipv4(vpc.vpc_cidr_block), _ec2.Port.tcp(22), "Allow SSH access from the VPC Only.") # We are using the latest AMAZON LINUX AMI # Benefit of having SSM Agent pre-installed ami_id = _ec2.AmazonLinuxImage(generation=_ec2.AmazonLinuxGeneration. AMAZON_LINUX_2).get_image(self).image_id # https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_iam/Role.html instace_profile_role = _iam.Role( self, 'ec2ssmroleid', assumed_by=_iam.ServicePrincipal('ec2.amazonaws.com'), role_name="instace_profile_role") instace_profile_role.add_managed_policy( _iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonSSMManagedInstanceCore')) instance_profile_role_additional_perms = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, resources=[ "arn:aws:logs:*:*:*", ], actions=["logs:Create*", "logs:PutLogEvents"]) instance_profile_role_additional_perms.sid = "PutBucketPolicy" instace_profile_role.add_to_policy( instance_profile_role_additional_perms) inst_profile_01 = _iam.CfnInstanceProfile( self, "instProfile01Id", roles=[instace_profile_role.role_name], ) # Let us bootstrap the server with the required agents try: with open("./bootstrap_scripts/install_agents.sh", mode='rb') as file: bootstrap_data = file.read() except OSError: print('Failed to get UserData script') install_agents = _ec2.UserData.for_linux() install_agents.add_commands(str(bootstrap_data, 'utf-8')) # The EC2 Instance to monitor for failed SSH Logins ssh_monitored_inst_01 = _ec2.CfnInstance( self, "sshMonitoredInstance01", image_id=ami_id, instance_type="t2.micro", monitoring=False, tags=[{ "key": "ServiceProvider", "value": "KonStone" }], iam_instance_profile=inst_profile_01.ref, network_interfaces=[{ "deviceIndex": "0", "associatePublicIpAddress": True, "subnetId": vpc.public_subnets[0].subnet_id, "groupSet": [weak_sg.security_group_id] }], #https: //github.com/aws/aws-cdk/issues/3419 user_data=core.Fn.base64(install_agents.render()), ) """ linux_ami = _ec2.GenericLinuxImage({ "cn-northwest-1": "ami-0f62e91915e16cfc2","eu-west-1": "ami-12345678"}) ssh_monitored_inst_01_02 = _ec2.Instance(self, "monitoredInstance02", instance_type=_ec2.InstanceType(instance_type_identifier="t2.micro"), instance_name="monitoredInstance02", machine_image=linux_ami, vpc=vpc, security_group=[weak_sg.security_group_id], # vpc_subnets=_ec2.SubnetSelection(subnet_type=_ec2.SubnetType.PUBLIC) vpc_subnets=vpc.public_subnets[0].subnet_id, # user_data=_ec2.UserData.custom(t_user_data) ) """ # The log group name to store logs info_sec_ops_log_group = _logs.LogGroup( self, "infoSecOpsLogGroupId", log_group_name=(f"/Mystique/InfoSec/Automation/" f"{ssh_monitored_inst_01.ref}"), retention=_logs.RetentionDays.ONE_WEEK) # Defines an AWS Lambda resource with open("lambda_src/quarantine_ec2_instance.py", encoding="utf8") as fp: quarantine_ec2_instance_fn_handler_code = fp.read() quarantine_ec2_instance_fn = _lambda.Function( self, id='quarantineEc2InstanceFnId', function_name="quarantine_ec2_instance", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(quarantine_ec2_instance_fn_handler_code), handler='index.lambda_handler', timeout=core.Duration.seconds(5)) quarantine_ec2_instance_fn_perms = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, resources=[ "*", ], actions=[ "ec2:RevokeSecurityGroupIngress", "ec2:DescribeSecurityGroupReferences", "ec2:RevokeSecurityGroupEgress", "ec2:ApplySecurityGroupsToClientVpnTargetNetwork", "ec2:DescribeSecurityGroups", "ec2:CreateSecurityGroup", "ec2:DescribeInstances", "ec2:CreateTags", "ec2:StopInstances", "ec2:CreateVolume", "ec2:CreateSnapshots", "ec2:CreateSnapshot", "ec2:DescribeSnapshots", "ec2:ModifyInstanceAttribute" ]) quarantine_ec2_instance_fn_perms.sid = "AllowLambdaToQuarantineEC2" quarantine_ec2_instance_fn.add_to_role_policy( quarantine_ec2_instance_fn_perms) info_sec_ops_topic = _sns.Topic(self, "infoSecOpsTopicId", display_name="InfoSecTopic", topic_name="InfoSecOpsTopic") # Ref: https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html ############################################################################### ################# STEP FUNCTIONS EXPERIMENTAL CODE - UNSTABLE ################# ############################################################################### quarantine_ec2_instance_task = _sfn.Task( self, "Quarantine EC2 Instance", task=_tasks.InvokeFunction(quarantine_ec2_instance_fn), result_path="$") notify_secops_task = _sfn.Task( self, "Notify InfoSecOps", task=_tasks.PublishToTopic( info_sec_ops_topic, integration_pattern=_sfn.ServiceIntegrationPattern. FIRE_AND_FORGET, message=_sfn.TaskInput.from_data_at("$.message"), subject="SSH Error Response Notification")) ssh_error_response_failure = _sfn.Fail( self, "SSH Error Response Actions Failed", cause="All Response Actions were NOT completed", error="Check Logs") ssh_error_response_success = _sfn.Succeed( self, "SSH Error Response Actions Succeeded", comment="All Response Action Completed Successfully", ) ssh_error_response_sfn_definition = quarantine_ec2_instance_task\ .next(notify_secops_task\ .next(_sfn.Choice(self, "SSH Errors Response Complete?")\ .when(_sfn.Condition.number_equals("$.SdkHttpMetadata.HttpStatusCode", 200),ssh_error_response_success)\ .when(_sfn.Condition.not_( _sfn.Condition.number_equals("$.SdkHttpMetadata.HttpStatusCode", 200)), ssh_error_response_failure)\ .otherwise(ssh_error_response_failure) ) ) ssh_error_response_statemachine = _sfn.StateMachine( self, "stateMachineId", definition=ssh_error_response_sfn_definition, timeout=core.Duration.minutes(5)) ############################################################################### ################# STEP FUNCTIONS EXPERIMENTAL CODE - UNSTABLE ################# ############################################################################### # LAMBDA TO TRIGGER STATE MACHINE - since state cannot be invoked by SNS with open("lambda_src/trigger_state_machine.py", encoding="utf8") as fp: trigger_state_machine_fn_handler_code = fp.read() trigger_state_machine_fn = _lambda.Function( self, id='sshErrorResponseFnId', function_name="trigger_ssh_error_response_state_machine_fn", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(trigger_state_machine_fn_handler_code), # code=_lambda.Code.asset("lambda_src/is_policy_permissive.py"), # code=_lambda.Code.asset('lambda_src'), # code=_lambda.InlineCode(code_body), handler='index.lambda_handler', timeout=core.Duration.seconds(5), environment={ "STATE_MACHINE_ARN": f"{ssh_error_response_statemachine.state_machine_arn}", }) trigger_state_machine_fn_perms = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, resources=[ f"{ssh_error_response_statemachine.state_machine_arn}", ], actions=["states:StartExecution"]) trigger_state_machine_fn_perms.sid = "PutBucketPolicy" trigger_state_machine_fn.add_to_role_policy( trigger_state_machine_fn_perms) """ version = trigger_state_machine_fn.add_version(name=datetime.now().isoformat()) trigger_state_machine_fn_alias = _lambda.Alias(self, 'lmdaAliasId', alias_name='MystiqueTestAlias', version=version ) """ # Lets add permission to SNS to trigger our lambda function trigger_lambda_perms = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, resources=[ trigger_state_machine_fn.function_arn, ], actions=[ "lambda:InvokeFunction", ]) trigger_lambda_perms.sid = "TriggerLambaFunction" # info_sec_ops_topic.add_to_resource_policy( trigger_lambda_perms ) # Subscribe InfoSecOps Email to topic info_sec_ops_topic.add_subscription( _subs.EmailSubscription(global_args.INFO_SEC_OPS_EMAIL)) # info_sec_ops_topic.add_subscription(_subs.LambdaSubscription(trigger_state_machine_fn)) trigger_state_machine_fn_alarm = trigger_state_machine_fn.metric_all_errors( ).create_alarm( self, "fn-error-alarm", threshold=5, alarm_name="trigger_state_machine_fn_error_alarm", evaluation_periods=5, period=core.Duration.minutes(1), ) subscribe_trigger_state_machine_fn_to_logs = _logs.SubscriptionFilter( self, "sshErrorLogSubscriptionId", log_group=info_sec_ops_log_group, destination=_logs_destination.LambdaDestination( trigger_state_machine_fn), filter_pattern=_logs.FilterPattern.space_delimited( "Mon", "day", "timestamp", "ip", "id", "status", "...").where_string("status", "=", "Invalid"), ) # https://pypi.org/project/aws-cdk.aws-logs/ # We are creating three filter # tooManySshDisconnects, invalidSshUser and invalidSshKey: # When a user tries to SSH with invalid username the next line is logged in the SSH log file: # Apr 20 02:39:35 ip-172-31-63-56 sshd[17136]: Received disconnect from xxx.xxx.xxx.xxx: 11: [preauth] too_many_ssh_disconnects_metric = _cloudwatch.Metric( namespace=f"{global_args.OWNER}", metric_name="tooManySshDisconnects") too_many_ssh_disconnects_filter = _logs.MetricFilter( self, "tooManySshDisconnectsFilterId", log_group=info_sec_ops_log_group, metric_namespace=too_many_ssh_disconnects_metric.namespace, metric_name=too_many_ssh_disconnects_metric.metric_name, filter_pattern=_logs.FilterPattern.space_delimited( "Mon", "day", "timestamp", "ip", "id", "msg1", "msg2", "...").where_string("msg2", "=", "disconnect"), metric_value="1") invalid_ssh_user_metric = _cloudwatch.Metric( namespace=f"{global_args.OWNER}", metric_name="invalidSshUser", ) invalid_ssh_user_filter = _logs.MetricFilter( self, "invalidSshUserFilterId", log_group=info_sec_ops_log_group, metric_namespace=invalid_ssh_user_metric.namespace, metric_name=invalid_ssh_user_metric.metric_name, filter_pattern=_logs.FilterPattern.space_delimited( "Mon", "day", "timestamp", "ip", "id", "status", "...").where_string("status", "=", "Invalid"), metric_value="1") invalid_ssh_key_metric = _cloudwatch.Metric( namespace=f"{global_args.OWNER}", metric_name="invalidSshKey") invalid_ssh_key_filter = _logs.MetricFilter( self, "invalidSshKeyFilterId", log_group=info_sec_ops_log_group, metric_namespace=invalid_ssh_key_metric.namespace, metric_name=invalid_ssh_key_metric.metric_name, filter_pattern=_logs.FilterPattern.space_delimited( "Mon", "day", "timestamp", "ip", "id", "msg1", "msg2", "...").where_string("msg1", "=", "Connection").where_string( "msg2", "=", "closed"), metric_value="1") # Now let us create alarms # alarm is raised there are more than 5(threshold) of the measured metrics in two(datapoint) of the last three seconds(evaluation): # Period=60Seconds, Eval=3, Threshold=5 too_many_ssh_disconnects_alarm = _cloudwatch.Alarm( self, "tooManySshDisconnectsAlarmId", alarm_name="too_many_ssh_disconnects_alarm", alarm_description= "The number disconnect requests is greater then 5, even 1 time in 3 minutes", metric=too_many_ssh_disconnects_metric, actions_enabled=True, period=core.Duration.minutes(1), threshold=5, evaluation_periods=3, datapoints_to_alarm=1, statistic="sum", comparison_operator=_cloudwatch.ComparisonOperator. GREATER_THAN_OR_EQUAL_TO_THRESHOLD) invalid_ssh_user_alarm = _cloudwatch.Alarm( self, "invalidSshUserAlarmId", alarm_name="too_many_invalid_ssh_users_alarm", alarm_description= "The number of invalid ssh users connecting is greater then 5, even 1 time in 3 minutes", metric=invalid_ssh_user_metric, actions_enabled=True, period=core.Duration.minutes(1), threshold=5, evaluation_periods=3, datapoints_to_alarm=1, statistic="sum", comparison_operator=_cloudwatch.ComparisonOperator. GREATER_THAN_THRESHOLD) invalid_ssh_user_alarm.add_alarm_action( _cloudwatch_actions.SnsAction(info_sec_ops_topic)) invalid_ssh_key_alarm = _cloudwatch.Alarm( self, "invalidSshKeyAlarmId", alarm_name="too_many_invalid_ssh_key_alarm", alarm_description= "The number of invalid ssh keys connecting is greater then 5, even 1 time in 3 minutes", metric=invalid_ssh_key_metric, actions_enabled=True, period=core.Duration.minutes(1), threshold=5, evaluation_periods=3, datapoints_to_alarm=1, statistic="sum", comparison_operator=_cloudwatch.ComparisonOperator. GREATER_THAN_OR_EQUAL_TO_THRESHOLD) invalid_ssh_key_alarm.add_alarm_action( _cloudwatch_actions.SnsAction(info_sec_ops_topic)) ########################################### ################# OUTPUTS ################# ########################################### output0 = core.CfnOutput( self, "SecuirtyAutomationFrom", value=f"{global_args.SOURCE_INFO}", description= "To know more about this automation stack, check out our github page." ) output1_1 = core.Fn.get_att( logical_name_of_resource="sshMonitoredInstance01", attribute_name="PublicIp") output1 = core.CfnOutput(self, "MonitoredInstance", value=output1_1.to_string(), description="Web Server Public IP to attack") output2 = core.CfnOutput( self, "SSHAlarms", value= (f"https://console.aws.amazon.com/cloudwatch/home?region=" f"{core.Aws.REGION}" f"#/configuration/" f"#alarmsV2:?search=ssh&alarmStateFilter=ALL&alarmTypeFilter=ALL" ), description="Check out the cloudwatch Alarms") output3 = core.CfnOutput( self, "SubscribeToNotificationTopic", value=(f"https://console.aws.amazon.com/sns/v3/home?" f"{core.Aws.REGION}" f"#/topic/" f"{info_sec_ops_topic.topic_arn}"), description= "Add your email to subscription and confirm subscription") output_test_1 = core.CfnOutput( self, "ToGenInvalidKeyErrors", value= (f"for i in {{1..30}}; do ssh -i $RANDOM ec2-user@{output1_1.to_string()}; sleep 2; done &" ), description= "Generates random key names and connects to server 30 times over 60 seconds" ) output_test_2 = core.CfnOutput( self, "ToGenInvalidUserErrors", value= (f"for i in {{1..30}}; do ssh ec2-user$RANDOM@{output1_1.to_string()}; sleep 2; done &" ), description= "Generates random user names and connects to server 30 times over 60 seconds" ) """
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The start of the image pipeline imageBucket = aws_s3.Bucket(self, "imageBucket") # Capture API activity with a trail imageBucketTrail = aws_cloudtrail.Trail(self, "imageBucketTrail", is_multi_region_trail=False) # Restrict to S3 data-plane events imageBucketTrail.add_s3_event_selector( include_management_events=False, prefixes=[f"{imageBucket.bucket_arn}/"], read_write_type=aws_cloudtrail.ReadWriteType.WRITE_ONLY) # Filter to just PutObject and CopyObject events imageBucketRule = aws_events.Rule( self, "imageBucketRule", event_pattern={ "source": ["aws.s3"], "detail": { "eventSource": ["s3.amazonaws.com"], "eventName": ["PutObject", "CopyObject"], "requestParameters": { "bucketName": [imageBucket.bucket_name] } } }) #-- # Lambda Layers #--------------------# opencvLayer = aws_lambda.LayerVersion( self, 'opencvLayer', code=aws_lambda.AssetCode('layers/opencvLayer'), compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6]) boto3Layer = aws_lambda.LayerVersion( self, 'boto3Layer', code=aws_lambda.AssetCode('layers/boto3Layer'), compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6]) #-- # Lambda Functions #--------------------# # Gather info about an image, name, extension, etc getImageInfoFunc = aws_lambda.Function( self, "getImageInfoFunc", code=aws_lambda.AssetCode('functions/getImageInfoFunc'), handler="lambda.handler", runtime=aws_lambda.Runtime.PYTHON_3_6) # The home for the website webBucket = aws_s3.Bucket(self, "webBucket", website_index_document='index.html') # Copy the image to the web bucket copyImageFunc = aws_lambda.Function( self, "copyImageFunc", code=aws_lambda.AssetCode('functions/copyImageFunc'), handler="lambda.handler", runtime=aws_lambda.Runtime.PYTHON_3_6, layers=[boto3Layer], environment={ 'OUTPUTBUCKET': webBucket.bucket_name, 'OUTPUTPREFIX': 'images/' }) # Grant permissions to read from the source and write to the desination imageBucket.grant_read(copyImageFunc) webBucket.grant_write(copyImageFunc) # Create a thumbnail of the image and place in the web bucket createThumbnailFunc = aws_lambda.Function( self, "createThumbnailFunc", code=aws_lambda.AssetCode('functions/createThumbnailFunc'), handler="lambda.handler", runtime=aws_lambda.Runtime.PYTHON_3_6, layers=[boto3Layer, opencvLayer], timeout=core.Duration.seconds(10), memory_size=256, environment={ 'OUTPUTBUCKET': webBucket.bucket_name, 'OUTPUTPREFIX': 'images/' }) # Grant permissions to read from the source and write to the desination imageBucket.grant_read(createThumbnailFunc) webBucket.grant_write(createThumbnailFunc) # Store page information pageTable = aws_dynamodb.Table( self, 'pageTable', partition_key={ 'name': 'pageName', 'type': aws_dynamodb.AttributeType.STRING }, billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST, stream=aws_dynamodb.StreamViewType.NEW_IMAGE) # Save page and image information updatePageInfoFunc = aws_lambda.Function( self, "updatePageInfoFunc", code=aws_lambda.AssetCode('functions/updatePageInfoFunc'), handler="lambda.handler", runtime=aws_lambda.Runtime.PYTHON_3_6, layers=[boto3Layer], environment={ 'PAGETABLE': pageTable.table_name, 'PAGEPREFIX': 'posts/' }) # Grant permissions to write to the page table pageTable.grant_write_data(updatePageInfoFunc) imagePipelineDone = aws_stepfunctions.Succeed(self, "Done processing image") updatePageInfoJob = aws_stepfunctions.Task( self, 'Update page info', task=aws_stepfunctions_tasks.InvokeFunction(updatePageInfoFunc)) updatePageInfoJob.next(imagePipelineDone) copyImageJob = aws_stepfunctions.Task( self, 'Copy image', task=aws_stepfunctions_tasks.InvokeFunction(copyImageFunc)) createThumbnailJob = aws_stepfunctions.Task( self, 'Create thumbnail', task=aws_stepfunctions_tasks.InvokeFunction(createThumbnailFunc)) # These tasks can be done in parallel processImage = aws_stepfunctions.Parallel(self, 'Process image', result_path="$.images") processImage.branch(copyImageJob) processImage.branch(createThumbnailJob) processImage.next(updatePageInfoJob) # Results of file extension check notPng = aws_stepfunctions.Succeed(self, "Not a PNG") # Verify the file extension checkForPng = aws_stepfunctions.Choice(self, 'Is a PNG?') checkForPng.when( aws_stepfunctions.Condition.string_equals('$.extension', 'png'), processImage) checkForPng.otherwise(notPng) # A single image pipeline job for testing getImageInfoJob = aws_stepfunctions.Task( self, 'Get image info', task=aws_stepfunctions_tasks.InvokeFunction(getImageInfoFunc)) getImageInfoJob.next(checkForPng) # Configure the image pipeline and starting state imagePipeline = aws_stepfunctions.StateMachine( self, "imagePipeline", definition=getImageInfoJob) # Matching events start the image pipline imageBucketRule.add_target( aws_events_targets.SfnStateMachine( imagePipeline, input=aws_events.RuleTargetInput.from_event_path( "$.detail.requestParameters")))
def __init__( # pylint: disable=too-many-arguments self, scope: Construct, stack_id: str, *, botocore_lambda_layer: aws_lambda_python.PythonLayerVersion, datasets_table: Table, deploy_env: str, storage_bucket: aws_s3.Bucket, storage_bucket_parameter: aws_ssm.StringParameter, validation_results_table: Table, **kwargs: Any, ) -> None: # pylint: disable=too-many-locals super().__init__(scope, stack_id, **kwargs) ############################################################################################ # PROCESSING ASSETS TABLE processing_assets_table = Table( self, f"{deploy_env}-processing-assets", deploy_env=deploy_env, parameter_name=ParameterName.PROCESSING_ASSETS_TABLE_NAME, sort_key=aws_dynamodb.Attribute( name="sk", type=aws_dynamodb.AttributeType.STRING), ) ############################################################################################ # BATCH JOB DEPENDENCIES batch_job_queue = BatchJobQueue( self, "batch-job-queue", deploy_env=deploy_env, processing_assets_table=processing_assets_table, ).job_queue s3_read_only_access_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonS3ReadOnlyAccess") ############################################################################################ # STATE MACHINE TASKS check_stac_metadata_task = LambdaTask( self, "check-stac-metadata-task", directory="check_stac_metadata", botocore_lambda_layer=botocore_lambda_layer, extra_environment={"DEPLOY_ENV": deploy_env}, ) assert check_stac_metadata_task.lambda_function.role check_stac_metadata_task.lambda_function.role.add_managed_policy( policy=s3_read_only_access_policy) for table in [processing_assets_table, validation_results_table]: table.grant_read_write_data( check_stac_metadata_task.lambda_function) table.grant( check_stac_metadata_task.lambda_function, "dynamodb:DescribeTable", ) content_iterator_task = LambdaTask( self, "content-iterator-task", directory="content_iterator", botocore_lambda_layer=botocore_lambda_layer, result_path="$.content", extra_environment={"DEPLOY_ENV": deploy_env}, ) check_files_checksums_directory = "check_files_checksums" check_files_checksums_default_payload_object = { "dataset_id.$": "$.dataset_id", "version_id.$": "$.version_id", "metadata_url.$": "$.metadata_url", "first_item.$": "$.content.first_item", "assets_table_name.$": "$.content.assets_table_name", "results_table_name.$": "$.content.results_table_name", } check_files_checksums_single_task = BatchSubmitJobTask( self, "check-files-checksums-single-task", deploy_env=deploy_env, directory=check_files_checksums_directory, s3_policy=s3_read_only_access_policy, job_queue=batch_job_queue, payload_object=check_files_checksums_default_payload_object, container_overrides_command=[ "--dataset-id", "Ref::dataset_id", "--version-id", "Ref::version_id", "--first-item", "Ref::first_item", "--assets-table-name", "Ref::assets_table_name", "--results-table-name", "Ref::results_table_name", ], ) array_size = int( aws_stepfunctions.JsonPath.number_at("$.content.iteration_size")) check_files_checksums_array_task = BatchSubmitJobTask( self, "check-files-checksums-array-task", deploy_env=deploy_env, directory=check_files_checksums_directory, s3_policy=s3_read_only_access_policy, job_queue=batch_job_queue, payload_object=check_files_checksums_default_payload_object, container_overrides_command=[ "--dataset-id", "Ref::dataset_id", "--version-id", "Ref::version_id", "--first-item", "Ref::first_item", "--assets-table-name", "Ref::assets_table_name", "--results-table-name", "Ref::results_table_name", ], array_size=array_size, ) for reader in [ content_iterator_task.lambda_function, check_files_checksums_single_task.job_role, check_files_checksums_array_task.job_role, ]: processing_assets_table.grant_read_data( reader) # type: ignore[arg-type] processing_assets_table.grant( reader, "dynamodb:DescribeTable" # type: ignore[arg-type] ) for writer in [ check_files_checksums_single_task.job_role, check_files_checksums_array_task.job_role, ]: validation_results_table.grant_read_write_data( writer) # type: ignore[arg-type] validation_results_table.grant( writer, "dynamodb:DescribeTable" # type: ignore[arg-type] ) validation_summary_task = LambdaTask( self, "validation-summary-task", directory="validation_summary", botocore_lambda_layer=botocore_lambda_layer, result_path="$.validation", extra_environment={"DEPLOY_ENV": deploy_env}, ) validation_results_table.grant_read_data( validation_summary_task.lambda_function) validation_results_table.grant(validation_summary_task.lambda_function, "dynamodb:DescribeTable") validation_failure_lambda_invoke = LambdaTask( self, "validation-failure-task", directory="validation_failure", botocore_lambda_layer=botocore_lambda_layer, result_path=aws_stepfunctions.JsonPath.DISCARD, ).lambda_invoke import_dataset_role = aws_iam.Role( self, "import-dataset", assumed_by=aws_iam.ServicePrincipal( # type: ignore[arg-type] "batchoperations.s3.amazonaws.com"), ) import_asset_file_function = ImportFileFunction( self, directory="import_asset_file", invoker=import_dataset_role, deploy_env=deploy_env, botocore_lambda_layer=botocore_lambda_layer, ) import_metadata_file_function = ImportFileFunction( self, directory="import_metadata_file", invoker=import_dataset_role, deploy_env=deploy_env, botocore_lambda_layer=botocore_lambda_layer, ) for storage_writer in [ import_dataset_role, import_asset_file_function.role, import_metadata_file_function.role, ]: storage_bucket.grant_read_write( storage_writer) # type: ignore[arg-type] import_dataset_task = LambdaTask( self, "import-dataset-task", directory="import_dataset", botocore_lambda_layer=botocore_lambda_layer, result_path="$.import_dataset", extra_environment={"DEPLOY_ENV": deploy_env}, ) assert import_dataset_task.lambda_function.role is not None import_dataset_task.lambda_function.role.add_to_policy( aws_iam.PolicyStatement( resources=[import_dataset_role.role_arn], actions=["iam:PassRole"], ), ) import_dataset_task.lambda_function.role.add_to_policy( aws_iam.PolicyStatement(resources=["*"], actions=["s3:CreateJob"])) storage_bucket.grant_read_write(import_dataset_task.lambda_function) for table in [datasets_table, processing_assets_table]: table.grant_read_data(import_dataset_task.lambda_function) table.grant(import_dataset_task.lambda_function, "dynamodb:DescribeTable") # Parameters import_asset_file_function_arn_parameter = aws_ssm.StringParameter( self, "import asset file function arn", string_value=import_asset_file_function.function_arn, description=f"Import asset file function ARN for {deploy_env}", parameter_name=ParameterName. PROCESSING_IMPORT_ASSET_FILE_FUNCTION_TASK_ARN.value, ) import_metadata_file_function_arn_parameter = aws_ssm.StringParameter( self, "import metadata file function arn", string_value=import_metadata_file_function.function_arn, description=f"Import metadata file function ARN for {deploy_env}", parameter_name=ParameterName. PROCESSING_IMPORT_METADATA_FILE_FUNCTION_TASK_ARN.value, ) import_dataset_role_arn_parameter = aws_ssm.StringParameter( self, "import dataset role arn", string_value=import_dataset_role.role_arn, description=f"Import dataset role ARN for {deploy_env}", parameter_name=ParameterName.PROCESSING_IMPORT_DATASET_ROLE_ARN. value, ) grant_parameter_read_access({ datasets_table.name_parameter: [import_dataset_task.lambda_function], import_asset_file_function_arn_parameter: [import_dataset_task.lambda_function], import_dataset_role_arn_parameter: [import_dataset_task.lambda_function], import_metadata_file_function_arn_parameter: [import_dataset_task.lambda_function], processing_assets_table.name_parameter: [ check_stac_metadata_task.lambda_function.role, content_iterator_task.lambda_function, import_dataset_task.lambda_function, ], storage_bucket_parameter: [ import_dataset_task.lambda_function, ], validation_results_table.name_parameter: [ check_stac_metadata_task.lambda_function.role, validation_summary_task.lambda_function, content_iterator_task.lambda_function, ], }) success_task = aws_stepfunctions.Succeed(self, "success") ############################################################################################ # STATE MACHINE dataset_version_creation_definition = ( check_stac_metadata_task.lambda_invoke.next( content_iterator_task.lambda_invoke).next( aws_stepfunctions.Choice( # type: ignore[arg-type] self, "check_files_checksums_maybe_array").when( aws_stepfunctions.Condition.number_equals( "$.content.iteration_size", 1), check_files_checksums_single_task.batch_submit_job, ).otherwise(check_files_checksums_array_task. batch_submit_job).afterwards()). next( aws_stepfunctions.Choice( self, "content_iteration_finished").when( aws_stepfunctions.Condition.number_equals( "$.content.next_item", -1), validation_summary_task.lambda_invoke.next( aws_stepfunctions.Choice( # type: ignore[arg-type] self, "validation_successful").when( aws_stepfunctions.Condition.boolean_equals( "$.validation.success", True), import_dataset_task.lambda_invoke.next( success_task # type: ignore[arg-type] ), ).otherwise(validation_failure_lambda_invoke)), ).otherwise(content_iterator_task.lambda_invoke))) self.state_machine = aws_stepfunctions.StateMachine( self, f"{deploy_env}-dataset-version-creation", definition= dataset_version_creation_definition, # type: ignore[arg-type] ) self.state_machine_parameter = aws_ssm.StringParameter( self, "state machine arn", description=f"State machine ARN for {deploy_env}", parameter_name=ParameterName. PROCESSING_DATASET_VERSION_CREATION_STEP_FUNCTION_ARN.value, # pylint:disable=line-too-long string_value=self.state_machine.state_machine_arn, ) Tags.of(self).add("ApplicationLayer", "processing") # type: ignore[arg-type]